duckdb 0.8.2-dev161.0 → 0.8.2-dev1764.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (504) hide show
  1. package/binding.gyp +15 -12
  2. package/binding.gyp.in +1 -1
  3. package/configure.py +1 -1
  4. package/duckdb_extension_config.cmake +10 -0
  5. package/package.json +1 -1
  6. package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
  7. package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
  8. package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
  9. package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
  10. package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
  11. package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
  12. package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
  13. package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
  14. package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
  15. package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
  16. package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
  17. package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
  18. package/src/duckdb/extension/json/include/json_common.hpp +47 -231
  19. package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
  20. package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
  21. package/src/duckdb/extension/json/json_common.cpp +272 -40
  22. package/src/duckdb/extension/json/json_functions/json_create.cpp +21 -2
  23. package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
  24. package/src/duckdb/extension/json/json_functions/json_transform.cpp +91 -38
  25. package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
  26. package/src/duckdb/extension/json/json_functions.cpp +24 -24
  27. package/src/duckdb/extension/json/json_scan.cpp +3 -6
  28. package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
  29. package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
  30. package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
  31. package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
  32. package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
  33. package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
  34. package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
  35. package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
  36. package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
  37. package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
  38. package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
  39. package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
  40. package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
  41. package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
  42. package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
  43. package/src/duckdb/extension/parquet/parquet_extension.cpp +194 -20
  44. package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
  45. package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
  46. package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
  47. package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
  48. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
  49. package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
  50. package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
  51. package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
  52. package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
  53. package/src/duckdb/src/common/allocator.cpp +14 -2
  54. package/src/duckdb/src/common/arrow/arrow_appender.cpp +79 -12
  55. package/src/duckdb/src/common/arrow/arrow_converter.cpp +44 -19
  56. package/src/duckdb/src/common/assert.cpp +3 -0
  57. package/src/duckdb/src/common/enum_util.cpp +4619 -4446
  58. package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
  59. package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
  60. package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
  61. package/src/duckdb/src/common/exception.cpp +2 -2
  62. package/src/duckdb/src/common/extra_type_info.cpp +506 -0
  63. package/src/duckdb/src/common/file_system.cpp +19 -0
  64. package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
  65. package/src/duckdb/src/common/local_file_system.cpp +14 -14
  66. package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
  67. package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
  68. package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
  69. package/src/duckdb/src/common/re2_regex.cpp +1 -1
  70. package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
  71. package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
  72. package/src/duckdb/src/common/sort/partition_state.cpp +70 -50
  73. package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
  74. package/src/duckdb/src/common/types/bit.cpp +51 -0
  75. package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
  76. package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
  77. package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
  78. package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
  79. package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
  80. package/src/duckdb/src/common/types/date.cpp +9 -0
  81. package/src/duckdb/src/common/types/list_segment.cpp +24 -74
  82. package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
  83. package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
  84. package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
  85. package/src/duckdb/src/common/types/uuid.cpp +2 -2
  86. package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
  87. package/src/duckdb/src/common/types/value.cpp +11 -6
  88. package/src/duckdb/src/common/types.cpp +9 -656
  89. package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
  90. package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
  91. package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
  92. package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
  93. package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
  94. package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
  95. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
  96. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
  97. package/src/duckdb/src/core_functions/function_list.cpp +4 -2
  98. package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
  99. package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
  100. package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
  101. package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
  102. package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
  103. package/src/duckdb/src/execution/expression_executor.cpp +1 -1
  104. package/src/duckdb/src/execution/index/art/art.cpp +149 -139
  105. package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
  106. package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
  107. package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
  108. package/src/duckdb/src/execution/index/art/node.cpp +113 -120
  109. package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
  110. package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
  111. package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
  112. package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
  113. package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
  114. package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
  115. package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
  116. package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
  117. package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
  118. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
  119. package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
  120. package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
  121. package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +444 -284
  122. package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
  123. package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
  124. package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +28 -12
  125. package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
  126. package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +23 -4
  127. package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +41 -5
  128. package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
  129. package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
  130. package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
  131. package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
  132. package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
  133. package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
  134. package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
  135. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
  136. package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
  137. package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
  138. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
  139. package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
  140. package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
  141. package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
  142. package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
  143. package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
  144. package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +56 -33
  145. package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +17 -13
  146. package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
  147. package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
  148. package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
  149. package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
  150. package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
  151. package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
  152. package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
  153. package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
  154. package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
  155. package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
  156. package/src/duckdb/src/function/function.cpp +3 -1
  157. package/src/duckdb/src/function/pragma/pragma_queries.cpp +5 -0
  158. package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
  159. package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
  160. package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
  161. package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
  162. package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
  163. package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
  164. package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
  165. package/src/duckdb/src/function/table/arrow.cpp +19 -0
  166. package/src/duckdb/src/function/table/arrow_conversion.cpp +35 -1
  167. package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
  168. package/src/duckdb/src/function/table/read_csv.cpp +100 -17
  169. package/src/duckdb/src/function/table/system/test_all_types.cpp +7 -0
  170. package/src/duckdb/src/function/table/system_functions.cpp +1 -0
  171. package/src/duckdb/src/function/table/table_scan.cpp +9 -0
  172. package/src/duckdb/src/function/table/version/pragma_version.cpp +46 -2
  173. package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
  174. package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
  175. package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
  176. package/src/duckdb/src/include/duckdb/common/dl.hpp +3 -1
  177. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +616 -584
  178. package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
  179. package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
  180. package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
  181. package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
  182. package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
  183. package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
  184. package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +219 -0
  185. package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
  186. package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
  187. package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
  188. package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
  189. package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
  190. package/src/duckdb/src/include/duckdb/common/opener_file_system.hpp +2 -2
  191. package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
  192. package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
  193. package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
  194. package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
  195. package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
  196. package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +23 -8
  197. package/src/duckdb/src/include/duckdb/common/string_util.hpp +11 -0
  198. package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
  199. package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
  200. package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
  201. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
  202. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
  203. package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
  204. package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
  205. package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
  206. package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
  207. package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +5 -2
  208. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
  209. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
  210. package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
  211. package/src/duckdb/src/include/duckdb/common/types/value.hpp +1 -0
  212. package/src/duckdb/src/include/duckdb/common/types.hpp +1 -15
  213. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
  214. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
  215. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
  216. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
  217. package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
  218. package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
  219. package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
  220. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
  221. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
  222. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
  223. package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
  224. package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
  225. package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
  226. package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
  227. package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
  228. package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
  229. package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
  230. package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
  231. package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
  232. package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
  233. package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
  234. package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
  235. package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
  236. package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
  237. package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
  238. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
  239. package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
  240. package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
  241. package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
  242. package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
  243. package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
  244. package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
  245. package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
  246. package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
  247. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +3 -10
  248. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +1 -1
  249. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +1 -1
  250. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp +12 -1
  251. package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
  252. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
  253. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
  254. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
  255. package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
  256. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
  257. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
  258. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
  259. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
  260. package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
  261. package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
  262. package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
  263. package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
  264. package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
  265. package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
  266. package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
  267. package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
  268. package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
  269. package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
  270. package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
  271. package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +4 -0
  272. package/src/duckdb/src/include/duckdb/main/client_config.hpp +5 -0
  273. package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
  274. package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
  275. package/src/duckdb/src/include/duckdb/main/extension/generated_extension_loader.hpp +22 -0
  276. package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +2 -0
  277. package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
  278. package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
  279. package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
  280. package/src/duckdb/src/include/duckdb/main/settings.hpp +39 -1
  281. package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
  282. package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
  283. package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
  284. package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
  285. package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
  286. package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
  287. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
  288. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
  289. package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
  290. package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
  291. package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
  292. package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
  293. package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
  294. package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
  295. package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
  296. package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
  297. package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
  298. package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
  299. package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
  300. package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
  301. package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
  302. package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
  303. package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
  304. package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
  305. package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
  306. package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
  307. package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
  308. package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
  309. package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
  310. package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
  311. package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
  312. package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
  313. package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
  314. package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
  315. package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
  316. package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
  317. package/src/duckdb/src/include/duckdb/planner/binder.hpp +12 -5
  318. package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
  319. package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
  320. package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
  321. package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
  322. package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
  323. package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
  324. package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
  325. package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
  326. package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
  327. package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
  328. package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
  329. package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
  330. package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
  331. package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
  332. package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
  333. package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
  334. package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
  335. package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
  336. package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
  337. package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
  338. package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
  339. package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
  340. package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
  341. package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
  342. package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
  343. package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
  344. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
  345. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
  346. package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
  347. package/src/duckdb/src/include/duckdb.h +28 -0
  348. package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
  349. package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
  350. package/src/duckdb/src/main/config.cpp +4 -0
  351. package/src/duckdb/src/main/database.cpp +1 -1
  352. package/src/duckdb/src/main/extension/extension_helper.cpp +93 -88
  353. package/src/duckdb/src/main/extension/extension_install.cpp +9 -0
  354. package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
  355. package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
  356. package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
  357. package/src/duckdb/src/main/relation.cpp +6 -5
  358. package/src/duckdb/src/main/settings/settings.cpp +79 -18
  359. package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
  360. package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
  361. package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
  362. package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
  363. package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
  364. package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
  365. package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
  366. package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
  367. package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
  368. package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
  369. package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
  370. package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
  371. package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
  372. package/src/duckdb/src/optimizer/optimizer.cpp +49 -14
  373. package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
  374. package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
  375. package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
  376. package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
  377. package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
  378. package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
  379. package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
  380. package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
  381. package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
  382. package/src/duckdb/src/parallel/executor.cpp +15 -0
  383. package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
  384. package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
  385. package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
  386. package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
  387. package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
  388. package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
  389. package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
  390. package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
  391. package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
  392. package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
  393. package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
  394. package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
  395. package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
  396. package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
  397. package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
  398. package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
  399. package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
  400. package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
  401. package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
  402. package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
  403. package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
  404. package/src/duckdb/src/parser/parser.cpp +8 -2
  405. package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
  406. package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
  407. package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
  408. package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
  409. package/src/duckdb/src/parser/query_node.cpp +15 -37
  410. package/src/duckdb/src/parser/result_modifier.cpp +0 -74
  411. package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
  412. package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
  413. package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
  414. package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
  415. package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
  416. package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
  417. package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
  418. package/src/duckdb/src/parser/tableref.cpp +0 -44
  419. package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
  420. package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
  421. package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
  422. package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
  423. package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
  424. package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
  425. package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
  426. package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
  427. package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
  428. package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
  429. package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
  430. package/src/duckdb/src/parser/transformer.cpp +15 -0
  431. package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
  432. package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
  433. package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
  434. package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
  435. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
  436. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
  437. package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
  438. package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
  439. package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
  440. package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
  441. package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +64 -26
  442. package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
  443. package/src/duckdb/src/planner/binder.cpp +44 -31
  444. package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
  445. package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
  446. package/src/duckdb/src/planner/expression_binder.cpp +3 -0
  447. package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
  448. package/src/duckdb/src/planner/logical_operator.cpp +5 -0
  449. package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
  450. package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
  451. package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
  452. package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
  453. package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
  454. package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
  455. package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
  456. package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
  457. package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
  458. package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
  459. package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
  460. package/src/duckdb/src/storage/compression/rle.cpp +0 -1
  461. package/src/duckdb/src/storage/data_table.cpp +1 -1
  462. package/src/duckdb/src/storage/local_storage.cpp +3 -3
  463. package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
  464. package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
  465. package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
  466. package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
  467. package/src/duckdb/src/storage/serialization/serialize_types.cpp +127 -0
  468. package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
  469. package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
  470. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  471. package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
  472. package/src/duckdb/src/storage/table/row_group.cpp +25 -9
  473. package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
  474. package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
  475. package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
  476. package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
  477. package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
  478. package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
  479. package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
  480. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
  481. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
  482. package/src/duckdb/ub_src_common.cpp +2 -0
  483. package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
  484. package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
  485. package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
  486. package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
  487. package/src/duckdb/ub_src_function_scalar.cpp +2 -0
  488. package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
  489. package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
  490. package/src/duckdb/ub_src_optimizer.cpp +6 -0
  491. package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
  492. package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
  493. package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
  494. package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
  495. package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
  496. package/src/duckdb/ub_src_planner_operator.cpp +4 -0
  497. package/src/duckdb/ub_src_storage_serialization.cpp +10 -0
  498. package/src/statement.cpp +10 -3
  499. package/test/columns.test.ts +24 -1
  500. package/test/test_all_types.test.ts +234 -0
  501. package/tsconfig.json +1 -0
  502. package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
  503. package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
  504. package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -9,15 +9,18 @@
9
9
  #pragma once
10
10
 
11
11
  #include "duckdb/main/relation.hpp"
12
+ #include "duckdb/common/enums/joinref_type.hpp"
12
13
 
13
14
  namespace duckdb {
14
15
 
15
16
  class CrossProductRelation : public Relation {
16
17
  public:
17
- DUCKDB_API CrossProductRelation(shared_ptr<Relation> left, shared_ptr<Relation> right);
18
+ DUCKDB_API CrossProductRelation(shared_ptr<Relation> left, shared_ptr<Relation> right,
19
+ JoinRefType join_ref_type = JoinRefType::CROSS);
18
20
 
19
21
  shared_ptr<Relation> left;
20
22
  shared_ptr<Relation> right;
23
+ JoinRefType ref_type;
21
24
  vector<ColumnDefinition> columns;
22
25
 
23
26
  public:
@@ -9,21 +9,24 @@
9
9
  #pragma once
10
10
 
11
11
  #include "duckdb/main/relation.hpp"
12
+ #include "duckdb/common/enums/joinref_type.hpp"
12
13
 
13
14
  namespace duckdb {
14
15
 
15
16
  class JoinRelation : public Relation {
16
17
  public:
17
18
  DUCKDB_API JoinRelation(shared_ptr<Relation> left, shared_ptr<Relation> right,
18
- unique_ptr<ParsedExpression> condition, JoinType type);
19
+ unique_ptr<ParsedExpression> condition, JoinType type,
20
+ JoinRefType join_ref_type = JoinRefType::REGULAR);
19
21
  DUCKDB_API JoinRelation(shared_ptr<Relation> left, shared_ptr<Relation> right, vector<string> using_columns,
20
- JoinType type);
22
+ JoinType type, JoinRefType join_ref_type = JoinRefType::REGULAR);
21
23
 
22
24
  shared_ptr<Relation> left;
23
25
  shared_ptr<Relation> right;
24
26
  unique_ptr<ParsedExpression> condition;
25
27
  vector<string> using_columns;
26
28
  JoinType join_type;
29
+ JoinRefType join_ref_type;
27
30
  vector<ColumnDefinition> columns;
28
31
 
29
32
  public:
@@ -12,6 +12,7 @@
12
12
  #include "duckdb/common/enums/join_type.hpp"
13
13
  #include "duckdb/common/enums/relation_type.hpp"
14
14
  #include "duckdb/common/winapi.hpp"
15
+ #include "duckdb/common/enums/joinref_type.hpp"
15
16
  #include "duckdb/main/query_result.hpp"
16
17
  #include "duckdb/parser/column_definition.hpp"
17
18
  #include "duckdb/common/named_parameter_map.hpp"
@@ -94,10 +95,11 @@ public:
94
95
 
95
96
  // JOIN operation
96
97
  DUCKDB_API shared_ptr<Relation> Join(const shared_ptr<Relation> &other, const string &condition,
97
- JoinType type = JoinType::INNER);
98
+ JoinType type = JoinType::INNER, JoinRefType ref_type = JoinRefType::REGULAR);
98
99
 
99
100
  // CROSS PRODUCT operation
100
- DUCKDB_API shared_ptr<Relation> CrossProduct(const shared_ptr<Relation> &other);
101
+ DUCKDB_API shared_ptr<Relation> CrossProduct(const shared_ptr<Relation> &other,
102
+ JoinRefType join_ref_type = JoinRefType::CROSS);
101
103
 
102
104
  // SET operations
103
105
  DUCKDB_API shared_ptr<Relation> Union(const shared_ptr<Relation> &other);
@@ -84,6 +84,15 @@ struct DebugAsOfIEJoin {
84
84
  static Value GetSetting(ClientContext &context);
85
85
  };
86
86
 
87
+ struct PreferRangeJoins {
88
+ static constexpr const char *Name = "prefer_range_joins"; // NOLINT
89
+ static constexpr const char *Description = "Force use of range joins with mixed predicates"; // NOLINT
90
+ static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN; // NOLINT
91
+ static void SetLocal(ClientContext &context, const Value &parameter);
92
+ static void ResetLocal(ClientContext &context);
93
+ static Value GetSetting(ClientContext &context);
94
+ };
95
+
87
96
  struct DebugWindowMode {
88
97
  static constexpr const char *Name = "debug_window_mode";
89
98
  static constexpr const char *Description = "DEBUG SETTING: switch window mode to use";
@@ -122,6 +131,15 @@ struct DefaultNullOrderSetting {
122
131
  static Value GetSetting(ClientContext &context);
123
132
  };
124
133
 
134
+ struct DisabledFileSystemsSetting {
135
+ static constexpr const char *Name = "disabled_filesystems";
136
+ static constexpr const char *Description = "Disable specific file systems preventing access (e.g. LocalFileSystem)";
137
+ static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
138
+ static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
139
+ static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
140
+ static Value GetSetting(ClientContext &context);
141
+ };
142
+
125
143
  struct DisabledOptimizersSetting {
126
144
  static constexpr const char *Name = "disabled_optimizers";
127
145
  static constexpr const char *Description = "DEBUG SETTING: disable a specific set of optimizers (comma separated)";
@@ -366,10 +384,20 @@ struct PerfectHashThresholdSetting {
366
384
  static Value GetSetting(ClientContext &context);
367
385
  };
368
386
 
387
+ struct PivotFilterThreshold {
388
+ static constexpr const char *Name = "pivot_filter_threshold";
389
+ static constexpr const char *Description =
390
+ "The threshold to switch from using filtered aggregates to LIST with a dedicated pivot operator";
391
+ static constexpr const LogicalTypeId InputType = LogicalTypeId::BIGINT;
392
+ static void SetLocal(ClientContext &context, const Value &parameter);
393
+ static void ResetLocal(ClientContext &context);
394
+ static Value GetSetting(ClientContext &context);
395
+ };
396
+
369
397
  struct PivotLimitSetting {
370
398
  static constexpr const char *Name = "pivot_limit";
371
399
  static constexpr const char *Description =
372
- "The maximum numer of pivot columns in a pivot statement (default: 100000)";
400
+ "The maximum number of pivot columns in a pivot statement (default: 100000)";
373
401
  static constexpr const LogicalTypeId InputType = LogicalTypeId::BIGINT;
374
402
  static void SetLocal(ClientContext &context, const Value &parameter);
375
403
  static void ResetLocal(ClientContext &context);
@@ -492,4 +520,14 @@ struct UsernameSetting {
492
520
  static Value GetSetting(ClientContext &context);
493
521
  };
494
522
 
523
+ struct FlushAllocatorSetting {
524
+ static constexpr const char *Name = "allocator_flush_threshold";
525
+ static constexpr const char *Description =
526
+ "Peak allocation threshold at which to flush the allocator after completing a task.";
527
+ static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
528
+ static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
529
+ static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
530
+ static Value GetSetting(ClientContext &context);
531
+ };
532
+
495
533
  } // namespace duckdb
@@ -0,0 +1,47 @@
1
+ //===----------------------------------------------------------------------===//
2
+ // DuckDB
3
+ //
4
+ // duckdb/optimizer/column_binding_replacer.hpp
5
+ //
6
+ //
7
+ //===----------------------------------------------------------------------===//
8
+
9
+ #pragma once
10
+
11
+ #include "duckdb/planner/logical_operator.hpp"
12
+
13
+ namespace duckdb {
14
+
15
+ struct ReplacementBinding {
16
+ public:
17
+ ReplacementBinding(ColumnBinding old_binding, ColumnBinding new_binding);
18
+ ReplacementBinding(ColumnBinding old_binding, ColumnBinding new_binding, LogicalType new_type);
19
+
20
+ public:
21
+ ColumnBinding old_binding;
22
+ ColumnBinding new_binding;
23
+
24
+ bool replace_type;
25
+ LogicalType new_type;
26
+ };
27
+
28
+ //! The ColumnBindingReplacer updates column bindings (e.g., after changing the operator plan), utility for optimizers
29
+ class ColumnBindingReplacer : LogicalOperatorVisitor {
30
+ public:
31
+ ColumnBindingReplacer();
32
+
33
+ public:
34
+ //! Update each operator of the plan
35
+ void VisitOperator(LogicalOperator &op) override;
36
+ //! Visit an expression and update its column bindings
37
+ void VisitExpression(unique_ptr<Expression> *expression) override;
38
+
39
+ public:
40
+ //! Contains all bindings that need to be updated
41
+ vector<ReplacementBinding> replacement_bindings;
42
+
43
+ //! Do not recurse further than this operator (optional)
44
+ optional_ptr<LogicalOperator> stop_operator;
45
+ };
46
+
47
+ } // namespace duckdb
@@ -0,0 +1,132 @@
1
+ //===----------------------------------------------------------------------===//
2
+ // DuckDB
3
+ //
4
+ // duckdb/optimizer/compressed_materialization.hpp
5
+ //
6
+ //
7
+ //===----------------------------------------------------------------------===//
8
+
9
+ #pragma once
10
+
11
+ #include "duckdb/common/unordered_set.hpp"
12
+ #include "duckdb/function/scalar/compressed_materialization_functions.hpp"
13
+ #include "duckdb/planner/column_binding_map.hpp"
14
+ #include "duckdb/storage/statistics/base_statistics.hpp"
15
+
16
+ namespace duckdb {
17
+
18
+ class LogicalOperator;
19
+ struct JoinCondition;
20
+
21
+ struct CMChildInfo {
22
+ public:
23
+ CMChildInfo(LogicalOperator &op, const column_binding_set_t &referenced_bindings);
24
+
25
+ public:
26
+ //! Bindings and types before compressing
27
+ vector<ColumnBinding> bindings_before;
28
+ vector<LogicalType> &types;
29
+ //! Whether the input binding is eligible for compression
30
+ vector<bool> can_compress;
31
+
32
+ //! Bindings after compressing (projection on top)
33
+ vector<ColumnBinding> bindings_after;
34
+ };
35
+
36
+ struct CMBindingInfo {
37
+ public:
38
+ explicit CMBindingInfo(ColumnBinding binding, const LogicalType &type);
39
+
40
+ public:
41
+ ColumnBinding binding;
42
+
43
+ //! Type before compressing
44
+ LogicalType type;
45
+ bool needs_decompression;
46
+ unique_ptr<BaseStatistics> stats;
47
+ };
48
+
49
+ struct CompressedMaterializationInfo {
50
+ public:
51
+ CompressedMaterializationInfo(LogicalOperator &op, vector<idx_t> &&child_idxs,
52
+ const column_binding_set_t &referenced_bindings);
53
+
54
+ public:
55
+ //! Mapping from incoming bindings to outgoing bindings
56
+ column_binding_map_t<CMBindingInfo> binding_map;
57
+
58
+ //! Operator child info
59
+ vector<idx_t> child_idxs;
60
+ vector<CMChildInfo> child_info;
61
+ };
62
+
63
+ struct CompressExpression {
64
+ public:
65
+ CompressExpression(unique_ptr<Expression> expression, unique_ptr<BaseStatistics> stats);
66
+
67
+ public:
68
+ unique_ptr<Expression> expression;
69
+ unique_ptr<BaseStatistics> stats;
70
+ };
71
+
72
+ typedef column_binding_map_t<unique_ptr<BaseStatistics>> statistics_map_t;
73
+
74
+ //! The CompressedMaterialization optimizer compressed columns using projections, based on available statistics,
75
+ //! but only if the data enters a materializing operator
76
+ class CompressedMaterialization {
77
+ public:
78
+ explicit CompressedMaterialization(ClientContext &context, Binder &binder, statistics_map_t &&statistics_map);
79
+
80
+ void Compress(unique_ptr<LogicalOperator> &op);
81
+
82
+ private:
83
+ //! Depth-first traversal of the plan
84
+ void CompressInternal(unique_ptr<LogicalOperator> &op);
85
+
86
+ //! Compress materializing operators
87
+ void CompressAggregate(unique_ptr<LogicalOperator> &op);
88
+ void CompressDistinct(unique_ptr<LogicalOperator> &op);
89
+ void CompressOrder(unique_ptr<LogicalOperator> &op);
90
+
91
+ //! Update statistics after compressing
92
+ void UpdateAggregateStats(unique_ptr<LogicalOperator> &op);
93
+ void UpdateOrderStats(unique_ptr<LogicalOperator> &op);
94
+
95
+ //! Adds bindings referenced in expression to referenced_bindings
96
+ static void GetReferencedBindings(const Expression &expression, column_binding_set_t &referenced_bindings);
97
+ //! Updates CMBindingInfo in the binding_map in info
98
+ void UpdateBindingInfo(CompressedMaterializationInfo &info, const ColumnBinding &binding, bool needs_decompression);
99
+
100
+ //! Create (de)compress projections around the operator
101
+ void CreateProjections(unique_ptr<LogicalOperator> &op, CompressedMaterializationInfo &info);
102
+ bool TryCompressChild(CompressedMaterializationInfo &info, const CMChildInfo &child_info,
103
+ vector<unique_ptr<CompressExpression>> &compress_expressions);
104
+ void CreateCompressProjection(unique_ptr<LogicalOperator> &child_op,
105
+ vector<unique_ptr<CompressExpression>> &&compress_exprs,
106
+ CompressedMaterializationInfo &info, CMChildInfo &child_info);
107
+ void CreateDecompressProjection(unique_ptr<LogicalOperator> &op, CompressedMaterializationInfo &info);
108
+
109
+ //! Create expressions that apply a scalar compression function
110
+ unique_ptr<CompressExpression> GetCompressExpression(const ColumnBinding &binding, const LogicalType &type,
111
+ const bool &can_compress);
112
+ unique_ptr<CompressExpression> GetCompressExpression(unique_ptr<Expression> input, const BaseStatistics &stats);
113
+ unique_ptr<CompressExpression> GetIntegralCompress(unique_ptr<Expression> input, const BaseStatistics &stats);
114
+ unique_ptr<CompressExpression> GetStringCompress(unique_ptr<Expression> input, const BaseStatistics &stats);
115
+
116
+ //! Create an expression that applies a scalar decompression function
117
+ unique_ptr<Expression> GetDecompressExpression(unique_ptr<Expression> input, const LogicalType &result_type,
118
+ const BaseStatistics &stats);
119
+ unique_ptr<Expression> GetIntegralDecompress(unique_ptr<Expression> input, const LogicalType &result_type,
120
+ const BaseStatistics &stats);
121
+ unique_ptr<Expression> GetStringDecompress(unique_ptr<Expression> input, const BaseStatistics &stats);
122
+
123
+ private:
124
+ ClientContext &context;
125
+ Binder &binder;
126
+ statistics_map_t statistics_map;
127
+ unordered_set<idx_t> compression_table_indices;
128
+ unordered_set<idx_t> decompression_table_indices;
129
+ optional_ptr<LogicalOperator> root;
130
+ };
131
+
132
+ } // namespace duckdb
@@ -8,36 +8,33 @@
8
8
 
9
9
  #pragma once
10
10
 
11
- #include "duckdb/parser/expression_map.hpp"
12
- #include "duckdb/planner/column_binding_map.hpp"
13
- #include "duckdb/planner/logical_operator.hpp"
14
- #include "duckdb/planner/logical_operator_visitor.hpp"
11
+ #include "duckdb/optimizer/column_binding_replacer.hpp"
15
12
 
16
13
  namespace duckdb {
17
14
 
18
- class Optimizer;
19
- class DeliminatorPlanUpdater;
15
+ struct DelimCandidate;
20
16
 
21
17
  //! The Deliminator optimizer traverses the logical operator tree and removes any redundant DelimGets/DelimJoins
22
18
  class Deliminator {
23
19
  public:
24
- explicit Deliminator(ClientContext &context) : context(context) {
20
+ Deliminator() {
25
21
  }
26
22
  //! Perform DelimJoin elimination
27
23
  unique_ptr<LogicalOperator> Optimize(unique_ptr<LogicalOperator> op);
28
24
 
29
25
  private:
30
- //! Find Joins with a DelimGet that can be removed
31
- void FindCandidates(unique_ptr<LogicalOperator> *op_ptr, vector<unique_ptr<LogicalOperator> *> &candidates);
32
- //! Try to remove a Join with a DelimGet, returns true if it was successful
33
- bool RemoveCandidate(unique_ptr<LogicalOperator> *plan, unique_ptr<LogicalOperator> *candidate,
34
- DeliminatorPlanUpdater &updater);
35
- //! Try to remove an inequality Join with a DelimGet, returns true if it was successful
36
- bool RemoveInequalityCandidate(unique_ptr<LogicalOperator> *plan, unique_ptr<LogicalOperator> *candidate,
37
- DeliminatorPlanUpdater &updater);
26
+ //! Finds DelimJoins and their corresponding DelimGets
27
+ void FindCandidates(unique_ptr<LogicalOperator> &op, vector<DelimCandidate> &candidates);
28
+ void FindJoinWithDelimGet(unique_ptr<LogicalOperator> &op, DelimCandidate &candidate);
29
+ //! Remove joins with a DelimGet
30
+ bool RemoveJoinWithDelimGet(LogicalDelimJoin &delim_join, const idx_t delim_get_count,
31
+ unique_ptr<LogicalOperator> &join, bool &all_equality_conditions);
32
+ bool RemoveInequalityJoinWithDelimGet(LogicalDelimJoin &delim_join, const idx_t delim_get_count,
33
+ unique_ptr<LogicalOperator> &join,
34
+ const vector<ReplacementBinding> &replacement_bindings);
38
35
 
39
36
  private:
40
- ClientContext &context;
37
+ optional_ptr<LogicalOperator> root;
41
38
  };
42
39
 
43
40
  } // namespace duckdb
@@ -19,8 +19,11 @@ class Optimizer;
19
19
  class FilterPushdown {
20
20
  public:
21
21
  explicit FilterPushdown(Optimizer &optimizer);
22
+
22
23
  //! Perform filter pushdown
23
24
  unique_ptr<LogicalOperator> Rewrite(unique_ptr<LogicalOperator> op);
25
+ //! Return a reference to the client context (from the optimizer)
26
+ ClientContext &GetContext();
24
27
 
25
28
  struct Filter {
26
29
  unordered_set<idx_t> bindings;
@@ -34,7 +34,7 @@ struct RelationsToTDom {
34
34
  bool has_tdom_hll;
35
35
  vector<FilterInfo *> filters;
36
36
 
37
- RelationsToTDom(column_binding_set_t column_binding_set)
37
+ RelationsToTDom(const column_binding_set_t &column_binding_set)
38
38
  : equivalent_relations(column_binding_set), tdom_hll(0), tdom_no_hll(NumericLimits<idx_t>::Maximum()),
39
39
  has_tdom_hll(false) {};
40
40
  };
@@ -27,7 +27,10 @@ public:
27
27
  T GetCardinality() const {
28
28
  throw NotImplementedException("Unsupported type for GetCardinality");
29
29
  }
30
- double GetCost() const;
30
+ template <class T>
31
+ T GetCost() const {
32
+ throw NotImplementedException("Unsupported type for GetCost");
33
+ }
31
34
  void SetCost(double new_cost);
32
35
  void SetCardinality(double cardinality);
33
36
 
@@ -45,4 +48,10 @@ double EstimatedProperties::GetCardinality() const;
45
48
  template <>
46
49
  idx_t EstimatedProperties::GetCardinality() const;
47
50
 
51
+ template <>
52
+ double EstimatedProperties::GetCost() const;
53
+
54
+ template <>
55
+ idx_t EstimatedProperties::GetCost() const;
56
+
48
57
  } // namespace duckdb
@@ -90,7 +90,7 @@ private:
90
90
  //! cancelling the dynamic programming step.
91
91
  bool TryEmitPair(JoinRelationSet &left, JoinRelationSet &right, const vector<reference<NeighborInfo>> &info);
92
92
 
93
- bool EnumerateCmpRecursive(JoinRelationSet &left, JoinRelationSet &right, unordered_set<idx_t> exclusion_set);
93
+ bool EnumerateCmpRecursive(JoinRelationSet &left, JoinRelationSet &right, unordered_set<idx_t> &exclusion_set);
94
94
  //! Emit a relation set node
95
95
  bool EmitCSG(JoinRelationSet &node);
96
96
  //! Enumerate the possible connected subgraphs that can be joined together in the join graph
@@ -53,7 +53,7 @@ public:
53
53
  //! Create or get a JoinRelationSet from a single node with the given index
54
54
  JoinRelationSet &GetJoinRelation(idx_t index);
55
55
  //! Create or get a JoinRelationSet from a set of relation bindings
56
- JoinRelationSet &GetJoinRelation(unordered_set<idx_t> &bindings);
56
+ JoinRelationSet &GetJoinRelation(const unordered_set<idx_t> &bindings);
57
57
  //! Create or get a JoinRelationSet from a (sorted, duplicate-free!) list of relations
58
58
  JoinRelationSet &GetJoinRelation(unsafe_unique_array<idx_t> relations, idx_t count);
59
59
  //! Union two sets of relations together and create a new relation set
@@ -75,6 +75,9 @@ private:
75
75
  //! Get the QueryEdge of a specific node
76
76
  QueryEdge &GetQueryEdge(JoinRelationSet &left);
77
77
 
78
+ void EnumerateNeighborsDFS(JoinRelationSet &node, reference<QueryEdge> info, idx_t index,
79
+ const std::function<bool(NeighborInfo &)> &callback);
80
+
78
81
  QueryEdge root;
79
82
  };
80
83
 
@@ -23,6 +23,8 @@ public:
23
23
  UNORDERED,
24
24
  //! Only some entries have to be matched, the order of the matches does not matter
25
25
  SOME,
26
+ //! Only some entries have to be matched. The order of the matches does matter.
27
+ SOME_ORDERED,
26
28
  //! Not initialized
27
29
  INVALID
28
30
  };
@@ -79,6 +81,17 @@ public:
79
81
  }
80
82
  }
81
83
  return true;
84
+ } else if (policy == Policy::SOME_ORDERED) {
85
+ if (entries.size() < matchers.size()) {
86
+ return false;
87
+ }
88
+ // now provided entries have to match in order
89
+ for (idx_t i = 0; i < matchers.size(); i++) {
90
+ if (!matchers[i]->Match(entries[i], bindings)) {
91
+ return false;
92
+ }
93
+ }
94
+ return true;
82
95
  } else {
83
96
  if (policy == Policy::UNORDERED && matchers.size() != entries.size()) {
84
97
  // unordered policy, count does not match: no match
@@ -22,7 +22,10 @@ class Optimizer {
22
22
  public:
23
23
  Optimizer(Binder &binder, ClientContext &context);
24
24
 
25
+ //! Optimize a plan by running specialized optimizers
25
26
  unique_ptr<LogicalOperator> Optimize(unique_ptr<LogicalOperator> plan);
27
+ //! Return a reference to the client context of this optimizer
28
+ ClientContext &GetContext();
26
29
 
27
30
  ClientContext &context;
28
31
  Binder &binder;
@@ -0,0 +1,40 @@
1
+ //===----------------------------------------------------------------------===//
2
+ // DuckDB
3
+ //
4
+ // duckdb/optimizer/remove_duplicate_groups.hpp
5
+ //
6
+ //
7
+ //===----------------------------------------------------------------------===//
8
+
9
+ #pragma once
10
+
11
+ #include "duckdb/planner/column_binding_map.hpp"
12
+ #include "duckdb/planner/logical_operator_visitor.hpp"
13
+
14
+ namespace duckdb {
15
+
16
+ class BoundColumnRefExpression;
17
+
18
+ //! The RemoveDuplicateGroups optimizer traverses the logical operator tree and removes any duplicate aggregate groups
19
+ //! Duplicate groups may be introduced when joins columns are removed, e.g., by Deliminator or RemoveUnusedColumns
20
+ class RemoveDuplicateGroups : public LogicalOperatorVisitor {
21
+ public:
22
+ RemoveDuplicateGroups() {
23
+ }
24
+
25
+ void VisitOperator(LogicalOperator &op) override;
26
+
27
+ private:
28
+ void VisitAggregate(LogicalAggregate &aggr);
29
+
30
+ protected:
31
+ unique_ptr<Expression> VisitReplace(BoundColumnRefExpression &expr, unique_ptr<Expression> *expr_ptr) override;
32
+
33
+ private:
34
+ //! The map of column references
35
+ column_binding_map_t<vector<reference<BoundColumnRefExpression>>> column_references;
36
+ //! Stored expressions (kept around so we don't have dangling pointers)
37
+ vector<unique_ptr<Expression>> stored_expressions;
38
+ };
39
+
40
+ } // namespace duckdb
@@ -18,6 +18,8 @@
18
18
  #include "duckdb/storage/statistics/node_statistics.hpp"
19
19
 
20
20
  namespace duckdb {
21
+
22
+ class Optimizer;
21
23
  class ClientContext;
22
24
  class LogicalOperator;
23
25
  class TableFilter;
@@ -25,10 +27,14 @@ struct BoundOrderByNode;
25
27
 
26
28
  class StatisticsPropagator {
27
29
  public:
28
- explicit StatisticsPropagator(ClientContext &context);
30
+ explicit StatisticsPropagator(Optimizer &optimizer);
29
31
 
30
32
  unique_ptr<NodeStatistics> PropagateStatistics(unique_ptr<LogicalOperator> &node_ptr);
31
33
 
34
+ column_binding_map_t<unique_ptr<BaseStatistics>> GetStatisticsMap() {
35
+ return std::move(statistics_map);
36
+ }
37
+
32
38
  private:
33
39
  //! Propagate statistics through an operator
34
40
  unique_ptr<NodeStatistics> PropagateStatistics(LogicalOperator &node, unique_ptr<LogicalOperator> *node_ptr);
@@ -75,6 +81,9 @@ private:
75
81
  //! Multiply the cardinalities together (i.e. new max cardinality is stats.max * new_stats.max): used for
76
82
  //! joins/cross products
77
83
  void MultiplyCardinalities(unique_ptr<NodeStatistics> &stats, NodeStatistics &new_stats);
84
+ //! Creates and pushes down a filter based on join statistics
85
+ void CreateFilterFromJoinStats(unique_ptr<LogicalOperator> &child, unique_ptr<Expression> &expr,
86
+ const BaseStatistics &stats_before, const BaseStatistics &stats_after);
78
87
 
79
88
  unique_ptr<BaseStatistics> PropagateExpression(unique_ptr<Expression> &expr);
80
89
  unique_ptr<BaseStatistics> PropagateExpression(Expression &expr, unique_ptr<Expression> *expr_ptr);
@@ -90,14 +99,13 @@ private:
90
99
  unique_ptr<BaseStatistics> PropagateExpression(BoundColumnRefExpression &expr, unique_ptr<Expression> *expr_ptr);
91
100
  unique_ptr<BaseStatistics> PropagateExpression(BoundOperatorExpression &expr, unique_ptr<Expression> *expr_ptr);
92
101
 
93
- void PropagateAndCompress(unique_ptr<Expression> &expr, unique_ptr<BaseStatistics> &stats);
94
-
95
102
  void ReplaceWithEmptyResult(unique_ptr<LogicalOperator> &node);
96
103
 
97
104
  bool ExpressionIsConstant(Expression &expr, const Value &val);
98
105
  bool ExpressionIsConstantOrNull(Expression &expr, const Value &val);
99
106
 
100
107
  private:
108
+ Optimizer &optimizer;
101
109
  ClientContext &context;
102
110
  //! The map of ColumnBinding -> statistics for the various nodes
103
111
  column_binding_map_t<unique_ptr<BaseStatistics>> statistics_map;
@@ -18,6 +18,8 @@ class TopN {
18
18
  public:
19
19
  //! Optimize ORDER BY + LIMIT to TopN
20
20
  unique_ptr<LogicalOperator> Optimize(unique_ptr<LogicalOperator> op);
21
+ //! Whether we can perform the optimization on this operator
22
+ static bool CanOptimize(LogicalOperator &op);
21
23
  };
22
24
 
23
25
  } // namespace duckdb
@@ -30,6 +30,8 @@ public:
30
30
  public:
31
31
  //! Duplicate eliminated join scan dependencies
32
32
  reference_map_t<const PhysicalOperator, reference<Pipeline>> delim_join_dependencies;
33
+ //! Materialized CTE scan dependencies
34
+ reference_map_t<const PhysicalOperator, reference<Pipeline>> cte_dependencies;
33
35
 
34
36
  public:
35
37
  void SetPipelineSource(Pipeline &pipeline, PhysicalOperator &op);
@@ -67,6 +67,9 @@ public:
67
67
  //! Send signals to n threads, signalling for them to wake up and attempt to execute a task
68
68
  void Signal(idx_t n);
69
69
 
70
+ //! Set the allocator flush threshold
71
+ void SetAllocatorFlushTreshold(idx_t threshold);
72
+
70
73
  private:
71
74
  void SetThreadsInternal(int32_t n);
72
75
 
@@ -80,6 +83,8 @@ private:
80
83
  vector<unique_ptr<SchedulerThread>> threads;
81
84
  //! Markers used by the various threads, if the markers are set to "false" the thread execution is stopped
82
85
  vector<unique_ptr<atomic<bool>>> markers;
86
+ //! The threshold after which to flush the allocator after completing a task
87
+ atomic<idx_t> allocator_flush_threshold;
83
88
  };
84
89
 
85
90
  } // namespace duckdb
@@ -9,6 +9,7 @@
9
9
  #pragma once
10
10
 
11
11
  #include "duckdb/parser/statement/select_statement.hpp"
12
+ #include "duckdb/common/enums/cte_materialize.hpp"
12
13
 
13
14
  namespace duckdb {
14
15
 
@@ -17,6 +18,7 @@ class SelectStatement;
17
18
  struct CommonTableExpressionInfo {
18
19
  vector<string> aliases;
19
20
  unique_ptr<SelectStatement> query;
21
+ CTEMaterialize materialized = CTEMaterialize::CTE_MATERIALIZE_DEFAULT;
20
22
 
21
23
  void FormatSerialize(FormatSerializer &serializer) const;
22
24
  static unique_ptr<CommonTableExpressionInfo> FormatDeserialize(FormatDeserializer &deserializer);
@@ -43,5 +43,8 @@ public:
43
43
  return "(" + entry.input->ToString() + " BETWEEN " + entry.lower->ToString() + " AND " +
44
44
  entry.upper->ToString() + ")";
45
45
  }
46
+
47
+ private:
48
+ BetweenExpression();
46
49
  };
47
50
  } // namespace duckdb
@@ -46,5 +46,8 @@ public:
46
46
  return (entry.try_cast ? "TRY_CAST(" : "CAST(") + entry.child->ToString() + " AS " +
47
47
  entry.cast_type.ToString() + ")";
48
48
  }
49
+
50
+ private:
51
+ CastExpression();
49
52
  };
50
53
  } // namespace duckdb