duckdb 0.8.2-dev150.0 → 0.8.2-dev1559.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (489) hide show
  1. package/binding.gyp +15 -12
  2. package/binding.gyp.in +1 -1
  3. package/configure.py +1 -1
  4. package/duckdb_extension_config.cmake +10 -0
  5. package/package.json +1 -1
  6. package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
  7. package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
  8. package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
  9. package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
  10. package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
  11. package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
  12. package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
  13. package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
  14. package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
  15. package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
  16. package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
  17. package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
  18. package/src/duckdb/extension/json/include/json_common.hpp +47 -231
  19. package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
  20. package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
  21. package/src/duckdb/extension/json/json_common.cpp +272 -40
  22. package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
  23. package/src/duckdb/extension/json/json_functions/json_transform.cpp +17 -37
  24. package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
  25. package/src/duckdb/extension/json/json_functions.cpp +24 -24
  26. package/src/duckdb/extension/json/json_scan.cpp +3 -6
  27. package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
  28. package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
  29. package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
  30. package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
  31. package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
  32. package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
  33. package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
  34. package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
  35. package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
  36. package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
  37. package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
  38. package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
  39. package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
  40. package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
  41. package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
  42. package/src/duckdb/extension/parquet/parquet_extension.cpp +192 -20
  43. package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
  44. package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
  45. package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
  46. package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
  47. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
  48. package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
  49. package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
  50. package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
  51. package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
  52. package/src/duckdb/src/common/allocator.cpp +14 -2
  53. package/src/duckdb/src/common/arrow/arrow_appender.cpp +5 -11
  54. package/src/duckdb/src/common/assert.cpp +3 -0
  55. package/src/duckdb/src/common/enum_util.cpp +4619 -4446
  56. package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
  57. package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
  58. package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
  59. package/src/duckdb/src/common/exception.cpp +2 -2
  60. package/src/duckdb/src/common/extra_type_info.cpp +506 -0
  61. package/src/duckdb/src/common/file_system.cpp +19 -0
  62. package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
  63. package/src/duckdb/src/common/local_file_system.cpp +14 -14
  64. package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
  65. package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
  66. package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
  67. package/src/duckdb/src/common/re2_regex.cpp +1 -1
  68. package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
  69. package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
  70. package/src/duckdb/src/common/sort/partition_state.cpp +44 -11
  71. package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
  72. package/src/duckdb/src/common/types/bit.cpp +51 -0
  73. package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
  74. package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
  75. package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
  76. package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
  77. package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
  78. package/src/duckdb/src/common/types/date.cpp +9 -0
  79. package/src/duckdb/src/common/types/list_segment.cpp +24 -74
  80. package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
  81. package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
  82. package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
  83. package/src/duckdb/src/common/types/uuid.cpp +2 -2
  84. package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
  85. package/src/duckdb/src/common/types.cpp +8 -655
  86. package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
  87. package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
  88. package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
  89. package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
  90. package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
  91. package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
  92. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
  93. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
  94. package/src/duckdb/src/core_functions/function_list.cpp +4 -2
  95. package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
  96. package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
  97. package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
  98. package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
  99. package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
  100. package/src/duckdb/src/execution/expression_executor.cpp +1 -1
  101. package/src/duckdb/src/execution/index/art/art.cpp +149 -139
  102. package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
  103. package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
  104. package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
  105. package/src/duckdb/src/execution/index/art/node.cpp +113 -120
  106. package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
  107. package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
  108. package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
  109. package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
  110. package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
  111. package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
  112. package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
  113. package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
  114. package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
  115. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
  116. package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
  117. package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
  118. package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +414 -283
  119. package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
  120. package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
  121. package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +28 -12
  122. package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
  123. package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +23 -4
  124. package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +41 -5
  125. package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
  126. package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
  127. package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
  128. package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
  129. package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
  130. package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
  131. package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
  132. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
  133. package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
  134. package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
  135. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
  136. package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
  137. package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
  138. package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
  139. package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
  140. package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
  141. package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +13 -22
  142. package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +17 -13
  143. package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
  144. package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
  145. package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
  146. package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
  147. package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
  148. package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
  149. package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
  150. package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
  151. package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
  152. package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
  153. package/src/duckdb/src/function/function.cpp +3 -1
  154. package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
  155. package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
  156. package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
  157. package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
  158. package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
  159. package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
  160. package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
  161. package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
  162. package/src/duckdb/src/function/table/read_csv.cpp +100 -17
  163. package/src/duckdb/src/function/table/table_scan.cpp +9 -0
  164. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  165. package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
  166. package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
  167. package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
  168. package/src/duckdb/src/include/duckdb/common/dl.hpp +3 -1
  169. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +616 -584
  170. package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
  171. package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
  172. package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
  173. package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
  174. package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
  175. package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
  176. package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +219 -0
  177. package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
  178. package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
  179. package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
  180. package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
  181. package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
  182. package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
  183. package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
  184. package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
  185. package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
  186. package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
  187. package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +13 -0
  188. package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
  189. package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
  190. package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
  191. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
  192. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
  193. package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
  194. package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
  195. package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
  196. package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
  197. package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +0 -1
  198. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
  199. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
  200. package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
  201. package/src/duckdb/src/include/duckdb/common/types.hpp +1 -15
  202. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
  203. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
  204. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
  205. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
  206. package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
  207. package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
  208. package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
  209. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
  210. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
  211. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
  212. package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
  213. package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
  214. package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
  215. package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
  216. package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
  217. package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
  218. package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
  219. package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
  220. package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
  221. package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
  222. package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
  223. package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
  224. package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
  225. package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
  226. package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
  227. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
  228. package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
  229. package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
  230. package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
  231. package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
  232. package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
  233. package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
  234. package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
  235. package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
  236. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +2 -10
  237. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +1 -1
  238. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +1 -1
  239. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp +12 -1
  240. package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
  241. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
  242. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
  243. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
  244. package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
  245. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
  246. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
  247. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
  248. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
  249. package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
  250. package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
  251. package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
  252. package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
  253. package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
  254. package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
  255. package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
  256. package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
  257. package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
  258. package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
  259. package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
  260. package/src/duckdb/src/include/duckdb/main/client_config.hpp +5 -0
  261. package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
  262. package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
  263. package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
  264. package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
  265. package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
  266. package/src/duckdb/src/include/duckdb/main/settings.hpp +39 -1
  267. package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
  268. package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
  269. package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
  270. package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
  271. package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
  272. package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
  273. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
  274. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
  275. package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
  276. package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
  277. package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
  278. package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
  279. package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
  280. package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
  281. package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
  282. package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
  283. package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
  284. package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
  285. package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
  286. package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
  287. package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
  288. package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
  289. package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
  290. package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
  291. package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
  292. package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
  293. package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
  294. package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
  295. package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
  296. package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
  297. package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
  298. package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
  299. package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
  300. package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
  301. package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
  302. package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
  303. package/src/duckdb/src/include/duckdb/planner/binder.hpp +12 -5
  304. package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
  305. package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
  306. package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
  307. package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
  308. package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
  309. package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
  310. package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
  311. package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
  312. package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
  313. package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
  314. package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
  315. package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
  316. package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
  317. package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
  318. package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
  319. package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
  320. package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
  321. package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
  322. package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
  323. package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
  324. package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
  325. package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
  326. package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
  327. package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
  328. package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
  329. package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
  330. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
  331. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
  332. package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
  333. package/src/duckdb/src/include/duckdb.h +28 -0
  334. package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
  335. package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
  336. package/src/duckdb/src/main/config.cpp +4 -0
  337. package/src/duckdb/src/main/database.cpp +1 -1
  338. package/src/duckdb/src/main/extension/extension_helper.cpp +96 -89
  339. package/src/duckdb/src/main/extension/extension_install.cpp +6 -0
  340. package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
  341. package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
  342. package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
  343. package/src/duckdb/src/main/relation.cpp +6 -5
  344. package/src/duckdb/src/main/settings/settings.cpp +79 -18
  345. package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
  346. package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
  347. package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
  348. package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
  349. package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
  350. package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
  351. package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
  352. package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
  353. package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
  354. package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
  355. package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
  356. package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
  357. package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
  358. package/src/duckdb/src/optimizer/optimizer.cpp +51 -14
  359. package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
  360. package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
  361. package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
  362. package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
  363. package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
  364. package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
  365. package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
  366. package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
  367. package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
  368. package/src/duckdb/src/parallel/executor.cpp +15 -0
  369. package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
  370. package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
  371. package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
  372. package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
  373. package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
  374. package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
  375. package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
  376. package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
  377. package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
  378. package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
  379. package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
  380. package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
  381. package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
  382. package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
  383. package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
  384. package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
  385. package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
  386. package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
  387. package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
  388. package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
  389. package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
  390. package/src/duckdb/src/parser/parser.cpp +8 -2
  391. package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
  392. package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
  393. package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
  394. package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
  395. package/src/duckdb/src/parser/query_node.cpp +15 -37
  396. package/src/duckdb/src/parser/result_modifier.cpp +0 -74
  397. package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
  398. package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
  399. package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
  400. package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
  401. package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
  402. package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
  403. package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
  404. package/src/duckdb/src/parser/tableref.cpp +0 -44
  405. package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
  406. package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
  407. package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
  408. package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
  409. package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
  410. package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
  411. package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
  412. package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
  413. package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
  414. package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
  415. package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
  416. package/src/duckdb/src/parser/transformer.cpp +15 -0
  417. package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
  418. package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
  419. package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
  420. package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
  421. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
  422. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
  423. package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
  424. package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
  425. package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
  426. package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
  427. package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +61 -26
  428. package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
  429. package/src/duckdb/src/planner/binder.cpp +44 -31
  430. package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
  431. package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
  432. package/src/duckdb/src/planner/expression_binder.cpp +3 -0
  433. package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
  434. package/src/duckdb/src/planner/logical_operator.cpp +5 -0
  435. package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
  436. package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
  437. package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
  438. package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
  439. package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
  440. package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
  441. package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
  442. package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
  443. package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
  444. package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
  445. package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
  446. package/src/duckdb/src/storage/compression/rle.cpp +0 -1
  447. package/src/duckdb/src/storage/data_table.cpp +1 -1
  448. package/src/duckdb/src/storage/local_storage.cpp +3 -3
  449. package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
  450. package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
  451. package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
  452. package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
  453. package/src/duckdb/src/storage/serialization/serialize_types.cpp +127 -0
  454. package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
  455. package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
  456. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  457. package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
  458. package/src/duckdb/src/storage/table/row_group.cpp +25 -9
  459. package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
  460. package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
  461. package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
  462. package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
  463. package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
  464. package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
  465. package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
  466. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
  467. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
  468. package/src/duckdb/ub_src_common.cpp +2 -0
  469. package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
  470. package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
  471. package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
  472. package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
  473. package/src/duckdb/ub_src_function_scalar.cpp +2 -0
  474. package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
  475. package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
  476. package/src/duckdb/ub_src_optimizer.cpp +6 -0
  477. package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
  478. package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
  479. package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
  480. package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
  481. package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
  482. package/src/duckdb/ub_src_planner_operator.cpp +4 -0
  483. package/src/duckdb/ub_src_storage_serialization.cpp +10 -0
  484. package/src/statement.cpp +10 -3
  485. package/test/test_all_types.test.ts +233 -0
  486. package/tsconfig.json +1 -0
  487. package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
  488. package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
  489. package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -32,6 +32,8 @@ public:
32
32
  //! Recursive CTEs require at least one ChunkScan, referencing the working_table.
33
33
  //! This data structure is used to establish it.
34
34
  unordered_map<idx_t, std::shared_ptr<ColumnDataCollection>> recursive_cte_tables;
35
+ //! Materialized CTE ids must be collected.
36
+ unordered_set<idx_t> materialized_ctes;
35
37
 
36
38
  public:
37
39
  //! Creates a plan from the logical operator. This involves resolving column bindings and generating physical
@@ -87,6 +89,7 @@ protected:
87
89
  unique_ptr<PhysicalOperator> CreatePlan(LogicalSimple &op);
88
90
  unique_ptr<PhysicalOperator> CreatePlan(LogicalUnnest &op);
89
91
  unique_ptr<PhysicalOperator> CreatePlan(LogicalRecursiveCTE &op);
92
+ unique_ptr<PhysicalOperator> CreatePlan(LogicalMaterializedCTE &op);
90
93
  unique_ptr<PhysicalOperator> CreatePlan(LogicalCTERef &op);
91
94
  unique_ptr<PhysicalOperator> CreatePlan(LogicalPivot &op);
92
95
 
@@ -8,10 +8,10 @@
8
8
 
9
9
  #pragma once
10
10
 
11
+ #include "duckdb/execution/operator/aggregate/grouped_aggregate_data.hpp"
11
12
  #include "duckdb/execution/partitionable_hashtable.hpp"
12
- #include "duckdb/parser/group_by_node.hpp"
13
13
  #include "duckdb/execution/physical_operator.hpp"
14
- #include "duckdb/execution/operator/aggregate/grouped_aggregate_data.hpp"
14
+ #include "duckdb/parser/group_by_node.hpp"
15
15
 
16
16
  namespace duckdb {
17
17
  class BufferManager;
@@ -57,11 +57,18 @@ public:
57
57
  OperatorSourceInput &input) const;
58
58
 
59
59
  static void SetMultiScan(GlobalSinkState &state);
60
- bool ForceSingleHT(GlobalSinkState &state) const;
60
+ static bool ForceSingleHT(GlobalSinkState &state);
61
+ static bool AnyPartitioned(GlobalSinkState &state);
62
+ static void GetRepartitionInfo(ClientContext &context, GlobalSinkState &state, idx_t &repartition_radix_bits,
63
+ idx_t &concurrent_repartitions, idx_t &tasks_per_partition);
61
64
 
62
65
  private:
63
66
  void SetGroupingValues();
64
67
  void PopulateGroupChunk(DataChunk &group_chunk, DataChunk &input_chunk) const;
68
+ void InitializeFinalizedHTs(ClientContext &context, GlobalSinkState &state) const;
69
+ void ScheduleRepartitionTasks(Executor &executor, const shared_ptr<Event> &event, GlobalSinkState &state,
70
+ vector<shared_ptr<Task>> &tasks, const idx_t repartition_radix_bits,
71
+ const idx_t concurrent_repartitions, const idx_t tasks_per_partition) const;
65
72
  };
66
73
 
67
74
  } // namespace duckdb
@@ -18,12 +18,13 @@ namespace duckdb {
18
18
 
19
19
  class WindowAggregateState {
20
20
  public:
21
- WindowAggregateState(AggregateObject aggr, const LogicalType &result_type_p);
21
+ WindowAggregateState(AggregateObject aggr, const LogicalType &result_type_p, idx_t partition_count);
22
22
  virtual ~WindowAggregateState();
23
23
 
24
24
  virtual void Sink(DataChunk &payload_chunk, SelectionVector *filter_sel, idx_t filtered);
25
25
  virtual void Finalize();
26
26
  virtual void Compute(Vector &result, idx_t rid, idx_t start, idx_t end);
27
+ virtual void Evaluate(const idx_t *begins, const idx_t *ends, Vector &result, idx_t count);
27
28
 
28
29
  protected:
29
30
  void AggregateInit();
@@ -33,20 +34,28 @@ protected:
33
34
  //! The result type of the window function
34
35
  LogicalType result_type;
35
36
 
37
+ //! The cardinality of the partition
38
+ const idx_t partition_count;
39
+ //! The size of a single aggregate state
40
+ const idx_t state_size;
36
41
  //! Data pointer that contains a single state, used for intermediate window segment aggregation
37
42
  vector<data_t> state;
38
43
  //! Reused result state container for the window functions
39
- Vector statev;
40
- //! A vector of pointers to "state", used for intermediate window segment aggregation
41
- Vector statep;
42
- //! Input data chunk, used for intermediate window segment aggregation
44
+ Vector statef;
45
+ //! Partition data chunk
43
46
  DataChunk inputs;
47
+
48
+ //! The filtered rows in inputs.
49
+ vector<validity_t> filter_bits;
50
+ ValidityMask filter_mask;
51
+ idx_t filter_pos;
52
+
53
+ //! Allocator for aggregates
54
+ ArenaAllocator allocator;
44
55
  };
45
56
 
46
57
  class WindowConstantAggregate : public WindowAggregateState {
47
58
  public:
48
- static bool IsConstantAggregate(const BoundWindowExpression &wexpr);
49
-
50
59
  WindowConstantAggregate(AggregateObject aggr, const LogicalType &result_type_p, const ValidityMask &partition_mask,
51
60
  const idx_t count);
52
61
  ~WindowConstantAggregate() override {
@@ -54,7 +63,7 @@ public:
54
63
 
55
64
  void Sink(DataChunk &payload_chunk, SelectionVector *filter_sel, idx_t filtered) override;
56
65
  void Finalize() override;
57
- void Compute(Vector &result, idx_t rid, idx_t start, idx_t end) override;
66
+ void Evaluate(const idx_t *begins, const idx_t *ends, Vector &result, idx_t count) override;
58
67
 
59
68
  private:
60
69
  //! Partition starts
@@ -65,52 +74,57 @@ private:
65
74
  idx_t partition;
66
75
  //! The current input row being built/read
67
76
  idx_t row;
77
+ //! A vector of pointers to "state", used for intermediate window segment aggregation
78
+ Vector statep;
79
+ //! Shared SV for evaluation
80
+ SelectionVector matches;
81
+ };
82
+
83
+ class WindowCustomAggregate : public WindowAggregateState {
84
+ public:
85
+ WindowCustomAggregate(AggregateObject aggr, const LogicalType &result_type_p, idx_t partition_count);
86
+ ~WindowCustomAggregate() override;
87
+
88
+ void Compute(Vector &result, idx_t rid, idx_t start, idx_t end) override;
89
+
90
+ private:
91
+ //! The frame boundaries, used for the window functions
92
+ FrameBounds frame;
68
93
  };
69
94
 
70
- class WindowSegmentTree {
95
+ class WindowSegmentTree : public WindowAggregateState {
71
96
  public:
72
97
  using FrameBounds = std::pair<idx_t, idx_t>;
73
98
 
74
- WindowSegmentTree(AggregateObject aggr, const LogicalType &result_type, DataChunk *input,
75
- const ValidityMask &filter_mask, WindowAggregationMode mode);
76
- ~WindowSegmentTree();
99
+ WindowSegmentTree(AggregateObject aggr, const LogicalType &result_type, idx_t count, WindowAggregationMode mode_p);
100
+ ~WindowSegmentTree() override;
77
101
 
78
- //! First row contains the result.
79
- void Compute(Vector &result, idx_t rid, idx_t start, idx_t end);
102
+ void Finalize() override;
103
+ void Evaluate(const idx_t *begins, const idx_t *ends, Vector &result, idx_t count) override;
80
104
 
81
105
  private:
82
106
  void ConstructTree();
83
- void ExtractFrame(idx_t begin, idx_t end);
84
- void WindowSegmentValue(idx_t l_idx, idx_t begin, idx_t end);
85
- void AggregateInit();
86
- void AggegateFinal(Vector &result, idx_t rid);
107
+ void ExtractFrame(idx_t begin, idx_t end, data_ptr_t current_state);
108
+ void FlushStates(bool combining);
109
+ void WindowSegmentValue(idx_t l_idx, idx_t begin, idx_t end, data_ptr_t current_state);
87
110
 
88
- //! Use the window API, if available
89
- inline bool UseWindowAPI() const {
90
- return mode < WindowAggregationMode::COMBINE;
91
- }
92
111
  //! Use the combine API, if available
93
112
  inline bool UseCombineAPI() const {
94
113
  return mode < WindowAggregationMode::SEPARATE;
95
114
  }
96
115
 
97
- //! The aggregate that the window function is computed over
98
- AggregateObject aggr;
99
- //! The result type of the window function
100
- LogicalType result_type;
101
-
102
- //! Data pointer that contains a single state, used for intermediate window segment aggregation
103
- vector<data_t> state;
104
- //! Input data chunk, used for intermediate window segment aggregation
105
- DataChunk inputs;
116
+ //! Input data chunk, used for leaf segment aggregation
117
+ DataChunk leaves;
106
118
  //! The filtered rows in inputs.
107
119
  SelectionVector filter_sel;
108
120
  //! A vector of pointers to "state", used for intermediate window segment aggregation
109
121
  Vector statep;
110
122
  //! The frame boundaries, used for the window functions
111
123
  FrameBounds frame;
112
- //! Reused result state container for the window functions
113
- Vector statev;
124
+ //! Reused state pointers for combining segment tree levels
125
+ Vector statel;
126
+ //! Count of buffered values
127
+ idx_t flush_count;
114
128
 
115
129
  //! The actual window segment tree: an array of aggregate states that represent all the intermediate nodes
116
130
  unsafe_unique_array<data_t> levels_flat_native;
@@ -120,17 +134,14 @@ private:
120
134
  //! The total number of internal nodes of the tree, stored in levels_flat_native
121
135
  idx_t internal_nodes;
122
136
 
123
- //! The (sorted) input chunk collection on which the tree is built
124
- DataChunk *input_ref;
125
-
126
- //! The filtered rows in input_ref.
127
- const ValidityMask &filter_mask;
128
-
129
137
  //! Use the window API, if available
130
138
  WindowAggregationMode mode;
131
139
 
140
+ //! Aggregate allocator
141
+ ArenaAllocator allocator;
142
+
132
143
  // TREE_FANOUT needs to cleanly divide STANDARD_VECTOR_SIZE
133
- static constexpr idx_t TREE_FANOUT = 64;
144
+ static constexpr idx_t TREE_FANOUT = 16;
134
145
  };
135
146
 
136
147
  } // namespace duckdb
@@ -8,10 +8,10 @@
8
8
 
9
9
  #pragma once
10
10
 
11
+ #include "duckdb/common/vector_operations/aggregate_executor.hpp"
11
12
  #include "duckdb/function/aggregate_state.hpp"
12
13
  #include "duckdb/planner/bound_result_modifier.hpp"
13
14
  #include "duckdb/planner/expression.hpp"
14
- #include "duckdb/common/vector_operations/aggregate_executor.hpp"
15
15
 
16
16
  namespace duckdb {
17
17
 
@@ -21,11 +21,11 @@ enum class AggregateOrderDependent : uint8_t { ORDER_DEPENDENT = 1, NOT_ORDER_DE
21
21
  class BoundAggregateExpression;
22
22
 
23
23
  struct AggregateInputData {
24
- AggregateInputData(optional_ptr<FunctionData> bind_data_p, Allocator &allocator_p)
24
+ AggregateInputData(optional_ptr<FunctionData> bind_data_p, ArenaAllocator &allocator_p)
25
25
  : bind_data(bind_data_p), allocator(allocator_p) {
26
26
  }
27
27
  optional_ptr<FunctionData> bind_data;
28
- Allocator &allocator;
28
+ ArenaAllocator &allocator;
29
29
  };
30
30
 
31
31
  struct AggregateUnaryInput {
@@ -57,6 +57,7 @@ private:
57
57
  void RegisterDistributiveAggregates();
58
58
 
59
59
  // scalar functions
60
+ void RegisterCompressedMaterializationFunctions();
60
61
  void RegisterGenericFunctions();
61
62
  void RegisterOperators();
62
63
  void RegisterStringFunctions();
@@ -0,0 +1,49 @@
1
+ //===----------------------------------------------------------------------===//
2
+ // DuckDB
3
+ //
4
+ // duckdb/function/scalar/compressed_materialization_functions.hpp
5
+ //
6
+ //
7
+ //===----------------------------------------------------------------------===//
8
+
9
+ #pragma once
10
+
11
+ #include "duckdb/function/built_in_functions.hpp"
12
+ #include "duckdb/function/function_set.hpp"
13
+
14
+ namespace duckdb {
15
+
16
+ struct CompressedMaterializationFunctions {
17
+ //! The types we compress integral types to
18
+ static const vector<LogicalType> IntegralTypes();
19
+ //! The types we compress strings to
20
+ static const vector<LogicalType> StringTypes();
21
+
22
+ static unique_ptr<FunctionData> Bind(ClientContext &context, ScalarFunction &bound_function,
23
+ vector<unique_ptr<Expression>> &arguments);
24
+ };
25
+
26
+ //! Needed for (de)serialization without binding
27
+ enum class CompressedMaterializationDirection : uint8_t { INVALID = 0, COMPRESS = 1, DECOMPRESS = 2 };
28
+
29
+ struct CMIntegralCompressFun {
30
+ static ScalarFunction GetFunction(const LogicalType &input_type, const LogicalType &result_type);
31
+ static void RegisterFunction(BuiltinFunctions &set);
32
+ };
33
+
34
+ struct CMIntegralDecompressFun {
35
+ static ScalarFunction GetFunction(const LogicalType &input_type, const LogicalType &result_type);
36
+ static void RegisterFunction(BuiltinFunctions &set);
37
+ };
38
+
39
+ struct CMStringCompressFun {
40
+ static ScalarFunction GetFunction(const LogicalType &result_type);
41
+ static void RegisterFunction(BuiltinFunctions &set);
42
+ };
43
+
44
+ struct CMStringDecompressFun {
45
+ static ScalarFunction GetFunction(const LogicalType &input_type);
46
+ static void RegisterFunction(BuiltinFunctions &set);
47
+ };
48
+
49
+ } // namespace duckdb
@@ -130,7 +130,7 @@ void ListContainsOrPosition(DataChunk &args, Vector &result) {
130
130
  TemplatedContainsOrPosition<int8_t, T, OP, LIST_ACCESSOR>(args, result, true);
131
131
  break;
132
132
  default:
133
- throw NotImplementedException("This function has not been implemented for physical type %s",
133
+ throw NotImplementedException("This function has not been implemented for logical type %s",
134
134
  TypeIdToString(physical_type));
135
135
  }
136
136
  }
@@ -99,6 +99,11 @@ struct ListPositionFun {
99
99
  static void RegisterFunction(BuiltinFunctions &set);
100
100
  };
101
101
 
102
+ struct ListResizeFun {
103
+ static ScalarFunction GetFunction();
104
+ static void RegisterFunction(BuiltinFunctions &set);
105
+ };
106
+
102
107
  struct StructExtractFun {
103
108
  static ScalarFunction GetFunction();
104
109
  static void RegisterFunction(BuiltinFunctions &set);
@@ -83,12 +83,14 @@ struct LengthFun {
83
83
  };
84
84
 
85
85
  struct LikeFun {
86
+ static ScalarFunction GetLikeFunction();
86
87
  static void RegisterFunction(BuiltinFunctions &set);
87
88
  DUCKDB_API static bool Glob(const char *s, idx_t slen, const char *pattern, idx_t plen,
88
89
  bool allow_question_mark = true);
89
90
  };
90
91
 
91
92
  struct LikeEscapeFun {
93
+ static ScalarFunction GetLikeEscapeFun();
92
94
  static void RegisterFunction(BuiltinFunctions &set);
93
95
  };
94
96
 
@@ -75,6 +75,8 @@ struct ClientConfig {
75
75
  bool force_no_cross_product = false;
76
76
  //! Force use of IEJoin to implement AsOfJoin, used for testing
77
77
  bool force_asof_iejoin = false;
78
+ //! Use range joins for inequalities, even if there are equality predicates
79
+ bool prefer_range_joins = false;
78
80
  //! If this context should also try to use the available replacement scans
79
81
  //! True by default
80
82
  bool use_replacement_scans = true;
@@ -96,6 +98,9 @@ struct ClientConfig {
96
98
  //! The maximum amount of pivot columns
97
99
  idx_t pivot_limit = 100000;
98
100
 
101
+ //! The threshold at which we switch from using filtered aggregates to LIST with a dedicated pivot operator
102
+ idx_t pivot_filter_threshold = 10;
103
+
99
104
  //! Whether or not the "/" division operator defaults to integer division or floating point division
100
105
  bool integer_division = false;
101
106
 
@@ -158,6 +158,8 @@ struct DBConfigOptions {
158
158
  bool lock_configuration = false;
159
159
  //! Whether to print bindings when printing the plan (debug mode only)
160
160
  static bool debug_print_bindings;
161
+ //! The peak allocation threshold at which to flush the allocator after completing a task (1 << 27, ~128MB)
162
+ idx_t allocator_flush_threshold = 134217728;
161
163
 
162
164
  bool operator==(const DBConfigOptions &other) const;
163
165
  };
@@ -169,8 +169,7 @@ public:
169
169
  template <typename TR, typename... Args>
170
170
  void CreateScalarFunction(const string &name, vector<LogicalType> args, LogicalType ret_type,
171
171
  TR (*udf_func)(Args...)) {
172
- scalar_function_t function =
173
- UDFWrapper::CreateScalarFunction<TR, Args...>(name, args, std::move(ret_type), udf_func);
172
+ scalar_function_t function = UDFWrapper::CreateScalarFunction<TR, Args...>(name, args, ret_type, udf_func);
174
173
  UDFWrapper::RegisterFunction(name, args, ret_type, function, *context);
175
174
  }
176
175
 
@@ -9,15 +9,18 @@
9
9
  #pragma once
10
10
 
11
11
  #include "duckdb/main/relation.hpp"
12
+ #include "duckdb/common/enums/joinref_type.hpp"
12
13
 
13
14
  namespace duckdb {
14
15
 
15
16
  class CrossProductRelation : public Relation {
16
17
  public:
17
- DUCKDB_API CrossProductRelation(shared_ptr<Relation> left, shared_ptr<Relation> right);
18
+ DUCKDB_API CrossProductRelation(shared_ptr<Relation> left, shared_ptr<Relation> right,
19
+ JoinRefType join_ref_type = JoinRefType::CROSS);
18
20
 
19
21
  shared_ptr<Relation> left;
20
22
  shared_ptr<Relation> right;
23
+ JoinRefType ref_type;
21
24
  vector<ColumnDefinition> columns;
22
25
 
23
26
  public:
@@ -9,21 +9,24 @@
9
9
  #pragma once
10
10
 
11
11
  #include "duckdb/main/relation.hpp"
12
+ #include "duckdb/common/enums/joinref_type.hpp"
12
13
 
13
14
  namespace duckdb {
14
15
 
15
16
  class JoinRelation : public Relation {
16
17
  public:
17
18
  DUCKDB_API JoinRelation(shared_ptr<Relation> left, shared_ptr<Relation> right,
18
- unique_ptr<ParsedExpression> condition, JoinType type);
19
+ unique_ptr<ParsedExpression> condition, JoinType type,
20
+ JoinRefType join_ref_type = JoinRefType::REGULAR);
19
21
  DUCKDB_API JoinRelation(shared_ptr<Relation> left, shared_ptr<Relation> right, vector<string> using_columns,
20
- JoinType type);
22
+ JoinType type, JoinRefType join_ref_type = JoinRefType::REGULAR);
21
23
 
22
24
  shared_ptr<Relation> left;
23
25
  shared_ptr<Relation> right;
24
26
  unique_ptr<ParsedExpression> condition;
25
27
  vector<string> using_columns;
26
28
  JoinType join_type;
29
+ JoinRefType join_ref_type;
27
30
  vector<ColumnDefinition> columns;
28
31
 
29
32
  public:
@@ -12,6 +12,7 @@
12
12
  #include "duckdb/common/enums/join_type.hpp"
13
13
  #include "duckdb/common/enums/relation_type.hpp"
14
14
  #include "duckdb/common/winapi.hpp"
15
+ #include "duckdb/common/enums/joinref_type.hpp"
15
16
  #include "duckdb/main/query_result.hpp"
16
17
  #include "duckdb/parser/column_definition.hpp"
17
18
  #include "duckdb/common/named_parameter_map.hpp"
@@ -94,10 +95,11 @@ public:
94
95
 
95
96
  // JOIN operation
96
97
  DUCKDB_API shared_ptr<Relation> Join(const shared_ptr<Relation> &other, const string &condition,
97
- JoinType type = JoinType::INNER);
98
+ JoinType type = JoinType::INNER, JoinRefType ref_type = JoinRefType::REGULAR);
98
99
 
99
100
  // CROSS PRODUCT operation
100
- DUCKDB_API shared_ptr<Relation> CrossProduct(const shared_ptr<Relation> &other);
101
+ DUCKDB_API shared_ptr<Relation> CrossProduct(const shared_ptr<Relation> &other,
102
+ JoinRefType join_ref_type = JoinRefType::CROSS);
101
103
 
102
104
  // SET operations
103
105
  DUCKDB_API shared_ptr<Relation> Union(const shared_ptr<Relation> &other);
@@ -84,6 +84,15 @@ struct DebugAsOfIEJoin {
84
84
  static Value GetSetting(ClientContext &context);
85
85
  };
86
86
 
87
+ struct PreferRangeJoins {
88
+ static constexpr const char *Name = "prefer_range_joins"; // NOLINT
89
+ static constexpr const char *Description = "Force use of range joins with mixed predicates"; // NOLINT
90
+ static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN; // NOLINT
91
+ static void SetLocal(ClientContext &context, const Value &parameter);
92
+ static void ResetLocal(ClientContext &context);
93
+ static Value GetSetting(ClientContext &context);
94
+ };
95
+
87
96
  struct DebugWindowMode {
88
97
  static constexpr const char *Name = "debug_window_mode";
89
98
  static constexpr const char *Description = "DEBUG SETTING: switch window mode to use";
@@ -122,6 +131,15 @@ struct DefaultNullOrderSetting {
122
131
  static Value GetSetting(ClientContext &context);
123
132
  };
124
133
 
134
+ struct DisabledFileSystemsSetting {
135
+ static constexpr const char *Name = "disabled_filesystems";
136
+ static constexpr const char *Description = "Disable specific file systems preventing access (e.g. LocalFileSystem)";
137
+ static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
138
+ static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
139
+ static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
140
+ static Value GetSetting(ClientContext &context);
141
+ };
142
+
125
143
  struct DisabledOptimizersSetting {
126
144
  static constexpr const char *Name = "disabled_optimizers";
127
145
  static constexpr const char *Description = "DEBUG SETTING: disable a specific set of optimizers (comma separated)";
@@ -366,10 +384,20 @@ struct PerfectHashThresholdSetting {
366
384
  static Value GetSetting(ClientContext &context);
367
385
  };
368
386
 
387
+ struct PivotFilterThreshold {
388
+ static constexpr const char *Name = "pivot_filter_threshold";
389
+ static constexpr const char *Description =
390
+ "The threshold to switch from using filtered aggregates to LIST with a dedicated pivot operator";
391
+ static constexpr const LogicalTypeId InputType = LogicalTypeId::BIGINT;
392
+ static void SetLocal(ClientContext &context, const Value &parameter);
393
+ static void ResetLocal(ClientContext &context);
394
+ static Value GetSetting(ClientContext &context);
395
+ };
396
+
369
397
  struct PivotLimitSetting {
370
398
  static constexpr const char *Name = "pivot_limit";
371
399
  static constexpr const char *Description =
372
- "The maximum numer of pivot columns in a pivot statement (default: 100000)";
400
+ "The maximum number of pivot columns in a pivot statement (default: 100000)";
373
401
  static constexpr const LogicalTypeId InputType = LogicalTypeId::BIGINT;
374
402
  static void SetLocal(ClientContext &context, const Value &parameter);
375
403
  static void ResetLocal(ClientContext &context);
@@ -492,4 +520,14 @@ struct UsernameSetting {
492
520
  static Value GetSetting(ClientContext &context);
493
521
  };
494
522
 
523
+ struct FlushAllocatorSetting {
524
+ static constexpr const char *Name = "allocator_flush_threshold";
525
+ static constexpr const char *Description =
526
+ "Peak allocation threshold at which to flush the allocator after completing a task.";
527
+ static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
528
+ static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
529
+ static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
530
+ static Value GetSetting(ClientContext &context);
531
+ };
532
+
495
533
  } // namespace duckdb
@@ -0,0 +1,47 @@
1
+ //===----------------------------------------------------------------------===//
2
+ // DuckDB
3
+ //
4
+ // duckdb/optimizer/column_binding_replacer.hpp
5
+ //
6
+ //
7
+ //===----------------------------------------------------------------------===//
8
+
9
+ #pragma once
10
+
11
+ #include "duckdb/planner/logical_operator.hpp"
12
+
13
+ namespace duckdb {
14
+
15
+ struct ReplacementBinding {
16
+ public:
17
+ ReplacementBinding(ColumnBinding old_binding, ColumnBinding new_binding);
18
+ ReplacementBinding(ColumnBinding old_binding, ColumnBinding new_binding, LogicalType new_type);
19
+
20
+ public:
21
+ ColumnBinding old_binding;
22
+ ColumnBinding new_binding;
23
+
24
+ bool replace_type;
25
+ LogicalType new_type;
26
+ };
27
+
28
+ //! The ColumnBindingReplacer updates column bindings (e.g., after changing the operator plan), utility for optimizers
29
+ class ColumnBindingReplacer : LogicalOperatorVisitor {
30
+ public:
31
+ ColumnBindingReplacer();
32
+
33
+ public:
34
+ //! Update each operator of the plan
35
+ void VisitOperator(LogicalOperator &op) override;
36
+ //! Visit an expression and update its column bindings
37
+ void VisitExpression(unique_ptr<Expression> *expression) override;
38
+
39
+ public:
40
+ //! Contains all bindings that need to be updated
41
+ vector<ReplacementBinding> replacement_bindings;
42
+
43
+ //! Do not recurse further than this operator (optional)
44
+ optional_ptr<LogicalOperator> stop_operator;
45
+ };
46
+
47
+ } // namespace duckdb