duckdb 0.8.2-dev161.0 → 0.8.2-dev1764.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (504) hide show
  1. package/binding.gyp +15 -12
  2. package/binding.gyp.in +1 -1
  3. package/configure.py +1 -1
  4. package/duckdb_extension_config.cmake +10 -0
  5. package/package.json +1 -1
  6. package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
  7. package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
  8. package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
  9. package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
  10. package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
  11. package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
  12. package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
  13. package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
  14. package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
  15. package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
  16. package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
  17. package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
  18. package/src/duckdb/extension/json/include/json_common.hpp +47 -231
  19. package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
  20. package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
  21. package/src/duckdb/extension/json/json_common.cpp +272 -40
  22. package/src/duckdb/extension/json/json_functions/json_create.cpp +21 -2
  23. package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
  24. package/src/duckdb/extension/json/json_functions/json_transform.cpp +91 -38
  25. package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
  26. package/src/duckdb/extension/json/json_functions.cpp +24 -24
  27. package/src/duckdb/extension/json/json_scan.cpp +3 -6
  28. package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
  29. package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
  30. package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
  31. package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
  32. package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
  33. package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
  34. package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
  35. package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
  36. package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
  37. package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
  38. package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
  39. package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
  40. package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
  41. package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
  42. package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
  43. package/src/duckdb/extension/parquet/parquet_extension.cpp +194 -20
  44. package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
  45. package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
  46. package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
  47. package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
  48. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
  49. package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
  50. package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
  51. package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
  52. package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
  53. package/src/duckdb/src/common/allocator.cpp +14 -2
  54. package/src/duckdb/src/common/arrow/arrow_appender.cpp +79 -12
  55. package/src/duckdb/src/common/arrow/arrow_converter.cpp +44 -19
  56. package/src/duckdb/src/common/assert.cpp +3 -0
  57. package/src/duckdb/src/common/enum_util.cpp +4619 -4446
  58. package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
  59. package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
  60. package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
  61. package/src/duckdb/src/common/exception.cpp +2 -2
  62. package/src/duckdb/src/common/extra_type_info.cpp +506 -0
  63. package/src/duckdb/src/common/file_system.cpp +19 -0
  64. package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
  65. package/src/duckdb/src/common/local_file_system.cpp +14 -14
  66. package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
  67. package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
  68. package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
  69. package/src/duckdb/src/common/re2_regex.cpp +1 -1
  70. package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
  71. package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
  72. package/src/duckdb/src/common/sort/partition_state.cpp +70 -50
  73. package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
  74. package/src/duckdb/src/common/types/bit.cpp +51 -0
  75. package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
  76. package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
  77. package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
  78. package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
  79. package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
  80. package/src/duckdb/src/common/types/date.cpp +9 -0
  81. package/src/duckdb/src/common/types/list_segment.cpp +24 -74
  82. package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
  83. package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
  84. package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
  85. package/src/duckdb/src/common/types/uuid.cpp +2 -2
  86. package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
  87. package/src/duckdb/src/common/types/value.cpp +11 -6
  88. package/src/duckdb/src/common/types.cpp +9 -656
  89. package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
  90. package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
  91. package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
  92. package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
  93. package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
  94. package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
  95. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
  96. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
  97. package/src/duckdb/src/core_functions/function_list.cpp +4 -2
  98. package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
  99. package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
  100. package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
  101. package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
  102. package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
  103. package/src/duckdb/src/execution/expression_executor.cpp +1 -1
  104. package/src/duckdb/src/execution/index/art/art.cpp +149 -139
  105. package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
  106. package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
  107. package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
  108. package/src/duckdb/src/execution/index/art/node.cpp +113 -120
  109. package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
  110. package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
  111. package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
  112. package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
  113. package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
  114. package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
  115. package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
  116. package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
  117. package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
  118. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
  119. package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
  120. package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
  121. package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +444 -284
  122. package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
  123. package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
  124. package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +28 -12
  125. package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
  126. package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +23 -4
  127. package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +41 -5
  128. package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
  129. package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
  130. package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
  131. package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
  132. package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
  133. package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
  134. package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
  135. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
  136. package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
  137. package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
  138. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
  139. package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
  140. package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
  141. package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
  142. package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
  143. package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
  144. package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +56 -33
  145. package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +17 -13
  146. package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
  147. package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
  148. package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
  149. package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
  150. package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
  151. package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
  152. package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
  153. package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
  154. package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
  155. package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
  156. package/src/duckdb/src/function/function.cpp +3 -1
  157. package/src/duckdb/src/function/pragma/pragma_queries.cpp +5 -0
  158. package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
  159. package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
  160. package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
  161. package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
  162. package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
  163. package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
  164. package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
  165. package/src/duckdb/src/function/table/arrow.cpp +19 -0
  166. package/src/duckdb/src/function/table/arrow_conversion.cpp +35 -1
  167. package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
  168. package/src/duckdb/src/function/table/read_csv.cpp +100 -17
  169. package/src/duckdb/src/function/table/system/test_all_types.cpp +7 -0
  170. package/src/duckdb/src/function/table/system_functions.cpp +1 -0
  171. package/src/duckdb/src/function/table/table_scan.cpp +9 -0
  172. package/src/duckdb/src/function/table/version/pragma_version.cpp +46 -2
  173. package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
  174. package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
  175. package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
  176. package/src/duckdb/src/include/duckdb/common/dl.hpp +3 -1
  177. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +616 -584
  178. package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
  179. package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
  180. package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
  181. package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
  182. package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
  183. package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
  184. package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +219 -0
  185. package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
  186. package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
  187. package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
  188. package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
  189. package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
  190. package/src/duckdb/src/include/duckdb/common/opener_file_system.hpp +2 -2
  191. package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
  192. package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
  193. package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
  194. package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
  195. package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
  196. package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +23 -8
  197. package/src/duckdb/src/include/duckdb/common/string_util.hpp +11 -0
  198. package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
  199. package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
  200. package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
  201. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
  202. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
  203. package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
  204. package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
  205. package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
  206. package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
  207. package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +5 -2
  208. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
  209. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
  210. package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
  211. package/src/duckdb/src/include/duckdb/common/types/value.hpp +1 -0
  212. package/src/duckdb/src/include/duckdb/common/types.hpp +1 -15
  213. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
  214. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
  215. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
  216. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
  217. package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
  218. package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
  219. package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
  220. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
  221. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
  222. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
  223. package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
  224. package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
  225. package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
  226. package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
  227. package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
  228. package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
  229. package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
  230. package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
  231. package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
  232. package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
  233. package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
  234. package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
  235. package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
  236. package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
  237. package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
  238. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
  239. package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
  240. package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
  241. package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
  242. package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
  243. package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
  244. package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
  245. package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
  246. package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
  247. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +3 -10
  248. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +1 -1
  249. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +1 -1
  250. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp +12 -1
  251. package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
  252. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
  253. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
  254. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
  255. package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
  256. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
  257. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
  258. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
  259. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
  260. package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
  261. package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
  262. package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
  263. package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
  264. package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
  265. package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
  266. package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
  267. package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
  268. package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
  269. package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
  270. package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
  271. package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +4 -0
  272. package/src/duckdb/src/include/duckdb/main/client_config.hpp +5 -0
  273. package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
  274. package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
  275. package/src/duckdb/src/include/duckdb/main/extension/generated_extension_loader.hpp +22 -0
  276. package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +2 -0
  277. package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
  278. package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
  279. package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
  280. package/src/duckdb/src/include/duckdb/main/settings.hpp +39 -1
  281. package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
  282. package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
  283. package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
  284. package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
  285. package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
  286. package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
  287. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
  288. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
  289. package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
  290. package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
  291. package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
  292. package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
  293. package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
  294. package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
  295. package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
  296. package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
  297. package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
  298. package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
  299. package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
  300. package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
  301. package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
  302. package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
  303. package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
  304. package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
  305. package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
  306. package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
  307. package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
  308. package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
  309. package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
  310. package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
  311. package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
  312. package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
  313. package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
  314. package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
  315. package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
  316. package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
  317. package/src/duckdb/src/include/duckdb/planner/binder.hpp +12 -5
  318. package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
  319. package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
  320. package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
  321. package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
  322. package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
  323. package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
  324. package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
  325. package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
  326. package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
  327. package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
  328. package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
  329. package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
  330. package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
  331. package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
  332. package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
  333. package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
  334. package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
  335. package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
  336. package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
  337. package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
  338. package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
  339. package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
  340. package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
  341. package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
  342. package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
  343. package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
  344. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
  345. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
  346. package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
  347. package/src/duckdb/src/include/duckdb.h +28 -0
  348. package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
  349. package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
  350. package/src/duckdb/src/main/config.cpp +4 -0
  351. package/src/duckdb/src/main/database.cpp +1 -1
  352. package/src/duckdb/src/main/extension/extension_helper.cpp +93 -88
  353. package/src/duckdb/src/main/extension/extension_install.cpp +9 -0
  354. package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
  355. package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
  356. package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
  357. package/src/duckdb/src/main/relation.cpp +6 -5
  358. package/src/duckdb/src/main/settings/settings.cpp +79 -18
  359. package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
  360. package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
  361. package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
  362. package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
  363. package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
  364. package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
  365. package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
  366. package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
  367. package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
  368. package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
  369. package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
  370. package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
  371. package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
  372. package/src/duckdb/src/optimizer/optimizer.cpp +49 -14
  373. package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
  374. package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
  375. package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
  376. package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
  377. package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
  378. package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
  379. package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
  380. package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
  381. package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
  382. package/src/duckdb/src/parallel/executor.cpp +15 -0
  383. package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
  384. package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
  385. package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
  386. package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
  387. package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
  388. package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
  389. package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
  390. package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
  391. package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
  392. package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
  393. package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
  394. package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
  395. package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
  396. package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
  397. package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
  398. package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
  399. package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
  400. package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
  401. package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
  402. package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
  403. package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
  404. package/src/duckdb/src/parser/parser.cpp +8 -2
  405. package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
  406. package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
  407. package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
  408. package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
  409. package/src/duckdb/src/parser/query_node.cpp +15 -37
  410. package/src/duckdb/src/parser/result_modifier.cpp +0 -74
  411. package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
  412. package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
  413. package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
  414. package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
  415. package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
  416. package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
  417. package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
  418. package/src/duckdb/src/parser/tableref.cpp +0 -44
  419. package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
  420. package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
  421. package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
  422. package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
  423. package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
  424. package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
  425. package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
  426. package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
  427. package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
  428. package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
  429. package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
  430. package/src/duckdb/src/parser/transformer.cpp +15 -0
  431. package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
  432. package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
  433. package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
  434. package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
  435. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
  436. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
  437. package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
  438. package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
  439. package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
  440. package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
  441. package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +64 -26
  442. package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
  443. package/src/duckdb/src/planner/binder.cpp +44 -31
  444. package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
  445. package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
  446. package/src/duckdb/src/planner/expression_binder.cpp +3 -0
  447. package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
  448. package/src/duckdb/src/planner/logical_operator.cpp +5 -0
  449. package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
  450. package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
  451. package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
  452. package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
  453. package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
  454. package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
  455. package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
  456. package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
  457. package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
  458. package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
  459. package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
  460. package/src/duckdb/src/storage/compression/rle.cpp +0 -1
  461. package/src/duckdb/src/storage/data_table.cpp +1 -1
  462. package/src/duckdb/src/storage/local_storage.cpp +3 -3
  463. package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
  464. package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
  465. package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
  466. package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
  467. package/src/duckdb/src/storage/serialization/serialize_types.cpp +127 -0
  468. package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
  469. package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
  470. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  471. package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
  472. package/src/duckdb/src/storage/table/row_group.cpp +25 -9
  473. package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
  474. package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
  475. package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
  476. package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
  477. package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
  478. package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
  479. package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
  480. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
  481. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
  482. package/src/duckdb/ub_src_common.cpp +2 -0
  483. package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
  484. package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
  485. package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
  486. package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
  487. package/src/duckdb/ub_src_function_scalar.cpp +2 -0
  488. package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
  489. package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
  490. package/src/duckdb/ub_src_optimizer.cpp +6 -0
  491. package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
  492. package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
  493. package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
  494. package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
  495. package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
  496. package/src/duckdb/ub_src_planner_operator.cpp +4 -0
  497. package/src/duckdb/ub_src_storage_serialization.cpp +10 -0
  498. package/src/statement.cpp +10 -3
  499. package/test/columns.test.ts +24 -1
  500. package/test/test_all_types.test.ts +234 -0
  501. package/tsconfig.json +1 -0
  502. package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
  503. package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
  504. package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -87,71 +87,76 @@ PartitionGlobalSinkState::PartitionGlobalSinkState(ClientContext &context,
87
87
  const vector<unique_ptr<BaseStatistics>> &partition_stats,
88
88
  idx_t estimated_cardinality)
89
89
  : context(context), buffer_manager(BufferManager::GetBufferManager(context)), allocator(Allocator::Get(context)),
90
- payload_types(payload_types), memory_per_thread(0), count(0) {
90
+ fixed_bits(0), payload_types(payload_types), memory_per_thread(0), max_bits(1), count(0) {
91
91
 
92
92
  GenerateOrderings(partitions, orders, partition_bys, order_bys, partition_stats);
93
93
 
94
94
  memory_per_thread = PhysicalOperator::GetMaxThreadMemory(context);
95
95
  external = ClientConfig::GetConfig(context).force_external;
96
96
 
97
+ const auto thread_pages = PreviousPowerOfTwo(memory_per_thread / (4 * idx_t(Storage::BLOCK_ALLOC_SIZE)));
98
+ while (max_bits < 10 && (thread_pages >> max_bits) > 1) {
99
+ ++max_bits;
100
+ }
101
+
97
102
  if (!orders.empty()) {
98
- grouping_types = payload_types;
99
- grouping_types.push_back(LogicalType::HASH);
103
+ auto types = payload_types;
104
+ types.push_back(LogicalType::HASH);
105
+ grouping_types.Initialize(types);
100
106
 
101
107
  ResizeGroupingData(estimated_cardinality);
102
108
  }
103
109
  }
104
110
 
111
+ void PartitionGlobalSinkState::SyncPartitioning(const PartitionGlobalSinkState &other) {
112
+ fixed_bits = other.grouping_data ? other.grouping_data->GetRadixBits() : 0;
113
+
114
+ const auto old_bits = grouping_data ? grouping_data->GetRadixBits() : 0;
115
+ if (fixed_bits != old_bits) {
116
+ const auto hash_col_idx = payload_types.size();
117
+ grouping_data = make_uniq<RadixPartitionedTupleData>(buffer_manager, grouping_types, fixed_bits, hash_col_idx);
118
+ }
119
+ }
120
+
121
+ unique_ptr<RadixPartitionedTupleData> PartitionGlobalSinkState::CreatePartition(idx_t new_bits) const {
122
+ const auto hash_col_idx = payload_types.size();
123
+ return make_uniq<RadixPartitionedTupleData>(buffer_manager, grouping_types, new_bits, hash_col_idx);
124
+ }
125
+
105
126
  void PartitionGlobalSinkState::ResizeGroupingData(idx_t cardinality) {
106
127
  // Have we started to combine? Then just live with it.
107
- if (grouping_data && !grouping_data->GetPartitions().empty()) {
128
+ if (fixed_bits || (grouping_data && !grouping_data->GetPartitions().empty())) {
108
129
  return;
109
130
  }
110
131
  // Is the average partition size too large?
111
132
  const idx_t partition_size = STANDARD_ROW_GROUPS_SIZE;
112
133
  const auto bits = grouping_data ? grouping_data->GetRadixBits() : 0;
113
134
  auto new_bits = bits ? bits : 4;
114
- while (new_bits < 10 && (cardinality / RadixPartitioning::NumberOfPartitions(new_bits)) > partition_size) {
135
+ while (new_bits < max_bits && (cardinality / RadixPartitioning::NumberOfPartitions(new_bits)) > partition_size) {
115
136
  ++new_bits;
116
137
  }
117
138
 
118
139
  // Repartition the grouping data
119
140
  if (new_bits != bits) {
120
- const auto hash_col_idx = payload_types.size();
121
- grouping_data = make_uniq<RadixPartitionedColumnData>(context, grouping_types, new_bits, hash_col_idx);
141
+ grouping_data = CreatePartition(new_bits);
122
142
  }
123
143
  }
124
144
 
125
145
  void PartitionGlobalSinkState::SyncLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append) {
126
146
  // We are done if the local_partition is right sized.
127
- auto &local_radix = local_partition->Cast<RadixPartitionedColumnData>();
128
- if (local_radix.GetRadixBits() == grouping_data->GetRadixBits()) {
147
+ auto &local_radix = local_partition->Cast<RadixPartitionedTupleData>();
148
+ const auto new_bits = grouping_data->GetRadixBits();
149
+ if (local_radix.GetRadixBits() == new_bits) {
129
150
  return;
130
151
  }
131
152
 
132
153
  // If the local partition is now too small, flush it and reallocate
133
- auto new_partition = grouping_data->CreateShared();
134
- auto new_append = make_uniq<PartitionedColumnDataAppendState>();
135
- new_partition->InitializeAppendState(*new_append);
136
-
154
+ auto new_partition = CreatePartition(new_bits);
137
155
  local_partition->FlushAppendState(*local_append);
138
- auto &local_groups = local_partition->GetPartitions();
139
- for (auto &local_group : local_groups) {
140
- ColumnDataScanState scanner;
141
- local_group->InitializeScan(scanner);
142
-
143
- DataChunk scan_chunk;
144
- local_group->InitializeScanChunk(scan_chunk);
145
- for (scan_chunk.Reset(); local_group->Scan(scanner, scan_chunk); scan_chunk.Reset()) {
146
- new_partition->Append(*new_append, scan_chunk);
147
- }
148
- }
149
-
150
- // The append state has stale pointers to the old local partition, so nuke it from orbit.
151
- new_partition->FlushAppendState(*new_append);
156
+ local_partition->Repartition(*new_partition);
152
157
 
153
158
  local_partition = std::move(new_partition);
154
- local_append = make_uniq<PartitionedColumnDataAppendState>();
159
+ local_append = make_uniq<PartitionedTupleDataAppendState>();
155
160
  local_partition->InitializeAppendState(*local_append);
156
161
  }
157
162
 
@@ -160,8 +165,8 @@ void PartitionGlobalSinkState::UpdateLocalPartition(GroupingPartition &local_par
160
165
  lock_guard<mutex> guard(lock);
161
166
 
162
167
  if (!local_partition) {
163
- local_partition = grouping_data->CreateShared();
164
- local_append = make_uniq<PartitionedColumnDataAppendState>();
168
+ local_partition = CreatePartition(grouping_data->GetRadixBits());
169
+ local_append = make_uniq<PartitionedTupleDataAppendState>();
165
170
  local_partition->InitializeAppendState(*local_append);
166
171
  return;
167
172
  }
@@ -186,9 +191,7 @@ void PartitionGlobalSinkState::CombineLocalPartition(GroupingPartition &local_pa
186
191
  grouping_data->Combine(*local_partition);
187
192
  }
188
193
 
189
- void PartitionGlobalSinkState::BuildSortState(ColumnDataCollection &group_data, PartitionGlobalHashGroup &hash_group) {
190
- auto &global_sort = *hash_group.global_sort;
191
-
194
+ void PartitionGlobalSinkState::BuildSortState(TupleDataCollection &group_data, GlobalSortState &global_sort) const {
192
195
  // Set up the sort expression computation.
193
196
  vector<LogicalType> sort_types;
194
197
  ExpressionExecutor executor(context);
@@ -213,16 +216,9 @@ void PartitionGlobalSinkState::BuildSortState(ColumnDataCollection &group_data,
213
216
  for (column_t i = 0; i < payload_types.size(); ++i) {
214
217
  column_ids.emplace_back(i);
215
218
  }
216
- ColumnDataConsumer scanner(group_data, column_ids);
217
- ColumnDataConsumerScanState chunk_state;
218
- chunk_state.current_chunk_state.properties = ColumnDataScanProperties::ALLOW_ZERO_COPY;
219
- scanner.InitializeScan();
220
- for (auto chunk_idx = scanner.ChunkCount(); chunk_idx-- > 0;) {
221
- if (!scanner.AssignChunk(chunk_state)) {
222
- break;
223
- }
224
- scanner.ScanChunk(chunk_state, payload_chunk);
225
-
219
+ TupleDataScanState chunk_state;
220
+ group_data.InitializeScan(chunk_state, column_ids);
221
+ while (group_data.Scan(chunk_state, payload_chunk)) {
226
222
  sort_chunk.Reset();
227
223
  executor.Execute(payload_chunk, sort_chunk);
228
224
 
@@ -230,10 +226,13 @@ void PartitionGlobalSinkState::BuildSortState(ColumnDataCollection &group_data,
230
226
  if (local_sort.SizeInBytes() > memory_per_thread) {
231
227
  local_sort.Sort(global_sort, true);
232
228
  }
233
- scanner.FinishChunk(chunk_state);
234
229
  }
235
230
 
236
231
  global_sort.AddLocalState(local_sort);
232
+ }
233
+
234
+ void PartitionGlobalSinkState::BuildSortState(TupleDataCollection &group_data, PartitionGlobalHashGroup &hash_group) {
235
+ BuildSortState(group_data, *hash_group.global_sort);
237
236
 
238
237
  hash_group.count += group_data.Count();
239
238
  }
@@ -482,18 +481,29 @@ public:
482
481
  TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override;
483
482
 
484
483
  private:
484
+ struct ExecutorCallback : public PartitionGlobalMergeStates::Callback {
485
+ explicit ExecutorCallback(Executor &executor) : executor(executor) {
486
+ }
487
+
488
+ bool HasError() const override {
489
+ return executor.HasError();
490
+ }
491
+
492
+ Executor &executor;
493
+ };
494
+
485
495
  shared_ptr<Event> event;
486
496
  PartitionLocalMergeState local_state;
487
497
  PartitionGlobalMergeStates &hash_groups;
488
498
  };
489
499
 
490
- TaskExecutionResult PartitionMergeTask::ExecuteTask(TaskExecutionMode mode) {
500
+ bool PartitionGlobalMergeStates::ExecuteTask(PartitionLocalMergeState &local_state, Callback &callback) {
491
501
  // Loop until all hash groups are done
492
502
  size_t sorted = 0;
493
- while (sorted < hash_groups.states.size()) {
503
+ while (sorted < states.size()) {
494
504
  // First check if there is an unfinished task for this thread
495
- if (executor.HasError()) {
496
- return TaskExecutionResult::TASK_ERROR;
505
+ if (callback.HasError()) {
506
+ return false;
497
507
  }
498
508
  if (!local_state.TaskFinished()) {
499
509
  local_state.ExecuteTask();
@@ -501,8 +511,8 @@ TaskExecutionResult PartitionMergeTask::ExecuteTask(TaskExecutionMode mode) {
501
511
  }
502
512
 
503
513
  // Thread is done with its assigned task, try to fetch new work
504
- for (auto group = sorted; group < hash_groups.states.size(); ++group) {
505
- auto &global_state = hash_groups.states[group];
514
+ for (auto group = sorted; group < states.size(); ++group) {
515
+ auto &global_state = states[group];
506
516
  if (global_state->IsSorted()) {
507
517
  // This hash group is done
508
518
  // Update the high water mark of densely completed groups
@@ -543,6 +553,16 @@ TaskExecutionResult PartitionMergeTask::ExecuteTask(TaskExecutionMode mode) {
543
553
  }
544
554
  }
545
555
 
556
+ return true;
557
+ }
558
+
559
+ TaskExecutionResult PartitionMergeTask::ExecuteTask(TaskExecutionMode mode) {
560
+ ExecutorCallback callback(executor);
561
+
562
+ if (!hash_groups.ExecuteTask(local_state, callback)) {
563
+ return TaskExecutionResult::TASK_ERROR;
564
+ }
565
+
546
566
  event->FinishTask();
547
567
  return TaskExecutionResult::TASK_FINISHED;
548
568
  }
@@ -1,11 +1,14 @@
1
1
  #include "duckdb/common/types/batched_data_collection.hpp"
2
+
3
+ #include "duckdb/common/optional_ptr.hpp"
2
4
  #include "duckdb/common/printer.hpp"
3
5
  #include "duckdb/storage/buffer_manager.hpp"
4
- #include "duckdb/common/optional_ptr.hpp"
5
6
 
6
7
  namespace duckdb {
7
8
 
8
- BatchedDataCollection::BatchedDataCollection(vector<LogicalType> types_p) : types(std::move(types_p)) {
9
+ BatchedDataCollection::BatchedDataCollection(ClientContext &context_p, vector<LogicalType> types_p,
10
+ bool buffer_managed_p)
11
+ : context(context_p), types(std::move(types_p)), buffer_managed(buffer_managed_p) {
9
12
  }
10
13
 
11
14
  void BatchedDataCollection::Append(DataChunk &input, idx_t batch_index) {
@@ -20,6 +23,8 @@ void BatchedDataCollection::Append(DataChunk &input, idx_t batch_index) {
20
23
  unique_ptr<ColumnDataCollection> new_collection;
21
24
  if (last_collection.collection) {
22
25
  new_collection = make_uniq<ColumnDataCollection>(*last_collection.collection);
26
+ } else if (buffer_managed) {
27
+ new_collection = make_uniq<ColumnDataCollection>(BufferManager::GetBufferManager(context), types);
23
28
  } else {
24
29
  new_collection = make_uniq<ColumnDataCollection>(Allocator::DefaultAllocator(), types);
25
30
  }
@@ -1,4 +1,6 @@
1
+ #include "duckdb/common/assert.hpp"
1
2
  #include "duckdb/common/operator/cast_operators.hpp"
3
+ #include "duckdb/common/typedefs.hpp"
2
4
  #include "duckdb/common/types/bit.hpp"
3
5
  #include "duckdb/common/types/string_type.hpp"
4
6
 
@@ -34,6 +36,13 @@ static inline idx_t GetBitSize(const string_t &str) {
34
36
  return str_len;
35
37
  }
36
38
 
39
+ uint8_t Bit::GetFirstByte(const string_t &str) {
40
+ D_ASSERT(str.GetSize() > 1);
41
+
42
+ auto data = const_data_ptr_cast(str.GetData());
43
+ return data[1] & ((1 << (8 - data[0])) - 1);
44
+ }
45
+
37
46
  void Bit::Finalize(string_t &str) {
38
47
  // bit strings require all padding bits to be set to 1
39
48
  // this method sets all padding bits to 1
@@ -146,6 +155,48 @@ string Bit::ToBit(string_t str) {
146
155
  return output_str.GetString();
147
156
  }
148
157
 
158
+ void Bit::BlobToBit(string_t blob, string_t &output_str) {
159
+ auto data = const_data_ptr_cast(blob.GetData());
160
+ auto output = output_str.GetDataWriteable();
161
+ idx_t size = blob.GetSize();
162
+
163
+ *output = 0; // No padding
164
+ memcpy(output + 1, data, size);
165
+ }
166
+
167
+ string Bit::BlobToBit(string_t blob) {
168
+ auto buffer = make_unsafe_uniq_array<char>(blob.GetSize() + 1);
169
+ string_t output_str(buffer.get(), blob.GetSize() + 1);
170
+ Bit::BlobToBit(blob, output_str);
171
+ return output_str.GetString();
172
+ }
173
+
174
+ void Bit::BitToBlob(string_t bit, string_t &output_blob) {
175
+ D_ASSERT(bit.GetSize() == output_blob.GetSize() + 1);
176
+
177
+ auto data = const_data_ptr_cast(bit.GetData());
178
+ auto output = output_blob.GetDataWriteable();
179
+ idx_t size = output_blob.GetSize();
180
+
181
+ output[0] = GetFirstByte(bit);
182
+ if (size > 2) {
183
+ ++output;
184
+ // First byte in bitstring contains amount of padded bits,
185
+ // second byte in bitstring is the padded byte,
186
+ // therefore the rest of the data starts at data + 2 (third byte)
187
+ memcpy(output, data + 2, size - 1);
188
+ }
189
+ }
190
+
191
+ string Bit::BitToBlob(string_t bit) {
192
+ D_ASSERT(bit.GetSize() > 1);
193
+
194
+ auto buffer = make_unsafe_uniq_array<char>(bit.GetSize() - 1);
195
+ string_t output_str(buffer.get(), bit.GetSize() - 1);
196
+ Bit::BitToBlob(bit, output_str);
197
+ return output_str.GetString();
198
+ }
199
+
149
200
  // **** scalar functions ****
150
201
  void Bit::BitString(const string_t &input, const idx_t &bit_length, string_t &result) {
151
202
  char *res_buf = result.GetDataWriteable();
@@ -1,8 +1,8 @@
1
1
  #include "duckdb/common/types/column/column_data_allocator.hpp"
2
2
 
3
3
  #include "duckdb/common/types/column/column_data_collection_segment.hpp"
4
- #include "duckdb/storage/buffer_manager.hpp"
5
4
  #include "duckdb/storage/buffer/block_handle.hpp"
5
+ #include "duckdb/storage/buffer_manager.hpp"
6
6
 
7
7
  namespace duckdb {
8
8
 
@@ -19,6 +19,7 @@ ColumnDataAllocator::ColumnDataAllocator(ClientContext &context, ColumnDataAlloc
19
19
  : type(allocator_type) {
20
20
  switch (type) {
21
21
  case ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR:
22
+ case ColumnDataAllocatorType::HYBRID:
22
23
  alloc.buffer_manager = &BufferManager::GetBufferManager(context);
23
24
  break;
24
25
  case ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR:
@@ -33,6 +34,7 @@ ColumnDataAllocator::ColumnDataAllocator(ColumnDataAllocator &other) {
33
34
  type = other.GetType();
34
35
  switch (type) {
35
36
  case ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR:
37
+ case ColumnDataAllocatorType::HYBRID:
36
38
  alloc.allocator = other.alloc.allocator;
37
39
  break;
38
40
  case ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR:
@@ -44,7 +46,7 @@ ColumnDataAllocator::ColumnDataAllocator(ColumnDataAllocator &other) {
44
46
  }
45
47
 
46
48
  BufferHandle ColumnDataAllocator::Pin(uint32_t block_id) {
47
- D_ASSERT(type == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR);
49
+ D_ASSERT(type == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR || type == ColumnDataAllocatorType::HYBRID);
48
50
  shared_ptr<BlockHandle> handle;
49
51
  if (shared) {
50
52
  // we only need to grab the lock when accessing the vector, because vector access is not thread-safe:
@@ -58,7 +60,7 @@ BufferHandle ColumnDataAllocator::Pin(uint32_t block_id) {
58
60
  }
59
61
 
60
62
  BufferHandle ColumnDataAllocator::AllocateBlock(idx_t size) {
61
- D_ASSERT(type == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR);
63
+ D_ASSERT(type == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR || type == ColumnDataAllocatorType::HYBRID);
62
64
  auto block_size = MaxValue<idx_t>(size, Storage::BLOCK_SIZE);
63
65
  BlockMetaData data;
64
66
  data.size = 0;
@@ -136,6 +138,7 @@ void ColumnDataAllocator::AllocateData(idx_t size, uint32_t &block_id, uint32_t
136
138
  ChunkManagementState *chunk_state) {
137
139
  switch (type) {
138
140
  case ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR:
141
+ case ColumnDataAllocatorType::HYBRID:
139
142
  if (shared) {
140
143
  lock_guard<mutex> guard(lock);
141
144
  AllocateBuffer(size, block_id, offset, chunk_state);
@@ -174,8 +177,8 @@ data_ptr_t ColumnDataAllocator::GetDataPointer(ChunkManagementState &state, uint
174
177
  return state.handles[block_id].Ptr() + offset;
175
178
  }
176
179
 
177
- void ColumnDataAllocator::UnswizzlePointers(ChunkManagementState &state, Vector &result, uint16_t v_offset,
178
- uint16_t count, uint32_t block_id, uint32_t offset) {
180
+ void ColumnDataAllocator::UnswizzlePointers(ChunkManagementState &state, Vector &result, idx_t v_offset, uint16_t count,
181
+ uint32_t block_id, uint32_t offset) {
179
182
  D_ASSERT(result.GetType().InternalType() == PhysicalType::VARCHAR);
180
183
  lock_guard<mutex> guard(lock);
181
184
 
@@ -225,7 +228,7 @@ Allocator &ColumnDataAllocator::GetAllocator() {
225
228
  }
226
229
 
227
230
  void ColumnDataAllocator::InitializeChunkState(ChunkManagementState &state, ChunkMetaData &chunk) {
228
- if (type != ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR) {
231
+ if (type != ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR && type != ColumnDataAllocatorType::HYBRID) {
229
232
  // nothing to pin
230
233
  return;
231
234
  }
@@ -333,7 +333,7 @@ struct StandardValueCopy : public BaseValueCopy<T> {
333
333
 
334
334
  struct StringValueCopy : public BaseValueCopy<string_t> {
335
335
  static string_t Operation(ColumnDataMetaData &meta_data, string_t input) {
336
- return input.IsInlined() ? input : meta_data.segment.heap.AddBlob(input);
336
+ return input.IsInlined() ? input : meta_data.segment.heap->AddBlob(input);
337
337
  }
338
338
  };
339
339
 
@@ -423,7 +423,8 @@ void ColumnDataCopy<string_t>(ColumnDataMetaData &meta_data, const UnifiedVector
423
423
  idx_t offset, idx_t copy_count) {
424
424
 
425
425
  const auto &allocator_type = meta_data.segment.allocator->GetType();
426
- if (allocator_type == ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR) {
426
+ if (allocator_type == ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR ||
427
+ allocator_type == ColumnDataAllocatorType::HYBRID) {
427
428
  // strings cannot be spilled to disk - use StringHeap
428
429
  TemplatedColumnDataCopy<StringValueCopy>(meta_data, source_data, source, offset, copy_count);
429
430
  return;
@@ -930,6 +931,7 @@ void ColumnDataCollection::Verify() {
930
931
  #endif
931
932
  }
932
933
 
934
+ // LCOV_EXCL_START
933
935
  string ColumnDataCollection::ToString() const {
934
936
  DataChunk chunk;
935
937
  InitializeScanChunk(chunk);
@@ -950,6 +952,7 @@ string ColumnDataCollection::ToString() const {
950
952
 
951
953
  return result;
952
954
  }
955
+ // LCOV_EXCL_STOP
953
956
 
954
957
  void ColumnDataCollection::Print() const {
955
958
  Printer::Print(ToString());
@@ -1030,6 +1033,18 @@ bool ColumnDataCollection::ResultEquals(const ColumnDataCollection &left, const
1030
1033
  return true;
1031
1034
  }
1032
1035
 
1036
+ vector<shared_ptr<StringHeap>> ColumnDataCollection::GetHeapReferences() {
1037
+ vector<shared_ptr<StringHeap>> result(segments.size(), nullptr);
1038
+ for (idx_t segment_idx = 0; segment_idx < segments.size(); segment_idx++) {
1039
+ result[segment_idx] = segments[segment_idx]->heap;
1040
+ }
1041
+ return result;
1042
+ }
1043
+
1044
+ ColumnDataAllocatorType ColumnDataCollection::GetAllocatorType() const {
1045
+ return allocator->GetType();
1046
+ }
1047
+
1033
1048
  const vector<unique_ptr<ColumnDataCollectionSegment>> &ColumnDataCollection::GetSegments() const {
1034
1049
  return segments;
1035
1050
  }
@@ -6,7 +6,8 @@ namespace duckdb {
6
6
 
7
7
  ColumnDataCollectionSegment::ColumnDataCollectionSegment(shared_ptr<ColumnDataAllocator> allocator_p,
8
8
  vector<LogicalType> types_p)
9
- : allocator(std::move(allocator_p)), types(std::move(types_p)), count(0), heap(allocator->GetAllocator()) {
9
+ : allocator(std::move(allocator_p)), types(std::move(types_p)), count(0),
10
+ heap(make_shared<StringHeap>(allocator->GetAllocator())) {
10
11
  }
11
12
 
12
13
  idx_t ColumnDataCollectionSegment::GetDataSize(idx_t type_size) {
@@ -26,7 +27,8 @@ VectorDataIndex ColumnDataCollectionSegment::AllocateVectorInternal(const Logica
26
27
  auto type_size = internal_type == PhysicalType::STRUCT ? 0 : GetTypeIdSize(internal_type);
27
28
  allocator->AllocateData(GetDataSize(type_size) + ValidityMask::STANDARD_MASK_SIZE, meta_data.block_id,
28
29
  meta_data.offset, chunk_state);
29
- if (allocator->GetType() == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR) {
30
+ if (allocator->GetType() == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR ||
31
+ allocator->GetType() == ColumnDataAllocatorType::HYBRID) {
30
32
  chunk_meta.block_ids.insert(meta_data.block_id);
31
33
  }
32
34
 
@@ -203,10 +205,17 @@ idx_t ColumnDataCollectionSegment::ReadVector(ChunkManagementState &state, Vecto
203
205
  }
204
206
  } else if (internal_type == PhysicalType::VARCHAR) {
205
207
  if (allocator->GetType() == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR) {
206
- for (auto &swizzle_segment : vdata.swizzle_data) {
207
- auto &string_heap_segment = GetVectorData(swizzle_segment.child_index);
208
- allocator->UnswizzlePointers(state, result, swizzle_segment.offset, swizzle_segment.count,
209
- string_heap_segment.block_id, string_heap_segment.offset);
208
+ auto next_index = vector_index;
209
+ idx_t offset = 0;
210
+ while (next_index.IsValid()) {
211
+ auto &current_vdata = GetVectorData(next_index);
212
+ for (auto &swizzle_segment : current_vdata.swizzle_data) {
213
+ auto &string_heap_segment = GetVectorData(swizzle_segment.child_index);
214
+ allocator->UnswizzlePointers(state, result, offset + swizzle_segment.offset, swizzle_segment.count,
215
+ string_heap_segment.block_id, string_heap_segment.offset);
216
+ }
217
+ offset += current_vdata.count;
218
+ next_index = current_vdata.next_data;
210
219
  }
211
220
  }
212
221
  if (state.properties == ColumnDataScanProperties::DISALLOW_ZERO_COPY) {
@@ -32,13 +32,13 @@ PartitionedColumnData::~PartitionedColumnData() {
32
32
 
33
33
  void PartitionedColumnData::InitializeAppendState(PartitionedColumnDataAppendState &state) const {
34
34
  state.partition_sel.Initialize();
35
- state.slice_chunk.Initialize(context, types);
35
+ state.slice_chunk.Initialize(BufferAllocator::Get(context), types);
36
36
  InitializeAppendStateInternal(state);
37
37
  }
38
38
 
39
39
  unique_ptr<DataChunk> PartitionedColumnData::CreatePartitionBuffer() const {
40
40
  auto result = make_uniq<DataChunk>();
41
- result->Initialize(BufferManager::GetBufferManager(context).GetBufferAllocator(), types, BufferSize());
41
+ result->Initialize(BufferAllocator::Get(context), types, BufferSize());
42
42
  return result;
43
43
  }
44
44
 
@@ -309,7 +309,7 @@ void DataChunk::Hash(Vector &result) {
309
309
 
310
310
  void DataChunk::Hash(vector<idx_t> &column_ids, Vector &result) {
311
311
  D_ASSERT(result.GetType().id() == LogicalType::HASH);
312
- D_ASSERT(column_ids.size() > 0);
312
+ D_ASSERT(!column_ids.empty());
313
313
 
314
314
  VectorOperations::Hash(data[column_ids[0]], result, size());
315
315
  for (idx_t i = 1; i < column_ids.size(); i++) {
@@ -327,7 +327,7 @@ void DataChunk::Verify() {
327
327
  #endif
328
328
  }
329
329
 
330
- void DataChunk::Print() {
330
+ void DataChunk::Print() const {
331
331
  Printer::Print(ToString());
332
332
  }
333
333
 
@@ -441,6 +441,15 @@ int64_t Date::EpochMicroseconds(date_t date) {
441
441
  return result;
442
442
  }
443
443
 
444
+ int64_t Date::EpochMilliseconds(date_t date) {
445
+ int64_t result;
446
+ const auto MILLIS_PER_DAY = Interval::MICROS_PER_DAY / Interval::MICROS_PER_MSEC;
447
+ if (!TryMultiplyOperator::Operation<int64_t, int64_t, int64_t>(date.days, MILLIS_PER_DAY, result)) {
448
+ throw ConversionException("Could not convert DATE (%s) to milliseconds", Date::ToString(date));
449
+ }
450
+ return result;
451
+ }
452
+
444
453
  int32_t Date::ExtractYear(date_t d, int32_t *last_year) {
445
454
  auto n = d.days;
446
455
  // cached look up: check if year of this date is the same as the last one we looked up