duckdb 0.8.2-dev157.0 → 0.8.2-dev1573.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (493) hide show
  1. package/binding.gyp +15 -12
  2. package/binding.gyp.in +1 -1
  3. package/configure.py +1 -1
  4. package/duckdb_extension_config.cmake +10 -0
  5. package/package.json +1 -1
  6. package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
  7. package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
  8. package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
  9. package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
  10. package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
  11. package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
  12. package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
  13. package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
  14. package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
  15. package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
  16. package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
  17. package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
  18. package/src/duckdb/extension/json/include/json_common.hpp +47 -231
  19. package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
  20. package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
  21. package/src/duckdb/extension/json/json_common.cpp +272 -40
  22. package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
  23. package/src/duckdb/extension/json/json_functions/json_transform.cpp +17 -37
  24. package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
  25. package/src/duckdb/extension/json/json_functions.cpp +24 -24
  26. package/src/duckdb/extension/json/json_scan.cpp +3 -6
  27. package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
  28. package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
  29. package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
  30. package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
  31. package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
  32. package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
  33. package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
  34. package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
  35. package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
  36. package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
  37. package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
  38. package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
  39. package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
  40. package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
  41. package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
  42. package/src/duckdb/extension/parquet/parquet_extension.cpp +192 -20
  43. package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
  44. package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
  45. package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
  46. package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
  47. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
  48. package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
  49. package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
  50. package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
  51. package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
  52. package/src/duckdb/src/common/allocator.cpp +14 -2
  53. package/src/duckdb/src/common/arrow/arrow_appender.cpp +5 -11
  54. package/src/duckdb/src/common/assert.cpp +3 -0
  55. package/src/duckdb/src/common/enum_util.cpp +4619 -4446
  56. package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
  57. package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
  58. package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
  59. package/src/duckdb/src/common/exception.cpp +2 -2
  60. package/src/duckdb/src/common/extra_type_info.cpp +506 -0
  61. package/src/duckdb/src/common/file_system.cpp +19 -0
  62. package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
  63. package/src/duckdb/src/common/local_file_system.cpp +14 -14
  64. package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
  65. package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
  66. package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
  67. package/src/duckdb/src/common/re2_regex.cpp +1 -1
  68. package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
  69. package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
  70. package/src/duckdb/src/common/sort/partition_state.cpp +44 -11
  71. package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
  72. package/src/duckdb/src/common/types/bit.cpp +51 -0
  73. package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
  74. package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
  75. package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
  76. package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
  77. package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
  78. package/src/duckdb/src/common/types/date.cpp +9 -0
  79. package/src/duckdb/src/common/types/list_segment.cpp +24 -74
  80. package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
  81. package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
  82. package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
  83. package/src/duckdb/src/common/types/uuid.cpp +2 -2
  84. package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
  85. package/src/duckdb/src/common/types.cpp +8 -655
  86. package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
  87. package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
  88. package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
  89. package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
  90. package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
  91. package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
  92. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
  93. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
  94. package/src/duckdb/src/core_functions/function_list.cpp +4 -2
  95. package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
  96. package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
  97. package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
  98. package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
  99. package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
  100. package/src/duckdb/src/execution/expression_executor.cpp +1 -1
  101. package/src/duckdb/src/execution/index/art/art.cpp +149 -139
  102. package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
  103. package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
  104. package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
  105. package/src/duckdb/src/execution/index/art/node.cpp +113 -120
  106. package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
  107. package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
  108. package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
  109. package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
  110. package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
  111. package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
  112. package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
  113. package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
  114. package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
  115. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
  116. package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
  117. package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
  118. package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +444 -284
  119. package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
  120. package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
  121. package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +28 -12
  122. package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
  123. package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +23 -4
  124. package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +41 -5
  125. package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
  126. package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
  127. package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
  128. package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
  129. package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
  130. package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
  131. package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
  132. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
  133. package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
  134. package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
  135. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
  136. package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
  137. package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
  138. package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
  139. package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
  140. package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
  141. package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +56 -33
  142. package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +17 -13
  143. package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
  144. package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
  145. package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
  146. package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
  147. package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
  148. package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
  149. package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
  150. package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
  151. package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
  152. package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
  153. package/src/duckdb/src/function/function.cpp +3 -1
  154. package/src/duckdb/src/function/pragma/pragma_queries.cpp +5 -0
  155. package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
  156. package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
  157. package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
  158. package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
  159. package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
  160. package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
  161. package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
  162. package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
  163. package/src/duckdb/src/function/table/read_csv.cpp +100 -17
  164. package/src/duckdb/src/function/table/system_functions.cpp +1 -0
  165. package/src/duckdb/src/function/table/table_scan.cpp +9 -0
  166. package/src/duckdb/src/function/table/version/pragma_version.cpp +46 -2
  167. package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
  168. package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
  169. package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
  170. package/src/duckdb/src/include/duckdb/common/dl.hpp +3 -1
  171. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +616 -584
  172. package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
  173. package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
  174. package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
  175. package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
  176. package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
  177. package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
  178. package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +219 -0
  179. package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
  180. package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
  181. package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
  182. package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
  183. package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
  184. package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
  185. package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
  186. package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
  187. package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
  188. package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
  189. package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +13 -0
  190. package/src/duckdb/src/include/duckdb/common/string_util.hpp +11 -0
  191. package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
  192. package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
  193. package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
  194. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
  195. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
  196. package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
  197. package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
  198. package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
  199. package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
  200. package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +0 -1
  201. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
  202. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
  203. package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
  204. package/src/duckdb/src/include/duckdb/common/types.hpp +1 -15
  205. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
  206. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
  207. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
  208. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
  209. package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
  210. package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
  211. package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
  212. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
  213. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
  214. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
  215. package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
  216. package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
  217. package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
  218. package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
  219. package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
  220. package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
  221. package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
  222. package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
  223. package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
  224. package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
  225. package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
  226. package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
  227. package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
  228. package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
  229. package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
  230. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
  231. package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
  232. package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
  233. package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
  234. package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
  235. package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
  236. package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
  237. package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
  238. package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
  239. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +3 -10
  240. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +1 -1
  241. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +1 -1
  242. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp +12 -1
  243. package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
  244. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
  245. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
  246. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
  247. package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
  248. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
  249. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
  250. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
  251. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
  252. package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
  253. package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
  254. package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
  255. package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
  256. package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
  257. package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
  258. package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
  259. package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
  260. package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
  261. package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
  262. package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
  263. package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +4 -0
  264. package/src/duckdb/src/include/duckdb/main/client_config.hpp +5 -0
  265. package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
  266. package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
  267. package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
  268. package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
  269. package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
  270. package/src/duckdb/src/include/duckdb/main/settings.hpp +39 -1
  271. package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
  272. package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
  273. package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
  274. package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
  275. package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
  276. package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
  277. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
  278. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
  279. package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
  280. package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
  281. package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
  282. package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
  283. package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
  284. package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
  285. package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
  286. package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
  287. package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
  288. package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
  289. package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
  290. package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
  291. package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
  292. package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
  293. package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
  294. package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
  295. package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
  296. package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
  297. package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
  298. package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
  299. package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
  300. package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
  301. package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
  302. package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
  303. package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
  304. package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
  305. package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
  306. package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
  307. package/src/duckdb/src/include/duckdb/planner/binder.hpp +12 -5
  308. package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
  309. package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
  310. package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
  311. package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
  312. package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
  313. package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
  314. package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
  315. package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
  316. package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
  317. package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
  318. package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
  319. package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
  320. package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
  321. package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
  322. package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
  323. package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
  324. package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
  325. package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
  326. package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
  327. package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
  328. package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
  329. package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
  330. package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
  331. package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
  332. package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
  333. package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
  334. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
  335. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
  336. package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
  337. package/src/duckdb/src/include/duckdb.h +28 -0
  338. package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
  339. package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
  340. package/src/duckdb/src/main/config.cpp +4 -0
  341. package/src/duckdb/src/main/database.cpp +1 -1
  342. package/src/duckdb/src/main/extension/extension_helper.cpp +96 -89
  343. package/src/duckdb/src/main/extension/extension_install.cpp +9 -0
  344. package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
  345. package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
  346. package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
  347. package/src/duckdb/src/main/relation.cpp +6 -5
  348. package/src/duckdb/src/main/settings/settings.cpp +79 -18
  349. package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
  350. package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
  351. package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
  352. package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
  353. package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
  354. package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
  355. package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
  356. package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
  357. package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
  358. package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
  359. package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
  360. package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
  361. package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
  362. package/src/duckdb/src/optimizer/optimizer.cpp +51 -14
  363. package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
  364. package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
  365. package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
  366. package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
  367. package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
  368. package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
  369. package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
  370. package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
  371. package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
  372. package/src/duckdb/src/parallel/executor.cpp +15 -0
  373. package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
  374. package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
  375. package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
  376. package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
  377. package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
  378. package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
  379. package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
  380. package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
  381. package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
  382. package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
  383. package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
  384. package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
  385. package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
  386. package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
  387. package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
  388. package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
  389. package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
  390. package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
  391. package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
  392. package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
  393. package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
  394. package/src/duckdb/src/parser/parser.cpp +8 -2
  395. package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
  396. package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
  397. package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
  398. package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
  399. package/src/duckdb/src/parser/query_node.cpp +15 -37
  400. package/src/duckdb/src/parser/result_modifier.cpp +0 -74
  401. package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
  402. package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
  403. package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
  404. package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
  405. package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
  406. package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
  407. package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
  408. package/src/duckdb/src/parser/tableref.cpp +0 -44
  409. package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
  410. package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
  411. package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
  412. package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
  413. package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
  414. package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
  415. package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
  416. package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
  417. package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
  418. package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
  419. package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
  420. package/src/duckdb/src/parser/transformer.cpp +15 -0
  421. package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
  422. package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
  423. package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
  424. package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
  425. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
  426. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
  427. package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
  428. package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
  429. package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
  430. package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
  431. package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +64 -26
  432. package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
  433. package/src/duckdb/src/planner/binder.cpp +44 -31
  434. package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
  435. package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
  436. package/src/duckdb/src/planner/expression_binder.cpp +3 -0
  437. package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
  438. package/src/duckdb/src/planner/logical_operator.cpp +5 -0
  439. package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
  440. package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
  441. package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
  442. package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
  443. package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
  444. package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
  445. package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
  446. package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
  447. package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
  448. package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
  449. package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
  450. package/src/duckdb/src/storage/compression/rle.cpp +0 -1
  451. package/src/duckdb/src/storage/data_table.cpp +1 -1
  452. package/src/duckdb/src/storage/local_storage.cpp +3 -3
  453. package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
  454. package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
  455. package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
  456. package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
  457. package/src/duckdb/src/storage/serialization/serialize_types.cpp +127 -0
  458. package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
  459. package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
  460. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  461. package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
  462. package/src/duckdb/src/storage/table/row_group.cpp +25 -9
  463. package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
  464. package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
  465. package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
  466. package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
  467. package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
  468. package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
  469. package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
  470. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
  471. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
  472. package/src/duckdb/ub_src_common.cpp +2 -0
  473. package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
  474. package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
  475. package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
  476. package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
  477. package/src/duckdb/ub_src_function_scalar.cpp +2 -0
  478. package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
  479. package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
  480. package/src/duckdb/ub_src_optimizer.cpp +6 -0
  481. package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
  482. package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
  483. package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
  484. package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
  485. package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
  486. package/src/duckdb/ub_src_planner_operator.cpp +4 -0
  487. package/src/duckdb/ub_src_storage_serialization.cpp +10 -0
  488. package/src/statement.cpp +10 -3
  489. package/test/test_all_types.test.ts +233 -0
  490. package/tsconfig.json +1 -0
  491. package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
  492. package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
  493. package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -87,7 +87,7 @@ PartitionGlobalSinkState::PartitionGlobalSinkState(ClientContext &context,
87
87
  const vector<unique_ptr<BaseStatistics>> &partition_stats,
88
88
  idx_t estimated_cardinality)
89
89
  : context(context), buffer_manager(BufferManager::GetBufferManager(context)), allocator(Allocator::Get(context)),
90
- payload_types(payload_types), memory_per_thread(0), count(0) {
90
+ fixed_bits(0), payload_types(payload_types), memory_per_thread(0), count(0) {
91
91
 
92
92
  GenerateOrderings(partitions, orders, partition_bys, order_bys, partition_stats);
93
93
 
@@ -102,9 +102,19 @@ PartitionGlobalSinkState::PartitionGlobalSinkState(ClientContext &context,
102
102
  }
103
103
  }
104
104
 
105
+ void PartitionGlobalSinkState::SyncPartitioning(const PartitionGlobalSinkState &other) {
106
+ fixed_bits = other.grouping_data ? other.grouping_data->GetRadixBits() : 0;
107
+
108
+ const auto old_bits = grouping_data ? grouping_data->GetRadixBits() : 0;
109
+ if (fixed_bits != old_bits) {
110
+ const auto hash_col_idx = payload_types.size();
111
+ grouping_data = make_uniq<RadixPartitionedColumnData>(context, grouping_types, fixed_bits, hash_col_idx);
112
+ }
113
+ }
114
+
105
115
  void PartitionGlobalSinkState::ResizeGroupingData(idx_t cardinality) {
106
116
  // Have we started to combine? Then just live with it.
107
- if (grouping_data && !grouping_data->GetPartitions().empty()) {
117
+ if (fixed_bits || (grouping_data && !grouping_data->GetPartitions().empty())) {
108
118
  return;
109
119
  }
110
120
  // Is the average partition size too large?
@@ -186,9 +196,7 @@ void PartitionGlobalSinkState::CombineLocalPartition(GroupingPartition &local_pa
186
196
  grouping_data->Combine(*local_partition);
187
197
  }
188
198
 
189
- void PartitionGlobalSinkState::BuildSortState(ColumnDataCollection &group_data, PartitionGlobalHashGroup &hash_group) {
190
- auto &global_sort = *hash_group.global_sort;
191
-
199
+ void PartitionGlobalSinkState::BuildSortState(ColumnDataCollection &group_data, GlobalSortState &global_sort) const {
192
200
  // Set up the sort expression computation.
193
201
  vector<LogicalType> sort_types;
194
202
  ExpressionExecutor executor(context);
@@ -234,6 +242,10 @@ void PartitionGlobalSinkState::BuildSortState(ColumnDataCollection &group_data,
234
242
  }
235
243
 
236
244
  global_sort.AddLocalState(local_sort);
245
+ }
246
+
247
+ void PartitionGlobalSinkState::BuildSortState(ColumnDataCollection &group_data, PartitionGlobalHashGroup &hash_group) {
248
+ BuildSortState(group_data, *hash_group.global_sort);
237
249
 
238
250
  hash_group.count += group_data.Count();
239
251
  }
@@ -482,18 +494,29 @@ public:
482
494
  TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override;
483
495
 
484
496
  private:
497
+ struct ExecutorCallback : public PartitionGlobalMergeStates::Callback {
498
+ explicit ExecutorCallback(Executor &executor) : executor(executor) {
499
+ }
500
+
501
+ bool HasError() const override {
502
+ return executor.HasError();
503
+ }
504
+
505
+ Executor &executor;
506
+ };
507
+
485
508
  shared_ptr<Event> event;
486
509
  PartitionLocalMergeState local_state;
487
510
  PartitionGlobalMergeStates &hash_groups;
488
511
  };
489
512
 
490
- TaskExecutionResult PartitionMergeTask::ExecuteTask(TaskExecutionMode mode) {
513
+ bool PartitionGlobalMergeStates::ExecuteTask(PartitionLocalMergeState &local_state, Callback &callback) {
491
514
  // Loop until all hash groups are done
492
515
  size_t sorted = 0;
493
- while (sorted < hash_groups.states.size()) {
516
+ while (sorted < states.size()) {
494
517
  // First check if there is an unfinished task for this thread
495
- if (executor.HasError()) {
496
- return TaskExecutionResult::TASK_ERROR;
518
+ if (callback.HasError()) {
519
+ return false;
497
520
  }
498
521
  if (!local_state.TaskFinished()) {
499
522
  local_state.ExecuteTask();
@@ -501,8 +524,8 @@ TaskExecutionResult PartitionMergeTask::ExecuteTask(TaskExecutionMode mode) {
501
524
  }
502
525
 
503
526
  // Thread is done with its assigned task, try to fetch new work
504
- for (auto group = sorted; group < hash_groups.states.size(); ++group) {
505
- auto &global_state = hash_groups.states[group];
527
+ for (auto group = sorted; group < states.size(); ++group) {
528
+ auto &global_state = states[group];
506
529
  if (global_state->IsSorted()) {
507
530
  // This hash group is done
508
531
  // Update the high water mark of densely completed groups
@@ -543,6 +566,16 @@ TaskExecutionResult PartitionMergeTask::ExecuteTask(TaskExecutionMode mode) {
543
566
  }
544
567
  }
545
568
 
569
+ return true;
570
+ }
571
+
572
+ TaskExecutionResult PartitionMergeTask::ExecuteTask(TaskExecutionMode mode) {
573
+ ExecutorCallback callback(executor);
574
+
575
+ if (!hash_groups.ExecuteTask(local_state, callback)) {
576
+ return TaskExecutionResult::TASK_ERROR;
577
+ }
578
+
546
579
  event->FinishTask();
547
580
  return TaskExecutionResult::TASK_FINISHED;
548
581
  }
@@ -1,11 +1,14 @@
1
1
  #include "duckdb/common/types/batched_data_collection.hpp"
2
+
3
+ #include "duckdb/common/optional_ptr.hpp"
2
4
  #include "duckdb/common/printer.hpp"
3
5
  #include "duckdb/storage/buffer_manager.hpp"
4
- #include "duckdb/common/optional_ptr.hpp"
5
6
 
6
7
  namespace duckdb {
7
8
 
8
- BatchedDataCollection::BatchedDataCollection(vector<LogicalType> types_p) : types(std::move(types_p)) {
9
+ BatchedDataCollection::BatchedDataCollection(ClientContext &context_p, vector<LogicalType> types_p,
10
+ bool buffer_managed_p)
11
+ : context(context_p), types(std::move(types_p)), buffer_managed(buffer_managed_p) {
9
12
  }
10
13
 
11
14
  void BatchedDataCollection::Append(DataChunk &input, idx_t batch_index) {
@@ -20,6 +23,8 @@ void BatchedDataCollection::Append(DataChunk &input, idx_t batch_index) {
20
23
  unique_ptr<ColumnDataCollection> new_collection;
21
24
  if (last_collection.collection) {
22
25
  new_collection = make_uniq<ColumnDataCollection>(*last_collection.collection);
26
+ } else if (buffer_managed) {
27
+ new_collection = make_uniq<ColumnDataCollection>(BufferManager::GetBufferManager(context), types);
23
28
  } else {
24
29
  new_collection = make_uniq<ColumnDataCollection>(Allocator::DefaultAllocator(), types);
25
30
  }
@@ -1,4 +1,6 @@
1
+ #include "duckdb/common/assert.hpp"
1
2
  #include "duckdb/common/operator/cast_operators.hpp"
3
+ #include "duckdb/common/typedefs.hpp"
2
4
  #include "duckdb/common/types/bit.hpp"
3
5
  #include "duckdb/common/types/string_type.hpp"
4
6
 
@@ -34,6 +36,13 @@ static inline idx_t GetBitSize(const string_t &str) {
34
36
  return str_len;
35
37
  }
36
38
 
39
+ uint8_t Bit::GetFirstByte(const string_t &str) {
40
+ D_ASSERT(str.GetSize() > 1);
41
+
42
+ auto data = const_data_ptr_cast(str.GetData());
43
+ return data[1] & ((1 << (8 - data[0])) - 1);
44
+ }
45
+
37
46
  void Bit::Finalize(string_t &str) {
38
47
  // bit strings require all padding bits to be set to 1
39
48
  // this method sets all padding bits to 1
@@ -146,6 +155,48 @@ string Bit::ToBit(string_t str) {
146
155
  return output_str.GetString();
147
156
  }
148
157
 
158
+ void Bit::BlobToBit(string_t blob, string_t &output_str) {
159
+ auto data = const_data_ptr_cast(blob.GetData());
160
+ auto output = output_str.GetDataWriteable();
161
+ idx_t size = blob.GetSize();
162
+
163
+ *output = 0; // No padding
164
+ memcpy(output + 1, data, size);
165
+ }
166
+
167
+ string Bit::BlobToBit(string_t blob) {
168
+ auto buffer = make_unsafe_uniq_array<char>(blob.GetSize() + 1);
169
+ string_t output_str(buffer.get(), blob.GetSize() + 1);
170
+ Bit::BlobToBit(blob, output_str);
171
+ return output_str.GetString();
172
+ }
173
+
174
+ void Bit::BitToBlob(string_t bit, string_t &output_blob) {
175
+ D_ASSERT(bit.GetSize() == output_blob.GetSize() + 1);
176
+
177
+ auto data = const_data_ptr_cast(bit.GetData());
178
+ auto output = output_blob.GetDataWriteable();
179
+ idx_t size = output_blob.GetSize();
180
+
181
+ output[0] = GetFirstByte(bit);
182
+ if (size > 2) {
183
+ ++output;
184
+ // First byte in bitstring contains amount of padded bits,
185
+ // second byte in bitstring is the padded byte,
186
+ // therefore the rest of the data starts at data + 2 (third byte)
187
+ memcpy(output, data + 2, size - 1);
188
+ }
189
+ }
190
+
191
+ string Bit::BitToBlob(string_t bit) {
192
+ D_ASSERT(bit.GetSize() > 1);
193
+
194
+ auto buffer = make_unsafe_uniq_array<char>(bit.GetSize() - 1);
195
+ string_t output_str(buffer.get(), bit.GetSize() - 1);
196
+ Bit::BitToBlob(bit, output_str);
197
+ return output_str.GetString();
198
+ }
199
+
149
200
  // **** scalar functions ****
150
201
  void Bit::BitString(const string_t &input, const idx_t &bit_length, string_t &result) {
151
202
  char *res_buf = result.GetDataWriteable();
@@ -1,8 +1,8 @@
1
1
  #include "duckdb/common/types/column/column_data_allocator.hpp"
2
2
 
3
3
  #include "duckdb/common/types/column/column_data_collection_segment.hpp"
4
- #include "duckdb/storage/buffer_manager.hpp"
5
4
  #include "duckdb/storage/buffer/block_handle.hpp"
5
+ #include "duckdb/storage/buffer_manager.hpp"
6
6
 
7
7
  namespace duckdb {
8
8
 
@@ -19,6 +19,7 @@ ColumnDataAllocator::ColumnDataAllocator(ClientContext &context, ColumnDataAlloc
19
19
  : type(allocator_type) {
20
20
  switch (type) {
21
21
  case ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR:
22
+ case ColumnDataAllocatorType::HYBRID:
22
23
  alloc.buffer_manager = &BufferManager::GetBufferManager(context);
23
24
  break;
24
25
  case ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR:
@@ -33,6 +34,7 @@ ColumnDataAllocator::ColumnDataAllocator(ColumnDataAllocator &other) {
33
34
  type = other.GetType();
34
35
  switch (type) {
35
36
  case ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR:
37
+ case ColumnDataAllocatorType::HYBRID:
36
38
  alloc.allocator = other.alloc.allocator;
37
39
  break;
38
40
  case ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR:
@@ -44,7 +46,7 @@ ColumnDataAllocator::ColumnDataAllocator(ColumnDataAllocator &other) {
44
46
  }
45
47
 
46
48
  BufferHandle ColumnDataAllocator::Pin(uint32_t block_id) {
47
- D_ASSERT(type == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR);
49
+ D_ASSERT(type == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR || type == ColumnDataAllocatorType::HYBRID);
48
50
  shared_ptr<BlockHandle> handle;
49
51
  if (shared) {
50
52
  // we only need to grab the lock when accessing the vector, because vector access is not thread-safe:
@@ -58,7 +60,7 @@ BufferHandle ColumnDataAllocator::Pin(uint32_t block_id) {
58
60
  }
59
61
 
60
62
  BufferHandle ColumnDataAllocator::AllocateBlock(idx_t size) {
61
- D_ASSERT(type == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR);
63
+ D_ASSERT(type == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR || type == ColumnDataAllocatorType::HYBRID);
62
64
  auto block_size = MaxValue<idx_t>(size, Storage::BLOCK_SIZE);
63
65
  BlockMetaData data;
64
66
  data.size = 0;
@@ -136,6 +138,7 @@ void ColumnDataAllocator::AllocateData(idx_t size, uint32_t &block_id, uint32_t
136
138
  ChunkManagementState *chunk_state) {
137
139
  switch (type) {
138
140
  case ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR:
141
+ case ColumnDataAllocatorType::HYBRID:
139
142
  if (shared) {
140
143
  lock_guard<mutex> guard(lock);
141
144
  AllocateBuffer(size, block_id, offset, chunk_state);
@@ -174,8 +177,8 @@ data_ptr_t ColumnDataAllocator::GetDataPointer(ChunkManagementState &state, uint
174
177
  return state.handles[block_id].Ptr() + offset;
175
178
  }
176
179
 
177
- void ColumnDataAllocator::UnswizzlePointers(ChunkManagementState &state, Vector &result, uint16_t v_offset,
178
- uint16_t count, uint32_t block_id, uint32_t offset) {
180
+ void ColumnDataAllocator::UnswizzlePointers(ChunkManagementState &state, Vector &result, idx_t v_offset, uint16_t count,
181
+ uint32_t block_id, uint32_t offset) {
179
182
  D_ASSERT(result.GetType().InternalType() == PhysicalType::VARCHAR);
180
183
  lock_guard<mutex> guard(lock);
181
184
 
@@ -225,7 +228,7 @@ Allocator &ColumnDataAllocator::GetAllocator() {
225
228
  }
226
229
 
227
230
  void ColumnDataAllocator::InitializeChunkState(ChunkManagementState &state, ChunkMetaData &chunk) {
228
- if (type != ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR) {
231
+ if (type != ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR && type != ColumnDataAllocatorType::HYBRID) {
229
232
  // nothing to pin
230
233
  return;
231
234
  }
@@ -333,7 +333,7 @@ struct StandardValueCopy : public BaseValueCopy<T> {
333
333
 
334
334
  struct StringValueCopy : public BaseValueCopy<string_t> {
335
335
  static string_t Operation(ColumnDataMetaData &meta_data, string_t input) {
336
- return input.IsInlined() ? input : meta_data.segment.heap.AddBlob(input);
336
+ return input.IsInlined() ? input : meta_data.segment.heap->AddBlob(input);
337
337
  }
338
338
  };
339
339
 
@@ -423,7 +423,8 @@ void ColumnDataCopy<string_t>(ColumnDataMetaData &meta_data, const UnifiedVector
423
423
  idx_t offset, idx_t copy_count) {
424
424
 
425
425
  const auto &allocator_type = meta_data.segment.allocator->GetType();
426
- if (allocator_type == ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR) {
426
+ if (allocator_type == ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR ||
427
+ allocator_type == ColumnDataAllocatorType::HYBRID) {
427
428
  // strings cannot be spilled to disk - use StringHeap
428
429
  TemplatedColumnDataCopy<StringValueCopy>(meta_data, source_data, source, offset, copy_count);
429
430
  return;
@@ -930,6 +931,7 @@ void ColumnDataCollection::Verify() {
930
931
  #endif
931
932
  }
932
933
 
934
+ // LCOV_EXCL_START
933
935
  string ColumnDataCollection::ToString() const {
934
936
  DataChunk chunk;
935
937
  InitializeScanChunk(chunk);
@@ -950,6 +952,7 @@ string ColumnDataCollection::ToString() const {
950
952
 
951
953
  return result;
952
954
  }
955
+ // LCOV_EXCL_STOP
953
956
 
954
957
  void ColumnDataCollection::Print() const {
955
958
  Printer::Print(ToString());
@@ -1030,6 +1033,18 @@ bool ColumnDataCollection::ResultEquals(const ColumnDataCollection &left, const
1030
1033
  return true;
1031
1034
  }
1032
1035
 
1036
+ vector<shared_ptr<StringHeap>> ColumnDataCollection::GetHeapReferences() {
1037
+ vector<shared_ptr<StringHeap>> result(segments.size(), nullptr);
1038
+ for (idx_t segment_idx = 0; segment_idx < segments.size(); segment_idx++) {
1039
+ result[segment_idx] = segments[segment_idx]->heap;
1040
+ }
1041
+ return result;
1042
+ }
1043
+
1044
+ ColumnDataAllocatorType ColumnDataCollection::GetAllocatorType() const {
1045
+ return allocator->GetType();
1046
+ }
1047
+
1033
1048
  const vector<unique_ptr<ColumnDataCollectionSegment>> &ColumnDataCollection::GetSegments() const {
1034
1049
  return segments;
1035
1050
  }
@@ -6,7 +6,8 @@ namespace duckdb {
6
6
 
7
7
  ColumnDataCollectionSegment::ColumnDataCollectionSegment(shared_ptr<ColumnDataAllocator> allocator_p,
8
8
  vector<LogicalType> types_p)
9
- : allocator(std::move(allocator_p)), types(std::move(types_p)), count(0), heap(allocator->GetAllocator()) {
9
+ : allocator(std::move(allocator_p)), types(std::move(types_p)), count(0),
10
+ heap(make_shared<StringHeap>(allocator->GetAllocator())) {
10
11
  }
11
12
 
12
13
  idx_t ColumnDataCollectionSegment::GetDataSize(idx_t type_size) {
@@ -26,7 +27,8 @@ VectorDataIndex ColumnDataCollectionSegment::AllocateVectorInternal(const Logica
26
27
  auto type_size = internal_type == PhysicalType::STRUCT ? 0 : GetTypeIdSize(internal_type);
27
28
  allocator->AllocateData(GetDataSize(type_size) + ValidityMask::STANDARD_MASK_SIZE, meta_data.block_id,
28
29
  meta_data.offset, chunk_state);
29
- if (allocator->GetType() == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR) {
30
+ if (allocator->GetType() == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR ||
31
+ allocator->GetType() == ColumnDataAllocatorType::HYBRID) {
30
32
  chunk_meta.block_ids.insert(meta_data.block_id);
31
33
  }
32
34
 
@@ -203,10 +205,17 @@ idx_t ColumnDataCollectionSegment::ReadVector(ChunkManagementState &state, Vecto
203
205
  }
204
206
  } else if (internal_type == PhysicalType::VARCHAR) {
205
207
  if (allocator->GetType() == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR) {
206
- for (auto &swizzle_segment : vdata.swizzle_data) {
207
- auto &string_heap_segment = GetVectorData(swizzle_segment.child_index);
208
- allocator->UnswizzlePointers(state, result, swizzle_segment.offset, swizzle_segment.count,
209
- string_heap_segment.block_id, string_heap_segment.offset);
208
+ auto next_index = vector_index;
209
+ idx_t offset = 0;
210
+ while (next_index.IsValid()) {
211
+ auto &current_vdata = GetVectorData(next_index);
212
+ for (auto &swizzle_segment : current_vdata.swizzle_data) {
213
+ auto &string_heap_segment = GetVectorData(swizzle_segment.child_index);
214
+ allocator->UnswizzlePointers(state, result, offset + swizzle_segment.offset, swizzle_segment.count,
215
+ string_heap_segment.block_id, string_heap_segment.offset);
216
+ }
217
+ offset += current_vdata.count;
218
+ next_index = current_vdata.next_data;
210
219
  }
211
220
  }
212
221
  if (state.properties == ColumnDataScanProperties::DISALLOW_ZERO_COPY) {
@@ -32,13 +32,13 @@ PartitionedColumnData::~PartitionedColumnData() {
32
32
 
33
33
  void PartitionedColumnData::InitializeAppendState(PartitionedColumnDataAppendState &state) const {
34
34
  state.partition_sel.Initialize();
35
- state.slice_chunk.Initialize(context, types);
35
+ state.slice_chunk.Initialize(BufferAllocator::Get(context), types);
36
36
  InitializeAppendStateInternal(state);
37
37
  }
38
38
 
39
39
  unique_ptr<DataChunk> PartitionedColumnData::CreatePartitionBuffer() const {
40
40
  auto result = make_uniq<DataChunk>();
41
- result->Initialize(BufferManager::GetBufferManager(context).GetBufferAllocator(), types, BufferSize());
41
+ result->Initialize(BufferAllocator::Get(context), types, BufferSize());
42
42
  return result;
43
43
  }
44
44
 
@@ -309,7 +309,7 @@ void DataChunk::Hash(Vector &result) {
309
309
 
310
310
  void DataChunk::Hash(vector<idx_t> &column_ids, Vector &result) {
311
311
  D_ASSERT(result.GetType().id() == LogicalType::HASH);
312
- D_ASSERT(column_ids.size() > 0);
312
+ D_ASSERT(!column_ids.empty());
313
313
 
314
314
  VectorOperations::Hash(data[column_ids[0]], result, size());
315
315
  for (idx_t i = 1; i < column_ids.size(); i++) {
@@ -327,7 +327,7 @@ void DataChunk::Verify() {
327
327
  #endif
328
328
  }
329
329
 
330
- void DataChunk::Print() {
330
+ void DataChunk::Print() const {
331
331
  Printer::Print(ToString());
332
332
  }
333
333
 
@@ -441,6 +441,15 @@ int64_t Date::EpochMicroseconds(date_t date) {
441
441
  return result;
442
442
  }
443
443
 
444
+ int64_t Date::EpochMilliseconds(date_t date) {
445
+ int64_t result;
446
+ const auto MILLIS_PER_DAY = Interval::MICROS_PER_DAY / Interval::MICROS_PER_MSEC;
447
+ if (!TryMultiplyOperator::Operation<int64_t, int64_t, int64_t>(date.days, MILLIS_PER_DAY, result)) {
448
+ throw ConversionException("Could not convert DATE (%s) to milliseconds", Date::ToString(date));
449
+ }
450
+ return result;
451
+ }
452
+
444
453
  int32_t Date::ExtractYear(date_t d, int32_t *last_year) {
445
454
  auto n = d.days;
446
455
  // cached look up: check if year of this date is the same as the last one we looked up
@@ -12,8 +12,8 @@ static idx_t GetAllocationSize(uint16_t capacity) {
12
12
  }
13
13
 
14
14
  template <class T>
15
- static data_ptr_t AllocatePrimitiveData(Allocator &allocator, uint16_t capacity) {
16
- return allocator.AllocateData(GetAllocationSize<T>(capacity));
15
+ static data_ptr_t AllocatePrimitiveData(ArenaAllocator &allocator, uint16_t capacity) {
16
+ return allocator.Allocate(GetAllocationSize<T>(capacity));
17
17
  }
18
18
 
19
19
  template <class T>
@@ -34,8 +34,8 @@ static idx_t GetAllocationSizeList(uint16_t capacity) {
34
34
  return AlignValue(sizeof(ListSegment) + capacity * (sizeof(bool) + sizeof(uint64_t)) + sizeof(LinkedList));
35
35
  }
36
36
 
37
- static data_ptr_t AllocateListData(Allocator &allocator, uint16_t capacity) {
38
- return allocator.AllocateData(GetAllocationSizeList(capacity));
37
+ static data_ptr_t AllocateListData(ArenaAllocator &allocator, uint16_t capacity) {
38
+ return allocator.Allocate(GetAllocationSizeList(capacity));
39
39
  }
40
40
 
41
41
  static uint64_t *GetListLengthData(ListSegment *segment) {
@@ -65,8 +65,8 @@ static idx_t GetAllocationSizeStruct(uint16_t capacity, idx_t child_count) {
65
65
  return AlignValue(sizeof(ListSegment) + capacity * sizeof(bool) + child_count * sizeof(ListSegment *));
66
66
  }
67
67
 
68
- static data_ptr_t AllocateStructData(Allocator &allocator, uint16_t capacity, idx_t child_count) {
69
- return allocator.AllocateData(GetAllocationSizeStruct(capacity, child_count));
68
+ static data_ptr_t AllocateStructData(ArenaAllocator &allocator, uint16_t capacity, idx_t child_count) {
69
+ return allocator.Allocate(GetAllocationSizeStruct(capacity, child_count));
70
70
  }
71
71
 
72
72
  static ListSegment **GetStructData(ListSegment *segment) {
@@ -98,20 +98,8 @@ static uint16_t GetCapacityForNewSegment(uint16_t capacity) {
98
98
  //===--------------------------------------------------------------------===//
99
99
  // Create & Destroy
100
100
  //===--------------------------------------------------------------------===//
101
- static void DestroyLinkedList(const ListSegmentFunctions &functions, Allocator &allocator, LinkedList &list) {
102
- auto segment = list.first_segment;
103
- while (segment) {
104
- auto next_segment = segment->next;
105
- functions.destroy(functions, segment, allocator);
106
- segment = next_segment;
107
- }
108
- list.first_segment = nullptr;
109
- list.last_segment = nullptr;
110
- list.total_capacity = 0;
111
- }
112
-
113
101
  template <class T>
114
- static ListSegment *CreatePrimitiveSegment(const ListSegmentFunctions &, Allocator &allocator, uint16_t capacity) {
102
+ static ListSegment *CreatePrimitiveSegment(const ListSegmentFunctions &, ArenaAllocator &allocator, uint16_t capacity) {
115
103
  // allocate data and set the header
116
104
  auto segment = (ListSegment *)AllocatePrimitiveData<T>(allocator, capacity);
117
105
  segment->capacity = capacity;
@@ -120,13 +108,7 @@ static ListSegment *CreatePrimitiveSegment(const ListSegmentFunctions &, Allocat
120
108
  return segment;
121
109
  }
122
110
 
123
- template <class T>
124
- void DestroyPrimitiveSegment(const ListSegmentFunctions &, ListSegment *segment, Allocator &allocator) {
125
- D_ASSERT(segment);
126
- allocator.FreeData(data_ptr_cast(segment), GetAllocationSize<T>(segment->capacity));
127
- }
128
-
129
- static ListSegment *CreateListSegment(const ListSegmentFunctions &, Allocator &allocator, uint16_t capacity) {
111
+ static ListSegment *CreateListSegment(const ListSegmentFunctions &, ArenaAllocator &allocator, uint16_t capacity) {
130
112
  // allocate data and set the header
131
113
  auto segment = reinterpret_cast<ListSegment *>(AllocateListData(allocator, capacity));
132
114
  segment->capacity = capacity;
@@ -141,16 +123,7 @@ static ListSegment *CreateListSegment(const ListSegmentFunctions &, Allocator &a
141
123
  return segment;
142
124
  }
143
125
 
144
- void DestroyListSegment(const ListSegmentFunctions &functions, ListSegment *segment, Allocator &allocator) {
145
- // destroy the child list
146
- auto linked_child_list = Load<LinkedList>(data_ptr_cast(GetListChildData(segment)));
147
- DestroyLinkedList(functions.child_functions[0], allocator, linked_child_list);
148
-
149
- // destroy the list segment itself
150
- allocator.FreeData(data_ptr_cast(segment), GetAllocationSizeList(segment->capacity));
151
- }
152
-
153
- static ListSegment *CreateStructSegment(const ListSegmentFunctions &functions, Allocator &allocator,
126
+ static ListSegment *CreateStructSegment(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
154
127
  uint16_t capacity) {
155
128
  // allocate data and set header
156
129
  auto segment =
@@ -170,21 +143,8 @@ static ListSegment *CreateStructSegment(const ListSegmentFunctions &functions, A
170
143
  return segment;
171
144
  }
172
145
 
173
- void DestroyStructSegment(const ListSegmentFunctions &functions, ListSegment *segment, Allocator &allocator) {
174
- // destroy the child entries
175
- auto child_segments = GetStructData(segment);
176
- for (idx_t i = 0; i < functions.child_functions.size(); i++) {
177
- auto child_function = functions.child_functions[i];
178
- auto child_segment = Load<ListSegment *>(data_ptr_cast(child_segments + i));
179
- child_function.destroy(child_function, child_segment, allocator);
180
- }
181
-
182
- // destroy the struct segment itself
183
- allocator.FreeData(data_ptr_cast(segment),
184
- GetAllocationSizeStruct(segment->capacity, functions.child_functions.size()));
185
- }
186
-
187
- static ListSegment *GetSegment(const ListSegmentFunctions &functions, Allocator &allocator, LinkedList &linked_list) {
146
+ static ListSegment *GetSegment(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
147
+ LinkedList &linked_list) {
188
148
  ListSegment *segment;
189
149
 
190
150
  // determine segment
@@ -214,7 +174,7 @@ static ListSegment *GetSegment(const ListSegmentFunctions &functions, Allocator
214
174
  // Append
215
175
  //===--------------------------------------------------------------------===//
216
176
  template <class T>
217
- static void WriteDataToPrimitiveSegment(const ListSegmentFunctions &functions, Allocator &allocator,
177
+ static void WriteDataToPrimitiveSegment(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
218
178
  ListSegment *segment, Vector &input, idx_t &entry_idx, idx_t &count) {
219
179
 
220
180
  // get the vector data and the source index of the entry that we want to write
@@ -232,8 +192,8 @@ static void WriteDataToPrimitiveSegment(const ListSegmentFunctions &functions, A
232
192
  }
233
193
  }
234
194
 
235
- static void WriteDataToVarcharSegment(const ListSegmentFunctions &functions, Allocator &allocator, ListSegment *segment,
236
- Vector &input, idx_t &entry_idx, idx_t &count) {
195
+ static void WriteDataToVarcharSegment(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
196
+ ListSegment *segment, Vector &input, idx_t &entry_idx, idx_t &count) {
237
197
 
238
198
  // get the vector data and the source index of the entry that we want to write
239
199
  auto input_data = FlatVector::GetData<string_t>(input);
@@ -275,8 +235,8 @@ static void WriteDataToVarcharSegment(const ListSegmentFunctions &functions, All
275
235
  Store<LinkedList>(child_segments, data_ptr_cast(GetListChildData(segment)));
276
236
  }
277
237
 
278
- static void WriteDataToListSegment(const ListSegmentFunctions &functions, Allocator &allocator, ListSegment *segment,
279
- Vector &input, idx_t &entry_idx, idx_t &count) {
238
+ static void WriteDataToListSegment(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
239
+ ListSegment *segment, Vector &input, idx_t &entry_idx, idx_t &count) {
280
240
 
281
241
  // get the vector data and the source index of the entry that we want to write
282
242
  auto input_data = FlatVector::GetData<list_entry_t>(input);
@@ -315,8 +275,8 @@ static void WriteDataToListSegment(const ListSegmentFunctions &functions, Alloca
315
275
  Store<uint64_t>(list_length, data_ptr_cast(list_length_data + segment->count));
316
276
  }
317
277
 
318
- static void WriteDataToStructSegment(const ListSegmentFunctions &functions, Allocator &allocator, ListSegment *segment,
319
- Vector &input, idx_t &entry_idx, idx_t &count) {
278
+ static void WriteDataToStructSegment(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
279
+ ListSegment *segment, Vector &input, idx_t &entry_idx, idx_t &count) {
320
280
 
321
281
  // write null validity
322
282
  auto null_mask = GetNullMask(segment);
@@ -338,8 +298,8 @@ static void WriteDataToStructSegment(const ListSegmentFunctions &functions, Allo
338
298
  }
339
299
  }
340
300
 
341
- void ListSegmentFunctions::AppendRow(Allocator &allocator, LinkedList &linked_list, Vector &input, idx_t &entry_idx,
342
- idx_t &count) const {
301
+ void ListSegmentFunctions::AppendRow(ArenaAllocator &allocator, LinkedList &linked_list, Vector &input,
302
+ idx_t &entry_idx, idx_t &count) const {
343
303
 
344
304
  D_ASSERT(input.GetVectorType() == VectorType::FLAT_VECTOR);
345
305
  auto &write_data_to_segment = *this;
@@ -503,7 +463,7 @@ void ListSegmentFunctions::BuildListVector(const LinkedList &linked_list, Vector
503
463
  //===--------------------------------------------------------------------===//
504
464
  template <class T>
505
465
  static ListSegment *CopyDataFromPrimitiveSegment(const ListSegmentFunctions &, const ListSegment *source,
506
- Allocator &allocator) {
466
+ ArenaAllocator &allocator) {
507
467
 
508
468
  auto target = (ListSegment *)AllocatePrimitiveData<T>(allocator, source->capacity);
509
469
  memcpy(target, source, sizeof(ListSegment) + source->capacity * (sizeof(bool) + sizeof(T)));
@@ -512,7 +472,7 @@ static ListSegment *CopyDataFromPrimitiveSegment(const ListSegmentFunctions &, c
512
472
  }
513
473
 
514
474
  static ListSegment *CopyDataFromListSegment(const ListSegmentFunctions &functions, const ListSegment *source,
515
- Allocator &allocator) {
475
+ ArenaAllocator &allocator) {
516
476
 
517
477
  // create an empty linked list for the child vector of target
518
478
  auto source_linked_child_list = Load<LinkedList>(const_data_ptr_cast(GetListChildData(source)));
@@ -538,7 +498,7 @@ static ListSegment *CopyDataFromListSegment(const ListSegmentFunctions &function
538
498
  }
539
499
 
540
500
  static ListSegment *CopyDataFromStructSegment(const ListSegmentFunctions &functions, const ListSegment *source,
541
- Allocator &allocator) {
501
+ ArenaAllocator &allocator) {
542
502
 
543
503
  auto source_child_count = functions.child_functions.size();
544
504
  auto target = reinterpret_cast<ListSegment *>(AllocateStructData(allocator, source->capacity, source_child_count));
@@ -560,7 +520,7 @@ static ListSegment *CopyDataFromStructSegment(const ListSegmentFunctions &functi
560
520
  }
561
521
 
562
522
  void ListSegmentFunctions::CopyLinkedList(const LinkedList &source_list, LinkedList &target_list,
563
- Allocator &allocator) const {
523
+ ArenaAllocator &allocator) const {
564
524
  auto &copy_data_from_segment = *this;
565
525
  auto source_segment = source_list.first_segment;
566
526
 
@@ -578,12 +538,6 @@ void ListSegmentFunctions::CopyLinkedList(const LinkedList &source_list, LinkedL
578
538
  }
579
539
  }
580
540
 
581
- //===--------------------------------------------------------------------===//
582
- // Destroy
583
- //===--------------------------------------------------------------------===//
584
- void ListSegmentFunctions::Destroy(Allocator &allocator, LinkedList &linked_list) const {
585
- DestroyLinkedList(*this, allocator, linked_list);
586
- }
587
541
  //===--------------------------------------------------------------------===//
588
542
  // Functions
589
543
  //===--------------------------------------------------------------------===//
@@ -593,7 +547,6 @@ void SegmentPrimitiveFunction(ListSegmentFunctions &functions) {
593
547
  functions.write_data = WriteDataToPrimitiveSegment<T>;
594
548
  functions.read_data = ReadDataFromPrimitiveSegment<T>;
595
549
  functions.copy_data = CopyDataFromPrimitiveSegment<T>;
596
- functions.destroy = DestroyPrimitiveSegment<T>;
597
550
  }
598
551
 
599
552
  void GetSegmentDataFunctions(ListSegmentFunctions &functions, const LogicalType &type) {
@@ -645,7 +598,6 @@ void GetSegmentDataFunctions(ListSegmentFunctions &functions, const LogicalType
645
598
  functions.write_data = WriteDataToVarcharSegment;
646
599
  functions.read_data = ReadDataFromVarcharSegment;
647
600
  functions.copy_data = CopyDataFromListSegment;
648
- functions.destroy = DestroyListSegment;
649
601
 
650
602
  functions.child_functions.emplace_back();
651
603
  SegmentPrimitiveFunction<char>(functions.child_functions.back());
@@ -656,7 +608,6 @@ void GetSegmentDataFunctions(ListSegmentFunctions &functions, const LogicalType
656
608
  functions.write_data = WriteDataToListSegment;
657
609
  functions.read_data = ReadDataFromListSegment;
658
610
  functions.copy_data = CopyDataFromListSegment;
659
- functions.destroy = DestroyListSegment;
660
611
 
661
612
  // recurse
662
613
  functions.child_functions.emplace_back();
@@ -668,7 +619,6 @@ void GetSegmentDataFunctions(ListSegmentFunctions &functions, const LogicalType
668
619
  functions.write_data = WriteDataToStructSegment;
669
620
  functions.read_data = ReadDataFromStructSegment;
670
621
  functions.copy_data = CopyDataFromStructSegment;
671
- functions.destroy = DestroyStructSegment;
672
622
 
673
623
  // recurse
674
624
  auto child_types = StructType::GetChildTypes(type);