duckdb 0.8.2-dev77.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1321) hide show
  1. package/README.md +7 -0
  2. package/binding.gyp +30 -15
  3. package/binding.gyp.in +1 -2
  4. package/configure.py +11 -3
  5. package/duckdb_extension_config.cmake +10 -0
  6. package/lib/duckdb.js +14 -4
  7. package/package.json +3 -1
  8. package/src/connection.cpp +67 -38
  9. package/src/data_chunk.cpp +1 -3
  10. package/src/database.cpp +9 -17
  11. package/src/duckdb/extension/icu/icu-dateadd.cpp +13 -21
  12. package/src/duckdb/extension/icu/icu-datefunc.cpp +10 -1
  13. package/src/duckdb/extension/icu/icu-datepart.cpp +216 -100
  14. package/src/duckdb/extension/icu/icu-datesub.cpp +13 -17
  15. package/src/duckdb/extension/icu/icu-datetrunc.cpp +8 -9
  16. package/src/duckdb/extension/icu/icu-list-range.cpp +7 -9
  17. package/src/duckdb/extension/icu/icu-makedate.cpp +27 -16
  18. package/src/duckdb/extension/icu/icu-strptime.cpp +36 -65
  19. package/src/duckdb/extension/icu/icu-table-range.cpp +11 -14
  20. package/src/duckdb/extension/icu/icu-timebucket.cpp +21 -23
  21. package/src/duckdb/extension/icu/icu-timezone.cpp +26 -37
  22. package/src/duckdb/extension/icu/icu_extension.cpp +29 -36
  23. package/src/duckdb/extension/icu/include/icu-dateadd.hpp +1 -1
  24. package/src/duckdb/extension/icu/include/icu-datepart.hpp +1 -1
  25. package/src/duckdb/extension/icu/include/icu-datesub.hpp +1 -1
  26. package/src/duckdb/extension/icu/include/icu-datetrunc.hpp +1 -1
  27. package/src/duckdb/extension/icu/include/icu-list-range.hpp +1 -1
  28. package/src/duckdb/extension/icu/include/icu-makedate.hpp +1 -1
  29. package/src/duckdb/extension/icu/include/icu-strptime.hpp +1 -1
  30. package/src/duckdb/extension/icu/include/icu-table-range.hpp +1 -1
  31. package/src/duckdb/extension/icu/include/icu-timebucket.hpp +1 -1
  32. package/src/duckdb/extension/icu/include/icu-timezone.hpp +1 -1
  33. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/gregocal.h +1 -1
  34. package/src/duckdb/extension/json/buffered_json_reader.cpp +82 -92
  35. package/src/duckdb/extension/json/include/buffered_json_reader.hpp +38 -52
  36. package/src/duckdb/extension/json/include/json_common.hpp +47 -231
  37. package/src/duckdb/extension/json/include/json_deserializer.hpp +9 -18
  38. package/src/duckdb/extension/json/include/json_enums.hpp +60 -0
  39. package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
  40. package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
  41. package/src/duckdb/extension/json/include/json_scan.hpp +21 -21
  42. package/src/duckdb/extension/json/include/json_serializer.hpp +12 -18
  43. package/src/duckdb/extension/json/include/json_transform.hpp +2 -2
  44. package/src/duckdb/extension/json/json_common.cpp +272 -40
  45. package/src/duckdb/extension/json/json_deserializer.cpp +37 -73
  46. package/src/duckdb/extension/json/json_enums.cpp +105 -0
  47. package/src/duckdb/extension/json/json_functions/copy_json.cpp +1 -1
  48. package/src/duckdb/extension/json/json_functions/json_create.cpp +21 -2
  49. package/src/duckdb/extension/json/json_functions/json_serialize_sql.cpp +4 -1
  50. package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
  51. package/src/duckdb/extension/json/json_functions/json_transform.cpp +98 -57
  52. package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
  53. package/src/duckdb/extension/json/json_functions/read_json.cpp +21 -2
  54. package/src/duckdb/extension/json/json_functions.cpp +30 -31
  55. package/src/duckdb/extension/json/json_scan.cpp +148 -175
  56. package/src/duckdb/extension/json/json_serializer.cpp +29 -72
  57. package/src/duckdb/extension/json/serialize_json.cpp +92 -0
  58. package/src/duckdb/extension/parquet/column_reader.cpp +37 -25
  59. package/src/duckdb/extension/parquet/column_writer.cpp +121 -104
  60. package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
  61. package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
  62. package/src/duckdb/extension/parquet/include/column_writer.hpp +11 -9
  63. package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
  64. package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
  65. package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
  66. package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
  67. package/src/duckdb/extension/parquet/include/parquet_reader.hpp +3 -2
  68. package/src/duckdb/extension/parquet/include/parquet_rle_bp_encoder.hpp +4 -4
  69. package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
  70. package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
  71. package/src/duckdb/extension/parquet/include/parquet_timestamp.hpp +1 -0
  72. package/src/duckdb/extension/parquet/include/parquet_writer.hpp +29 -5
  73. package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
  74. package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
  75. package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
  76. package/src/duckdb/extension/parquet/parquet_extension.cpp +340 -93
  77. package/src/duckdb/extension/parquet/parquet_reader.cpp +10 -10
  78. package/src/duckdb/extension/parquet/parquet_statistics.cpp +25 -8
  79. package/src/duckdb/extension/parquet/parquet_timestamp.cpp +6 -0
  80. package/src/duckdb/extension/parquet/parquet_writer.cpp +170 -33
  81. package/src/duckdb/extension/parquet/serialize_parquet.cpp +26 -0
  82. package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
  83. package/src/duckdb/src/catalog/catalog.cpp +177 -70
  84. package/src/duckdb/src/catalog/catalog_entry/duck_index_entry.cpp +5 -0
  85. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +8 -11
  86. package/src/duckdb/src/catalog/catalog_entry/index_catalog_entry.cpp +17 -42
  87. package/src/duckdb/src/catalog/catalog_entry/macro_catalog_entry.cpp +2 -11
  88. package/src/duckdb/src/catalog/catalog_entry/schema_catalog_entry.cpp +4 -15
  89. package/src/duckdb/src/catalog/catalog_entry/sequence_catalog_entry.cpp +11 -29
  90. package/src/duckdb/src/catalog/catalog_entry/table_catalog_entry.cpp +11 -44
  91. package/src/duckdb/src/catalog/catalog_entry/type_catalog_entry.cpp +8 -29
  92. package/src/duckdb/src/catalog/catalog_entry/view_catalog_entry.cpp +11 -28
  93. package/src/duckdb/src/catalog/catalog_entry.cpp +16 -1
  94. package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
  95. package/src/duckdb/src/catalog/catalog_set.cpp +16 -77
  96. package/src/duckdb/src/catalog/default/default_functions.cpp +23 -2
  97. package/src/duckdb/src/catalog/dependency_manager.cpp +0 -36
  98. package/src/duckdb/src/catalog/duck_catalog.cpp +4 -0
  99. package/src/duckdb/src/common/adbc/adbc.cpp +541 -171
  100. package/src/duckdb/src/common/adbc/driver_manager.cpp +92 -39
  101. package/src/duckdb/src/common/adbc/nanoarrow/allocator.cpp +57 -0
  102. package/src/duckdb/src/common/adbc/nanoarrow/metadata.cpp +121 -0
  103. package/src/duckdb/src/common/adbc/nanoarrow/schema.cpp +474 -0
  104. package/src/duckdb/src/common/adbc/nanoarrow/single_batch_array_stream.cpp +84 -0
  105. package/src/duckdb/src/common/allocator.cpp +14 -2
  106. package/src/duckdb/src/common/arrow/appender/bool_data.cpp +44 -0
  107. package/src/duckdb/src/common/arrow/appender/list_data.cpp +78 -0
  108. package/src/duckdb/src/common/arrow/appender/map_data.cpp +86 -0
  109. package/src/duckdb/src/common/arrow/appender/struct_data.cpp +45 -0
  110. package/src/duckdb/src/common/arrow/appender/union_data.cpp +70 -0
  111. package/src/duckdb/src/common/arrow/arrow_appender.cpp +95 -666
  112. package/src/duckdb/src/common/arrow/arrow_converter.cpp +68 -38
  113. package/src/duckdb/src/common/arrow/arrow_wrapper.cpp +37 -42
  114. package/src/duckdb/src/common/assert.cpp +3 -0
  115. package/src/duckdb/src/common/constants.cpp +2 -2
  116. package/src/duckdb/src/common/crypto/md5.cpp +2 -12
  117. package/src/duckdb/src/common/enum_util.cpp +4967 -4412
  118. package/src/duckdb/src/common/enums/date_part_specifier.cpp +2 -0
  119. package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
  120. package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
  121. package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
  122. package/src/duckdb/src/common/exception.cpp +15 -2
  123. package/src/duckdb/src/common/extra_type_info.cpp +315 -0
  124. package/src/duckdb/src/common/file_buffer.cpp +2 -2
  125. package/src/duckdb/src/common/file_system.cpp +46 -12
  126. package/src/duckdb/src/common/filename_pattern.cpp +1 -1
  127. package/src/duckdb/src/common/gzip_file_system.cpp +7 -12
  128. package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
  129. package/src/duckdb/src/common/http_state.cpp +78 -0
  130. package/src/duckdb/src/common/local_file_system.cpp +36 -28
  131. package/src/duckdb/src/common/multi_file_reader.cpp +175 -70
  132. package/src/duckdb/src/common/operator/cast_operators.cpp +110 -1
  133. package/src/duckdb/src/common/operator/string_cast.cpp +45 -8
  134. package/src/duckdb/src/common/radix_partitioning.cpp +34 -39
  135. package/src/duckdb/src/common/re2_regex.cpp +1 -1
  136. package/src/duckdb/src/common/row_operations/row_aggregate.cpp +18 -3
  137. package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
  138. package/src/duckdb/src/common/row_operations/row_matcher.cpp +375 -0
  139. package/src/duckdb/src/common/serializer/binary_deserializer.cpp +63 -73
  140. package/src/duckdb/src/common/serializer/binary_serializer.cpp +88 -78
  141. package/src/duckdb/src/common/serializer/buffered_file_reader.cpp +3 -20
  142. package/src/duckdb/src/common/serializer/buffered_file_writer.cpp +1 -1
  143. package/src/duckdb/src/common/serializer/memory_stream.cpp +61 -0
  144. package/src/duckdb/src/common/serializer/serializer.cpp +15 -0
  145. package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
  146. package/src/duckdb/src/common/sort/partition_state.cpp +210 -100
  147. package/src/duckdb/src/common/sort/sort_state.cpp +1 -1
  148. package/src/duckdb/src/common/sort/sorted_block.cpp +1 -1
  149. package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
  150. package/src/duckdb/src/common/types/bit.cpp +51 -0
  151. package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
  152. package/src/duckdb/src/common/types/column/column_data_collection.cpp +68 -2
  153. package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +20 -6
  154. package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
  155. package/src/duckdb/src/common/types/data_chunk.cpp +77 -36
  156. package/src/duckdb/src/common/types/date.cpp +15 -0
  157. package/src/duckdb/src/common/types/hugeint.cpp +40 -0
  158. package/src/duckdb/src/common/types/hyperloglog.cpp +9 -7
  159. package/src/duckdb/src/common/types/interval.cpp +6 -0
  160. package/src/duckdb/src/common/types/list_segment.cpp +56 -198
  161. package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +251 -131
  162. package/src/duckdb/src/common/types/row/row_data_collection_scanner.cpp +35 -5
  163. package/src/duckdb/src/common/types/row/row_layout.cpp +3 -31
  164. package/src/duckdb/src/common/types/row/tuple_data_allocator.cpp +43 -35
  165. package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +74 -41
  166. package/src/duckdb/src/common/types/row/tuple_data_layout.cpp +11 -1
  167. package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +46 -45
  168. package/src/duckdb/src/common/types/row/tuple_data_segment.cpp +21 -16
  169. package/src/duckdb/src/common/types/string_heap.cpp +4 -0
  170. package/src/duckdb/src/common/types/time.cpp +105 -0
  171. package/src/duckdb/src/common/types/timestamp.cpp +7 -0
  172. package/src/duckdb/src/common/types/uuid.cpp +2 -2
  173. package/src/duckdb/src/common/types/validity_mask.cpp +89 -0
  174. package/src/duckdb/src/common/types/value.cpp +103 -189
  175. package/src/duckdb/src/common/types/vector.cpp +79 -216
  176. package/src/duckdb/src/common/types.cpp +50 -745
  177. package/src/duckdb/src/common/vector_operations/vector_hash.cpp +1 -0
  178. package/src/duckdb/src/common/virtual_file_system.cpp +142 -1
  179. package/src/duckdb/src/core_functions/aggregate/algebraic/avg.cpp +0 -1
  180. package/src/duckdb/src/core_functions/aggregate/distributive/string_agg.cpp +6 -7
  181. package/src/duckdb/src/core_functions/aggregate/holistic/approximate_quantile.cpp +11 -8
  182. package/src/duckdb/src/core_functions/aggregate/holistic/mode.cpp +6 -8
  183. package/src/duckdb/src/core_functions/aggregate/holistic/quantile.cpp +137 -92
  184. package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +15 -10
  185. package/src/duckdb/src/core_functions/aggregate/nested/histogram.cpp +1 -0
  186. package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +83 -59
  187. package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
  188. package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
  189. package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
  190. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
  191. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
  192. package/src/duckdb/src/core_functions/function_list.cpp +20 -6
  193. package/src/duckdb/src/core_functions/scalar/date/date_diff.cpp +2 -0
  194. package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +450 -126
  195. package/src/duckdb/src/core_functions/scalar/date/date_sub.cpp +2 -0
  196. package/src/duckdb/src/core_functions/scalar/date/date_trunc.cpp +4 -0
  197. package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
  198. package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
  199. package/src/duckdb/src/core_functions/scalar/date/strftime.cpp +12 -2
  200. package/src/duckdb/src/core_functions/scalar/debug/vector_type.cpp +23 -0
  201. package/src/duckdb/src/core_functions/scalar/enum/enum_functions.cpp +16 -12
  202. package/src/duckdb/src/core_functions/scalar/generic/current_setting.cpp +3 -1
  203. package/src/duckdb/src/core_functions/scalar/generic/hash.cpp +3 -0
  204. package/src/duckdb/src/core_functions/scalar/list/array_slice.cpp +324 -82
  205. package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +25 -18
  206. package/src/duckdb/src/core_functions/scalar/list/list_cosine_similarity.cpp +78 -0
  207. package/src/duckdb/src/core_functions/scalar/list/list_distance.cpp +72 -0
  208. package/src/duckdb/src/core_functions/scalar/list/list_inner_product.cpp +70 -0
  209. package/src/duckdb/src/core_functions/scalar/list/list_lambdas.cpp +18 -6
  210. package/src/duckdb/src/core_functions/scalar/list/list_sort.cpp +10 -1
  211. package/src/duckdb/src/core_functions/scalar/map/map_concat.cpp +0 -2
  212. package/src/duckdb/src/core_functions/scalar/map/map_entries.cpp +2 -2
  213. package/src/duckdb/src/core_functions/scalar/string/repeat.cpp +8 -5
  214. package/src/duckdb/src/core_functions/scalar/string/sha256.cpp +32 -0
  215. package/src/duckdb/src/core_functions/scalar/string/to_base.cpp +66 -0
  216. package/src/duckdb/src/core_functions/scalar/struct/struct_pack.cpp +24 -14
  217. package/src/duckdb/src/core_functions/scalar/union/union_tag.cpp +1 -1
  218. package/src/duckdb/src/execution/aggregate_hashtable.cpp +229 -348
  219. package/src/duckdb/src/execution/column_binding_resolver.cpp +11 -7
  220. package/src/duckdb/src/execution/expression_executor/execute_parameter.cpp +2 -2
  221. package/src/duckdb/src/execution/expression_executor.cpp +1 -1
  222. package/src/duckdb/src/execution/index/art/art.cpp +279 -296
  223. package/src/duckdb/src/execution/index/art/art_key.cpp +0 -11
  224. package/src/duckdb/src/execution/index/art/iterator.cpp +127 -217
  225. package/src/duckdb/src/execution/index/art/leaf.cpp +228 -272
  226. package/src/duckdb/src/execution/index/art/node.cpp +225 -313
  227. package/src/duckdb/src/execution/index/art/node16.cpp +34 -75
  228. package/src/duckdb/src/execution/index/art/node256.cpp +40 -63
  229. package/src/duckdb/src/execution/index/art/node4.cpp +44 -77
  230. package/src/duckdb/src/execution/index/art/node48.cpp +45 -79
  231. package/src/duckdb/src/execution/index/art/prefix.cpp +246 -347
  232. package/src/duckdb/src/execution/index/fixed_size_allocator.cpp +323 -0
  233. package/src/duckdb/src/execution/index/fixed_size_buffer.cpp +285 -0
  234. package/src/duckdb/src/execution/join_hashtable.cpp +80 -69
  235. package/src/duckdb/src/execution/nested_loop_join/nested_loop_join_inner.cpp +20 -27
  236. package/src/duckdb/src/execution/nested_loop_join/nested_loop_join_mark.cpp +21 -9
  237. package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
  238. package/src/duckdb/src/execution/operator/aggregate/physical_hash_aggregate.cpp +257 -324
  239. package/src/duckdb/src/execution/operator/aggregate/physical_perfecthash_aggregate.cpp +6 -4
  240. package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
  241. package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +231 -190
  242. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +368 -1070
  243. package/src/duckdb/src/execution/operator/{persistent → csv_scanner}/base_csv_reader.cpp +162 -175
  244. package/src/duckdb/src/execution/operator/csv_scanner/buffered_csv_reader.cpp +434 -0
  245. package/src/duckdb/src/execution/operator/csv_scanner/csv_buffer.cpp +89 -0
  246. package/src/duckdb/src/execution/operator/csv_scanner/csv_buffer_manager.cpp +90 -0
  247. package/src/duckdb/src/execution/operator/csv_scanner/csv_file_handle.cpp +95 -0
  248. package/src/duckdb/src/execution/operator/csv_scanner/csv_reader_options.cpp +494 -0
  249. package/src/duckdb/src/execution/operator/csv_scanner/csv_state_machine.cpp +35 -0
  250. package/src/duckdb/src/execution/operator/csv_scanner/csv_state_machine_cache.cpp +106 -0
  251. package/src/duckdb/src/execution/operator/{persistent → csv_scanner}/parallel_csv_reader.cpp +69 -51
  252. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp +61 -0
  253. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp +339 -0
  254. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/header_detection.cpp +171 -0
  255. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_detection.cpp +415 -0
  256. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_refinement.cpp +196 -0
  257. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_replacement.cpp +39 -0
  258. package/src/duckdb/src/execution/operator/filter/physical_filter.cpp +1 -1
  259. package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +12 -9
  260. package/src/duckdb/src/execution/operator/helper/physical_explain_analyze.cpp +2 -2
  261. package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +10 -8
  262. package/src/duckdb/src/execution/operator/helper/physical_load.cpp +2 -1
  263. package/src/duckdb/src/execution/operator/helper/physical_materialized_collector.cpp +7 -5
  264. package/src/duckdb/src/execution/operator/helper/physical_reset.cpp +3 -1
  265. package/src/duckdb/src/execution/operator/helper/physical_set.cpp +3 -1
  266. package/src/duckdb/src/execution/operator/helper/physical_vacuum.cpp +7 -5
  267. package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +465 -289
  268. package/src/duckdb/src/execution/operator/join/physical_blockwise_nl_join.cpp +2 -2
  269. package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -2
  270. package/src/duckdb/src/execution/operator/join/physical_delim_join.cpp +13 -6
  271. package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +37 -19
  272. package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +35 -17
  273. package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
  274. package/src/duckdb/src/execution/operator/join/physical_nested_loop_join.cpp +7 -4
  275. package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +31 -10
  276. package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +41 -5
  277. package/src/duckdb/src/execution/operator/order/physical_order.cpp +7 -5
  278. package/src/duckdb/src/execution/operator/order/physical_top_n.cpp +7 -5
  279. package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
  280. package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +14 -10
  281. package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp +69 -47
  282. package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp +9 -7
  283. package/src/duckdb/src/execution/operator/persistent/physical_export.cpp +1 -1
  284. package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +14 -12
  285. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +12 -12
  286. package/src/duckdb/src/execution/operator/persistent/physical_update.cpp +4 -2
  287. package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
  288. package/src/duckdb/src/execution/operator/projection/physical_unnest.cpp +24 -27
  289. package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
  290. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -12
  291. package/src/duckdb/src/execution/operator/schema/physical_attach.cpp +6 -2
  292. package/src/duckdb/src/execution/operator/schema/physical_create_art_index.cpp +198 -0
  293. package/src/duckdb/src/execution/operator/schema/physical_create_type.cpp +2 -6
  294. package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
  295. package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +16 -7
  296. package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +37 -6
  297. package/src/duckdb/src/execution/physical_operator.cpp +20 -16
  298. package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
  299. package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +57 -35
  300. package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +45 -25
  301. package/src/duckdb/src/execution/physical_plan/plan_create_index.cpp +45 -34
  302. package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
  303. package/src/duckdb/src/execution/physical_plan/plan_delim_join.cpp +2 -5
  304. package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
  305. package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
  306. package/src/duckdb/src/execution/physical_plan_generator.cpp +6 -11
  307. package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +658 -363
  308. package/src/duckdb/src/execution/reservoir_sample.cpp +3 -9
  309. package/src/duckdb/src/execution/window_executor.cpp +1285 -0
  310. package/src/duckdb/src/execution/window_segment_tree.cpp +408 -144
  311. package/src/duckdb/src/function/aggregate/distributive/count.cpp +2 -13
  312. package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +6 -12
  313. package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
  314. package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
  315. package/src/duckdb/src/function/cast/cast_function_set.cpp +1 -0
  316. package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
  317. package/src/duckdb/src/function/cast/string_cast.cpp +2 -2
  318. package/src/duckdb/src/function/cast/struct_cast.cpp +8 -0
  319. package/src/duckdb/src/function/cast/time_casts.cpp +19 -6
  320. package/src/duckdb/src/function/cast/union/from_struct.cpp +114 -0
  321. package/src/duckdb/src/function/cast/union_casts.cpp +25 -36
  322. package/src/duckdb/src/function/cast/vector_cast_helpers.cpp +15 -4
  323. package/src/duckdb/src/function/function.cpp +3 -1
  324. package/src/duckdb/src/function/function_binder.cpp +18 -8
  325. package/src/duckdb/src/function/macro_function.cpp +0 -42
  326. package/src/duckdb/src/function/pragma/pragma_functions.cpp +5 -0
  327. package/src/duckdb/src/function/pragma/pragma_queries.cpp +15 -1
  328. package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +214 -0
  329. package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +250 -0
  330. package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
  331. package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
  332. package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
  333. package/src/duckdb/src/function/scalar/operators/add.cpp +9 -0
  334. package/src/duckdb/src/function/scalar/operators/arithmetic.cpp +15 -14
  335. package/src/duckdb/src/function/scalar/strftime_format.cpp +33 -12
  336. package/src/duckdb/src/function/scalar/string/like.cpp +12 -7
  337. package/src/duckdb/src/function/scalar/string/suffix.cpp +1 -1
  338. package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +19 -11
  339. package/src/duckdb/src/function/scalar_function.cpp +3 -19
  340. package/src/duckdb/src/function/scalar_macro_function.cpp +0 -10
  341. package/src/duckdb/src/function/table/arrow/arrow_duck_schema.cpp +57 -0
  342. package/src/duckdb/src/function/table/arrow.cpp +131 -100
  343. package/src/duckdb/src/function/table/arrow_conversion.cpp +95 -74
  344. package/src/duckdb/src/function/table/copy_csv.cpp +133 -127
  345. package/src/duckdb/src/function/table/read_csv.cpp +198 -381
  346. package/src/duckdb/src/function/table/system/duckdb_columns.cpp +3 -1
  347. package/src/duckdb/src/function/table/system/pragma_metadata_info.cpp +83 -0
  348. package/src/duckdb/src/function/table/system/pragma_storage_info.cpp +5 -0
  349. package/src/duckdb/src/function/table/system/test_all_types.cpp +48 -21
  350. package/src/duckdb/src/function/table/system_functions.cpp +2 -0
  351. package/src/duckdb/src/function/table/table_scan.cpp +29 -24
  352. package/src/duckdb/src/function/table/version/pragma_version.cpp +49 -2
  353. package/src/duckdb/src/function/table_macro_function.cpp +0 -10
  354. package/src/duckdb/src/include/duckdb/catalog/catalog.hpp +23 -5
  355. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/duck_index_entry.hpp +2 -0
  356. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/index_catalog_entry.hpp +3 -3
  357. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/macro_catalog_entry.hpp +1 -4
  358. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/schema_catalog_entry.hpp +2 -7
  359. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/sequence_catalog_entry.hpp +1 -8
  360. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp +2 -13
  361. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/type_catalog_entry.hpp +1 -6
  362. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/view_catalog_entry.hpp +2 -5
  363. package/src/duckdb/src/include/duckdb/catalog/catalog_entry.hpp +9 -0
  364. package/src/duckdb/src/include/duckdb/catalog/catalog_set.hpp +0 -6
  365. package/src/duckdb/src/include/duckdb/catalog/duck_catalog.hpp +1 -0
  366. package/src/duckdb/src/include/duckdb/common/adbc/adbc.h +1 -0
  367. package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +4 -1
  368. package/src/duckdb/src/include/duckdb/common/adbc/single_batch_array_stream.hpp +16 -0
  369. package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
  370. package/src/duckdb/src/include/duckdb/common/arrow/appender/append_data.hpp +109 -0
  371. package/src/duckdb/src/include/duckdb/common/arrow/appender/bool_data.hpp +15 -0
  372. package/src/duckdb/src/include/duckdb/common/arrow/appender/enum_data.hpp +69 -0
  373. package/src/duckdb/src/include/duckdb/common/arrow/appender/list.hpp +8 -0
  374. package/src/duckdb/src/include/duckdb/common/arrow/appender/list_data.hpp +18 -0
  375. package/src/duckdb/src/include/duckdb/common/arrow/appender/map_data.hpp +18 -0
  376. package/src/duckdb/src/include/duckdb/common/arrow/appender/scalar_data.hpp +88 -0
  377. package/src/duckdb/src/include/duckdb/common/arrow/appender/struct_data.hpp +18 -0
  378. package/src/duckdb/src/include/duckdb/common/arrow/appender/union_data.hpp +21 -0
  379. package/src/duckdb/src/include/duckdb/common/arrow/appender/varchar_data.hpp +105 -0
  380. package/src/duckdb/src/include/duckdb/common/arrow/arrow_appender.hpp +9 -4
  381. package/src/duckdb/src/include/duckdb/common/arrow/arrow_converter.hpp +3 -5
  382. package/src/duckdb/src/include/duckdb/common/arrow/arrow_wrapper.hpp +5 -3
  383. package/src/duckdb/src/include/duckdb/common/arrow/nanoarrow/nanoarrow.h +462 -0
  384. package/src/duckdb/src/include/duckdb/common/arrow/nanoarrow/nanoarrow.hpp +14 -0
  385. package/src/duckdb/src/include/duckdb/common/arrow/result_arrow_wrapper.hpp +4 -0
  386. package/src/duckdb/src/include/duckdb/common/assert.hpp +1 -1
  387. package/src/duckdb/src/include/duckdb/common/bitpacking.hpp +70 -55
  388. package/src/duckdb/src/include/duckdb/common/box_renderer.hpp +1 -1
  389. package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
  390. package/src/duckdb/src/include/duckdb/common/case_insensitive_map.hpp +1 -0
  391. package/src/duckdb/src/include/duckdb/common/constants.hpp +6 -15
  392. package/src/duckdb/src/include/duckdb/common/dl.hpp +3 -1
  393. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +689 -577
  394. package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
  395. package/src/duckdb/src/include/duckdb/common/enums/date_part_specifier.hpp +18 -2
  396. package/src/duckdb/src/include/duckdb/common/enums/index_type.hpp +4 -3
  397. package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
  398. package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
  399. package/src/duckdb/src/include/duckdb/common/enums/operator_result_type.hpp +5 -1
  400. package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
  401. package/src/duckdb/src/include/duckdb/common/enums/pending_execution_result.hpp +1 -1
  402. package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
  403. package/src/duckdb/src/include/duckdb/common/exception.hpp +15 -1
  404. package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
  405. package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +185 -0
  406. package/src/duckdb/src/include/duckdb/common/file_opener.hpp +9 -0
  407. package/src/duckdb/src/include/duckdb/common/file_system.hpp +10 -8
  408. package/src/duckdb/src/include/duckdb/common/filename_pattern.hpp +1 -1
  409. package/src/duckdb/src/include/duckdb/common/fixed_size_map.hpp +208 -0
  410. package/src/duckdb/src/include/duckdb/common/helper.hpp +8 -3
  411. package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
  412. package/src/duckdb/src/include/duckdb/common/http_state.hpp +61 -28
  413. package/src/duckdb/src/include/duckdb/common/hugeint.hpp +15 -0
  414. package/src/duckdb/src/include/duckdb/common/index_vector.hpp +12 -0
  415. package/src/duckdb/src/include/duckdb/common/limits.hpp +52 -149
  416. package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +14 -7
  417. package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -44
  418. package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
  419. package/src/duckdb/src/include/duckdb/common/numeric_utils.hpp +48 -0
  420. package/src/duckdb/src/include/duckdb/common/opener_file_system.hpp +6 -2
  421. package/src/duckdb/src/include/duckdb/common/operator/add.hpp +5 -2
  422. package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +92 -4
  423. package/src/duckdb/src/include/duckdb/common/operator/comparison_operators.hpp +38 -2
  424. package/src/duckdb/src/include/duckdb/common/operator/multiply.hpp +3 -2
  425. package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
  426. package/src/duckdb/src/include/duckdb/common/operator/string_cast.hpp +1 -1
  427. package/src/duckdb/src/include/duckdb/common/operator/subtract.hpp +3 -2
  428. package/src/duckdb/src/include/duckdb/common/optional_idx.hpp +4 -1
  429. package/src/duckdb/src/include/duckdb/common/perfect_map_set.hpp +2 -1
  430. package/src/duckdb/src/include/duckdb/common/printer.hpp +11 -0
  431. package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
  432. package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
  433. package/src/duckdb/src/include/duckdb/common/row_operations/row_matcher.hpp +63 -0
  434. package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
  435. package/src/duckdb/src/include/duckdb/common/serializer/binary_deserializer.hpp +103 -41
  436. package/src/duckdb/src/include/duckdb/common/serializer/binary_serializer.hpp +59 -42
  437. package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_reader.hpp +4 -10
  438. package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_writer.hpp +3 -3
  439. package/src/duckdb/src/include/duckdb/common/serializer/deserialization_data.hpp +181 -0
  440. package/src/duckdb/src/include/duckdb/common/serializer/{format_deserializer.hpp → deserializer.hpp} +222 -134
  441. package/src/duckdb/src/include/duckdb/common/serializer/encoding_util.hpp +132 -0
  442. package/src/duckdb/src/include/duckdb/common/serializer/memory_stream.hpp +62 -0
  443. package/src/duckdb/src/include/duckdb/common/serializer/read_stream.hpp +38 -0
  444. package/src/duckdb/src/include/duckdb/common/serializer/serialization_traits.hpp +172 -19
  445. package/src/duckdb/src/include/duckdb/common/serializer/serializer.hpp +307 -0
  446. package/src/duckdb/src/include/duckdb/common/serializer/write_stream.hpp +36 -0
  447. package/src/duckdb/src/include/duckdb/common/shared_ptr.hpp +8 -0
  448. package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +48 -17
  449. package/src/duckdb/src/include/duckdb/common/stack_checker.hpp +34 -0
  450. package/src/duckdb/src/include/duckdb/common/string_util.hpp +11 -0
  451. package/src/duckdb/src/include/duckdb/common/type_util.hpp +8 -0
  452. package/src/duckdb/src/include/duckdb/common/typedefs.hpp +8 -0
  453. package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
  454. package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
  455. package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +11 -1
  456. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +12 -1
  457. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +3 -1
  458. package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
  459. package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +4 -6
  460. package/src/duckdb/src/include/duckdb/common/types/date.hpp +9 -5
  461. package/src/duckdb/src/include/duckdb/common/types/datetime.hpp +46 -3
  462. package/src/duckdb/src/include/duckdb/common/types/hyperloglog.hpp +6 -6
  463. package/src/duckdb/src/include/duckdb/common/types/interval.hpp +7 -0
  464. package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +11 -15
  465. package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +46 -11
  466. package/src/duckdb/src/include/duckdb/common/types/row/row_data_collection_scanner.hpp +10 -1
  467. package/src/duckdb/src/include/duckdb/common/types/row/row_layout.hpp +1 -23
  468. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_allocator.hpp +14 -8
  469. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +15 -5
  470. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_layout.hpp +7 -0
  471. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_segment.hpp +13 -8
  472. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +8 -4
  473. package/src/duckdb/src/include/duckdb/common/types/string_heap.hpp +3 -0
  474. package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
  475. package/src/duckdb/src/include/duckdb/common/types/time.hpp +5 -0
  476. package/src/duckdb/src/include/duckdb/common/types/timestamp.hpp +16 -10
  477. package/src/duckdb/src/include/duckdb/common/types/validity_mask.hpp +7 -1
  478. package/src/duckdb/src/include/duckdb/common/types/value.hpp +8 -7
  479. package/src/duckdb/src/include/duckdb/common/types/vector.hpp +13 -11
  480. package/src/duckdb/src/include/duckdb/common/types.hpp +8 -35
  481. package/src/duckdb/src/include/duckdb/common/vector.hpp +2 -2
  482. package/src/duckdb/src/include/duckdb/common/vector_operations/aggregate_executor.hpp +7 -2
  483. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +40 -97
  484. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
  485. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
  486. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
  487. package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +4 -2
  488. package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
  489. package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
  490. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
  491. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
  492. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
  493. package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +7 -5
  494. package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +7 -5
  495. package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +44 -15
  496. package/src/duckdb/src/include/duckdb/core_functions/scalar/debug_functions.hpp +27 -0
  497. package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +10 -8
  498. package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +15 -13
  499. package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +66 -13
  500. package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +6 -4
  501. package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +38 -36
  502. package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +6 -4
  503. package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +6 -4
  504. package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +34 -14
  505. package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +10 -5
  506. package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +5 -3
  507. package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +132 -131
  508. package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
  509. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +27 -16
  510. package/src/duckdb/src/include/duckdb/execution/index/art/art_key.hpp +0 -1
  511. package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +36 -32
  512. package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +47 -56
  513. package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +67 -64
  514. package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +17 -20
  515. package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +17 -25
  516. package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +19 -22
  517. package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +17 -26
  518. package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +60 -55
  519. package/src/duckdb/src/include/duckdb/execution/index/fixed_size_allocator.hpp +120 -0
  520. package/src/duckdb/src/include/duckdb/execution/index/fixed_size_buffer.hpp +109 -0
  521. package/src/duckdb/src/include/duckdb/execution/index/index_pointer.hpp +96 -0
  522. package/src/duckdb/src/include/duckdb/execution/join_hashtable.hpp +14 -8
  523. package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_hash_aggregate.hpp +8 -7
  524. package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_perfecthash_aggregate.hpp +1 -1
  525. package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_ungrouped_aggregate.hpp +3 -3
  526. package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_window.hpp +6 -5
  527. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_batch_collector.hpp +2 -2
  528. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_explain_analyze.hpp +1 -1
  529. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_limit.hpp +1 -1
  530. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_materialized_collector.hpp +1 -1
  531. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_vacuum.hpp +2 -2
  532. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +5 -12
  533. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_blockwise_nl_join.hpp +1 -1
  534. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_delim_join.hpp +2 -2
  535. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_hash_join.hpp +2 -2
  536. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +3 -3
  537. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_nested_loop_join.hpp +2 -2
  538. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +3 -3
  539. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp +12 -1
  540. package/src/duckdb/src/include/duckdb/execution/operator/order/physical_order.hpp +2 -2
  541. package/src/duckdb/src/include/duckdb/execution/operator/order/physical_top_n.hpp +2 -2
  542. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
  543. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_copy_to_file.hpp +2 -2
  544. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_insert.hpp +2 -2
  545. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_copy_to_file.hpp +2 -2
  546. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_fixed_batch_copy.hpp +2 -2
  547. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_insert.hpp +2 -2
  548. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_update.hpp +1 -1
  549. package/src/duckdb/src/include/duckdb/execution/operator/{persistent → scan/csv}/base_csv_reader.hpp +23 -19
  550. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/buffered_csv_reader.hpp +72 -0
  551. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_buffer.hpp +110 -0
  552. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_buffer_manager.hpp +103 -0
  553. package/src/duckdb/src/include/duckdb/execution/operator/{persistent → scan/csv}/csv_file_handle.hpp +8 -15
  554. package/src/duckdb/src/include/duckdb/execution/operator/{persistent → scan/csv}/csv_line_info.hpp +9 -4
  555. package/src/duckdb/src/include/duckdb/execution/operator/{persistent → scan/csv}/csv_reader_options.hpp +79 -33
  556. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_sniffer.hpp +129 -0
  557. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_state_machine.hpp +75 -0
  558. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_state_machine_cache.hpp +51 -0
  559. package/src/duckdb/src/include/duckdb/execution/operator/{persistent → scan/csv}/parallel_csv_reader.hpp +23 -28
  560. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/quote_rules.hpp +21 -0
  561. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
  562. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -5
  563. package/src/duckdb/src/include/duckdb/execution/operator/schema/{physical_create_index.hpp → physical_create_art_index.hpp} +14 -7
  564. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
  565. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
  566. package/src/duckdb/src/include/duckdb/execution/perfect_aggregate_hashtable.hpp +4 -2
  567. package/src/duckdb/src/include/duckdb/execution/physical_operator.hpp +6 -5
  568. package/src/duckdb/src/include/duckdb/execution/physical_operator_states.hpp +11 -0
  569. package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +6 -2
  570. package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +18 -21
  571. package/src/duckdb/src/include/duckdb/execution/window_executor.hpp +313 -0
  572. package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +79 -63
  573. package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +3 -5
  574. package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
  575. package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
  576. package/src/duckdb/src/include/duckdb/function/cast/bound_cast_data.hpp +32 -0
  577. package/src/duckdb/src/include/duckdb/function/compression_function.hpp +36 -4
  578. package/src/duckdb/src/include/duckdb/function/copy_function.hpp +11 -4
  579. package/src/duckdb/src/include/duckdb/function/function_serialization.hpp +58 -61
  580. package/src/duckdb/src/include/duckdb/function/macro_function.hpp +1 -4
  581. package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
  582. package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
  583. package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +14 -10
  584. package/src/duckdb/src/include/duckdb/function/scalar/strftime_format.hpp +14 -5
  585. package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
  586. package/src/duckdb/src/include/duckdb/function/scalar_function.hpp +2 -7
  587. package/src/duckdb/src/include/duckdb/function/scalar_macro_function.hpp +2 -4
  588. package/src/duckdb/src/include/duckdb/function/table/arrow/arrow_duck_schema.hpp +99 -0
  589. package/src/duckdb/src/include/duckdb/function/table/arrow.hpp +11 -36
  590. package/src/duckdb/src/include/duckdb/function/table/read_csv.hpp +22 -22
  591. package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +9 -1
  592. package/src/duckdb/src/include/duckdb/function/table_function.hpp +2 -3
  593. package/src/duckdb/src/include/duckdb/function/table_macro_function.hpp +2 -4
  594. package/src/duckdb/src/include/duckdb/function/udf_function.hpp +2 -1
  595. package/src/duckdb/src/include/duckdb/main/attached_database.hpp +1 -1
  596. package/src/duckdb/src/include/duckdb/main/capi/capi_internal.hpp +4 -3
  597. package/src/duckdb/src/include/duckdb/main/chunk_scan_state/query_result.hpp +29 -0
  598. package/src/duckdb/src/include/duckdb/main/chunk_scan_state.hpp +43 -0
  599. package/src/duckdb/src/include/duckdb/main/client_config.hpp +9 -2
  600. package/src/duckdb/src/include/duckdb/main/client_context.hpp +16 -14
  601. package/src/duckdb/src/include/duckdb/main/client_context_file_opener.hpp +1 -0
  602. package/src/duckdb/src/include/duckdb/main/client_data.hpp +2 -1
  603. package/src/duckdb/src/include/duckdb/main/client_properties.hpp +25 -0
  604. package/src/duckdb/src/include/duckdb/main/config.hpp +19 -1
  605. package/src/duckdb/src/include/duckdb/main/connection.hpp +3 -4
  606. package/src/duckdb/src/include/duckdb/main/extension/generated_extension_loader.hpp +27 -0
  607. package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +258 -144
  608. package/src/duckdb/src/include/duckdb/main/extension_helper.hpp +42 -6
  609. package/src/duckdb/src/include/duckdb/main/extension_util.hpp +18 -0
  610. package/src/duckdb/src/include/duckdb/main/pending_query_result.hpp +5 -0
  611. package/src/duckdb/src/include/duckdb/main/prepared_statement.hpp +73 -5
  612. package/src/duckdb/src/include/duckdb/main/prepared_statement_data.hpp +7 -6
  613. package/src/duckdb/src/include/duckdb/main/query_result.hpp +3 -28
  614. package/src/duckdb/src/include/duckdb/main/relation/aggregate_relation.hpp +4 -1
  615. package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
  616. package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
  617. package/src/duckdb/src/include/duckdb/main/relation/read_csv_relation.hpp +7 -7
  618. package/src/duckdb/src/include/duckdb/main/relation/table_function_relation.hpp +1 -0
  619. package/src/duckdb/src/include/duckdb/main/relation.hpp +17 -4
  620. package/src/duckdb/src/include/duckdb/main/settings.hpp +72 -12
  621. package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
  622. package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
  623. package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
  624. package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +7 -0
  625. package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +38 -64
  626. package/src/duckdb/src/include/duckdb/optimizer/join_order/cost_model.hpp +37 -0
  627. package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
  628. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_node.hpp +14 -29
  629. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +8 -22
  630. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -12
  631. package/src/duckdb/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp +89 -0
  632. package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +19 -30
  633. package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph_manager.hpp +113 -0
  634. package/src/duckdb/src/include/duckdb/optimizer/join_order/relation_manager.hpp +73 -0
  635. package/src/duckdb/src/include/duckdb/optimizer/join_order/relation_statistics_helper.hpp +73 -0
  636. package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
  637. package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
  638. package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
  639. package/src/duckdb/src/include/duckdb/optimizer/rule/empty_needle_removal.hpp +1 -1
  640. package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
  641. package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
  642. package/src/duckdb/src/include/duckdb/parallel/event.hpp +12 -1
  643. package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -3
  644. package/src/duckdb/src/include/duckdb/parallel/pipeline_executor.hpp +3 -2
  645. package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +9 -1
  646. package/src/duckdb/src/include/duckdb/parser/column_definition.hpp +4 -6
  647. package/src/duckdb/src/include/duckdb/parser/column_list.hpp +3 -3
  648. package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +4 -2
  649. package/src/duckdb/src/include/duckdb/parser/constraint.hpp +3 -8
  650. package/src/duckdb/src/include/duckdb/parser/constraints/check_constraint.hpp +2 -2
  651. package/src/duckdb/src/include/duckdb/parser/constraints/foreign_key_constraint.hpp +5 -4
  652. package/src/duckdb/src/include/duckdb/parser/constraints/not_null_constraint.hpp +2 -4
  653. package/src/duckdb/src/include/duckdb/parser/constraints/unique_constraint.hpp +5 -4
  654. package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +5 -5
  655. package/src/duckdb/src/include/duckdb/parser/expression/bound_expression.hpp +1 -4
  656. package/src/duckdb/src/include/duckdb/parser/expression/case_expression.hpp +4 -6
  657. package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +5 -4
  658. package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +5 -4
  659. package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +5 -4
  660. package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +5 -4
  661. package/src/duckdb/src/include/duckdb/parser/expression/conjunction_expression.hpp +2 -4
  662. package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +4 -4
  663. package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +2 -3
  664. package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +5 -4
  665. package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +5 -4
  666. package/src/duckdb/src/include/duckdb/parser/expression/operator_expression.hpp +22 -7
  667. package/src/duckdb/src/include/duckdb/parser/expression/parameter_expression.hpp +19 -5
  668. package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +5 -4
  669. package/src/duckdb/src/include/duckdb/parser/expression/star_expression.hpp +2 -4
  670. package/src/duckdb/src/include/duckdb/parser/expression/subquery_expression.hpp +2 -4
  671. package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +5 -4
  672. package/src/duckdb/src/include/duckdb/parser/group_by_node.hpp +11 -0
  673. package/src/duckdb/src/include/duckdb/parser/parsed_data/alter_info.hpp +12 -4
  674. package/src/duckdb/src/include/duckdb/parser/parsed_data/alter_scalar_function_info.hpp +0 -2
  675. package/src/duckdb/src/include/duckdb/parser/parsed_data/alter_table_function_info.hpp +0 -2
  676. package/src/duckdb/src/include/duckdb/parser/parsed_data/alter_table_info.hpp +66 -29
  677. package/src/duckdb/src/include/duckdb/parser/parsed_data/attach_info.hpp +6 -2
  678. package/src/duckdb/src/include/duckdb/parser/parsed_data/copy_info.hpp +8 -1
  679. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_collation_info.hpp +0 -3
  680. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_copy_function_info.hpp +0 -3
  681. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_function_info.hpp +0 -5
  682. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_index_info.hpp +7 -4
  683. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_info.hpp +7 -12
  684. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_macro_info.hpp +2 -5
  685. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_schema_info.hpp +2 -9
  686. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_sequence_info.hpp +2 -4
  687. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_table_info.hpp +3 -5
  688. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_type_info.hpp +2 -4
  689. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_view_info.hpp +2 -4
  690. package/src/duckdb/src/include/duckdb/parser/parsed_data/detach_info.hpp +6 -1
  691. package/src/duckdb/src/include/duckdb/parser/parsed_data/drop_info.hpp +6 -2
  692. package/src/duckdb/src/include/duckdb/parser/parsed_data/exported_table_data.hpp +7 -0
  693. package/src/duckdb/src/include/duckdb/parser/parsed_data/load_info.hpp +14 -19
  694. package/src/duckdb/src/include/duckdb/parser/parsed_data/parse_info.hpp +22 -0
  695. package/src/duckdb/src/include/duckdb/parser/parsed_data/pragma_info.hpp +10 -0
  696. package/src/duckdb/src/include/duckdb/parser/parsed_data/sample_options.hpp +2 -4
  697. package/src/duckdb/src/include/duckdb/parser/parsed_data/show_select_info.hpp +7 -0
  698. package/src/duckdb/src/include/duckdb/parser/parsed_data/transaction_info.hpp +8 -1
  699. package/src/duckdb/src/include/duckdb/parser/parsed_data/vacuum_info.hpp +9 -1
  700. package/src/duckdb/src/include/duckdb/parser/parsed_expression.hpp +3 -16
  701. package/src/duckdb/src/include/duckdb/parser/parser.hpp +4 -0
  702. package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +52 -0
  703. package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
  704. package/src/duckdb/src/include/duckdb/parser/query_node/recursive_cte_node.hpp +2 -4
  705. package/src/duckdb/src/include/duckdb/parser/query_node/select_node.hpp +2 -4
  706. package/src/duckdb/src/include/duckdb/parser/query_node/set_operation_node.hpp +2 -4
  707. package/src/duckdb/src/include/duckdb/parser/query_node.hpp +9 -15
  708. package/src/duckdb/src/include/duckdb/parser/result_modifier.hpp +14 -32
  709. package/src/duckdb/src/include/duckdb/parser/statement/execute_statement.hpp +1 -1
  710. package/src/duckdb/src/include/duckdb/parser/statement/select_statement.hpp +4 -9
  711. package/src/duckdb/src/include/duckdb/parser/tableref/basetableref.hpp +2 -6
  712. package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +2 -5
  713. package/src/duckdb/src/include/duckdb/parser/tableref/expressionlistref.hpp +2 -6
  714. package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +3 -7
  715. package/src/duckdb/src/include/duckdb/parser/tableref/pivotref.hpp +6 -14
  716. package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +5 -6
  717. package/src/duckdb/src/include/duckdb/parser/tableref/table_function_ref.hpp +2 -6
  718. package/src/duckdb/src/include/duckdb/parser/tableref.hpp +2 -10
  719. package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
  720. package/src/duckdb/src/include/duckdb/parser/transformer.hpp +24 -26
  721. package/src/duckdb/src/include/duckdb/planner/binder.hpp +16 -5
  722. package/src/duckdb/src/include/duckdb/planner/bound_constraint.hpp +0 -9
  723. package/src/duckdb/src/include/duckdb/planner/bound_parameter_map.hpp +30 -2
  724. package/src/duckdb/src/include/duckdb/planner/bound_result_modifier.hpp +2 -2
  725. package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
  726. package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +9 -0
  727. package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
  728. package/src/duckdb/src/include/duckdb/planner/expression/bound_aggregate_expression.hpp +3 -2
  729. package/src/duckdb/src/include/duckdb/planner/expression/bound_between_expression.hpp +6 -2
  730. package/src/duckdb/src/include/duckdb/planner/expression/bound_case_expression.hpp +3 -3
  731. package/src/duckdb/src/include/duckdb/planner/expression/bound_cast_expression.hpp +5 -2
  732. package/src/duckdb/src/include/duckdb/planner/expression/bound_columnref_expression.hpp +2 -5
  733. package/src/duckdb/src/include/duckdb/planner/expression/bound_comparison_expression.hpp +3 -2
  734. package/src/duckdb/src/include/duckdb/planner/expression/bound_conjunction_expression.hpp +2 -2
  735. package/src/duckdb/src/include/duckdb/planner/expression/bound_constant_expression.hpp +2 -2
  736. package/src/duckdb/src/include/duckdb/planner/expression/bound_default_expression.hpp +2 -2
  737. package/src/duckdb/src/include/duckdb/planner/expression/bound_function_expression.hpp +3 -2
  738. package/src/duckdb/src/include/duckdb/planner/expression/bound_lambda_expression.hpp +2 -2
  739. package/src/duckdb/src/include/duckdb/planner/expression/bound_lambdaref_expression.hpp +2 -5
  740. package/src/duckdb/src/include/duckdb/planner/expression/bound_operator_expression.hpp +2 -2
  741. package/src/duckdb/src/include/duckdb/planner/expression/bound_parameter_data.hpp +12 -29
  742. package/src/duckdb/src/include/duckdb/planner/expression/bound_parameter_expression.hpp +9 -5
  743. package/src/duckdb/src/include/duckdb/planner/expression/bound_reference_expression.hpp +2 -2
  744. package/src/duckdb/src/include/duckdb/planner/expression/bound_subquery_expression.hpp +0 -3
  745. package/src/duckdb/src/include/duckdb/planner/expression/bound_unnest_expression.hpp +2 -2
  746. package/src/duckdb/src/include/duckdb/planner/expression/bound_window_expression.hpp +2 -2
  747. package/src/duckdb/src/include/duckdb/planner/expression/list.hpp +1 -0
  748. package/src/duckdb/src/include/duckdb/planner/expression.hpp +2 -11
  749. package/src/duckdb/src/include/duckdb/planner/expression_binder/base_select_binder.hpp +1 -0
  750. package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
  751. package/src/duckdb/src/include/duckdb/planner/expression_binder.hpp +13 -1
  752. package/src/duckdb/src/include/duckdb/planner/extension_callback.hpp +26 -0
  753. package/src/duckdb/src/include/duckdb/planner/filter/conjunction_filter.hpp +4 -4
  754. package/src/duckdb/src/include/duckdb/planner/filter/constant_filter.hpp +2 -2
  755. package/src/duckdb/src/include/duckdb/planner/filter/null_filter.hpp +4 -4
  756. package/src/duckdb/src/include/duckdb/planner/joinside.hpp +1 -3
  757. package/src/duckdb/src/include/duckdb/planner/logical_operator.hpp +2 -16
  758. package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -2
  759. package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +3 -3
  760. package/src/duckdb/src/include/duckdb/planner/operator/logical_aggregate.hpp +3 -2
  761. package/src/duckdb/src/include/duckdb/planner/operator/logical_any_join.hpp +3 -2
  762. package/src/duckdb/src/include/duckdb/planner/operator/logical_column_data_get.hpp +3 -2
  763. package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +12 -11
  764. package/src/duckdb/src/include/duckdb/planner/operator/logical_copy_to_file.hpp +2 -2
  765. package/src/duckdb/src/include/duckdb/planner/operator/logical_create.hpp +8 -8
  766. package/src/duckdb/src/include/duckdb/planner/operator/logical_create_index.hpp +11 -25
  767. package/src/duckdb/src/include/duckdb/planner/operator/logical_create_table.hpp +8 -8
  768. package/src/duckdb/src/include/duckdb/planner/operator/logical_cross_product.hpp +2 -2
  769. package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +9 -4
  770. package/src/duckdb/src/include/duckdb/planner/operator/logical_delete.hpp +6 -2
  771. package/src/duckdb/src/include/duckdb/planner/operator/logical_delim_get.hpp +3 -2
  772. package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +42 -0
  773. package/src/duckdb/src/include/duckdb/planner/operator/logical_distinct.hpp +6 -12
  774. package/src/duckdb/src/include/duckdb/planner/operator/logical_dummy_scan.hpp +2 -2
  775. package/src/duckdb/src/include/duckdb/planner/operator/logical_empty_result.hpp +2 -2
  776. package/src/duckdb/src/include/duckdb/planner/operator/logical_execute.hpp +0 -2
  777. package/src/duckdb/src/include/duckdb/planner/operator/logical_explain.hpp +3 -2
  778. package/src/duckdb/src/include/duckdb/planner/operator/logical_export.hpp +0 -3
  779. package/src/duckdb/src/include/duckdb/planner/operator/logical_expression_get.hpp +3 -2
  780. package/src/duckdb/src/include/duckdb/planner/operator/logical_extension_operator.hpp +7 -1
  781. package/src/duckdb/src/include/duckdb/planner/operator/logical_filter.hpp +3 -2
  782. package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +11 -3
  783. package/src/duckdb/src/include/duckdb/planner/operator/logical_insert.hpp +5 -2
  784. package/src/duckdb/src/include/duckdb/planner/operator/logical_join.hpp +1 -3
  785. package/src/duckdb/src/include/duckdb/planner/operator/logical_limit.hpp +2 -2
  786. package/src/duckdb/src/include/duckdb/planner/operator/logical_limit_percent.hpp +2 -2
  787. package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +50 -0
  788. package/src/duckdb/src/include/duckdb/planner/operator/logical_order.hpp +6 -37
  789. package/src/duckdb/src/include/duckdb/planner/operator/logical_pivot.hpp +6 -2
  790. package/src/duckdb/src/include/duckdb/planner/operator/logical_positional_join.hpp +2 -2
  791. package/src/duckdb/src/include/duckdb/planner/operator/logical_pragma.hpp +0 -2
  792. package/src/duckdb/src/include/duckdb/planner/operator/logical_prepare.hpp +0 -2
  793. package/src/duckdb/src/include/duckdb/planner/operator/logical_projection.hpp +3 -2
  794. package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +10 -9
  795. package/src/duckdb/src/include/duckdb/planner/operator/logical_reset.hpp +3 -2
  796. package/src/duckdb/src/include/duckdb/planner/operator/logical_sample.hpp +5 -2
  797. package/src/duckdb/src/include/duckdb/planner/operator/logical_set.hpp +3 -2
  798. package/src/duckdb/src/include/duckdb/planner/operator/logical_set_operation.hpp +3 -2
  799. package/src/duckdb/src/include/duckdb/planner/operator/logical_show.hpp +2 -2
  800. package/src/duckdb/src/include/duckdb/planner/operator/logical_simple.hpp +2 -2
  801. package/src/duckdb/src/include/duckdb/planner/operator/logical_top_n.hpp +4 -2
  802. package/src/duckdb/src/include/duckdb/planner/operator/logical_unnest.hpp +2 -2
  803. package/src/duckdb/src/include/duckdb/planner/operator/logical_update.hpp +6 -2
  804. package/src/duckdb/src/include/duckdb/planner/operator/logical_window.hpp +3 -2
  805. package/src/duckdb/src/include/duckdb/planner/operator_extension.hpp +1 -2
  806. package/src/duckdb/src/include/duckdb/planner/parsed_data/bound_create_table_info.hpp +0 -5
  807. package/src/duckdb/src/include/duckdb/planner/planner.hpp +5 -4
  808. package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
  809. package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
  810. package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
  811. package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
  812. package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
  813. package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
  814. package/src/duckdb/src/include/duckdb/planner/table_filter.hpp +4 -7
  815. package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
  816. package/src/duckdb/src/include/duckdb/planner/tableref/bound_pivotref.hpp +3 -0
  817. package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +2 -1
  818. package/src/duckdb/src/include/duckdb/storage/block.hpp +36 -4
  819. package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +11 -11
  820. package/src/duckdb/src/include/duckdb/storage/buffer/block_handle.hpp +3 -0
  821. package/src/duckdb/src/include/duckdb/storage/checkpoint/row_group_writer.hpp +8 -7
  822. package/src/duckdb/src/include/duckdb/storage/checkpoint/string_checkpoint_state.hpp +27 -4
  823. package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_reader.hpp +2 -2
  824. package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp +7 -7
  825. package/src/duckdb/src/include/duckdb/storage/checkpoint/write_overflow_strings_to_disk.hpp +4 -2
  826. package/src/duckdb/src/include/duckdb/storage/checkpoint_manager.hpp +29 -24
  827. package/src/duckdb/src/include/duckdb/storage/compression/bitpacking.hpp +1 -8
  828. package/src/duckdb/src/include/duckdb/storage/data_pointer.hpp +31 -4
  829. package/src/duckdb/src/include/duckdb/storage/data_table.hpp +4 -4
  830. package/src/duckdb/src/include/duckdb/storage/database_size.hpp +6 -0
  831. package/src/duckdb/src/include/duckdb/storage/in_memory_block_manager.hpp +2 -2
  832. package/src/duckdb/src/include/duckdb/storage/index.hpp +12 -10
  833. package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp +91 -0
  834. package/src/duckdb/src/include/duckdb/storage/metadata/metadata_reader.hpp +57 -0
  835. package/src/duckdb/src/include/duckdb/storage/metadata/metadata_writer.hpp +52 -0
  836. package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
  837. package/src/duckdb/src/include/duckdb/storage/partial_block_manager.hpp +37 -21
  838. package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +8 -5
  839. package/src/duckdb/src/include/duckdb/storage/statistics/base_statistics.hpp +3 -8
  840. package/src/duckdb/src/include/duckdb/storage/statistics/column_statistics.hpp +3 -1
  841. package/src/duckdb/src/include/duckdb/storage/statistics/distinct_statistics.hpp +4 -7
  842. package/src/duckdb/src/include/duckdb/storage/statistics/list_stats.hpp +2 -4
  843. package/src/duckdb/src/include/duckdb/storage/statistics/node_statistics.hpp +0 -26
  844. package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats.hpp +2 -4
  845. package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +6 -4
  846. package/src/duckdb/src/include/duckdb/storage/statistics/struct_stats.hpp +2 -4
  847. package/src/duckdb/src/include/duckdb/storage/storage_info.hpp +25 -8
  848. package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp +4 -2
  849. package/src/duckdb/src/include/duckdb/storage/string_uncompressed.hpp +6 -1
  850. package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +28 -16
  851. package/src/duckdb/src/include/duckdb/storage/table/column_checkpoint_state.hpp +5 -20
  852. package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp +3 -3
  853. package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp +7 -3
  854. package/src/duckdb/src/include/duckdb/storage/table/list_column_data.hpp +1 -1
  855. package/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp +2 -2
  856. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +21 -21
  857. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +4 -4
  858. package/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp +2 -2
  859. package/src/duckdb/src/include/duckdb/storage/table/row_version_manager.hpp +59 -0
  860. package/src/duckdb/src/include/duckdb/storage/table/standard_column_data.hpp +2 -2
  861. package/src/duckdb/src/include/duckdb/storage/table/table_index_list.hpp +1 -1
  862. package/src/duckdb/src/include/duckdb/storage/table/table_statistics.hpp +4 -2
  863. package/src/duckdb/src/include/duckdb/storage/table/update_segment.hpp +1 -1
  864. package/src/duckdb/src/include/duckdb/storage/table_io_manager.hpp +3 -0
  865. package/src/duckdb/src/include/duckdb/storage/table_storage_info.hpp +1 -0
  866. package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp +29 -32
  867. package/src/duckdb/src/include/duckdb/transaction/commit_state.hpp +1 -6
  868. package/src/duckdb/src/include/duckdb/transaction/delete_info.hpp +3 -2
  869. package/src/duckdb/src/include/duckdb/transaction/duck_transaction.hpp +4 -2
  870. package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +3 -4
  871. package/src/duckdb/src/include/duckdb/transaction/undo_buffer.hpp +0 -1
  872. package/src/duckdb/src/include/duckdb/verification/prepared_statement_verifier.hpp +1 -1
  873. package/src/duckdb/src/include/duckdb/verification/statement_verifier.hpp +0 -1
  874. package/src/duckdb/src/include/duckdb.h +101 -4
  875. package/src/duckdb/src/main/appender.cpp +3 -1
  876. package/src/duckdb/src/main/attached_database.cpp +2 -2
  877. package/src/duckdb/src/main/capi/arrow-c.cpp +196 -8
  878. package/src/duckdb/src/main/capi/duckdb-c.cpp +16 -0
  879. package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
  880. package/src/duckdb/src/main/capi/logical_types-c.cpp +22 -0
  881. package/src/duckdb/src/main/capi/pending-c.cpp +23 -0
  882. package/src/duckdb/src/main/capi/prepared-c.cpp +114 -30
  883. package/src/duckdb/src/main/capi/result-c.cpp +3 -1
  884. package/src/duckdb/src/main/chunk_scan_state/query_result.cpp +53 -0
  885. package/src/duckdb/src/main/chunk_scan_state.cpp +48 -0
  886. package/src/duckdb/src/main/client_context.cpp +42 -25
  887. package/src/duckdb/src/main/client_context_file_opener.cpp +17 -0
  888. package/src/duckdb/src/main/client_verify.cpp +17 -1
  889. package/src/duckdb/src/main/config.cpp +10 -4
  890. package/src/duckdb/src/main/connection.cpp +5 -7
  891. package/src/duckdb/src/main/database.cpp +9 -12
  892. package/src/duckdb/src/main/db_instance_cache.cpp +14 -6
  893. package/src/duckdb/src/main/extension/extension_helper.cpp +195 -87
  894. package/src/duckdb/src/main/extension/extension_install.cpp +79 -15
  895. package/src/duckdb/src/main/extension/extension_load.cpp +63 -14
  896. package/src/duckdb/src/main/extension/extension_util.cpp +72 -0
  897. package/src/duckdb/src/main/pending_query_result.cpp +9 -1
  898. package/src/duckdb/src/main/prepared_statement.cpp +38 -11
  899. package/src/duckdb/src/main/prepared_statement_data.cpp +23 -18
  900. package/src/duckdb/src/main/query_result.cpp +16 -31
  901. package/src/duckdb/src/main/relation/aggregate_relation.cpp +20 -10
  902. package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
  903. package/src/duckdb/src/main/relation/join_relation.cpp +6 -6
  904. package/src/duckdb/src/main/relation/read_csv_relation.cpp +38 -13
  905. package/src/duckdb/src/main/relation/table_function_relation.cpp +8 -2
  906. package/src/duckdb/src/main/relation.cpp +35 -11
  907. package/src/duckdb/src/main/settings/settings.cpp +134 -43
  908. package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
  909. package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +11 -7
  910. package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
  911. package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
  912. package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
  913. package/src/duckdb/src/optimizer/compressed_materialization.cpp +477 -0
  914. package/src/duckdb/src/optimizer/deliminator.cpp +180 -323
  915. package/src/duckdb/src/optimizer/filter_pushdown.cpp +23 -6
  916. package/src/duckdb/src/optimizer/join_order/cardinality_estimator.cpp +79 -325
  917. package/src/duckdb/src/optimizer/join_order/cost_model.cpp +19 -0
  918. package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
  919. package/src/duckdb/src/optimizer/join_order/join_node.cpp +5 -37
  920. package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +48 -1047
  921. package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
  922. package/src/duckdb/src/optimizer/join_order/plan_enumerator.cpp +552 -0
  923. package/src/duckdb/src/optimizer/join_order/query_graph.cpp +52 -41
  924. package/src/duckdb/src/optimizer/join_order/query_graph_manager.cpp +409 -0
  925. package/src/duckdb/src/optimizer/join_order/relation_manager.cpp +356 -0
  926. package/src/duckdb/src/optimizer/join_order/relation_statistics_helper.cpp +351 -0
  927. package/src/duckdb/src/optimizer/optimizer.cpp +49 -14
  928. package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
  929. package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
  930. package/src/duckdb/src/optimizer/pushdown/pushdown_projection.cpp +34 -7
  931. package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
  932. package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
  933. package/src/duckdb/src/optimizer/rule/date_part_simplification.cpp +0 -3
  934. package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
  935. package/src/duckdb/src/optimizer/statistics/expression/propagate_cast.cpp +14 -0
  936. package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
  937. package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
  938. package/src/duckdb/src/optimizer/statistics/operator/propagate_projection.cpp +0 -1
  939. package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
  940. package/src/duckdb/src/optimizer/topn_optimizer.cpp +27 -10
  941. package/src/duckdb/src/optimizer/unnest_rewriter.cpp +3 -5
  942. package/src/duckdb/src/parallel/executor.cpp +25 -1
  943. package/src/duckdb/src/parallel/pipeline.cpp +0 -17
  944. package/src/duckdb/src/parallel/pipeline_executor.cpp +27 -7
  945. package/src/duckdb/src/parallel/pipeline_finish_event.cpp +55 -1
  946. package/src/duckdb/src/parallel/task_scheduler.cpp +18 -2
  947. package/src/duckdb/src/parser/column_definition.cpp +18 -55
  948. package/src/duckdb/src/parser/column_list.cpp +8 -13
  949. package/src/duckdb/src/parser/constraint.cpp +0 -33
  950. package/src/duckdb/src/parser/constraints/check_constraint.cpp +0 -11
  951. package/src/duckdb/src/parser/constraints/foreign_key_constraint.cpp +3 -27
  952. package/src/duckdb/src/parser/constraints/not_null_constraint.cpp +0 -11
  953. package/src/duckdb/src/parser/constraints/unique_constraint.cpp +3 -24
  954. package/src/duckdb/src/parser/expression/between_expression.cpp +5 -31
  955. package/src/duckdb/src/parser/expression/case_expression.cpp +2 -54
  956. package/src/duckdb/src/parser/expression/cast_expression.cpp +5 -30
  957. package/src/duckdb/src/parser/expression/collate_expression.cpp +5 -27
  958. package/src/duckdb/src/parser/expression/columnref_expression.cpp +5 -25
  959. package/src/duckdb/src/parser/expression/comparison_expression.cpp +5 -27
  960. package/src/duckdb/src/parser/expression/conjunction_expression.cpp +2 -25
  961. package/src/duckdb/src/parser/expression/constant_expression.cpp +5 -23
  962. package/src/duckdb/src/parser/expression/default_expression.cpp +2 -13
  963. package/src/duckdb/src/parser/expression/function_expression.cpp +5 -64
  964. package/src/duckdb/src/parser/expression/lambda_expression.cpp +6 -28
  965. package/src/duckdb/src/parser/expression/operator_expression.cpp +2 -25
  966. package/src/duckdb/src/parser/expression/parameter_expression.cpp +7 -30
  967. package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +6 -23
  968. package/src/duckdb/src/parser/expression/star_expression.cpp +2 -62
  969. package/src/duckdb/src/parser/expression/subquery_expression.cpp +2 -46
  970. package/src/duckdb/src/parser/expression/window_expression.cpp +5 -92
  971. package/src/duckdb/src/parser/keyword_helper.cpp +1 -1
  972. package/src/duckdb/src/parser/parsed_data/alter_info.cpp +4 -36
  973. package/src/duckdb/src/parser/parsed_data/alter_scalar_function_info.cpp +0 -18
  974. package/src/duckdb/src/parser/parsed_data/alter_table_function_info.cpp +0 -13
  975. package/src/duckdb/src/parser/parsed_data/alter_table_info.cpp +38 -188
  976. package/src/duckdb/src/parser/parsed_data/attach_info.cpp +0 -29
  977. package/src/duckdb/src/parser/parsed_data/create_collation_info.cpp +0 -4
  978. package/src/duckdb/src/parser/parsed_data/create_copy_function_info.cpp +0 -4
  979. package/src/duckdb/src/parser/parsed_data/create_index_info.cpp +2 -34
  980. package/src/duckdb/src/parser/parsed_data/create_info.cpp +0 -47
  981. package/src/duckdb/src/parser/parsed_data/create_macro_info.cpp +1 -28
  982. package/src/duckdb/src/parser/parsed_data/create_sequence_info.cpp +0 -29
  983. package/src/duckdb/src/parser/parsed_data/create_table_info.cpp +0 -23
  984. package/src/duckdb/src/parser/parsed_data/create_type_info.cpp +0 -22
  985. package/src/duckdb/src/parser/parsed_data/create_view_info.cpp +0 -23
  986. package/src/duckdb/src/parser/parsed_data/detach_info.cpp +1 -20
  987. package/src/duckdb/src/parser/parsed_data/drop_info.cpp +1 -27
  988. package/src/duckdb/src/parser/parsed_data/sample_options.cpp +2 -43
  989. package/src/duckdb/src/parser/parsed_data/transaction_info.cpp +2 -15
  990. package/src/duckdb/src/parser/parsed_data/vacuum_info.cpp +1 -21
  991. package/src/duckdb/src/parser/parsed_expression.cpp +2 -149
  992. package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
  993. package/src/duckdb/src/parser/parser.cpp +62 -36
  994. package/src/duckdb/src/parser/query_node/cte_node.cpp +41 -0
  995. package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +2 -40
  996. package/src/duckdb/src/parser/query_node/select_node.cpp +2 -76
  997. package/src/duckdb/src/parser/query_node/set_operation_node.cpp +2 -32
  998. package/src/duckdb/src/parser/query_node.cpp +12 -104
  999. package/src/duckdb/src/parser/result_modifier.cpp +2 -183
  1000. package/src/duckdb/src/parser/statement/execute_statement.cpp +2 -2
  1001. package/src/duckdb/src/parser/statement/select_statement.cpp +2 -23
  1002. package/src/duckdb/src/parser/tableref/basetableref.cpp +2 -40
  1003. package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -13
  1004. package/src/duckdb/src/parser/tableref/expressionlistref.cpp +2 -43
  1005. package/src/duckdb/src/parser/tableref/joinref.cpp +5 -46
  1006. package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -116
  1007. package/src/duckdb/src/parser/tableref/subqueryref.cpp +5 -28
  1008. package/src/duckdb/src/parser/tableref/table_function.cpp +2 -32
  1009. package/src/duckdb/src/parser/tableref.cpp +2 -96
  1010. package/src/duckdb/src/parser/transform/constraint/transform_constraint.cpp +55 -38
  1011. package/src/duckdb/src/parser/transform/expression/transform_array_access.cpp +13 -4
  1012. package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
  1013. package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
  1014. package/src/duckdb/src/parser/transform/expression/transform_function.cpp +3 -0
  1015. package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
  1016. package/src/duckdb/src/parser/transform/expression/transform_param_ref.cpp +45 -26
  1017. package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
  1018. package/src/duckdb/src/parser/transform/helpers/transform_typename.cpp +16 -1
  1019. package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
  1020. package/src/duckdb/src/parser/transform/statement/transform_create_index.cpp +32 -17
  1021. package/src/duckdb/src/parser/transform/statement/transform_create_type.cpp +1 -1
  1022. package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
  1023. package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
  1024. package/src/duckdb/src/parser/transform/statement/transform_load.cpp +1 -0
  1025. package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
  1026. package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
  1027. package/src/duckdb/src/parser/transform/statement/transform_prepare.cpp +28 -6
  1028. package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
  1029. package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
  1030. package/src/duckdb/src/parser/transformer.cpp +44 -25
  1031. package/src/duckdb/src/planner/binder/expression/bind_aggregate_expression.cpp +1 -4
  1032. package/src/duckdb/src/planner/binder/expression/bind_comparison_expression.cpp +1 -0
  1033. package/src/duckdb/src/planner/binder/expression/bind_function_expression.cpp +14 -5
  1034. package/src/duckdb/src/planner/binder/expression/bind_macro_expression.cpp +5 -3
  1035. package/src/duckdb/src/planner/binder/expression/bind_parameter_expression.cpp +12 -22
  1036. package/src/duckdb/src/planner/binder/expression/bind_subquery_expression.cpp +1 -4
  1037. package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
  1038. package/src/duckdb/src/planner/binder/query_node/bind_select_node.cpp +28 -6
  1039. package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
  1040. package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
  1041. package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
  1042. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +36 -33
  1043. package/src/duckdb/src/planner/binder/statement/bind_copy.cpp +1 -1
  1044. package/src/duckdb/src/planner/binder/statement/bind_create.cpp +14 -52
  1045. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +0 -23
  1046. package/src/duckdb/src/planner/binder/statement/bind_drop.cpp +3 -0
  1047. package/src/duckdb/src/planner/binder/statement/bind_execute.cpp +13 -7
  1048. package/src/duckdb/src/planner/binder/statement/bind_export.cpp +168 -30
  1049. package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +93 -28
  1050. package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
  1051. package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -50
  1052. package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
  1053. package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +79 -35
  1054. package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
  1055. package/src/duckdb/src/planner/binder.cpp +49 -32
  1056. package/src/duckdb/src/planner/bound_parameter_map.cpp +67 -0
  1057. package/src/duckdb/src/planner/bound_result_modifier.cpp +0 -33
  1058. package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +20 -21
  1059. package/src/duckdb/src/planner/expression/bound_between_expression.cpp +4 -19
  1060. package/src/duckdb/src/planner/expression/bound_case_expression.cpp +0 -36
  1061. package/src/duckdb/src/planner/expression/bound_cast_expression.cpp +13 -23
  1062. package/src/duckdb/src/planner/expression/bound_columnref_expression.cpp +0 -20
  1063. package/src/duckdb/src/planner/expression/bound_comparison_expression.cpp +0 -13
  1064. package/src/duckdb/src/planner/expression/bound_conjunction_expression.cpp +0 -13
  1065. package/src/duckdb/src/planner/expression/bound_constant_expression.cpp +0 -11
  1066. package/src/duckdb/src/planner/expression/bound_expression.cpp +1 -5
  1067. package/src/duckdb/src/planner/expression/bound_function_expression.cpp +18 -16
  1068. package/src/duckdb/src/planner/expression/bound_lambda_expression.cpp +0 -4
  1069. package/src/duckdb/src/planner/expression/bound_lambdaref_expression.cpp +0 -23
  1070. package/src/duckdb/src/planner/expression/bound_operator_expression.cpp +0 -16
  1071. package/src/duckdb/src/planner/expression/bound_parameter_expression.cpp +23 -33
  1072. package/src/duckdb/src/planner/expression/bound_reference_expression.cpp +0 -16
  1073. package/src/duckdb/src/planner/expression/bound_subquery_expression.cpp +0 -4
  1074. package/src/duckdb/src/planner/expression/bound_unnest_expression.cpp +0 -15
  1075. package/src/duckdb/src/planner/expression/bound_window_expression.cpp +41 -50
  1076. package/src/duckdb/src/planner/expression.cpp +0 -74
  1077. package/src/duckdb/src/planner/expression_binder/base_select_binder.cpp +14 -6
  1078. package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
  1079. package/src/duckdb/src/planner/expression_binder/order_binder.cpp +5 -4
  1080. package/src/duckdb/src/planner/expression_binder.cpp +23 -0
  1081. package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
  1082. package/src/duckdb/src/planner/filter/conjunction_filter.cpp +0 -21
  1083. package/src/duckdb/src/planner/filter/constant_filter.cpp +0 -12
  1084. package/src/duckdb/src/planner/filter/null_filter.cpp +0 -14
  1085. package/src/duckdb/src/planner/joinside.cpp +0 -24
  1086. package/src/duckdb/src/planner/logical_operator.cpp +18 -206
  1087. package/src/duckdb/src/planner/logical_operator_visitor.cpp +5 -6
  1088. package/src/duckdb/src/planner/operator/logical_aggregate.cpp +0 -46
  1089. package/src/duckdb/src/planner/operator/logical_any_join.cpp +0 -14
  1090. package/src/duckdb/src/planner/operator/logical_column_data_get.cpp +0 -23
  1091. package/src/duckdb/src/planner/operator/logical_comparison_join.cpp +0 -21
  1092. package/src/duckdb/src/planner/operator/logical_copy_to_file.cpp +4 -50
  1093. package/src/duckdb/src/planner/operator/logical_create.cpp +10 -8
  1094. package/src/duckdb/src/planner/operator/logical_create_index.cpp +30 -26
  1095. package/src/duckdb/src/planner/operator/logical_create_table.cpp +12 -6
  1096. package/src/duckdb/src/planner/operator/logical_cross_product.cpp +0 -9
  1097. package/src/duckdb/src/planner/operator/logical_cteref.cpp +0 -16
  1098. package/src/duckdb/src/planner/operator/logical_delete.cpp +4 -18
  1099. package/src/duckdb/src/planner/operator/logical_delim_get.cpp +0 -12
  1100. package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +22 -0
  1101. package/src/duckdb/src/planner/operator/logical_distinct.cpp +10 -13
  1102. package/src/duckdb/src/planner/operator/logical_dummy_scan.cpp +0 -10
  1103. package/src/duckdb/src/planner/operator/logical_empty_result.cpp +0 -15
  1104. package/src/duckdb/src/planner/operator/logical_expression_get.cpp +0 -24
  1105. package/src/duckdb/src/planner/operator/logical_extension_operator.cpp +32 -6
  1106. package/src/duckdb/src/planner/operator/logical_filter.cpp +0 -15
  1107. package/src/duckdb/src/planner/operator/logical_get.cpp +63 -70
  1108. package/src/duckdb/src/planner/operator/logical_insert.cpp +4 -74
  1109. package/src/duckdb/src/planner/operator/logical_join.cpp +0 -17
  1110. package/src/duckdb/src/planner/operator/logical_limit.cpp +0 -16
  1111. package/src/duckdb/src/planner/operator/logical_limit_percent.cpp +0 -16
  1112. package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +9 -0
  1113. package/src/duckdb/src/planner/operator/logical_order.cpp +35 -11
  1114. package/src/duckdb/src/planner/operator/logical_pivot.cpp +3 -20
  1115. package/src/duckdb/src/planner/operator/logical_positional_join.cpp +0 -10
  1116. package/src/duckdb/src/planner/operator/logical_pragma.cpp +0 -8
  1117. package/src/duckdb/src/planner/operator/logical_prepare.cpp +0 -8
  1118. package/src/duckdb/src/planner/operator/logical_projection.cpp +0 -12
  1119. package/src/duckdb/src/planner/operator/logical_recursive_cte.cpp +0 -15
  1120. package/src/duckdb/src/planner/operator/logical_reset.cpp +0 -12
  1121. package/src/duckdb/src/planner/operator/logical_sample.cpp +3 -11
  1122. package/src/duckdb/src/planner/operator/logical_set.cpp +0 -14
  1123. package/src/duckdb/src/planner/operator/logical_set_operation.cpp +0 -13
  1124. package/src/duckdb/src/planner/operator/logical_simple.cpp +0 -60
  1125. package/src/duckdb/src/planner/operator/logical_top_n.cpp +0 -14
  1126. package/src/duckdb/src/planner/operator/logical_unnest.cpp +0 -14
  1127. package/src/duckdb/src/planner/operator/logical_update.cpp +4 -25
  1128. package/src/duckdb/src/planner/operator/logical_window.cpp +0 -13
  1129. package/src/duckdb/src/planner/planner.cpp +26 -24
  1130. package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
  1131. package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
  1132. package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
  1133. package/src/duckdb/src/planner/table_filter.cpp +0 -59
  1134. package/src/duckdb/src/storage/arena_allocator.cpp +13 -2
  1135. package/src/duckdb/src/storage/buffer/block_manager.cpp +13 -9
  1136. package/src/duckdb/src/storage/checkpoint/row_group_writer.cpp +5 -18
  1137. package/src/duckdb/src/storage/checkpoint/table_data_reader.cpp +10 -8
  1138. package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +25 -25
  1139. package/src/duckdb/src/storage/checkpoint/write_overflow_strings_to_disk.cpp +53 -26
  1140. package/src/duckdb/src/storage/checkpoint_manager.cpp +314 -267
  1141. package/src/duckdb/src/storage/compression/bitpacking.cpp +138 -107
  1142. package/src/duckdb/src/storage/compression/bitpacking_hugeint.cpp +295 -0
  1143. package/src/duckdb/src/storage/compression/fixed_size_uncompressed.cpp +6 -1
  1144. package/src/duckdb/src/storage/compression/fsst.cpp +1 -1
  1145. package/src/duckdb/src/storage/compression/rle.cpp +66 -16
  1146. package/src/duckdb/src/storage/compression/string_uncompressed.cpp +84 -51
  1147. package/src/duckdb/src/storage/compression/validity_uncompressed.cpp +4 -4
  1148. package/src/duckdb/src/storage/data_pointer.cpp +20 -0
  1149. package/src/duckdb/src/storage/data_table.cpp +60 -39
  1150. package/src/duckdb/src/storage/index.cpp +11 -27
  1151. package/src/duckdb/src/storage/local_storage.cpp +17 -16
  1152. package/src/duckdb/src/storage/metadata/metadata_manager.cpp +322 -0
  1153. package/src/duckdb/src/storage/metadata/metadata_reader.cpp +89 -0
  1154. package/src/duckdb/src/storage/metadata/metadata_writer.cpp +92 -0
  1155. package/src/duckdb/src/storage/partial_block_manager.cpp +42 -15
  1156. package/src/duckdb/src/storage/serialization/serialize_constraint.cpp +98 -0
  1157. package/src/duckdb/src/storage/serialization/serialize_create_info.cpp +198 -0
  1158. package/src/duckdb/src/storage/serialization/serialize_expression.cpp +283 -0
  1159. package/src/duckdb/src/storage/serialization/serialize_logical_operator.cpp +748 -0
  1160. package/src/duckdb/src/storage/serialization/serialize_macro_function.cpp +62 -0
  1161. package/src/duckdb/src/storage/serialization/serialize_nodes.cpp +459 -0
  1162. package/src/duckdb/src/storage/serialization/serialize_parse_info.cpp +421 -0
  1163. package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +342 -0
  1164. package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
  1165. package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +97 -0
  1166. package/src/duckdb/src/storage/serialization/serialize_statement.cpp +22 -0
  1167. package/src/duckdb/src/storage/serialization/serialize_storage.cpp +78 -0
  1168. package/src/duckdb/src/storage/serialization/serialize_table_filter.cpp +97 -0
  1169. package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +164 -0
  1170. package/src/duckdb/src/storage/serialization/serialize_types.cpp +127 -0
  1171. package/src/duckdb/src/storage/single_file_block_manager.cpp +93 -64
  1172. package/src/duckdb/src/storage/standard_buffer_manager.cpp +10 -16
  1173. package/src/duckdb/src/storage/statistics/base_statistics.cpp +62 -53
  1174. package/src/duckdb/src/storage/statistics/column_statistics.cpp +10 -7
  1175. package/src/duckdb/src/storage/statistics/distinct_statistics.cpp +0 -26
  1176. package/src/duckdb/src/storage/statistics/list_stats.cpp +12 -7
  1177. package/src/duckdb/src/storage/statistics/numeric_stats.cpp +53 -43
  1178. package/src/duckdb/src/storage/statistics/string_stats.cpp +37 -18
  1179. package/src/duckdb/src/storage/statistics/struct_stats.cpp +17 -12
  1180. package/src/duckdb/src/storage/storage_info.cpp +3 -2
  1181. package/src/duckdb/src/storage/storage_manager.cpp +22 -7
  1182. package/src/duckdb/src/storage/table/chunk_info.cpp +59 -36
  1183. package/src/duckdb/src/storage/table/column_checkpoint_state.cpp +31 -34
  1184. package/src/duckdb/src/storage/table/column_data.cpp +44 -41
  1185. package/src/duckdb/src/storage/table/column_data_checkpointer.cpp +7 -11
  1186. package/src/duckdb/src/storage/table/column_segment.cpp +21 -12
  1187. package/src/duckdb/src/storage/table/list_column_data.cpp +24 -14
  1188. package/src/duckdb/src/storage/table/persistent_table_data.cpp +1 -2
  1189. package/src/duckdb/src/storage/table/row_group.cpp +131 -209
  1190. package/src/duckdb/src/storage/table/row_group_collection.cpp +34 -25
  1191. package/src/duckdb/src/storage/table/row_version_manager.cpp +228 -0
  1192. package/src/duckdb/src/storage/table/standard_column_data.cpp +10 -7
  1193. package/src/duckdb/src/storage/table/struct_column_data.cpp +22 -11
  1194. package/src/duckdb/src/storage/table/table_statistics.cpp +19 -9
  1195. package/src/duckdb/src/storage/table/update_segment.cpp +3 -3
  1196. package/src/duckdb/src/storage/table_index_list.cpp +2 -2
  1197. package/src/duckdb/src/storage/wal_replay.cpp +135 -127
  1198. package/src/duckdb/src/storage/write_ahead_log.cpp +136 -64
  1199. package/src/duckdb/src/transaction/cleanup_state.cpp +2 -1
  1200. package/src/duckdb/src/transaction/commit_state.cpp +32 -13
  1201. package/src/duckdb/src/transaction/duck_transaction.cpp +5 -2
  1202. package/src/duckdb/src/transaction/rollback_state.cpp +2 -1
  1203. package/src/duckdb/src/transaction/undo_buffer.cpp +3 -5
  1204. package/src/duckdb/src/verification/deserialized_statement_verifier.cpp +11 -6
  1205. package/src/duckdb/src/verification/prepared_statement_verifier.cpp +16 -11
  1206. package/src/duckdb/src/verification/statement_verifier.cpp +0 -3
  1207. package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
  1208. package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +5 -2
  1209. package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
  1210. package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
  1211. package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +11 -0
  1212. package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
  1213. package/src/duckdb/third_party/libpg_query/pg_functions.cpp +13 -0
  1214. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +11019 -10364
  1215. package/src/duckdb/third_party/libpg_query/src_backend_parser_scansup.cpp +9 -0
  1216. package/src/duckdb/third_party/mbedtls/include/mbedtls_wrapper.hpp +14 -1
  1217. package/src/duckdb/third_party/mbedtls/mbedtls_wrapper.cpp +54 -2
  1218. package/src/duckdb/third_party/parquet/parquet_types.h +1 -1
  1219. package/src/duckdb/third_party/utf8proc/include/utf8proc_wrapper.hpp +8 -0
  1220. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +4 -4
  1221. package/src/duckdb/ub_src_common.cpp +3 -3
  1222. package/src/duckdb/ub_src_common_adbc_nanoarrow.cpp +8 -0
  1223. package/src/duckdb/ub_src_common_arrow_appender.cpp +10 -0
  1224. package/src/duckdb/ub_src_common_row_operations.cpp +1 -1
  1225. package/src/duckdb/ub_src_common_serializer.cpp +3 -3
  1226. package/src/duckdb/ub_src_core_functions_scalar_debug.cpp +2 -0
  1227. package/src/duckdb/ub_src_core_functions_scalar_list.cpp +6 -0
  1228. package/src/duckdb/ub_src_core_functions_scalar_string.cpp +4 -0
  1229. package/src/duckdb/ub_src_execution.cpp +2 -2
  1230. package/src/duckdb/ub_src_execution_index.cpp +4 -0
  1231. package/src/duckdb/ub_src_execution_index_art.cpp +0 -8
  1232. package/src/duckdb/ub_src_execution_operator_csv_scanner.cpp +18 -0
  1233. package/src/duckdb/ub_src_execution_operator_csv_scanner_sniffer.cpp +12 -0
  1234. package/src/duckdb/ub_src_execution_operator_persistent.cpp +1 -11
  1235. package/src/duckdb/ub_src_execution_operator_schema.cpp +1 -1
  1236. package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
  1237. package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
  1238. package/src/duckdb/ub_src_function_cast_union.cpp +2 -0
  1239. package/src/duckdb/ub_src_function_scalar.cpp +2 -0
  1240. package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
  1241. package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
  1242. package/src/duckdb/ub_src_function_table_arrow.cpp +2 -0
  1243. package/src/duckdb/ub_src_function_table_system.cpp +2 -0
  1244. package/src/duckdb/ub_src_main.cpp +2 -0
  1245. package/src/duckdb/ub_src_main_chunk_scan_state.cpp +2 -0
  1246. package/src/duckdb/ub_src_optimizer.cpp +6 -0
  1247. package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
  1248. package/src/duckdb/ub_src_optimizer_join_order.cpp +10 -0
  1249. package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
  1250. package/src/duckdb/ub_src_parser.cpp +0 -2
  1251. package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
  1252. package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
  1253. package/src/duckdb/ub_src_planner.cpp +2 -2
  1254. package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
  1255. package/src/duckdb/ub_src_planner_expression.cpp +0 -2
  1256. package/src/duckdb/ub_src_planner_operator.cpp +3 -11
  1257. package/src/duckdb/ub_src_storage.cpp +2 -4
  1258. package/src/duckdb/ub_src_storage_compression.cpp +2 -0
  1259. package/src/duckdb/ub_src_storage_metadata.cpp +6 -0
  1260. package/src/duckdb/ub_src_storage_serialization.cpp +30 -0
  1261. package/src/duckdb/ub_src_storage_table.cpp +2 -0
  1262. package/src/duckdb_node.cpp +6 -12
  1263. package/src/duckdb_node.hpp +25 -13
  1264. package/src/statement.cpp +27 -25
  1265. package/test/close_hang.test.ts +39 -0
  1266. package/test/columns.test.ts +25 -3
  1267. package/test/database_fail.test.ts +26 -0
  1268. package/test/extension.test.ts +1 -1
  1269. package/test/prepare.test.ts +10 -1
  1270. package/test/test_all_types.test.ts +234 -0
  1271. package/test/worker.js +7 -0
  1272. package/tsconfig.json +1 -0
  1273. package/src/duckdb/src/common/field_writer.cpp +0 -97
  1274. package/src/duckdb/src/common/row_operations/row_match.cpp +0 -359
  1275. package/src/duckdb/src/common/serializer/buffered_deserializer.cpp +0 -27
  1276. package/src/duckdb/src/common/serializer/buffered_serializer.cpp +0 -36
  1277. package/src/duckdb/src/common/serializer.cpp +0 -24
  1278. package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +0 -242
  1279. package/src/duckdb/src/execution/index/art/leaf_segment.cpp +0 -52
  1280. package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
  1281. package/src/duckdb/src/execution/index/art/swizzleable_pointer.cpp +0 -22
  1282. package/src/duckdb/src/execution/operator/persistent/buffered_csv_reader.cpp +0 -1487
  1283. package/src/duckdb/src/execution/operator/persistent/csv_buffer.cpp +0 -72
  1284. package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +0 -158
  1285. package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +0 -260
  1286. package/src/duckdb/src/execution/operator/schema/physical_create_index.cpp +0 -193
  1287. package/src/duckdb/src/execution/partitionable_hashtable.cpp +0 -172
  1288. package/src/duckdb/src/include/duckdb/common/arrow/arrow_options.hpp +0 -25
  1289. package/src/duckdb/src/include/duckdb/common/field_writer.hpp +0 -383
  1290. package/src/duckdb/src/include/duckdb/common/serializer/buffered_deserializer.hpp +0 -41
  1291. package/src/duckdb/src/include/duckdb/common/serializer/buffered_serializer.hpp +0 -50
  1292. package/src/duckdb/src/include/duckdb/common/serializer/format_serializer.hpp +0 -248
  1293. package/src/duckdb/src/include/duckdb/common/serializer.hpp +0 -158
  1294. package/src/duckdb/src/include/duckdb/execution/index/art/fixed_size_allocator.hpp +0 -117
  1295. package/src/duckdb/src/include/duckdb/execution/index/art/leaf_segment.hpp +0 -38
  1296. package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
  1297. package/src/duckdb/src/include/duckdb/execution/index/art/swizzleable_pointer.hpp +0 -58
  1298. package/src/duckdb/src/include/duckdb/execution/operator/persistent/buffered_csv_reader.hpp +0 -133
  1299. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_buffer.hpp +0 -74
  1300. package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +0 -69
  1301. package/src/duckdb/src/include/duckdb/planner/operator/logical_asof_join.hpp +0 -27
  1302. package/src/duckdb/src/include/duckdb/planner/operator/logical_delim_join.hpp +0 -32
  1303. package/src/duckdb/src/include/duckdb/planner/plan_serialization.hpp +0 -44
  1304. package/src/duckdb/src/include/duckdb/storage/meta_block_reader.hpp +0 -49
  1305. package/src/duckdb/src/include/duckdb/storage/meta_block_writer.hpp +0 -50
  1306. package/src/duckdb/src/include/duckdb/verification/deserialized_statement_verifier_v2.hpp +0 -26
  1307. package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
  1308. package/src/duckdb/src/parser/common_table_expression_info.cpp +0 -19
  1309. package/src/duckdb/src/planner/expression/bound_default_expression.cpp +0 -16
  1310. package/src/duckdb/src/planner/operator/logical_asof_join.cpp +0 -14
  1311. package/src/duckdb/src/planner/operator/logical_delim_join.cpp +0 -27
  1312. package/src/duckdb/src/planner/operator/logical_execute.cpp +0 -12
  1313. package/src/duckdb/src/planner/operator/logical_explain.cpp +0 -22
  1314. package/src/duckdb/src/planner/operator/logical_export.cpp +0 -13
  1315. package/src/duckdb/src/planner/operator/logical_show.cpp +0 -21
  1316. package/src/duckdb/src/planner/parsed_data/bound_create_table_info.cpp +0 -20
  1317. package/src/duckdb/src/planner/plan_serialization.cpp +0 -20
  1318. package/src/duckdb/src/storage/meta_block_reader.cpp +0 -78
  1319. package/src/duckdb/src/storage/meta_block_writer.cpp +0 -80
  1320. package/src/duckdb/src/verification/deserialized_statement_verifier_v2.cpp +0 -20
  1321. package/src/duckdb/ub_src_planner_parsed_data.cpp +0 -2
@@ -14,7 +14,7 @@
14
14
  #include "duckdb/common/vector_operations/vector_operations.hpp"
15
15
  #include "duckdb/common/windows_undefs.hpp"
16
16
  #include "duckdb/execution/expression_executor.hpp"
17
- #include "duckdb/execution/partitionable_hashtable.hpp"
17
+ #include "duckdb/execution/window_executor.hpp"
18
18
  #include "duckdb/execution/window_segment_tree.hpp"
19
19
  #include "duckdb/main/client_config.hpp"
20
20
  #include "duckdb/main/config.hpp"
@@ -32,7 +32,7 @@ namespace duckdb {
32
32
  class WindowGlobalSinkState : public GlobalSinkState {
33
33
  public:
34
34
  WindowGlobalSinkState(const PhysicalWindow &op, ClientContext &context)
35
- : mode(DBConfig::GetConfig(context).options.window_mode) {
35
+ : op(op), mode(DBConfig::GetConfig(context).options.window_mode) {
36
36
 
37
37
  D_ASSERT(op.select_list[0]->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);
38
38
  auto &wexpr = op.select_list[0]->Cast<BoundWindowExpression>();
@@ -42,6 +42,7 @@ public:
42
42
  wexpr.partitions_stats, op.estimated_cardinality);
43
43
  }
44
44
 
45
+ const PhysicalWindow &op;
45
46
  unique_ptr<PartitionGlobalSinkState> global_partition;
46
47
  WindowAggregationMode mode;
47
48
  };
@@ -78,977 +79,40 @@ PhysicalWindow::PhysicalWindow(vector<LogicalType> types, vector<unique_ptr<Expr
78
79
  }
79
80
  }
80
81
 
81
- static idx_t FindNextStart(const ValidityMask &mask, idx_t l, const idx_t r, idx_t &n) {
82
- if (mask.AllValid()) {
83
- auto start = MinValue(l + n - 1, r);
84
- n -= MinValue(n, r - l);
85
- return start;
86
- }
87
-
88
- while (l < r) {
89
- // If l is aligned with the start of a block, and the block is blank, then skip forward one block.
90
- idx_t entry_idx;
91
- idx_t shift;
92
- mask.GetEntryIndex(l, entry_idx, shift);
93
-
94
- const auto block = mask.GetValidityEntry(entry_idx);
95
- if (mask.NoneValid(block) && !shift) {
96
- l += ValidityMask::BITS_PER_VALUE;
97
- continue;
98
- }
99
-
100
- // Loop over the block
101
- for (; shift < ValidityMask::BITS_PER_VALUE && l < r; ++shift, ++l) {
102
- if (mask.RowIsValid(block, shift) && --n == 0) {
103
- return MinValue(l, r);
104
- }
105
- }
106
- }
107
-
108
- // Didn't find a start so return the end of the range
109
- return r;
110
- }
111
-
112
- static idx_t FindPrevStart(const ValidityMask &mask, const idx_t l, idx_t r, idx_t &n) {
113
- if (mask.AllValid()) {
114
- auto start = (r <= l + n) ? l : r - n;
115
- n -= r - start;
116
- return start;
117
- }
118
-
119
- while (l < r) {
120
- // If r is aligned with the start of a block, and the previous block is blank,
121
- // then skip backwards one block.
122
- idx_t entry_idx;
123
- idx_t shift;
124
- mask.GetEntryIndex(r - 1, entry_idx, shift);
125
-
126
- const auto block = mask.GetValidityEntry(entry_idx);
127
- if (mask.NoneValid(block) && (shift + 1 == ValidityMask::BITS_PER_VALUE)) {
128
- // r is nonzero (> l) and word aligned, so this will not underflow.
129
- r -= ValidityMask::BITS_PER_VALUE;
130
- continue;
131
- }
132
-
133
- // Loop backwards over the block
134
- // shift is probing r-1 >= l >= 0
135
- for (++shift; shift-- > 0; --r) {
136
- if (mask.RowIsValid(block, shift) && --n == 0) {
137
- return MaxValue(l, r - 1);
138
- }
139
- }
140
- }
141
-
142
- // Didn't find a start so return the start of the range
143
- return l;
144
- }
145
-
146
- static void PrepareInputExpressions(vector<unique_ptr<Expression>> &exprs, ExpressionExecutor &executor,
147
- DataChunk &chunk) {
148
- if (exprs.empty()) {
149
- return;
150
- }
151
-
152
- vector<LogicalType> types;
153
- for (idx_t expr_idx = 0; expr_idx < exprs.size(); ++expr_idx) {
154
- types.push_back(exprs[expr_idx]->return_type);
155
- executor.AddExpression(*exprs[expr_idx]);
156
- }
157
-
158
- if (!types.empty()) {
159
- auto &allocator = executor.GetAllocator();
160
- chunk.Initialize(allocator, types);
161
- }
162
- }
163
-
164
- static void PrepareInputExpression(Expression &expr, ExpressionExecutor &executor, DataChunk &chunk) {
165
- vector<LogicalType> types;
166
- types.push_back(expr.return_type);
167
- executor.AddExpression(expr);
168
-
169
- auto &allocator = executor.GetAllocator();
170
- chunk.Initialize(allocator, types);
171
- }
172
-
173
- struct WindowInputExpression {
174
- WindowInputExpression(optional_ptr<Expression> expr_p, ClientContext &context)
175
- : expr(expr_p), ptype(PhysicalType::INVALID), scalar(true), executor(context) {
176
- if (expr) {
177
- PrepareInputExpression(*expr, executor, chunk);
178
- ptype = expr->return_type.InternalType();
179
- scalar = expr->IsScalar();
180
- }
181
- }
182
-
183
- void Execute(DataChunk &input_chunk) {
184
- if (expr) {
185
- chunk.Reset();
186
- executor.Execute(input_chunk, chunk);
187
- chunk.Verify();
188
- }
189
- }
190
-
191
- template <typename T>
192
- inline T GetCell(idx_t i) const {
193
- D_ASSERT(!chunk.data.empty());
194
- const auto data = FlatVector::GetData<T>(chunk.data[0]);
195
- return data[scalar ? 0 : i];
196
- }
197
-
198
- inline bool CellIsNull(idx_t i) const {
199
- D_ASSERT(!chunk.data.empty());
200
- if (chunk.data[0].GetVectorType() == VectorType::CONSTANT_VECTOR) {
201
- return ConstantVector::IsNull(chunk.data[0]);
202
- }
203
- return FlatVector::IsNull(chunk.data[0], i);
204
- }
205
-
206
- inline void CopyCell(Vector &target, idx_t target_offset) const {
207
- D_ASSERT(!chunk.data.empty());
208
- auto &source = chunk.data[0];
209
- auto source_offset = scalar ? 0 : target_offset;
210
- VectorOperations::Copy(source, target, source_offset + 1, source_offset, target_offset);
211
- }
212
-
213
- optional_ptr<Expression> expr;
214
- PhysicalType ptype;
215
- bool scalar;
216
- ExpressionExecutor executor;
217
- DataChunk chunk;
218
- };
219
-
220
- struct WindowInputColumn {
221
- WindowInputColumn(Expression *expr_p, ClientContext &context, idx_t capacity_p)
222
- : input_expr(expr_p, context), count(0), capacity(capacity_p) {
223
- if (input_expr.expr) {
224
- target = make_uniq<Vector>(input_expr.chunk.data[0].GetType(), capacity);
225
- }
226
- }
227
-
228
- void Append(DataChunk &input_chunk) {
229
- if (input_expr.expr) {
230
- const auto source_count = input_chunk.size();
231
- D_ASSERT(count + source_count <= capacity);
232
- if (!input_expr.scalar || !count) {
233
- input_expr.Execute(input_chunk);
234
- auto &source = input_expr.chunk.data[0];
235
- VectorOperations::Copy(source, *target, source_count, 0, count);
236
- }
237
- count += source_count;
238
- }
239
- }
240
-
241
- inline bool CellIsNull(idx_t i) {
242
- D_ASSERT(target);
243
- D_ASSERT(i < count);
244
- return FlatVector::IsNull(*target, input_expr.scalar ? 0 : i);
245
- }
246
-
247
- template <typename T>
248
- inline T GetCell(idx_t i) const {
249
- D_ASSERT(target);
250
- D_ASSERT(i < count);
251
- const auto data = FlatVector::GetData<T>(*target);
252
- return data[input_expr.scalar ? 0 : i];
253
- }
254
-
255
- WindowInputExpression input_expr;
256
-
257
- private:
258
- unique_ptr<Vector> target;
259
- idx_t count;
260
- idx_t capacity;
261
- };
262
-
263
- static inline bool BoundaryNeedsPeer(const WindowBoundary &boundary) {
264
- switch (boundary) {
265
- case WindowBoundary::CURRENT_ROW_RANGE:
266
- case WindowBoundary::EXPR_PRECEDING_RANGE:
267
- case WindowBoundary::EXPR_FOLLOWING_RANGE:
268
- return true;
269
- default:
270
- return false;
271
- }
272
- }
273
-
274
- struct WindowBoundariesState {
275
- static inline bool IsScalar(const unique_ptr<Expression> &expr) {
276
- return expr ? expr->IsScalar() : true;
277
- }
278
-
279
- WindowBoundariesState(BoundWindowExpression &wexpr, const idx_t input_size)
280
- : type(wexpr.type), input_size(input_size), start_boundary(wexpr.start), end_boundary(wexpr.end),
281
- partition_count(wexpr.partitions.size()), order_count(wexpr.orders.size()),
282
- range_sense(wexpr.orders.empty() ? OrderType::INVALID : wexpr.orders[0].type),
283
- has_preceding_range(wexpr.start == WindowBoundary::EXPR_PRECEDING_RANGE ||
284
- wexpr.end == WindowBoundary::EXPR_PRECEDING_RANGE),
285
- has_following_range(wexpr.start == WindowBoundary::EXPR_FOLLOWING_RANGE ||
286
- wexpr.end == WindowBoundary::EXPR_FOLLOWING_RANGE),
287
- needs_peer(BoundaryNeedsPeer(wexpr.end) || wexpr.type == ExpressionType::WINDOW_CUME_DIST) {
288
- }
289
-
290
- void Update(const idx_t row_idx, WindowInputColumn &range_collection, const idx_t source_offset,
291
- WindowInputExpression &boundary_start, WindowInputExpression &boundary_end,
292
- const ValidityMask &partition_mask, const ValidityMask &order_mask);
293
-
294
- // Cached lookups
295
- const ExpressionType type;
296
- const idx_t input_size;
297
- const WindowBoundary start_boundary;
298
- const WindowBoundary end_boundary;
299
- const size_t partition_count;
300
- const size_t order_count;
301
- const OrderType range_sense;
302
- const bool has_preceding_range;
303
- const bool has_following_range;
304
- const bool needs_peer;
305
-
306
- idx_t partition_start = 0;
307
- idx_t partition_end = 0;
308
- idx_t peer_start = 0;
309
- idx_t peer_end = 0;
310
- idx_t valid_start = 0;
311
- idx_t valid_end = 0;
312
- int64_t window_start = -1;
313
- int64_t window_end = -1;
314
- bool is_same_partition = false;
315
- bool is_peer = false;
316
- };
317
-
318
- static bool WindowNeedsRank(const BoundWindowExpression &wexpr) {
319
- return wexpr.type == ExpressionType::WINDOW_PERCENT_RANK || wexpr.type == ExpressionType::WINDOW_RANK ||
320
- wexpr.type == ExpressionType::WINDOW_RANK_DENSE || wexpr.type == ExpressionType::WINDOW_CUME_DIST;
321
- }
322
-
323
- template <typename T>
324
- static T GetCell(DataChunk &chunk, idx_t column, idx_t index) {
325
- D_ASSERT(chunk.ColumnCount() > column);
326
- auto &source = chunk.data[column];
327
- const auto data = FlatVector::GetData<T>(source);
328
- return data[index];
329
- }
330
-
331
- static bool CellIsNull(DataChunk &chunk, idx_t column, idx_t index) {
332
- D_ASSERT(chunk.ColumnCount() > column);
333
- auto &source = chunk.data[column];
334
- return FlatVector::IsNull(source, index);
335
- }
336
-
337
- static void CopyCell(DataChunk &chunk, idx_t column, idx_t index, Vector &target, idx_t target_offset) {
338
- D_ASSERT(chunk.ColumnCount() > column);
339
- auto &source = chunk.data[column];
340
- VectorOperations::Copy(source, target, index + 1, index, target_offset);
341
- }
342
-
343
- template <typename T>
344
- struct WindowColumnIterator {
345
- using iterator = WindowColumnIterator<T>;
346
- using iterator_category = std::forward_iterator_tag;
347
- using difference_type = std::ptrdiff_t;
348
- using value_type = T;
349
- using reference = T;
350
- using pointer = idx_t;
351
-
352
- explicit WindowColumnIterator(WindowInputColumn &coll_p, pointer pos_p = 0) : coll(&coll_p), pos(pos_p) {
353
- }
354
-
355
- inline reference operator*() const {
356
- return coll->GetCell<T>(pos);
357
- }
358
- inline explicit operator pointer() const {
359
- return pos;
360
- }
361
-
362
- inline iterator &operator++() {
363
- ++pos;
364
- return *this;
365
- }
366
- inline iterator operator++(int) {
367
- auto result = *this;
368
- ++(*this);
369
- return result;
370
- }
371
-
372
- friend inline bool operator==(const iterator &a, const iterator &b) {
373
- return a.pos == b.pos;
374
- }
375
- friend inline bool operator!=(const iterator &a, const iterator &b) {
376
- return a.pos != b.pos;
377
- }
378
-
379
- private:
380
- optional_ptr<WindowInputColumn> coll;
381
- pointer pos;
382
- };
383
-
384
- template <typename T, typename OP>
385
- struct OperationCompare : public std::function<bool(T, T)> {
386
- inline bool operator()(const T &lhs, const T &val) const {
387
- return OP::template Operation(lhs, val);
388
- }
389
- };
390
-
391
- template <typename T, typename OP, bool FROM>
392
- static idx_t FindTypedRangeBound(WindowInputColumn &over, const idx_t order_begin, const idx_t order_end,
393
- WindowInputExpression &boundary, const idx_t boundary_row) {
394
- D_ASSERT(!boundary.CellIsNull(boundary_row));
395
- const auto val = boundary.GetCell<T>(boundary_row);
396
-
397
- OperationCompare<T, OP> comp;
398
- WindowColumnIterator<T> begin(over, order_begin);
399
- WindowColumnIterator<T> end(over, order_end);
400
- if (FROM) {
401
- return idx_t(std::lower_bound(begin, end, val, comp));
402
- } else {
403
- return idx_t(std::upper_bound(begin, end, val, comp));
404
- }
405
- }
406
-
407
- template <typename OP, bool FROM>
408
- static idx_t FindRangeBound(WindowInputColumn &over, const idx_t order_begin, const idx_t order_end,
409
- WindowInputExpression &boundary, const idx_t expr_idx) {
410
- D_ASSERT(boundary.chunk.ColumnCount() == 1);
411
- D_ASSERT(boundary.chunk.data[0].GetType().InternalType() == over.input_expr.ptype);
412
-
413
- switch (over.input_expr.ptype) {
414
- case PhysicalType::INT8:
415
- return FindTypedRangeBound<int8_t, OP, FROM>(over, order_begin, order_end, boundary, expr_idx);
416
- case PhysicalType::INT16:
417
- return FindTypedRangeBound<int16_t, OP, FROM>(over, order_begin, order_end, boundary, expr_idx);
418
- case PhysicalType::INT32:
419
- return FindTypedRangeBound<int32_t, OP, FROM>(over, order_begin, order_end, boundary, expr_idx);
420
- case PhysicalType::INT64:
421
- return FindTypedRangeBound<int64_t, OP, FROM>(over, order_begin, order_end, boundary, expr_idx);
422
- case PhysicalType::UINT8:
423
- return FindTypedRangeBound<uint8_t, OP, FROM>(over, order_begin, order_end, boundary, expr_idx);
424
- case PhysicalType::UINT16:
425
- return FindTypedRangeBound<uint16_t, OP, FROM>(over, order_begin, order_end, boundary, expr_idx);
426
- case PhysicalType::UINT32:
427
- return FindTypedRangeBound<uint32_t, OP, FROM>(over, order_begin, order_end, boundary, expr_idx);
428
- case PhysicalType::UINT64:
429
- return FindTypedRangeBound<uint64_t, OP, FROM>(over, order_begin, order_end, boundary, expr_idx);
430
- case PhysicalType::INT128:
431
- return FindTypedRangeBound<hugeint_t, OP, FROM>(over, order_begin, order_end, boundary, expr_idx);
432
- case PhysicalType::FLOAT:
433
- return FindTypedRangeBound<float, OP, FROM>(over, order_begin, order_end, boundary, expr_idx);
434
- case PhysicalType::DOUBLE:
435
- return FindTypedRangeBound<double, OP, FROM>(over, order_begin, order_end, boundary, expr_idx);
436
- case PhysicalType::INTERVAL:
437
- return FindTypedRangeBound<interval_t, OP, FROM>(over, order_begin, order_end, boundary, expr_idx);
438
- default:
439
- throw InternalException("Unsupported column type for RANGE");
440
- }
441
- }
442
-
443
- template <bool FROM>
444
- static idx_t FindOrderedRangeBound(WindowInputColumn &over, const OrderType range_sense, const idx_t order_begin,
445
- const idx_t order_end, WindowInputExpression &boundary, const idx_t expr_idx) {
446
- switch (range_sense) {
447
- case OrderType::ASCENDING:
448
- return FindRangeBound<LessThan, FROM>(over, order_begin, order_end, boundary, expr_idx);
449
- case OrderType::DESCENDING:
450
- return FindRangeBound<GreaterThan, FROM>(over, order_begin, order_end, boundary, expr_idx);
451
- default:
452
- throw InternalException("Unsupported ORDER BY sense for RANGE");
453
- }
454
- }
455
-
456
- void WindowBoundariesState::Update(const idx_t row_idx, WindowInputColumn &range_collection, const idx_t expr_idx,
457
- WindowInputExpression &boundary_start, WindowInputExpression &boundary_end,
458
- const ValidityMask &partition_mask, const ValidityMask &order_mask) {
459
-
460
- auto &bounds = *this;
461
- if (bounds.partition_count + bounds.order_count > 0) {
462
-
463
- // determine partition and peer group boundaries to ultimately figure out window size
464
- bounds.is_same_partition = !partition_mask.RowIsValidUnsafe(row_idx);
465
- bounds.is_peer = !order_mask.RowIsValidUnsafe(row_idx);
466
-
467
- // when the partition changes, recompute the boundaries
468
- if (!bounds.is_same_partition) {
469
- bounds.partition_start = row_idx;
470
- bounds.peer_start = row_idx;
471
-
472
- // find end of partition
473
- bounds.partition_end = bounds.input_size;
474
- if (bounds.partition_count) {
475
- idx_t n = 1;
476
- bounds.partition_end = FindNextStart(partition_mask, bounds.partition_start + 1, bounds.input_size, n);
477
- }
478
-
479
- // Find valid ordering values for the new partition
480
- // so we can exclude NULLs from RANGE expression computations
481
- bounds.valid_start = bounds.partition_start;
482
- bounds.valid_end = bounds.partition_end;
483
-
484
- if ((bounds.valid_start < bounds.valid_end) && bounds.has_preceding_range) {
485
- // Exclude any leading NULLs
486
- if (range_collection.CellIsNull(bounds.valid_start)) {
487
- idx_t n = 1;
488
- bounds.valid_start = FindNextStart(order_mask, bounds.valid_start + 1, bounds.valid_end, n);
489
- }
490
- }
491
-
492
- if ((bounds.valid_start < bounds.valid_end) && bounds.has_following_range) {
493
- // Exclude any trailing NULLs
494
- if (range_collection.CellIsNull(bounds.valid_end - 1)) {
495
- idx_t n = 1;
496
- bounds.valid_end = FindPrevStart(order_mask, bounds.valid_start, bounds.valid_end, n);
497
- }
498
- }
499
-
500
- } else if (!bounds.is_peer) {
501
- bounds.peer_start = row_idx;
502
- }
503
-
504
- if (bounds.needs_peer) {
505
- bounds.peer_end = bounds.partition_end;
506
- if (bounds.order_count) {
507
- idx_t n = 1;
508
- bounds.peer_end = FindNextStart(order_mask, bounds.peer_start + 1, bounds.partition_end, n);
509
- }
510
- }
511
-
512
- } else {
513
- bounds.is_same_partition = false;
514
- bounds.is_peer = true;
515
- bounds.partition_end = bounds.input_size;
516
- bounds.peer_end = bounds.partition_end;
517
- }
518
-
519
- // determine window boundaries depending on the type of expression
520
- bounds.window_start = -1;
521
- bounds.window_end = -1;
522
-
523
- switch (bounds.start_boundary) {
524
- case WindowBoundary::UNBOUNDED_PRECEDING:
525
- bounds.window_start = bounds.partition_start;
526
- break;
527
- case WindowBoundary::CURRENT_ROW_ROWS:
528
- bounds.window_start = row_idx;
529
- break;
530
- case WindowBoundary::CURRENT_ROW_RANGE:
531
- bounds.window_start = bounds.peer_start;
532
- break;
533
- case WindowBoundary::EXPR_PRECEDING_ROWS: {
534
- if (!TrySubtractOperator::Operation(int64_t(row_idx), boundary_start.GetCell<int64_t>(expr_idx),
535
- bounds.window_start)) {
536
- throw OutOfRangeException("Overflow computing ROWS PRECEDING start");
537
- }
538
- break;
539
- }
540
- case WindowBoundary::EXPR_FOLLOWING_ROWS: {
541
- if (!TryAddOperator::Operation(int64_t(row_idx), boundary_start.GetCell<int64_t>(expr_idx),
542
- bounds.window_start)) {
543
- throw OutOfRangeException("Overflow computing ROWS FOLLOWING start");
544
- }
545
- break;
546
- }
547
- case WindowBoundary::EXPR_PRECEDING_RANGE: {
548
- if (boundary_start.CellIsNull(expr_idx)) {
549
- bounds.window_start = bounds.peer_start;
550
- } else {
551
- bounds.window_start = FindOrderedRangeBound<true>(range_collection, bounds.range_sense, bounds.valid_start,
552
- row_idx, boundary_start, expr_idx);
553
- }
554
- break;
555
- }
556
- case WindowBoundary::EXPR_FOLLOWING_RANGE: {
557
- if (boundary_start.CellIsNull(expr_idx)) {
558
- bounds.window_start = bounds.peer_start;
559
- } else {
560
- bounds.window_start = FindOrderedRangeBound<true>(range_collection, bounds.range_sense, row_idx,
561
- bounds.valid_end, boundary_start, expr_idx);
562
- }
82
+ static unique_ptr<WindowExecutor> WindowExecutorFactory(BoundWindowExpression &wexpr, ClientContext &context,
83
+ const ValidityMask &partition_mask,
84
+ const ValidityMask &order_mask, const idx_t payload_count,
85
+ WindowAggregationMode mode) {
86
+ switch (wexpr.type) {
87
+ case ExpressionType::WINDOW_AGGREGATE:
88
+ return make_uniq<WindowAggregateExecutor>(wexpr, context, payload_count, partition_mask, order_mask, mode);
89
+ case ExpressionType::WINDOW_ROW_NUMBER:
90
+ return make_uniq<WindowRowNumberExecutor>(wexpr, context, payload_count, partition_mask, order_mask);
91
+ case ExpressionType::WINDOW_RANK_DENSE:
92
+ return make_uniq<WindowDenseRankExecutor>(wexpr, context, payload_count, partition_mask, order_mask);
93
+ case ExpressionType::WINDOW_RANK:
94
+ return make_uniq<WindowRankExecutor>(wexpr, context, payload_count, partition_mask, order_mask);
95
+ case ExpressionType::WINDOW_PERCENT_RANK:
96
+ return make_uniq<WindowPercentRankExecutor>(wexpr, context, payload_count, partition_mask, order_mask);
97
+ case ExpressionType::WINDOW_CUME_DIST:
98
+ return make_uniq<WindowCumeDistExecutor>(wexpr, context, payload_count, partition_mask, order_mask);
99
+ case ExpressionType::WINDOW_NTILE:
100
+ return make_uniq<WindowNtileExecutor>(wexpr, context, payload_count, partition_mask, order_mask);
101
+ case ExpressionType::WINDOW_LEAD:
102
+ case ExpressionType::WINDOW_LAG:
103
+ return make_uniq<WindowLeadLagExecutor>(wexpr, context, payload_count, partition_mask, order_mask);
104
+ case ExpressionType::WINDOW_FIRST_VALUE:
105
+ return make_uniq<WindowFirstValueExecutor>(wexpr, context, payload_count, partition_mask, order_mask);
106
+ case ExpressionType::WINDOW_LAST_VALUE:
107
+ return make_uniq<WindowLastValueExecutor>(wexpr, context, payload_count, partition_mask, order_mask);
108
+ case ExpressionType::WINDOW_NTH_VALUE:
109
+ return make_uniq<WindowNthValueExecutor>(wexpr, context, payload_count, partition_mask, order_mask);
563
110
  break;
564
- }
565
111
  default:
566
- throw InternalException("Unsupported window start boundary");
567
- }
568
-
569
- switch (bounds.end_boundary) {
570
- case WindowBoundary::CURRENT_ROW_ROWS:
571
- bounds.window_end = row_idx + 1;
572
- break;
573
- case WindowBoundary::CURRENT_ROW_RANGE:
574
- bounds.window_end = bounds.peer_end;
575
- break;
576
- case WindowBoundary::UNBOUNDED_FOLLOWING:
577
- bounds.window_end = bounds.partition_end;
578
- break;
579
- case WindowBoundary::EXPR_PRECEDING_ROWS:
580
- if (!TrySubtractOperator::Operation(int64_t(row_idx + 1), boundary_end.GetCell<int64_t>(expr_idx),
581
- bounds.window_end)) {
582
- throw OutOfRangeException("Overflow computing ROWS PRECEDING end");
583
- }
584
- break;
585
- case WindowBoundary::EXPR_FOLLOWING_ROWS:
586
- if (!TryAddOperator::Operation(int64_t(row_idx + 1), boundary_end.GetCell<int64_t>(expr_idx),
587
- bounds.window_end)) {
588
- throw OutOfRangeException("Overflow computing ROWS FOLLOWING end");
589
- }
590
- break;
591
- case WindowBoundary::EXPR_PRECEDING_RANGE: {
592
- if (boundary_end.CellIsNull(expr_idx)) {
593
- bounds.window_end = bounds.peer_end;
594
- } else {
595
- bounds.window_end = FindOrderedRangeBound<false>(range_collection, bounds.range_sense, bounds.valid_start,
596
- row_idx, boundary_end, expr_idx);
597
- }
598
- break;
599
- }
600
- case WindowBoundary::EXPR_FOLLOWING_RANGE: {
601
- if (boundary_end.CellIsNull(expr_idx)) {
602
- bounds.window_end = bounds.peer_end;
603
- } else {
604
- bounds.window_end = FindOrderedRangeBound<false>(range_collection, bounds.range_sense, row_idx,
605
- bounds.valid_end, boundary_end, expr_idx);
606
- }
607
- break;
608
- }
609
- default:
610
- throw InternalException("Unsupported window end boundary");
611
- }
612
-
613
- // clamp windows to partitions if they should exceed
614
- if (bounds.window_start < (int64_t)bounds.partition_start) {
615
- bounds.window_start = bounds.partition_start;
616
- }
617
- if (bounds.window_start > (int64_t)bounds.partition_end) {
618
- bounds.window_start = bounds.partition_end;
619
- }
620
- if (bounds.window_end < (int64_t)bounds.partition_start) {
621
- bounds.window_end = bounds.partition_start;
622
- }
623
- if (bounds.window_end > (int64_t)bounds.partition_end) {
624
- bounds.window_end = bounds.partition_end;
625
- }
626
-
627
- if (bounds.window_start < 0 || bounds.window_end < 0) {
628
- throw InternalException("Failed to compute window boundaries");
112
+ throw InternalException("Window aggregate type %s", ExpressionTypeToString(wexpr.type));
629
113
  }
630
114
  }
631
115
 
632
- struct WindowExecutor {
633
- static bool IsConstantAggregate(const BoundWindowExpression &wexpr);
634
-
635
- WindowExecutor(BoundWindowExpression &wexpr, ClientContext &context, const ValidityMask &partition_mask,
636
- const idx_t count);
637
-
638
- void Sink(DataChunk &input_chunk, const idx_t input_idx, const idx_t total_count);
639
- void Finalize(WindowAggregationMode mode);
640
-
641
- void Evaluate(idx_t row_idx, DataChunk &input_chunk, Vector &result, const ValidityMask &partition_mask,
642
- const ValidityMask &order_mask);
643
-
644
- // The function
645
- BoundWindowExpression &wexpr;
646
-
647
- // Frame management
648
- WindowBoundariesState bounds;
649
- uint64_t dense_rank = 1;
650
- uint64_t rank_equal = 0;
651
- uint64_t rank = 1;
652
-
653
- // Expression collections
654
- DataChunk payload_collection;
655
- ExpressionExecutor payload_executor;
656
- DataChunk payload_chunk;
657
-
658
- ExpressionExecutor filter_executor;
659
- ValidityMask filter_mask;
660
- vector<validity_t> filter_bits;
661
- SelectionVector filter_sel;
662
-
663
- // LEAD/LAG Evaluation
664
- WindowInputExpression leadlag_offset;
665
- WindowInputExpression leadlag_default;
666
-
667
- // evaluate boundaries if present. Parser has checked boundary types.
668
- WindowInputExpression boundary_start;
669
- WindowInputExpression boundary_end;
670
-
671
- // evaluate RANGE expressions, if needed
672
- WindowInputColumn range;
673
-
674
- // IGNORE NULLS
675
- ValidityMask ignore_nulls;
676
-
677
- // build a segment tree for frame-adhering aggregates
678
- // see http://www.vldb.org/pvldb/vol8/p1058-leis.pdf
679
- unique_ptr<WindowSegmentTree> segment_tree = nullptr;
680
-
681
- // all aggregate values are the same for each partition
682
- unique_ptr<WindowConstantAggregate> constant_aggregate = nullptr;
683
- };
684
-
685
- bool WindowExecutor::IsConstantAggregate(const BoundWindowExpression &wexpr) {
686
- if (!wexpr.aggregate) {
687
- return false;
688
- }
689
-
690
- // COUNT(*) is already handled efficiently by segment trees.
691
- if (wexpr.children.empty()) {
692
- return false;
693
- }
694
-
695
- /*
696
- The default framing option is RANGE UNBOUNDED PRECEDING, which
697
- is the same as RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT
698
- ROW; it sets the frame to be all rows from the partition start
699
- up through the current row's last peer (a row that the window's
700
- ORDER BY clause considers equivalent to the current row; all
701
- rows are peers if there is no ORDER BY). In general, UNBOUNDED
702
- PRECEDING means that the frame starts with the first row of the
703
- partition, and similarly UNBOUNDED FOLLOWING means that the
704
- frame ends with the last row of the partition, regardless of
705
- RANGE, ROWS or GROUPS mode. In ROWS mode, CURRENT ROW means that
706
- the frame starts or ends with the current row; but in RANGE or
707
- GROUPS mode it means that the frame starts or ends with the
708
- current row's first or last peer in the ORDER BY ordering. The
709
- offset PRECEDING and offset FOLLOWING options vary in meaning
710
- depending on the frame mode.
711
- */
712
- switch (wexpr.start) {
713
- case WindowBoundary::UNBOUNDED_PRECEDING:
714
- break;
715
- case WindowBoundary::CURRENT_ROW_RANGE:
716
- if (!wexpr.orders.empty()) {
717
- return false;
718
- }
719
- break;
720
- default:
721
- return false;
722
- }
723
-
724
- switch (wexpr.end) {
725
- case WindowBoundary::UNBOUNDED_FOLLOWING:
726
- break;
727
- case WindowBoundary::CURRENT_ROW_RANGE:
728
- if (!wexpr.orders.empty()) {
729
- return false;
730
- }
731
- break;
732
- default:
733
- return false;
734
- }
735
-
736
- return true;
737
- }
738
-
739
- WindowExecutor::WindowExecutor(BoundWindowExpression &wexpr, ClientContext &context, const ValidityMask &partition_mask,
740
- const idx_t count)
741
- : wexpr(wexpr), bounds(wexpr, count), payload_collection(), payload_executor(context), filter_executor(context),
742
- leadlag_offset(wexpr.offset_expr.get(), context), leadlag_default(wexpr.default_expr.get(), context),
743
- boundary_start(wexpr.start_expr.get(), context), boundary_end(wexpr.end_expr.get(), context),
744
- range((bounds.has_preceding_range || bounds.has_following_range) ? wexpr.orders[0].expression.get() : nullptr,
745
- context, count)
746
-
747
- {
748
- // TODO we could evaluate those expressions in parallel
749
-
750
- // Check for constant aggregate
751
- if (IsConstantAggregate(wexpr)) {
752
- constant_aggregate =
753
- make_uniq<WindowConstantAggregate>(AggregateObject(wexpr), wexpr.return_type, partition_mask, count);
754
- }
755
-
756
- // evaluate the FILTER clause and stuff it into a large mask for compactness and reuse
757
- if (wexpr.filter_expr) {
758
- // Start with all invalid and set the ones that pass
759
- filter_bits.resize(ValidityMask::ValidityMaskSize(count), 0);
760
- filter_mask.Initialize(filter_bits.data());
761
- filter_executor.AddExpression(*wexpr.filter_expr);
762
- filter_sel.Initialize(STANDARD_VECTOR_SIZE);
763
- }
764
-
765
- // TODO: child may be a scalar, don't need to materialize the whole collection then
766
-
767
- // evaluate inner expressions of window functions, could be more complex
768
- PrepareInputExpressions(wexpr.children, payload_executor, payload_chunk);
769
-
770
- auto types = payload_chunk.GetTypes();
771
- if (!types.empty()) {
772
- payload_collection.Initialize(Allocator::Get(context), types);
773
- }
774
- }
775
-
776
- void WindowExecutor::Sink(DataChunk &input_chunk, const idx_t input_idx, const idx_t total_count) {
777
- // Single pass over the input to produce the global data.
778
- // Vectorisation for the win...
779
-
780
- // Set up a validity mask for IGNORE NULLS
781
- bool check_nulls = false;
782
- if (wexpr.ignore_nulls) {
783
- switch (wexpr.type) {
784
- case ExpressionType::WINDOW_LEAD:
785
- case ExpressionType::WINDOW_LAG:
786
- case ExpressionType::WINDOW_FIRST_VALUE:
787
- case ExpressionType::WINDOW_LAST_VALUE:
788
- case ExpressionType::WINDOW_NTH_VALUE:
789
- check_nulls = true;
790
- break;
791
- default:
792
- break;
793
- }
794
- }
795
-
796
- const auto count = input_chunk.size();
797
-
798
- idx_t filtered = 0;
799
- SelectionVector *filtering = nullptr;
800
- if (wexpr.filter_expr) {
801
- filtering = &filter_sel;
802
- filtered = filter_executor.SelectExpression(input_chunk, filter_sel);
803
- for (idx_t f = 0; f < filtered; ++f) {
804
- filter_mask.SetValid(input_idx + filter_sel[f]);
805
- }
806
- }
807
-
808
- if (!wexpr.children.empty()) {
809
- payload_chunk.Reset();
810
- payload_executor.Execute(input_chunk, payload_chunk);
811
- payload_chunk.Verify();
812
- if (constant_aggregate) {
813
- constant_aggregate->Sink(payload_chunk, filtering, filtered);
814
- } else {
815
- payload_collection.Append(payload_chunk, true);
816
- }
817
-
818
- // process payload chunks while they are still piping hot
819
- if (check_nulls) {
820
- UnifiedVectorFormat vdata;
821
- payload_chunk.data[0].ToUnifiedFormat(count, vdata);
822
- if (!vdata.validity.AllValid()) {
823
- // Lazily materialise the contents when we find the first NULL
824
- if (ignore_nulls.AllValid()) {
825
- ignore_nulls.Initialize(total_count);
826
- }
827
- // Write to the current position
828
- if (input_idx % ValidityMask::BITS_PER_VALUE == 0) {
829
- // If we are at the edge of an output entry, just copy the entries
830
- auto dst = ignore_nulls.GetData() + ignore_nulls.EntryCount(input_idx);
831
- auto src = vdata.validity.GetData();
832
- for (auto entry_count = vdata.validity.EntryCount(count); entry_count-- > 0;) {
833
- *dst++ = *src++;
834
- }
835
- } else {
836
- // If not, we have ragged data and need to copy one bit at a time.
837
- for (idx_t i = 0; i < count; ++i) {
838
- ignore_nulls.Set(input_idx + i, vdata.validity.RowIsValid(i));
839
- }
840
- }
841
- }
842
- }
843
- }
844
-
845
- range.Append(input_chunk);
846
- }
847
-
848
- void WindowExecutor::Finalize(WindowAggregationMode mode) {
849
- // build a segment tree for frame-adhering aggregates
850
- // see http://www.vldb.org/pvldb/vol8/p1058-leis.pdf
851
- if (constant_aggregate) {
852
- constant_aggregate->Finalize();
853
- } else if (wexpr.aggregate) {
854
- segment_tree = make_uniq<WindowSegmentTree>(AggregateObject(wexpr), wexpr.return_type, &payload_collection,
855
- filter_mask, mode);
856
- }
857
- }
858
-
859
- void WindowExecutor::Evaluate(idx_t row_idx, DataChunk &input_chunk, Vector &result, const ValidityMask &partition_mask,
860
- const ValidityMask &order_mask) {
861
- // Evaluate the row-level arguments
862
- boundary_start.Execute(input_chunk);
863
- boundary_end.Execute(input_chunk);
864
-
865
- leadlag_offset.Execute(input_chunk);
866
- leadlag_default.Execute(input_chunk);
867
-
868
- // this is the main loop, go through all sorted rows and compute window function result
869
- for (idx_t output_offset = 0; output_offset < input_chunk.size(); ++output_offset, ++row_idx) {
870
- // special case, OVER (), aggregate over everything
871
- bounds.Update(row_idx, range, output_offset, boundary_start, boundary_end, partition_mask, order_mask);
872
- if (WindowNeedsRank(wexpr)) {
873
- if (!bounds.is_same_partition || row_idx == 0) { // special case for first row, need to init
874
- dense_rank = 1;
875
- rank = 1;
876
- rank_equal = 0;
877
- } else if (!bounds.is_peer) {
878
- dense_rank++;
879
- rank += rank_equal;
880
- rank_equal = 0;
881
- }
882
- rank_equal++;
883
- }
884
-
885
- // if no values are read for window, result is NULL
886
- if (bounds.window_start >= bounds.window_end) {
887
- FlatVector::SetNull(result, output_offset, true);
888
- continue;
889
- }
890
-
891
- switch (wexpr.type) {
892
- case ExpressionType::WINDOW_AGGREGATE: {
893
- if (constant_aggregate) {
894
- constant_aggregate->Compute(result, output_offset, bounds.window_start, bounds.window_end);
895
- } else {
896
- segment_tree->Compute(result, output_offset, bounds.window_start, bounds.window_end);
897
- }
898
- break;
899
- }
900
- case ExpressionType::WINDOW_ROW_NUMBER: {
901
- auto rdata = FlatVector::GetData<int64_t>(result);
902
- rdata[output_offset] = row_idx - bounds.partition_start + 1;
903
- break;
904
- }
905
- case ExpressionType::WINDOW_RANK_DENSE: {
906
- auto rdata = FlatVector::GetData<int64_t>(result);
907
- rdata[output_offset] = dense_rank;
908
- break;
909
- }
910
- case ExpressionType::WINDOW_RANK: {
911
- auto rdata = FlatVector::GetData<int64_t>(result);
912
- rdata[output_offset] = rank;
913
- break;
914
- }
915
- case ExpressionType::WINDOW_PERCENT_RANK: {
916
- int64_t denom = (int64_t)bounds.partition_end - bounds.partition_start - 1;
917
- double percent_rank = denom > 0 ? ((double)rank - 1) / denom : 0;
918
- auto rdata = FlatVector::GetData<double>(result);
919
- rdata[output_offset] = percent_rank;
920
- break;
921
- }
922
- case ExpressionType::WINDOW_CUME_DIST: {
923
- int64_t denom = (int64_t)bounds.partition_end - bounds.partition_start;
924
- double cume_dist = denom > 0 ? ((double)(bounds.peer_end - bounds.partition_start)) / denom : 0;
925
- auto rdata = FlatVector::GetData<double>(result);
926
- rdata[output_offset] = cume_dist;
927
- break;
928
- }
929
- case ExpressionType::WINDOW_NTILE: {
930
- D_ASSERT(payload_collection.ColumnCount() == 1);
931
- if (CellIsNull(payload_collection, 0, row_idx)) {
932
- FlatVector::SetNull(result, output_offset, true);
933
- } else {
934
- auto n_param = GetCell<int64_t>(payload_collection, 0, row_idx);
935
- if (n_param < 1) {
936
- throw InvalidInputException("Argument for ntile must be greater than zero");
937
- }
938
- // With thanks from SQLite's ntileValueFunc()
939
- int64_t n_total = bounds.partition_end - bounds.partition_start;
940
- if (n_param > n_total) {
941
- // more groups allowed than we have values
942
- // map every entry to a unique group
943
- n_param = n_total;
944
- }
945
- int64_t n_size = (n_total / n_param);
946
- // find the row idx within the group
947
- D_ASSERT(row_idx >= bounds.partition_start);
948
- int64_t adjusted_row_idx = row_idx - bounds.partition_start;
949
- // now compute the ntile
950
- int64_t n_large = n_total - n_param * n_size;
951
- int64_t i_small = n_large * (n_size + 1);
952
- int64_t result_ntile;
953
-
954
- D_ASSERT((n_large * (n_size + 1) + (n_param - n_large) * n_size) == n_total);
955
-
956
- if (adjusted_row_idx < i_small) {
957
- result_ntile = 1 + adjusted_row_idx / (n_size + 1);
958
- } else {
959
- result_ntile = 1 + n_large + (adjusted_row_idx - i_small) / n_size;
960
- }
961
- // result has to be between [1, NTILE]
962
- D_ASSERT(result_ntile >= 1 && result_ntile <= n_param);
963
- auto rdata = FlatVector::GetData<int64_t>(result);
964
- rdata[output_offset] = result_ntile;
965
- }
966
- break;
967
- }
968
- case ExpressionType::WINDOW_LEAD:
969
- case ExpressionType::WINDOW_LAG: {
970
- int64_t offset = 1;
971
- if (wexpr.offset_expr) {
972
- offset = leadlag_offset.GetCell<int64_t>(output_offset);
973
- }
974
- int64_t val_idx = (int64_t)row_idx;
975
- if (wexpr.type == ExpressionType::WINDOW_LEAD) {
976
- val_idx += offset;
977
- } else {
978
- val_idx -= offset;
979
- }
980
-
981
- idx_t delta = 0;
982
- if (val_idx < (int64_t)row_idx) {
983
- // Count backwards
984
- delta = idx_t(row_idx - val_idx);
985
- val_idx = FindPrevStart(ignore_nulls, bounds.partition_start, row_idx, delta);
986
- } else if (val_idx > (int64_t)row_idx) {
987
- delta = idx_t(val_idx - row_idx);
988
- val_idx = FindNextStart(ignore_nulls, row_idx + 1, bounds.partition_end, delta);
989
- }
990
- // else offset is zero, so don't move.
991
-
992
- if (!delta) {
993
- CopyCell(payload_collection, 0, val_idx, result, output_offset);
994
- } else if (wexpr.default_expr) {
995
- leadlag_default.CopyCell(result, output_offset);
996
- } else {
997
- FlatVector::SetNull(result, output_offset, true);
998
- }
999
- break;
1000
- }
1001
- case ExpressionType::WINDOW_FIRST_VALUE: {
1002
- // Same as NTH_VALUE(..., 1)
1003
- idx_t n = 1;
1004
- const auto first_idx = FindNextStart(ignore_nulls, bounds.window_start, bounds.window_end, n);
1005
- if (!n) {
1006
- CopyCell(payload_collection, 0, first_idx, result, output_offset);
1007
- } else {
1008
- FlatVector::SetNull(result, output_offset, true);
1009
- }
1010
- break;
1011
- }
1012
- case ExpressionType::WINDOW_LAST_VALUE: {
1013
- idx_t n = 1;
1014
- const auto last_idx = FindPrevStart(ignore_nulls, bounds.window_start, bounds.window_end, n);
1015
- if (!n) {
1016
- CopyCell(payload_collection, 0, last_idx, result, output_offset);
1017
- } else {
1018
- FlatVector::SetNull(result, output_offset, true);
1019
- }
1020
- break;
1021
- }
1022
- case ExpressionType::WINDOW_NTH_VALUE: {
1023
- D_ASSERT(payload_collection.ColumnCount() == 2);
1024
- // Returns value evaluated at the row that is the n'th row of the window frame (counting from 1);
1025
- // returns NULL if there is no such row.
1026
- if (CellIsNull(payload_collection, 1, row_idx)) {
1027
- FlatVector::SetNull(result, output_offset, true);
1028
- } else {
1029
- auto n_param = GetCell<int64_t>(payload_collection, 1, row_idx);
1030
- if (n_param < 1) {
1031
- FlatVector::SetNull(result, output_offset, true);
1032
- } else {
1033
- auto n = idx_t(n_param);
1034
- const auto nth_index = FindNextStart(ignore_nulls, bounds.window_start, bounds.window_end, n);
1035
- if (!n) {
1036
- CopyCell(payload_collection, 0, nth_index, result, output_offset);
1037
- } else {
1038
- FlatVector::SetNull(result, output_offset, true);
1039
- }
1040
- }
1041
- }
1042
- break;
1043
- }
1044
- default:
1045
- throw InternalException("Window aggregate type %s", ExpressionTypeToString(wexpr.type));
1046
- }
1047
- }
1048
-
1049
- result.Verify(input_chunk.size());
1050
- }
1051
-
1052
116
  //===--------------------------------------------------------------------===//
1053
117
  // Sink
1054
118
  //===--------------------------------------------------------------------===//
@@ -1060,9 +124,11 @@ SinkResultType PhysicalWindow::Sink(ExecutionContext &context, DataChunk &chunk,
1060
124
  return SinkResultType::NEED_MORE_INPUT;
1061
125
  }
1062
126
 
1063
- void PhysicalWindow::Combine(ExecutionContext &context, GlobalSinkState &gstate_p, LocalSinkState &lstate_p) const {
1064
- auto &lstate = lstate_p.Cast<WindowLocalSinkState>();
127
+ SinkCombineResultType PhysicalWindow::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
128
+ auto &lstate = input.local_state.Cast<WindowLocalSinkState>();
1065
129
  lstate.Combine();
130
+
131
+ return SinkCombineResultType::FINISHED;
1066
132
  }
1067
133
 
1068
134
  unique_ptr<LocalSinkState> PhysicalWindow::GetLocalSinkState(ExecutionContext &context) const {
@@ -1075,8 +141,8 @@ unique_ptr<GlobalSinkState> PhysicalWindow::GetGlobalSinkState(ClientContext &co
1075
141
  }
1076
142
 
1077
143
  SinkFinalizeType PhysicalWindow::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
1078
- GlobalSinkState &gstate_p) const {
1079
- auto &state = gstate_p.Cast<WindowGlobalSinkState>();
144
+ OperatorSinkFinalizeInput &input) const {
145
+ auto &state = input.global_state.Cast<WindowGlobalSinkState>();
1080
146
 
1081
147
  // Did we get any data?
1082
148
  if (!state.global_partition->count) {
@@ -1090,8 +156,7 @@ SinkFinalizeType PhysicalWindow::Finalize(Pipeline &pipeline, Event &event, Clie
1090
156
  }
1091
157
 
1092
158
  // Find the first group to sort
1093
- auto &groups = state.global_partition->grouping_data->GetPartitions();
1094
- if (groups.empty()) {
159
+ if (!state.global_partition->HasMergeTasks()) {
1095
160
  // Empty input!
1096
161
  return SinkFinalizeType::NO_OUTPUT_POSSIBLE;
1097
162
  }
@@ -1106,64 +171,97 @@ SinkFinalizeType PhysicalWindow::Finalize(Pipeline &pipeline, Event &event, Clie
1106
171
  //===--------------------------------------------------------------------===//
1107
172
  // Source
1108
173
  //===--------------------------------------------------------------------===//
174
+ class WindowPartitionSourceState;
175
+
1109
176
  class WindowGlobalSourceState : public GlobalSourceState {
1110
177
  public:
1111
- explicit WindowGlobalSourceState(WindowGlobalSinkState &gsink) : gsink(*gsink.global_partition), next_bin(0) {
1112
- }
178
+ using HashGroupSourcePtr = unique_ptr<WindowPartitionSourceState>;
179
+ using ScannerPtr = unique_ptr<RowDataCollectionScanner>;
180
+ using Task = std::pair<WindowPartitionSourceState *, ScannerPtr>;
1113
181
 
1114
- PartitionGlobalSinkState &gsink;
1115
- //! The output read position.
1116
- atomic<idx_t> next_bin;
182
+ WindowGlobalSourceState(ClientContext &context_p, WindowGlobalSinkState &gsink_p);
183
+
184
+ //! Get the next task
185
+ Task NextTask(idx_t hash_bin);
186
+
187
+ //! Context for executing computations
188
+ ClientContext &context;
189
+ //! All the sunk data
190
+ WindowGlobalSinkState &gsink;
191
+ //! The next group to build.
192
+ atomic<idx_t> next_build;
193
+ //! The built groups
194
+ vector<HashGroupSourcePtr> built;
195
+ //! Serialise access to the built hash groups
196
+ mutable mutex built_lock;
197
+ //! The number of unfinished tasks
198
+ atomic<idx_t> tasks_remaining;
1117
199
 
1118
200
  public:
1119
201
  idx_t MaxThreads() override {
1120
- // If there is only one partition, we have to process it on one thread.
1121
- if (!gsink.grouping_data) {
1122
- return 1;
1123
- }
202
+ return tasks_remaining;
203
+ }
1124
204
 
1125
- // If there is not a lot of data, process serially.
1126
- if (gsink.count < STANDARD_ROW_GROUPS_SIZE) {
1127
- return 1;
205
+ private:
206
+ Task CreateTask(idx_t hash_bin);
207
+ Task StealWork();
208
+ };
209
+
210
+ WindowGlobalSourceState::WindowGlobalSourceState(ClientContext &context_p, WindowGlobalSinkState &gsink_p)
211
+ : context(context_p), gsink(gsink_p), next_build(0), tasks_remaining(0) {
212
+ auto &hash_groups = gsink.global_partition->hash_groups;
213
+
214
+ auto &gpart = gsink.global_partition;
215
+ if (hash_groups.empty()) {
216
+ // OVER()
217
+ built.resize(1);
218
+ if (gpart->rows) {
219
+ tasks_remaining += gpart->rows->blocks.size();
1128
220
  }
221
+ } else {
222
+ built.resize(hash_groups.size());
223
+ idx_t batch_base = 0;
224
+ for (auto &hash_group : hash_groups) {
225
+ if (!hash_group) {
226
+ continue;
227
+ }
228
+ auto &global_sort_state = *hash_group->global_sort;
229
+ if (global_sort_state.sorted_blocks.empty()) {
230
+ continue;
231
+ }
232
+
233
+ D_ASSERT(global_sort_state.sorted_blocks.size() == 1);
234
+ auto &sb = *global_sort_state.sorted_blocks[0];
235
+ auto &sd = *sb.payload_data;
236
+ tasks_remaining += sd.data_blocks.size();
1129
237
 
1130
- return gsink.hash_groups.size();
238
+ hash_group->batch_base = batch_base;
239
+ batch_base += sd.data_blocks.size();
240
+ }
1131
241
  }
1132
- };
242
+ }
1133
243
 
1134
- // Per-thread read state
1135
- class WindowLocalSourceState : public LocalSourceState {
244
+ // Per-bin evaluation state (build and evaluate)
245
+ class WindowPartitionSourceState {
1136
246
  public:
1137
247
  using HashGroupPtr = unique_ptr<PartitionGlobalHashGroup>;
1138
- using WindowExecutorPtr = unique_ptr<WindowExecutor>;
1139
- using WindowExecutors = vector<WindowExecutorPtr>;
248
+ using ExecutorPtr = unique_ptr<WindowExecutor>;
249
+ using Executors = vector<ExecutorPtr>;
1140
250
 
1141
- WindowLocalSourceState(const PhysicalWindow &op_p, ExecutionContext &context, WindowGlobalSourceState &gsource)
1142
- : context(context.client), op(op_p), gsink(gsource.gsink) {
1143
-
1144
- vector<LogicalType> output_types;
1145
- for (idx_t expr_idx = 0; expr_idx < op.select_list.size(); ++expr_idx) {
1146
- D_ASSERT(op.select_list[expr_idx]->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);
1147
- auto &wexpr = op.select_list[expr_idx]->Cast<BoundWindowExpression>();
1148
- output_types.emplace_back(wexpr.return_type);
1149
- }
1150
- output_chunk.Initialize(Allocator::Get(context.client), output_types);
1151
-
1152
- const auto &input_types = gsink.payload_types;
1153
- layout.Initialize(input_types);
1154
- input_chunk.Initialize(gsink.allocator, input_types);
251
+ WindowPartitionSourceState(ClientContext &context, WindowGlobalSourceState &gsource)
252
+ : context(context), op(gsource.gsink.op), gsource(gsource), read_block_idx(0), unscanned(0) {
253
+ layout.Initialize(gsource.gsink.global_partition->payload_types);
1155
254
  }
1156
255
 
256
+ unique_ptr<RowDataCollectionScanner> GetScanner() const;
1157
257
  void MaterializeSortedData();
1158
- void GeneratePartition(WindowGlobalSinkState &gstate, const idx_t hash_bin);
1159
- void Scan(DataChunk &chunk);
258
+ void BuildPartition(WindowGlobalSinkState &gstate, const idx_t hash_bin);
1160
259
 
1161
- HashGroupPtr hash_group;
1162
260
  ClientContext &context;
1163
261
  const PhysicalWindow &op;
262
+ WindowGlobalSourceState &gsource;
1164
263
 
1165
- PartitionGlobalSinkState &gsink;
1166
-
264
+ HashGroupPtr hash_group;
1167
265
  //! The generated input chunks
1168
266
  unique_ptr<RowDataCollection> rows;
1169
267
  unique_ptr<RowDataCollection> heap;
@@ -1174,20 +272,21 @@ public:
1174
272
  //! The order boundary mask
1175
273
  vector<validity_t> order_bits;
1176
274
  ValidityMask order_mask;
275
+ //! External paging
276
+ bool external;
1177
277
  //! The current execution functions
1178
- WindowExecutors window_execs;
278
+ Executors executors;
1179
279
 
1180
- //! The read partition
280
+ //! The bin number
1181
281
  idx_t hash_bin;
1182
- //! The read cursor
1183
- unique_ptr<RowDataCollectionScanner> scanner;
1184
- //! Buffer for the inputs
1185
- DataChunk input_chunk;
1186
- //! Buffer for window results
1187
- DataChunk output_chunk;
282
+
283
+ //! The next block to read.
284
+ mutable atomic<idx_t> read_block_idx;
285
+ //! The number of remaining unscanned blocks.
286
+ atomic<idx_t> unscanned;
1188
287
  };
1189
288
 
1190
- void WindowLocalSourceState::MaterializeSortedData() {
289
+ void WindowPartitionSourceState::MaterializeSortedData() {
1191
290
  auto &global_sort_state = *hash_group->global_sort;
1192
291
  if (global_sort_state.sorted_blocks.empty()) {
1193
292
  return;
@@ -1226,7 +325,21 @@ void WindowLocalSourceState::MaterializeSortedData() {
1226
325
  [&](idx_t c, const unique_ptr<RowDataBlock> &b) { return c + b->count; });
1227
326
  }
1228
327
 
1229
- void WindowLocalSourceState::GeneratePartition(WindowGlobalSinkState &gstate, const idx_t hash_bin_p) {
328
+ unique_ptr<RowDataCollectionScanner> WindowPartitionSourceState::GetScanner() const {
329
+ auto &gsink = *gsource.gsink.global_partition;
330
+ if ((gsink.rows && !hash_bin) || hash_bin < gsink.hash_groups.size()) {
331
+ const auto block_idx = read_block_idx++;
332
+ if (block_idx >= rows->blocks.size()) {
333
+ return nullptr;
334
+ }
335
+ // Second pass can flush
336
+ --gsource.tasks_remaining;
337
+ return make_uniq<RowDataCollectionScanner>(*rows, *heap, layout, external, block_idx, true);
338
+ }
339
+ return nullptr;
340
+ }
341
+
342
+ void WindowPartitionSourceState::BuildPartition(WindowGlobalSinkState &gstate, const idx_t hash_bin_p) {
1230
343
  // Get rid of any stale data
1231
344
  hash_bin = hash_bin_p;
1232
345
 
@@ -1236,11 +349,12 @@ void WindowLocalSourceState::GeneratePartition(WindowGlobalSinkState &gstate, co
1236
349
  // 3. Multiple partitions (sorting and hashing)
1237
350
 
1238
351
  // How big is the partition?
352
+ auto &gpart = *gsource.gsink.global_partition;
1239
353
  idx_t count = 0;
1240
- if (hash_bin < gsink.hash_groups.size() && gsink.hash_groups[hash_bin]) {
1241
- count = gsink.hash_groups[hash_bin]->count;
1242
- } else if (gsink.rows && !hash_bin) {
1243
- count = gsink.count;
354
+ if (hash_bin < gpart.hash_groups.size() && gpart.hash_groups[hash_bin]) {
355
+ count = gpart.hash_groups[hash_bin]->count;
356
+ } else if (gpart.rows && !hash_bin) {
357
+ count = gpart.count;
1244
358
  } else {
1245
359
  return;
1246
360
  }
@@ -1256,19 +370,20 @@ void WindowLocalSourceState::GeneratePartition(WindowGlobalSinkState &gstate, co
1256
370
  order_mask.Initialize(order_bits.data());
1257
371
 
1258
372
  // Scan the sorted data into new Collections
1259
- auto external = gsink.external;
1260
- if (gsink.rows && !hash_bin) {
373
+ external = gpart.external;
374
+ if (gpart.rows && !hash_bin) {
1261
375
  // Simple mask
1262
376
  partition_mask.SetValidUnsafe(0);
1263
377
  order_mask.SetValidUnsafe(0);
1264
378
  // No partition - align the heap blocks with the row blocks
1265
- rows = gsink.rows->CloneEmpty(gsink.rows->keep_pinned);
1266
- heap = gsink.strings->CloneEmpty(gsink.strings->keep_pinned);
1267
- RowDataCollectionScanner::AlignHeapBlocks(*rows, *heap, *gsink.rows, *gsink.strings, layout);
379
+ rows = gpart.rows->CloneEmpty(gpart.rows->keep_pinned);
380
+ heap = gpart.strings->CloneEmpty(gpart.strings->keep_pinned);
381
+ RowDataCollectionScanner::AlignHeapBlocks(*rows, *heap, *gpart.rows, *gpart.strings, layout);
1268
382
  external = true;
1269
- } else if (hash_bin < gsink.hash_groups.size() && gsink.hash_groups[hash_bin]) {
383
+ } else if (hash_bin < gpart.hash_groups.size()) {
1270
384
  // Overwrite the collections with the sorted data
1271
- hash_group = std::move(gsink.hash_groups[hash_bin]);
385
+ D_ASSERT(gpart.hash_groups[hash_bin].get());
386
+ hash_group = std::move(gpart.hash_groups[hash_bin]);
1272
387
  hash_group->ComputeMasks(partition_mask, order_mask);
1273
388
  external = hash_group->global_sort->external;
1274
389
  MaterializeSortedData();
@@ -1277,17 +392,18 @@ void WindowLocalSourceState::GeneratePartition(WindowGlobalSinkState &gstate, co
1277
392
  }
1278
393
 
1279
394
  // Create the executors for each function
1280
- window_execs.clear();
395
+ executors.clear();
1281
396
  for (idx_t expr_idx = 0; expr_idx < op.select_list.size(); ++expr_idx) {
1282
397
  D_ASSERT(op.select_list[expr_idx]->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);
1283
398
  auto &wexpr = op.select_list[expr_idx]->Cast<BoundWindowExpression>();
1284
- auto wexec = make_uniq<WindowExecutor>(wexpr, context, partition_mask, count);
1285
- window_execs.emplace_back(std::move(wexec));
399
+ auto wexec = WindowExecutorFactory(wexpr, context, partition_mask, order_mask, count, gstate.mode);
400
+ executors.emplace_back(std::move(wexec));
1286
401
  }
1287
402
 
1288
403
  // First pass over the input without flushing
1289
- // TODO: Factor out the constructor data as global state
1290
- scanner = make_uniq<RowDataCollectionScanner>(*rows, *heap, layout, external, false);
404
+ DataChunk input_chunk;
405
+ input_chunk.Initialize(gpart.allocator, gpart.payload_types);
406
+ auto scanner = make_uniq<RowDataCollectionScanner>(*rows, *heap, layout, external, false);
1291
407
  idx_t input_idx = 0;
1292
408
  while (true) {
1293
409
  input_chunk.Reset();
@@ -1297,38 +413,221 @@ void WindowLocalSourceState::GeneratePartition(WindowGlobalSinkState &gstate, co
1297
413
  }
1298
414
 
1299
415
  // TODO: Parallelization opportunity
1300
- for (auto &wexec : window_execs) {
416
+ for (auto &wexec : executors) {
1301
417
  wexec->Sink(input_chunk, input_idx, scanner->Count());
1302
418
  }
1303
419
  input_idx += input_chunk.size();
1304
420
  }
1305
421
 
1306
422
  // TODO: Parallelization opportunity
1307
- for (auto &wexec : window_execs) {
1308
- wexec->Finalize(gstate.mode);
423
+ for (auto &wexec : executors) {
424
+ wexec->Finalize();
1309
425
  }
1310
426
 
1311
427
  // External scanning assumes all blocks are swizzled.
1312
428
  scanner->ReSwizzle();
1313
429
 
1314
- // Second pass can flush
1315
- scanner->Reset(true);
430
+ // Start the block countdown
431
+ unscanned = rows->blocks.size();
432
+ }
433
+
434
+ // Per-thread scan state
435
+ class WindowLocalSourceState : public LocalSourceState {
436
+ public:
437
+ using ReadStatePtr = unique_ptr<WindowExecutorState>;
438
+ using ReadStates = vector<ReadStatePtr>;
439
+
440
+ explicit WindowLocalSourceState(WindowGlobalSourceState &gsource);
441
+ void UpdateBatchIndex();
442
+ bool NextPartition();
443
+ void Scan(DataChunk &chunk);
444
+
445
+ //! The shared source state
446
+ WindowGlobalSourceState &gsource;
447
+ //! The current bin being processed
448
+ idx_t hash_bin;
449
+ //! The current batch index (for output reordering)
450
+ idx_t batch_index;
451
+ //! The current source being processed
452
+ optional_ptr<WindowPartitionSourceState> partition_source;
453
+ //! The read cursor
454
+ unique_ptr<RowDataCollectionScanner> scanner;
455
+ //! Buffer for the inputs
456
+ DataChunk input_chunk;
457
+ //! Executor read states.
458
+ ReadStates read_states;
459
+ //! Buffer for window results
460
+ DataChunk output_chunk;
461
+ };
462
+
463
+ WindowLocalSourceState::WindowLocalSourceState(WindowGlobalSourceState &gsource)
464
+ : gsource(gsource), hash_bin(gsource.built.size()), batch_index(0) {
465
+ auto &gsink = *gsource.gsink.global_partition;
466
+ auto &op = gsource.gsink.op;
467
+
468
+ input_chunk.Initialize(gsink.allocator, gsink.payload_types);
469
+
470
+ vector<LogicalType> output_types;
471
+ for (idx_t expr_idx = 0; expr_idx < op.select_list.size(); ++expr_idx) {
472
+ D_ASSERT(op.select_list[expr_idx]->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);
473
+ auto &wexpr = op.select_list[expr_idx]->Cast<BoundWindowExpression>();
474
+ output_types.emplace_back(wexpr.return_type);
475
+ }
476
+ output_chunk.Initialize(Allocator::Get(gsource.context), output_types);
477
+ }
478
+
479
+ WindowGlobalSourceState::Task WindowGlobalSourceState::CreateTask(idx_t hash_bin) {
480
+ // Build outside the lock so no one tries to steal before we are done.
481
+ auto partition_source = make_uniq<WindowPartitionSourceState>(context, *this);
482
+ partition_source->BuildPartition(gsink, hash_bin);
483
+ Task result(partition_source.get(), partition_source->GetScanner());
484
+
485
+ // Is there any data to scan?
486
+ if (result.second) {
487
+ lock_guard<mutex> built_guard(built_lock);
488
+ built[hash_bin] = std::move(partition_source);
489
+
490
+ return result;
491
+ }
492
+
493
+ return Task();
494
+ }
495
+
496
+ WindowGlobalSourceState::Task WindowGlobalSourceState::StealWork() {
497
+ for (idx_t hash_bin = 0; hash_bin < built.size(); ++hash_bin) {
498
+ lock_guard<mutex> built_guard(built_lock);
499
+ auto &partition_source = built[hash_bin];
500
+ if (!partition_source) {
501
+ continue;
502
+ }
503
+
504
+ Task result(partition_source.get(), partition_source->GetScanner());
505
+
506
+ // Is there any data to scan?
507
+ if (result.second) {
508
+ return result;
509
+ }
510
+ }
511
+
512
+ // Nothing to steal
513
+ return Task();
514
+ }
515
+
516
+ WindowGlobalSourceState::Task WindowGlobalSourceState::NextTask(idx_t hash_bin) {
517
+ auto &hash_groups = gsink.global_partition->hash_groups;
518
+ const auto bin_count = built.size();
519
+
520
+ // Flush unneeded data
521
+ if (hash_bin < bin_count) {
522
+ // Lock and delete when all blocks have been scanned
523
+ // We do this here instead of in NextScan so the WindowLocalSourceState
524
+ // has a chance to delete its state objects first,
525
+ // which may reference the partition_source
526
+
527
+ // Delete data outside the lock in case it is slow
528
+ HashGroupSourcePtr killed;
529
+ lock_guard<mutex> built_guard(built_lock);
530
+ auto &partition_source = built[hash_bin];
531
+ if (partition_source && !partition_source->unscanned) {
532
+ killed = std::move(partition_source);
533
+ }
534
+ }
535
+
536
+ hash_bin = next_build++;
537
+ if (hash_bin < bin_count) {
538
+ // Find a non-empty hash group.
539
+ for (; hash_bin < hash_groups.size(); hash_bin = next_build++) {
540
+ if (hash_groups[hash_bin] && hash_groups[hash_bin]->count) {
541
+ auto result = CreateTask(hash_bin);
542
+ if (result.second) {
543
+ return result;
544
+ }
545
+ }
546
+ }
547
+
548
+ // OVER() doesn't have a hash_group
549
+ if (hash_groups.empty()) {
550
+ auto result = CreateTask(hash_bin);
551
+ if (result.second) {
552
+ return result;
553
+ }
554
+ }
555
+ }
556
+
557
+ // Work stealing
558
+ while (!context.interrupted && tasks_remaining) {
559
+ auto result = StealWork();
560
+ if (result.second) {
561
+ return result;
562
+ }
563
+
564
+ // If there is nothing to steal but there are unfinished partitions,
565
+ // yield until any pending builds are done.
566
+ TaskScheduler::YieldThread();
567
+ }
568
+
569
+ return Task();
570
+ }
571
+
572
+ void WindowLocalSourceState::UpdateBatchIndex() {
573
+ D_ASSERT(partition_source);
574
+ D_ASSERT(scanner.get());
575
+
576
+ batch_index = partition_source->hash_group ? partition_source->hash_group->batch_base : 0;
577
+ batch_index += scanner->BlockIndex();
578
+ }
579
+
580
+ bool WindowLocalSourceState::NextPartition() {
581
+ // Release old states before the source
582
+ scanner.reset();
583
+ read_states.clear();
584
+
585
+ // Get a partition_source that is not finished
586
+ while (!scanner) {
587
+ auto task = gsource.NextTask(hash_bin);
588
+ if (!task.first) {
589
+ return false;
590
+ }
591
+ partition_source = task.first;
592
+ scanner = std::move(task.second);
593
+ hash_bin = partition_source->hash_bin;
594
+ UpdateBatchIndex();
595
+ }
596
+
597
+ for (auto &wexec : partition_source->executors) {
598
+ read_states.emplace_back(wexec->GetExecutorState());
599
+ }
600
+
601
+ return true;
1316
602
  }
1317
603
 
1318
604
  void WindowLocalSourceState::Scan(DataChunk &result) {
1319
605
  D_ASSERT(scanner);
1320
606
  if (!scanner->Remaining()) {
1321
- return;
607
+ lock_guard<mutex> built_guard(gsource.built_lock);
608
+ --partition_source->unscanned;
609
+ scanner = partition_source->GetScanner();
610
+
611
+ if (!scanner) {
612
+ partition_source = nullptr;
613
+ read_states.clear();
614
+ return;
615
+ }
616
+
617
+ UpdateBatchIndex();
1322
618
  }
1323
619
 
1324
620
  const auto position = scanner->Scanned();
1325
621
  input_chunk.Reset();
1326
622
  scanner->Scan(input_chunk);
1327
623
 
624
+ auto &executors = partition_source->executors;
1328
625
  output_chunk.Reset();
1329
- for (idx_t expr_idx = 0; expr_idx < window_execs.size(); ++expr_idx) {
1330
- auto &executor = *window_execs[expr_idx];
1331
- executor.Evaluate(position, input_chunk, output_chunk.data[expr_idx], partition_mask, order_mask);
626
+ for (idx_t expr_idx = 0; expr_idx < executors.size(); ++expr_idx) {
627
+ auto &executor = *executors[expr_idx];
628
+ auto &lstate = *read_states[expr_idx];
629
+ auto &result = output_chunk.data[expr_idx];
630
+ executor.Evaluate(position, input_chunk, result, lstate);
1332
631
  }
1333
632
  output_chunk.SetCardinality(input_chunk);
1334
633
  output_chunk.Verify();
@@ -1345,43 +644,42 @@ void WindowLocalSourceState::Scan(DataChunk &result) {
1345
644
  }
1346
645
 
1347
646
  unique_ptr<LocalSourceState> PhysicalWindow::GetLocalSourceState(ExecutionContext &context,
1348
- GlobalSourceState &gstate_p) const {
1349
- auto &gstate = gstate_p.Cast<WindowGlobalSourceState>();
1350
- return make_uniq<WindowLocalSourceState>(*this, context, gstate);
647
+ GlobalSourceState &gsource_p) const {
648
+ auto &gsource = gsource_p.Cast<WindowGlobalSourceState>();
649
+ return make_uniq<WindowLocalSourceState>(gsource);
1351
650
  }
1352
651
 
1353
652
  unique_ptr<GlobalSourceState> PhysicalWindow::GetGlobalSourceState(ClientContext &context) const {
1354
653
  auto &gsink = sink_state->Cast<WindowGlobalSinkState>();
1355
- return make_uniq<WindowGlobalSourceState>(gsink);
654
+ return make_uniq<WindowGlobalSourceState>(context, gsink);
655
+ }
656
+
657
+ bool PhysicalWindow::SupportsBatchIndex() const {
658
+ // We can only preserve order for single partitioning
659
+ // or work stealing causes out of order batch numbers
660
+ auto &wexpr = select_list[0]->Cast<BoundWindowExpression>();
661
+ return wexpr.partitions.empty() && !wexpr.orders.empty();
662
+ }
663
+
664
+ OrderPreservationType PhysicalWindow::SourceOrder() const {
665
+ return SupportsBatchIndex() ? OrderPreservationType::FIXED_ORDER : OrderPreservationType::NO_ORDER;
666
+ }
667
+
668
+ idx_t PhysicalWindow::GetBatchIndex(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate_p,
669
+ LocalSourceState &lstate_p) const {
670
+ auto &lstate = lstate_p.Cast<WindowLocalSourceState>();
671
+ return lstate.batch_index;
1356
672
  }
1357
673
 
1358
674
  SourceResultType PhysicalWindow::GetData(ExecutionContext &context, DataChunk &chunk,
1359
675
  OperatorSourceInput &input) const {
1360
676
  auto &lsource = input.local_state.Cast<WindowLocalSourceState>();
1361
- auto &gsource = input.global_state.Cast<WindowGlobalSourceState>();
1362
- auto &gsink = sink_state->Cast<WindowGlobalSinkState>();
1363
-
1364
- auto &hash_groups = gsink.global_partition->hash_groups;
1365
- const auto bin_count = hash_groups.empty() ? 1 : hash_groups.size();
1366
-
1367
677
  while (chunk.size() == 0) {
1368
678
  // Move to the next bin if we are done.
1369
- while (!lsource.scanner || !lsource.scanner->Remaining()) {
1370
- lsource.scanner.reset();
1371
- lsource.rows.reset();
1372
- lsource.heap.reset();
1373
- lsource.hash_group.reset();
1374
- auto hash_bin = gsource.next_bin++;
1375
- if (hash_bin >= bin_count) {
679
+ while (!lsource.scanner) {
680
+ if (!lsource.NextPartition()) {
1376
681
  return chunk.size() > 0 ? SourceResultType::HAVE_MORE_OUTPUT : SourceResultType::FINISHED;
1377
682
  }
1378
-
1379
- for (; hash_bin < hash_groups.size(); hash_bin = gsource.next_bin++) {
1380
- if (hash_groups[hash_bin]) {
1381
- break;
1382
- }
1383
- }
1384
- lsource.GeneratePartition(gsink, hash_bin);
1385
683
  }
1386
684
 
1387
685
  lsource.Scan(chunk);