duckdb 0.8.2-dev37.0 → 0.8.2-dev3989.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1087) hide show
  1. package/README.md +7 -0
  2. package/binding.gyp +29 -13
  3. package/binding.gyp.in +1 -1
  4. package/configure.py +11 -3
  5. package/duckdb_extension_config.cmake +10 -0
  6. package/package.json +1 -1
  7. package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
  8. package/src/duckdb/extension/icu/icu-datefunc.cpp +10 -1
  9. package/src/duckdb/extension/icu/icu-datepart.cpp +162 -41
  10. package/src/duckdb/extension/icu/icu-datesub.cpp +3 -2
  11. package/src/duckdb/extension/icu/icu-datetrunc.cpp +2 -1
  12. package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
  13. package/src/duckdb/extension/icu/icu-makedate.cpp +19 -6
  14. package/src/duckdb/extension/icu/icu-strptime.cpp +5 -24
  15. package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
  16. package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
  17. package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
  18. package/src/duckdb/extension/icu/icu_extension.cpp +10 -12
  19. package/src/duckdb/extension/json/buffered_json_reader.cpp +2 -0
  20. package/src/duckdb/extension/json/include/buffered_json_reader.hpp +5 -19
  21. package/src/duckdb/extension/json/include/json_common.hpp +47 -231
  22. package/src/duckdb/extension/json/include/json_deserializer.hpp +7 -16
  23. package/src/duckdb/extension/json/include/json_enums.hpp +60 -0
  24. package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
  25. package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
  26. package/src/duckdb/extension/json/include/json_scan.hpp +14 -10
  27. package/src/duckdb/extension/json/include/json_serializer.hpp +9 -15
  28. package/src/duckdb/extension/json/include/json_transform.hpp +3 -0
  29. package/src/duckdb/extension/json/json_common.cpp +272 -40
  30. package/src/duckdb/extension/json/json_deserializer.cpp +37 -73
  31. package/src/duckdb/extension/json/json_enums.cpp +105 -0
  32. package/src/duckdb/extension/json/json_functions/json_create.cpp +21 -2
  33. package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
  34. package/src/duckdb/extension/json/json_functions/json_transform.cpp +93 -38
  35. package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
  36. package/src/duckdb/extension/json/json_functions.cpp +26 -25
  37. package/src/duckdb/extension/json/json_scan.cpp +47 -6
  38. package/src/duckdb/extension/json/json_serializer.cpp +29 -72
  39. package/src/duckdb/extension/json/serialize_json.cpp +92 -0
  40. package/src/duckdb/extension/parquet/column_reader.cpp +37 -25
  41. package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
  42. package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
  43. package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
  44. package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
  45. package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
  46. package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
  47. package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
  48. package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
  49. package/src/duckdb/extension/parquet/include/parquet_reader.hpp +4 -0
  50. package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
  51. package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
  52. package/src/duckdb/extension/parquet/include/parquet_timestamp.hpp +1 -0
  53. package/src/duckdb/extension/parquet/include/parquet_writer.hpp +28 -5
  54. package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
  55. package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
  56. package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
  57. package/src/duckdb/extension/parquet/parquet_extension.cpp +258 -40
  58. package/src/duckdb/extension/parquet/parquet_reader.cpp +10 -10
  59. package/src/duckdb/extension/parquet/parquet_statistics.cpp +25 -8
  60. package/src/duckdb/extension/parquet/parquet_timestamp.cpp +6 -0
  61. package/src/duckdb/extension/parquet/parquet_writer.cpp +149 -31
  62. package/src/duckdb/extension/parquet/serialize_parquet.cpp +26 -0
  63. package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
  64. package/src/duckdb/src/catalog/catalog.cpp +147 -70
  65. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +8 -11
  66. package/src/duckdb/src/catalog/catalog_entry/index_catalog_entry.cpp +17 -41
  67. package/src/duckdb/src/catalog/catalog_entry/macro_catalog_entry.cpp +2 -10
  68. package/src/duckdb/src/catalog/catalog_entry/schema_catalog_entry.cpp +4 -14
  69. package/src/duckdb/src/catalog/catalog_entry/sequence_catalog_entry.cpp +11 -28
  70. package/src/duckdb/src/catalog/catalog_entry/table_catalog_entry.cpp +11 -42
  71. package/src/duckdb/src/catalog/catalog_entry/type_catalog_entry.cpp +7 -26
  72. package/src/duckdb/src/catalog/catalog_entry/view_catalog_entry.cpp +11 -27
  73. package/src/duckdb/src/catalog/catalog_entry.cpp +25 -1
  74. package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
  75. package/src/duckdb/src/catalog/catalog_set.cpp +0 -63
  76. package/src/duckdb/src/catalog/default/default_functions.cpp +21 -0
  77. package/src/duckdb/src/catalog/dependency_manager.cpp +0 -36
  78. package/src/duckdb/src/common/adbc/adbc.cpp +541 -171
  79. package/src/duckdb/src/common/adbc/driver_manager.cpp +92 -39
  80. package/src/duckdb/src/common/adbc/nanoarrow/allocator.cpp +57 -0
  81. package/src/duckdb/src/common/adbc/nanoarrow/metadata.cpp +121 -0
  82. package/src/duckdb/src/common/adbc/nanoarrow/schema.cpp +474 -0
  83. package/src/duckdb/src/common/adbc/nanoarrow/single_batch_array_stream.cpp +84 -0
  84. package/src/duckdb/src/common/allocator.cpp +14 -2
  85. package/src/duckdb/src/common/arrow/appender/bool_data.cpp +44 -0
  86. package/src/duckdb/src/common/arrow/appender/list_data.cpp +78 -0
  87. package/src/duckdb/src/common/arrow/appender/map_data.cpp +86 -0
  88. package/src/duckdb/src/common/arrow/appender/struct_data.cpp +45 -0
  89. package/src/duckdb/src/common/arrow/appender/union_data.cpp +70 -0
  90. package/src/duckdb/src/common/arrow/arrow_appender.cpp +95 -666
  91. package/src/duckdb/src/common/arrow/arrow_converter.cpp +65 -37
  92. package/src/duckdb/src/common/arrow/arrow_wrapper.cpp +37 -42
  93. package/src/duckdb/src/common/assert.cpp +3 -0
  94. package/src/duckdb/src/common/constants.cpp +2 -1
  95. package/src/duckdb/src/common/enum_util.cpp +4979 -4458
  96. package/src/duckdb/src/common/enums/date_part_specifier.cpp +2 -0
  97. package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
  98. package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
  99. package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
  100. package/src/duckdb/src/common/exception.cpp +15 -2
  101. package/src/duckdb/src/common/extra_type_info.cpp +487 -0
  102. package/src/duckdb/src/common/field_writer.cpp +1 -1
  103. package/src/duckdb/src/common/file_buffer.cpp +1 -1
  104. package/src/duckdb/src/common/file_system.cpp +46 -12
  105. package/src/duckdb/src/common/filename_pattern.cpp +1 -1
  106. package/src/duckdb/src/common/gzip_file_system.cpp +7 -12
  107. package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
  108. package/src/duckdb/src/common/http_state.cpp +78 -0
  109. package/src/duckdb/src/common/local_file_system.cpp +36 -28
  110. package/src/duckdb/src/common/multi_file_reader.cpp +193 -20
  111. package/src/duckdb/src/common/operator/cast_operators.cpp +92 -1
  112. package/src/duckdb/src/common/operator/string_cast.cpp +45 -8
  113. package/src/duckdb/src/common/radix_partitioning.cpp +34 -39
  114. package/src/duckdb/src/common/re2_regex.cpp +1 -1
  115. package/src/duckdb/src/common/row_operations/row_aggregate.cpp +18 -3
  116. package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
  117. package/src/duckdb/src/common/serializer/binary_deserializer.cpp +63 -73
  118. package/src/duckdb/src/common/serializer/binary_serializer.cpp +85 -80
  119. package/src/duckdb/src/common/serializer/buffered_file_reader.cpp +0 -9
  120. package/src/duckdb/src/common/serializer/format_serializer.cpp +15 -0
  121. package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
  122. package/src/duckdb/src/common/sort/partition_state.cpp +102 -74
  123. package/src/duckdb/src/common/sort/sort_state.cpp +1 -1
  124. package/src/duckdb/src/common/sort/sorted_block.cpp +1 -1
  125. package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
  126. package/src/duckdb/src/common/types/bit.cpp +51 -0
  127. package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
  128. package/src/duckdb/src/common/types/column/column_data_collection.cpp +68 -2
  129. package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +20 -6
  130. package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
  131. package/src/duckdb/src/common/types/data_chunk.cpp +46 -10
  132. package/src/duckdb/src/common/types/date.cpp +15 -0
  133. package/src/duckdb/src/common/types/hugeint.cpp +40 -0
  134. package/src/duckdb/src/common/types/hyperloglog.cpp +21 -0
  135. package/src/duckdb/src/common/types/interval.cpp +6 -0
  136. package/src/duckdb/src/common/types/list_segment.cpp +56 -198
  137. package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +251 -131
  138. package/src/duckdb/src/common/types/row/row_data_collection_scanner.cpp +35 -5
  139. package/src/duckdb/src/common/types/row/row_layout.cpp +3 -31
  140. package/src/duckdb/src/common/types/row/tuple_data_allocator.cpp +40 -32
  141. package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +41 -26
  142. package/src/duckdb/src/common/types/row/tuple_data_layout.cpp +11 -1
  143. package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
  144. package/src/duckdb/src/common/types/row/tuple_data_segment.cpp +21 -16
  145. package/src/duckdb/src/common/types/string_heap.cpp +4 -0
  146. package/src/duckdb/src/common/types/time.cpp +105 -0
  147. package/src/duckdb/src/common/types/timestamp.cpp +7 -0
  148. package/src/duckdb/src/common/types/uuid.cpp +2 -2
  149. package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
  150. package/src/duckdb/src/common/types/value.cpp +99 -60
  151. package/src/duckdb/src/common/types/vector.cpp +73 -80
  152. package/src/duckdb/src/common/types.cpp +38 -724
  153. package/src/duckdb/src/common/virtual_file_system.cpp +142 -1
  154. package/src/duckdb/src/core_functions/aggregate/holistic/approximate_quantile.cpp +26 -0
  155. package/src/duckdb/src/core_functions/aggregate/holistic/mode.cpp +5 -7
  156. package/src/duckdb/src/core_functions/aggregate/holistic/quantile.cpp +64 -19
  157. package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +30 -0
  158. package/src/duckdb/src/core_functions/aggregate/nested/histogram.cpp +1 -0
  159. package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +83 -59
  160. package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
  161. package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
  162. package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
  163. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
  164. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
  165. package/src/duckdb/src/core_functions/function_list.cpp +10 -4
  166. package/src/duckdb/src/core_functions/scalar/date/date_diff.cpp +2 -0
  167. package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +380 -89
  168. package/src/duckdb/src/core_functions/scalar/date/date_sub.cpp +2 -0
  169. package/src/duckdb/src/core_functions/scalar/date/date_trunc.cpp +4 -0
  170. package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
  171. package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
  172. package/src/duckdb/src/core_functions/scalar/date/strftime.cpp +10 -0
  173. package/src/duckdb/src/core_functions/scalar/debug/vector_type.cpp +23 -0
  174. package/src/duckdb/src/core_functions/scalar/enum/enum_functions.cpp +16 -12
  175. package/src/duckdb/src/core_functions/scalar/generic/current_setting.cpp +3 -1
  176. package/src/duckdb/src/core_functions/scalar/list/array_slice.cpp +314 -82
  177. package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
  178. package/src/duckdb/src/core_functions/scalar/list/list_lambdas.cpp +23 -3
  179. package/src/duckdb/src/core_functions/scalar/map/map_entries.cpp +2 -2
  180. package/src/duckdb/src/core_functions/scalar/string/to_base.cpp +66 -0
  181. package/src/duckdb/src/core_functions/scalar/union/union_tag.cpp +1 -1
  182. package/src/duckdb/src/execution/aggregate_hashtable.cpp +226 -346
  183. package/src/duckdb/src/execution/column_binding_resolver.cpp +10 -7
  184. package/src/duckdb/src/execution/expression_executor/execute_parameter.cpp +2 -2
  185. package/src/duckdb/src/execution/expression_executor.cpp +1 -1
  186. package/src/duckdb/src/execution/index/art/art.cpp +219 -259
  187. package/src/duckdb/src/execution/index/art/art_key.cpp +0 -11
  188. package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +11 -15
  189. package/src/duckdb/src/execution/index/art/iterator.cpp +130 -214
  190. package/src/duckdb/src/execution/index/art/leaf.cpp +300 -266
  191. package/src/duckdb/src/execution/index/art/node.cpp +211 -205
  192. package/src/duckdb/src/execution/index/art/node16.cpp +10 -19
  193. package/src/duckdb/src/execution/index/art/node256.cpp +10 -18
  194. package/src/duckdb/src/execution/index/art/node4.cpp +21 -23
  195. package/src/duckdb/src/execution/index/art/node48.cpp +10 -20
  196. package/src/duckdb/src/execution/index/art/prefix.cpp +308 -338
  197. package/src/duckdb/src/execution/join_hashtable.cpp +9 -10
  198. package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
  199. package/src/duckdb/src/execution/operator/aggregate/physical_hash_aggregate.cpp +250 -317
  200. package/src/duckdb/src/execution/operator/aggregate/physical_perfecthash_aggregate.cpp +6 -4
  201. package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
  202. package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +231 -190
  203. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +367 -1068
  204. package/src/duckdb/src/execution/operator/{persistent → csv_scanner}/base_csv_reader.cpp +157 -174
  205. package/src/duckdb/src/execution/operator/csv_scanner/buffered_csv_reader.cpp +434 -0
  206. package/src/duckdb/src/execution/operator/csv_scanner/csv_buffer.cpp +80 -0
  207. package/src/duckdb/src/execution/operator/csv_scanner/csv_buffer_manager.cpp +90 -0
  208. package/src/duckdb/src/execution/operator/csv_scanner/csv_file_handle.cpp +95 -0
  209. package/src/duckdb/src/execution/operator/{persistent → csv_scanner}/csv_reader_options.cpp +67 -28
  210. package/src/duckdb/src/execution/operator/csv_scanner/csv_state_machine.cpp +35 -0
  211. package/src/duckdb/src/execution/operator/csv_scanner/csv_state_machine_cache.cpp +107 -0
  212. package/src/duckdb/src/execution/operator/{persistent → csv_scanner}/parallel_csv_reader.cpp +46 -47
  213. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp +52 -0
  214. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp +336 -0
  215. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/header_detection.cpp +165 -0
  216. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_detection.cpp +398 -0
  217. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_refinement.cpp +175 -0
  218. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_replacement.cpp +39 -0
  219. package/src/duckdb/src/execution/operator/filter/physical_filter.cpp +1 -1
  220. package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +12 -9
  221. package/src/duckdb/src/execution/operator/helper/physical_explain_analyze.cpp +2 -2
  222. package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +10 -8
  223. package/src/duckdb/src/execution/operator/helper/physical_load.cpp +2 -1
  224. package/src/duckdb/src/execution/operator/helper/physical_materialized_collector.cpp +7 -5
  225. package/src/duckdb/src/execution/operator/helper/physical_reset.cpp +3 -1
  226. package/src/duckdb/src/execution/operator/helper/physical_set.cpp +3 -1
  227. package/src/duckdb/src/execution/operator/helper/physical_vacuum.cpp +7 -5
  228. package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +449 -288
  229. package/src/duckdb/src/execution/operator/join/physical_blockwise_nl_join.cpp +2 -2
  230. package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -2
  231. package/src/duckdb/src/execution/operator/join/physical_delim_join.cpp +13 -6
  232. package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +28 -15
  233. package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +35 -17
  234. package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
  235. package/src/duckdb/src/execution/operator/join/physical_nested_loop_join.cpp +7 -4
  236. package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +31 -10
  237. package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +41 -5
  238. package/src/duckdb/src/execution/operator/order/physical_order.cpp +7 -5
  239. package/src/duckdb/src/execution/operator/order/physical_top_n.cpp +7 -5
  240. package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
  241. package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +14 -10
  242. package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp +11 -9
  243. package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp +9 -7
  244. package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +14 -12
  245. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +11 -11
  246. package/src/duckdb/src/execution/operator/persistent/physical_update.cpp +4 -2
  247. package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
  248. package/src/duckdb/src/execution/operator/projection/physical_unnest.cpp +24 -27
  249. package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
  250. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -12
  251. package/src/duckdb/src/execution/operator/schema/physical_attach.cpp +2 -1
  252. package/src/duckdb/src/execution/operator/schema/physical_create_art_index.cpp +198 -0
  253. package/src/duckdb/src/execution/operator/schema/physical_create_type.cpp +2 -6
  254. package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
  255. package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +16 -7
  256. package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +37 -6
  257. package/src/duckdb/src/execution/physical_operator.cpp +20 -16
  258. package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
  259. package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +57 -35
  260. package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +32 -15
  261. package/src/duckdb/src/execution/physical_plan/plan_create_index.cpp +45 -34
  262. package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
  263. package/src/duckdb/src/execution/physical_plan/plan_delim_join.cpp +2 -5
  264. package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
  265. package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
  266. package/src/duckdb/src/execution/physical_plan_generator.cpp +6 -11
  267. package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +636 -349
  268. package/src/duckdb/src/execution/window_executor.cpp +1285 -0
  269. package/src/duckdb/src/execution/window_segment_tree.cpp +408 -144
  270. package/src/duckdb/src/function/aggregate/distributive/count.cpp +2 -13
  271. package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +6 -12
  272. package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
  273. package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
  274. package/src/duckdb/src/function/cast/cast_function_set.cpp +1 -0
  275. package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
  276. package/src/duckdb/src/function/cast/string_cast.cpp +2 -2
  277. package/src/duckdb/src/function/cast/time_casts.cpp +7 -6
  278. package/src/duckdb/src/function/cast/vector_cast_helpers.cpp +7 -2
  279. package/src/duckdb/src/function/function.cpp +3 -1
  280. package/src/duckdb/src/function/pragma/pragma_queries.cpp +7 -1
  281. package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
  282. package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
  283. package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
  284. package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
  285. package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
  286. package/src/duckdb/src/function/scalar/operators/add.cpp +9 -0
  287. package/src/duckdb/src/function/scalar/operators/arithmetic.cpp +6 -3
  288. package/src/duckdb/src/function/scalar/strftime_format.cpp +4 -4
  289. package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
  290. package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +39 -5
  291. package/src/duckdb/src/function/scalar_function.cpp +5 -20
  292. package/src/duckdb/src/function/table/arrow/arrow_duck_schema.cpp +57 -0
  293. package/src/duckdb/src/function/table/arrow.cpp +110 -88
  294. package/src/duckdb/src/function/table/arrow_conversion.cpp +86 -73
  295. package/src/duckdb/src/function/table/copy_csv.cpp +102 -97
  296. package/src/duckdb/src/function/table/read_csv.cpp +263 -141
  297. package/src/duckdb/src/function/table/system/test_all_types.cpp +48 -21
  298. package/src/duckdb/src/function/table/system_functions.cpp +1 -0
  299. package/src/duckdb/src/function/table/table_scan.cpp +42 -0
  300. package/src/duckdb/src/function/table/version/pragma_version.cpp +49 -2
  301. package/src/duckdb/src/function/table_function.cpp +4 -3
  302. package/src/duckdb/src/include/duckdb/catalog/catalog.hpp +20 -5
  303. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/index_catalog_entry.hpp +3 -3
  304. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/macro_catalog_entry.hpp +1 -4
  305. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/schema_catalog_entry.hpp +2 -5
  306. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/sequence_catalog_entry.hpp +1 -6
  307. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp +2 -13
  308. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/type_catalog_entry.hpp +1 -4
  309. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/view_catalog_entry.hpp +2 -5
  310. package/src/duckdb/src/include/duckdb/catalog/catalog_entry.hpp +14 -0
  311. package/src/duckdb/src/include/duckdb/catalog/catalog_set.hpp +0 -6
  312. package/src/duckdb/src/include/duckdb/common/adbc/adbc.h +1 -0
  313. package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +4 -1
  314. package/src/duckdb/src/include/duckdb/common/adbc/single_batch_array_stream.hpp +16 -0
  315. package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
  316. package/src/duckdb/src/include/duckdb/common/arrow/appender/append_data.hpp +109 -0
  317. package/src/duckdb/src/include/duckdb/common/arrow/appender/bool_data.hpp +15 -0
  318. package/src/duckdb/src/include/duckdb/common/arrow/appender/enum_data.hpp +69 -0
  319. package/src/duckdb/src/include/duckdb/common/arrow/appender/list.hpp +8 -0
  320. package/src/duckdb/src/include/duckdb/common/arrow/appender/list_data.hpp +18 -0
  321. package/src/duckdb/src/include/duckdb/common/arrow/appender/map_data.hpp +18 -0
  322. package/src/duckdb/src/include/duckdb/common/arrow/appender/scalar_data.hpp +88 -0
  323. package/src/duckdb/src/include/duckdb/common/arrow/appender/struct_data.hpp +18 -0
  324. package/src/duckdb/src/include/duckdb/common/arrow/appender/union_data.hpp +21 -0
  325. package/src/duckdb/src/include/duckdb/common/arrow/appender/varchar_data.hpp +105 -0
  326. package/src/duckdb/src/include/duckdb/common/arrow/arrow_appender.hpp +9 -4
  327. package/src/duckdb/src/include/duckdb/common/arrow/arrow_converter.hpp +3 -5
  328. package/src/duckdb/src/include/duckdb/common/arrow/arrow_wrapper.hpp +5 -3
  329. package/src/duckdb/src/include/duckdb/common/arrow/nanoarrow/nanoarrow.h +462 -0
  330. package/src/duckdb/src/include/duckdb/common/arrow/nanoarrow/nanoarrow.hpp +14 -0
  331. package/src/duckdb/src/include/duckdb/common/arrow/result_arrow_wrapper.hpp +4 -0
  332. package/src/duckdb/src/include/duckdb/common/assert.hpp +1 -1
  333. package/src/duckdb/src/include/duckdb/common/bitpacking.hpp +70 -55
  334. package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
  335. package/src/duckdb/src/include/duckdb/common/case_insensitive_map.hpp +1 -0
  336. package/src/duckdb/src/include/duckdb/common/constants.hpp +4 -0
  337. package/src/duckdb/src/include/duckdb/common/dl.hpp +3 -1
  338. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +681 -577
  339. package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
  340. package/src/duckdb/src/include/duckdb/common/enums/date_part_specifier.hpp +9 -1
  341. package/src/duckdb/src/include/duckdb/common/enums/index_type.hpp +4 -3
  342. package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
  343. package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
  344. package/src/duckdb/src/include/duckdb/common/enums/operator_result_type.hpp +5 -1
  345. package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
  346. package/src/duckdb/src/include/duckdb/common/enums/pending_execution_result.hpp +1 -1
  347. package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
  348. package/src/duckdb/src/include/duckdb/common/exception.hpp +15 -1
  349. package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
  350. package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +215 -0
  351. package/src/duckdb/src/include/duckdb/common/field_writer.hpp +0 -4
  352. package/src/duckdb/src/include/duckdb/common/file_opener.hpp +9 -0
  353. package/src/duckdb/src/include/duckdb/common/file_system.hpp +10 -8
  354. package/src/duckdb/src/include/duckdb/common/filename_pattern.hpp +1 -1
  355. package/src/duckdb/src/include/duckdb/common/fixed_size_map.hpp +208 -0
  356. package/src/duckdb/src/include/duckdb/common/helper.hpp +8 -3
  357. package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
  358. package/src/duckdb/src/include/duckdb/common/http_state.hpp +61 -28
  359. package/src/duckdb/src/include/duckdb/common/hugeint.hpp +15 -0
  360. package/src/duckdb/src/include/duckdb/common/index_vector.hpp +12 -0
  361. package/src/duckdb/src/include/duckdb/common/limits.hpp +52 -149
  362. package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +11 -5
  363. package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +12 -42
  364. package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
  365. package/src/duckdb/src/include/duckdb/common/numeric_utils.hpp +48 -0
  366. package/src/duckdb/src/include/duckdb/common/opener_file_system.hpp +6 -2
  367. package/src/duckdb/src/include/duckdb/common/operator/add.hpp +5 -2
  368. package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +65 -4
  369. package/src/duckdb/src/include/duckdb/common/operator/multiply.hpp +3 -2
  370. package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
  371. package/src/duckdb/src/include/duckdb/common/operator/string_cast.hpp +1 -1
  372. package/src/duckdb/src/include/duckdb/common/operator/subtract.hpp +3 -2
  373. package/src/duckdb/src/include/duckdb/common/optional_idx.hpp +3 -0
  374. package/src/duckdb/src/include/duckdb/common/perfect_map_set.hpp +2 -1
  375. package/src/duckdb/src/include/duckdb/common/printer.hpp +11 -0
  376. package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
  377. package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
  378. package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
  379. package/src/duckdb/src/include/duckdb/common/serializer/binary_deserializer.hpp +71 -30
  380. package/src/duckdb/src/include/duckdb/common/serializer/binary_serializer.hpp +48 -39
  381. package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_reader.hpp +0 -4
  382. package/src/duckdb/src/include/duckdb/common/serializer/deserialization_data.hpp +128 -0
  383. package/src/duckdb/src/include/duckdb/common/serializer/encoding_util.hpp +132 -0
  384. package/src/duckdb/src/include/duckdb/common/serializer/format_deserializer.hpp +186 -133
  385. package/src/duckdb/src/include/duckdb/common/serializer/format_serializer.hpp +166 -121
  386. package/src/duckdb/src/include/duckdb/common/serializer/serialization_traits.hpp +27 -4
  387. package/src/duckdb/src/include/duckdb/common/serializer.hpp +0 -7
  388. package/src/duckdb/src/include/duckdb/common/shared_ptr.hpp +8 -0
  389. package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +34 -13
  390. package/src/duckdb/src/include/duckdb/common/stack_checker.hpp +34 -0
  391. package/src/duckdb/src/include/duckdb/common/string_util.hpp +11 -0
  392. package/src/duckdb/src/include/duckdb/common/type_util.hpp +8 -0
  393. package/src/duckdb/src/include/duckdb/common/typedefs.hpp +8 -0
  394. package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
  395. package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
  396. package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +11 -1
  397. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +12 -1
  398. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +3 -1
  399. package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
  400. package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +6 -3
  401. package/src/duckdb/src/include/duckdb/common/types/date.hpp +9 -5
  402. package/src/duckdb/src/include/duckdb/common/types/datetime.hpp +46 -3
  403. package/src/duckdb/src/include/duckdb/common/types/hyperloglog.hpp +7 -1
  404. package/src/duckdb/src/include/duckdb/common/types/interval.hpp +7 -0
  405. package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +11 -15
  406. package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +46 -11
  407. package/src/duckdb/src/include/duckdb/common/types/row/row_data_collection_scanner.hpp +10 -1
  408. package/src/duckdb/src/include/duckdb/common/types/row/row_layout.hpp +1 -23
  409. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_allocator.hpp +14 -8
  410. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +7 -3
  411. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_layout.hpp +7 -0
  412. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_segment.hpp +13 -8
  413. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +6 -2
  414. package/src/duckdb/src/include/duckdb/common/types/string_heap.hpp +3 -0
  415. package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
  416. package/src/duckdb/src/include/duckdb/common/types/time.hpp +5 -0
  417. package/src/duckdb/src/include/duckdb/common/types/timestamp.hpp +16 -10
  418. package/src/duckdb/src/include/duckdb/common/types/value.hpp +7 -2
  419. package/src/duckdb/src/include/duckdb/common/types/vector.hpp +10 -3
  420. package/src/duckdb/src/include/duckdb/common/types.hpp +6 -25
  421. package/src/duckdb/src/include/duckdb/common/vector.hpp +2 -2
  422. package/src/duckdb/src/include/duckdb/common/vector_operations/aggregate_executor.hpp +7 -2
  423. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +40 -97
  424. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
  425. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
  426. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
  427. package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +4 -2
  428. package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
  429. package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
  430. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
  431. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
  432. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
  433. package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
  434. package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
  435. package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +40 -11
  436. package/src/duckdb/src/include/duckdb/core_functions/scalar/debug_functions.hpp +27 -0
  437. package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
  438. package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
  439. package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +7 -5
  440. package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
  441. package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +6 -4
  442. package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +4 -2
  443. package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
  444. package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +12 -1
  445. package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
  446. package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
  447. package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +128 -131
  448. package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
  449. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +13 -12
  450. package/src/duckdb/src/include/duckdb/execution/index/art/art_key.hpp +0 -1
  451. package/src/duckdb/src/include/duckdb/execution/index/art/fixed_size_allocator.hpp +22 -24
  452. package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +32 -28
  453. package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +46 -51
  454. package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +134 -53
  455. package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +5 -7
  456. package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +5 -7
  457. package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +7 -9
  458. package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +5 -7
  459. package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
  460. package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_hash_aggregate.hpp +8 -7
  461. package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_perfecthash_aggregate.hpp +1 -1
  462. package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_ungrouped_aggregate.hpp +3 -3
  463. package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_window.hpp +6 -5
  464. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_batch_collector.hpp +2 -2
  465. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_explain_analyze.hpp +1 -1
  466. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_limit.hpp +1 -1
  467. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_materialized_collector.hpp +1 -1
  468. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_vacuum.hpp +2 -2
  469. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +5 -12
  470. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_blockwise_nl_join.hpp +1 -1
  471. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_delim_join.hpp +2 -2
  472. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_hash_join.hpp +2 -2
  473. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +3 -3
  474. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_nested_loop_join.hpp +2 -2
  475. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +3 -3
  476. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp +12 -1
  477. package/src/duckdb/src/include/duckdb/execution/operator/order/physical_order.hpp +2 -2
  478. package/src/duckdb/src/include/duckdb/execution/operator/order/physical_top_n.hpp +2 -2
  479. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
  480. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_copy_to_file.hpp +2 -2
  481. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_insert.hpp +2 -2
  482. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_copy_to_file.hpp +2 -2
  483. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_fixed_batch_copy.hpp +2 -2
  484. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_insert.hpp +2 -2
  485. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_update.hpp +1 -1
  486. package/src/duckdb/src/include/duckdb/execution/operator/{persistent → scan/csv}/base_csv_reader.hpp +19 -19
  487. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/buffered_csv_reader.hpp +72 -0
  488. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_buffer.hpp +110 -0
  489. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_buffer_manager.hpp +103 -0
  490. package/src/duckdb/src/include/duckdb/execution/operator/{persistent → scan/csv}/csv_file_handle.hpp +8 -15
  491. package/src/duckdb/src/include/duckdb/execution/operator/{persistent → scan/csv}/csv_line_info.hpp +5 -4
  492. package/src/duckdb/src/include/duckdb/execution/operator/{persistent → scan/csv}/csv_reader_options.hpp +61 -28
  493. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_sniffer.hpp +127 -0
  494. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_state_machine.hpp +75 -0
  495. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_state_machine_cache.hpp +51 -0
  496. package/src/duckdb/src/include/duckdb/execution/operator/{persistent → scan/csv}/parallel_csv_reader.hpp +22 -28
  497. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/quote_rules.hpp +21 -0
  498. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
  499. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -5
  500. package/src/duckdb/src/include/duckdb/execution/operator/schema/{physical_create_index.hpp → physical_create_art_index.hpp} +14 -7
  501. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
  502. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
  503. package/src/duckdb/src/include/duckdb/execution/perfect_aggregate_hashtable.hpp +4 -2
  504. package/src/duckdb/src/include/duckdb/execution/physical_operator.hpp +6 -5
  505. package/src/duckdb/src/include/duckdb/execution/physical_operator_states.hpp +11 -0
  506. package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +6 -2
  507. package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +19 -21
  508. package/src/duckdb/src/include/duckdb/execution/window_executor.hpp +313 -0
  509. package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +79 -63
  510. package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +12 -4
  511. package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
  512. package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
  513. package/src/duckdb/src/include/duckdb/function/copy_function.hpp +6 -1
  514. package/src/duckdb/src/include/duckdb/function/function_serialization.hpp +80 -0
  515. package/src/duckdb/src/include/duckdb/function/macro_function.hpp +3 -0
  516. package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
  517. package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
  518. package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
  519. package/src/duckdb/src/include/duckdb/function/scalar/strftime_format.hpp +12 -4
  520. package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
  521. package/src/duckdb/src/include/duckdb/function/scalar_function.hpp +8 -3
  522. package/src/duckdb/src/include/duckdb/function/scalar_macro_function.hpp +3 -0
  523. package/src/duckdb/src/include/duckdb/function/table/arrow/arrow_duck_schema.hpp +99 -0
  524. package/src/duckdb/src/include/duckdb/function/table/arrow.hpp +6 -36
  525. package/src/duckdb/src/include/duckdb/function/table/read_csv.hpp +24 -12
  526. package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +5 -1
  527. package/src/duckdb/src/include/duckdb/function/table_function.hpp +8 -0
  528. package/src/duckdb/src/include/duckdb/function/table_macro_function.hpp +3 -0
  529. package/src/duckdb/src/include/duckdb/function/udf_function.hpp +2 -1
  530. package/src/duckdb/src/include/duckdb/main/attached_database.hpp +1 -1
  531. package/src/duckdb/src/include/duckdb/main/capi/capi_internal.hpp +4 -3
  532. package/src/duckdb/src/include/duckdb/main/chunk_scan_state/query_result.hpp +29 -0
  533. package/src/duckdb/src/include/duckdb/main/chunk_scan_state.hpp +43 -0
  534. package/src/duckdb/src/include/duckdb/main/client_config.hpp +7 -2
  535. package/src/duckdb/src/include/duckdb/main/client_context.hpp +16 -14
  536. package/src/duckdb/src/include/duckdb/main/client_context_file_opener.hpp +1 -0
  537. package/src/duckdb/src/include/duckdb/main/client_data.hpp +2 -1
  538. package/src/duckdb/src/include/duckdb/main/client_properties.hpp +25 -0
  539. package/src/duckdb/src/include/duckdb/main/config.hpp +16 -1
  540. package/src/duckdb/src/include/duckdb/main/connection.hpp +3 -4
  541. package/src/duckdb/src/include/duckdb/main/extension/generated_extension_loader.hpp +27 -0
  542. package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +210 -144
  543. package/src/duckdb/src/include/duckdb/main/extension_helper.hpp +41 -6
  544. package/src/duckdb/src/include/duckdb/main/extension_util.hpp +4 -0
  545. package/src/duckdb/src/include/duckdb/main/pending_query_result.hpp +5 -0
  546. package/src/duckdb/src/include/duckdb/main/prepared_statement.hpp +73 -5
  547. package/src/duckdb/src/include/duckdb/main/prepared_statement_data.hpp +6 -6
  548. package/src/duckdb/src/include/duckdb/main/query_result.hpp +2 -27
  549. package/src/duckdb/src/include/duckdb/main/relation/aggregate_relation.hpp +4 -1
  550. package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
  551. package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
  552. package/src/duckdb/src/include/duckdb/main/relation/read_csv_relation.hpp +6 -6
  553. package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
  554. package/src/duckdb/src/include/duckdb/main/settings.hpp +71 -11
  555. package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
  556. package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
  557. package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
  558. package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +7 -0
  559. package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +38 -64
  560. package/src/duckdb/src/include/duckdb/optimizer/join_order/cost_model.hpp +37 -0
  561. package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
  562. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_node.hpp +14 -29
  563. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +8 -22
  564. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -12
  565. package/src/duckdb/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp +89 -0
  566. package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +19 -30
  567. package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph_manager.hpp +113 -0
  568. package/src/duckdb/src/include/duckdb/optimizer/join_order/relation_manager.hpp +73 -0
  569. package/src/duckdb/src/include/duckdb/optimizer/join_order/relation_statistics_helper.hpp +73 -0
  570. package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
  571. package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
  572. package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
  573. package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
  574. package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
  575. package/src/duckdb/src/include/duckdb/parallel/event.hpp +12 -1
  576. package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -3
  577. package/src/duckdb/src/include/duckdb/parallel/pipeline_executor.hpp +3 -2
  578. package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +9 -1
  579. package/src/duckdb/src/include/duckdb/parser/column_definition.hpp +6 -5
  580. package/src/duckdb/src/include/duckdb/parser/column_list.hpp +4 -0
  581. package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
  582. package/src/duckdb/src/include/duckdb/parser/constraint.hpp +5 -0
  583. package/src/duckdb/src/include/duckdb/parser/constraints/check_constraint.hpp +3 -0
  584. package/src/duckdb/src/include/duckdb/parser/constraints/foreign_key_constraint.hpp +6 -0
  585. package/src/duckdb/src/include/duckdb/parser/constraints/not_null_constraint.hpp +3 -0
  586. package/src/duckdb/src/include/duckdb/parser/constraints/unique_constraint.hpp +6 -0
  587. package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +4 -1
  588. package/src/duckdb/src/include/duckdb/parser/expression/case_expression.hpp +1 -1
  589. package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +4 -1
  590. package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +4 -1
  591. package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +4 -1
  592. package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +4 -1
  593. package/src/duckdb/src/include/duckdb/parser/expression/conjunction_expression.hpp +1 -1
  594. package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +4 -1
  595. package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
  596. package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +4 -1
  597. package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +4 -1
  598. package/src/duckdb/src/include/duckdb/parser/expression/operator_expression.hpp +21 -4
  599. package/src/duckdb/src/include/duckdb/parser/expression/parameter_expression.hpp +18 -2
  600. package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +4 -1
  601. package/src/duckdb/src/include/duckdb/parser/expression/star_expression.hpp +1 -1
  602. package/src/duckdb/src/include/duckdb/parser/expression/subquery_expression.hpp +1 -1
  603. package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +4 -1
  604. package/src/duckdb/src/include/duckdb/parser/group_by_node.hpp +11 -0
  605. package/src/duckdb/src/include/duckdb/parser/parsed_data/alter_info.hpp +12 -1
  606. package/src/duckdb/src/include/duckdb/parser/parsed_data/alter_table_info.hpp +66 -2
  607. package/src/duckdb/src/include/duckdb/parser/parsed_data/attach_info.hpp +8 -1
  608. package/src/duckdb/src/include/duckdb/parser/parsed_data/copy_info.hpp +8 -1
  609. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_index_info.hpp +9 -1
  610. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_info.hpp +9 -2
  611. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_macro_info.hpp +3 -0
  612. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_schema_info.hpp +3 -0
  613. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_sequence_info.hpp +3 -0
  614. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_table_info.hpp +3 -0
  615. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_type_info.hpp +3 -0
  616. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_view_info.hpp +3 -0
  617. package/src/duckdb/src/include/duckdb/parser/parsed_data/detach_info.hpp +7 -0
  618. package/src/duckdb/src/include/duckdb/parser/parsed_data/drop_info.hpp +7 -0
  619. package/src/duckdb/src/include/duckdb/parser/parsed_data/exported_table_data.hpp +7 -0
  620. package/src/duckdb/src/include/duckdb/parser/parsed_data/load_info.hpp +17 -3
  621. package/src/duckdb/src/include/duckdb/parser/parsed_data/parse_info.hpp +22 -0
  622. package/src/duckdb/src/include/duckdb/parser/parsed_data/pragma_info.hpp +10 -0
  623. package/src/duckdb/src/include/duckdb/parser/parsed_data/show_select_info.hpp +7 -0
  624. package/src/duckdb/src/include/duckdb/parser/parsed_data/transaction_info.hpp +10 -0
  625. package/src/duckdb/src/include/duckdb/parser/parsed_data/vacuum_info.hpp +10 -0
  626. package/src/duckdb/src/include/duckdb/parser/parser.hpp +4 -0
  627. package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
  628. package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
  629. package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
  630. package/src/duckdb/src/include/duckdb/parser/statement/execute_statement.hpp +1 -1
  631. package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
  632. package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
  633. package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
  634. package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
  635. package/src/duckdb/src/include/duckdb/parser/transformer.hpp +23 -26
  636. package/src/duckdb/src/include/duckdb/planner/binder.hpp +16 -5
  637. package/src/duckdb/src/include/duckdb/planner/bound_constraint.hpp +0 -8
  638. package/src/duckdb/src/include/duckdb/planner/bound_parameter_map.hpp +2 -1
  639. package/src/duckdb/src/include/duckdb/planner/bound_result_modifier.hpp +6 -0
  640. package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
  641. package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +9 -0
  642. package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
  643. package/src/duckdb/src/include/duckdb/planner/expression/bound_aggregate_expression.hpp +3 -0
  644. package/src/duckdb/src/include/duckdb/planner/expression/bound_between_expression.hpp +6 -0
  645. package/src/duckdb/src/include/duckdb/planner/expression/bound_case_expression.hpp +6 -0
  646. package/src/duckdb/src/include/duckdb/planner/expression/bound_cast_expression.hpp +6 -0
  647. package/src/duckdb/src/include/duckdb/planner/expression/bound_columnref_expression.hpp +3 -0
  648. package/src/duckdb/src/include/duckdb/planner/expression/bound_comparison_expression.hpp +3 -0
  649. package/src/duckdb/src/include/duckdb/planner/expression/bound_conjunction_expression.hpp +3 -0
  650. package/src/duckdb/src/include/duckdb/planner/expression/bound_constant_expression.hpp +3 -0
  651. package/src/duckdb/src/include/duckdb/planner/expression/bound_default_expression.hpp +3 -0
  652. package/src/duckdb/src/include/duckdb/planner/expression/bound_function_expression.hpp +4 -0
  653. package/src/duckdb/src/include/duckdb/planner/expression/bound_lambda_expression.hpp +3 -1
  654. package/src/duckdb/src/include/duckdb/planner/expression/bound_lambdaref_expression.hpp +3 -0
  655. package/src/duckdb/src/include/duckdb/planner/expression/bound_operator_expression.hpp +3 -0
  656. package/src/duckdb/src/include/duckdb/planner/expression/bound_parameter_data.hpp +24 -6
  657. package/src/duckdb/src/include/duckdb/planner/expression/bound_parameter_expression.hpp +9 -2
  658. package/src/duckdb/src/include/duckdb/planner/expression/bound_reference_expression.hpp +3 -0
  659. package/src/duckdb/src/include/duckdb/planner/expression/bound_unnest_expression.hpp +3 -0
  660. package/src/duckdb/src/include/duckdb/planner/expression/bound_window_expression.hpp +3 -0
  661. package/src/duckdb/src/include/duckdb/planner/expression/list.hpp +1 -0
  662. package/src/duckdb/src/include/duckdb/planner/expression.hpp +3 -0
  663. package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
  664. package/src/duckdb/src/include/duckdb/planner/expression_binder.hpp +13 -1
  665. package/src/duckdb/src/include/duckdb/planner/filter/conjunction_filter.hpp +4 -0
  666. package/src/duckdb/src/include/duckdb/planner/filter/constant_filter.hpp +2 -0
  667. package/src/duckdb/src/include/duckdb/planner/filter/null_filter.hpp +4 -0
  668. package/src/duckdb/src/include/duckdb/planner/joinside.hpp +3 -0
  669. package/src/duckdb/src/include/duckdb/planner/logical_operator.hpp +3 -2
  670. package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -2
  671. package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +3 -3
  672. package/src/duckdb/src/include/duckdb/planner/operator/logical_aggregate.hpp +3 -0
  673. package/src/duckdb/src/include/duckdb/planner/operator/logical_any_join.hpp +3 -0
  674. package/src/duckdb/src/include/duckdb/planner/operator/logical_column_data_get.hpp +4 -0
  675. package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +12 -7
  676. package/src/duckdb/src/include/duckdb/planner/operator/logical_copy_to_file.hpp +2 -0
  677. package/src/duckdb/src/include/duckdb/planner/operator/logical_create.hpp +9 -6
  678. package/src/duckdb/src/include/duckdb/planner/operator/logical_create_index.hpp +12 -23
  679. package/src/duckdb/src/include/duckdb/planner/operator/logical_create_table.hpp +10 -6
  680. package/src/duckdb/src/include/duckdb/planner/operator/logical_cross_product.hpp +3 -0
  681. package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +9 -2
  682. package/src/duckdb/src/include/duckdb/planner/operator/logical_delete.hpp +7 -0
  683. package/src/duckdb/src/include/duckdb/planner/operator/logical_delim_get.hpp +3 -0
  684. package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
  685. package/src/duckdb/src/include/duckdb/planner/operator/logical_distinct.hpp +6 -10
  686. package/src/duckdb/src/include/duckdb/planner/operator/logical_dummy_scan.hpp +2 -0
  687. package/src/duckdb/src/include/duckdb/planner/operator/logical_empty_result.hpp +2 -0
  688. package/src/duckdb/src/include/duckdb/planner/operator/logical_explain.hpp +4 -0
  689. package/src/duckdb/src/include/duckdb/planner/operator/logical_expression_get.hpp +3 -0
  690. package/src/duckdb/src/include/duckdb/planner/operator/logical_extension_operator.hpp +8 -0
  691. package/src/duckdb/src/include/duckdb/planner/operator/logical_filter.hpp +3 -0
  692. package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +11 -1
  693. package/src/duckdb/src/include/duckdb/planner/operator/logical_insert.hpp +6 -0
  694. package/src/duckdb/src/include/duckdb/planner/operator/logical_limit.hpp +3 -0
  695. package/src/duckdb/src/include/duckdb/planner/operator/logical_limit_percent.hpp +3 -0
  696. package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +52 -0
  697. package/src/duckdb/src/include/duckdb/planner/operator/logical_order.hpp +7 -35
  698. package/src/duckdb/src/include/duckdb/planner/operator/logical_pivot.hpp +6 -0
  699. package/src/duckdb/src/include/duckdb/planner/operator/logical_positional_join.hpp +3 -0
  700. package/src/duckdb/src/include/duckdb/planner/operator/logical_projection.hpp +3 -0
  701. package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +10 -7
  702. package/src/duckdb/src/include/duckdb/planner/operator/logical_reset.hpp +4 -0
  703. package/src/duckdb/src/include/duckdb/planner/operator/logical_sample.hpp +6 -0
  704. package/src/duckdb/src/include/duckdb/planner/operator/logical_set.hpp +4 -0
  705. package/src/duckdb/src/include/duckdb/planner/operator/logical_set_operation.hpp +4 -0
  706. package/src/duckdb/src/include/duckdb/planner/operator/logical_show.hpp +3 -0
  707. package/src/duckdb/src/include/duckdb/planner/operator/logical_simple.hpp +3 -0
  708. package/src/duckdb/src/include/duckdb/planner/operator/logical_top_n.hpp +4 -0
  709. package/src/duckdb/src/include/duckdb/planner/operator/logical_unnest.hpp +2 -0
  710. package/src/duckdb/src/include/duckdb/planner/operator/logical_update.hpp +6 -0
  711. package/src/duckdb/src/include/duckdb/planner/operator/logical_window.hpp +3 -0
  712. package/src/duckdb/src/include/duckdb/planner/operator_extension.hpp +1 -0
  713. package/src/duckdb/src/include/duckdb/planner/planner.hpp +4 -3
  714. package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
  715. package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
  716. package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
  717. package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
  718. package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
  719. package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
  720. package/src/duckdb/src/include/duckdb/planner/table_filter.hpp +7 -1
  721. package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
  722. package/src/duckdb/src/include/duckdb/planner/tableref/bound_pivotref.hpp +3 -0
  723. package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +2 -1
  724. package/src/duckdb/src/include/duckdb/storage/block.hpp +33 -4
  725. package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +11 -11
  726. package/src/duckdb/src/include/duckdb/storage/buffer/block_handle.hpp +3 -0
  727. package/src/duckdb/src/include/duckdb/storage/checkpoint/row_group_writer.hpp +5 -5
  728. package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_reader.hpp +2 -2
  729. package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp +3 -3
  730. package/src/duckdb/src/include/duckdb/storage/checkpoint_manager.hpp +19 -16
  731. package/src/duckdb/src/include/duckdb/storage/data_pointer.hpp +1 -1
  732. package/src/duckdb/src/include/duckdb/storage/data_table.hpp +2 -2
  733. package/src/duckdb/src/include/duckdb/storage/in_memory_block_manager.hpp +2 -2
  734. package/src/duckdb/src/include/duckdb/storage/index.hpp +2 -2
  735. package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp +88 -0
  736. package/src/duckdb/src/include/duckdb/storage/metadata/metadata_reader.hpp +54 -0
  737. package/src/duckdb/src/include/duckdb/storage/metadata/metadata_writer.hpp +45 -0
  738. package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
  739. package/src/duckdb/src/include/duckdb/storage/partial_block_manager.hpp +2 -2
  740. package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +8 -5
  741. package/src/duckdb/src/include/duckdb/storage/statistics/base_statistics.hpp +7 -3
  742. package/src/duckdb/src/include/duckdb/storage/statistics/column_statistics.hpp +4 -0
  743. package/src/duckdb/src/include/duckdb/storage/statistics/distinct_statistics.hpp +5 -0
  744. package/src/duckdb/src/include/duckdb/storage/statistics/list_stats.hpp +3 -0
  745. package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats.hpp +3 -0
  746. package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +7 -0
  747. package/src/duckdb/src/include/duckdb/storage/statistics/struct_stats.hpp +3 -0
  748. package/src/duckdb/src/include/duckdb/storage/storage_info.hpp +2 -2
  749. package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp +2 -2
  750. package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +18 -3
  751. package/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp +2 -2
  752. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +8 -3
  753. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +3 -3
  754. package/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp +2 -2
  755. package/src/duckdb/src/include/duckdb/storage/table/table_index_list.hpp +1 -1
  756. package/src/duckdb/src/include/duckdb/storage/table/table_statistics.hpp +5 -0
  757. package/src/duckdb/src/include/duckdb/storage/table_io_manager.hpp +3 -0
  758. package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp +3 -4
  759. package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
  760. package/src/duckdb/src/include/duckdb/verification/deserialized_statement_verifier_v2.hpp +6 -0
  761. package/src/duckdb/src/include/duckdb/verification/prepared_statement_verifier.hpp +1 -1
  762. package/src/duckdb/src/include/duckdb/verification/statement_verifier.hpp +1 -0
  763. package/src/duckdb/src/include/duckdb.h +98 -1
  764. package/src/duckdb/src/main/appender.cpp +3 -1
  765. package/src/duckdb/src/main/attached_database.cpp +2 -2
  766. package/src/duckdb/src/main/capi/arrow-c.cpp +196 -8
  767. package/src/duckdb/src/main/capi/duckdb-c.cpp +16 -0
  768. package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
  769. package/src/duckdb/src/main/capi/logical_types-c.cpp +22 -0
  770. package/src/duckdb/src/main/capi/pending-c.cpp +23 -0
  771. package/src/duckdb/src/main/capi/prepared-c.cpp +106 -28
  772. package/src/duckdb/src/main/capi/result-c.cpp +3 -1
  773. package/src/duckdb/src/main/chunk_scan_state/query_result.cpp +53 -0
  774. package/src/duckdb/src/main/chunk_scan_state.cpp +48 -0
  775. package/src/duckdb/src/main/client_context.cpp +42 -19
  776. package/src/duckdb/src/main/client_context_file_opener.cpp +17 -0
  777. package/src/duckdb/src/main/client_verify.cpp +18 -0
  778. package/src/duckdb/src/main/config.cpp +9 -3
  779. package/src/duckdb/src/main/connection.cpp +3 -3
  780. package/src/duckdb/src/main/database.cpp +3 -12
  781. package/src/duckdb/src/main/db_instance_cache.cpp +14 -6
  782. package/src/duckdb/src/main/extension/extension_helper.cpp +164 -88
  783. package/src/duckdb/src/main/extension/extension_install.cpp +76 -15
  784. package/src/duckdb/src/main/extension/extension_load.cpp +62 -13
  785. package/src/duckdb/src/main/extension/extension_util.cpp +16 -0
  786. package/src/duckdb/src/main/pending_query_result.cpp +9 -1
  787. package/src/duckdb/src/main/prepared_statement.cpp +38 -11
  788. package/src/duckdb/src/main/prepared_statement_data.cpp +23 -18
  789. package/src/duckdb/src/main/query_result.cpp +0 -21
  790. package/src/duckdb/src/main/relation/aggregate_relation.cpp +20 -10
  791. package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
  792. package/src/duckdb/src/main/relation/join_relation.cpp +6 -6
  793. package/src/duckdb/src/main/relation/read_csv_relation.cpp +19 -13
  794. package/src/duckdb/src/main/relation.cpp +10 -9
  795. package/src/duckdb/src/main/settings/settings.cpp +125 -33
  796. package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
  797. package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +2 -4
  798. package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
  799. package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
  800. package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
  801. package/src/duckdb/src/optimizer/compressed_materialization.cpp +477 -0
  802. package/src/duckdb/src/optimizer/deliminator.cpp +180 -323
  803. package/src/duckdb/src/optimizer/filter_pushdown.cpp +23 -6
  804. package/src/duckdb/src/optimizer/join_order/cardinality_estimator.cpp +79 -325
  805. package/src/duckdb/src/optimizer/join_order/cost_model.cpp +19 -0
  806. package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
  807. package/src/duckdb/src/optimizer/join_order/join_node.cpp +5 -37
  808. package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +48 -1047
  809. package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
  810. package/src/duckdb/src/optimizer/join_order/plan_enumerator.cpp +552 -0
  811. package/src/duckdb/src/optimizer/join_order/query_graph.cpp +52 -41
  812. package/src/duckdb/src/optimizer/join_order/query_graph_manager.cpp +409 -0
  813. package/src/duckdb/src/optimizer/join_order/relation_manager.cpp +356 -0
  814. package/src/duckdb/src/optimizer/join_order/relation_statistics_helper.cpp +351 -0
  815. package/src/duckdb/src/optimizer/optimizer.cpp +49 -14
  816. package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
  817. package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
  818. package/src/duckdb/src/optimizer/pushdown/pushdown_projection.cpp +34 -7
  819. package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
  820. package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
  821. package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
  822. package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
  823. package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
  824. package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
  825. package/src/duckdb/src/optimizer/topn_optimizer.cpp +27 -10
  826. package/src/duckdb/src/optimizer/unnest_rewriter.cpp +3 -5
  827. package/src/duckdb/src/parallel/executor.cpp +25 -1
  828. package/src/duckdb/src/parallel/pipeline.cpp +0 -17
  829. package/src/duckdb/src/parallel/pipeline_executor.cpp +33 -13
  830. package/src/duckdb/src/parallel/pipeline_finish_event.cpp +55 -1
  831. package/src/duckdb/src/parallel/task_scheduler.cpp +18 -2
  832. package/src/duckdb/src/parser/column_definition.cpp +20 -32
  833. package/src/duckdb/src/parser/column_list.cpp +8 -0
  834. package/src/duckdb/src/parser/constraints/foreign_key_constraint.cpp +3 -0
  835. package/src/duckdb/src/parser/constraints/unique_constraint.cpp +3 -0
  836. package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
  837. package/src/duckdb/src/parser/expression/case_expression.cpp +0 -25
  838. package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
  839. package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
  840. package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
  841. package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
  842. package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
  843. package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
  844. package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
  845. package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
  846. package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
  847. package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
  848. package/src/duckdb/src/parser/expression/parameter_expression.cpp +7 -19
  849. package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
  850. package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
  851. package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
  852. package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
  853. package/src/duckdb/src/parser/parsed_data/alter_info.cpp +5 -2
  854. package/src/duckdb/src/parser/parsed_data/alter_table_info.cpp +38 -0
  855. package/src/duckdb/src/parser/parsed_data/create_index_info.cpp +17 -1
  856. package/src/duckdb/src/parser/parsed_data/create_sequence_info.cpp +2 -0
  857. package/src/duckdb/src/parser/parsed_data/detach_info.cpp +1 -1
  858. package/src/duckdb/src/parser/parsed_data/drop_info.cpp +1 -1
  859. package/src/duckdb/src/parser/parsed_data/sample_options.cpp +0 -18
  860. package/src/duckdb/src/parser/parsed_data/transaction_info.cpp +4 -1
  861. package/src/duckdb/src/parser/parsed_data/vacuum_info.cpp +1 -1
  862. package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
  863. package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
  864. package/src/duckdb/src/parser/parser.cpp +62 -36
  865. package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
  866. package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
  867. package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
  868. package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
  869. package/src/duckdb/src/parser/query_node.cpp +15 -47
  870. package/src/duckdb/src/parser/result_modifier.cpp +0 -87
  871. package/src/duckdb/src/parser/statement/execute_statement.cpp +2 -2
  872. package/src/duckdb/src/parser/statement/select_statement.cpp +0 -10
  873. package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
  874. package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
  875. package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
  876. package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
  877. package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -55
  878. package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
  879. package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
  880. package/src/duckdb/src/parser/tableref.cpp +0 -44
  881. package/src/duckdb/src/parser/transform/constraint/transform_constraint.cpp +55 -38
  882. package/src/duckdb/src/parser/transform/expression/transform_array_access.cpp +13 -4
  883. package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
  884. package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
  885. package/src/duckdb/src/parser/transform/expression/transform_function.cpp +3 -0
  886. package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
  887. package/src/duckdb/src/parser/transform/expression/transform_param_ref.cpp +45 -26
  888. package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
  889. package/src/duckdb/src/parser/transform/helpers/transform_typename.cpp +16 -1
  890. package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
  891. package/src/duckdb/src/parser/transform/statement/transform_create_index.cpp +32 -17
  892. package/src/duckdb/src/parser/transform/statement/transform_create_type.cpp +1 -1
  893. package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
  894. package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
  895. package/src/duckdb/src/parser/transform/statement/transform_load.cpp +1 -0
  896. package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
  897. package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
  898. package/src/duckdb/src/parser/transform/statement/transform_prepare.cpp +28 -6
  899. package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
  900. package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
  901. package/src/duckdb/src/parser/transformer.cpp +44 -25
  902. package/src/duckdb/src/planner/binder/expression/bind_macro_expression.cpp +5 -3
  903. package/src/duckdb/src/planner/binder/expression/bind_parameter_expression.cpp +10 -10
  904. package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
  905. package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
  906. package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
  907. package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
  908. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +36 -33
  909. package/src/duckdb/src/planner/binder/statement/bind_copy.cpp +1 -1
  910. package/src/duckdb/src/planner/binder/statement/bind_create.cpp +14 -52
  911. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +0 -23
  912. package/src/duckdb/src/planner/binder/statement/bind_execute.cpp +13 -7
  913. package/src/duckdb/src/planner/binder/statement/bind_export.cpp +70 -29
  914. package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +93 -28
  915. package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
  916. package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -50
  917. package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
  918. package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +67 -31
  919. package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
  920. package/src/duckdb/src/planner/binder.cpp +44 -31
  921. package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +24 -1
  922. package/src/duckdb/src/planner/expression/bound_between_expression.cpp +4 -0
  923. package/src/duckdb/src/planner/expression/bound_cast_expression.cpp +13 -8
  924. package/src/duckdb/src/planner/expression/bound_function_expression.cpp +22 -0
  925. package/src/duckdb/src/planner/expression/bound_parameter_expression.cpp +28 -20
  926. package/src/duckdb/src/planner/expression/bound_window_expression.cpp +48 -4
  927. package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
  928. package/src/duckdb/src/planner/expression_binder/order_binder.cpp +5 -4
  929. package/src/duckdb/src/planner/expression_binder.cpp +23 -0
  930. package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
  931. package/src/duckdb/src/planner/logical_operator.cpp +19 -7
  932. package/src/duckdb/src/planner/logical_operator_visitor.cpp +5 -6
  933. package/src/duckdb/src/planner/operator/logical_comparison_join.cpp +4 -2
  934. package/src/duckdb/src/planner/operator/logical_copy_to_file.cpp +8 -0
  935. package/src/duckdb/src/planner/operator/logical_create.cpp +14 -0
  936. package/src/duckdb/src/planner/operator/logical_create_index.cpp +36 -7
  937. package/src/duckdb/src/planner/operator/logical_create_table.cpp +16 -0
  938. package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
  939. package/src/duckdb/src/planner/operator/logical_delete.cpp +9 -2
  940. package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
  941. package/src/duckdb/src/planner/operator/logical_distinct.cpp +13 -0
  942. package/src/duckdb/src/planner/operator/logical_explain.cpp +1 -1
  943. package/src/duckdb/src/planner/operator/logical_extension_operator.cpp +39 -0
  944. package/src/duckdb/src/planner/operator/logical_get.cpp +82 -4
  945. package/src/duckdb/src/planner/operator/logical_insert.cpp +8 -2
  946. package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +22 -0
  947. package/src/duckdb/src/planner/operator/logical_order.cpp +39 -0
  948. package/src/duckdb/src/planner/operator/logical_pivot.cpp +3 -0
  949. package/src/duckdb/src/planner/operator/logical_recursive_cte.cpp +5 -5
  950. package/src/duckdb/src/planner/operator/logical_sample.cpp +3 -0
  951. package/src/duckdb/src/planner/operator/logical_update.cpp +8 -2
  952. package/src/duckdb/src/planner/parsed_data/bound_create_table_info.cpp +4 -2
  953. package/src/duckdb/src/planner/planner.cpp +18 -7
  954. package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
  955. package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
  956. package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
  957. package/src/duckdb/src/storage/arena_allocator.cpp +13 -2
  958. package/src/duckdb/src/storage/buffer/block_manager.cpp +13 -9
  959. package/src/duckdb/src/storage/checkpoint/row_group_writer.cpp +1 -1
  960. package/src/duckdb/src/storage/checkpoint/table_data_reader.cpp +3 -4
  961. package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +7 -7
  962. package/src/duckdb/src/storage/checkpoint_manager.cpp +78 -72
  963. package/src/duckdb/src/storage/compression/bitpacking.cpp +87 -63
  964. package/src/duckdb/src/storage/compression/bitpacking_hugeint.cpp +295 -0
  965. package/src/duckdb/src/storage/compression/fsst.cpp +1 -1
  966. package/src/duckdb/src/storage/compression/rle.cpp +52 -13
  967. package/src/duckdb/src/storage/data_table.cpp +36 -25
  968. package/src/duckdb/src/storage/index.cpp +4 -26
  969. package/src/duckdb/src/storage/local_storage.cpp +3 -4
  970. package/src/duckdb/src/storage/metadata/metadata_manager.cpp +267 -0
  971. package/src/duckdb/src/storage/metadata/metadata_reader.cpp +80 -0
  972. package/src/duckdb/src/storage/metadata/metadata_writer.cpp +86 -0
  973. package/src/duckdb/src/storage/serialization/serialize_constraint.cpp +98 -0
  974. package/src/duckdb/src/storage/serialization/serialize_create_info.cpp +194 -0
  975. package/src/duckdb/src/storage/serialization/serialize_expression.cpp +283 -0
  976. package/src/duckdb/src/storage/serialization/serialize_logical_operator.cpp +762 -0
  977. package/src/duckdb/src/storage/serialization/serialize_macro_function.cpp +62 -0
  978. package/src/duckdb/src/storage/serialization/serialize_nodes.cpp +461 -0
  979. package/src/duckdb/src/storage/serialization/serialize_parse_info.cpp +421 -0
  980. package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +342 -0
  981. package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
  982. package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +97 -0
  983. package/src/duckdb/src/storage/serialization/serialize_statement.cpp +22 -0
  984. package/src/duckdb/src/storage/serialization/serialize_storage.cpp +39 -0
  985. package/src/duckdb/src/storage/serialization/serialize_table_filter.cpp +97 -0
  986. package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +164 -0
  987. package/src/duckdb/src/storage/serialization/serialize_types.cpp +127 -0
  988. package/src/duckdb/src/storage/single_file_block_manager.cpp +69 -51
  989. package/src/duckdb/src/storage/statistics/base_statistics.cpp +67 -4
  990. package/src/duckdb/src/storage/statistics/column_statistics.cpp +16 -0
  991. package/src/duckdb/src/storage/statistics/list_stats.cpp +21 -0
  992. package/src/duckdb/src/storage/statistics/numeric_stats.cpp +126 -1
  993. package/src/duckdb/src/storage/statistics/string_stats.cpp +44 -2
  994. package/src/duckdb/src/storage/statistics/struct_stats.cpp +27 -0
  995. package/src/duckdb/src/storage/storage_info.cpp +3 -2
  996. package/src/duckdb/src/storage/storage_manager.cpp +11 -5
  997. package/src/duckdb/src/storage/table/chunk_info.cpp +99 -3
  998. package/src/duckdb/src/storage/table/column_data_checkpointer.cpp +3 -3
  999. package/src/duckdb/src/storage/table/list_column_data.cpp +6 -3
  1000. package/src/duckdb/src/storage/table/persistent_table_data.cpp +1 -2
  1001. package/src/duckdb/src/storage/table/row_group.cpp +102 -20
  1002. package/src/duckdb/src/storage/table/row_group_collection.cpp +23 -19
  1003. package/src/duckdb/src/storage/table/table_statistics.cpp +21 -0
  1004. package/src/duckdb/src/storage/table/update_segment.cpp +1 -1
  1005. package/src/duckdb/src/storage/table_index_list.cpp +1 -1
  1006. package/src/duckdb/src/storage/wal_replay.cpp +26 -26
  1007. package/src/duckdb/src/storage/write_ahead_log.cpp +3 -2
  1008. package/src/duckdb/src/verification/deserialized_statement_verifier_v2.cpp +15 -1
  1009. package/src/duckdb/src/verification/prepared_statement_verifier.cpp +16 -11
  1010. package/src/duckdb/src/verification/statement_verifier.cpp +2 -0
  1011. package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
  1012. package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +5 -2
  1013. package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
  1014. package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
  1015. package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +11 -0
  1016. package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
  1017. package/src/duckdb/third_party/libpg_query/pg_functions.cpp +13 -0
  1018. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +11019 -10364
  1019. package/src/duckdb/third_party/libpg_query/src_backend_parser_scansup.cpp +9 -0
  1020. package/src/duckdb/third_party/mbedtls/include/mbedtls_wrapper.hpp +10 -0
  1021. package/src/duckdb/third_party/mbedtls/mbedtls_wrapper.cpp +31 -1
  1022. package/src/duckdb/third_party/utf8proc/include/utf8proc_wrapper.hpp +8 -0
  1023. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +4 -4
  1024. package/src/duckdb/ub_src_common.cpp +4 -0
  1025. package/src/duckdb/ub_src_common_adbc_nanoarrow.cpp +8 -0
  1026. package/src/duckdb/ub_src_common_arrow_appender.cpp +10 -0
  1027. package/src/duckdb/ub_src_common_serializer.cpp +2 -0
  1028. package/src/duckdb/ub_src_core_functions_scalar_debug.cpp +2 -0
  1029. package/src/duckdb/ub_src_core_functions_scalar_string.cpp +2 -0
  1030. package/src/duckdb/ub_src_execution.cpp +2 -2
  1031. package/src/duckdb/ub_src_execution_index_art.cpp +0 -6
  1032. package/src/duckdb/ub_src_execution_operator_csv_scanner.cpp +18 -0
  1033. package/src/duckdb/ub_src_execution_operator_csv_scanner_sniffer.cpp +12 -0
  1034. package/src/duckdb/ub_src_execution_operator_persistent.cpp +1 -11
  1035. package/src/duckdb/ub_src_execution_operator_schema.cpp +1 -1
  1036. package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
  1037. package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
  1038. package/src/duckdb/ub_src_function_scalar.cpp +2 -0
  1039. package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
  1040. package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
  1041. package/src/duckdb/ub_src_function_table_arrow.cpp +2 -0
  1042. package/src/duckdb/ub_src_main.cpp +2 -0
  1043. package/src/duckdb/ub_src_main_chunk_scan_state.cpp +2 -0
  1044. package/src/duckdb/ub_src_optimizer.cpp +6 -0
  1045. package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
  1046. package/src/duckdb/ub_src_optimizer_join_order.cpp +10 -0
  1047. package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
  1048. package/src/duckdb/ub_src_parser.cpp +0 -2
  1049. package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
  1050. package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
  1051. package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
  1052. package/src/duckdb/ub_src_planner_operator.cpp +3 -3
  1053. package/src/duckdb/ub_src_storage.cpp +0 -4
  1054. package/src/duckdb/ub_src_storage_compression.cpp +2 -0
  1055. package/src/duckdb/ub_src_storage_metadata.cpp +6 -0
  1056. package/src/duckdb/ub_src_storage_serialization.cpp +30 -0
  1057. package/src/duckdb_node.hpp +1 -0
  1058. package/src/statement.cpp +10 -5
  1059. package/test/columns.test.ts +25 -3
  1060. package/test/extension.test.ts +1 -1
  1061. package/test/test_all_types.test.ts +234 -0
  1062. package/tsconfig.json +1 -0
  1063. package/src/duckdb/src/execution/index/art/leaf_segment.cpp +0 -52
  1064. package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
  1065. package/src/duckdb/src/execution/index/art/swizzleable_pointer.cpp +0 -22
  1066. package/src/duckdb/src/execution/operator/persistent/buffered_csv_reader.cpp +0 -1487
  1067. package/src/duckdb/src/execution/operator/persistent/csv_buffer.cpp +0 -72
  1068. package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +0 -158
  1069. package/src/duckdb/src/execution/operator/schema/physical_create_index.cpp +0 -193
  1070. package/src/duckdb/src/execution/partitionable_hashtable.cpp +0 -172
  1071. package/src/duckdb/src/include/duckdb/common/arrow/arrow_options.hpp +0 -25
  1072. package/src/duckdb/src/include/duckdb/execution/index/art/leaf_segment.hpp +0 -38
  1073. package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
  1074. package/src/duckdb/src/include/duckdb/execution/index/art/swizzleable_pointer.hpp +0 -58
  1075. package/src/duckdb/src/include/duckdb/execution/operator/persistent/buffered_csv_reader.hpp +0 -133
  1076. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_buffer.hpp +0 -74
  1077. package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +0 -69
  1078. package/src/duckdb/src/include/duckdb/planner/operator/logical_asof_join.hpp +0 -27
  1079. package/src/duckdb/src/include/duckdb/planner/operator/logical_delim_join.hpp +0 -32
  1080. package/src/duckdb/src/include/duckdb/storage/meta_block_reader.hpp +0 -49
  1081. package/src/duckdb/src/include/duckdb/storage/meta_block_writer.hpp +0 -50
  1082. package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
  1083. package/src/duckdb/src/parser/common_table_expression_info.cpp +0 -19
  1084. package/src/duckdb/src/planner/operator/logical_asof_join.cpp +0 -14
  1085. package/src/duckdb/src/planner/operator/logical_delim_join.cpp +0 -27
  1086. package/src/duckdb/src/storage/meta_block_reader.cpp +0 -78
  1087. package/src/duckdb/src/storage/meta_block_writer.cpp +0 -80
@@ -0,0 +1,1285 @@
1
+ #include "duckdb/execution/window_executor.hpp"
2
+
3
+ #include "duckdb/common/operator/add.hpp"
4
+ #include "duckdb/common/operator/subtract.hpp"
5
+
6
+ namespace duckdb {
7
+
8
+ static idx_t FindNextStart(const ValidityMask &mask, idx_t l, const idx_t r, idx_t &n) {
9
+ if (mask.AllValid()) {
10
+ auto start = MinValue(l + n - 1, r);
11
+ n -= MinValue(n, r - l);
12
+ return start;
13
+ }
14
+
15
+ while (l < r) {
16
+ // If l is aligned with the start of a block, and the block is blank, then skip forward one block.
17
+ idx_t entry_idx;
18
+ idx_t shift;
19
+ mask.GetEntryIndex(l, entry_idx, shift);
20
+
21
+ const auto block = mask.GetValidityEntry(entry_idx);
22
+ if (mask.NoneValid(block) && !shift) {
23
+ l += ValidityMask::BITS_PER_VALUE;
24
+ continue;
25
+ }
26
+
27
+ // Loop over the block
28
+ for (; shift < ValidityMask::BITS_PER_VALUE && l < r; ++shift, ++l) {
29
+ if (mask.RowIsValid(block, shift) && --n == 0) {
30
+ return MinValue(l, r);
31
+ }
32
+ }
33
+ }
34
+
35
+ // Didn't find a start so return the end of the range
36
+ return r;
37
+ }
38
+
39
+ static idx_t FindPrevStart(const ValidityMask &mask, const idx_t l, idx_t r, idx_t &n) {
40
+ if (mask.AllValid()) {
41
+ auto start = (r <= l + n) ? l : r - n;
42
+ n -= r - start;
43
+ return start;
44
+ }
45
+
46
+ while (l < r) {
47
+ // If r is aligned with the start of a block, and the previous block is blank,
48
+ // then skip backwards one block.
49
+ idx_t entry_idx;
50
+ idx_t shift;
51
+ mask.GetEntryIndex(r - 1, entry_idx, shift);
52
+
53
+ const auto block = mask.GetValidityEntry(entry_idx);
54
+ if (mask.NoneValid(block) && (shift + 1 == ValidityMask::BITS_PER_VALUE)) {
55
+ // r is nonzero (> l) and word aligned, so this will not underflow.
56
+ r -= ValidityMask::BITS_PER_VALUE;
57
+ continue;
58
+ }
59
+
60
+ // Loop backwards over the block
61
+ // shift is probing r-1 >= l >= 0
62
+ for (++shift; shift-- > 0; --r) {
63
+ if (mask.RowIsValid(block, shift) && --n == 0) {
64
+ return MaxValue(l, r - 1);
65
+ }
66
+ }
67
+ }
68
+
69
+ // Didn't find a start so return the start of the range
70
+ return l;
71
+ }
72
+
73
+ template <typename T>
74
+ static T GetCell(const DataChunk &chunk, idx_t column, idx_t index) {
75
+ D_ASSERT(chunk.ColumnCount() > column);
76
+ auto &source = chunk.data[column];
77
+ const auto data = FlatVector::GetData<T>(source);
78
+ return data[index];
79
+ }
80
+
81
+ static bool CellIsNull(const DataChunk &chunk, idx_t column, idx_t index) {
82
+ D_ASSERT(chunk.ColumnCount() > column);
83
+ auto &source = chunk.data[column];
84
+ return FlatVector::IsNull(source, index);
85
+ }
86
+
87
+ static void CopyCell(const DataChunk &chunk, idx_t column, idx_t index, Vector &target, idx_t target_offset) {
88
+ D_ASSERT(chunk.ColumnCount() > column);
89
+ auto &source = chunk.data[column];
90
+ VectorOperations::Copy(source, target, index + 1, index, target_offset);
91
+ }
92
+
93
+ //===--------------------------------------------------------------------===//
94
+ // WindowColumnIterator
95
+ //===--------------------------------------------------------------------===//
96
+ template <typename T>
97
+ struct WindowColumnIterator {
98
+ using iterator = WindowColumnIterator<T>;
99
+ using iterator_category = std::random_access_iterator_tag;
100
+ using difference_type = std::ptrdiff_t;
101
+ using value_type = T;
102
+ using reference = T;
103
+ using pointer = idx_t;
104
+
105
+ explicit WindowColumnIterator(const WindowInputColumn &coll_p, pointer pos_p = 0) : coll(&coll_p), pos(pos_p) {
106
+ }
107
+
108
+ // Forward iterator
109
+ inline reference operator*() const {
110
+ return coll->GetCell<T>(pos);
111
+ }
112
+ inline explicit operator pointer() const {
113
+ return pos;
114
+ }
115
+
116
+ inline iterator &operator++() {
117
+ ++pos;
118
+ return *this;
119
+ }
120
+ inline iterator operator++(int) {
121
+ auto result = *this;
122
+ ++(*this);
123
+ return result;
124
+ }
125
+
126
+ // Bidirectional iterator
127
+ inline iterator &operator--() {
128
+ --pos;
129
+ return *this;
130
+ }
131
+ inline iterator operator--(int) {
132
+ auto result = *this;
133
+ --(*this);
134
+ return result;
135
+ }
136
+
137
+ // Random Access
138
+ inline iterator &operator+=(difference_type n) {
139
+ pos += n;
140
+ return *this;
141
+ }
142
+ inline iterator &operator-=(difference_type n) {
143
+ pos -= n;
144
+ return *this;
145
+ }
146
+
147
+ inline reference operator[](difference_type m) const {
148
+ return coll->GetCell<T>(pos + m);
149
+ }
150
+
151
+ friend inline iterator &operator+(const iterator &a, difference_type n) {
152
+ return iterator(a.coll, a.pos + n);
153
+ }
154
+
155
+ friend inline iterator &operator-(const iterator &a, difference_type n) {
156
+ return iterator(a.coll, a.pos - n);
157
+ }
158
+
159
+ friend inline iterator &operator+(difference_type n, const iterator &a) {
160
+ return a + n;
161
+ }
162
+ friend inline difference_type operator-(const iterator &a, const iterator &b) {
163
+ return difference_type(a.pos - b.pos);
164
+ }
165
+
166
+ friend inline bool operator==(const iterator &a, const iterator &b) {
167
+ return a.pos == b.pos;
168
+ }
169
+ friend inline bool operator!=(const iterator &a, const iterator &b) {
170
+ return a.pos != b.pos;
171
+ }
172
+ friend inline bool operator<(const iterator &a, const iterator &b) {
173
+ return a.pos < b.pos;
174
+ }
175
+ friend inline bool operator<=(const iterator &a, const iterator &b) {
176
+ return a.pos <= b.pos;
177
+ }
178
+ friend inline bool operator>(const iterator &a, const iterator &b) {
179
+ return a.pos > b.pos;
180
+ }
181
+ friend inline bool operator>=(const iterator &a, const iterator &b) {
182
+ return a.pos >= b.pos;
183
+ }
184
+
185
+ private:
186
+ optional_ptr<const WindowInputColumn> coll;
187
+ pointer pos;
188
+ };
189
+
190
+ template <typename T, typename OP>
191
+ struct OperationCompare : public std::function<bool(T, T)> {
192
+ inline bool operator()(const T &lhs, const T &val) const {
193
+ return OP::template Operation(lhs, val);
194
+ }
195
+ };
196
+
197
+ template <typename T, typename OP, bool FROM>
198
+ static idx_t FindTypedRangeBound(const WindowInputColumn &over, const idx_t order_begin, const idx_t order_end,
199
+ WindowInputExpression &boundary, const idx_t chunk_idx, const FrameBounds &prev) {
200
+ D_ASSERT(!boundary.CellIsNull(chunk_idx));
201
+ const auto val = boundary.GetCell<T>(chunk_idx);
202
+
203
+ OperationCompare<T, OP> comp;
204
+ WindowColumnIterator<T> begin(over, order_begin);
205
+ WindowColumnIterator<T> end(over, order_end);
206
+
207
+ if (order_begin < prev.start && prev.start < order_end) {
208
+ const auto first = over.GetCell<T>(prev.start);
209
+ if (!comp(val, first)) {
210
+ // prev.first <= val, so we can start further forward
211
+ begin += (prev.start - order_begin);
212
+ }
213
+ }
214
+ if (order_begin <= prev.end && prev.end < order_end) {
215
+ const auto second = over.GetCell<T>(prev.end);
216
+ if (!comp(second, val)) {
217
+ // val <= prev.second, so we can end further back
218
+ // (prev.second is the largest peer)
219
+ end -= (order_end - prev.end - 1);
220
+ }
221
+ }
222
+
223
+ if (FROM) {
224
+ return idx_t(std::lower_bound(begin, end, val, comp));
225
+ } else {
226
+ return idx_t(std::upper_bound(begin, end, val, comp));
227
+ }
228
+ }
229
+
230
+ template <typename OP, bool FROM>
231
+ static idx_t FindRangeBound(const WindowInputColumn &over, const idx_t order_begin, const idx_t order_end,
232
+ WindowInputExpression &boundary, const idx_t chunk_idx, const FrameBounds &prev) {
233
+ D_ASSERT(boundary.chunk.ColumnCount() == 1);
234
+ D_ASSERT(boundary.chunk.data[0].GetType().InternalType() == over.input_expr.ptype);
235
+
236
+ switch (over.input_expr.ptype) {
237
+ case PhysicalType::INT8:
238
+ return FindTypedRangeBound<int8_t, OP, FROM>(over, order_begin, order_end, boundary, chunk_idx, prev);
239
+ case PhysicalType::INT16:
240
+ return FindTypedRangeBound<int16_t, OP, FROM>(over, order_begin, order_end, boundary, chunk_idx, prev);
241
+ case PhysicalType::INT32:
242
+ return FindTypedRangeBound<int32_t, OP, FROM>(over, order_begin, order_end, boundary, chunk_idx, prev);
243
+ case PhysicalType::INT64:
244
+ return FindTypedRangeBound<int64_t, OP, FROM>(over, order_begin, order_end, boundary, chunk_idx, prev);
245
+ case PhysicalType::UINT8:
246
+ return FindTypedRangeBound<uint8_t, OP, FROM>(over, order_begin, order_end, boundary, chunk_idx, prev);
247
+ case PhysicalType::UINT16:
248
+ return FindTypedRangeBound<uint16_t, OP, FROM>(over, order_begin, order_end, boundary, chunk_idx, prev);
249
+ case PhysicalType::UINT32:
250
+ return FindTypedRangeBound<uint32_t, OP, FROM>(over, order_begin, order_end, boundary, chunk_idx, prev);
251
+ case PhysicalType::UINT64:
252
+ return FindTypedRangeBound<uint64_t, OP, FROM>(over, order_begin, order_end, boundary, chunk_idx, prev);
253
+ case PhysicalType::INT128:
254
+ return FindTypedRangeBound<hugeint_t, OP, FROM>(over, order_begin, order_end, boundary, chunk_idx, prev);
255
+ case PhysicalType::FLOAT:
256
+ return FindTypedRangeBound<float, OP, FROM>(over, order_begin, order_end, boundary, chunk_idx, prev);
257
+ case PhysicalType::DOUBLE:
258
+ return FindTypedRangeBound<double, OP, FROM>(over, order_begin, order_end, boundary, chunk_idx, prev);
259
+ case PhysicalType::INTERVAL:
260
+ return FindTypedRangeBound<interval_t, OP, FROM>(over, order_begin, order_end, boundary, chunk_idx, prev);
261
+ default:
262
+ throw InternalException("Unsupported column type for RANGE");
263
+ }
264
+ }
265
+
266
+ template <bool FROM>
267
+ static idx_t FindOrderedRangeBound(const WindowInputColumn &over, const OrderType range_sense, const idx_t order_begin,
268
+ const idx_t order_end, WindowInputExpression &boundary, const idx_t chunk_idx,
269
+ const FrameBounds &prev) {
270
+ switch (range_sense) {
271
+ case OrderType::ASCENDING:
272
+ return FindRangeBound<LessThan, FROM>(over, order_begin, order_end, boundary, chunk_idx, prev);
273
+ case OrderType::DESCENDING:
274
+ return FindRangeBound<GreaterThan, FROM>(over, order_begin, order_end, boundary, chunk_idx, prev);
275
+ default:
276
+ throw InternalException("Unsupported ORDER BY sense for RANGE");
277
+ }
278
+ }
279
+
280
+ struct WindowBoundariesState {
281
+ static inline bool IsScalar(const unique_ptr<Expression> &expr) {
282
+ return expr ? expr->IsScalar() : true;
283
+ }
284
+
285
+ static inline bool BoundaryNeedsPeer(const WindowBoundary &boundary) {
286
+ switch (boundary) {
287
+ case WindowBoundary::CURRENT_ROW_RANGE:
288
+ case WindowBoundary::EXPR_PRECEDING_RANGE:
289
+ case WindowBoundary::EXPR_FOLLOWING_RANGE:
290
+ return true;
291
+ default:
292
+ return false;
293
+ }
294
+ }
295
+
296
+ WindowBoundariesState(BoundWindowExpression &wexpr, const idx_t input_size);
297
+
298
+ void Update(const idx_t row_idx, const WindowInputColumn &range_collection, const idx_t chunk_idx,
299
+ WindowInputExpression &boundary_start, WindowInputExpression &boundary_end,
300
+ const ValidityMask &partition_mask, const ValidityMask &order_mask);
301
+
302
+ void Bounds(DataChunk &bounds, idx_t row_idx, const WindowInputColumn &range, const idx_t count,
303
+ WindowInputExpression &boundary_start, WindowInputExpression &boundary_end,
304
+ const ValidityMask &partition_mask, const ValidityMask &order_mask);
305
+
306
+ // Cached lookups
307
+ const ExpressionType type;
308
+ const idx_t input_size;
309
+ const WindowBoundary start_boundary;
310
+ const WindowBoundary end_boundary;
311
+ const size_t partition_count;
312
+ const size_t order_count;
313
+ const OrderType range_sense;
314
+ const bool has_preceding_range;
315
+ const bool has_following_range;
316
+ const bool needs_peer;
317
+
318
+ idx_t next_pos = 0;
319
+ idx_t partition_start = 0;
320
+ idx_t partition_end = 0;
321
+ idx_t peer_start = 0;
322
+ idx_t peer_end = 0;
323
+ idx_t valid_start = 0;
324
+ idx_t valid_end = 0;
325
+ int64_t window_start = -1;
326
+ int64_t window_end = -1;
327
+ FrameBounds prev;
328
+ };
329
+
330
+ //===--------------------------------------------------------------------===//
331
+ // WindowBoundariesState
332
+ //===--------------------------------------------------------------------===//
333
+ void WindowBoundariesState::Update(const idx_t row_idx, const WindowInputColumn &range_collection,
334
+ const idx_t chunk_idx, WindowInputExpression &boundary_start,
335
+ WindowInputExpression &boundary_end, const ValidityMask &partition_mask,
336
+ const ValidityMask &order_mask) {
337
+
338
+ if (partition_count + order_count > 0) {
339
+
340
+ // determine partition and peer group boundaries to ultimately figure out window size
341
+ const auto is_same_partition = !partition_mask.RowIsValidUnsafe(row_idx);
342
+ const auto is_peer = !order_mask.RowIsValidUnsafe(row_idx);
343
+ const auto is_jump = (next_pos != row_idx);
344
+
345
+ // when the partition changes, recompute the boundaries
346
+ if (!is_same_partition || is_jump) {
347
+ if (is_jump) {
348
+ idx_t n = 1;
349
+ partition_start = FindPrevStart(partition_mask, 0, row_idx + 1, n);
350
+ n = 1;
351
+ peer_start = FindPrevStart(order_mask, 0, row_idx + 1, n);
352
+ } else {
353
+ partition_start = row_idx;
354
+ peer_start = row_idx;
355
+ }
356
+
357
+ // find end of partition
358
+ partition_end = input_size;
359
+ if (partition_count) {
360
+ idx_t n = 1;
361
+ partition_end = FindNextStart(partition_mask, partition_start + 1, input_size, n);
362
+ }
363
+
364
+ // Find valid ordering values for the new partition
365
+ // so we can exclude NULLs from RANGE expression computations
366
+ valid_start = partition_start;
367
+ valid_end = partition_end;
368
+
369
+ if ((valid_start < valid_end) && has_preceding_range) {
370
+ // Exclude any leading NULLs
371
+ if (range_collection.CellIsNull(valid_start)) {
372
+ idx_t n = 1;
373
+ valid_start = FindNextStart(order_mask, valid_start + 1, valid_end, n);
374
+ }
375
+ }
376
+
377
+ if ((valid_start < valid_end) && has_following_range) {
378
+ // Exclude any trailing NULLs
379
+ if (range_collection.CellIsNull(valid_end - 1)) {
380
+ idx_t n = 1;
381
+ valid_end = FindPrevStart(order_mask, valid_start, valid_end, n);
382
+ }
383
+
384
+ // Reset range hints
385
+ prev.start = valid_start;
386
+ prev.end = valid_end;
387
+ }
388
+ } else if (!is_peer) {
389
+ peer_start = row_idx;
390
+ }
391
+
392
+ if (needs_peer) {
393
+ peer_end = partition_end;
394
+ if (order_count) {
395
+ idx_t n = 1;
396
+ peer_end = FindNextStart(order_mask, peer_start + 1, partition_end, n);
397
+ }
398
+ }
399
+
400
+ } else {
401
+ // OVER()
402
+ partition_end = input_size;
403
+ peer_end = partition_end;
404
+ }
405
+ next_pos = row_idx + 1;
406
+
407
+ // determine window boundaries depending on the type of expression
408
+ window_start = -1;
409
+ window_end = -1;
410
+
411
+ switch (start_boundary) {
412
+ case WindowBoundary::UNBOUNDED_PRECEDING:
413
+ window_start = partition_start;
414
+ break;
415
+ case WindowBoundary::CURRENT_ROW_ROWS:
416
+ window_start = row_idx;
417
+ break;
418
+ case WindowBoundary::CURRENT_ROW_RANGE:
419
+ window_start = peer_start;
420
+ break;
421
+ case WindowBoundary::EXPR_PRECEDING_ROWS: {
422
+ if (!TrySubtractOperator::Operation(int64_t(row_idx), boundary_start.GetCell<int64_t>(chunk_idx),
423
+ window_start)) {
424
+ throw OutOfRangeException("Overflow computing ROWS PRECEDING start");
425
+ }
426
+ break;
427
+ }
428
+ case WindowBoundary::EXPR_FOLLOWING_ROWS: {
429
+ if (!TryAddOperator::Operation(int64_t(row_idx), boundary_start.GetCell<int64_t>(chunk_idx), window_start)) {
430
+ throw OutOfRangeException("Overflow computing ROWS FOLLOWING start");
431
+ }
432
+ break;
433
+ }
434
+ case WindowBoundary::EXPR_PRECEDING_RANGE: {
435
+ if (boundary_start.CellIsNull(chunk_idx)) {
436
+ window_start = peer_start;
437
+ } else {
438
+ prev.start = FindOrderedRangeBound<true>(range_collection, range_sense, valid_start, row_idx,
439
+ boundary_start, chunk_idx, prev);
440
+ window_start = prev.start;
441
+ }
442
+ break;
443
+ }
444
+ case WindowBoundary::EXPR_FOLLOWING_RANGE: {
445
+ if (boundary_start.CellIsNull(chunk_idx)) {
446
+ window_start = peer_start;
447
+ } else {
448
+ prev.start = FindOrderedRangeBound<true>(range_collection, range_sense, row_idx, valid_end, boundary_start,
449
+ chunk_idx, prev);
450
+ window_start = prev.start;
451
+ }
452
+ break;
453
+ }
454
+ default:
455
+ throw InternalException("Unsupported window start boundary");
456
+ }
457
+
458
+ switch (end_boundary) {
459
+ case WindowBoundary::CURRENT_ROW_ROWS:
460
+ window_end = row_idx + 1;
461
+ break;
462
+ case WindowBoundary::CURRENT_ROW_RANGE:
463
+ window_end = peer_end;
464
+ break;
465
+ case WindowBoundary::UNBOUNDED_FOLLOWING:
466
+ window_end = partition_end;
467
+ break;
468
+ case WindowBoundary::EXPR_PRECEDING_ROWS:
469
+ if (!TrySubtractOperator::Operation(int64_t(row_idx + 1), boundary_end.GetCell<int64_t>(chunk_idx),
470
+ window_end)) {
471
+ throw OutOfRangeException("Overflow computing ROWS PRECEDING end");
472
+ }
473
+ break;
474
+ case WindowBoundary::EXPR_FOLLOWING_ROWS:
475
+ if (!TryAddOperator::Operation(int64_t(row_idx + 1), boundary_end.GetCell<int64_t>(chunk_idx), window_end)) {
476
+ throw OutOfRangeException("Overflow computing ROWS FOLLOWING end");
477
+ }
478
+ break;
479
+ case WindowBoundary::EXPR_PRECEDING_RANGE: {
480
+ if (boundary_end.CellIsNull(chunk_idx)) {
481
+ window_end = peer_end;
482
+ } else {
483
+ prev.end = FindOrderedRangeBound<false>(range_collection, range_sense, valid_start, row_idx, boundary_end,
484
+ chunk_idx, prev);
485
+ window_end = prev.end;
486
+ }
487
+ break;
488
+ }
489
+ case WindowBoundary::EXPR_FOLLOWING_RANGE: {
490
+ if (boundary_end.CellIsNull(chunk_idx)) {
491
+ window_end = peer_end;
492
+ } else {
493
+ prev.end = FindOrderedRangeBound<false>(range_collection, range_sense, row_idx, valid_end, boundary_end,
494
+ chunk_idx, prev);
495
+ window_end = prev.end;
496
+ }
497
+ break;
498
+ }
499
+ default:
500
+ throw InternalException("Unsupported window end boundary");
501
+ }
502
+
503
+ // clamp windows to partitions if they should exceed
504
+ if (window_start < (int64_t)partition_start) {
505
+ window_start = partition_start;
506
+ }
507
+ if (window_start > (int64_t)partition_end) {
508
+ window_start = partition_end;
509
+ }
510
+ if (window_end < (int64_t)partition_start) {
511
+ window_end = partition_start;
512
+ }
513
+ if (window_end > (int64_t)partition_end) {
514
+ window_end = partition_end;
515
+ }
516
+
517
+ if (window_start < 0 || window_end < 0) {
518
+ throw InternalException("Failed to compute window boundaries");
519
+ }
520
+ }
521
+
522
+ static bool HasPrecedingRange(BoundWindowExpression &wexpr) {
523
+ return (wexpr.start == WindowBoundary::EXPR_PRECEDING_RANGE || wexpr.end == WindowBoundary::EXPR_PRECEDING_RANGE);
524
+ }
525
+
526
+ static bool HasFollowingRange(BoundWindowExpression &wexpr) {
527
+ return (wexpr.start == WindowBoundary::EXPR_FOLLOWING_RANGE || wexpr.end == WindowBoundary::EXPR_FOLLOWING_RANGE);
528
+ }
529
+
530
+ WindowBoundariesState::WindowBoundariesState(BoundWindowExpression &wexpr, const idx_t input_size)
531
+ : type(wexpr.type), input_size(input_size), start_boundary(wexpr.start), end_boundary(wexpr.end),
532
+ partition_count(wexpr.partitions.size()), order_count(wexpr.orders.size()),
533
+ range_sense(wexpr.orders.empty() ? OrderType::INVALID : wexpr.orders[0].type),
534
+ has_preceding_range(HasPrecedingRange(wexpr)), has_following_range(HasFollowingRange(wexpr)),
535
+ needs_peer(BoundaryNeedsPeer(wexpr.end) || wexpr.type == ExpressionType::WINDOW_CUME_DIST) {
536
+ }
537
+
538
+ void WindowBoundariesState::Bounds(DataChunk &bounds, idx_t row_idx, const WindowInputColumn &range, const idx_t count,
539
+ WindowInputExpression &boundary_start, WindowInputExpression &boundary_end,
540
+ const ValidityMask &partition_mask, const ValidityMask &order_mask) {
541
+ bounds.Reset();
542
+ D_ASSERT(bounds.ColumnCount() == 6);
543
+ auto partition_begin_data = FlatVector::GetData<idx_t>(bounds.data[PARTITION_BEGIN]);
544
+ auto partition_end_data = FlatVector::GetData<idx_t>(bounds.data[PARTITION_END]);
545
+ auto peer_begin_data = FlatVector::GetData<idx_t>(bounds.data[PEER_BEGIN]);
546
+ auto peer_end_data = FlatVector::GetData<idx_t>(bounds.data[PEER_END]);
547
+ auto window_begin_data = FlatVector::GetData<int64_t>(bounds.data[WINDOW_BEGIN]);
548
+ auto window_end_data = FlatVector::GetData<int64_t>(bounds.data[WINDOW_END]);
549
+ for (idx_t chunk_idx = 0; chunk_idx < count; ++chunk_idx, ++row_idx) {
550
+ Update(row_idx, range, chunk_idx, boundary_start, boundary_end, partition_mask, order_mask);
551
+ *partition_begin_data++ = partition_start;
552
+ *partition_end_data++ = partition_end;
553
+ if (needs_peer) {
554
+ *peer_begin_data++ = peer_start;
555
+ *peer_end_data++ = peer_end;
556
+ }
557
+ *window_begin_data++ = window_start;
558
+ *window_end_data++ = window_end;
559
+ }
560
+ bounds.SetCardinality(count);
561
+ }
562
+
563
+ //===--------------------------------------------------------------------===//
564
+ // WindowExecutorBoundsState
565
+ //===--------------------------------------------------------------------===//
566
+ class WindowExecutorBoundsState : public WindowExecutorState {
567
+ public:
568
+ WindowExecutorBoundsState(BoundWindowExpression &wexpr, ClientContext &context, const idx_t count,
569
+ const ValidityMask &partition_mask_p, const ValidityMask &order_mask_p);
570
+ ~WindowExecutorBoundsState() override {
571
+ }
572
+
573
+ virtual void UpdateBounds(idx_t row_idx, DataChunk &input_chunk, const WindowInputColumn &range);
574
+
575
+ // Frame management
576
+ const ValidityMask &partition_mask;
577
+ const ValidityMask &order_mask;
578
+ DataChunk bounds;
579
+ WindowBoundariesState state;
580
+
581
+ // evaluate boundaries if present. Parser has checked boundary types.
582
+ WindowInputExpression boundary_start;
583
+ WindowInputExpression boundary_end;
584
+ };
585
+
586
+ WindowExecutorBoundsState::WindowExecutorBoundsState(BoundWindowExpression &wexpr, ClientContext &context,
587
+ const idx_t payload_count, const ValidityMask &partition_mask_p,
588
+ const ValidityMask &order_mask_p)
589
+ : partition_mask(partition_mask_p), order_mask(order_mask_p), state(wexpr, payload_count),
590
+ boundary_start(wexpr.start_expr.get(), context), boundary_end(wexpr.end_expr.get(), context) {
591
+ vector<LogicalType> bounds_types(6, LogicalType(LogicalTypeId::UBIGINT));
592
+ bounds.Initialize(Allocator::Get(context), bounds_types);
593
+ }
594
+
595
+ void WindowExecutorBoundsState::UpdateBounds(idx_t row_idx, DataChunk &input_chunk, const WindowInputColumn &range) {
596
+ // Evaluate the row-level arguments
597
+ boundary_start.Execute(input_chunk);
598
+ boundary_end.Execute(input_chunk);
599
+
600
+ const auto count = input_chunk.size();
601
+ bounds.Reset();
602
+ state.Bounds(bounds, row_idx, range, count, boundary_start, boundary_end, partition_mask, order_mask);
603
+ }
604
+
605
+ //===--------------------------------------------------------------------===//
606
+ // WindowExecutor
607
+ //===--------------------------------------------------------------------===//
608
+ static void PrepareInputExpressions(vector<unique_ptr<Expression>> &exprs, ExpressionExecutor &executor,
609
+ DataChunk &chunk) {
610
+ if (exprs.empty()) {
611
+ return;
612
+ }
613
+
614
+ vector<LogicalType> types;
615
+ for (idx_t expr_idx = 0; expr_idx < exprs.size(); ++expr_idx) {
616
+ types.push_back(exprs[expr_idx]->return_type);
617
+ executor.AddExpression(*exprs[expr_idx]);
618
+ }
619
+
620
+ if (!types.empty()) {
621
+ auto &allocator = executor.GetAllocator();
622
+ chunk.Initialize(allocator, types);
623
+ }
624
+ }
625
+
626
+ WindowExecutor::WindowExecutor(BoundWindowExpression &wexpr, ClientContext &context, const idx_t payload_count,
627
+ const ValidityMask &partition_mask, const ValidityMask &order_mask)
628
+ : wexpr(wexpr), context(context), payload_count(payload_count), partition_mask(partition_mask),
629
+ order_mask(order_mask), payload_collection(), payload_executor(context),
630
+ range((HasPrecedingRange(wexpr) || HasFollowingRange(wexpr)) ? wexpr.orders[0].expression.get() : nullptr,
631
+ context, payload_count) {
632
+ // TODO: child may be a scalar, don't need to materialize the whole collection then
633
+
634
+ // evaluate inner expressions of window functions, could be more complex
635
+ PrepareInputExpressions(wexpr.children, payload_executor, payload_chunk);
636
+
637
+ auto types = payload_chunk.GetTypes();
638
+ if (!types.empty()) {
639
+ payload_collection.Initialize(Allocator::Get(context), types);
640
+ }
641
+ }
642
+
643
+ unique_ptr<WindowExecutorState> WindowExecutor::GetExecutorState() const {
644
+ return make_uniq<WindowExecutorBoundsState>(wexpr, context, payload_count, partition_mask, order_mask);
645
+ }
646
+
647
+ //===--------------------------------------------------------------------===//
648
+ // WindowAggregateExecutor
649
+ //===--------------------------------------------------------------------===//
650
+ bool WindowAggregateExecutor::IsConstantAggregate() {
651
+ if (!wexpr.aggregate) {
652
+ return false;
653
+ }
654
+
655
+ // COUNT(*) is already handled efficiently by segment trees.
656
+ if (wexpr.children.empty()) {
657
+ return false;
658
+ }
659
+
660
+ /*
661
+ The default framing option is RANGE UNBOUNDED PRECEDING, which
662
+ is the same as RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT
663
+ ROW; it sets the frame to be all rows from the partition start
664
+ up through the current row's last peer (a row that the window's
665
+ ORDER BY clause considers equivalent to the current row; all
666
+ rows are peers if there is no ORDER BY). In general, UNBOUNDED
667
+ PRECEDING means that the frame starts with the first row of the
668
+ partition, and similarly UNBOUNDED FOLLOWING means that the
669
+ frame ends with the last row of the partition, regardless of
670
+ RANGE, ROWS or GROUPS mode. In ROWS mode, CURRENT ROW means that
671
+ the frame starts or ends with the current row; but in RANGE or
672
+ GROUPS mode it means that the frame starts or ends with the
673
+ current row's first or last peer in the ORDER BY ordering. The
674
+ offset PRECEDING and offset FOLLOWING options vary in meaning
675
+ depending on the frame mode.
676
+ */
677
+ switch (wexpr.start) {
678
+ case WindowBoundary::UNBOUNDED_PRECEDING:
679
+ break;
680
+ case WindowBoundary::CURRENT_ROW_RANGE:
681
+ if (!wexpr.orders.empty()) {
682
+ return false;
683
+ }
684
+ break;
685
+ default:
686
+ return false;
687
+ }
688
+
689
+ switch (wexpr.end) {
690
+ case WindowBoundary::UNBOUNDED_FOLLOWING:
691
+ break;
692
+ case WindowBoundary::CURRENT_ROW_RANGE:
693
+ if (!wexpr.orders.empty()) {
694
+ return false;
695
+ }
696
+ break;
697
+ default:
698
+ return false;
699
+ }
700
+
701
+ return true;
702
+ }
703
+
704
+ bool WindowAggregateExecutor::IsCustomAggregate() {
705
+ if (!wexpr.aggregate) {
706
+ return false;
707
+ }
708
+
709
+ if (!AggregateObject(wexpr).function.window) {
710
+ return false;
711
+ }
712
+
713
+ return (mode < WindowAggregationMode::COMBINE);
714
+ }
715
+
716
+ void WindowExecutor::Evaluate(idx_t row_idx, DataChunk &input_chunk, Vector &result,
717
+ WindowExecutorState &lstate) const {
718
+ auto &lbstate = lstate.Cast<WindowExecutorBoundsState>();
719
+ lbstate.UpdateBounds(row_idx, input_chunk, range);
720
+
721
+ const auto count = input_chunk.size();
722
+ EvaluateInternal(lstate, result, count, row_idx);
723
+
724
+ result.Verify(count);
725
+ }
726
+
727
+ WindowAggregateExecutor::WindowAggregateExecutor(BoundWindowExpression &wexpr, ClientContext &context,
728
+ const idx_t count, const ValidityMask &partition_mask,
729
+ const ValidityMask &order_mask, WindowAggregationMode mode)
730
+ : WindowExecutor(wexpr, context, count, partition_mask, order_mask), mode(mode), filter_executor(context) {
731
+ // TODO we could evaluate those expressions in parallel
732
+
733
+ // Check for constant aggregate
734
+ if (IsConstantAggregate()) {
735
+ aggregator =
736
+ make_uniq<WindowConstantAggregator>(AggregateObject(wexpr), wexpr.return_type, partition_mask, count);
737
+ } else if (IsCustomAggregate()) {
738
+ aggregator = make_uniq<WindowCustomAggregator>(AggregateObject(wexpr), wexpr.return_type, count);
739
+ } else if (wexpr.aggregate) {
740
+ // build a segment tree for frame-adhering aggregates
741
+ // see http://www.vldb.org/pvldb/vol8/p1058-leis.pdf
742
+ aggregator = make_uniq<WindowSegmentTree>(AggregateObject(wexpr), wexpr.return_type, count, mode);
743
+ }
744
+
745
+ // evaluate the FILTER clause and stuff it into a large mask for compactness and reuse
746
+ if (wexpr.filter_expr) {
747
+ filter_executor.AddExpression(*wexpr.filter_expr);
748
+ filter_sel.Initialize(STANDARD_VECTOR_SIZE);
749
+ }
750
+ }
751
+
752
+ void WindowAggregateExecutor::Sink(DataChunk &input_chunk, const idx_t input_idx, const idx_t total_count) {
753
+ idx_t filtered = 0;
754
+ SelectionVector *filtering = nullptr;
755
+ if (wexpr.filter_expr) {
756
+ filtering = &filter_sel;
757
+ filtered = filter_executor.SelectExpression(input_chunk, filter_sel);
758
+ }
759
+
760
+ if (!wexpr.children.empty()) {
761
+ payload_chunk.Reset();
762
+ payload_executor.Execute(input_chunk, payload_chunk);
763
+ payload_chunk.Verify();
764
+ } else if (aggregator) {
765
+ // Zero-argument aggregate (e.g., COUNT(*)
766
+ payload_chunk.SetCardinality(input_chunk);
767
+ }
768
+
769
+ D_ASSERT(aggregator);
770
+ aggregator->Sink(payload_chunk, filtering, filtered);
771
+
772
+ WindowExecutor::Sink(input_chunk, input_idx, total_count);
773
+ }
774
+
775
+ void WindowAggregateExecutor::Finalize() {
776
+ D_ASSERT(aggregator);
777
+ aggregator->Finalize();
778
+ }
779
+
780
+ class WindowAggregateState : public WindowExecutorBoundsState {
781
+ public:
782
+ WindowAggregateState(BoundWindowExpression &wexpr, ClientContext &context, const idx_t payload_count,
783
+ const ValidityMask &partition_mask, const ValidityMask &order_mask,
784
+ const WindowAggregator &aggregator)
785
+ : WindowExecutorBoundsState(wexpr, context, payload_count, partition_mask, order_mask),
786
+ aggregator_state(aggregator.GetLocalState()) {
787
+ }
788
+
789
+ public:
790
+ unique_ptr<WindowAggregatorState> aggregator_state;
791
+
792
+ void NextRank(idx_t partition_begin, idx_t peer_begin, idx_t row_idx);
793
+ };
794
+
795
+ unique_ptr<WindowExecutorState> WindowAggregateExecutor::GetExecutorState() const {
796
+ return make_uniq<WindowAggregateState>(wexpr, context, payload_count, partition_mask, order_mask, *aggregator);
797
+ }
798
+
799
+ void WindowAggregateExecutor::EvaluateInternal(WindowExecutorState &lstate, Vector &result, idx_t count,
800
+ idx_t row_idx) const {
801
+ auto &lastate = lstate.Cast<WindowAggregateState>();
802
+ D_ASSERT(aggregator);
803
+ auto window_begin = FlatVector::GetData<const idx_t>(lastate.bounds.data[WINDOW_BEGIN]);
804
+ auto window_end = FlatVector::GetData<const idx_t>(lastate.bounds.data[WINDOW_END]);
805
+ aggregator->Evaluate(*lastate.aggregator_state, window_begin, window_end, result, count);
806
+ }
807
+
808
+ //===--------------------------------------------------------------------===//
809
+ // WindowRowNumberExecutor
810
+ //===--------------------------------------------------------------------===//
811
+ WindowRowNumberExecutor::WindowRowNumberExecutor(BoundWindowExpression &wexpr, ClientContext &context,
812
+ const idx_t payload_count, const ValidityMask &partition_mask,
813
+ const ValidityMask &order_mask)
814
+ : WindowExecutor(wexpr, context, payload_count, partition_mask, order_mask) {
815
+ }
816
+
817
+ void WindowRowNumberExecutor::EvaluateInternal(WindowExecutorState &lstate, Vector &result, idx_t count,
818
+ idx_t row_idx) const {
819
+ auto &lbstate = lstate.Cast<WindowExecutorBoundsState>();
820
+ auto partition_begin = FlatVector::GetData<const idx_t>(lbstate.bounds.data[PARTITION_BEGIN]);
821
+ auto rdata = FlatVector::GetData<int64_t>(result);
822
+ for (idx_t i = 0; i < count; ++i, ++row_idx) {
823
+ rdata[i] = row_idx - partition_begin[i] + 1;
824
+ }
825
+ }
826
+
827
+ //===--------------------------------------------------------------------===//
828
+ // WindowPeerState
829
+ //===--------------------------------------------------------------------===//
830
+ class WindowPeerState : public WindowExecutorBoundsState {
831
+ public:
832
+ WindowPeerState(BoundWindowExpression &wexpr, ClientContext &context, const idx_t payload_count,
833
+ const ValidityMask &partition_mask, const ValidityMask &order_mask)
834
+ : WindowExecutorBoundsState(wexpr, context, payload_count, partition_mask, order_mask) {
835
+ }
836
+
837
+ public:
838
+ uint64_t dense_rank = 1;
839
+ uint64_t rank_equal = 0;
840
+ uint64_t rank = 1;
841
+
842
+ void NextRank(idx_t partition_begin, idx_t peer_begin, idx_t row_idx);
843
+ };
844
+
845
+ void WindowPeerState::NextRank(idx_t partition_begin, idx_t peer_begin, idx_t row_idx) {
846
+ if (partition_begin == row_idx) {
847
+ dense_rank = 1;
848
+ rank = 1;
849
+ rank_equal = 0;
850
+ } else if (peer_begin == row_idx) {
851
+ dense_rank++;
852
+ rank += rank_equal;
853
+ rank_equal = 0;
854
+ }
855
+ rank_equal++;
856
+ }
857
+
858
+ WindowRankExecutor::WindowRankExecutor(BoundWindowExpression &wexpr, ClientContext &context, const idx_t payload_count,
859
+ const ValidityMask &partition_mask, const ValidityMask &order_mask)
860
+ : WindowExecutor(wexpr, context, payload_count, partition_mask, order_mask) {
861
+ }
862
+
863
+ unique_ptr<WindowExecutorState> WindowRankExecutor::GetExecutorState() const {
864
+ return make_uniq<WindowPeerState>(wexpr, context, payload_count, partition_mask, order_mask);
865
+ }
866
+
867
+ void WindowRankExecutor::EvaluateInternal(WindowExecutorState &lstate, Vector &result, idx_t count,
868
+ idx_t row_idx) const {
869
+ auto &lpeer = lstate.Cast<WindowPeerState>();
870
+ auto partition_begin = FlatVector::GetData<const idx_t>(lpeer.bounds.data[PARTITION_BEGIN]);
871
+ auto peer_begin = FlatVector::GetData<const idx_t>(lpeer.bounds.data[PEER_BEGIN]);
872
+ auto rdata = FlatVector::GetData<int64_t>(result);
873
+
874
+ // Reset to "previous" row
875
+ lpeer.rank = (peer_begin[0] - partition_begin[0]) + 1;
876
+ lpeer.rank_equal = (row_idx - peer_begin[0]);
877
+
878
+ for (idx_t i = 0; i < count; ++i, ++row_idx) {
879
+ lpeer.NextRank(partition_begin[i], peer_begin[i], row_idx);
880
+ rdata[i] = lpeer.rank;
881
+ }
882
+ }
883
+
884
+ WindowDenseRankExecutor::WindowDenseRankExecutor(BoundWindowExpression &wexpr, ClientContext &context,
885
+ const idx_t payload_count, const ValidityMask &partition_mask,
886
+ const ValidityMask &order_mask)
887
+ : WindowExecutor(wexpr, context, payload_count, partition_mask, order_mask) {
888
+ }
889
+
890
+ unique_ptr<WindowExecutorState> WindowDenseRankExecutor::GetExecutorState() const {
891
+ return make_uniq<WindowPeerState>(wexpr, context, payload_count, partition_mask, order_mask);
892
+ }
893
+
894
+ void WindowDenseRankExecutor::EvaluateInternal(WindowExecutorState &lstate, Vector &result, idx_t count,
895
+ idx_t row_idx) const {
896
+ auto &lpeer = lstate.Cast<WindowPeerState>();
897
+ auto partition_begin = FlatVector::GetData<const idx_t>(lpeer.bounds.data[PARTITION_BEGIN]);
898
+ auto peer_begin = FlatVector::GetData<const idx_t>(lpeer.bounds.data[PEER_BEGIN]);
899
+ auto rdata = FlatVector::GetData<int64_t>(result);
900
+
901
+ // Reset to "previous" row
902
+ lpeer.rank = (peer_begin[0] - partition_begin[0]) + 1;
903
+ lpeer.rank_equal = (row_idx - peer_begin[0]);
904
+
905
+ // The previous dense rank is the number of order mask bits in [partition_begin, row_idx)
906
+ lpeer.dense_rank = 0;
907
+
908
+ auto order_begin = partition_begin[0];
909
+ idx_t begin_idx;
910
+ idx_t begin_offset;
911
+ order_mask.GetEntryIndex(order_begin, begin_idx, begin_offset);
912
+
913
+ auto order_end = row_idx;
914
+ idx_t end_idx;
915
+ idx_t end_offset;
916
+ order_mask.GetEntryIndex(order_end, end_idx, end_offset);
917
+
918
+ // If they are in the same entry, just loop
919
+ if (begin_idx == end_idx) {
920
+ const auto entry = order_mask.GetValidityEntry(begin_idx);
921
+ for (; begin_offset < end_offset; ++begin_offset) {
922
+ lpeer.dense_rank += order_mask.RowIsValid(entry, begin_offset);
923
+ }
924
+ } else {
925
+ // Count the ragged bits at the start of the partition
926
+ if (begin_offset) {
927
+ const auto entry = order_mask.GetValidityEntry(begin_idx);
928
+ for (; begin_offset < order_mask.BITS_PER_VALUE; ++begin_offset) {
929
+ lpeer.dense_rank += order_mask.RowIsValid(entry, begin_offset);
930
+ ++order_begin;
931
+ }
932
+ ++begin_idx;
933
+ }
934
+
935
+ // Count the the aligned bits.
936
+ ValidityMask tail_mask(order_mask.GetData() + begin_idx);
937
+ lpeer.dense_rank += tail_mask.CountValid(order_end - order_begin);
938
+ }
939
+
940
+ for (idx_t i = 0; i < count; ++i, ++row_idx) {
941
+ lpeer.NextRank(partition_begin[i], peer_begin[i], row_idx);
942
+ rdata[i] = lpeer.dense_rank;
943
+ }
944
+ }
945
+
946
+ WindowPercentRankExecutor::WindowPercentRankExecutor(BoundWindowExpression &wexpr, ClientContext &context,
947
+ const idx_t payload_count, const ValidityMask &partition_mask,
948
+ const ValidityMask &order_mask)
949
+ : WindowExecutor(wexpr, context, payload_count, partition_mask, order_mask) {
950
+ }
951
+
952
+ unique_ptr<WindowExecutorState> WindowPercentRankExecutor::GetExecutorState() const {
953
+ return make_uniq<WindowPeerState>(wexpr, context, payload_count, partition_mask, order_mask);
954
+ }
955
+
956
+ void WindowPercentRankExecutor::EvaluateInternal(WindowExecutorState &lstate, Vector &result, idx_t count,
957
+ idx_t row_idx) const {
958
+ auto &lpeer = lstate.Cast<WindowPeerState>();
959
+ auto partition_begin = FlatVector::GetData<const idx_t>(lpeer.bounds.data[PARTITION_BEGIN]);
960
+ auto partition_end = FlatVector::GetData<const idx_t>(lpeer.bounds.data[PARTITION_END]);
961
+ auto peer_begin = FlatVector::GetData<const idx_t>(lpeer.bounds.data[PEER_BEGIN]);
962
+ auto rdata = FlatVector::GetData<double>(result);
963
+
964
+ // Reset to "previous" row
965
+ lpeer.rank = (peer_begin[0] - partition_begin[0]) + 1;
966
+ lpeer.rank_equal = (row_idx - peer_begin[0]);
967
+
968
+ for (idx_t i = 0; i < count; ++i, ++row_idx) {
969
+ lpeer.NextRank(partition_begin[i], peer_begin[i], row_idx);
970
+ int64_t denom = partition_end[i] - partition_begin[i] - 1;
971
+ double percent_rank = denom > 0 ? ((double)lpeer.rank - 1) / denom : 0;
972
+ rdata[i] = percent_rank;
973
+ }
974
+ }
975
+
976
+ //===--------------------------------------------------------------------===//
977
+ // WindowCumeDistExecutor
978
+ //===--------------------------------------------------------------------===//
979
+ WindowCumeDistExecutor::WindowCumeDistExecutor(BoundWindowExpression &wexpr, ClientContext &context,
980
+ const idx_t payload_count, const ValidityMask &partition_mask,
981
+ const ValidityMask &order_mask)
982
+ : WindowExecutor(wexpr, context, payload_count, partition_mask, order_mask) {
983
+ }
984
+
985
+ void WindowCumeDistExecutor::EvaluateInternal(WindowExecutorState &lstate, Vector &result, idx_t count,
986
+ idx_t row_idx) const {
987
+ auto &lbstate = lstate.Cast<WindowExecutorBoundsState>();
988
+ auto partition_begin = FlatVector::GetData<const idx_t>(lbstate.bounds.data[PARTITION_BEGIN]);
989
+ auto partition_end = FlatVector::GetData<const idx_t>(lbstate.bounds.data[PARTITION_END]);
990
+ auto peer_end = FlatVector::GetData<const idx_t>(lbstate.bounds.data[PEER_END]);
991
+ auto rdata = FlatVector::GetData<double>(result);
992
+ for (idx_t i = 0; i < count; ++i, ++row_idx) {
993
+ int64_t denom = partition_end[i] - partition_begin[i];
994
+ double cume_dist = denom > 0 ? ((double)(peer_end[i] - partition_begin[i])) / denom : 0;
995
+ rdata[i] = cume_dist;
996
+ }
997
+ }
998
+
999
+ //===--------------------------------------------------------------------===//
1000
+ // WindowValueExecutor
1001
+ //===--------------------------------------------------------------------===//
1002
+ WindowValueExecutor::WindowValueExecutor(BoundWindowExpression &wexpr, ClientContext &context,
1003
+ const idx_t payload_count, const ValidityMask &partition_mask,
1004
+ const ValidityMask &order_mask)
1005
+ : WindowExecutor(wexpr, context, payload_count, partition_mask, order_mask) {
1006
+ }
1007
+
1008
+ WindowNtileExecutor::WindowNtileExecutor(BoundWindowExpression &wexpr, ClientContext &context,
1009
+ const idx_t payload_count, const ValidityMask &partition_mask,
1010
+ const ValidityMask &order_mask)
1011
+ : WindowValueExecutor(wexpr, context, payload_count, partition_mask, order_mask) {
1012
+ }
1013
+
1014
+ void WindowValueExecutor::Sink(DataChunk &input_chunk, const idx_t input_idx, const idx_t total_count) {
1015
+ // Single pass over the input to produce the global data.
1016
+ // Vectorisation for the win...
1017
+
1018
+ // Set up a validity mask for IGNORE NULLS
1019
+ bool check_nulls = false;
1020
+ if (wexpr.ignore_nulls) {
1021
+ switch (wexpr.type) {
1022
+ case ExpressionType::WINDOW_LEAD:
1023
+ case ExpressionType::WINDOW_LAG:
1024
+ case ExpressionType::WINDOW_FIRST_VALUE:
1025
+ case ExpressionType::WINDOW_LAST_VALUE:
1026
+ case ExpressionType::WINDOW_NTH_VALUE:
1027
+ check_nulls = true;
1028
+ break;
1029
+ default:
1030
+ break;
1031
+ }
1032
+ }
1033
+
1034
+ if (!wexpr.children.empty()) {
1035
+ payload_chunk.Reset();
1036
+ payload_executor.Execute(input_chunk, payload_chunk);
1037
+ payload_chunk.Verify();
1038
+ payload_collection.Append(payload_chunk, true);
1039
+
1040
+ // process payload chunks while they are still piping hot
1041
+ if (check_nulls) {
1042
+ const auto count = input_chunk.size();
1043
+
1044
+ UnifiedVectorFormat vdata;
1045
+ payload_chunk.data[0].ToUnifiedFormat(count, vdata);
1046
+ if (!vdata.validity.AllValid()) {
1047
+ // Lazily materialise the contents when we find the first NULL
1048
+ if (ignore_nulls.AllValid()) {
1049
+ ignore_nulls.Initialize(total_count);
1050
+ }
1051
+ // Write to the current position
1052
+ if (input_idx % ValidityMask::BITS_PER_VALUE == 0) {
1053
+ // If we are at the edge of an output entry, just copy the entries
1054
+ auto dst = ignore_nulls.GetData() + ignore_nulls.EntryCount(input_idx);
1055
+ auto src = vdata.validity.GetData();
1056
+ for (auto entry_count = vdata.validity.EntryCount(count); entry_count-- > 0;) {
1057
+ *dst++ = *src++;
1058
+ }
1059
+ } else {
1060
+ // If not, we have ragged data and need to copy one bit at a time.
1061
+ for (idx_t i = 0; i < count; ++i) {
1062
+ ignore_nulls.Set(input_idx + i, vdata.validity.RowIsValid(i));
1063
+ }
1064
+ }
1065
+ }
1066
+ }
1067
+ }
1068
+
1069
+ WindowExecutor::Sink(input_chunk, input_idx, total_count);
1070
+ }
1071
+
1072
+ void WindowNtileExecutor::EvaluateInternal(WindowExecutorState &lstate, Vector &result, idx_t count,
1073
+ idx_t row_idx) const {
1074
+ D_ASSERT(payload_collection.ColumnCount() == 1);
1075
+ auto &lbstate = lstate.Cast<WindowExecutorBoundsState>();
1076
+ auto partition_begin = FlatVector::GetData<const idx_t>(lbstate.bounds.data[PARTITION_BEGIN]);
1077
+ auto partition_end = FlatVector::GetData<const idx_t>(lbstate.bounds.data[PARTITION_END]);
1078
+ auto rdata = FlatVector::GetData<int64_t>(result);
1079
+ for (idx_t i = 0; i < count; ++i, ++row_idx) {
1080
+ if (CellIsNull(payload_collection, 0, row_idx)) {
1081
+ FlatVector::SetNull(result, i, true);
1082
+ } else {
1083
+ auto n_param = GetCell<int64_t>(payload_collection, 0, row_idx);
1084
+ if (n_param < 1) {
1085
+ throw InvalidInputException("Argument for ntile must be greater than zero");
1086
+ }
1087
+ // With thanks from SQLite's ntileValueFunc()
1088
+ int64_t n_total = partition_end[i] - partition_begin[i];
1089
+ if (n_param > n_total) {
1090
+ // more groups allowed than we have values
1091
+ // map every entry to a unique group
1092
+ n_param = n_total;
1093
+ }
1094
+ int64_t n_size = (n_total / n_param);
1095
+ // find the row idx within the group
1096
+ D_ASSERT(row_idx >= partition_begin[i]);
1097
+ int64_t adjusted_row_idx = row_idx - partition_begin[i];
1098
+ // now compute the ntile
1099
+ int64_t n_large = n_total - n_param * n_size;
1100
+ int64_t i_small = n_large * (n_size + 1);
1101
+ int64_t result_ntile;
1102
+
1103
+ D_ASSERT((n_large * (n_size + 1) + (n_param - n_large) * n_size) == n_total);
1104
+
1105
+ if (adjusted_row_idx < i_small) {
1106
+ result_ntile = 1 + adjusted_row_idx / (n_size + 1);
1107
+ } else {
1108
+ result_ntile = 1 + n_large + (adjusted_row_idx - i_small) / n_size;
1109
+ }
1110
+ // result has to be between [1, NTILE]
1111
+ D_ASSERT(result_ntile >= 1 && result_ntile <= n_param);
1112
+ rdata[i] = result_ntile;
1113
+ }
1114
+ }
1115
+ }
1116
+
1117
+ //===--------------------------------------------------------------------===//
1118
+ // WindowLeadLagState
1119
+ //===--------------------------------------------------------------------===//
1120
+ class WindowLeadLagState : public WindowExecutorBoundsState {
1121
+ public:
1122
+ WindowLeadLagState(BoundWindowExpression &wexpr, ClientContext &context, const idx_t payload_count,
1123
+ const ValidityMask &partition_mask, const ValidityMask &order_mask)
1124
+ : WindowExecutorBoundsState(wexpr, context, payload_count, partition_mask, order_mask),
1125
+ leadlag_offset(wexpr.offset_expr.get(), context), leadlag_default(wexpr.default_expr.get(), context) {
1126
+ }
1127
+
1128
+ void UpdateBounds(idx_t row_idx, DataChunk &input_chunk, const WindowInputColumn &range) override;
1129
+
1130
+ public:
1131
+ // LEAD/LAG Evaluation
1132
+ WindowInputExpression leadlag_offset;
1133
+ WindowInputExpression leadlag_default;
1134
+ };
1135
+
1136
+ void WindowLeadLagState::UpdateBounds(idx_t row_idx, DataChunk &input_chunk, const WindowInputColumn &range) {
1137
+ // Evaluate the row-level arguments
1138
+ leadlag_offset.Execute(input_chunk);
1139
+ leadlag_default.Execute(input_chunk);
1140
+
1141
+ WindowExecutorBoundsState::UpdateBounds(row_idx, input_chunk, range);
1142
+ }
1143
+
1144
+ WindowLeadLagExecutor::WindowLeadLagExecutor(BoundWindowExpression &wexpr, ClientContext &context,
1145
+ const idx_t payload_count, const ValidityMask &partition_mask,
1146
+ const ValidityMask &order_mask)
1147
+ : WindowValueExecutor(wexpr, context, payload_count, partition_mask, order_mask) {
1148
+ }
1149
+
1150
+ unique_ptr<WindowExecutorState> WindowLeadLagExecutor::GetExecutorState() const {
1151
+ return make_uniq<WindowLeadLagState>(wexpr, context, payload_count, partition_mask, order_mask);
1152
+ }
1153
+
1154
+ void WindowLeadLagExecutor::EvaluateInternal(WindowExecutorState &lstate, Vector &result, idx_t count,
1155
+ idx_t row_idx) const {
1156
+ auto &llstate = lstate.Cast<WindowLeadLagState>();
1157
+
1158
+ auto partition_begin = FlatVector::GetData<const idx_t>(llstate.bounds.data[PARTITION_BEGIN]);
1159
+ auto partition_end = FlatVector::GetData<const idx_t>(llstate.bounds.data[PARTITION_END]);
1160
+ for (idx_t i = 0; i < count; ++i, ++row_idx) {
1161
+ int64_t offset = 1;
1162
+ if (wexpr.offset_expr) {
1163
+ offset = llstate.leadlag_offset.GetCell<int64_t>(i);
1164
+ }
1165
+ int64_t val_idx = (int64_t)row_idx;
1166
+ if (wexpr.type == ExpressionType::WINDOW_LEAD) {
1167
+ val_idx = AddOperatorOverflowCheck::Operation<int64_t, int64_t, int64_t>(val_idx, offset);
1168
+ } else {
1169
+ val_idx = SubtractOperatorOverflowCheck::Operation<int64_t, int64_t, int64_t>(val_idx, offset);
1170
+ }
1171
+
1172
+ idx_t delta = 0;
1173
+ if (val_idx < (int64_t)row_idx) {
1174
+ // Count backwards
1175
+ delta = idx_t(row_idx - val_idx);
1176
+ val_idx = FindPrevStart(ignore_nulls, partition_begin[i], row_idx, delta);
1177
+ } else if (val_idx > (int64_t)row_idx) {
1178
+ delta = idx_t(val_idx - row_idx);
1179
+ val_idx = FindNextStart(ignore_nulls, row_idx + 1, partition_end[i], delta);
1180
+ }
1181
+ // else offset is zero, so don't move.
1182
+
1183
+ if (!delta) {
1184
+ CopyCell(payload_collection, 0, val_idx, result, i);
1185
+ } else if (wexpr.default_expr) {
1186
+ llstate.leadlag_default.CopyCell(result, i);
1187
+ } else {
1188
+ FlatVector::SetNull(result, i, true);
1189
+ }
1190
+ }
1191
+ }
1192
+
1193
+ WindowFirstValueExecutor::WindowFirstValueExecutor(BoundWindowExpression &wexpr, ClientContext &context,
1194
+ const idx_t payload_count, const ValidityMask &partition_mask,
1195
+ const ValidityMask &order_mask)
1196
+ : WindowValueExecutor(wexpr, context, payload_count, partition_mask, order_mask) {
1197
+ }
1198
+
1199
+ void WindowFirstValueExecutor::EvaluateInternal(WindowExecutorState &lstate, Vector &result, idx_t count,
1200
+ idx_t row_idx) const {
1201
+ auto &lbstate = lstate.Cast<WindowExecutorBoundsState>();
1202
+ auto window_begin = FlatVector::GetData<const idx_t>(lbstate.bounds.data[WINDOW_BEGIN]);
1203
+ auto window_end = FlatVector::GetData<const idx_t>(lbstate.bounds.data[WINDOW_END]);
1204
+ for (idx_t i = 0; i < count; ++i, ++row_idx) {
1205
+ if (window_begin[i] >= window_end[i]) {
1206
+ FlatVector::SetNull(result, i, true);
1207
+ continue;
1208
+ }
1209
+ // Same as NTH_VALUE(..., 1)
1210
+ idx_t n = 1;
1211
+ const auto first_idx = FindNextStart(ignore_nulls, window_begin[i], window_end[i], n);
1212
+ if (!n) {
1213
+ CopyCell(payload_collection, 0, first_idx, result, i);
1214
+ } else {
1215
+ FlatVector::SetNull(result, i, true);
1216
+ }
1217
+ }
1218
+ }
1219
+
1220
+ WindowLastValueExecutor::WindowLastValueExecutor(BoundWindowExpression &wexpr, ClientContext &context,
1221
+ const idx_t payload_count, const ValidityMask &partition_mask,
1222
+ const ValidityMask &order_mask)
1223
+ : WindowValueExecutor(wexpr, context, payload_count, partition_mask, order_mask) {
1224
+ }
1225
+
1226
+ void WindowLastValueExecutor::EvaluateInternal(WindowExecutorState &lstate, Vector &result, idx_t count,
1227
+ idx_t row_idx) const {
1228
+ auto &lbstate = lstate.Cast<WindowExecutorBoundsState>();
1229
+ auto window_begin = FlatVector::GetData<const idx_t>(lbstate.bounds.data[WINDOW_BEGIN]);
1230
+ auto window_end = FlatVector::GetData<const idx_t>(lbstate.bounds.data[WINDOW_END]);
1231
+ for (idx_t i = 0; i < count; ++i, ++row_idx) {
1232
+ if (window_begin[i] >= window_end[i]) {
1233
+ FlatVector::SetNull(result, i, true);
1234
+ continue;
1235
+ }
1236
+ idx_t n = 1;
1237
+ const auto last_idx = FindPrevStart(ignore_nulls, window_begin[i], window_end[i], n);
1238
+ if (!n) {
1239
+ CopyCell(payload_collection, 0, last_idx, result, i);
1240
+ } else {
1241
+ FlatVector::SetNull(result, i, true);
1242
+ }
1243
+ }
1244
+ }
1245
+
1246
+ WindowNthValueExecutor::WindowNthValueExecutor(BoundWindowExpression &wexpr, ClientContext &context,
1247
+ const idx_t payload_count, const ValidityMask &partition_mask,
1248
+ const ValidityMask &order_mask)
1249
+ : WindowValueExecutor(wexpr, context, payload_count, partition_mask, order_mask) {
1250
+ }
1251
+
1252
+ void WindowNthValueExecutor::EvaluateInternal(WindowExecutorState &lstate, Vector &result, idx_t count,
1253
+ idx_t row_idx) const {
1254
+ D_ASSERT(payload_collection.ColumnCount() == 2);
1255
+
1256
+ auto &lbstate = lstate.Cast<WindowExecutorBoundsState>();
1257
+ auto window_begin = FlatVector::GetData<const idx_t>(lbstate.bounds.data[WINDOW_BEGIN]);
1258
+ auto window_end = FlatVector::GetData<const idx_t>(lbstate.bounds.data[WINDOW_END]);
1259
+ for (idx_t i = 0; i < count; ++i, ++row_idx) {
1260
+ if (window_begin[i] >= window_end[i]) {
1261
+ FlatVector::SetNull(result, i, true);
1262
+ continue;
1263
+ }
1264
+ // Returns value evaluated at the row that is the n'th row of the window frame (counting from 1);
1265
+ // returns NULL if there is no such row.
1266
+ if (CellIsNull(payload_collection, 1, row_idx)) {
1267
+ FlatVector::SetNull(result, i, true);
1268
+ } else {
1269
+ auto n_param = GetCell<int64_t>(payload_collection, 1, row_idx);
1270
+ if (n_param < 1) {
1271
+ FlatVector::SetNull(result, i, true);
1272
+ } else {
1273
+ auto n = idx_t(n_param);
1274
+ const auto nth_index = FindNextStart(ignore_nulls, window_begin[i], window_end[i], n);
1275
+ if (!n) {
1276
+ CopyCell(payload_collection, 0, nth_index, result, i);
1277
+ } else {
1278
+ FlatVector::SetNull(result, i, true);
1279
+ }
1280
+ }
1281
+ }
1282
+ }
1283
+ }
1284
+
1285
+ } // namespace duckdb