duckdb 0.8.2-dev145.0 → 0.8.2-dev1493.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (476) hide show
  1. package/binding.gyp +15 -12
  2. package/binding.gyp.in +1 -1
  3. package/configure.py +1 -1
  4. package/duckdb_extension_config.cmake +10 -0
  5. package/package.json +1 -1
  6. package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
  7. package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
  8. package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
  9. package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
  10. package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
  11. package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
  12. package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
  13. package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
  14. package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
  15. package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
  16. package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
  17. package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
  18. package/src/duckdb/extension/json/include/json_common.hpp +47 -231
  19. package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
  20. package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
  21. package/src/duckdb/extension/json/json_common.cpp +272 -40
  22. package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
  23. package/src/duckdb/extension/json/json_functions/json_transform.cpp +17 -37
  24. package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
  25. package/src/duckdb/extension/json/json_functions.cpp +24 -24
  26. package/src/duckdb/extension/json/json_scan.cpp +3 -6
  27. package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
  28. package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
  29. package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
  30. package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
  31. package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
  32. package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
  33. package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
  34. package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
  35. package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
  36. package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
  37. package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
  38. package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
  39. package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
  40. package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
  41. package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
  42. package/src/duckdb/extension/parquet/parquet_extension.cpp +192 -20
  43. package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
  44. package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
  45. package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
  46. package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
  47. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
  48. package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
  49. package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
  50. package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
  51. package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
  52. package/src/duckdb/src/common/allocator.cpp +14 -2
  53. package/src/duckdb/src/common/arrow/arrow_appender.cpp +5 -11
  54. package/src/duckdb/src/common/assert.cpp +3 -0
  55. package/src/duckdb/src/common/enum_util.cpp +42 -5
  56. package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
  57. package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
  58. package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
  59. package/src/duckdb/src/common/exception.cpp +2 -2
  60. package/src/duckdb/src/common/file_system.cpp +19 -0
  61. package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
  62. package/src/duckdb/src/common/local_file_system.cpp +2 -2
  63. package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
  64. package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
  65. package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
  66. package/src/duckdb/src/common/re2_regex.cpp +1 -1
  67. package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
  68. package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
  69. package/src/duckdb/src/common/sort/partition_state.cpp +44 -11
  70. package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
  71. package/src/duckdb/src/common/types/bit.cpp +51 -0
  72. package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
  73. package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
  74. package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
  75. package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
  76. package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
  77. package/src/duckdb/src/common/types/date.cpp +9 -0
  78. package/src/duckdb/src/common/types/list_segment.cpp +24 -74
  79. package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
  80. package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
  81. package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
  82. package/src/duckdb/src/common/types/uuid.cpp +2 -2
  83. package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
  84. package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
  85. package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
  86. package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
  87. package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
  88. package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
  89. package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
  90. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
  91. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
  92. package/src/duckdb/src/core_functions/function_list.cpp +4 -2
  93. package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
  94. package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
  95. package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
  96. package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
  97. package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
  98. package/src/duckdb/src/execution/expression_executor.cpp +1 -1
  99. package/src/duckdb/src/execution/index/art/art.cpp +149 -139
  100. package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
  101. package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
  102. package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
  103. package/src/duckdb/src/execution/index/art/node.cpp +113 -120
  104. package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
  105. package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
  106. package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
  107. package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
  108. package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
  109. package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
  110. package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
  111. package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
  112. package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
  113. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
  114. package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
  115. package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
  116. package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +414 -283
  117. package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
  118. package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
  119. package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
  120. package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +22 -3
  121. package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +2 -2
  122. package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
  123. package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
  124. package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
  125. package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
  126. package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
  127. package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
  128. package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
  129. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
  130. package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
  131. package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
  132. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
  133. package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
  134. package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
  135. package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
  136. package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
  137. package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
  138. package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
  139. package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
  140. package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
  141. package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
  142. package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
  143. package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
  144. package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
  145. package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
  146. package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
  147. package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
  148. package/src/duckdb/src/function/function.cpp +3 -1
  149. package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
  150. package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
  151. package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
  152. package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
  153. package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
  154. package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
  155. package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
  156. package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
  157. package/src/duckdb/src/function/table/read_csv.cpp +100 -17
  158. package/src/duckdb/src/function/table/table_scan.cpp +9 -0
  159. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  160. package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
  161. package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
  162. package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
  163. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +8 -0
  164. package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
  165. package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
  166. package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
  167. package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
  168. package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
  169. package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
  170. package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
  171. package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
  172. package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
  173. package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
  174. package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
  175. package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
  176. package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
  177. package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
  178. package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
  179. package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
  180. package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +13 -0
  181. package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
  182. package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
  183. package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
  184. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
  185. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
  186. package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
  187. package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
  188. package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
  189. package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
  190. package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +0 -1
  191. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
  192. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
  193. package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
  194. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
  195. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
  196. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
  197. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
  198. package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
  199. package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
  200. package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
  201. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
  202. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
  203. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
  204. package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
  205. package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
  206. package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
  207. package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
  208. package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
  209. package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
  210. package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
  211. package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
  212. package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
  213. package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
  214. package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
  215. package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
  216. package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
  217. package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
  218. package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
  219. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
  220. package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
  221. package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
  222. package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
  223. package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
  224. package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
  225. package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
  226. package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
  227. package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
  228. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +2 -10
  229. package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
  230. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
  231. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
  232. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
  233. package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
  234. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
  235. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
  236. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
  237. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
  238. package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
  239. package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
  240. package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
  241. package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
  242. package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
  243. package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
  244. package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
  245. package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
  246. package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
  247. package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
  248. package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
  249. package/src/duckdb/src/include/duckdb/main/client_config.hpp +3 -0
  250. package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
  251. package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
  252. package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
  253. package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
  254. package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
  255. package/src/duckdb/src/include/duckdb/main/settings.hpp +30 -1
  256. package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
  257. package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
  258. package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
  259. package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
  260. package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
  261. package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
  262. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
  263. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
  264. package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
  265. package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
  266. package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
  267. package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
  268. package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
  269. package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
  270. package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
  271. package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
  272. package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
  273. package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
  274. package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
  275. package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
  276. package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
  277. package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
  278. package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
  279. package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
  280. package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
  281. package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
  282. package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
  283. package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
  284. package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
  285. package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
  286. package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
  287. package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
  288. package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
  289. package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
  290. package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
  291. package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
  292. package/src/duckdb/src/include/duckdb/planner/binder.hpp +8 -5
  293. package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
  294. package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
  295. package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
  296. package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
  297. package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
  298. package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
  299. package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
  300. package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
  301. package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
  302. package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
  303. package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
  304. package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
  305. package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
  306. package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
  307. package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
  308. package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
  309. package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
  310. package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
  311. package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
  312. package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
  313. package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
  314. package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
  315. package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
  316. package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
  317. package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
  318. package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
  319. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
  320. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
  321. package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
  322. package/src/duckdb/src/include/duckdb.h +28 -0
  323. package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
  324. package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
  325. package/src/duckdb/src/main/config.cpp +3 -0
  326. package/src/duckdb/src/main/database.cpp +1 -1
  327. package/src/duckdb/src/main/extension/extension_helper.cpp +96 -89
  328. package/src/duckdb/src/main/extension/extension_install.cpp +6 -0
  329. package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
  330. package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
  331. package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
  332. package/src/duckdb/src/main/relation.cpp +6 -5
  333. package/src/duckdb/src/main/settings/settings.cpp +64 -18
  334. package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
  335. package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
  336. package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
  337. package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
  338. package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
  339. package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
  340. package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
  341. package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
  342. package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
  343. package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
  344. package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
  345. package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
  346. package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
  347. package/src/duckdb/src/optimizer/optimizer.cpp +51 -14
  348. package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
  349. package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
  350. package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
  351. package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
  352. package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
  353. package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
  354. package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
  355. package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
  356. package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
  357. package/src/duckdb/src/parallel/executor.cpp +15 -0
  358. package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
  359. package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
  360. package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
  361. package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
  362. package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
  363. package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
  364. package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
  365. package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
  366. package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
  367. package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
  368. package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
  369. package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
  370. package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
  371. package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
  372. package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
  373. package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
  374. package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
  375. package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
  376. package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
  377. package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
  378. package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
  379. package/src/duckdb/src/parser/parser.cpp +8 -2
  380. package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
  381. package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
  382. package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
  383. package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
  384. package/src/duckdb/src/parser/query_node.cpp +15 -37
  385. package/src/duckdb/src/parser/result_modifier.cpp +0 -74
  386. package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
  387. package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
  388. package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
  389. package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
  390. package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
  391. package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
  392. package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
  393. package/src/duckdb/src/parser/tableref.cpp +0 -44
  394. package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
  395. package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
  396. package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
  397. package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
  398. package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
  399. package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
  400. package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
  401. package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
  402. package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
  403. package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
  404. package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
  405. package/src/duckdb/src/parser/transformer.cpp +15 -0
  406. package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
  407. package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
  408. package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
  409. package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
  410. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
  411. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
  412. package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
  413. package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
  414. package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
  415. package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
  416. package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +61 -26
  417. package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
  418. package/src/duckdb/src/planner/binder.cpp +5 -0
  419. package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
  420. package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
  421. package/src/duckdb/src/planner/expression_binder.cpp +3 -0
  422. package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
  423. package/src/duckdb/src/planner/logical_operator.cpp +5 -0
  424. package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
  425. package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
  426. package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
  427. package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
  428. package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
  429. package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
  430. package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
  431. package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
  432. package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
  433. package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
  434. package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
  435. package/src/duckdb/src/storage/compression/rle.cpp +0 -1
  436. package/src/duckdb/src/storage/data_table.cpp +1 -1
  437. package/src/duckdb/src/storage/local_storage.cpp +3 -3
  438. package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
  439. package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
  440. package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
  441. package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
  442. package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
  443. package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
  444. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  445. package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
  446. package/src/duckdb/src/storage/table/row_group.cpp +25 -9
  447. package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
  448. package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
  449. package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
  450. package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
  451. package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
  452. package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
  453. package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
  454. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
  455. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
  456. package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
  457. package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
  458. package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
  459. package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
  460. package/src/duckdb/ub_src_function_scalar.cpp +2 -0
  461. package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
  462. package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
  463. package/src/duckdb/ub_src_optimizer.cpp +6 -0
  464. package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
  465. package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
  466. package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
  467. package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
  468. package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
  469. package/src/duckdb/ub_src_planner_operator.cpp +4 -0
  470. package/src/duckdb/ub_src_storage_serialization.cpp +8 -0
  471. package/src/statement.cpp +10 -3
  472. package/test/test_all_types.test.ts +233 -0
  473. package/tsconfig.json +1 -0
  474. package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
  475. package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
  476. package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -7,111 +7,58 @@
7
7
 
8
8
  namespace duckdb {
9
9
 
10
- void IteratorCurrentKey::Push(const uint8_t byte) {
11
- if (cur_key_pos == key.size()) {
12
- key.push_back(byte);
13
- }
14
- D_ASSERT(cur_key_pos < key.size());
15
- key[cur_key_pos++] = byte;
16
- }
17
-
18
- void IteratorCurrentKey::Pop(const idx_t n) {
19
- cur_key_pos -= n;
20
- D_ASSERT(cur_key_pos <= key.size());
21
- }
22
-
23
- uint8_t &IteratorCurrentKey::operator[](idx_t idx) {
24
- if (idx >= key.size()) {
25
- key.push_back(0);
26
- }
27
- D_ASSERT(idx < key.size());
28
- return key[idx];
29
- }
30
-
31
- bool IteratorCurrentKey::operator>(const ARTKey &k) const {
32
- for (idx_t i = 0; i < MinValue<idx_t>(cur_key_pos, k.len); i++) {
33
- if (key[i] > k.data[i]) {
10
+ bool IteratorKey::operator>(const ARTKey &k) const {
11
+ for (idx_t i = 0; i < MinValue<idx_t>(key_bytes.size(), k.len); i++) {
12
+ if (key_bytes[i] > k.data[i]) {
34
13
  return true;
35
- } else if (key[i] < k.data[i]) {
14
+ } else if (key_bytes[i] < k.data[i]) {
36
15
  return false;
37
16
  }
38
17
  }
39
- return cur_key_pos > k.len;
18
+ return key_bytes.size() > k.len;
40
19
  }
41
20
 
42
- bool IteratorCurrentKey::operator>=(const ARTKey &k) const {
43
- for (idx_t i = 0; i < MinValue<idx_t>(cur_key_pos, k.len); i++) {
44
- if (key[i] > k.data[i]) {
21
+ bool IteratorKey::operator>=(const ARTKey &k) const {
22
+ for (idx_t i = 0; i < MinValue<idx_t>(key_bytes.size(), k.len); i++) {
23
+ if (key_bytes[i] > k.data[i]) {
45
24
  return true;
46
- } else if (key[i] < k.data[i]) {
25
+ } else if (key_bytes[i] < k.data[i]) {
47
26
  return false;
48
27
  }
49
28
  }
50
- return cur_key_pos >= k.len;
29
+ return key_bytes.size() >= k.len;
51
30
  }
52
31
 
53
- bool IteratorCurrentKey::operator==(const ARTKey &k) const {
54
- if (cur_key_pos != k.len) {
55
- return false;
56
- }
57
- for (idx_t i = 0; i < cur_key_pos; i++) {
58
- if (key[i] != k.data[i]) {
32
+ bool IteratorKey::operator==(const ARTKey &k) const {
33
+ // NOTE: we only use this for finding the LowerBound, in which case the length
34
+ // has to be equal
35
+ D_ASSERT(key_bytes.size() == k.len);
36
+ for (idx_t i = 0; i < key_bytes.size(); i++) {
37
+ if (key_bytes[i] != k.data[i]) {
59
38
  return false;
60
39
  }
61
40
  }
62
41
  return true;
63
42
  }
64
43
 
65
- void Iterator::FindMinimum(Node &node) {
66
-
67
- // reconstruct the prefix
68
- // FIXME: get all bytes at once to increase performance
69
- auto &node_prefix = node.GetPrefix(*art);
70
- for (idx_t i = 0; i < node_prefix.count; i++) {
71
- cur_key.Push(node_prefix.GetByte(*art, i));
72
- }
73
-
74
- // found the minimum
75
- if (node.DecodeARTNodeType() == NType::LEAF) {
76
- last_leaf = Node::GetAllocator(*art, NType::LEAF).Get<Leaf>(node);
77
- return;
78
- }
79
-
80
- // go to the leftmost entry in the current node
81
- uint8_t byte = 0;
82
- auto next = node.GetNextChild(*art, byte);
83
- D_ASSERT(next);
84
- cur_key.Push(byte);
85
-
86
- // recurse
87
- nodes.emplace(node, byte);
88
- FindMinimum(*next);
89
- }
90
-
91
- void Iterator::PushKey(const Node &node, const uint8_t byte) {
92
- if (node.DecodeARTNodeType() != NType::LEAF) {
93
- cur_key.Push(byte);
94
- }
95
- }
96
-
97
- bool Iterator::Scan(const ARTKey &key, const idx_t &max_count, vector<row_t> &result_ids, const bool &is_inclusive) {
44
+ bool Iterator::Scan(const ARTKey &upper_bound, const idx_t max_count, vector<row_t> &result_ids, const bool equal) {
98
45
 
99
46
  bool has_next;
100
47
  do {
101
- if (!key.Empty()) {
48
+ if (!upper_bound.Empty()) {
102
49
  // no more row IDs within the key bounds
103
- if (is_inclusive) {
104
- if (cur_key > key) {
50
+ if (equal) {
51
+ if (current_key > upper_bound) {
105
52
  return true;
106
53
  }
107
54
  } else {
108
- if (cur_key >= key) {
55
+ if (current_key >= upper_bound) {
109
56
  return true;
110
57
  }
111
58
  }
112
59
  }
113
60
 
114
- // adding more elements would exceed the max count
61
+ // adding more elements would exceed the maximum count
115
62
  if (result_ids.size() + last_leaf->count > max_count) {
116
63
  return false;
117
64
  }
@@ -130,172 +77,147 @@ bool Iterator::Scan(const ARTKey &key, const idx_t &max_count, vector<row_t> &re
130
77
  return true;
131
78
  }
132
79
 
133
- void Iterator::PopNode() {
134
- auto cur_node = nodes.top();
135
- idx_t elements_to_pop = cur_node.node.GetPrefix(*art).count + (nodes.size() != 1);
136
- cur_key.Pop(elements_to_pop);
137
- nodes.pop();
138
- }
80
+ void Iterator::FindMinimum(Node &node) {
139
81
 
140
- bool Iterator::Next() {
141
- if (!nodes.empty()) {
142
- auto cur_node = nodes.top().node;
143
- if (cur_node.DecodeARTNodeType() == NType::LEAF) {
144
- // pop leaf
145
- // we must pop the prefix size + the key to the node, unless we are popping the root
146
- PopNode();
147
- }
82
+ D_ASSERT(node.IsSet());
83
+ if (node.IsSwizzled()) {
84
+ node.Deserialize(*art);
148
85
  }
149
86
 
150
- // look for the next leaf
151
- while (!nodes.empty()) {
152
-
153
- // cur_node
154
- auto &top = nodes.top();
155
- Node node = top.node;
156
-
157
- // found a leaf: move to next node
158
- if (node.DecodeARTNodeType() == NType::LEAF) {
159
- last_leaf = Node::GetAllocator(*art, NType::LEAF).Get<Leaf>(node);
160
- return true;
161
- }
162
-
163
- // find next node
164
- if (top.byte == NumericLimits<uint8_t>::Maximum()) {
165
- // no node found: move up the tree, pop prefix and key of current node
166
- PopNode();
167
- continue;
168
- }
169
-
170
- top.byte == 0 ? top.byte : top.byte++;
171
- auto next_node = node.GetNextChild(*art, top.byte);
172
-
173
- if (next_node) {
174
- // add the next node's key byte
175
- PushKey(node, top.byte);
176
-
177
- // add prefix of new node
178
- // FIXME: get all bytes at once to increase performance
179
- auto &next_node_prefix = next_node->GetPrefix(*art);
180
- for (idx_t i = 0; i < next_node_prefix.count; i++) {
181
- cur_key.Push(next_node_prefix.GetByte(*art, i));
182
- }
183
-
184
- // next node found: push it
185
- nodes.emplace(*next_node, 0);
186
- } else {
87
+ // found the minimum
88
+ if (node.DecodeARTNodeType() == NType::LEAF) {
89
+ last_leaf = Node::GetAllocator(*art, NType::LEAF).Get<Leaf>(node);
90
+ return;
91
+ }
187
92
 
188
- // no node found: move up the tree, pop prefix and key of current node
189
- PopNode();
93
+ // traverse the prefix
94
+ if (node.DecodeARTNodeType() == NType::PREFIX) {
95
+ auto &prefix = Prefix::Get(*art, node);
96
+ for (idx_t i = 0; i < prefix.data[Node::PREFIX_SIZE]; i++) {
97
+ current_key.Push(prefix.data[i]);
190
98
  }
99
+ nodes.emplace(node, 0);
100
+ return FindMinimum(prefix.ptr);
191
101
  }
192
- return false;
102
+
103
+ // go to the leftmost entry in the current node and recurse
104
+ uint8_t byte = 0;
105
+ auto next = node.GetNextChild(*art, byte);
106
+ D_ASSERT(next);
107
+ current_key.Push(byte);
108
+ nodes.emplace(node, byte);
109
+ FindMinimum(*next);
193
110
  }
194
111
 
195
- bool Iterator::LowerBound(Node node, const ARTKey &key, const bool &is_inclusive) {
112
+ bool Iterator::LowerBound(Node &node, const ARTKey &key, const bool equal, idx_t depth) {
196
113
 
197
114
  if (!node.IsSet()) {
198
115
  return false;
199
116
  }
200
117
 
201
- idx_t depth = 0;
202
- bool equal = true;
203
- while (true) {
118
+ if (node.IsSwizzled()) {
119
+ node.Deserialize(*art);
120
+ }
204
121
 
205
- nodes.emplace(node, 0);
206
- auto &top = nodes.top();
122
+ // we found the lower bound
123
+ if (node.DecodeARTNodeType() == NType::LEAF) {
124
+ if (!equal && current_key == key) {
125
+ return Next();
126
+ }
127
+ last_leaf = Node::GetAllocator(*art, NType::LEAF).Get<Leaf>(node);
128
+ return true;
129
+ }
207
130
 
208
- // reconstruct the prefix
209
- // FIXME: get all bytes at once to increase performance
210
- reference<Prefix> node_prefix(top.node.GetPrefix(*art));
211
- for (idx_t i = 0; i < node_prefix.get().count; i++) {
212
- cur_key.Push(node_prefix.get().GetByte(*art, i));
131
+ if (node.DecodeARTNodeType() != NType::PREFIX) {
132
+ auto next_byte = key[depth];
133
+ auto child = node.GetNextChild(*art, next_byte);
134
+ if (!child) {
135
+ // the key is greater than any key in this subtree
136
+ return Next();
213
137
  }
214
138
 
215
- // greater case: find leftmost leaf node directly
216
- if (!equal) {
217
- while (node.DecodeARTNodeType() != NType::LEAF) {
139
+ current_key.Push(next_byte);
140
+ nodes.emplace(node, next_byte);
218
141
 
219
- uint8_t byte = 0;
220
- auto next_node = *node.GetNextChild(*art, byte);
221
- D_ASSERT(next_node.IsSet());
142
+ if (next_byte > key[depth]) {
143
+ // we only need to find the minimum from here
144
+ // because all keys will be greater than the lower bound
145
+ FindMinimum(*child);
146
+ return true;
147
+ }
222
148
 
223
- PushKey(node, byte);
224
- nodes.emplace(node, byte);
225
- node = next_node;
149
+ // recurse into the child
150
+ return LowerBound(*child, key, equal, depth + 1);
151
+ }
226
152
 
227
- // reconstruct the prefix
228
- node_prefix = node.GetPrefix(*art);
229
- for (idx_t i = 0; i < node_prefix.get().count; i++) {
230
- cur_key.Push(node_prefix.get().GetByte(*art, i));
231
- }
153
+ // resolve the prefix
154
+ auto &prefix = Prefix::Get(*art, node);
155
+ for (idx_t i = 0; i < prefix.data[Node::PREFIX_SIZE]; i++) {
156
+ current_key.Push(prefix.data[i]);
157
+ }
158
+ nodes.emplace(node, 0);
232
159
 
233
- auto &c_top = nodes.top();
234
- c_top.node = node;
235
- }
160
+ for (idx_t i = 0; i < prefix.data[Node::PREFIX_SIZE]; i++) {
161
+ // the key down to this node is less than the lower bound, the next key will be
162
+ // greater than the lower bound
163
+ if (prefix.data[i] < key[depth + i]) {
164
+ return Next();
165
+ }
166
+ // we only need to find the minimum from here
167
+ // because all keys will be greater than the lower bound
168
+ if (prefix.data[i] > key[depth + i]) {
169
+ FindMinimum(prefix.ptr);
170
+ return true;
236
171
  }
172
+ }
237
173
 
238
- if (node.DecodeARTNodeType() == NType::LEAF) {
239
- // found a leaf node: check if it is bigger or equal than the current key
240
- last_leaf = Node::GetAllocator(*art, NType::LEAF).Get<Leaf>(node);
241
-
242
- // if the search is not inclusive the leaf node could still be equal to the current value
243
- // check if leaf is equal to the current key
244
- if (cur_key == key) {
245
- // if it's not inclusive check if there is a next leaf
246
- if (!is_inclusive && !Next()) {
247
- return false;
248
- } else {
249
- return true;
250
- }
251
- }
174
+ // recurse into the child
175
+ depth += prefix.data[Node::PREFIX_SIZE];
176
+ return LowerBound(prefix.ptr, key, equal, depth);
177
+ }
252
178
 
253
- if (cur_key > key) {
254
- return true;
255
- }
256
- // Case1: When the ART has only one leaf node, the Next() will return false
257
- // Case2: This means the previous node prefix(if any) + a_key(one element of of key array of previous node)
258
- // == key[q..=w].
259
- // But key[w+1..=z] maybe greater than leaf node prefix.
260
- // One fact is key[w] is alawys equal to a_key and the next element
261
- // of key array of previous node is always > a_key So we just call Next() once.
179
+ bool Iterator::Next() {
262
180
 
263
- return Next();
264
- }
181
+ while (!nodes.empty()) {
265
182
 
266
- // equal case:
267
- node_prefix = node.GetPrefix(*art);
268
- auto mismatch_pos = node_prefix.get().KeyMismatchPosition(*art, key, depth);
269
- if (mismatch_pos != node_prefix.get().count) {
270
- if (node_prefix.get().GetByte(*art, mismatch_pos) < key[depth + mismatch_pos]) {
271
- // less
272
- PopNode();
273
- return Next();
274
- }
275
- // greater
276
- top.byte = 0;
277
- return Next();
183
+ auto &top = nodes.top();
184
+ D_ASSERT(top.node.DecodeARTNodeType() != NType::LEAF);
185
+
186
+ if (top.node.DecodeARTNodeType() == NType::PREFIX) {
187
+ PopNode();
188
+ continue;
278
189
  }
279
190
 
280
- // prefix matches, search inside the child for the key
281
- depth += node_prefix.get().count;
282
- top.byte = key[depth];
283
- auto child = node.GetNextChild(*art, top.byte);
284
- equal = key[depth] == top.byte;
191
+ if (top.byte == NumericLimits<uint8_t>::Maximum()) {
192
+ // no node found: move up the tree, pop key byte of current node
193
+ PopNode();
194
+ continue;
195
+ }
285
196
 
286
- // the maximum key byte of the current node is less than the key
287
- // fall back to the previous node
288
- if (!child) {
197
+ top.byte++;
198
+ auto next_node = top.node.GetNextChild(*art, top.byte);
199
+ if (!next_node) {
289
200
  PopNode();
290
- return Next();
201
+ continue;
291
202
  }
292
203
 
293
- PushKey(node, top.byte);
294
- node = *child;
204
+ current_key.Pop(1);
205
+ current_key.Push(top.byte);
295
206
 
296
- // all children of this node qualify as greater or equal
297
- depth++;
207
+ FindMinimum(*next_node);
208
+ return true;
298
209
  }
210
+ return false;
211
+ }
212
+
213
+ void Iterator::PopNode() {
214
+ if (nodes.top().node.DecodeARTNodeType() == NType::PREFIX) {
215
+ auto prefix_byte_count = Prefix::Get(*art, nodes.top().node).data[Node::PREFIX_SIZE];
216
+ current_key.Pop(prefix_byte_count);
217
+ } else {
218
+ current_key.Pop(1);
219
+ }
220
+ nodes.pop();
299
221
  }
300
222
 
301
223
  } // namespace duckdb
@@ -1,15 +1,14 @@
1
1
  #include "duckdb/execution/index/art/leaf.hpp"
2
2
 
3
3
  #include "duckdb/execution/index/art/art.hpp"
4
- #include "duckdb/execution/index/art/art_key.hpp"
5
- #include "duckdb/execution/index/art/leaf_segment.hpp"
6
4
  #include "duckdb/execution/index/art/node.hpp"
5
+ #include "duckdb/execution/index/art/leaf_segment.hpp"
7
6
  #include "duckdb/storage/meta_block_reader.hpp"
8
7
  #include "duckdb/storage/meta_block_writer.hpp"
9
8
 
10
9
  namespace duckdb {
11
10
 
12
- Leaf &Leaf::New(ART &art, Node &node, const ARTKey &key, const uint32_t depth, const row_t row_id) {
11
+ Leaf &Leaf::New(ART &art, Node &node, const row_t row_id) {
13
12
 
14
13
  node.SetPtr(Node::GetAllocator(art, NType::LEAF).New());
15
14
  node.type = (uint8_t)NType::LEAF;
@@ -18,40 +17,24 @@ Leaf &Leaf::New(ART &art, Node &node, const ARTKey &key, const uint32_t depth, c
18
17
  // set the fields of the leaf
19
18
  leaf.count = 1;
20
19
  leaf.row_ids.inlined = row_id;
21
-
22
- // initialize the prefix
23
- D_ASSERT(key.len >= depth);
24
- leaf.prefix.Initialize(art, key, depth, key.len - depth);
25
-
26
20
  return leaf;
27
21
  }
28
22
 
29
- Leaf &Leaf::New(ART &art, Node &node, const ARTKey &key, const uint32_t depth, const row_t *row_ids,
30
- const idx_t count) {
23
+ Leaf &Leaf::New(ART &art, Node &node, const row_t *row_ids, const idx_t count) {
31
24
 
32
- // inlined leaf
33
- D_ASSERT(count >= 1);
34
- if (count == 1) {
35
- return Leaf::New(art, node, key, depth, row_ids[0]);
36
- }
25
+ D_ASSERT(count > 1);
37
26
 
38
27
  node.SetPtr(Node::GetAllocator(art, NType::LEAF).New());
39
28
  node.type = (uint8_t)NType::LEAF;
40
29
  auto &leaf = Leaf::Get(art, node);
41
30
 
42
- // set the fields of the leaf
31
+ // reset the count to copy the row IDs
43
32
  leaf.count = 0;
44
-
45
- // copy the row IDs
46
33
  reference<LeafSegment> segment(LeafSegment::New(art, leaf.row_ids.ptr));
47
34
  for (idx_t i = 0; i < count; i++) {
48
35
  segment = segment.get().Append(art, leaf.count, row_ids[i]);
49
36
  }
50
37
 
51
- // set the prefix
52
- D_ASSERT(key.len >= depth);
53
- leaf.prefix.Initialize(art, key, depth, key.len - depth);
54
-
55
38
  return leaf;
56
39
  }
57
40
 
@@ -130,12 +113,7 @@ void Leaf::Merge(ART &art, Node &other) {
130
113
 
131
114
  void Leaf::Insert(ART &art, const row_t row_id) {
132
115
 
133
- if (count == 0) {
134
- row_ids.inlined = row_id;
135
- count++;
136
- return;
137
- }
138
-
116
+ D_ASSERT(count != 0);
139
117
  if (count == 1) {
140
118
  MoveInlinedToSegment(art);
141
119
  }
@@ -162,10 +140,7 @@ void Leaf::Remove(ART &art, const row_t row_id) {
162
140
  // possibly inline the row ID
163
141
  if (count == 2) {
164
142
  auto &segment = LeafSegment::Get(art, row_ids.ptr);
165
- if (segment.row_ids[0] != row_id && segment.row_ids[1] != row_id) {
166
- return;
167
- }
168
-
143
+ D_ASSERT(segment.row_ids[0] == row_id || segment.row_ids[1] == row_id);
169
144
  auto remaining_row_id = segment.row_ids[0] == row_id ? segment.row_ids[1] : segment.row_ids[0];
170
145
  Node::Free(art, row_ids.ptr);
171
146
  row_ids.inlined = remaining_row_id;
@@ -176,9 +151,7 @@ void Leaf::Remove(ART &art, const row_t row_id) {
176
151
  // find the row ID, and the segment containing that row ID (stored in ptr)
177
152
  auto ptr = row_ids.ptr;
178
153
  auto copy_idx = FindRowId(art, ptr, row_id);
179
- if (copy_idx == (uint32_t)DConstants::INVALID_INDEX) {
180
- return;
181
- }
154
+ D_ASSERT(copy_idx != (uint32_t)DConstants::INVALID_INDEX);
182
155
  copy_idx++;
183
156
 
184
157
  // iterate all remaining segments and move the row IDs one field to the left
@@ -303,7 +276,6 @@ BlockPointer Leaf::Serialize(const ART &art, MetaBlockWriter &writer) const {
303
276
  auto block_pointer = writer.GetBlockPointer();
304
277
  writer.Write(NType::LEAF);
305
278
  writer.Write<uint32_t>(count);
306
- prefix.Serialize(art, writer);
307
279
 
308
280
  if (IsInlined()) {
309
281
  writer.Write(row_ids.inlined);
@@ -336,7 +308,6 @@ BlockPointer Leaf::Serialize(const ART &art, MetaBlockWriter &writer) const {
336
308
  void Leaf::Deserialize(ART &art, MetaBlockReader &reader) {
337
309
 
338
310
  auto count_p = reader.Read<uint32_t>();
339
- prefix.Deserialize(art, reader);
340
311
 
341
312
  // inlined
342
313
  if (count_p == 1) {