duckdb 0.8.2-dev150.0 → 0.8.2-dev1549.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (489) hide show
  1. package/binding.gyp +15 -12
  2. package/binding.gyp.in +1 -1
  3. package/configure.py +1 -1
  4. package/duckdb_extension_config.cmake +10 -0
  5. package/package.json +1 -1
  6. package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
  7. package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
  8. package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
  9. package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
  10. package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
  11. package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
  12. package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
  13. package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
  14. package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
  15. package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
  16. package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
  17. package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
  18. package/src/duckdb/extension/json/include/json_common.hpp +47 -231
  19. package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
  20. package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
  21. package/src/duckdb/extension/json/json_common.cpp +272 -40
  22. package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
  23. package/src/duckdb/extension/json/json_functions/json_transform.cpp +17 -37
  24. package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
  25. package/src/duckdb/extension/json/json_functions.cpp +24 -24
  26. package/src/duckdb/extension/json/json_scan.cpp +3 -6
  27. package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
  28. package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
  29. package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
  30. package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
  31. package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
  32. package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
  33. package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
  34. package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
  35. package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
  36. package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
  37. package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
  38. package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
  39. package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
  40. package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
  41. package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
  42. package/src/duckdb/extension/parquet/parquet_extension.cpp +192 -20
  43. package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
  44. package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
  45. package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
  46. package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
  47. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
  48. package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
  49. package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
  50. package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
  51. package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
  52. package/src/duckdb/src/common/allocator.cpp +14 -2
  53. package/src/duckdb/src/common/arrow/arrow_appender.cpp +5 -11
  54. package/src/duckdb/src/common/assert.cpp +3 -0
  55. package/src/duckdb/src/common/enum_util.cpp +4619 -4446
  56. package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
  57. package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
  58. package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
  59. package/src/duckdb/src/common/exception.cpp +2 -2
  60. package/src/duckdb/src/common/extra_type_info.cpp +506 -0
  61. package/src/duckdb/src/common/file_system.cpp +19 -0
  62. package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
  63. package/src/duckdb/src/common/local_file_system.cpp +14 -14
  64. package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
  65. package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
  66. package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
  67. package/src/duckdb/src/common/re2_regex.cpp +1 -1
  68. package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
  69. package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
  70. package/src/duckdb/src/common/sort/partition_state.cpp +44 -11
  71. package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
  72. package/src/duckdb/src/common/types/bit.cpp +51 -0
  73. package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
  74. package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
  75. package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
  76. package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
  77. package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
  78. package/src/duckdb/src/common/types/date.cpp +9 -0
  79. package/src/duckdb/src/common/types/list_segment.cpp +24 -74
  80. package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
  81. package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
  82. package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
  83. package/src/duckdb/src/common/types/uuid.cpp +2 -2
  84. package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
  85. package/src/duckdb/src/common/types.cpp +8 -655
  86. package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
  87. package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
  88. package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
  89. package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
  90. package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
  91. package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
  92. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
  93. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
  94. package/src/duckdb/src/core_functions/function_list.cpp +4 -2
  95. package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
  96. package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
  97. package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
  98. package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
  99. package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
  100. package/src/duckdb/src/execution/expression_executor.cpp +1 -1
  101. package/src/duckdb/src/execution/index/art/art.cpp +149 -139
  102. package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
  103. package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
  104. package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
  105. package/src/duckdb/src/execution/index/art/node.cpp +113 -120
  106. package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
  107. package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
  108. package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
  109. package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
  110. package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
  111. package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
  112. package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
  113. package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
  114. package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
  115. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
  116. package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
  117. package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
  118. package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +414 -283
  119. package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
  120. package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
  121. package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +28 -12
  122. package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
  123. package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +23 -4
  124. package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +41 -5
  125. package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
  126. package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
  127. package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
  128. package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
  129. package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
  130. package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
  131. package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
  132. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
  133. package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
  134. package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
  135. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
  136. package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
  137. package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
  138. package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
  139. package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
  140. package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
  141. package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +13 -22
  142. package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +17 -13
  143. package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
  144. package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
  145. package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
  146. package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
  147. package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
  148. package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
  149. package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
  150. package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
  151. package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
  152. package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
  153. package/src/duckdb/src/function/function.cpp +3 -1
  154. package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
  155. package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
  156. package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
  157. package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
  158. package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
  159. package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
  160. package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
  161. package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
  162. package/src/duckdb/src/function/table/read_csv.cpp +100 -17
  163. package/src/duckdb/src/function/table/table_scan.cpp +9 -0
  164. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  165. package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
  166. package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
  167. package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
  168. package/src/duckdb/src/include/duckdb/common/dl.hpp +3 -1
  169. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +616 -584
  170. package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
  171. package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
  172. package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
  173. package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
  174. package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
  175. package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
  176. package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +219 -0
  177. package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
  178. package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
  179. package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
  180. package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
  181. package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
  182. package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
  183. package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
  184. package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
  185. package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
  186. package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
  187. package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +13 -0
  188. package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
  189. package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
  190. package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
  191. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
  192. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
  193. package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
  194. package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
  195. package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
  196. package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
  197. package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +0 -1
  198. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
  199. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
  200. package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
  201. package/src/duckdb/src/include/duckdb/common/types.hpp +1 -15
  202. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
  203. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
  204. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
  205. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
  206. package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
  207. package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
  208. package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
  209. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
  210. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
  211. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
  212. package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
  213. package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
  214. package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
  215. package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
  216. package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
  217. package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
  218. package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
  219. package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
  220. package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
  221. package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
  222. package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
  223. package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
  224. package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
  225. package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
  226. package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
  227. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
  228. package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
  229. package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
  230. package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
  231. package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
  232. package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
  233. package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
  234. package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
  235. package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
  236. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +2 -10
  237. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +1 -1
  238. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +1 -1
  239. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp +12 -1
  240. package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
  241. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
  242. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
  243. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
  244. package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
  245. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
  246. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
  247. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
  248. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
  249. package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
  250. package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
  251. package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
  252. package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
  253. package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
  254. package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
  255. package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
  256. package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
  257. package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
  258. package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
  259. package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
  260. package/src/duckdb/src/include/duckdb/main/client_config.hpp +5 -0
  261. package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
  262. package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
  263. package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
  264. package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
  265. package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
  266. package/src/duckdb/src/include/duckdb/main/settings.hpp +39 -1
  267. package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
  268. package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
  269. package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
  270. package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
  271. package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
  272. package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
  273. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
  274. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
  275. package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
  276. package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
  277. package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
  278. package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
  279. package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
  280. package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
  281. package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
  282. package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
  283. package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
  284. package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
  285. package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
  286. package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
  287. package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
  288. package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
  289. package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
  290. package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
  291. package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
  292. package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
  293. package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
  294. package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
  295. package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
  296. package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
  297. package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
  298. package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
  299. package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
  300. package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
  301. package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
  302. package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
  303. package/src/duckdb/src/include/duckdb/planner/binder.hpp +8 -5
  304. package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
  305. package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
  306. package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
  307. package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
  308. package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
  309. package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
  310. package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
  311. package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
  312. package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
  313. package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
  314. package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
  315. package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
  316. package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
  317. package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
  318. package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
  319. package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
  320. package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
  321. package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
  322. package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
  323. package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
  324. package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
  325. package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
  326. package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
  327. package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
  328. package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
  329. package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
  330. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
  331. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
  332. package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
  333. package/src/duckdb/src/include/duckdb.h +28 -0
  334. package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
  335. package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
  336. package/src/duckdb/src/main/config.cpp +4 -0
  337. package/src/duckdb/src/main/database.cpp +1 -1
  338. package/src/duckdb/src/main/extension/extension_helper.cpp +96 -89
  339. package/src/duckdb/src/main/extension/extension_install.cpp +6 -0
  340. package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
  341. package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
  342. package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
  343. package/src/duckdb/src/main/relation.cpp +6 -5
  344. package/src/duckdb/src/main/settings/settings.cpp +79 -18
  345. package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
  346. package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
  347. package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
  348. package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
  349. package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
  350. package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
  351. package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
  352. package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
  353. package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
  354. package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
  355. package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
  356. package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
  357. package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
  358. package/src/duckdb/src/optimizer/optimizer.cpp +51 -14
  359. package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
  360. package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
  361. package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
  362. package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
  363. package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
  364. package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
  365. package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
  366. package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
  367. package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
  368. package/src/duckdb/src/parallel/executor.cpp +15 -0
  369. package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
  370. package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
  371. package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
  372. package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
  373. package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
  374. package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
  375. package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
  376. package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
  377. package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
  378. package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
  379. package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
  380. package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
  381. package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
  382. package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
  383. package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
  384. package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
  385. package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
  386. package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
  387. package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
  388. package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
  389. package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
  390. package/src/duckdb/src/parser/parser.cpp +8 -2
  391. package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
  392. package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
  393. package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
  394. package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
  395. package/src/duckdb/src/parser/query_node.cpp +15 -37
  396. package/src/duckdb/src/parser/result_modifier.cpp +0 -74
  397. package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
  398. package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
  399. package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
  400. package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
  401. package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
  402. package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
  403. package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
  404. package/src/duckdb/src/parser/tableref.cpp +0 -44
  405. package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
  406. package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
  407. package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
  408. package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
  409. package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
  410. package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
  411. package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
  412. package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
  413. package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
  414. package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
  415. package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
  416. package/src/duckdb/src/parser/transformer.cpp +15 -0
  417. package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
  418. package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
  419. package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
  420. package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
  421. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
  422. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
  423. package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
  424. package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
  425. package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
  426. package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
  427. package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +61 -26
  428. package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
  429. package/src/duckdb/src/planner/binder.cpp +5 -0
  430. package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
  431. package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
  432. package/src/duckdb/src/planner/expression_binder.cpp +3 -0
  433. package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
  434. package/src/duckdb/src/planner/logical_operator.cpp +5 -0
  435. package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
  436. package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
  437. package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
  438. package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
  439. package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
  440. package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
  441. package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
  442. package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
  443. package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
  444. package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
  445. package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
  446. package/src/duckdb/src/storage/compression/rle.cpp +0 -1
  447. package/src/duckdb/src/storage/data_table.cpp +1 -1
  448. package/src/duckdb/src/storage/local_storage.cpp +3 -3
  449. package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
  450. package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
  451. package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
  452. package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
  453. package/src/duckdb/src/storage/serialization/serialize_types.cpp +127 -0
  454. package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
  455. package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
  456. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  457. package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
  458. package/src/duckdb/src/storage/table/row_group.cpp +25 -9
  459. package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
  460. package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
  461. package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
  462. package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
  463. package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
  464. package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
  465. package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
  466. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
  467. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
  468. package/src/duckdb/ub_src_common.cpp +2 -0
  469. package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
  470. package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
  471. package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
  472. package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
  473. package/src/duckdb/ub_src_function_scalar.cpp +2 -0
  474. package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
  475. package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
  476. package/src/duckdb/ub_src_optimizer.cpp +6 -0
  477. package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
  478. package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
  479. package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
  480. package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
  481. package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
  482. package/src/duckdb/ub_src_planner_operator.cpp +4 -0
  483. package/src/duckdb/ub_src_storage_serialization.cpp +10 -0
  484. package/src/statement.cpp +10 -3
  485. package/test/test_all_types.test.ts +233 -0
  486. package/tsconfig.json +1 -0
  487. package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
  488. package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
  489. package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -5,7 +5,6 @@
5
5
  #include "duckdb/execution/expression_executor.hpp"
6
6
  #include "duckdb/storage/arena_allocator.hpp"
7
7
  #include "duckdb/execution/index/art/art_key.hpp"
8
- #include "duckdb/execution/index/art/prefix_segment.hpp"
9
8
  #include "duckdb/execution/index/art/leaf_segment.hpp"
10
9
  #include "duckdb/execution/index/art/prefix.hpp"
11
10
  #include "duckdb/execution/index/art/leaf.hpp"
@@ -44,7 +43,7 @@ ART::ART(const vector<column_t> &column_ids, TableIOManager &table_io_manager,
44
43
  }
45
44
 
46
45
  // initialize all allocators
47
- allocators.emplace_back(make_uniq<FixedSizeAllocator>(sizeof(PrefixSegment), buffer_manager.GetBufferAllocator()));
46
+ allocators.emplace_back(make_uniq<FixedSizeAllocator>(sizeof(Prefix), buffer_manager.GetBufferAllocator()));
48
47
  allocators.emplace_back(make_uniq<FixedSizeAllocator>(sizeof(LeafSegment), buffer_manager.GetBufferAllocator()));
49
48
  allocators.emplace_back(make_uniq<FixedSizeAllocator>(sizeof(Leaf), buffer_manager.GetBufferAllocator()));
50
49
  allocators.emplace_back(make_uniq<FixedSizeAllocator>(sizeof(Node4), buffer_manager.GetBufferAllocator()));
@@ -306,10 +305,12 @@ bool Construct(ART &art, vector<ARTKey> &keys, row_t *row_ids, Node &node, KeySe
306
305
  return false;
307
306
  }
308
307
 
308
+ reference<Node> ref_node(node);
309
+ Prefix::New(art, ref_node, start_key, prefix_start, start_key.len - prefix_start);
309
310
  if (single_row_id) {
310
- Leaf::New(art, node, start_key, prefix_start, row_ids[key_section.start]);
311
+ Leaf::New(art, ref_node, row_ids[key_section.start]);
311
312
  } else {
312
- Leaf::New(art, node, start_key, prefix_start, row_ids + key_section.start, num_row_ids);
313
+ Leaf::New(art, ref_node, row_ids + key_section.start, num_row_ids);
313
314
  }
314
315
  return true;
315
316
  }
@@ -320,17 +321,20 @@ bool Construct(ART &art, vector<ARTKey> &keys, row_t *row_ids, Node &node, KeySe
320
321
  vector<KeySection> child_sections;
321
322
  GetChildSections(child_sections, keys, key_section);
322
323
 
323
- auto node_type = Node::GetARTNodeTypeByCount(child_sections.size());
324
- Node::New(art, node, node_type);
325
-
324
+ // set the prefix
325
+ reference<Node> ref_node(node);
326
326
  auto prefix_length = key_section.depth - prefix_start;
327
- node.GetPrefix(art).Initialize(art, start_key, prefix_start, prefix_length);
327
+ Prefix::New(art, ref_node, start_key, prefix_start, prefix_length);
328
+
329
+ // set the node
330
+ auto node_type = Node::GetARTNodeTypeByCount(child_sections.size());
331
+ Node::New(art, ref_node, node_type);
328
332
 
329
333
  // recurse on each child section
330
334
  for (auto &child_section : child_sections) {
331
335
  Node new_child;
332
336
  auto no_violation = Construct(art, keys, row_ids, new_child, child_section, has_constraint);
333
- Node::InsertChild(art, node, child_section.key_byte, new_child);
337
+ Node::InsertChild(art, ref_node, child_section.key_byte, new_child);
334
338
  if (!no_violation) {
335
339
  return false;
336
340
  }
@@ -485,73 +489,75 @@ bool ART::InsertToLeaf(Node &leaf_node, const row_t &row_id) {
485
489
 
486
490
  bool ART::Insert(Node &node, const ARTKey &key, idx_t depth, const row_t &row_id) {
487
491
 
492
+ // node is currently empty, create a leaf here with the key
488
493
  if (!node.IsSet()) {
489
- // node is currently empty, create a leaf here with the key
490
- Leaf::New(*this, node, key, depth, row_id);
494
+ D_ASSERT(depth <= key.len);
495
+ reference<Node> ref_node(node);
496
+ Prefix::New(*this, ref_node, key, depth, key.len - depth);
497
+ Leaf::New(*this, ref_node, row_id);
491
498
  return true;
492
499
  }
493
500
 
494
- if (node.DecodeARTNodeType() == NType::LEAF) {
501
+ auto node_type = node.DecodeARTNodeType();
495
502
 
496
- // add a row ID to a leaf, if they have the same key
497
- auto &leaf = Leaf::Get(*this, node);
498
- auto mismatch_position = leaf.prefix.KeyMismatchPosition(*this, key, depth);
499
- if (mismatch_position == leaf.prefix.count && depth + leaf.prefix.count == key.len) {
500
- return InsertToLeaf(node, row_id);
501
- }
503
+ // insert the row ID into this leaf
504
+ if (node_type == NType::LEAF) {
505
+ return InsertToLeaf(node, row_id);
506
+ }
502
507
 
503
- // replace leaf with Node4 and store both leaves in it
504
- auto old_node = node;
505
- auto &new_n4 = Node4::New(*this, node);
506
- new_n4.prefix.Initialize(*this, key, depth, mismatch_position);
508
+ if (node_type != NType::PREFIX) {
509
+ D_ASSERT(depth < key.len);
510
+ auto child = node.GetChild(*this, key[depth]);
507
511
 
508
- auto key_byte = old_node.GetPrefix(*this).Reduce(*this, mismatch_position);
509
- Node4::InsertChild(*this, node, key_byte, old_node);
512
+ // recurse, if a child exists at key[depth]
513
+ if (child) {
514
+ bool success = Insert(*child, key, depth + 1, row_id);
515
+ node.ReplaceChild(*this, key[depth], *child);
516
+ return success;
517
+ }
510
518
 
519
+ // insert a new leaf node at key[depth]
511
520
  Node leaf_node;
512
- Leaf::New(*this, leaf_node, key, depth + mismatch_position + 1, row_id);
513
- Node4::InsertChild(*this, node, key[depth + mismatch_position], leaf_node);
514
-
521
+ if (depth + 1 < key.len) {
522
+ reference<Node> ref_node(leaf_node);
523
+ Prefix::New(*this, ref_node, key, depth + 1, key.len - depth - 1);
524
+ Leaf::New(*this, ref_node, row_id);
525
+ } else {
526
+ Leaf::New(*this, leaf_node, row_id);
527
+ }
528
+ Node::InsertChild(*this, node, key[depth], leaf_node);
515
529
  return true;
516
530
  }
517
531
 
518
- // handle prefix of inner node
519
- auto &old_node_prefix = node.GetPrefix(*this);
520
- if (old_node_prefix.count) {
521
-
522
- auto mismatch_position = old_node_prefix.KeyMismatchPosition(*this, key, depth);
523
- if (mismatch_position != old_node_prefix.count) {
524
-
525
- // prefix differs, create new node
526
- auto old_node = node;
527
- auto &new_n4 = Node4::New(*this, node);
528
- new_n4.prefix.Initialize(*this, key, depth, mismatch_position);
532
+ // this is a prefix node, traverse
533
+ reference<Node> next_node(node);
534
+ auto mismatch_position = Prefix::Traverse(*this, next_node, key, depth);
529
535
 
530
- auto key_byte = old_node_prefix.Reduce(*this, mismatch_position);
531
- Node4::InsertChild(*this, node, key_byte, old_node);
532
-
533
- Node leaf_node;
534
- Leaf::New(*this, leaf_node, key, depth + mismatch_position + 1, row_id);
535
- Node4::InsertChild(*this, node, key[depth + mismatch_position], leaf_node);
536
-
537
- return true;
538
- }
539
- depth += node.GetPrefix(*this).count;
536
+ // prefix matches key
537
+ if (next_node.get().DecodeARTNodeType() != NType::PREFIX) {
538
+ return Insert(next_node, key, depth, row_id);
540
539
  }
541
540
 
542
- // recurse
543
- D_ASSERT(depth < key.len);
544
- auto child = node.GetChild(*this, key[depth]);
545
- if (child) {
546
- bool success = Insert(*child, key, depth + 1, row_id);
547
- node.ReplaceChild(*this, key[depth], *child);
548
- return success;
549
- }
541
+ // prefix does not match the key, we need to create a new Node4; this new Node4 has two children,
542
+ // the remaining part of the prefix, and the new leaf
543
+ Node remaining_prefix;
544
+ auto prefix_byte = Prefix::GetByte(*this, next_node, mismatch_position);
545
+ Prefix::Split(*this, next_node, remaining_prefix, mismatch_position);
546
+ Node4::New(*this, next_node);
547
+
548
+ // insert remaining prefix
549
+ Node4::InsertChild(*this, next_node, prefix_byte, remaining_prefix);
550
550
 
551
- // insert at position
551
+ // insert new leaf
552
552
  Node leaf_node;
553
- Leaf::New(*this, leaf_node, key, depth + 1, row_id);
554
- Node::InsertChild(*this, node, key[depth], leaf_node);
553
+ if (depth + 1 < key.len) {
554
+ reference<Node> ref_node(leaf_node);
555
+ Prefix::New(*this, ref_node, key, depth + 1, key.len - depth - 1);
556
+ Leaf::New(*this, ref_node, row_id);
557
+ } else {
558
+ Leaf::New(*this, leaf_node, row_id);
559
+ }
560
+ Node4::InsertChild(*this, next_node, key[depth], leaf_node);
555
561
  return true;
556
562
  }
557
563
 
@@ -614,9 +620,18 @@ void ART::Erase(Node &node, const ARTKey &key, idx_t depth, const row_t &row_id)
614
620
  return;
615
621
  }
616
622
 
617
- // delete a row ID from a leaf
618
- if (node.DecodeARTNodeType() == NType::LEAF) {
619
- auto &leaf = Leaf::Get(*this, node);
623
+ // handle prefix
624
+ reference<Node> next_node(node);
625
+ if (next_node.get().DecodeARTNodeType() == NType::PREFIX) {
626
+ Prefix::Traverse(*this, next_node, key, depth);
627
+ if (next_node.get().DecodeARTNodeType() == NType::PREFIX) {
628
+ return;
629
+ }
630
+ }
631
+
632
+ // delete a row ID from a leaf (root is leaf with possible prefix nodes)
633
+ if (next_node.get().DecodeARTNodeType() == NType::LEAF) {
634
+ auto &leaf = Leaf::Get(*this, next_node.get());
620
635
  leaf.Remove(*this, row_id);
621
636
 
622
637
  if (leaf.count == 0) {
@@ -626,34 +641,35 @@ void ART::Erase(Node &node, const ARTKey &key, idx_t depth, const row_t &row_id)
626
641
  return;
627
642
  }
628
643
 
629
- // handle prefix
630
- auto &node_prefix = node.GetPrefix(*this);
631
- if (node_prefix.count) {
632
- if (node_prefix.KeyMismatchPosition(*this, key, depth) != node_prefix.count) {
633
- return;
634
- }
635
- depth += node_prefix.count;
636
- }
637
-
638
- auto child = node.GetChild(*this, key[depth]);
644
+ D_ASSERT(depth < key.len);
645
+ auto child = next_node.get().GetChild(*this, key[depth]);
639
646
  if (child) {
640
647
  D_ASSERT(child->IsSet());
641
648
 
642
- if (child->DecodeARTNodeType() == NType::LEAF) {
649
+ auto temp_depth = depth + 1;
650
+ reference<Node> child_node(*child);
651
+ if (child_node.get().DecodeARTNodeType() == NType::PREFIX) {
652
+ Prefix::Traverse(*this, child_node, key, temp_depth);
653
+ if (child_node.get().DecodeARTNodeType() == NType::PREFIX) {
654
+ return;
655
+ }
656
+ }
657
+
658
+ if (child_node.get().DecodeARTNodeType() == NType::LEAF) {
643
659
  // leaf found, remove entry
644
- auto &leaf = Leaf::Get(*this, *child);
660
+ auto &leaf = Leaf::Get(*this, child_node.get());
645
661
  leaf.Remove(*this, row_id);
646
662
 
647
663
  if (leaf.count == 0) {
648
664
  // leaf is empty, delete leaf, decrement node counter and maybe shrink node
649
- Node::DeleteChild(*this, node, key[depth]);
665
+ Node::DeleteChild(*this, next_node, node, key[depth]);
650
666
  }
651
667
  return;
652
668
  }
653
669
 
654
670
  // recurse
655
671
  Erase(*child, key, depth + 1, row_id);
656
- node.ReplaceChild(*this, key[depth], *child);
672
+ next_node.get().ReplaceChild(*this, key[depth], *child);
657
673
  }
658
674
  }
659
675
 
@@ -733,35 +749,28 @@ void ART::SearchEqualJoinNoFetch(ARTKey &key, idx_t &result_size) {
733
749
  Node ART::Lookup(Node node, const ARTKey &key, idx_t depth) {
734
750
 
735
751
  while (node.IsSet()) {
736
- if (node.DecodeARTNodeType() == NType::LEAF) {
737
- auto &leaf = Leaf::Get(*this, node);
738
752
 
739
- // check if leaf contains key
740
- for (idx_t i = 0; i < leaf.prefix.count; i++) {
741
- if (leaf.prefix.GetByte(*this, i) != key[i + depth]) {
742
- return Node();
743
- }
753
+ // traverse prefix, if exists
754
+ reference<Node> next_node(node);
755
+ if (next_node.get().DecodeARTNodeType() == NType::PREFIX) {
756
+ Prefix::Traverse(*this, next_node, key, depth);
757
+ if (next_node.get().DecodeARTNodeType() == NType::PREFIX) {
758
+ return Node();
744
759
  }
745
- return node;
746
760
  }
747
- auto &node_prefix = node.GetPrefix(*this);
748
- if (node_prefix.count) {
749
- for (idx_t pos = 0; pos < node_prefix.count; pos++) {
750
- if (key[depth + pos] != node_prefix.GetByte(*this, pos)) {
751
- // prefix mismatch, subtree of node does not contain key
752
- return Node();
753
- }
754
- }
755
- depth += node_prefix.count;
761
+
762
+ if (next_node.get().DecodeARTNodeType() == NType::LEAF) {
763
+ return next_node.get();
756
764
  }
757
765
 
758
- // prefix matches key, but no child at byte, does not contain key
759
- auto child = node.GetChild(*this, key[depth]);
766
+ D_ASSERT(depth < key.len);
767
+ auto child = next_node.get().GetChild(*this, key[depth]);
760
768
  if (!child) {
769
+ // prefix matches key, but no child at byte, ART/subtree does not contain key
761
770
  return Node();
762
771
  }
763
772
 
764
- // recurse into child
773
+ // lookup in child node
765
774
  node = *child;
766
775
  D_ASSERT(node.IsSet());
767
776
  depth++;
@@ -771,21 +780,21 @@ Node ART::Lookup(Node node, const ARTKey &key, idx_t depth) {
771
780
  }
772
781
 
773
782
  //===--------------------------------------------------------------------===//
774
- // Greater Than
775
- // Returns: True (If found leaf >= key)
776
- // False (Otherwise)
783
+ // Greater Than and Less Than
777
784
  //===--------------------------------------------------------------------===//
778
785
 
779
- bool ART::SearchGreater(ARTIndexScanState &state, ARTKey &key, bool inclusive, idx_t max_count,
780
- vector<row_t> &result_ids) {
786
+ bool ART::SearchGreater(ARTIndexScanState &state, ARTKey &key, bool equal, idx_t max_count, vector<row_t> &result_ids) {
781
787
 
782
- auto &it = state.iterator;
788
+ if (!tree->IsSet()) {
789
+ return true;
790
+ }
791
+ Iterator &it = state.iterator;
783
792
 
784
- // greater than scan: first set the iterator to the node at which we will start our scan by finding the lowest node
785
- // that satisfies our requirement
793
+ // find the lowest value that satisfies the predicate
786
794
  if (!it.art) {
787
795
  it.art = this;
788
- if (!it.LowerBound(*tree, key, inclusive)) {
796
+ if (!it.LowerBound(*tree, key, equal, 0)) {
797
+ // early-out, if the maximum value in the ART is lower than the lower bound
789
798
  return true;
790
799
  }
791
800
  }
@@ -796,86 +805,84 @@ bool ART::SearchGreater(ARTIndexScanState &state, ARTKey &key, bool inclusive, i
796
805
  return it.Scan(empty_key, max_count, result_ids, false);
797
806
  }
798
807
 
799
- //===--------------------------------------------------------------------===//
800
- // Less Than
801
- //===--------------------------------------------------------------------===//
802
-
803
- bool ART::SearchLess(ARTIndexScanState &state, ARTKey &upper_bound, bool inclusive, idx_t max_count,
808
+ bool ART::SearchLess(ARTIndexScanState &state, ARTKey &upper_bound, bool equal, idx_t max_count,
804
809
  vector<row_t> &result_ids) {
805
810
 
806
811
  if (!tree->IsSet()) {
807
812
  return true;
808
813
  }
809
-
810
- auto &it = state.iterator;
814
+ Iterator &it = state.iterator;
811
815
 
812
816
  if (!it.art) {
813
817
  it.art = this;
814
- // first find the minimum value in the ART: we start scanning from this value
818
+ // find the minimum value in the ART: we start scanning from this value
815
819
  it.FindMinimum(*tree);
816
- // early out min value higher than upper bound query
817
- if (it.cur_key > upper_bound) {
820
+ // early-out, if the minimum value is higher than the upper bound
821
+ if (it.current_key > upper_bound) {
818
822
  return true;
819
823
  }
820
824
  }
821
825
 
822
826
  // now continue the scan until we reach the upper bound
823
- return it.Scan(upper_bound, max_count, result_ids, inclusive);
827
+ return it.Scan(upper_bound, max_count, result_ids, equal);
824
828
  }
825
829
 
826
830
  //===--------------------------------------------------------------------===//
827
831
  // Closed Range Query
828
832
  //===--------------------------------------------------------------------===//
829
833
 
830
- bool ART::SearchCloseRange(ARTIndexScanState &state, ARTKey &lower_bound, ARTKey &upper_bound, bool left_inclusive,
831
- bool right_inclusive, idx_t max_count, vector<row_t> &result_ids) {
832
- auto &it = state.iterator;
834
+ bool ART::SearchCloseRange(ARTIndexScanState &state, ARTKey &lower_bound, ARTKey &upper_bound, bool left_equal,
835
+ bool right_equal, idx_t max_count, vector<row_t> &result_ids) {
836
+
837
+ Iterator &it = state.iterator;
833
838
 
834
- // first find the first node that satisfies the left predicate
839
+ // find the first node that satisfies the left predicate
835
840
  if (!it.art) {
836
841
  it.art = this;
837
- if (!it.LowerBound(*tree, lower_bound, left_inclusive)) {
842
+ if (!it.LowerBound(*tree, lower_bound, left_equal, 0)) {
843
+ // early-out, if the maximum value in the ART is lower than the lower bound
838
844
  return true;
839
845
  }
840
846
  }
841
847
 
842
848
  // now continue the scan until we reach the upper bound
843
- return it.Scan(upper_bound, max_count, result_ids, right_inclusive);
849
+ return it.Scan(upper_bound, max_count, result_ids, right_equal);
844
850
  }
845
851
 
846
- bool ART::Scan(const Transaction &transaction, const DataTable &table, IndexScanState &table_state,
847
- const idx_t max_count, vector<row_t> &result_ids) {
848
- auto &state = table_state.Cast<ARTIndexScanState>();
852
+ bool ART::Scan(const Transaction &transaction, const DataTable &table, IndexScanState &state, const idx_t max_count,
853
+ vector<row_t> &result_ids) {
854
+
855
+ auto &scan_state = state.Cast<ARTIndexScanState>();
849
856
  vector<row_t> row_ids;
850
857
  bool success;
851
858
 
852
859
  // FIXME: the key directly owning the data for a single key might be more efficient
853
- D_ASSERT(state.values[0].type().InternalType() == types[0]);
860
+ D_ASSERT(scan_state.values[0].type().InternalType() == types[0]);
854
861
  ArenaAllocator arena_allocator(Allocator::Get(db));
855
- auto key = CreateKey(arena_allocator, types[0], state.values[0]);
862
+ auto key = CreateKey(arena_allocator, types[0], scan_state.values[0]);
856
863
 
857
- if (state.values[1].IsNull()) {
864
+ if (scan_state.values[1].IsNull()) {
858
865
 
859
866
  // single predicate
860
867
  lock_guard<mutex> l(lock);
861
- switch (state.expressions[0]) {
868
+ switch (scan_state.expressions[0]) {
862
869
  case ExpressionType::COMPARE_EQUAL:
863
870
  success = SearchEqual(key, max_count, row_ids);
864
871
  break;
865
872
  case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
866
- success = SearchGreater(state, key, true, max_count, row_ids);
873
+ success = SearchGreater(scan_state, key, true, max_count, row_ids);
867
874
  break;
868
875
  case ExpressionType::COMPARE_GREATERTHAN:
869
- success = SearchGreater(state, key, false, max_count, row_ids);
876
+ success = SearchGreater(scan_state, key, false, max_count, row_ids);
870
877
  break;
871
878
  case ExpressionType::COMPARE_LESSTHANOREQUALTO:
872
- success = SearchLess(state, key, true, max_count, row_ids);
879
+ success = SearchLess(scan_state, key, true, max_count, row_ids);
873
880
  break;
874
881
  case ExpressionType::COMPARE_LESSTHAN:
875
- success = SearchLess(state, key, false, max_count, row_ids);
882
+ success = SearchLess(scan_state, key, false, max_count, row_ids);
876
883
  break;
877
884
  default:
878
- throw InternalException("Operation not implemented");
885
+ throw InternalException("Index scan type not implemented");
879
886
  }
880
887
 
881
888
  } else {
@@ -883,12 +890,12 @@ bool ART::Scan(const Transaction &transaction, const DataTable &table, IndexScan
883
890
  // two predicates
884
891
  lock_guard<mutex> l(lock);
885
892
 
886
- D_ASSERT(state.values[1].type().InternalType() == types[0]);
887
- auto upper_bound = CreateKey(arena_allocator, types[0], state.values[1]);
893
+ D_ASSERT(scan_state.values[1].type().InternalType() == types[0]);
894
+ auto upper_bound = CreateKey(arena_allocator, types[0], scan_state.values[1]);
888
895
 
889
- bool left_inclusive = state.expressions[0] == ExpressionType ::COMPARE_GREATERTHANOREQUALTO;
890
- bool right_inclusive = state.expressions[1] == ExpressionType ::COMPARE_LESSTHANOREQUALTO;
891
- success = SearchCloseRange(state, key, upper_bound, left_inclusive, right_inclusive, max_count, row_ids);
896
+ bool left_equal = scan_state.expressions[0] == ExpressionType ::COMPARE_GREATERTHANOREQUALTO;
897
+ bool right_equal = scan_state.expressions[1] == ExpressionType ::COMPARE_LESSTHANOREQUALTO;
898
+ success = SearchCloseRange(scan_state, key, upper_bound, left_equal, right_equal, max_count, row_ids);
892
899
  }
893
900
 
894
901
  if (!success) {
@@ -1136,6 +1143,9 @@ bool ART::MergeIndexes(IndexLock &state, Index &other_index) {
1136
1143
  //===--------------------------------------------------------------------===//
1137
1144
 
1138
1145
  string ART::VerifyAndToString(IndexLock &state, const bool only_verify) {
1146
+ // FIXME: this can be improved by counting the allocations of each node type,
1147
+ // FIXME: and by asserting that each fixed-size allocator lists an equal number of
1148
+ // FIXME: allocations of that type
1139
1149
  return VerifyAndToStringInternal(only_verify);
1140
1150
  }
1141
1151
 
@@ -114,7 +114,7 @@ void FixedSizeAllocator::Merge(FixedSizeAllocator &other) {
114
114
  }
115
115
  other.buffers.clear();
116
116
 
117
- // merge the vectors containing all buffers with free space
117
+ // merge the buffers with free spaces
118
118
  for (auto &buffer_id : other.buffers_with_free_space) {
119
119
  buffers_with_free_space.insert(buffer_id + buffer_count);
120
120
  }