duckdb 0.8.2-dev161.0 → 0.8.2-dev1764.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (504) hide show
  1. package/binding.gyp +15 -12
  2. package/binding.gyp.in +1 -1
  3. package/configure.py +1 -1
  4. package/duckdb_extension_config.cmake +10 -0
  5. package/package.json +1 -1
  6. package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
  7. package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
  8. package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
  9. package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
  10. package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
  11. package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
  12. package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
  13. package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
  14. package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
  15. package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
  16. package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
  17. package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
  18. package/src/duckdb/extension/json/include/json_common.hpp +47 -231
  19. package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
  20. package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
  21. package/src/duckdb/extension/json/json_common.cpp +272 -40
  22. package/src/duckdb/extension/json/json_functions/json_create.cpp +21 -2
  23. package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
  24. package/src/duckdb/extension/json/json_functions/json_transform.cpp +91 -38
  25. package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
  26. package/src/duckdb/extension/json/json_functions.cpp +24 -24
  27. package/src/duckdb/extension/json/json_scan.cpp +3 -6
  28. package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
  29. package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
  30. package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
  31. package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
  32. package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
  33. package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
  34. package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
  35. package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
  36. package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
  37. package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
  38. package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
  39. package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
  40. package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
  41. package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
  42. package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
  43. package/src/duckdb/extension/parquet/parquet_extension.cpp +194 -20
  44. package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
  45. package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
  46. package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
  47. package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
  48. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
  49. package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
  50. package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
  51. package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
  52. package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
  53. package/src/duckdb/src/common/allocator.cpp +14 -2
  54. package/src/duckdb/src/common/arrow/arrow_appender.cpp +79 -12
  55. package/src/duckdb/src/common/arrow/arrow_converter.cpp +44 -19
  56. package/src/duckdb/src/common/assert.cpp +3 -0
  57. package/src/duckdb/src/common/enum_util.cpp +4619 -4446
  58. package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
  59. package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
  60. package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
  61. package/src/duckdb/src/common/exception.cpp +2 -2
  62. package/src/duckdb/src/common/extra_type_info.cpp +506 -0
  63. package/src/duckdb/src/common/file_system.cpp +19 -0
  64. package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
  65. package/src/duckdb/src/common/local_file_system.cpp +14 -14
  66. package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
  67. package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
  68. package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
  69. package/src/duckdb/src/common/re2_regex.cpp +1 -1
  70. package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
  71. package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
  72. package/src/duckdb/src/common/sort/partition_state.cpp +70 -50
  73. package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
  74. package/src/duckdb/src/common/types/bit.cpp +51 -0
  75. package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
  76. package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
  77. package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
  78. package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
  79. package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
  80. package/src/duckdb/src/common/types/date.cpp +9 -0
  81. package/src/duckdb/src/common/types/list_segment.cpp +24 -74
  82. package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
  83. package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
  84. package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
  85. package/src/duckdb/src/common/types/uuid.cpp +2 -2
  86. package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
  87. package/src/duckdb/src/common/types/value.cpp +11 -6
  88. package/src/duckdb/src/common/types.cpp +9 -656
  89. package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
  90. package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
  91. package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
  92. package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
  93. package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
  94. package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
  95. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
  96. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
  97. package/src/duckdb/src/core_functions/function_list.cpp +4 -2
  98. package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
  99. package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
  100. package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
  101. package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
  102. package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
  103. package/src/duckdb/src/execution/expression_executor.cpp +1 -1
  104. package/src/duckdb/src/execution/index/art/art.cpp +149 -139
  105. package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
  106. package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
  107. package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
  108. package/src/duckdb/src/execution/index/art/node.cpp +113 -120
  109. package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
  110. package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
  111. package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
  112. package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
  113. package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
  114. package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
  115. package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
  116. package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
  117. package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
  118. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
  119. package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
  120. package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
  121. package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +444 -284
  122. package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
  123. package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
  124. package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +28 -12
  125. package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
  126. package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +23 -4
  127. package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +41 -5
  128. package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
  129. package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
  130. package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
  131. package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
  132. package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
  133. package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
  134. package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
  135. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
  136. package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
  137. package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
  138. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
  139. package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
  140. package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
  141. package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
  142. package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
  143. package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
  144. package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +56 -33
  145. package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +17 -13
  146. package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
  147. package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
  148. package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
  149. package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
  150. package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
  151. package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
  152. package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
  153. package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
  154. package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
  155. package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
  156. package/src/duckdb/src/function/function.cpp +3 -1
  157. package/src/duckdb/src/function/pragma/pragma_queries.cpp +5 -0
  158. package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
  159. package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
  160. package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
  161. package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
  162. package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
  163. package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
  164. package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
  165. package/src/duckdb/src/function/table/arrow.cpp +19 -0
  166. package/src/duckdb/src/function/table/arrow_conversion.cpp +35 -1
  167. package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
  168. package/src/duckdb/src/function/table/read_csv.cpp +100 -17
  169. package/src/duckdb/src/function/table/system/test_all_types.cpp +7 -0
  170. package/src/duckdb/src/function/table/system_functions.cpp +1 -0
  171. package/src/duckdb/src/function/table/table_scan.cpp +9 -0
  172. package/src/duckdb/src/function/table/version/pragma_version.cpp +46 -2
  173. package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
  174. package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
  175. package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
  176. package/src/duckdb/src/include/duckdb/common/dl.hpp +3 -1
  177. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +616 -584
  178. package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
  179. package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
  180. package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
  181. package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
  182. package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
  183. package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
  184. package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +219 -0
  185. package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
  186. package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
  187. package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
  188. package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
  189. package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
  190. package/src/duckdb/src/include/duckdb/common/opener_file_system.hpp +2 -2
  191. package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
  192. package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
  193. package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
  194. package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
  195. package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
  196. package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +23 -8
  197. package/src/duckdb/src/include/duckdb/common/string_util.hpp +11 -0
  198. package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
  199. package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
  200. package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
  201. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
  202. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
  203. package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
  204. package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
  205. package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
  206. package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
  207. package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +5 -2
  208. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
  209. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
  210. package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
  211. package/src/duckdb/src/include/duckdb/common/types/value.hpp +1 -0
  212. package/src/duckdb/src/include/duckdb/common/types.hpp +1 -15
  213. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
  214. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
  215. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
  216. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
  217. package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
  218. package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
  219. package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
  220. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
  221. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
  222. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
  223. package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
  224. package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
  225. package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
  226. package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
  227. package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
  228. package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
  229. package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
  230. package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
  231. package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
  232. package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
  233. package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
  234. package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
  235. package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
  236. package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
  237. package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
  238. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
  239. package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
  240. package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
  241. package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
  242. package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
  243. package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
  244. package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
  245. package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
  246. package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
  247. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +3 -10
  248. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +1 -1
  249. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +1 -1
  250. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp +12 -1
  251. package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
  252. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
  253. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
  254. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
  255. package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
  256. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
  257. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
  258. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
  259. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
  260. package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
  261. package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
  262. package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
  263. package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
  264. package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
  265. package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
  266. package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
  267. package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
  268. package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
  269. package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
  270. package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
  271. package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +4 -0
  272. package/src/duckdb/src/include/duckdb/main/client_config.hpp +5 -0
  273. package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
  274. package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
  275. package/src/duckdb/src/include/duckdb/main/extension/generated_extension_loader.hpp +22 -0
  276. package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +2 -0
  277. package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
  278. package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
  279. package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
  280. package/src/duckdb/src/include/duckdb/main/settings.hpp +39 -1
  281. package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
  282. package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
  283. package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
  284. package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
  285. package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
  286. package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
  287. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
  288. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
  289. package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
  290. package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
  291. package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
  292. package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
  293. package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
  294. package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
  295. package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
  296. package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
  297. package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
  298. package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
  299. package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
  300. package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
  301. package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
  302. package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
  303. package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
  304. package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
  305. package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
  306. package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
  307. package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
  308. package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
  309. package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
  310. package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
  311. package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
  312. package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
  313. package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
  314. package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
  315. package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
  316. package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
  317. package/src/duckdb/src/include/duckdb/planner/binder.hpp +12 -5
  318. package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
  319. package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
  320. package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
  321. package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
  322. package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
  323. package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
  324. package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
  325. package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
  326. package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
  327. package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
  328. package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
  329. package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
  330. package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
  331. package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
  332. package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
  333. package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
  334. package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
  335. package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
  336. package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
  337. package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
  338. package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
  339. package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
  340. package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
  341. package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
  342. package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
  343. package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
  344. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
  345. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
  346. package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
  347. package/src/duckdb/src/include/duckdb.h +28 -0
  348. package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
  349. package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
  350. package/src/duckdb/src/main/config.cpp +4 -0
  351. package/src/duckdb/src/main/database.cpp +1 -1
  352. package/src/duckdb/src/main/extension/extension_helper.cpp +93 -88
  353. package/src/duckdb/src/main/extension/extension_install.cpp +9 -0
  354. package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
  355. package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
  356. package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
  357. package/src/duckdb/src/main/relation.cpp +6 -5
  358. package/src/duckdb/src/main/settings/settings.cpp +79 -18
  359. package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
  360. package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
  361. package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
  362. package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
  363. package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
  364. package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
  365. package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
  366. package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
  367. package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
  368. package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
  369. package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
  370. package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
  371. package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
  372. package/src/duckdb/src/optimizer/optimizer.cpp +49 -14
  373. package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
  374. package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
  375. package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
  376. package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
  377. package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
  378. package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
  379. package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
  380. package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
  381. package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
  382. package/src/duckdb/src/parallel/executor.cpp +15 -0
  383. package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
  384. package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
  385. package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
  386. package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
  387. package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
  388. package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
  389. package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
  390. package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
  391. package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
  392. package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
  393. package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
  394. package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
  395. package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
  396. package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
  397. package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
  398. package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
  399. package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
  400. package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
  401. package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
  402. package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
  403. package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
  404. package/src/duckdb/src/parser/parser.cpp +8 -2
  405. package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
  406. package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
  407. package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
  408. package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
  409. package/src/duckdb/src/parser/query_node.cpp +15 -37
  410. package/src/duckdb/src/parser/result_modifier.cpp +0 -74
  411. package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
  412. package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
  413. package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
  414. package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
  415. package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
  416. package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
  417. package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
  418. package/src/duckdb/src/parser/tableref.cpp +0 -44
  419. package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
  420. package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
  421. package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
  422. package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
  423. package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
  424. package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
  425. package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
  426. package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
  427. package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
  428. package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
  429. package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
  430. package/src/duckdb/src/parser/transformer.cpp +15 -0
  431. package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
  432. package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
  433. package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
  434. package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
  435. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
  436. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
  437. package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
  438. package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
  439. package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
  440. package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
  441. package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +64 -26
  442. package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
  443. package/src/duckdb/src/planner/binder.cpp +44 -31
  444. package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
  445. package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
  446. package/src/duckdb/src/planner/expression_binder.cpp +3 -0
  447. package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
  448. package/src/duckdb/src/planner/logical_operator.cpp +5 -0
  449. package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
  450. package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
  451. package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
  452. package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
  453. package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
  454. package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
  455. package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
  456. package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
  457. package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
  458. package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
  459. package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
  460. package/src/duckdb/src/storage/compression/rle.cpp +0 -1
  461. package/src/duckdb/src/storage/data_table.cpp +1 -1
  462. package/src/duckdb/src/storage/local_storage.cpp +3 -3
  463. package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
  464. package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
  465. package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
  466. package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
  467. package/src/duckdb/src/storage/serialization/serialize_types.cpp +127 -0
  468. package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
  469. package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
  470. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  471. package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
  472. package/src/duckdb/src/storage/table/row_group.cpp +25 -9
  473. package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
  474. package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
  475. package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
  476. package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
  477. package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
  478. package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
  479. package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
  480. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
  481. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
  482. package/src/duckdb/ub_src_common.cpp +2 -0
  483. package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
  484. package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
  485. package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
  486. package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
  487. package/src/duckdb/ub_src_function_scalar.cpp +2 -0
  488. package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
  489. package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
  490. package/src/duckdb/ub_src_optimizer.cpp +6 -0
  491. package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
  492. package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
  493. package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
  494. package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
  495. package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
  496. package/src/duckdb/ub_src_planner_operator.cpp +4 -0
  497. package/src/duckdb/ub_src_storage_serialization.cpp +10 -0
  498. package/src/statement.cpp +10 -3
  499. package/test/columns.test.ts +24 -1
  500. package/test/test_all_types.test.ts +234 -0
  501. package/tsconfig.json +1 -0
  502. package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
  503. package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
  504. package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -14,9 +14,8 @@
14
14
 
15
15
  namespace duckdb {
16
16
 
17
+ //! Keeps track of the byte leading to the currently active child of the node
17
18
  struct IteratorEntry {
18
- IteratorEntry() {
19
- }
20
19
  IteratorEntry(Node node, uint8_t byte) : node(node), byte(byte) {
21
20
  }
22
21
 
@@ -24,16 +23,23 @@ struct IteratorEntry {
24
23
  uint8_t byte = 0;
25
24
  };
26
25
 
27
- //! Keeps track of the current key in the iterator
28
- class IteratorCurrentKey {
26
+ //! Keeps track of the current key in the iterator leading down to the top node in the stack
27
+ class IteratorKey {
29
28
  public:
30
- //! Push byte into current key
31
- void Push(const uint8_t key);
32
- //! Pops n elements from the key
33
- void Pop(const idx_t n);
29
+ //! Pushes a byte into the current key
30
+ inline void Push(const uint8_t key_byte) {
31
+ key_bytes.push_back(key_byte);
32
+ }
33
+ //! Pops n bytes from the current key
34
+ inline void Pop(const idx_t n) {
35
+ key_bytes.resize(key_bytes.size() - n);
36
+ }
34
37
 
35
38
  //! Subscript operator
36
- uint8_t &operator[](idx_t idx);
39
+ inline uint8_t &operator[](idx_t idx) {
40
+ D_ASSERT(idx < key_bytes.size());
41
+ return key_bytes[idx];
42
+ }
37
43
  //! Greater than operator
38
44
  bool operator>(const ARTKey &k) const;
39
45
  //! Greater than or equal to operator
@@ -42,37 +48,35 @@ public:
42
48
  bool operator==(const ARTKey &k) const;
43
49
 
44
50
  private:
45
- //! The current key position
46
- idx_t cur_key_pos = 0;
47
- //! The current key corresponding to the current leaf
48
- vector<uint8_t> key;
51
+ vector<uint8_t> key_bytes;
49
52
  };
50
53
 
51
54
  class Iterator {
52
55
  public:
53
- //! All information about the current key
54
- IteratorCurrentKey cur_key;
56
+ //! Holds the current key leading down to the top node on the stack
57
+ IteratorKey current_key;
55
58
  //! Pointer to the ART
56
- ART *art = nullptr;
59
+ optional_ptr<ART> art = nullptr;
57
60
 
58
- //! Scan the tree
59
- bool Scan(const ARTKey &key, const idx_t &max_count, vector<row_t> &result_ids, const bool &is_inclusive);
60
- //! Finds the minimum value of the tree
61
+ //! Scans the tree, starting at the current top node on the stack, and ending at upper_bound.
62
+ //! If upper_bound is the empty ARTKey, than there is no upper bound
63
+ bool Scan(const ARTKey &upper_bound, const idx_t max_count, vector<row_t> &result_ids, const bool equal);
64
+ //! Finds the minimum (leaf) of the current subtree
61
65
  void FindMinimum(Node &node);
62
- //! Goes to the lower bound of the tree
63
- bool LowerBound(Node node, const ARTKey &key, const bool &is_inclusive);
66
+ //! Finds the lower bound of the ART and adds the nodes to the stack. Returns false, if the lower
67
+ //! bound exceeds the maximum value of the ART
68
+ bool LowerBound(Node &node, const ARTKey &key, const bool equal, idx_t depth);
64
69
 
65
70
  private:
66
- //! Stack of iterator entries
71
+ //! Stack of nodes from the root to the currently active node
67
72
  stack<IteratorEntry> nodes;
68
73
  //! Last visited leaf
69
- Leaf *last_leaf = nullptr;
74
+ optional_ptr<Leaf> last_leaf = nullptr;
70
75
 
71
- //! Go to the next node
76
+ //! Goes to the next leaf in the ART and sets it as last_leaf,
77
+ //! returns false if there is no next leaf
72
78
  bool Next();
73
- //! Push part of the key to the current key
74
- void PushKey(const Node &node, const uint8_t byte);
75
- //! Pop node from the stack of iterator entries
79
+ //! Pop the top node from the stack of iterator entries and adjust the current key
76
80
  void PopNode();
77
81
  };
78
82
  } // namespace duckdb
@@ -8,16 +8,13 @@
8
8
 
9
9
  #pragma once
10
10
 
11
- #include "duckdb/execution/index/art/art.hpp"
12
11
  #include "duckdb/execution/index/art/fixed_size_allocator.hpp"
12
+ #include "duckdb/execution/index/art/art.hpp"
13
13
  #include "duckdb/execution/index/art/node.hpp"
14
- #include "duckdb/execution/index/art/prefix.hpp"
15
14
 
16
15
  namespace duckdb {
17
16
 
18
17
  // classes
19
- class Node;
20
- class ARTKey;
21
18
  class MetaBlockWriter;
22
19
  class MetaBlockReader;
23
20
 
@@ -28,8 +25,6 @@ class Leaf {
28
25
  public:
29
26
  //! Number of row IDs
30
27
  uint32_t count;
31
- //! Compressed path (prefix)
32
- Prefix prefix;
33
28
  union {
34
29
  //! The pointer to the head of the list of leaf segments
35
30
  Node ptr;
@@ -38,13 +33,10 @@ public:
38
33
  } row_ids;
39
34
 
40
35
  public:
41
- //! Get a new leaf node, might cause a new buffer allocation, and initializes a leaf holding one
42
- //! row ID and a prefix starting at depth
43
- static Leaf &New(ART &art, Node &node, const ARTKey &key, const uint32_t depth, const row_t row_id);
44
- //! Get a new leaf node, might cause a new buffer allocation, and initializes a leaf holding
45
- //! n_row_ids row IDs and a prefix starting at depth
46
- static Leaf &New(ART &art, Node &node, const ARTKey &key, const uint32_t depth, const row_t *row_ids,
47
- const idx_t count);
36
+ //! Get a new leaf node, might cause a new buffer allocation, and initializes a leaf holding one row ID
37
+ static Leaf &New(ART &art, Node &node, const row_t row_id);
38
+ //! Get a new leaf node, might cause a new buffer allocation, and initializes a leaf holding n_row_ids row IDs
39
+ static Leaf &New(ART &art, Node &node, const row_t *row_ids, const idx_t count);
48
40
  //! Free the leaf
49
41
  static void Free(ART &art, Node &node);
50
42
  //! Get a reference to the leaf
@@ -72,7 +64,7 @@ public:
72
64
  //! and sets the ptr to point to the segment containing the row ID
73
65
  uint32_t FindRowId(const ART &art, Node &ptr, const row_t row_id) const;
74
66
 
75
- //! Returns the string representation of a leaf
67
+ //! Returns the string representation of the node, or only traverses and verifies the node and its subtree
76
68
  string VerifyAndToString(const ART &art, const bool only_verify) const;
77
69
 
78
70
  //! Serialize this leaf
@@ -15,7 +15,7 @@ namespace duckdb {
15
15
 
16
16
  // classes
17
17
  enum class NType : uint8_t {
18
- PREFIX_SEGMENT = 1,
18
+ PREFIX = 1,
19
19
  LEAF_SEGMENT = 2,
20
20
  LEAF = 3,
21
21
  NODE_4 = 4,
@@ -38,11 +38,6 @@ struct ARTFlags;
38
38
  //! The remaining bytes are the position in the respective ART buffer.
39
39
  class Node : public SwizzleablePointer {
40
40
  public:
41
- // constants (this allows testing performance with different ART node sizes)
42
-
43
- //! Node prefixes (NOTE: this should always hold: PREFIX_SEGMENT_SIZE >= PREFIX_INLINE_BYTES)
44
- static constexpr uint32_t PREFIX_INLINE_BYTES = 8;
45
- static constexpr uint32_t PREFIX_SEGMENT_SIZE = 32;
46
41
  //! Node thresholds
47
42
  static constexpr uint8_t NODE_48_SHRINK_THRESHOLD = 12;
48
43
  static constexpr uint8_t NODE_256_SHRINK_THRESHOLD = 36;
@@ -54,6 +49,7 @@ public:
54
49
  //! Other constants
55
50
  static constexpr uint8_t EMPTY_MARKER = 48;
56
51
  static constexpr uint32_t LEAF_SEGMENT_SIZE = 8;
52
+ static constexpr uint8_t PREFIX_SIZE = 15;
57
53
 
58
54
  public:
59
55
  //! Constructs an empty ARTNode
@@ -73,7 +69,7 @@ public:
73
69
  //! Retrieve the node type from the leftmost byte
74
70
  inline NType DecodeARTNodeType() const {
75
71
  D_ASSERT(!IsSwizzled());
76
- D_ASSERT(type >= (uint8_t)NType::PREFIX_SEGMENT);
72
+ D_ASSERT(type >= (uint8_t)NType::PREFIX);
77
73
  D_ASSERT(type <= (uint8_t)NType::NODE_256);
78
74
  return NType(type);
79
75
  }
@@ -91,7 +87,7 @@ public:
91
87
  //! Insert the child node at byte
92
88
  static void InsertChild(ART &art, Node &node, const uint8_t byte, const Node child);
93
89
  //! Delete the child node at the respective byte
94
- static void DeleteChild(ART &art, Node &node, const uint8_t byte);
90
+ static void DeleteChild(ART &art, Node &node, Node &prefix, const uint8_t byte);
95
91
 
96
92
  //! Get the child for the respective byte in the node
97
93
  optional_ptr<Node> GetChild(ART &art, const uint8_t byte) const;
@@ -107,8 +103,6 @@ public:
107
103
  string VerifyAndToString(ART &art, const bool only_verify);
108
104
  //! Returns the capacity of the node
109
105
  idx_t GetCapacity() const;
110
- //! Returns a pointer to the prefix of the node
111
- Prefix &GetPrefix(ART &art);
112
106
  //! Returns the matching node type for a given count
113
107
  static NType GetARTNodeTypeByCount(const idx_t count);
114
108
  //! Get references to the different allocators
@@ -8,10 +8,9 @@
8
8
 
9
9
  #pragma once
10
10
 
11
- #include "duckdb/execution/index/art/art.hpp"
12
11
  #include "duckdb/execution/index/art/fixed_size_allocator.hpp"
12
+ #include "duckdb/execution/index/art/art.hpp"
13
13
  #include "duckdb/execution/index/art/node.hpp"
14
- #include "duckdb/execution/index/art/prefix.hpp"
15
14
 
16
15
  namespace duckdb {
17
16
 
@@ -20,8 +19,6 @@ class Node16 {
20
19
  public:
21
20
  //! Number of non-null children
22
21
  uint8_t count;
23
- //! Compressed path (prefix)
24
- Prefix prefix;
25
22
  //! Array containing all partial key bytes
26
23
  uint8_t key[Node::NODE_16_CAPACITY];
27
24
  //! ART node pointers to the child nodes
@@ -57,10 +54,10 @@ public:
57
54
  //! Get the first child that is greater or equal to the specific byte
58
55
  optional_ptr<Node> GetNextChild(uint8_t &byte);
59
56
 
60
- //! Serialize an ART node
57
+ //! Serialize this node
61
58
  BlockPointer Serialize(ART &art, MetaBlockWriter &writer);
62
59
  //! Deserialize this node
63
- void Deserialize(ART &art, MetaBlockReader &reader);
60
+ void Deserialize(MetaBlockReader &reader);
64
61
 
65
62
  //! Vacuum the children of the node
66
63
  void Vacuum(ART &art, const ARTFlags &flags);
@@ -8,10 +8,9 @@
8
8
 
9
9
  #pragma once
10
10
 
11
- #include "duckdb/execution/index/art/art.hpp"
12
11
  #include "duckdb/execution/index/art/fixed_size_allocator.hpp"
12
+ #include "duckdb/execution/index/art/art.hpp"
13
13
  #include "duckdb/execution/index/art/node.hpp"
14
- #include "duckdb/execution/index/art/prefix.hpp"
15
14
 
16
15
  namespace duckdb {
17
16
 
@@ -20,8 +19,6 @@ class Node256 {
20
19
  public:
21
20
  //! Number of non-null children
22
21
  uint16_t count;
23
- //! Compressed path (prefix)
24
- Prefix prefix;
25
22
  //! ART node pointers to the child nodes
26
23
  Node children[Node::NODE_256_CAPACITY];
27
24
 
@@ -60,10 +57,10 @@ public:
60
57
  //! Get the first child that is greater or equal to the specific byte
61
58
  optional_ptr<Node> GetNextChild(uint8_t &byte);
62
59
 
63
- //! Serialize an ART node
60
+ //! Serialize this node
64
61
  BlockPointer Serialize(ART &art, MetaBlockWriter &writer);
65
62
  //! Deserialize this node
66
- void Deserialize(ART &art, MetaBlockReader &reader);
63
+ void Deserialize(MetaBlockReader &reader);
67
64
 
68
65
  //! Vacuum the children of the node
69
66
  void Vacuum(ART &art, const ARTFlags &flags);
@@ -8,10 +8,9 @@
8
8
 
9
9
  #pragma once
10
10
 
11
- #include "duckdb/execution/index/art/art.hpp"
12
11
  #include "duckdb/execution/index/art/fixed_size_allocator.hpp"
12
+ #include "duckdb/execution/index/art/art.hpp"
13
13
  #include "duckdb/execution/index/art/node.hpp"
14
- #include "duckdb/execution/index/art/prefix.hpp"
15
14
 
16
15
  namespace duckdb {
17
16
 
@@ -20,8 +19,6 @@ class Node4 {
20
19
  public:
21
20
  //! Number of non-null children
22
21
  uint8_t count;
23
- //! Compressed path (prefix)
24
- Prefix prefix;
25
22
  //! Array containing all partial key bytes
26
23
  uint8_t key[Node::NODE_4_CAPACITY];
27
24
  //! ART node pointers to the child nodes
@@ -39,13 +36,13 @@ public:
39
36
  //! Initializes all fields of the node while shrinking a Node16 to a Node4
40
37
  static Node4 &ShrinkNode16(ART &art, Node &node4, Node &node16);
41
38
 
42
- //! Initializes a merge by incrementing the buffer IDs of the node
39
+ //! Initializes a merge by incrementing the buffer IDs of the child nodes
43
40
  void InitializeMerge(ART &art, const ARTFlags &flags);
44
41
 
45
42
  //! Insert a child node at byte
46
43
  static void InsertChild(ART &art, Node &node, const uint8_t byte, const Node child);
47
44
  //! Delete the child node at the respective byte
48
- static void DeleteChild(ART &art, Node &node, const uint8_t byte);
45
+ static void DeleteChild(ART &art, Node &node, Node &prefix, const uint8_t byte);
49
46
 
50
47
  //! Replace the child node at the respective byte
51
48
  void ReplaceChild(const uint8_t byte, const Node child);
@@ -55,10 +52,10 @@ public:
55
52
  //! Get the first child that is greater or equal to the specific byte
56
53
  optional_ptr<Node> GetNextChild(uint8_t &byte);
57
54
 
58
- //! Serialize an ART node
55
+ //! Serialize this node
59
56
  BlockPointer Serialize(ART &art, MetaBlockWriter &writer);
60
57
  //! Deserialize this node
61
- void Deserialize(ART &art, MetaBlockReader &reader);
58
+ void Deserialize(MetaBlockReader &reader);
62
59
 
63
60
  //! Vacuum the children of the node
64
61
  void Vacuum(ART &art, const ARTFlags &flags);
@@ -8,10 +8,9 @@
8
8
 
9
9
  #pragma once
10
10
 
11
- #include "duckdb/execution/index/art/art.hpp"
12
11
  #include "duckdb/execution/index/art/fixed_size_allocator.hpp"
12
+ #include "duckdb/execution/index/art/art.hpp"
13
13
  #include "duckdb/execution/index/art/node.hpp"
14
- #include "duckdb/execution/index/art/prefix.hpp"
15
14
 
16
15
  namespace duckdb {
17
16
 
@@ -21,8 +20,6 @@ class Node48 {
21
20
  public:
22
21
  //! Number of non-null children
23
22
  uint8_t count;
24
- //! Compressed path (prefix)
25
- Prefix prefix;
26
23
  //! Array containing all possible partial key bytes, those not set have an EMPTY_MARKER
27
24
  uint8_t child_index[Node::NODE_256_CAPACITY];
28
25
  //! ART node pointers to the child nodes
@@ -67,10 +64,10 @@ public:
67
64
  //! Get the first child that is greater or equal to the specific byte
68
65
  optional_ptr<Node> GetNextChild(uint8_t &byte);
69
66
 
70
- //! Serialize an ART node
67
+ //! Serialize this node
71
68
  BlockPointer Serialize(ART &art, MetaBlockWriter &writer);
72
69
  //! Deserialize this node
73
- void Deserialize(ART &art, MetaBlockReader &reader);
70
+ void Deserialize(MetaBlockReader &reader);
74
71
 
75
72
  //! Vacuum the children of the node
76
73
  void Vacuum(ART &art, const ARTFlags &flags);
@@ -7,78 +7,89 @@
7
7
  //===----------------------------------------------------------------------===//
8
8
  #pragma once
9
9
 
10
+ #include "duckdb/execution/index/art/art.hpp"
10
11
  #include "duckdb/execution/index/art/node.hpp"
11
12
 
12
13
  namespace duckdb {
13
14
 
14
15
  // classes
15
16
  class ARTKey;
16
- class PrefixSegment;
17
17
 
18
+ //! The Prefix is a special node type that contains up to PREFIX_SIZE bytes, and one byte for the count,
19
+ //! and a Node pointer. This pointer either points to another prefix
20
+ //! node or the 'actual' ART node.
18
21
  class Prefix {
19
22
  public:
20
- //! Number of bytes in this prefix
21
- uint32_t count;
22
- union {
23
- //! Pointer to the head of the list of prefix segments
24
- Node ptr;
25
- //! Inlined prefix bytes
26
- uint8_t inlined[Node::PREFIX_INLINE_BYTES];
27
- } data;
23
+ //! Up to PREFIX_SIZE bytes of prefix data and the count
24
+ uint8_t data[Node::PREFIX_SIZE + 1];
25
+ //! A pointer to the next ART node
26
+ Node ptr;
28
27
 
29
28
  public:
30
- //! Delete all prefix segments (if not inlined) and reset all fields
31
- void Free(ART &art);
32
- //! Initializes all the fields of an empty prefix
33
- inline void Initialize() {
34
- count = 0;
29
+ //! Get a new empty prefix node, might cause a new buffer allocation
30
+ static Prefix &New(ART &art, Node &node);
31
+ //! Create a new prefix node containing a single byte and a pointer to a next node
32
+ static Prefix &New(ART &art, Node &node, uint8_t byte, Node next);
33
+ //! Get a new chain of prefix nodes, might cause new buffer allocations,
34
+ //! with the node parameter holding the tail of the chain
35
+ static void New(ART &art, reference<Node> &node, const ARTKey &key, const uint32_t depth, uint32_t count);
36
+ //! Free the node (and its subtree)
37
+ static void Free(ART &art, Node &node);
38
+ //! Get a reference to the prefix
39
+ static inline Prefix &Get(const ART &art, const Node ptr) {
40
+ return *Node::GetAllocator(art, NType::PREFIX).Get<Prefix>(ptr);
35
41
  }
36
- //! Initialize a prefix from an ART key
37
- void Initialize(ART &art, const ARTKey &key, const uint32_t depth, const uint32_t count_p);
38
- //! Initialize a prefix from another prefix up to count
39
- void Initialize(ART &art, const Prefix &other, const uint32_t count_p);
40
42
 
41
- //! Initializes a merge by incrementing the buffer IDs of the prefix segments
42
- void InitializeMerge(ART &art, const idx_t buffer_count);
43
+ //! Initializes a merge by incrementing the buffer ID of the child node(s)
44
+ inline void InitializeMerge(ART &art, const ARTFlags &flags) {
45
+ ptr.InitializeMerge(art, flags);
46
+ }
43
47
 
44
- //! Move a prefix into this prefix
45
- inline void Move(Prefix &other) {
46
- count = other.count;
47
- data = other.data;
48
- other.Initialize();
48
+ //! Appends a byte and a child_prefix to prefix. If there is no prefix, than it pushes the
49
+ //! byte on top of child_prefix. If there is no child_prefix, then it creates a new
50
+ //! prefix node containing that byte
51
+ static void Concatenate(ART &art, Node &prefix_node, const uint8_t byte, Node &child_prefix_node);
52
+ //! Traverse a prefix and a key until (1) encountering a non-prefix node, or (2) encountering
53
+ //! a mismatching byte, in which case depth indexes the mismatching byte in the key
54
+ static idx_t Traverse(ART &art, reference<Node> &prefix_node, const ARTKey &key, idx_t &depth);
55
+ //! Traverse two prefixes to find (1) that they match (so far), or (2) that they have a mismatching position,
56
+ //! or (3) that one prefix contains the other prefix
57
+ static bool Traverse(ART &art, reference<Node> &l_node, reference<Node> &r_node, idx_t &mismatch_position);
58
+ //! Returns the byte at position
59
+ static inline uint8_t GetByte(const ART &art, const Node &prefix_node, const idx_t position) {
60
+ auto prefix = Prefix::Get(art, prefix_node);
61
+ D_ASSERT(position < Node::PREFIX_SIZE);
62
+ D_ASSERT(position < prefix.data[Node::PREFIX_SIZE]);
63
+ return prefix.data[position];
49
64
  }
50
- //! Append a prefix to this prefix
51
- void Append(ART &art, const Prefix &other);
52
- //! Concatenate prefix with a partial key byte and another prefix: other.prefix + byte + this->prefix
53
- void Concatenate(ART &art, const uint8_t byte, const Prefix &other);
54
- //! Removes the first n bytes, and returns the new first byte
55
- uint8_t Reduce(ART &art, const idx_t reduce_count);
65
+ //! Removes the first n bytes from the prefix and shifts all subsequent bytes in the
66
+ //! prefix node(s) by n. Frees empty prefix nodes
67
+ static void Reduce(ART &art, Node &prefix_node, const idx_t n);
68
+ //! Splits the prefix at position. prefix_node then references the ptr (if any bytes left before
69
+ //! the split), or stays unchanged (no bytes left before the split). child_node references
70
+ //! the node after the split, which is either a new prefix node, or ptr
71
+ static void Split(ART &art, reference<Node> &prefix_node, Node &child_node, idx_t position);
56
72
 
57
- //! Get the byte at position
58
- uint8_t GetByte(const ART &art, const idx_t position) const;
59
- //! Compare the key with the prefix of the node, return the position where they mismatch
60
- uint32_t KeyMismatchPosition(const ART &art, const ARTKey &key, const uint32_t depth) const;
61
- //! Compare this prefix to another prefix, return the position where they mismatch, or count otherwise
62
- uint32_t MismatchPosition(const ART &art, const Prefix &other) const;
73
+ //! Returns the string representation of the node, or only traverses and verifies the node and its subtree
74
+ string VerifyAndToString(ART &art, const bool only_verify);
63
75
 
64
- //! Serialize this prefix
65
- void Serialize(const ART &art, MetaBlockWriter &writer) const;
66
- //! Deserialize this prefix
67
- void Deserialize(ART &art, MetaBlockReader &reader);
76
+ //! Serialize this node
77
+ BlockPointer Serialize(ART &art, MetaBlockWriter &writer);
78
+ //! Deserialize this node
79
+ void Deserialize(MetaBlockReader &reader);
68
80
 
69
- //! Vacuum the prefix segments of a prefix, if not inlined
70
- void Vacuum(ART &art);
81
+ //! Vacuum the child of the node
82
+ inline void Vacuum(ART &art, const ARTFlags &flags) {
83
+ Node::Vacuum(art, ptr, flags);
84
+ }
71
85
 
72
86
  private:
73
- //! Returns whether this prefix is inlined
74
- inline bool IsInlined() const {
75
- return count <= Node::PREFIX_INLINE_BYTES;
76
- }
77
- //! Moves all inlined bytes onto a prefix segment, does not change the size
78
- //! so this will be an (temporarily) invalid prefix
79
- PrefixSegment &MoveInlinedToSegment(ART &art);
80
- //! Inlines up to eight bytes on the first prefix segment
81
- void MoveSegmentToInlined(ART &art);
87
+ //! Appends the byte to this prefix node, or creates a subsequent prefix node,
88
+ //! if this node is full
89
+ Prefix &Append(ART &art, const uint8_t byte);
90
+ //! Appends the other_prefix and all its subsequent prefix nodes to this prefix node.
91
+ //! Also frees all copied/appended nodes
92
+ void Append(ART &art, Node other_prefix);
82
93
  };
83
94
 
84
95
  } // namespace duckdb
@@ -13,8 +13,7 @@
13
13
 
14
14
  namespace duckdb {
15
15
 
16
- //! PhysicalAsOfJoin represents a piecewise merge loop join between
17
- //! two tables
16
+ //! PhysicalAsOfJoin represents an as-of join between two tables
18
17
  class PhysicalAsOfJoin : public PhysicalComparisonJoin {
19
18
  public:
20
19
  static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::ASOF_JOIN;
@@ -24,6 +23,7 @@ public:
24
23
 
25
24
  vector<LogicalType> join_key_types;
26
25
  vector<column_t> null_sensitive;
26
+ ExpressionType comparison_type;
27
27
 
28
28
  // Equalities
29
29
  vector<unique_ptr<Expression>> lhs_partitions;
@@ -58,7 +58,7 @@ public:
58
58
  SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
59
59
 
60
60
  bool IsSource() const override {
61
- return IsRightOuterJoin(join_type);
61
+ return true;
62
62
  }
63
63
  bool ParallelSource() const override {
64
64
  return true;
@@ -79,13 +79,6 @@ public:
79
79
  bool ParallelSink() const override {
80
80
  return true;
81
81
  }
82
-
83
- private:
84
- // resolve joins that output max N elements (SEMI, ANTI, MARK)
85
- void ResolveSimpleJoin(ExecutionContext &context, DataChunk &input, DataChunk &chunk, OperatorState &state) const;
86
- // resolve joins that can potentially output N*M elements (INNER, LEFT, FULL)
87
- OperatorResultType ResolveComplexJoin(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
88
- OperatorState &state) const;
89
82
  };
90
83
 
91
84
  } // namespace duckdb
@@ -20,7 +20,7 @@ public:
20
20
  static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::IE_JOIN;
21
21
 
22
22
  public:
23
- PhysicalIEJoin(LogicalOperator &op, unique_ptr<PhysicalOperator> left, unique_ptr<PhysicalOperator> right,
23
+ PhysicalIEJoin(LogicalComparisonJoin &op, unique_ptr<PhysicalOperator> left, unique_ptr<PhysicalOperator> right,
24
24
  vector<JoinCondition> cond, JoinType join_type, idx_t estimated_cardinality);
25
25
 
26
26
  vector<LogicalType> join_key_types;
@@ -22,7 +22,7 @@ public:
22
22
  static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::PIECEWISE_MERGE_JOIN;
23
23
 
24
24
  public:
25
- PhysicalPiecewiseMergeJoin(LogicalOperator &op, unique_ptr<PhysicalOperator> left,
25
+ PhysicalPiecewiseMergeJoin(LogicalComparisonJoin &op, unique_ptr<PhysicalOperator> left,
26
26
  unique_ptr<PhysicalOperator> right, vector<JoinCondition> cond, JoinType join_type,
27
27
  idx_t estimated_cardinality);
28
28
 
@@ -89,10 +89,18 @@ public:
89
89
  };
90
90
 
91
91
  public:
92
- PhysicalRangeJoin(LogicalOperator &op, PhysicalOperatorType type, unique_ptr<PhysicalOperator> left,
92
+ PhysicalRangeJoin(LogicalComparisonJoin &op, PhysicalOperatorType type, unique_ptr<PhysicalOperator> left,
93
93
  unique_ptr<PhysicalOperator> right, vector<JoinCondition> cond, JoinType join_type,
94
94
  idx_t estimated_cardinality);
95
95
 
96
+ // Projection mappings
97
+ using ProjectionMapping = vector<column_t>;
98
+ ProjectionMapping left_projection_map;
99
+ ProjectionMapping right_projection_map;
100
+
101
+ //! The full set of types (left + right child)
102
+ vector<LogicalType> unprojected_types;
103
+
96
104
  public:
97
105
  // Gather the result values and slice the payload columns to those values.
98
106
  // Returns a buffer handle to the pinned heap block (if any)
@@ -102,6 +110,9 @@ public:
102
110
  // Apply a tail condition to the current selection
103
111
  static idx_t SelectJoinTail(const ExpressionType &condition, Vector &left, Vector &right,
104
112
  const SelectionVector *sel, idx_t count, SelectionVector *true_sel);
113
+
114
+ //! Utility to project full width internal chunks to projected results
115
+ void ProjectResult(DataChunk &chunk, DataChunk &result) const;
105
116
  };
106
117
 
107
118
  } // namespace duckdb
@@ -37,7 +37,7 @@ class BaseCSVReader {
37
37
  public:
38
38
  BaseCSVReader(ClientContext &context, BufferedCSVReaderOptions options,
39
39
  const vector<LogicalType> &requested_types = vector<LogicalType>());
40
- ~BaseCSVReader();
40
+ virtual ~BaseCSVReader();
41
41
 
42
42
  ClientContext &context;
43
43
  FileSystem &fs;
@@ -75,7 +75,7 @@ public:
75
75
  }
76
76
 
77
77
  //! Get the 1-indexed global line number for the given local error line
78
- virtual idx_t GetLineError(idx_t line_error, idx_t buffer_idx) {
78
+ virtual idx_t GetLineError(idx_t line_error, idx_t buffer_idx, bool stop_at_first = true) {
79
79
  return line_error + 1;
80
80
  };
81
81
 
@@ -18,11 +18,12 @@ public:
18
18
  bool CanItGetLine(idx_t file_idx, idx_t batch_idx);
19
19
 
20
20
  //! Return the 1-indexed line number
21
- idx_t GetLine(idx_t batch_idx, idx_t line_error = 0, idx_t file_idx = 0, idx_t cur_start = 0, bool verify = true);
21
+ idx_t GetLine(idx_t batch_idx, idx_t line_error = 0, idx_t file_idx = 0, idx_t cur_start = 0, bool verify = true,
22
+ bool stop_at_first = true);
22
23
  //! Verify if the CSV File was read correctly from [0,batch_idx] batches.
23
24
  void Verify(idx_t file_idx, idx_t batch_idx, idx_t cur_first_pos);
24
- //! Lines read per batch, <batch_index,count>
25
- unordered_map<idx_t, idx_t> lines_read;
25
+ //! Lines read per batch, <file_index, <batch_index, count>>
26
+ vector<unordered_map<idx_t, idx_t>> lines_read;
26
27
  //! Set of batches that have been initialized but are not yet finished.
27
28
  vector<set<idx_t>> current_batches;
28
29
  //! Pointer to CSV Reader Mutex
@@ -55,6 +55,14 @@ struct BufferedCSVReaderOptions {
55
55
  bool header = false;
56
56
  //! Whether or not we should ignore InvalidInput errors
57
57
  bool ignore_errors = false;
58
+ //! Rejects table name
59
+ string rejects_table_name;
60
+ //! Rejects table entry limit (0 = no limit)
61
+ idx_t rejects_limit = 0;
62
+ //! Columns to use as recovery key for rejected rows when reading with ignore_errors = true
63
+ vector<string> rejects_recovery_columns;
64
+ //! Index of the recovery columns
65
+ vector<idx_t> rejects_recovery_column_ids;
58
66
  //! Expected number of columns
59
67
  idx_t num_cols = 0;
60
68
  //! Number of samples to buffer
@@ -155,7 +163,6 @@ struct BufferedCSVReaderOptions {
155
163
  //! set - argument(s) to the option
156
164
  //! expected_names - names expected if the option is "columns"
157
165
  void SetReadOption(const string &loption, const Value &value, vector<string> &expected_names);
158
-
159
166
  void SetWriteOption(const string &loption, const Value &value);
160
167
  void SetDateFormat(LogicalTypeId type, const string &format, bool read_format);
161
168