duckdb 0.8.2-dev145.0 → 0.8.2-dev1493.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (476) hide show
  1. package/binding.gyp +15 -12
  2. package/binding.gyp.in +1 -1
  3. package/configure.py +1 -1
  4. package/duckdb_extension_config.cmake +10 -0
  5. package/package.json +1 -1
  6. package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
  7. package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
  8. package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
  9. package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
  10. package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
  11. package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
  12. package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
  13. package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
  14. package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
  15. package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
  16. package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
  17. package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
  18. package/src/duckdb/extension/json/include/json_common.hpp +47 -231
  19. package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
  20. package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
  21. package/src/duckdb/extension/json/json_common.cpp +272 -40
  22. package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
  23. package/src/duckdb/extension/json/json_functions/json_transform.cpp +17 -37
  24. package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
  25. package/src/duckdb/extension/json/json_functions.cpp +24 -24
  26. package/src/duckdb/extension/json/json_scan.cpp +3 -6
  27. package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
  28. package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
  29. package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
  30. package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
  31. package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
  32. package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
  33. package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
  34. package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
  35. package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
  36. package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
  37. package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
  38. package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
  39. package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
  40. package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
  41. package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
  42. package/src/duckdb/extension/parquet/parquet_extension.cpp +192 -20
  43. package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
  44. package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
  45. package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
  46. package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
  47. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
  48. package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
  49. package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
  50. package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
  51. package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
  52. package/src/duckdb/src/common/allocator.cpp +14 -2
  53. package/src/duckdb/src/common/arrow/arrow_appender.cpp +5 -11
  54. package/src/duckdb/src/common/assert.cpp +3 -0
  55. package/src/duckdb/src/common/enum_util.cpp +42 -5
  56. package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
  57. package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
  58. package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
  59. package/src/duckdb/src/common/exception.cpp +2 -2
  60. package/src/duckdb/src/common/file_system.cpp +19 -0
  61. package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
  62. package/src/duckdb/src/common/local_file_system.cpp +2 -2
  63. package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
  64. package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
  65. package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
  66. package/src/duckdb/src/common/re2_regex.cpp +1 -1
  67. package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
  68. package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
  69. package/src/duckdb/src/common/sort/partition_state.cpp +44 -11
  70. package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
  71. package/src/duckdb/src/common/types/bit.cpp +51 -0
  72. package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
  73. package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
  74. package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
  75. package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
  76. package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
  77. package/src/duckdb/src/common/types/date.cpp +9 -0
  78. package/src/duckdb/src/common/types/list_segment.cpp +24 -74
  79. package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
  80. package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
  81. package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
  82. package/src/duckdb/src/common/types/uuid.cpp +2 -2
  83. package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
  84. package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
  85. package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
  86. package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
  87. package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
  88. package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
  89. package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
  90. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
  91. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
  92. package/src/duckdb/src/core_functions/function_list.cpp +4 -2
  93. package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
  94. package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
  95. package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
  96. package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
  97. package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
  98. package/src/duckdb/src/execution/expression_executor.cpp +1 -1
  99. package/src/duckdb/src/execution/index/art/art.cpp +149 -139
  100. package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
  101. package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
  102. package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
  103. package/src/duckdb/src/execution/index/art/node.cpp +113 -120
  104. package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
  105. package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
  106. package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
  107. package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
  108. package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
  109. package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
  110. package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
  111. package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
  112. package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
  113. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
  114. package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
  115. package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
  116. package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +414 -283
  117. package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
  118. package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
  119. package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
  120. package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +22 -3
  121. package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +2 -2
  122. package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
  123. package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
  124. package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
  125. package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
  126. package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
  127. package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
  128. package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
  129. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
  130. package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
  131. package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
  132. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
  133. package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
  134. package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
  135. package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
  136. package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
  137. package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
  138. package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
  139. package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
  140. package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
  141. package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
  142. package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
  143. package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
  144. package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
  145. package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
  146. package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
  147. package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
  148. package/src/duckdb/src/function/function.cpp +3 -1
  149. package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
  150. package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
  151. package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
  152. package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
  153. package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
  154. package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
  155. package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
  156. package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
  157. package/src/duckdb/src/function/table/read_csv.cpp +100 -17
  158. package/src/duckdb/src/function/table/table_scan.cpp +9 -0
  159. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  160. package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
  161. package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
  162. package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
  163. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +8 -0
  164. package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
  165. package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
  166. package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
  167. package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
  168. package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
  169. package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
  170. package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
  171. package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
  172. package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
  173. package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
  174. package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
  175. package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
  176. package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
  177. package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
  178. package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
  179. package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
  180. package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +13 -0
  181. package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
  182. package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
  183. package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
  184. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
  185. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
  186. package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
  187. package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
  188. package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
  189. package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
  190. package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +0 -1
  191. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
  192. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
  193. package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
  194. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
  195. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
  196. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
  197. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
  198. package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
  199. package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
  200. package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
  201. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
  202. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
  203. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
  204. package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
  205. package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
  206. package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
  207. package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
  208. package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
  209. package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
  210. package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
  211. package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
  212. package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
  213. package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
  214. package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
  215. package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
  216. package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
  217. package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
  218. package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
  219. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
  220. package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
  221. package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
  222. package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
  223. package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
  224. package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
  225. package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
  226. package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
  227. package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
  228. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +2 -10
  229. package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
  230. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
  231. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
  232. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
  233. package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
  234. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
  235. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
  236. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
  237. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
  238. package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
  239. package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
  240. package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
  241. package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
  242. package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
  243. package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
  244. package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
  245. package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
  246. package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
  247. package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
  248. package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
  249. package/src/duckdb/src/include/duckdb/main/client_config.hpp +3 -0
  250. package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
  251. package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
  252. package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
  253. package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
  254. package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
  255. package/src/duckdb/src/include/duckdb/main/settings.hpp +30 -1
  256. package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
  257. package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
  258. package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
  259. package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
  260. package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
  261. package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
  262. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
  263. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
  264. package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
  265. package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
  266. package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
  267. package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
  268. package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
  269. package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
  270. package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
  271. package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
  272. package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
  273. package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
  274. package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
  275. package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
  276. package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
  277. package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
  278. package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
  279. package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
  280. package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
  281. package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
  282. package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
  283. package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
  284. package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
  285. package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
  286. package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
  287. package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
  288. package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
  289. package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
  290. package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
  291. package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
  292. package/src/duckdb/src/include/duckdb/planner/binder.hpp +8 -5
  293. package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
  294. package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
  295. package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
  296. package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
  297. package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
  298. package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
  299. package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
  300. package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
  301. package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
  302. package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
  303. package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
  304. package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
  305. package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
  306. package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
  307. package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
  308. package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
  309. package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
  310. package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
  311. package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
  312. package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
  313. package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
  314. package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
  315. package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
  316. package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
  317. package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
  318. package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
  319. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
  320. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
  321. package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
  322. package/src/duckdb/src/include/duckdb.h +28 -0
  323. package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
  324. package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
  325. package/src/duckdb/src/main/config.cpp +3 -0
  326. package/src/duckdb/src/main/database.cpp +1 -1
  327. package/src/duckdb/src/main/extension/extension_helper.cpp +96 -89
  328. package/src/duckdb/src/main/extension/extension_install.cpp +6 -0
  329. package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
  330. package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
  331. package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
  332. package/src/duckdb/src/main/relation.cpp +6 -5
  333. package/src/duckdb/src/main/settings/settings.cpp +64 -18
  334. package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
  335. package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
  336. package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
  337. package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
  338. package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
  339. package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
  340. package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
  341. package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
  342. package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
  343. package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
  344. package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
  345. package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
  346. package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
  347. package/src/duckdb/src/optimizer/optimizer.cpp +51 -14
  348. package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
  349. package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
  350. package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
  351. package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
  352. package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
  353. package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
  354. package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
  355. package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
  356. package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
  357. package/src/duckdb/src/parallel/executor.cpp +15 -0
  358. package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
  359. package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
  360. package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
  361. package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
  362. package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
  363. package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
  364. package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
  365. package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
  366. package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
  367. package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
  368. package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
  369. package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
  370. package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
  371. package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
  372. package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
  373. package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
  374. package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
  375. package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
  376. package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
  377. package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
  378. package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
  379. package/src/duckdb/src/parser/parser.cpp +8 -2
  380. package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
  381. package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
  382. package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
  383. package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
  384. package/src/duckdb/src/parser/query_node.cpp +15 -37
  385. package/src/duckdb/src/parser/result_modifier.cpp +0 -74
  386. package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
  387. package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
  388. package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
  389. package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
  390. package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
  391. package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
  392. package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
  393. package/src/duckdb/src/parser/tableref.cpp +0 -44
  394. package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
  395. package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
  396. package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
  397. package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
  398. package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
  399. package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
  400. package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
  401. package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
  402. package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
  403. package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
  404. package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
  405. package/src/duckdb/src/parser/transformer.cpp +15 -0
  406. package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
  407. package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
  408. package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
  409. package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
  410. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
  411. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
  412. package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
  413. package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
  414. package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
  415. package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
  416. package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +61 -26
  417. package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
  418. package/src/duckdb/src/planner/binder.cpp +5 -0
  419. package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
  420. package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
  421. package/src/duckdb/src/planner/expression_binder.cpp +3 -0
  422. package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
  423. package/src/duckdb/src/planner/logical_operator.cpp +5 -0
  424. package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
  425. package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
  426. package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
  427. package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
  428. package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
  429. package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
  430. package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
  431. package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
  432. package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
  433. package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
  434. package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
  435. package/src/duckdb/src/storage/compression/rle.cpp +0 -1
  436. package/src/duckdb/src/storage/data_table.cpp +1 -1
  437. package/src/duckdb/src/storage/local_storage.cpp +3 -3
  438. package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
  439. package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
  440. package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
  441. package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
  442. package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
  443. package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
  444. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  445. package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
  446. package/src/duckdb/src/storage/table/row_group.cpp +25 -9
  447. package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
  448. package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
  449. package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
  450. package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
  451. package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
  452. package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
  453. package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
  454. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
  455. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
  456. package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
  457. package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
  458. package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
  459. package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
  460. package/src/duckdb/ub_src_function_scalar.cpp +2 -0
  461. package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
  462. package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
  463. package/src/duckdb/ub_src_optimizer.cpp +6 -0
  464. package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
  465. package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
  466. package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
  467. package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
  468. package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
  469. package/src/duckdb/ub_src_planner_operator.cpp +4 -0
  470. package/src/duckdb/ub_src_storage_serialization.cpp +8 -0
  471. package/src/statement.cpp +10 -3
  472. package/test/test_all_types.test.ts +233 -0
  473. package/tsconfig.json +1 -0
  474. package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
  475. package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
  476. package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -14,7 +14,8 @@ namespace duckdb {
14
14
  LogicalGet::LogicalGet(idx_t table_index, TableFunction function, unique_ptr<FunctionData> bind_data,
15
15
  vector<LogicalType> returned_types, vector<string> returned_names)
16
16
  : LogicalOperator(LogicalOperatorType::LOGICAL_GET), table_index(table_index), function(std::move(function)),
17
- bind_data(std::move(bind_data)), returned_types(std::move(returned_types)), names(std::move(returned_names)) {
17
+ bind_data(std::move(bind_data)), returned_types(std::move(returned_types)), names(std::move(returned_names)),
18
+ extra_info() {
18
19
  }
19
20
 
20
21
  optional_ptr<TableCatalogEntry> LogicalGet::GetTable() const {
@@ -22,7 +23,7 @@ optional_ptr<TableCatalogEntry> LogicalGet::GetTable() const {
22
23
  }
23
24
 
24
25
  string LogicalGet::ParamsToString() const {
25
- string result;
26
+ string result = "";
26
27
  for (auto &kv : table_filters.filters) {
27
28
  auto &column_index = kv.first;
28
29
  auto &filter = kv.second;
@@ -31,10 +32,14 @@ string LogicalGet::ParamsToString() const {
31
32
  }
32
33
  result += "\n";
33
34
  }
35
+ if (!extra_info.file_filters.empty()) {
36
+ result += "\n[INFOSEPARATOR]\n";
37
+ result += "File Filters: " + extra_info.file_filters;
38
+ }
34
39
  if (!function.to_string) {
35
- return string();
40
+ return result;
36
41
  }
37
- return function.to_string(bind_data.get());
42
+ return result + "\n" + function.to_string(bind_data.get());
38
43
  }
39
44
 
40
45
  vector<ColumnBinding> LogicalGet::GetColumnBindings() {
@@ -0,0 +1,21 @@
1
+ #include "duckdb/planner/operator/logical_materialized_cte.hpp"
2
+
3
+ #include "duckdb/common/field_writer.hpp"
4
+
5
+ namespace duckdb {
6
+
7
+ void LogicalMaterializedCTE::Serialize(FieldWriter &writer) const {
8
+ writer.WriteField(table_index);
9
+ }
10
+
11
+ unique_ptr<LogicalOperator> LogicalMaterializedCTE::Deserialize(LogicalDeserializationState &state,
12
+ FieldReader &reader) {
13
+ auto table_index = reader.ReadRequired<idx_t>();
14
+ return unique_ptr<LogicalMaterializedCTE>(new LogicalMaterializedCTE(table_index));
15
+ }
16
+
17
+ vector<idx_t> LogicalMaterializedCTE::GetTableIndex() const {
18
+ return vector<idx_t> {table_index};
19
+ }
20
+
21
+ } // namespace duckdb
@@ -10,6 +10,7 @@
10
10
  #include "duckdb/planner/operator/list.hpp"
11
11
  #include "duckdb/planner/subquery/has_correlated_expressions.hpp"
12
12
  #include "duckdb/planner/subquery/rewrite_correlated_expressions.hpp"
13
+ #include "duckdb/planner/operator/logical_dependent_join.hpp"
13
14
 
14
15
  namespace duckdb {
15
16
 
@@ -24,20 +25,32 @@ FlattenDependentJoins::FlattenDependentJoins(Binder &binder, const vector<Correl
24
25
  }
25
26
  }
26
27
 
27
- bool FlattenDependentJoins::DetectCorrelatedExpressions(LogicalOperator *op, bool lateral) {
28
+ bool FlattenDependentJoins::DetectCorrelatedExpressions(LogicalOperator *op, bool lateral, idx_t lateral_depth) {
29
+
30
+ bool is_lateral_join = false;
31
+
28
32
  D_ASSERT(op);
29
33
  // check if this entry has correlated expressions
30
- HasCorrelatedExpressions visitor(correlated_columns, lateral);
34
+ if (op->type == LogicalOperatorType::LOGICAL_DEPENDENT_JOIN) {
35
+ is_lateral_join = true;
36
+ }
37
+ HasCorrelatedExpressions visitor(correlated_columns, lateral, lateral_depth);
31
38
  visitor.VisitOperator(*op);
32
39
  bool has_correlation = visitor.has_correlated_expressions;
40
+ int child_idx = 0;
33
41
  // now visit the children of this entry and check if they have correlated expressions
34
42
  for (auto &child : op->children) {
43
+ auto new_lateral_depth = lateral_depth;
44
+ if (is_lateral_join && child_idx == 1) {
45
+ new_lateral_depth = lateral_depth + 1;
46
+ }
35
47
  // we OR the property with its children such that has_correlation is true if either
36
48
  // (1) this node has a correlated expression or
37
49
  // (2) one of its children has a correlated expression
38
- if (DetectCorrelatedExpressions(child.get(), lateral)) {
50
+ if (DetectCorrelatedExpressions(child.get(), lateral, new_lateral_depth)) {
39
51
  has_correlation = true;
40
52
  }
53
+ child_idx++;
41
54
  }
42
55
  // set the entry in the map
43
56
  has_correlated_expressions[op] = has_correlation;
@@ -46,7 +59,7 @@ bool FlattenDependentJoins::DetectCorrelatedExpressions(LogicalOperator *op, boo
46
59
 
47
60
  unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoin(unique_ptr<LogicalOperator> plan) {
48
61
  bool propagate_null_values = true;
49
- auto result = PushDownDependentJoinInternal(std::move(plan), propagate_null_values);
62
+ auto result = PushDownDependentJoinInternal(std::move(plan), propagate_null_values, 0);
50
63
  if (!replacement_map.empty()) {
51
64
  // check if we have to replace any COUNT aggregates into "CASE WHEN X IS NULL THEN 0 ELSE COUNT END"
52
65
  RewriteCountAggregates aggr(replacement_map);
@@ -70,8 +83,10 @@ bool SubqueryDependentFilter(Expression *expr) {
70
83
  }
71
84
  return false;
72
85
  }
86
+
73
87
  unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal(unique_ptr<LogicalOperator> plan,
74
- bool &parent_propagate_null_values) {
88
+ bool &parent_propagate_null_values,
89
+ idx_t lateral_depth) {
75
90
  // first check if the logical operator has correlated expressions
76
91
  auto entry = has_correlated_expressions.find(plan.get());
77
92
  D_ASSERT(entry != has_correlated_expressions.end());
@@ -95,10 +110,11 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
95
110
  for (auto &expr : plan->expressions) {
96
111
  any_join |= SubqueryDependentFilter(expr.get());
97
112
  }
98
- plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
113
+ plan->children[0] =
114
+ PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
99
115
 
100
116
  // then we replace any correlated expressions with the corresponding entry in the correlated_map
101
- RewriteCorrelatedExpressions rewriter(base_binding, correlated_map);
117
+ RewriteCorrelatedExpressions rewriter(base_binding, correlated_map, lateral_depth);
102
118
  rewriter.VisitOperator(*plan);
103
119
  return plan;
104
120
  }
@@ -108,10 +124,11 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
108
124
  for (auto &expr : plan->expressions) {
109
125
  parent_propagate_null_values &= expr->PropagatesNullValues();
110
126
  }
111
- plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
127
+ plan->children[0] =
128
+ PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
112
129
 
113
130
  // then we replace any correlated expressions with the corresponding entry in the correlated_map
114
- RewriteCorrelatedExpressions rewriter(base_binding, correlated_map);
131
+ RewriteCorrelatedExpressions rewriter(base_binding, correlated_map, lateral_depth);
115
132
  rewriter.VisitOperator(*plan);
116
133
  // now we add all the columns of the delim_scan to the projection list
117
134
  auto &proj = plan->Cast<LogicalProjection>();
@@ -134,9 +151,10 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
134
151
  for (auto &expr : plan->expressions) {
135
152
  parent_propagate_null_values &= expr->PropagatesNullValues();
136
153
  }
137
- plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
154
+ plan->children[0] =
155
+ PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
138
156
  // then we replace any correlated expressions with the corresponding entry in the correlated_map
139
- RewriteCorrelatedExpressions rewriter(base_binding, correlated_map);
157
+ RewriteCorrelatedExpressions rewriter(base_binding, correlated_map, lateral_depth);
140
158
  rewriter.VisitOperator(*plan);
141
159
  // now we add all the columns of the delim_scan to the grouping operators AND the projection list
142
160
  idx_t delim_table_index;
@@ -231,22 +249,24 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
231
249
  bool right_has_correlation = has_correlated_expressions.find(plan->children[1].get())->second;
232
250
  if (!right_has_correlation) {
233
251
  // only left has correlation: push into left
234
- plan->children[0] =
235
- PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
252
+ plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]),
253
+ parent_propagate_null_values, lateral_depth);
236
254
  return plan;
237
255
  }
238
256
  if (!left_has_correlation) {
239
257
  // only right has correlation: push into right
240
- plan->children[1] =
241
- PushDownDependentJoinInternal(std::move(plan->children[1]), parent_propagate_null_values);
258
+ plan->children[1] = PushDownDependentJoinInternal(std::move(plan->children[1]),
259
+ parent_propagate_null_values, lateral_depth);
242
260
  return plan;
243
261
  }
244
262
  // both sides have correlation
245
263
  // turn into an inner join
246
264
  auto join = make_uniq<LogicalComparisonJoin>(JoinType::INNER);
247
- plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
265
+ plan->children[0] =
266
+ PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
248
267
  auto left_binding = this->base_binding;
249
- plan->children[1] = PushDownDependentJoinInternal(std::move(plan->children[1]), parent_propagate_null_values);
268
+ plan->children[1] =
269
+ PushDownDependentJoinInternal(std::move(plan->children[1]), parent_propagate_null_values, lateral_depth);
250
270
  // add the correlated columns to the join conditions
251
271
  for (idx_t i = 0; i < correlated_columns.size(); i++) {
252
272
  JoinCondition cond;
@@ -261,6 +281,26 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
261
281
  join->children.push_back(std::move(plan->children[1]));
262
282
  return std::move(join);
263
283
  }
284
+ case LogicalOperatorType::LOGICAL_DEPENDENT_JOIN: {
285
+ auto &dependent_join = plan->Cast<LogicalJoin>();
286
+ if (!((dependent_join.join_type == JoinType::INNER) || (dependent_join.join_type == JoinType::LEFT))) {
287
+ throw Exception("Dependent join can only be INNER or LEFT type");
288
+ }
289
+ D_ASSERT(plan->children.size() == 2);
290
+ // Push all the bindings down to the left side so the right side knows where to refer DELIM_GET from
291
+ plan->children[0] =
292
+ PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
293
+
294
+ // Normal rewriter like in other joins
295
+ RewriteCorrelatedExpressions rewriter(this->base_binding, correlated_map, lateral_depth);
296
+ rewriter.VisitOperator(*plan);
297
+
298
+ // Recursive rewriter to visit right side of lateral join and update bindings from left
299
+ RewriteCorrelatedExpressions recursive_rewriter(this->base_binding, correlated_map, lateral_depth + 1, true);
300
+ recursive_rewriter.VisitOperator(*plan->children[1]);
301
+
302
+ return plan;
303
+ }
264
304
  case LogicalOperatorType::LOGICAL_ANY_JOIN:
265
305
  case LogicalOperatorType::LOGICAL_ASOF_JOIN:
266
306
  case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: {
@@ -274,30 +314,33 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
274
314
  // inner join
275
315
  if (!right_has_correlation) {
276
316
  // only left has correlation: push into left
277
- plan->children[0] =
278
- PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
317
+ plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]),
318
+ parent_propagate_null_values, lateral_depth);
319
+ // Remove the correlated columns coming from outside for current join node
279
320
  return plan;
280
321
  }
281
322
  if (!left_has_correlation) {
282
323
  // only right has correlation: push into right
283
- plan->children[1] =
284
- PushDownDependentJoinInternal(std::move(plan->children[1]), parent_propagate_null_values);
324
+ plan->children[1] = PushDownDependentJoinInternal(std::move(plan->children[1]),
325
+ parent_propagate_null_values, lateral_depth);
326
+ // Remove the correlated columns coming from outside for current join node
285
327
  return plan;
286
328
  }
287
329
  } else if (join.join_type == JoinType::LEFT) {
288
330
  // left outer join
289
331
  if (!right_has_correlation) {
290
332
  // only left has correlation: push into left
291
- plan->children[0] =
292
- PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
333
+ plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]),
334
+ parent_propagate_null_values, lateral_depth);
335
+ // Remove the correlated columns coming from outside for current join node
293
336
  return plan;
294
337
  }
295
338
  } else if (join.join_type == JoinType::RIGHT) {
296
339
  // left outer join
297
340
  if (!left_has_correlation) {
298
341
  // only right has correlation: push into right
299
- plan->children[1] =
300
- PushDownDependentJoinInternal(std::move(plan->children[1]), parent_propagate_null_values);
342
+ plan->children[1] = PushDownDependentJoinInternal(std::move(plan->children[1]),
343
+ parent_propagate_null_values, lateral_depth);
301
344
  return plan;
302
345
  }
303
346
  } else if (join.join_type == JoinType::MARK) {
@@ -305,10 +348,10 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
305
348
  throw Exception("MARK join with correlation in RHS not supported");
306
349
  }
307
350
  // push the child into the LHS
308
- plan->children[0] =
309
- PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
351
+ plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]),
352
+ parent_propagate_null_values, lateral_depth);
310
353
  // rewrite expressions in the join conditions
311
- RewriteCorrelatedExpressions rewriter(base_binding, correlated_map);
354
+ RewriteCorrelatedExpressions rewriter(base_binding, correlated_map, lateral_depth);
312
355
  rewriter.VisitOperator(*plan);
313
356
  return plan;
314
357
  } else {
@@ -316,9 +359,11 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
316
359
  }
317
360
  // both sides have correlation
318
361
  // push into both sides
319
- plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
362
+ plan->children[0] =
363
+ PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
320
364
  auto left_binding = this->base_binding;
321
- plan->children[1] = PushDownDependentJoinInternal(std::move(plan->children[1]), parent_propagate_null_values);
365
+ plan->children[1] =
366
+ PushDownDependentJoinInternal(std::move(plan->children[1]), parent_propagate_null_values, lateral_depth);
322
367
  auto right_binding = this->base_binding;
323
368
  // NOTE: for OUTER JOINS it matters what the BASE BINDING is after the join
324
369
  // for the LEFT OUTER JOIN, we want the LEFT side to be the base binding after we push
@@ -354,7 +399,7 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
354
399
  }
355
400
  }
356
401
  // then we replace any correlated expressions with the corresponding entry in the correlated_map
357
- RewriteCorrelatedExpressions rewriter(right_binding, correlated_map);
402
+ RewriteCorrelatedExpressions rewriter(right_binding, correlated_map, lateral_depth);
358
403
  rewriter.VisitOperator(*plan);
359
404
  return plan;
360
405
  }
@@ -371,9 +416,11 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
371
416
  // this is done for an optimization to avoid having to compute the total order
372
417
  if (plan->children[0]->type == LogicalOperatorType::LOGICAL_ORDER_BY) {
373
418
  order_by = unique_ptr_cast<LogicalOperator, LogicalOrder>(std::move(plan->children[0]));
374
- child = PushDownDependentJoinInternal(std::move(order_by->children[0]), parent_propagate_null_values);
419
+ child = PushDownDependentJoinInternal(std::move(order_by->children[0]), parent_propagate_null_values,
420
+ lateral_depth);
375
421
  } else {
376
- child = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
422
+ child = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values,
423
+ lateral_depth);
377
424
  }
378
425
  auto child_column_count = child->GetColumnBindings().size();
379
426
  // we push a row_number() OVER (PARTITION BY [correlated columns])
@@ -437,7 +484,8 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
437
484
  case LogicalOperatorType::LOGICAL_WINDOW: {
438
485
  auto &window = plan->Cast<LogicalWindow>();
439
486
  // push into children
440
- plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
487
+ plan->children[0] =
488
+ PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
441
489
  // add the correlated columns to the PARTITION BY clauses in the Window
442
490
  for (auto &expr : window.expressions) {
443
491
  D_ASSERT(expr->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);
@@ -488,9 +536,10 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
488
536
  case LogicalOperatorType::LOGICAL_EXPRESSION_GET: {
489
537
  // expression get
490
538
  // first we flatten the dependent join in the child
491
- plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
539
+ plan->children[0] =
540
+ PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
492
541
  // then we replace any correlated expressions with the corresponding entry in the correlated_map
493
- RewriteCorrelatedExpressions rewriter(base_binding, correlated_map);
542
+ RewriteCorrelatedExpressions rewriter(base_binding, correlated_map, lateral_depth);
494
543
  rewriter.VisitOperator(*plan);
495
544
  // now we add all the correlated columns to each of the expressions of the expression scan
496
545
  auto &expr_get = plan->Cast<LogicalExpressionGet>();
@@ -519,7 +568,7 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
519
568
  throw InternalException("Flatten dependent joins - logical get encountered without children");
520
569
  }
521
570
  plan->children[0] = PushDownDependentJoin(std::move(plan->children[0]));
522
- for (idx_t i = 0; i < (perform_delim ? correlated_columns.size() : 1); i++) {
571
+ for (idx_t i = 0; i < correlated_columns.size(); i++) {
523
572
  get.projected_input.push_back(this->delim_offset + i);
524
573
  }
525
574
  this->delim_offset = get.returned_types.size();
@@ -527,7 +576,10 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
527
576
  return plan;
528
577
  }
529
578
  case LogicalOperatorType::LOGICAL_RECURSIVE_CTE: {
530
- throw BinderException("Recursive CTEs not supported in correlated subquery");
579
+ throw BinderException("Recursive CTEs not (yet) supported in correlated subquery");
580
+ }
581
+ case LogicalOperatorType::LOGICAL_MATERIALIZED_CTE: {
582
+ throw BinderException("Materialized CTEs not (yet) supported in correlated subquery");
531
583
  }
532
584
  case LogicalOperatorType::LOGICAL_DELIM_JOIN: {
533
585
  throw BinderException("Nested lateral joins or lateral joins in correlated subqueries are not (yet) supported");
@@ -7,8 +7,10 @@
7
7
 
8
8
  namespace duckdb {
9
9
 
10
- HasCorrelatedExpressions::HasCorrelatedExpressions(const vector<CorrelatedColumnInfo> &correlated, bool lateral)
11
- : has_correlated_expressions(false), lateral(lateral), correlated_columns(correlated) {
10
+ HasCorrelatedExpressions::HasCorrelatedExpressions(const vector<CorrelatedColumnInfo> &correlated, bool lateral,
11
+ idx_t lateral_depth)
12
+ : has_correlated_expressions(false), lateral(lateral), correlated_columns(correlated),
13
+ lateral_depth(lateral_depth) {
12
14
  }
13
15
 
14
16
  void HasCorrelatedExpressions::VisitOperator(LogicalOperator &op) {
@@ -17,18 +19,31 @@ void HasCorrelatedExpressions::VisitOperator(LogicalOperator &op) {
17
19
 
18
20
  unique_ptr<Expression> HasCorrelatedExpressions::VisitReplace(BoundColumnRefExpression &expr,
19
21
  unique_ptr<Expression> *expr_ptr) {
20
- if (expr.depth == 0) {
22
+ // Indicates local correlations (all correlations within a child) for the root
23
+ if (expr.depth <= lateral_depth) {
21
24
  return nullptr;
22
25
  }
23
- if (expr.depth > 1) {
26
+
27
+ // Should never happen
28
+ if (expr.depth > 1 + lateral_depth) {
24
29
  if (lateral) {
25
- throw BinderException("Nested lateral joins are not (yet) supported");
30
+ throw BinderException("Invalid lateral depth encountered for an expression");
26
31
  }
27
32
  throw InternalException("Expression with depth > 1 detected in non-lateral join");
28
33
  }
34
+ // Note: This is added, since we only want to set has_correlated_expressions to true when the
35
+ // BoundSubqueryExpression has the same bindings as one of the correlated_columns from the left hand side
36
+ // (correlated_columns is the correlated_columns from left hand side)
37
+ bool found_match = false;
38
+ for (idx_t i = 0; i < correlated_columns.size(); i++) {
39
+ if (correlated_columns[i].binding == expr.binding) {
40
+ found_match = true;
41
+ break;
42
+ }
43
+ }
29
44
  // correlated column reference
30
- D_ASSERT(expr.depth == 1);
31
- has_correlated_expressions = true;
45
+ D_ASSERT(expr.depth == lateral_depth + 1);
46
+ has_correlated_expressions = found_match;
32
47
  return nullptr;
33
48
  }
34
49
 
@@ -5,34 +5,67 @@
5
5
  #include "duckdb/planner/expression/bound_constant_expression.hpp"
6
6
  #include "duckdb/planner/expression/bound_operator_expression.hpp"
7
7
  #include "duckdb/planner/expression/bound_subquery_expression.hpp"
8
+ #include "duckdb/planner/query_node/bound_select_node.hpp"
8
9
  #include "duckdb/planner/expression_iterator.hpp"
10
+ #include "duckdb/planner/tableref/bound_joinref.hpp"
11
+ #include "duckdb/planner/operator/logical_dependent_join.hpp"
9
12
 
10
13
  namespace duckdb {
11
14
 
12
15
  RewriteCorrelatedExpressions::RewriteCorrelatedExpressions(ColumnBinding base_binding,
13
- column_binding_map_t<idx_t> &correlated_map)
14
- : base_binding(base_binding), correlated_map(correlated_map) {
16
+ column_binding_map_t<idx_t> &correlated_map,
17
+ idx_t lateral_depth, bool recursive_rewrite)
18
+ : base_binding(base_binding), correlated_map(correlated_map), lateral_depth(lateral_depth),
19
+ recursive_rewrite(recursive_rewrite) {
15
20
  }
16
21
 
17
22
  void RewriteCorrelatedExpressions::VisitOperator(LogicalOperator &op) {
23
+ if (recursive_rewrite) {
24
+ // Update column bindings from left child of lateral to right child
25
+ if (op.type == LogicalOperatorType::LOGICAL_DEPENDENT_JOIN) {
26
+ D_ASSERT(op.children.size() == 2);
27
+ VisitOperator(*op.children[0]);
28
+ lateral_depth++;
29
+ VisitOperator(*op.children[1]);
30
+ lateral_depth--;
31
+ } else {
32
+ VisitOperatorChildren(op);
33
+ }
34
+ }
35
+ // update the bindings in the correlated columns of the dependendent join
36
+ if (op.type == LogicalOperatorType::LOGICAL_DEPENDENT_JOIN) {
37
+ auto &plan = op.Cast<LogicalDependentJoin>();
38
+ for (auto &corr : plan.correlated_columns) {
39
+ auto entry = correlated_map.find(corr.binding);
40
+ if (entry != correlated_map.end()) {
41
+ corr.binding = ColumnBinding(base_binding.table_index, base_binding.column_index + entry->second);
42
+ }
43
+ }
44
+ }
18
45
  VisitOperatorExpressions(op);
19
46
  }
20
47
 
21
48
  unique_ptr<Expression> RewriteCorrelatedExpressions::VisitReplace(BoundColumnRefExpression &expr,
22
49
  unique_ptr<Expression> *expr_ptr) {
23
- if (expr.depth == 0) {
50
+ if (expr.depth <= lateral_depth) {
51
+ // Indicates local correlations not relevant for the current the rewrite
24
52
  return nullptr;
25
53
  }
26
54
  // correlated column reference
27
55
  // replace with the entry referring to the duplicate eliminated scan
28
- // if this assertion occurs it generally means the correlated expressions were not propagated correctly
29
- // through different binders
30
- D_ASSERT(expr.depth == 1);
56
+ // if this assertion occurs it generally means the bindings are inappropriate set in the binder or
57
+ // we either missed to account for lateral binder or over-counted for the lateral binder
58
+ D_ASSERT(expr.depth == 1 + lateral_depth);
31
59
  auto entry = correlated_map.find(expr.binding);
32
60
  D_ASSERT(entry != correlated_map.end());
33
61
 
34
62
  expr.binding = ColumnBinding(base_binding.table_index, base_binding.column_index + entry->second);
35
- expr.depth = 0;
63
+ if (recursive_rewrite) {
64
+ D_ASSERT(expr.depth > 1);
65
+ expr.depth--;
66
+ } else {
67
+ expr.depth = 0;
68
+ }
36
69
  return nullptr;
37
70
  }
38
71
 
@@ -53,6 +86,21 @@ RewriteCorrelatedExpressions::RewriteCorrelatedRecursive::RewriteCorrelatedRecur
53
86
  : parent(parent), base_binding(base_binding), correlated_map(correlated_map) {
54
87
  }
55
88
 
89
+ void RewriteCorrelatedExpressions::RewriteCorrelatedRecursive::RewriteJoinRefRecursive(BoundTableRef &ref) {
90
+ // recursively rewrite bindings in the correlated columns for the table ref and all the children
91
+ if (ref.type == TableReferenceType::JOIN) {
92
+ auto &bound_join = ref.Cast<BoundJoinRef>();
93
+ for (auto &corr : bound_join.correlated_columns) {
94
+ auto entry = correlated_map.find(corr.binding);
95
+ if (entry != correlated_map.end()) {
96
+ corr.binding = ColumnBinding(base_binding.table_index, base_binding.column_index + entry->second);
97
+ }
98
+ }
99
+ RewriteJoinRefRecursive(*bound_join.left);
100
+ RewriteJoinRefRecursive(*bound_join.right);
101
+ }
102
+ }
103
+
56
104
  void RewriteCorrelatedExpressions::RewriteCorrelatedRecursive::RewriteCorrelatedSubquery(
57
105
  BoundSubqueryExpression &expr) {
58
106
  // rewrite the binding in the correlated list of the subquery)
@@ -62,6 +110,16 @@ void RewriteCorrelatedExpressions::RewriteCorrelatedRecursive::RewriteCorrelated
62
110
  corr.binding = ColumnBinding(base_binding.table_index, base_binding.column_index + entry->second);
63
111
  }
64
112
  }
113
+ // TODO: Cleanup and find a better way to do this
114
+ auto &node = *expr.subquery;
115
+ if (node.type == QueryNodeType::SELECT_NODE) {
116
+ // Found an unplanned select node, need to update column bindings correlated columns in the from tables
117
+ auto &bound_select = node.Cast<BoundSelectNode>();
118
+ if (bound_select.from_table) {
119
+ BoundTableRef &table_ref = *bound_select.from_table;
120
+ RewriteJoinRefRecursive(table_ref);
121
+ }
122
+ }
65
123
  // now rewrite any correlated BoundColumnRef expressions inside the subquery
66
124
  ExpressionIterator::EnumerateQueryNodeChildren(*expr.subquery,
67
125
  [&](Expression &child) { RewriteCorrelatedExpressions(child); });
@@ -109,7 +109,6 @@ data_ptr_t ArenaAllocator::ReallocateAligned(data_ptr_t pointer, idx_t old_size,
109
109
  }
110
110
 
111
111
  void ArenaAllocator::Reset() {
112
-
113
112
  if (head) {
114
113
  // destroy all chunks except the current one
115
114
  if (head->next) {
@@ -148,7 +147,7 @@ ArenaChunk *ArenaAllocator::GetTail() {
148
147
  return tail;
149
148
  }
150
149
 
151
- bool ArenaAllocator::IsEmpty() {
150
+ bool ArenaAllocator::IsEmpty() const {
152
151
  return head == nullptr;
153
152
  }
154
153
 
@@ -79,4 +79,7 @@ void BlockManager::UnregisterBlock(block_id_t block_id, bool can_destroy) {
79
79
  }
80
80
  }
81
81
 
82
+ void BlockManager::Truncate() {
83
+ }
84
+
82
85
  } // namespace duckdb
@@ -111,6 +111,9 @@ void SingleFileCheckpointWriter::CreateCheckpoint() {
111
111
  // truncate the WAL
112
112
  wal->Truncate(0);
113
113
 
114
+ // truncate the file
115
+ block_manager.Truncate();
116
+
114
117
  // mark all blocks written as part of the metadata as modified
115
118
  metadata_writer->MarkWrittenBlocks();
116
119
  table_metadata_writer->MarkWrittenBlocks();
@@ -272,7 +272,6 @@ struct RLEScanState : public SegmentScanState {
272
272
  }
273
273
 
274
274
  BufferHandle handle;
275
- uint32_t rle_offset;
276
275
  idx_t entry_pos;
277
276
  idx_t position_in_entry;
278
277
  uint32_t rle_count_offset;
@@ -56,7 +56,7 @@ DataTable::DataTable(AttachedDatabase &db, shared_ptr<TableIOManager> table_io_m
56
56
  row_groups->Verify();
57
57
  }
58
58
 
59
- DataTable::DataTable(ClientContext &context, DataTable &parent, ColumnDefinition &new_column, Expression *default_value)
59
+ DataTable::DataTable(ClientContext &context, DataTable &parent, ColumnDefinition &new_column, Expression &default_value)
60
60
  : info(parent.info), db(parent.db), is_root(true) {
61
61
  // add the column definitions from this DataTable
62
62
  for (auto &column_def : parent.column_definitions) {
@@ -61,11 +61,11 @@ LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &paren
61
61
  }
62
62
 
63
63
  LocalTableStorage::LocalTableStorage(ClientContext &context, DataTable &new_dt, LocalTableStorage &parent,
64
- ColumnDefinition &new_column, optional_ptr<Expression> default_value)
64
+ ColumnDefinition &new_column, Expression &default_value)
65
65
  : table_ref(new_dt), allocator(Allocator::Get(new_dt.db)), deleted_rows(parent.deleted_rows),
66
66
  optimistic_writer(new_dt, parent.optimistic_writer), optimistic_writers(std::move(parent.optimistic_writers)),
67
67
  merged_storage(parent.merged_storage) {
68
- row_groups = parent.row_groups->AddColumn(context, new_column, default_value.get());
68
+ row_groups = parent.row_groups->AddColumn(context, new_column, default_value);
69
69
  parent.row_groups.reset();
70
70
  indexes.Move(parent.indexes);
71
71
  }
@@ -508,7 +508,7 @@ void LocalStorage::MoveStorage(DataTable &old_dt, DataTable &new_dt) {
508
508
  }
509
509
 
510
510
  void LocalStorage::AddColumn(DataTable &old_dt, DataTable &new_dt, ColumnDefinition &new_column,
511
- optional_ptr<Expression> default_value) {
511
+ Expression &default_value) {
512
512
  // check if there are any pending appends for the old version of the table
513
513
  auto storage = table_manager.MoveEntry(old_dt);
514
514
  if (!storage) {