duckdb 0.8.2-dev150.0 → 0.8.2-dev1549.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (489) hide show
  1. package/binding.gyp +15 -12
  2. package/binding.gyp.in +1 -1
  3. package/configure.py +1 -1
  4. package/duckdb_extension_config.cmake +10 -0
  5. package/package.json +1 -1
  6. package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
  7. package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
  8. package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
  9. package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
  10. package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
  11. package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
  12. package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
  13. package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
  14. package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
  15. package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
  16. package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
  17. package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
  18. package/src/duckdb/extension/json/include/json_common.hpp +47 -231
  19. package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
  20. package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
  21. package/src/duckdb/extension/json/json_common.cpp +272 -40
  22. package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
  23. package/src/duckdb/extension/json/json_functions/json_transform.cpp +17 -37
  24. package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
  25. package/src/duckdb/extension/json/json_functions.cpp +24 -24
  26. package/src/duckdb/extension/json/json_scan.cpp +3 -6
  27. package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
  28. package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
  29. package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
  30. package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
  31. package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
  32. package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
  33. package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
  34. package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
  35. package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
  36. package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
  37. package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
  38. package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
  39. package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
  40. package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
  41. package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
  42. package/src/duckdb/extension/parquet/parquet_extension.cpp +192 -20
  43. package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
  44. package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
  45. package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
  46. package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
  47. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
  48. package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
  49. package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
  50. package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
  51. package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
  52. package/src/duckdb/src/common/allocator.cpp +14 -2
  53. package/src/duckdb/src/common/arrow/arrow_appender.cpp +5 -11
  54. package/src/duckdb/src/common/assert.cpp +3 -0
  55. package/src/duckdb/src/common/enum_util.cpp +4619 -4446
  56. package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
  57. package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
  58. package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
  59. package/src/duckdb/src/common/exception.cpp +2 -2
  60. package/src/duckdb/src/common/extra_type_info.cpp +506 -0
  61. package/src/duckdb/src/common/file_system.cpp +19 -0
  62. package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
  63. package/src/duckdb/src/common/local_file_system.cpp +14 -14
  64. package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
  65. package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
  66. package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
  67. package/src/duckdb/src/common/re2_regex.cpp +1 -1
  68. package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
  69. package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
  70. package/src/duckdb/src/common/sort/partition_state.cpp +44 -11
  71. package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
  72. package/src/duckdb/src/common/types/bit.cpp +51 -0
  73. package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
  74. package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
  75. package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
  76. package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
  77. package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
  78. package/src/duckdb/src/common/types/date.cpp +9 -0
  79. package/src/duckdb/src/common/types/list_segment.cpp +24 -74
  80. package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
  81. package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
  82. package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
  83. package/src/duckdb/src/common/types/uuid.cpp +2 -2
  84. package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
  85. package/src/duckdb/src/common/types.cpp +8 -655
  86. package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
  87. package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
  88. package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
  89. package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
  90. package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
  91. package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
  92. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
  93. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
  94. package/src/duckdb/src/core_functions/function_list.cpp +4 -2
  95. package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
  96. package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
  97. package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
  98. package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
  99. package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
  100. package/src/duckdb/src/execution/expression_executor.cpp +1 -1
  101. package/src/duckdb/src/execution/index/art/art.cpp +149 -139
  102. package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
  103. package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
  104. package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
  105. package/src/duckdb/src/execution/index/art/node.cpp +113 -120
  106. package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
  107. package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
  108. package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
  109. package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
  110. package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
  111. package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
  112. package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
  113. package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
  114. package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
  115. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
  116. package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
  117. package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
  118. package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +414 -283
  119. package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
  120. package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
  121. package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +28 -12
  122. package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
  123. package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +23 -4
  124. package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +41 -5
  125. package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
  126. package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
  127. package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
  128. package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
  129. package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
  130. package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
  131. package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
  132. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
  133. package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
  134. package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
  135. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
  136. package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
  137. package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
  138. package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
  139. package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
  140. package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
  141. package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +13 -22
  142. package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +17 -13
  143. package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
  144. package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
  145. package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
  146. package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
  147. package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
  148. package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
  149. package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
  150. package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
  151. package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
  152. package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
  153. package/src/duckdb/src/function/function.cpp +3 -1
  154. package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
  155. package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
  156. package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
  157. package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
  158. package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
  159. package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
  160. package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
  161. package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
  162. package/src/duckdb/src/function/table/read_csv.cpp +100 -17
  163. package/src/duckdb/src/function/table/table_scan.cpp +9 -0
  164. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  165. package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
  166. package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
  167. package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
  168. package/src/duckdb/src/include/duckdb/common/dl.hpp +3 -1
  169. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +616 -584
  170. package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
  171. package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
  172. package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
  173. package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
  174. package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
  175. package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
  176. package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +219 -0
  177. package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
  178. package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
  179. package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
  180. package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
  181. package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
  182. package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
  183. package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
  184. package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
  185. package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
  186. package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
  187. package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +13 -0
  188. package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
  189. package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
  190. package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
  191. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
  192. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
  193. package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
  194. package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
  195. package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
  196. package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
  197. package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +0 -1
  198. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
  199. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
  200. package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
  201. package/src/duckdb/src/include/duckdb/common/types.hpp +1 -15
  202. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
  203. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
  204. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
  205. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
  206. package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
  207. package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
  208. package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
  209. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
  210. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
  211. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
  212. package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
  213. package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
  214. package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
  215. package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
  216. package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
  217. package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
  218. package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
  219. package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
  220. package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
  221. package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
  222. package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
  223. package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
  224. package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
  225. package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
  226. package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
  227. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
  228. package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
  229. package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
  230. package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
  231. package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
  232. package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
  233. package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
  234. package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
  235. package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
  236. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +2 -10
  237. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +1 -1
  238. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +1 -1
  239. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp +12 -1
  240. package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
  241. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
  242. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
  243. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
  244. package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
  245. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
  246. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
  247. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
  248. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
  249. package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
  250. package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
  251. package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
  252. package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
  253. package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
  254. package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
  255. package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
  256. package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
  257. package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
  258. package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
  259. package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
  260. package/src/duckdb/src/include/duckdb/main/client_config.hpp +5 -0
  261. package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
  262. package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
  263. package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
  264. package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
  265. package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
  266. package/src/duckdb/src/include/duckdb/main/settings.hpp +39 -1
  267. package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
  268. package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
  269. package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
  270. package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
  271. package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
  272. package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
  273. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
  274. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
  275. package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
  276. package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
  277. package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
  278. package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
  279. package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
  280. package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
  281. package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
  282. package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
  283. package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
  284. package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
  285. package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
  286. package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
  287. package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
  288. package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
  289. package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
  290. package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
  291. package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
  292. package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
  293. package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
  294. package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
  295. package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
  296. package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
  297. package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
  298. package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
  299. package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
  300. package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
  301. package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
  302. package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
  303. package/src/duckdb/src/include/duckdb/planner/binder.hpp +8 -5
  304. package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
  305. package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
  306. package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
  307. package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
  308. package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
  309. package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
  310. package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
  311. package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
  312. package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
  313. package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
  314. package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
  315. package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
  316. package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
  317. package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
  318. package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
  319. package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
  320. package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
  321. package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
  322. package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
  323. package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
  324. package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
  325. package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
  326. package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
  327. package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
  328. package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
  329. package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
  330. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
  331. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
  332. package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
  333. package/src/duckdb/src/include/duckdb.h +28 -0
  334. package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
  335. package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
  336. package/src/duckdb/src/main/config.cpp +4 -0
  337. package/src/duckdb/src/main/database.cpp +1 -1
  338. package/src/duckdb/src/main/extension/extension_helper.cpp +96 -89
  339. package/src/duckdb/src/main/extension/extension_install.cpp +6 -0
  340. package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
  341. package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
  342. package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
  343. package/src/duckdb/src/main/relation.cpp +6 -5
  344. package/src/duckdb/src/main/settings/settings.cpp +79 -18
  345. package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
  346. package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
  347. package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
  348. package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
  349. package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
  350. package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
  351. package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
  352. package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
  353. package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
  354. package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
  355. package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
  356. package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
  357. package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
  358. package/src/duckdb/src/optimizer/optimizer.cpp +51 -14
  359. package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
  360. package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
  361. package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
  362. package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
  363. package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
  364. package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
  365. package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
  366. package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
  367. package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
  368. package/src/duckdb/src/parallel/executor.cpp +15 -0
  369. package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
  370. package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
  371. package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
  372. package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
  373. package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
  374. package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
  375. package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
  376. package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
  377. package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
  378. package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
  379. package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
  380. package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
  381. package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
  382. package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
  383. package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
  384. package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
  385. package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
  386. package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
  387. package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
  388. package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
  389. package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
  390. package/src/duckdb/src/parser/parser.cpp +8 -2
  391. package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
  392. package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
  393. package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
  394. package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
  395. package/src/duckdb/src/parser/query_node.cpp +15 -37
  396. package/src/duckdb/src/parser/result_modifier.cpp +0 -74
  397. package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
  398. package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
  399. package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
  400. package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
  401. package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
  402. package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
  403. package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
  404. package/src/duckdb/src/parser/tableref.cpp +0 -44
  405. package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
  406. package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
  407. package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
  408. package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
  409. package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
  410. package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
  411. package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
  412. package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
  413. package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
  414. package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
  415. package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
  416. package/src/duckdb/src/parser/transformer.cpp +15 -0
  417. package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
  418. package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
  419. package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
  420. package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
  421. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
  422. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
  423. package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
  424. package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
  425. package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
  426. package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
  427. package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +61 -26
  428. package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
  429. package/src/duckdb/src/planner/binder.cpp +5 -0
  430. package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
  431. package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
  432. package/src/duckdb/src/planner/expression_binder.cpp +3 -0
  433. package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
  434. package/src/duckdb/src/planner/logical_operator.cpp +5 -0
  435. package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
  436. package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
  437. package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
  438. package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
  439. package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
  440. package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
  441. package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
  442. package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
  443. package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
  444. package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
  445. package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
  446. package/src/duckdb/src/storage/compression/rle.cpp +0 -1
  447. package/src/duckdb/src/storage/data_table.cpp +1 -1
  448. package/src/duckdb/src/storage/local_storage.cpp +3 -3
  449. package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
  450. package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
  451. package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
  452. package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
  453. package/src/duckdb/src/storage/serialization/serialize_types.cpp +127 -0
  454. package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
  455. package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
  456. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  457. package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
  458. package/src/duckdb/src/storage/table/row_group.cpp +25 -9
  459. package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
  460. package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
  461. package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
  462. package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
  463. package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
  464. package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
  465. package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
  466. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
  467. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
  468. package/src/duckdb/ub_src_common.cpp +2 -0
  469. package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
  470. package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
  471. package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
  472. package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
  473. package/src/duckdb/ub_src_function_scalar.cpp +2 -0
  474. package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
  475. package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
  476. package/src/duckdb/ub_src_optimizer.cpp +6 -0
  477. package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
  478. package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
  479. package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
  480. package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
  481. package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
  482. package/src/duckdb/ub_src_planner_operator.cpp +4 -0
  483. package/src/duckdb/ub_src_storage_serialization.cpp +10 -0
  484. package/src/statement.cpp +10 -3
  485. package/test/test_all_types.test.ts +233 -0
  486. package/tsconfig.json +1 -0
  487. package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
  488. package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
  489. package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -18,6 +18,7 @@
18
18
  #include "duckdb/planner/tableref/bound_pivotref.hpp"
19
19
  #include "duckdb/planner/expression/bound_aggregate_expression.hpp"
20
20
  #include "duckdb/main/client_config.hpp"
21
+ #include "duckdb/catalog/catalog_entry/aggregate_function_catalog_entry.hpp"
21
22
 
22
23
  namespace duckdb {
23
24
 
@@ -66,18 +67,10 @@ static void ExtractPivotExpressions(ParsedExpression &expr, case_insensitive_set
66
67
  expr, [&](ParsedExpression &child) { ExtractPivotExpressions(child, handled_columns); });
67
68
  }
68
69
 
69
- struct PivotBindState {
70
- vector<string> internal_group_names;
71
- vector<string> group_names;
72
- vector<string> aggregate_names;
73
- vector<string> internal_aggregate_names;
74
- };
75
-
76
- static unique_ptr<SelectNode> PivotInitialAggregate(PivotBindState &bind_state, PivotRef &ref,
77
- vector<unique_ptr<ParsedExpression>> all_columns,
78
- const case_insensitive_set_t &handled_columns) {
79
- auto subquery_stage1 = make_uniq<SelectNode>();
80
- subquery_stage1->from_table = std::move(ref.source);
70
+ static unique_ptr<SelectNode> ConstructInitialGrouping(PivotRef &ref, vector<unique_ptr<ParsedExpression>> all_columns,
71
+ const case_insensitive_set_t &handled_columns) {
72
+ auto subquery = make_uniq<SelectNode>();
73
+ subquery->from_table = std::move(ref.source);
81
74
  if (ref.groups.empty()) {
82
75
  // if rows are not specified any columns that are not pivoted/aggregated on are added to the GROUP BY clause
83
76
  for (auto &entry : all_columns) {
@@ -87,19 +80,74 @@ static unique_ptr<SelectNode> PivotInitialAggregate(PivotBindState &bind_state,
87
80
  auto &columnref = entry->Cast<ColumnRefExpression>();
88
81
  if (handled_columns.find(columnref.GetColumnName()) == handled_columns.end()) {
89
82
  // not handled - add to grouping set
90
- subquery_stage1->groups.group_expressions.push_back(
91
- make_uniq<ConstantExpression>(Value::INTEGER(subquery_stage1->select_list.size() + 1)));
92
- subquery_stage1->select_list.push_back(make_uniq<ColumnRefExpression>(columnref.GetColumnName()));
83
+ subquery->groups.group_expressions.push_back(
84
+ make_uniq<ConstantExpression>(Value::INTEGER(subquery->select_list.size() + 1)));
85
+ subquery->select_list.push_back(make_uniq<ColumnRefExpression>(columnref.GetColumnName()));
93
86
  }
94
87
  }
95
88
  } else {
96
89
  // if rows are specified only the columns mentioned in rows are added as groups
97
90
  for (auto &row : ref.groups) {
98
- subquery_stage1->groups.group_expressions.push_back(
99
- make_uniq<ConstantExpression>(Value::INTEGER(subquery_stage1->select_list.size() + 1)));
100
- subquery_stage1->select_list.push_back(make_uniq<ColumnRefExpression>(row));
91
+ subquery->groups.group_expressions.push_back(
92
+ make_uniq<ConstantExpression>(Value::INTEGER(subquery->select_list.size() + 1)));
93
+ subquery->select_list.push_back(make_uniq<ColumnRefExpression>(row));
101
94
  }
102
95
  }
96
+ return subquery;
97
+ }
98
+
99
+ static unique_ptr<SelectNode> PivotFilteredAggregate(PivotRef &ref, vector<unique_ptr<ParsedExpression>> all_columns,
100
+ const case_insensitive_set_t &handled_columns,
101
+ vector<PivotValueElement> pivot_values) {
102
+ auto subquery = ConstructInitialGrouping(ref, std::move(all_columns), handled_columns);
103
+
104
+ // push the filtered aggregates
105
+ for (auto &pivot_value : pivot_values) {
106
+ unique_ptr<ParsedExpression> filter;
107
+ idx_t pivot_value_idx = 0;
108
+ for (auto &pivot_column : ref.pivots) {
109
+ for (auto &pivot_expr : pivot_column.pivot_expressions) {
110
+ auto column_ref = make_uniq<CastExpression>(LogicalType::VARCHAR, pivot_expr->Copy());
111
+ auto constant_value = make_uniq<ConstantExpression>(pivot_value.values[pivot_value_idx++]);
112
+ auto comp_expr = make_uniq<ComparisonExpression>(ExpressionType::COMPARE_NOT_DISTINCT_FROM,
113
+ std::move(column_ref), std::move(constant_value));
114
+ if (filter) {
115
+ filter = make_uniq<ConjunctionExpression>(ExpressionType::CONJUNCTION_AND, std::move(filter),
116
+ std::move(comp_expr));
117
+ } else {
118
+ filter = std::move(comp_expr);
119
+ }
120
+ }
121
+ }
122
+ for (auto &aggregate : ref.aggregates) {
123
+ auto copied_aggr = aggregate->Copy();
124
+ auto &aggr = copied_aggr->Cast<FunctionExpression>();
125
+ aggr.filter = filter->Copy();
126
+ auto &aggr_name = aggregate->alias;
127
+ auto name = pivot_value.name;
128
+ if (ref.aggregates.size() > 1 || !aggr_name.empty()) {
129
+ // if there are multiple aggregates specified we add the name of the aggregate as well
130
+ name += "_" + (aggr_name.empty() ? aggregate->GetName() : aggr_name);
131
+ }
132
+ aggr.alias = name;
133
+ subquery->select_list.push_back(std::move(copied_aggr));
134
+ }
135
+ }
136
+ return subquery;
137
+ }
138
+
139
+ struct PivotBindState {
140
+ vector<string> internal_group_names;
141
+ vector<string> group_names;
142
+ vector<string> aggregate_names;
143
+ vector<string> internal_aggregate_names;
144
+ };
145
+
146
+ static unique_ptr<SelectNode> PivotInitialAggregate(PivotBindState &bind_state, PivotRef &ref,
147
+ vector<unique_ptr<ParsedExpression>> all_columns,
148
+ const case_insensitive_set_t &handled_columns) {
149
+ auto subquery_stage1 = ConstructInitialGrouping(ref, std::move(all_columns), handled_columns);
150
+
103
151
  idx_t group_count = 0;
104
152
  for (auto &expr : subquery_stage1->select_list) {
105
153
  bind_state.group_names.push_back(expr->GetName());
@@ -134,6 +182,15 @@ static unique_ptr<SelectNode> PivotInitialAggregate(PivotBindState &bind_state,
134
182
  return subquery_stage1;
135
183
  }
136
184
 
185
+ unique_ptr<ParsedExpression> ConstructPivotExpression(unique_ptr<ParsedExpression> pivot_expr) {
186
+ auto cast = make_uniq<CastExpression>(LogicalType::VARCHAR, std::move(pivot_expr));
187
+ vector<unique_ptr<ParsedExpression>> coalesce_children;
188
+ coalesce_children.push_back(std::move(cast));
189
+ coalesce_children.push_back(make_uniq<ConstantExpression>(Value("NULL")));
190
+ auto coalesce = make_uniq<OperatorExpression>(ExpressionType::OPERATOR_COALESCE, std::move(coalesce_children));
191
+ return std::move(coalesce);
192
+ }
193
+
137
194
  static unique_ptr<SelectNode> PivotListAggregate(PivotBindState &bind_state, PivotRef &ref,
138
195
  unique_ptr<SelectNode> subquery_stage1) {
139
196
  auto subquery_stage2 = make_uniq<SelectNode>();
@@ -166,13 +223,7 @@ static unique_ptr<SelectNode> PivotListAggregate(PivotBindState &bind_state, Piv
166
223
  for (auto &pivot : ref.pivots) {
167
224
  for (auto &pivot_expr : pivot.pivot_expressions) {
168
225
  // coalesce(pivot::VARCHAR, 'NULL')
169
- auto cast = make_uniq<CastExpression>(LogicalType::VARCHAR, std::move(pivot_expr));
170
- vector<unique_ptr<ParsedExpression>> coalesce_children;
171
- coalesce_children.push_back(std::move(cast));
172
- coalesce_children.push_back(make_uniq<ConstantExpression>(Value("NULL")));
173
- auto coalesce =
174
- make_uniq<OperatorExpression>(ExpressionType::OPERATOR_COALESCE, std::move(coalesce_children));
175
-
226
+ auto coalesce = ConstructPivotExpression(std::move(pivot_expr));
176
227
  if (!expr) {
177
228
  expr = std::move(coalesce);
178
229
  } else {
@@ -250,9 +301,8 @@ unique_ptr<BoundTableRef> Binder::BindBoundPivot(PivotRef &ref) {
250
301
  auto &aggregates = result->bound_pivot.aggregates;
251
302
  ExtractPivotAggregates(*result->child, aggregates);
252
303
  if (aggregates.size() != ref.bound_aggregate_names.size()) {
253
- throw BinderException("Pivot aggregate count mismatch. Expected %llu aggregates but found %llu. Are all pivot "
254
- "expressions aggregate functions?",
255
- ref.bound_aggregate_names.size(), aggregates.size());
304
+ throw InternalException("Pivot aggregate count mismatch (expected %llu, found %llu)",
305
+ ref.bound_aggregate_names.size(), aggregates.size());
256
306
  }
257
307
 
258
308
  vector<string> child_names;
@@ -313,6 +363,10 @@ unique_ptr<SelectNode> Binder::BindPivot(PivotRef &ref, vector<unique_ptr<Parsed
313
363
  if (aggr->IsWindow()) {
314
364
  throw BinderException(FormatError(*aggr, "Pivot expression cannot contain window functions"));
315
365
  }
366
+ // bind the function as an aggregate to ensure it is an aggregate and not a scalar function
367
+ auto &aggr_function = aggr->Cast<FunctionExpression>();
368
+ (void)Catalog::GetEntry<AggregateFunctionCatalogEntry>(context, aggr_function.catalog, aggr_function.schema,
369
+ aggr_function.function_name);
316
370
  ExtractPivotExpressions(*aggr, handled_columns);
317
371
  }
318
372
 
@@ -361,40 +415,53 @@ unique_ptr<SelectNode> Binder::BindPivot(PivotRef &ref, vector<unique_ptr<Parsed
361
415
  pivots.insert(val);
362
416
  }
363
417
  }
364
- auto pivot_limit = ClientConfig::GetConfig(context).pivot_limit;
418
+ auto &client_config = ClientConfig::GetConfig(context);
419
+ auto pivot_limit = client_config.pivot_limit;
365
420
  if (total_pivots >= pivot_limit) {
366
421
  throw BinderException("Pivot column limit of %llu exceeded. Use SET pivot_limit=X to increase the limit.",
367
- ClientConfig::GetConfig(context).pivot_limit);
422
+ client_config.pivot_limit);
368
423
  }
369
424
 
370
425
  // construct the required pivot values recursively
371
426
  vector<PivotValueElement> pivot_values;
372
427
  ConstructPivots(ref, pivot_values);
373
428
 
429
+ unique_ptr<SelectNode> pivot_node;
374
430
  // pivots have three components
375
431
  // - the pivots (i.e. future column names)
376
432
  // - the groups (i.e. the future row names
377
433
  // - the aggregates (i.e. the values of the pivot columns)
378
434
 
379
- // executing a pivot statement happens in three stages
380
- // 1) execute the query "SELECT {groups}, {pivots}, {aggregates} FROM {from_clause} GROUP BY {groups}, {pivots}
381
- // this computes all values that are required in the final result, but not yet in the correct orientation
382
- // 2) execute the query "SELECT {groups}, LIST({pivots}), LIST({aggregates}) FROM [Q1] GROUP BY {groups}
383
- // this pushes all pivots and aggregates that belong to a specific group together in an aligned manner
384
- // 3) push a PIVOT operator, that performs the actual pivoting of the values into the different columns
385
-
386
- PivotBindState bind_state;
387
- // Pivot Stage 1
388
- // SELECT {groups}, {pivots}, {aggregates} FROM {from_clause} GROUP BY {groups}, {pivots}
389
- auto subquery_stage1 = PivotInitialAggregate(bind_state, ref, std::move(all_columns), handled_columns);
390
-
391
- // Pivot stage 2
392
- // SELECT {groups}, LIST({pivots}), LIST({aggregates}) FROM [Q1] GROUP BY {groups}
393
- auto subquery_stage2 = PivotListAggregate(bind_state, ref, std::move(subquery_stage1));
394
-
395
- // Pivot stage 3
396
- // construct the final pivot operator
397
- auto pivot_node = PivotFinalOperator(bind_state, ref, std::move(subquery_stage2), std::move(pivot_values));
435
+ // we have two ways of executing a pivot statement
436
+ // (1) the straightforward manner of filtered aggregates SUM(..) FILTER (pivot_value=X)
437
+ // (2) computing the aggregates once, then using LIST to group the aggregates together with the PIVOT operator
438
+ // -> filtered aggregates are faster when there are FEW pivot values
439
+ // -> LIST is faster when there are MANY pivot values
440
+ // we switch dynamically based on the number of pivots to compute
441
+ if (pivot_values.size() <= client_config.pivot_filter_threshold) {
442
+ // use a set of filtered aggregates
443
+ pivot_node = PivotFilteredAggregate(ref, std::move(all_columns), handled_columns, std::move(pivot_values));
444
+ } else {
445
+ // executing a pivot statement happens in three stages
446
+ // 1) execute the query "SELECT {groups}, {pivots}, {aggregates} FROM {from_clause} GROUP BY {groups}, {pivots}
447
+ // this computes all values that are required in the final result, but not yet in the correct orientation
448
+ // 2) execute the query "SELECT {groups}, LIST({pivots}), LIST({aggregates}) FROM [Q1] GROUP BY {groups}
449
+ // this pushes all pivots and aggregates that belong to a specific group together in an aligned manner
450
+ // 3) push a PIVOT operator, that performs the actual pivoting of the values into the different columns
451
+
452
+ PivotBindState bind_state;
453
+ // Pivot Stage 1
454
+ // SELECT {groups}, {pivots}, {aggregates} FROM {from_clause} GROUP BY {groups}, {pivots}
455
+ auto subquery_stage1 = PivotInitialAggregate(bind_state, ref, std::move(all_columns), handled_columns);
456
+
457
+ // Pivot stage 2
458
+ // SELECT {groups}, LIST({pivots}), LIST({aggregates}) FROM [Q1] GROUP BY {groups}
459
+ auto subquery_stage2 = PivotListAggregate(bind_state, ref, std::move(subquery_stage1));
460
+
461
+ // Pivot stage 3
462
+ // construct the final pivot operator
463
+ pivot_node = PivotFinalOperator(bind_state, ref, std::move(subquery_stage2), std::move(pivot_values));
464
+ }
398
465
  return pivot_node;
399
466
  }
400
467
 
@@ -13,7 +13,7 @@ unique_ptr<LogicalOperator> Binder::CreatePlan(BoundCTERef &ref) {
13
13
  types.push_back(type);
14
14
  }
15
15
 
16
- return make_uniq<LogicalCTERef>(index, ref.cte_index, types, ref.bound_columns);
16
+ return make_uniq<LogicalCTERef>(index, ref.cte_index, types, ref.bound_columns, ref.materialized_cte);
17
17
  }
18
18
 
19
19
  } // namespace duckdb
@@ -10,11 +10,14 @@
10
10
  #include "duckdb/planner/operator/logical_asof_join.hpp"
11
11
  #include "duckdb/planner/operator/logical_comparison_join.hpp"
12
12
  #include "duckdb/planner/operator/logical_cross_product.hpp"
13
+ #include "duckdb/planner/operator/logical_dependent_join.hpp"
13
14
  #include "duckdb/planner/operator/logical_filter.hpp"
14
15
  #include "duckdb/planner/operator/logical_positional_join.hpp"
15
16
  #include "duckdb/planner/tableref/bound_joinref.hpp"
16
17
  #include "duckdb/main/client_context.hpp"
17
18
  #include "duckdb/planner/expression_binder/lateral_binder.hpp"
19
+ #include "duckdb/planner/subquery/recursive_dependent_join_planner.hpp"
20
+ #include "duckdb/execution/expression_executor.hpp"
18
21
 
19
22
  namespace duckdb {
20
23
 
@@ -44,13 +47,11 @@ static bool CreateJoinCondition(Expression &expr, const unordered_set<idx_t> &le
44
47
  return false;
45
48
  }
46
49
 
47
- void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<LogicalOperator> &left_child,
48
- unique_ptr<LogicalOperator> &right_child,
49
- const unordered_set<idx_t> &left_bindings,
50
- const unordered_set<idx_t> &right_bindings,
51
- vector<unique_ptr<Expression>> &expressions,
52
- vector<JoinCondition> &conditions,
53
- vector<unique_ptr<Expression>> &arbitrary_expressions) {
50
+ void LogicalComparisonJoin::ExtractJoinConditions(
51
+ ClientContext &context, JoinType type, unique_ptr<LogicalOperator> &left_child,
52
+ unique_ptr<LogicalOperator> &right_child, const unordered_set<idx_t> &left_bindings,
53
+ const unordered_set<idx_t> &right_bindings, vector<unique_ptr<Expression>> &expressions,
54
+ vector<JoinCondition> &conditions, vector<unique_ptr<Expression>> &arbitrary_expressions) {
54
55
  for (auto &expr : expressions) {
55
56
  auto total_side = JoinSide::GetJoinSide(*expr, left_bindings, right_bindings);
56
57
  if (total_side != JoinSide::BOTH) {
@@ -68,6 +69,15 @@ void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<Logi
68
69
  filter.expressions.push_back(std::move(expr));
69
70
  continue;
70
71
  }
72
+ // if the join is a LEFT JOIN and the join expression constantly evaluates to TRUE,
73
+ // then we do not add it to the arbitrary expressions
74
+ if (type == JoinType::LEFT && expr->IsFoldable()) {
75
+ Value result;
76
+ ExpressionExecutor::TryEvaluateScalar(context, *expr, result);
77
+ if (!result.IsNull() && result == Value(true)) {
78
+ continue;
79
+ }
80
+ }
71
81
  } else if ((expr->type >= ExpressionType::COMPARE_EQUAL &&
72
82
  expr->type <= ExpressionType::COMPARE_GREATERTHANOREQUALTO) ||
73
83
  expr->type == ExpressionType::COMPARE_DISTINCT_FROM ||
@@ -82,7 +92,8 @@ void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<Logi
82
92
  }
83
93
  }
84
94
 
85
- void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<LogicalOperator> &left_child,
95
+ void LogicalComparisonJoin::ExtractJoinConditions(ClientContext &context, JoinType type,
96
+ unique_ptr<LogicalOperator> &left_child,
86
97
  unique_ptr<LogicalOperator> &right_child,
87
98
  vector<unique_ptr<Expression>> &expressions,
88
99
  vector<JoinCondition> &conditions,
@@ -90,11 +101,12 @@ void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<Logi
90
101
  unordered_set<idx_t> left_bindings, right_bindings;
91
102
  LogicalJoin::GetTableReferences(*left_child, left_bindings);
92
103
  LogicalJoin::GetTableReferences(*right_child, right_bindings);
93
- return ExtractJoinConditions(type, left_child, right_child, left_bindings, right_bindings, expressions, conditions,
94
- arbitrary_expressions);
104
+ return ExtractJoinConditions(context, type, left_child, right_child, left_bindings, right_bindings, expressions,
105
+ conditions, arbitrary_expressions);
95
106
  }
96
107
 
97
- void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<LogicalOperator> &left_child,
108
+ void LogicalComparisonJoin::ExtractJoinConditions(ClientContext &context, JoinType type,
109
+ unique_ptr<LogicalOperator> &left_child,
98
110
  unique_ptr<LogicalOperator> &right_child,
99
111
  unique_ptr<Expression> condition, vector<JoinCondition> &conditions,
100
112
  vector<unique_ptr<Expression>> &arbitrary_expressions) {
@@ -102,10 +114,12 @@ void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<Logi
102
114
  vector<unique_ptr<Expression>> expressions;
103
115
  expressions.push_back(std::move(condition));
104
116
  LogicalFilter::SplitPredicates(expressions);
105
- return ExtractJoinConditions(type, left_child, right_child, expressions, conditions, arbitrary_expressions);
117
+ return ExtractJoinConditions(context, type, left_child, right_child, expressions, conditions,
118
+ arbitrary_expressions);
106
119
  }
107
120
 
108
- unique_ptr<LogicalOperator> LogicalComparisonJoin::CreateJoin(JoinType type, JoinRefType reftype,
121
+ unique_ptr<LogicalOperator> LogicalComparisonJoin::CreateJoin(ClientContext &context, JoinType type,
122
+ JoinRefType reftype,
109
123
  unique_ptr<LogicalOperator> left_child,
110
124
  unique_ptr<LogicalOperator> right_child,
111
125
  vector<JoinCondition> conditions,
@@ -216,29 +230,37 @@ static bool HasCorrelatedColumns(Expression &expression) {
216
230
  return has_correlated_columns;
217
231
  }
218
232
 
219
- unique_ptr<LogicalOperator> LogicalComparisonJoin::CreateJoin(JoinType type, JoinRefType reftype,
233
+ unique_ptr<LogicalOperator> LogicalComparisonJoin::CreateJoin(ClientContext &context, JoinType type,
234
+ JoinRefType reftype,
220
235
  unique_ptr<LogicalOperator> left_child,
221
236
  unique_ptr<LogicalOperator> right_child,
222
237
  unique_ptr<Expression> condition) {
223
238
  vector<JoinCondition> conditions;
224
239
  vector<unique_ptr<Expression>> arbitrary_expressions;
225
- LogicalComparisonJoin::ExtractJoinConditions(type, left_child, right_child, std::move(condition), conditions,
226
- arbitrary_expressions);
227
- return LogicalComparisonJoin::CreateJoin(type, reftype, std::move(left_child), std::move(right_child),
240
+ LogicalComparisonJoin::ExtractJoinConditions(context, type, left_child, right_child, std::move(condition),
241
+ conditions, arbitrary_expressions);
242
+ return LogicalComparisonJoin::CreateJoin(context, type, reftype, std::move(left_child), std::move(right_child),
228
243
  std::move(conditions), std::move(arbitrary_expressions));
229
244
  }
230
245
 
231
246
  unique_ptr<LogicalOperator> Binder::CreatePlan(BoundJoinRef &ref) {
247
+ auto old_is_outside_flattened = is_outside_flattened;
248
+ // Plan laterals from outermost to innermost
249
+ if (ref.lateral) {
250
+ // Set the flag to ensure that children do not flatten before the root
251
+ is_outside_flattened = false;
252
+ }
232
253
  auto left = CreatePlan(*ref.left);
233
254
  auto right = CreatePlan(*ref.right);
255
+ is_outside_flattened = old_is_outside_flattened;
256
+
257
+ // For joins, depth of the bindings will be one higher on the right because of the lateral binder
258
+ // If the current join does not have correlations between left and right, then the right bindings
259
+ // have depth 1 too high and can be reduced by 1 throughout
234
260
  if (!ref.lateral && !ref.correlated_columns.empty()) {
235
- // non-lateral join with correlated columns
236
- // this happens if there is a join (or cross product) in a correlated subquery
237
- // due to the lateral binder the expression depth of all correlated columns in the "ref.correlated_columns" set
238
- // is 1 too high
239
- // we reduce expression depth of all columns in the "ref.correlated_columns" set by 1
240
261
  LateralBinder::ReduceExpressionDepth(*right, ref.correlated_columns);
241
262
  }
263
+
242
264
  if (ref.type == JoinType::RIGHT && ref.ref_type != JoinRefType::ASOF &&
243
265
  ClientConfig::GetConfig(context).enable_optimizer) {
244
266
  // we turn any right outer joins into left outer joins for optimization purposes
@@ -247,9 +269,22 @@ unique_ptr<LogicalOperator> Binder::CreatePlan(BoundJoinRef &ref) {
247
269
  std::swap(left, right);
248
270
  }
249
271
  if (ref.lateral) {
250
- // lateral join
251
- return PlanLateralJoin(std::move(left), std::move(right), ref.correlated_columns, ref.type,
252
- std::move(ref.condition));
272
+ if (!is_outside_flattened) {
273
+ // If outer dependent joins is yet to be flattened, only plan the lateral
274
+ has_unplanned_dependent_joins = true;
275
+ return LogicalDependentJoin::Create(std::move(left), std::move(right), ref.correlated_columns, ref.type,
276
+ std::move(ref.condition));
277
+ } else {
278
+ // All outer dependent joins have been planned and flattened, so plan and flatten lateral and recursively
279
+ // plan the children
280
+ auto new_plan = PlanLateralJoin(std::move(left), std::move(right), ref.correlated_columns, ref.type,
281
+ std::move(ref.condition));
282
+ if (has_unplanned_dependent_joins) {
283
+ RecursiveDependentJoinPlanner plan(*this);
284
+ plan.VisitOperator(*new_plan);
285
+ }
286
+ return new_plan;
287
+ }
253
288
  }
254
289
  switch (ref.ref_type) {
255
290
  case JoinRefType::CROSS:
@@ -275,7 +310,7 @@ unique_ptr<LogicalOperator> Binder::CreatePlan(BoundJoinRef &ref) {
275
310
  }
276
311
 
277
312
  // now create the join operator from the join condition
278
- auto result = LogicalComparisonJoin::CreateJoin(ref.type, ref.ref_type, std::move(left), std::move(right),
313
+ auto result = LogicalComparisonJoin::CreateJoin(context, ref.type, ref.ref_type, std::move(left), std::move(right),
279
314
  std::move(ref.condition));
280
315
 
281
316
  optional_ptr<LogicalOperator> join;
@@ -6,10 +6,10 @@ namespace duckdb {
6
6
  unique_ptr<LogicalOperator> Binder::CreatePlan(BoundSubqueryRef &ref) {
7
7
  // generate the logical plan for the subquery
8
8
  // this happens separately from the current LogicalPlan generation
9
- ref.binder->plan_subquery = plan_subquery;
9
+ ref.binder->is_outside_flattened = is_outside_flattened;
10
10
  auto subquery = ref.binder->CreatePlan(*ref.subquery);
11
- if (ref.binder->has_unplanned_subqueries) {
12
- has_unplanned_subqueries = true;
11
+ if (ref.binder->has_unplanned_dependent_joins) {
12
+ has_unplanned_dependent_joins = true;
13
13
  }
14
14
  return subquery;
15
15
  }
@@ -117,6 +117,9 @@ unique_ptr<BoundQueryNode> Binder::BindNode(QueryNode &node) {
117
117
  case QueryNodeType::RECURSIVE_CTE_NODE:
118
118
  result = BindNode(node.Cast<RecursiveCTENode>());
119
119
  break;
120
+ case QueryNodeType::CTE_NODE:
121
+ result = BindNode(node.Cast<CTENode>());
122
+ break;
120
123
  default:
121
124
  D_ASSERT(node.type == QueryNodeType::SET_OPERATION_NODE);
122
125
  result = BindNode(node.Cast<SetOperationNode>());
@@ -145,6 +148,8 @@ unique_ptr<LogicalOperator> Binder::CreatePlan(BoundQueryNode &node) {
145
148
  return CreatePlan(node.Cast<BoundSetOperationNode>());
146
149
  case QueryNodeType::RECURSIVE_CTE_NODE:
147
150
  return CreatePlan(node.Cast<BoundRecursiveCTENode>());
151
+ case QueryNodeType::CTE_NODE:
152
+ return CreatePlan(node.Cast<BoundCTENode>());
148
153
  default:
149
154
  throw InternalException("Unsupported bound query node type");
150
155
  }
@@ -15,7 +15,7 @@ BoundAggregateExpression::BoundAggregateExpression(AggregateFunction function, v
15
15
  : Expression(ExpressionType::BOUND_AGGREGATE, ExpressionClass::BOUND_AGGREGATE, function.return_type),
16
16
  function(std::move(function)), children(std::move(children)), bind_info(std::move(bind_info)),
17
17
  aggr_type(aggr_type), filter(std::move(filter)) {
18
- D_ASSERT(!function.name.empty());
18
+ D_ASSERT(!this->function.name.empty());
19
19
  }
20
20
 
21
21
  string BoundAggregateExpression::ToString() const {
@@ -31,41 +31,10 @@ BindResult LateralBinder::BindColumnRef(unique_ptr<ParsedExpression> &expr_ptr,
31
31
  if (result.HasError()) {
32
32
  return result;
33
33
  }
34
- if (depth > 1) {
35
- throw BinderException("Nested lateral joins are not supported yet");
36
- }
37
34
  ExtractCorrelatedColumns(*result.expression);
38
35
  return result;
39
36
  }
40
37
 
41
- vector<CorrelatedColumnInfo> LateralBinder::ExtractCorrelatedColumns(Binder &binder) {
42
-
43
- if (correlated_columns.empty()) {
44
- return binder.correlated_columns;
45
- }
46
-
47
- // clear outer
48
- correlated_columns.clear();
49
- auto all_correlated_columns = binder.correlated_columns;
50
-
51
- // remove outer from inner
52
- for (auto &corr_column : correlated_columns) {
53
- auto entry = std::find(binder.correlated_columns.begin(), binder.correlated_columns.end(), corr_column);
54
- if (entry != binder.correlated_columns.end()) {
55
- binder.correlated_columns.erase(entry);
56
- }
57
- }
58
-
59
- // add inner to outer
60
- for (auto &corr_column : binder.correlated_columns) {
61
- correlated_columns.push_back(corr_column);
62
- }
63
-
64
- // clear inner
65
- binder.correlated_columns.clear();
66
- return all_correlated_columns;
67
- }
68
-
69
38
  BindResult LateralBinder::BindExpression(unique_ptr<ParsedExpression> &expr_ptr, idx_t depth, bool root_expression) {
70
39
  auto &expr = *expr_ptr;
71
40
  switch (expr.GetExpressionClass()) {
@@ -125,7 +94,11 @@ protected:
125
94
  ReduceColumnRefDepth(expr.Cast<BoundColumnRefExpression>());
126
95
  }
127
96
  if (expr.GetExpressionClass() == ExpressionClass::BOUND_SUBQUERY) {
97
+ auto &subquery_ref = expr.Cast<BoundSubqueryExpression>();
128
98
  ReduceExpressionSubquery(expr.Cast<BoundSubqueryExpression>());
99
+ // Recursively update the depth in the bindings of the children nodes
100
+ ExpressionIterator::EnumerateQueryNodeChildren(
101
+ *subquery_ref.subquery, [&](Expression &child_expr) { ReduceExpressionDepth(child_expr); });
129
102
  }
130
103
  }
131
104
 
@@ -79,9 +79,12 @@ bool ExpressionBinder::BindCorrelatedColumns(unique_ptr<ParsedExpression> &expr)
79
79
  auto &active_binders = binder.GetActiveBinders();
80
80
  // make a copy of the set of binders, so we can restore it later
81
81
  auto binders = active_binders;
82
+
83
+ // we already failed with the current binder
82
84
  active_binders.pop_back();
83
85
  idx_t depth = 1;
84
86
  bool success = false;
87
+
85
88
  while (!active_binders.empty()) {
86
89
  auto &next_binder = active_binders.back().get();
87
90
  ExpressionBinder::QualifyColumnNames(next_binder.binder, expr);
@@ -5,6 +5,7 @@
5
5
  #include "duckdb/planner/query_node/bound_select_node.hpp"
6
6
  #include "duckdb/planner/query_node/bound_set_operation_node.hpp"
7
7
  #include "duckdb/planner/query_node/bound_recursive_cte_node.hpp"
8
+ #include "duckdb/planner/query_node/bound_cte_node.hpp"
8
9
  #include "duckdb/planner/tableref/list.hpp"
9
10
 
10
11
  namespace duckdb {
@@ -198,6 +199,11 @@ void ExpressionIterator::EnumerateQueryNodeChildren(BoundQueryNode &node,
198
199
  EnumerateQueryNodeChildren(*cte_node.right, callback);
199
200
  break;
200
201
  }
202
+ case QueryNodeType::CTE_NODE: {
203
+ auto &cte_node = node.Cast<BoundCTENode>();
204
+ EnumerateQueryNodeChildren(*cte_node.child, callback);
205
+ break;
206
+ }
201
207
  case QueryNodeType::SELECT_NODE: {
202
208
  auto &bound_select = node.Cast<BoundSelectNode>();
203
209
  for (auto &expr : bound_select.select_list) {
@@ -8,6 +8,7 @@
8
8
  #include "duckdb/parser/parser.hpp"
9
9
  #include "duckdb/planner/operator/list.hpp"
10
10
  #include "duckdb/planner/operator/logical_extension_operator.hpp"
11
+ #include "duckdb/planner/operator/logical_dependent_join.hpp"
11
12
 
12
13
  namespace duckdb {
13
14
 
@@ -283,6 +284,9 @@ unique_ptr<LogicalOperator> LogicalOperator::Deserialize(Deserializer &deseriali
283
284
  case LogicalOperatorType::LOGICAL_RECURSIVE_CTE:
284
285
  result = LogicalRecursiveCTE::Deserialize(state, reader);
285
286
  break;
287
+ case LogicalOperatorType::LOGICAL_MATERIALIZED_CTE:
288
+ result = LogicalMaterializedCTE::Deserialize(state, reader);
289
+ break;
286
290
  case LogicalOperatorType::LOGICAL_INSERT:
287
291
  result = LogicalInsert::Deserialize(state, reader);
288
292
  break;
@@ -352,6 +356,7 @@ unique_ptr<LogicalOperator> LogicalOperator::Deserialize(Deserializer &deseriali
352
356
  case LogicalOperatorType::LOGICAL_PIVOT:
353
357
  result = LogicalPivot::Deserialize(state, reader);
354
358
  break;
359
+ case LogicalOperatorType::LOGICAL_DEPENDENT_JOIN:
355
360
  case LogicalOperatorType::LOGICAL_INVALID:
356
361
  /* no default here to trigger a warning if we forget to implement deserialize for a new operator */
357
362
  throw SerializationException("Invalid type for operator deserialization");
@@ -19,6 +19,7 @@ void LogicalOperatorVisitor::VisitOperatorChildren(LogicalOperator &op) {
19
19
 
20
20
  void LogicalOperatorVisitor::EnumerateExpressions(LogicalOperator &op,
21
21
  const std::function<void(unique_ptr<Expression> *child)> &callback) {
22
+
22
23
  switch (op.type) {
23
24
  case LogicalOperatorType::LOGICAL_EXPRESSION_GET: {
24
25
  auto &get = op.Cast<LogicalExpressionGet>();
@@ -67,6 +68,7 @@ void LogicalOperatorVisitor::EnumerateExpressions(LogicalOperator &op,
67
68
  }
68
69
  case LogicalOperatorType::LOGICAL_ASOF_JOIN:
69
70
  case LogicalOperatorType::LOGICAL_DELIM_JOIN:
71
+ case LogicalOperatorType::LOGICAL_DEPENDENT_JOIN:
70
72
  case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: {
71
73
  if (op.type == LogicalOperatorType::LOGICAL_DELIM_JOIN) {
72
74
  auto &delim_join = op.Cast<LogicalDelimJoin>();
@@ -10,6 +10,7 @@ void LogicalCTERef::Serialize(FieldWriter &writer) const {
10
10
  writer.WriteField(cte_index);
11
11
  writer.WriteRegularSerializableList(chunk_types);
12
12
  writer.WriteList<string>(bound_columns);
13
+ writer.WriteField(materialized_cte);
13
14
  }
14
15
 
15
16
  unique_ptr<LogicalOperator> LogicalCTERef::Deserialize(LogicalDeserializationState &state, FieldReader &reader) {
@@ -17,7 +18,8 @@ unique_ptr<LogicalOperator> LogicalCTERef::Deserialize(LogicalDeserializationSta
17
18
  auto cte_index = reader.ReadRequired<idx_t>();
18
19
  auto chunk_types = reader.ReadRequiredSerializableList<LogicalType, LogicalType>();
19
20
  auto bound_columns = reader.ReadRequiredList<string>();
20
- return make_uniq<LogicalCTERef>(table_index, cte_index, chunk_types, bound_columns);
21
+ auto materialized_cte = reader.ReadField<CTEMaterialize>(CTEMaterialize::CTE_MATERIALIZE_DEFAULT);
22
+ return make_uniq<LogicalCTERef>(table_index, cte_index, chunk_types, bound_columns, materialized_cte);
21
23
  }
22
24
 
23
25
  vector<idx_t> LogicalCTERef::GetTableIndex() const {
@@ -0,0 +1,26 @@
1
+ #include "duckdb/planner/operator/logical_dependent_join.hpp"
2
+
3
+ namespace duckdb {
4
+
5
+ LogicalDependentJoin::LogicalDependentJoin(unique_ptr<LogicalOperator> left, unique_ptr<LogicalOperator> right,
6
+ vector<CorrelatedColumnInfo> correlated_columns, JoinType type,
7
+ unique_ptr<Expression> condition)
8
+ : LogicalComparisonJoin(type, LogicalOperatorType::LOGICAL_DEPENDENT_JOIN), join_condition(std::move(condition)),
9
+ correlated_columns(std::move(correlated_columns)) {
10
+ children.push_back(std::move(left));
11
+ children.push_back(std::move(right));
12
+ }
13
+
14
+ unique_ptr<LogicalOperator> LogicalDependentJoin::Create(unique_ptr<LogicalOperator> left,
15
+ unique_ptr<LogicalOperator> right,
16
+ vector<CorrelatedColumnInfo> correlated_columns, JoinType type,
17
+ unique_ptr<Expression> condition) {
18
+ return make_uniq<LogicalDependentJoin>(std::move(left), std::move(right), std::move(correlated_columns), type,
19
+ std::move(condition));
20
+ }
21
+
22
+ void LogicalDependentJoin::Serialize(FieldWriter &writer) const {
23
+ throw InternalException("LogicalDependentJoin should not exist in any plan");
24
+ }
25
+
26
+ } // namespace duckdb