duckdb 0.8.2-dev161.0 → 0.8.2-dev1724.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (504) hide show
  1. package/binding.gyp +15 -12
  2. package/binding.gyp.in +1 -1
  3. package/configure.py +1 -1
  4. package/duckdb_extension_config.cmake +10 -0
  5. package/package.json +1 -1
  6. package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
  7. package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
  8. package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
  9. package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
  10. package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
  11. package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
  12. package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
  13. package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
  14. package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
  15. package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
  16. package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
  17. package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
  18. package/src/duckdb/extension/json/include/json_common.hpp +47 -231
  19. package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
  20. package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
  21. package/src/duckdb/extension/json/json_common.cpp +272 -40
  22. package/src/duckdb/extension/json/json_functions/json_create.cpp +21 -2
  23. package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
  24. package/src/duckdb/extension/json/json_functions/json_transform.cpp +91 -38
  25. package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
  26. package/src/duckdb/extension/json/json_functions.cpp +24 -24
  27. package/src/duckdb/extension/json/json_scan.cpp +3 -6
  28. package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
  29. package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
  30. package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
  31. package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
  32. package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
  33. package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
  34. package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
  35. package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
  36. package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
  37. package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
  38. package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
  39. package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
  40. package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
  41. package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
  42. package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
  43. package/src/duckdb/extension/parquet/parquet_extension.cpp +194 -20
  44. package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
  45. package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
  46. package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
  47. package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
  48. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
  49. package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
  50. package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
  51. package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
  52. package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
  53. package/src/duckdb/src/common/allocator.cpp +14 -2
  54. package/src/duckdb/src/common/arrow/arrow_appender.cpp +79 -12
  55. package/src/duckdb/src/common/arrow/arrow_converter.cpp +44 -19
  56. package/src/duckdb/src/common/assert.cpp +3 -0
  57. package/src/duckdb/src/common/enum_util.cpp +4619 -4446
  58. package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
  59. package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
  60. package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
  61. package/src/duckdb/src/common/exception.cpp +2 -2
  62. package/src/duckdb/src/common/extra_type_info.cpp +506 -0
  63. package/src/duckdb/src/common/file_system.cpp +19 -0
  64. package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
  65. package/src/duckdb/src/common/local_file_system.cpp +14 -14
  66. package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
  67. package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
  68. package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
  69. package/src/duckdb/src/common/re2_regex.cpp +1 -1
  70. package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
  71. package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
  72. package/src/duckdb/src/common/sort/partition_state.cpp +44 -11
  73. package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
  74. package/src/duckdb/src/common/types/bit.cpp +51 -0
  75. package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
  76. package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
  77. package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
  78. package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
  79. package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
  80. package/src/duckdb/src/common/types/date.cpp +9 -0
  81. package/src/duckdb/src/common/types/list_segment.cpp +24 -74
  82. package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
  83. package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
  84. package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
  85. package/src/duckdb/src/common/types/uuid.cpp +2 -2
  86. package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
  87. package/src/duckdb/src/common/types/value.cpp +11 -6
  88. package/src/duckdb/src/common/types.cpp +9 -656
  89. package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
  90. package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
  91. package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
  92. package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
  93. package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
  94. package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
  95. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
  96. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
  97. package/src/duckdb/src/core_functions/function_list.cpp +4 -2
  98. package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
  99. package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
  100. package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
  101. package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
  102. package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
  103. package/src/duckdb/src/execution/expression_executor.cpp +1 -1
  104. package/src/duckdb/src/execution/index/art/art.cpp +149 -139
  105. package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
  106. package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
  107. package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
  108. package/src/duckdb/src/execution/index/art/node.cpp +113 -120
  109. package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
  110. package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
  111. package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
  112. package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
  113. package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
  114. package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
  115. package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
  116. package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
  117. package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
  118. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
  119. package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
  120. package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
  121. package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +444 -284
  122. package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
  123. package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
  124. package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +28 -12
  125. package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
  126. package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +23 -4
  127. package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +41 -5
  128. package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
  129. package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
  130. package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
  131. package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
  132. package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
  133. package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
  134. package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
  135. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
  136. package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
  137. package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
  138. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
  139. package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
  140. package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
  141. package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
  142. package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
  143. package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
  144. package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +56 -33
  145. package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +17 -13
  146. package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
  147. package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
  148. package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
  149. package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
  150. package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
  151. package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
  152. package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
  153. package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
  154. package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
  155. package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
  156. package/src/duckdb/src/function/function.cpp +3 -1
  157. package/src/duckdb/src/function/pragma/pragma_queries.cpp +5 -0
  158. package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
  159. package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
  160. package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
  161. package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
  162. package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
  163. package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
  164. package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
  165. package/src/duckdb/src/function/table/arrow.cpp +19 -0
  166. package/src/duckdb/src/function/table/arrow_conversion.cpp +35 -1
  167. package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
  168. package/src/duckdb/src/function/table/read_csv.cpp +100 -17
  169. package/src/duckdb/src/function/table/system/test_all_types.cpp +7 -0
  170. package/src/duckdb/src/function/table/system_functions.cpp +1 -0
  171. package/src/duckdb/src/function/table/table_scan.cpp +9 -0
  172. package/src/duckdb/src/function/table/version/pragma_version.cpp +46 -2
  173. package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
  174. package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
  175. package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
  176. package/src/duckdb/src/include/duckdb/common/dl.hpp +3 -1
  177. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +616 -584
  178. package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
  179. package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
  180. package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
  181. package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
  182. package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
  183. package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
  184. package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +219 -0
  185. package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
  186. package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
  187. package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
  188. package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
  189. package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
  190. package/src/duckdb/src/include/duckdb/common/opener_file_system.hpp +2 -2
  191. package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
  192. package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
  193. package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
  194. package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
  195. package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
  196. package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +13 -0
  197. package/src/duckdb/src/include/duckdb/common/string_util.hpp +11 -0
  198. package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
  199. package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
  200. package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
  201. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
  202. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
  203. package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
  204. package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
  205. package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
  206. package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
  207. package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +0 -1
  208. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
  209. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
  210. package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
  211. package/src/duckdb/src/include/duckdb/common/types/value.hpp +1 -0
  212. package/src/duckdb/src/include/duckdb/common/types.hpp +1 -15
  213. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
  214. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
  215. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
  216. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
  217. package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
  218. package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
  219. package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
  220. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
  221. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
  222. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
  223. package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
  224. package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
  225. package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
  226. package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
  227. package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
  228. package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
  229. package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
  230. package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
  231. package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
  232. package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
  233. package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
  234. package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
  235. package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
  236. package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
  237. package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
  238. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
  239. package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
  240. package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
  241. package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
  242. package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
  243. package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
  244. package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
  245. package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
  246. package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
  247. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +3 -10
  248. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +1 -1
  249. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +1 -1
  250. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp +12 -1
  251. package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
  252. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
  253. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
  254. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
  255. package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
  256. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
  257. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
  258. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
  259. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
  260. package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
  261. package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
  262. package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
  263. package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
  264. package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
  265. package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
  266. package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
  267. package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
  268. package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
  269. package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
  270. package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
  271. package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +4 -0
  272. package/src/duckdb/src/include/duckdb/main/client_config.hpp +5 -0
  273. package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
  274. package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
  275. package/src/duckdb/src/include/duckdb/main/extension/generated_extension_loader.hpp +22 -0
  276. package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +2 -0
  277. package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
  278. package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
  279. package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
  280. package/src/duckdb/src/include/duckdb/main/settings.hpp +39 -1
  281. package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
  282. package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
  283. package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
  284. package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
  285. package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
  286. package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
  287. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
  288. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
  289. package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
  290. package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
  291. package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
  292. package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
  293. package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
  294. package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
  295. package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
  296. package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
  297. package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
  298. package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
  299. package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
  300. package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
  301. package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
  302. package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
  303. package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
  304. package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
  305. package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
  306. package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
  307. package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
  308. package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
  309. package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
  310. package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
  311. package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
  312. package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
  313. package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
  314. package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
  315. package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
  316. package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
  317. package/src/duckdb/src/include/duckdb/planner/binder.hpp +12 -5
  318. package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
  319. package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
  320. package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
  321. package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
  322. package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
  323. package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
  324. package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
  325. package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
  326. package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
  327. package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
  328. package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
  329. package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
  330. package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
  331. package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
  332. package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
  333. package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
  334. package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
  335. package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
  336. package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
  337. package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
  338. package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
  339. package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
  340. package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
  341. package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
  342. package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
  343. package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
  344. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
  345. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
  346. package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
  347. package/src/duckdb/src/include/duckdb.h +28 -0
  348. package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
  349. package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
  350. package/src/duckdb/src/main/config.cpp +4 -0
  351. package/src/duckdb/src/main/database.cpp +1 -1
  352. package/src/duckdb/src/main/extension/extension_helper.cpp +93 -88
  353. package/src/duckdb/src/main/extension/extension_install.cpp +9 -0
  354. package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
  355. package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
  356. package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
  357. package/src/duckdb/src/main/relation.cpp +6 -5
  358. package/src/duckdb/src/main/settings/settings.cpp +79 -18
  359. package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
  360. package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
  361. package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
  362. package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
  363. package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
  364. package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
  365. package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
  366. package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
  367. package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
  368. package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
  369. package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
  370. package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
  371. package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
  372. package/src/duckdb/src/optimizer/optimizer.cpp +51 -14
  373. package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
  374. package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
  375. package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
  376. package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
  377. package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
  378. package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
  379. package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
  380. package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
  381. package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
  382. package/src/duckdb/src/parallel/executor.cpp +15 -0
  383. package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
  384. package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
  385. package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
  386. package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
  387. package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
  388. package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
  389. package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
  390. package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
  391. package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
  392. package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
  393. package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
  394. package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
  395. package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
  396. package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
  397. package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
  398. package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
  399. package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
  400. package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
  401. package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
  402. package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
  403. package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
  404. package/src/duckdb/src/parser/parser.cpp +8 -2
  405. package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
  406. package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
  407. package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
  408. package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
  409. package/src/duckdb/src/parser/query_node.cpp +15 -37
  410. package/src/duckdb/src/parser/result_modifier.cpp +0 -74
  411. package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
  412. package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
  413. package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
  414. package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
  415. package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
  416. package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
  417. package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
  418. package/src/duckdb/src/parser/tableref.cpp +0 -44
  419. package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
  420. package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
  421. package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
  422. package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
  423. package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
  424. package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
  425. package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
  426. package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
  427. package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
  428. package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
  429. package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
  430. package/src/duckdb/src/parser/transformer.cpp +15 -0
  431. package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
  432. package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
  433. package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
  434. package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
  435. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
  436. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
  437. package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
  438. package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
  439. package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
  440. package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
  441. package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +64 -26
  442. package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
  443. package/src/duckdb/src/planner/binder.cpp +44 -31
  444. package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
  445. package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
  446. package/src/duckdb/src/planner/expression_binder.cpp +3 -0
  447. package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
  448. package/src/duckdb/src/planner/logical_operator.cpp +5 -0
  449. package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
  450. package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
  451. package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
  452. package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
  453. package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
  454. package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
  455. package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
  456. package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
  457. package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
  458. package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
  459. package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
  460. package/src/duckdb/src/storage/compression/rle.cpp +0 -1
  461. package/src/duckdb/src/storage/data_table.cpp +1 -1
  462. package/src/duckdb/src/storage/local_storage.cpp +3 -3
  463. package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
  464. package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
  465. package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
  466. package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
  467. package/src/duckdb/src/storage/serialization/serialize_types.cpp +127 -0
  468. package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
  469. package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
  470. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  471. package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
  472. package/src/duckdb/src/storage/table/row_group.cpp +25 -9
  473. package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
  474. package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
  475. package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
  476. package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
  477. package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
  478. package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
  479. package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
  480. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
  481. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
  482. package/src/duckdb/ub_src_common.cpp +2 -0
  483. package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
  484. package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
  485. package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
  486. package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
  487. package/src/duckdb/ub_src_function_scalar.cpp +2 -0
  488. package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
  489. package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
  490. package/src/duckdb/ub_src_optimizer.cpp +6 -0
  491. package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
  492. package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
  493. package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
  494. package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
  495. package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
  496. package/src/duckdb/ub_src_planner_operator.cpp +4 -0
  497. package/src/duckdb/ub_src_storage_serialization.cpp +10 -0
  498. package/src/statement.cpp +10 -3
  499. package/test/columns.test.ts +24 -1
  500. package/test/test_all_types.test.ts +234 -0
  501. package/tsconfig.json +1 -0
  502. package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
  503. package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
  504. package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -1,6 +1,7 @@
1
1
  #include "duckdb/optimizer/deliminator.hpp"
2
2
 
3
3
  #include "duckdb/optimizer/join_order/join_order_optimizer.hpp"
4
+ #include "duckdb/optimizer/remove_duplicate_groups.hpp"
4
5
  #include "duckdb/planner/expression/bound_cast_expression.hpp"
5
6
  #include "duckdb/planner/expression/bound_columnref_expression.hpp"
6
7
  #include "duckdb/planner/expression/bound_conjunction_expression.hpp"
@@ -12,35 +13,18 @@
12
13
 
13
14
  namespace duckdb {
14
15
 
15
- class DeliminatorPlanUpdater : LogicalOperatorVisitor {
16
+ struct DelimCandidate {
16
17
  public:
17
- explicit DeliminatorPlanUpdater(ClientContext &context) : context(context) {
18
+ explicit DelimCandidate(LogicalDelimJoin &delim_join) : delim_join(delim_join), delim_get_count(0) {
18
19
  }
19
- //! Update the plan after a DelimGet has been removed
20
- void VisitOperator(LogicalOperator &op) override;
21
- void VisitExpression(unique_ptr<Expression> *expression) override;
22
20
 
23
21
  public:
24
- ClientContext &context;
25
-
26
- expression_map_t<Expression *> expr_map;
27
- column_binding_map_t<bool> projection_map;
28
- column_binding_map_t<Expression *> reverse_proj_or_agg_map;
29
- unique_ptr<LogicalOperator> temp_ptr;
22
+ LogicalDelimJoin &delim_join;
23
+ vector<reference<unique_ptr<LogicalOperator>>> joins;
24
+ idx_t delim_get_count;
30
25
  };
31
26
 
32
- static idx_t DelimGetCount(LogicalOperator &op) {
33
- if (op.type == LogicalOperatorType::LOGICAL_DELIM_GET) {
34
- return 1;
35
- }
36
- idx_t child_count = 0;
37
- for (auto &child : op.children) {
38
- child_count += DelimGetCount(*child);
39
- }
40
- return child_count;
41
- }
42
-
43
- static bool IsEqualityJoinCondition(JoinCondition &cond) {
27
+ static bool IsEqualityJoinCondition(const JoinCondition &cond) {
44
28
  switch (cond.comparison) {
45
29
  case ExpressionType::COMPARE_EQUAL:
46
30
  case ExpressionType::COMPARE_NOT_DISTINCT_FROM:
@@ -50,119 +34,55 @@ static bool IsEqualityJoinCondition(JoinCondition &cond) {
50
34
  }
51
35
  }
52
36
 
53
- static bool InequalityDelimJoinCanBeEliminated(JoinType &join_type) {
54
- switch (join_type) {
55
- case JoinType::ANTI:
56
- case JoinType::MARK:
57
- case JoinType::SEMI:
58
- case JoinType::SINGLE:
59
- return true;
60
- default:
61
- return false;
62
- }
63
- }
37
+ unique_ptr<LogicalOperator> Deliminator::Optimize(unique_ptr<LogicalOperator> op) {
38
+ root = op;
64
39
 
65
- void DeliminatorPlanUpdater::VisitOperator(LogicalOperator &op) {
66
- VisitOperatorChildren(op);
67
- VisitOperatorExpressions(op);
68
- if (op.type == LogicalOperatorType::LOGICAL_DELIM_JOIN && DelimGetCount(op) == 0) {
69
- auto &delim_join = op.Cast<LogicalDelimJoin>();
70
- auto &decs = delim_join.duplicate_eliminated_columns;
71
- for (auto &cond : delim_join.conditions) {
72
- if (!IsEqualityJoinCondition(cond)) {
73
- continue;
74
- }
75
- auto rhs = cond.right.get();
76
- while (rhs->type == ExpressionType::OPERATOR_CAST) {
77
- auto &cast = rhs->Cast<BoundCastExpression>();
78
- rhs = cast.child.get();
79
- }
80
- if (rhs->type != ExpressionType::BOUND_COLUMN_REF) {
81
- throw InternalException("Error in Deliminator: expected a bound column reference");
82
- }
83
- auto &colref = rhs->Cast<BoundColumnRefExpression>();
84
- if (projection_map.find(colref.binding) != projection_map.end()) {
85
- // value on the right is a projection of removed DelimGet
86
- for (idx_t i = 0; i < decs.size(); i++) {
87
- if (decs[i]->Equals(*cond.left)) {
88
- // the value on the left no longer needs to be a duplicate-eliminated column
89
- decs.erase(decs.begin() + i);
90
- break;
40
+ vector<DelimCandidate> candidates;
41
+ FindCandidates(op, candidates);
42
+
43
+ for (auto &candidate : candidates) {
44
+ auto &delim_join = candidate.delim_join;
45
+
46
+ bool all_removed = true;
47
+ bool all_equality_conditions = true;
48
+ for (auto &join : candidate.joins) {
49
+ all_removed =
50
+ RemoveJoinWithDelimGet(delim_join, candidate.delim_get_count, join, all_equality_conditions) &&
51
+ all_removed;
52
+ }
53
+
54
+ // Change type if there are no more duplicate-eliminated columns
55
+ if (candidate.joins.size() == candidate.delim_get_count && all_removed) {
56
+ delim_join.duplicate_eliminated_columns.clear();
57
+ if (all_equality_conditions) {
58
+ for (auto &cond : delim_join.conditions) {
59
+ if (IsEqualityJoinCondition(cond)) {
60
+ cond.comparison = ExpressionType::COMPARE_NOT_DISTINCT_FROM;
91
61
  }
92
62
  }
93
- // whether we applied an IS NOT NULL filter
94
- cond.comparison = ExpressionType::COMPARE_NOT_DISTINCT_FROM;
95
63
  }
96
- }
97
- // change type if there are no more duplicate-eliminated columns
98
- if (decs.empty()) {
99
64
  delim_join.type = LogicalOperatorType::LOGICAL_COMPARISON_JOIN;
100
- // sub-plans with DelimGets are not re-orderable (yet), however, we removed all DelimGet of this DelimJoin
101
- // the DelimGets are on the RHS of the DelimJoin, so we can call the JoinOrderOptimizer on the RHS now
102
- JoinOrderOptimizer optimizer(context);
103
- delim_join.children[1] = optimizer.Optimize(std::move(delim_join.children[1]));
104
65
  }
105
66
  }
106
- }
107
-
108
- void DeliminatorPlanUpdater::VisitExpression(unique_ptr<Expression> *expression) {
109
- auto &expr = **expression;
110
- auto entry = expr_map.find(expr);
111
- if (entry != expr_map.end()) {
112
- *expression = entry->second->Copy();
113
- } else {
114
- VisitExpressionChildren(**expression);
115
- }
116
- }
117
67
 
118
- unique_ptr<LogicalOperator> Deliminator::Optimize(unique_ptr<LogicalOperator> op) {
119
- vector<unique_ptr<LogicalOperator> *> candidates;
120
- FindCandidates(&op, candidates);
121
-
122
- for (auto &candidate : candidates) {
123
- DeliminatorPlanUpdater updater(context);
124
- if (RemoveCandidate(&op, candidate, updater)) {
125
- updater.VisitOperator(*op);
126
- }
127
- }
128
68
  return op;
129
69
  }
130
70
 
131
- void Deliminator::FindCandidates(unique_ptr<LogicalOperator> *op_ptr,
132
- vector<unique_ptr<LogicalOperator> *> &candidates) {
133
- auto op = op_ptr->get();
134
- // search children before adding, so the deepest candidates get added first
71
+ void Deliminator::FindCandidates(unique_ptr<LogicalOperator> &op, vector<DelimCandidate> &candidates) {
72
+ // Search children before adding, so the deepest candidates get added first
135
73
  for (auto &child : op->children) {
136
- FindCandidates(&child, candidates);
137
- }
138
- // search for projection/aggregate
139
- if (op->type != LogicalOperatorType::LOGICAL_PROJECTION &&
140
- op->type != LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY) {
141
- return;
142
- }
143
- // followed by a join
144
- if (op->children[0]->type != LogicalOperatorType::LOGICAL_COMPARISON_JOIN) {
145
- return;
146
- }
147
- auto &join = *op->children[0];
148
- // with a DelimGet as a direct child (left or right)
149
- if (join.children[0]->type == LogicalOperatorType::LOGICAL_DELIM_GET ||
150
- join.children[1]->type == LogicalOperatorType::LOGICAL_DELIM_GET) {
151
- candidates.push_back(op_ptr);
152
- return;
153
- }
154
- // or a filter followed by a DelimGet (left)
155
- if (join.children[0]->type == LogicalOperatorType::LOGICAL_FILTER &&
156
- join.children[0]->children[0]->type == LogicalOperatorType::LOGICAL_DELIM_GET) {
157
- candidates.push_back(op_ptr);
158
- return;
74
+ FindCandidates(child, candidates);
159
75
  }
160
- // filter followed by a DelimGet (right)
161
- if (join.children[1]->type == LogicalOperatorType::LOGICAL_FILTER &&
162
- join.children[1]->children[0]->type == LogicalOperatorType::LOGICAL_DELIM_GET) {
163
- candidates.push_back(op_ptr);
76
+
77
+ if (op->type != LogicalOperatorType::LOGICAL_DELIM_JOIN) {
164
78
  return;
165
79
  }
80
+
81
+ candidates.emplace_back(op->Cast<LogicalDelimJoin>());
82
+ auto &candidate = candidates.back();
83
+
84
+ // DelimGets are in the RHS
85
+ FindJoinWithDelimGet(op->children[1], candidate);
166
86
  }
167
87
 
168
88
  static bool OperatorIsDelimGet(LogicalOperator &op) {
@@ -176,6 +96,23 @@ static bool OperatorIsDelimGet(LogicalOperator &op) {
176
96
  return false;
177
97
  }
178
98
 
99
+ void Deliminator::FindJoinWithDelimGet(unique_ptr<LogicalOperator> &op, DelimCandidate &candidate) {
100
+ if (op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN) {
101
+ FindJoinWithDelimGet(op->children[0], candidate);
102
+ } else if (op->type == LogicalOperatorType::LOGICAL_DELIM_GET) {
103
+ candidate.delim_get_count++;
104
+ } else {
105
+ for (auto &child : op->children) {
106
+ FindJoinWithDelimGet(child, candidate);
107
+ }
108
+ }
109
+
110
+ if (op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN &&
111
+ (OperatorIsDelimGet(*op->children[0]) || OperatorIsDelimGet(*op->children[1]))) {
112
+ candidate.joins.emplace_back(op);
113
+ }
114
+ }
115
+
179
116
  static bool ChildJoinTypeCanBeDeliminated(JoinType &join_type) {
180
117
  switch (join_type) {
181
118
  case JoinType::INNER:
@@ -186,255 +123,173 @@ static bool ChildJoinTypeCanBeDeliminated(JoinType &join_type) {
186
123
  }
187
124
  }
188
125
 
189
- bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *plan, unique_ptr<LogicalOperator> *candidate,
190
- DeliminatorPlanUpdater &updater) {
191
- auto &proj_or_agg = **candidate;
192
- auto &join = proj_or_agg.children[0]->Cast<LogicalComparisonJoin>();
193
- if (!ChildJoinTypeCanBeDeliminated(join.join_type)) {
126
+ bool Deliminator::RemoveJoinWithDelimGet(LogicalDelimJoin &delim_join, const idx_t delim_get_count,
127
+ unique_ptr<LogicalOperator> &join, bool &all_equality_conditions) {
128
+ auto &comparison_join = join->Cast<LogicalComparisonJoin>();
129
+ if (!ChildJoinTypeCanBeDeliminated(comparison_join.join_type)) {
194
130
  return false;
195
131
  }
196
132
 
197
- // get the index (left or right) of the DelimGet side of the join
198
- idx_t delim_idx = OperatorIsDelimGet(*join.children[0]) ? 0 : 1;
199
- D_ASSERT(OperatorIsDelimGet(*join.children[delim_idx]));
200
- // get the filter (if any)
133
+ // Get the index (left or right) of the DelimGet side of the join
134
+ const idx_t delim_idx = OperatorIsDelimGet(*join->children[0]) ? 0 : 1;
135
+
136
+ // Get the filter (if any)
201
137
  optional_ptr<LogicalFilter> filter;
202
- if (join.children[delim_idx]->type == LogicalOperatorType::LOGICAL_FILTER) {
203
- filter = &join.children[delim_idx]->Cast<LogicalFilter>();
138
+ vector<unique_ptr<Expression>> filter_expressions;
139
+ if (join->children[delim_idx]->type == LogicalOperatorType::LOGICAL_FILTER) {
140
+ filter = &join->children[delim_idx]->Cast<LogicalFilter>();
141
+ for (auto &expr : filter->expressions) {
142
+ filter_expressions.emplace_back(expr->Copy());
143
+ }
204
144
  }
205
- auto &delim_get = (filter ? filter->children[0] : join.children[delim_idx])->Cast<LogicalDelimGet>();
206
- if (join.conditions.size() != delim_get.chunk_types.size()) {
207
- // joining with DelimGet adds new information
208
- return false;
145
+
146
+ auto &delim_get = (filter ? filter->children[0] : join->children[delim_idx])->Cast<LogicalDelimGet>();
147
+ if (comparison_join.conditions.size() != delim_get.chunk_types.size()) {
148
+ return false; // Joining with DelimGet adds new information
209
149
  }
210
- // check if joining with the DelimGet is redundant, and collect relevant column information
211
- bool all_equality_conditions = true;
212
- vector<reference<Expression>> nulls_are_not_equal_exprs;
213
- for (auto &cond : join.conditions) {
150
+
151
+ // Check if joining with the DelimGet is redundant, and collect relevant column information
152
+ ColumnBindingReplacer replacer;
153
+ auto &replacement_bindings = replacer.replacement_bindings;
154
+ for (auto &cond : comparison_join.conditions) {
214
155
  all_equality_conditions = all_equality_conditions && IsEqualityJoinCondition(cond);
215
156
  auto &delim_side = delim_idx == 0 ? *cond.left : *cond.right;
216
157
  auto &other_side = delim_idx == 0 ? *cond.right : *cond.left;
217
- if (delim_side.type != ExpressionType::BOUND_COLUMN_REF) {
218
- // non-colref e.g. expression -(4, 1) in 4-i=j where i is from DelimGet
219
- // FIXME: might be possible to also eliminate these
158
+ if (delim_side.type != ExpressionType::BOUND_COLUMN_REF ||
159
+ other_side.type != ExpressionType::BOUND_COLUMN_REF) {
220
160
  return false;
221
161
  }
222
- updater.expr_map[delim_side] = &other_side;
162
+ auto &delim_colref = delim_side.Cast<BoundColumnRefExpression>();
163
+ auto &other_colref = other_side.Cast<BoundColumnRefExpression>();
164
+ replacement_bindings.emplace_back(delim_colref.binding, other_colref.binding);
165
+
223
166
  if (cond.comparison != ExpressionType::COMPARE_NOT_DISTINCT_FROM) {
224
- nulls_are_not_equal_exprs.push_back(other_side);
167
+ auto is_not_null_expr =
168
+ make_uniq<BoundOperatorExpression>(ExpressionType::OPERATOR_IS_NOT_NULL, LogicalType::BOOLEAN);
169
+ is_not_null_expr->children.push_back(other_side.Copy());
170
+ filter_expressions.push_back(std::move(is_not_null_expr));
225
171
  }
226
172
  }
227
173
 
228
- // removed DelimGet columns are assigned a new ColumnBinding by Projection/Aggregation, keep track here
229
- if (proj_or_agg.type == LogicalOperatorType::LOGICAL_PROJECTION) {
230
- for (auto &cb : proj_or_agg.GetColumnBindings()) {
231
- updater.projection_map[cb] = true;
232
- updater.reverse_proj_or_agg_map[cb] = proj_or_agg.expressions[cb.column_index].get();
233
- for (auto &expr : nulls_are_not_equal_exprs) {
234
- if (proj_or_agg.expressions[cb.column_index]->Equals(expr.get())) {
235
- updater.projection_map[cb] = false;
236
- break;
237
- }
238
- }
239
- }
240
- } else {
241
- auto &agg = proj_or_agg.Cast<LogicalAggregate>();
242
-
243
- // Create a vector of all exprs in the agg
244
- vector<Expression *> all_agg_exprs;
245
- all_agg_exprs.reserve(agg.groups.size() + agg.expressions.size());
246
- for (auto &expr : agg.groups) {
247
- all_agg_exprs.push_back(expr.get());
248
- }
249
- for (auto &expr : agg.expressions) {
250
- all_agg_exprs.push_back(expr.get());
251
- }
252
-
253
- for (auto &cb : agg.GetColumnBindings()) {
254
- updater.projection_map[cb] = true;
255
- updater.reverse_proj_or_agg_map[cb] = all_agg_exprs[cb.column_index];
256
- for (auto &expr : nulls_are_not_equal_exprs) {
257
- if ((cb.table_index == agg.group_index && agg.groups[cb.column_index]->Equals(expr.get())) ||
258
- (cb.table_index == agg.aggregate_index && agg.expressions[cb.column_index]->Equals(expr.get()))) {
259
- updater.projection_map[cb] = false;
260
- break;
261
- }
262
- }
263
- }
174
+ if (!all_equality_conditions &&
175
+ !RemoveInequalityJoinWithDelimGet(delim_join, delim_get_count, join, replacement_bindings)) {
176
+ return false;
264
177
  }
265
178
 
266
- if (!all_equality_conditions) {
267
- // we can get rid of an inequality join with a DelimGet, but only under specific circumstances
268
- if (!RemoveInequalityCandidate(plan, candidate, updater)) {
269
- return false;
270
- }
179
+ unique_ptr<LogicalOperator> replacement_op = std::move(comparison_join.children[1 - delim_idx]);
180
+ if (!filter_expressions.empty()) { // Create filter if necessary
181
+ auto new_filter = make_uniq<LogicalFilter>();
182
+ new_filter->expressions = std::move(filter_expressions);
183
+ new_filter->children.emplace_back(std::move(replacement_op));
184
+ replacement_op = std::move(new_filter);
271
185
  }
272
186
 
273
- // make a filter if needed
274
- if (!nulls_are_not_equal_exprs.empty() || filter != nullptr) {
275
- auto filter_op = make_uniq<LogicalFilter>();
276
- if (!nulls_are_not_equal_exprs.empty()) {
277
- // add an IS NOT NULL filter that was implicitly in JoinCondition::null_values_are_equal
278
- for (auto &expr : nulls_are_not_equal_exprs) {
279
- auto is_not_null_expr =
280
- make_uniq<BoundOperatorExpression>(ExpressionType::OPERATOR_IS_NOT_NULL, LogicalType::BOOLEAN);
281
- is_not_null_expr->children.push_back(expr.get().Copy());
282
- filter_op->expressions.push_back(std::move(is_not_null_expr));
283
- }
284
- }
285
- if (filter != nullptr) {
286
- for (auto &expr : filter->expressions) {
287
- filter_op->expressions.push_back(std::move(expr));
288
- }
289
- }
290
- filter_op->children.push_back(std::move(join.children[1 - delim_idx]));
291
- join.children[1 - delim_idx] = std::move(filter_op);
292
- }
293
- // temporarily save deleted operator so its expressions are still available
294
- updater.temp_ptr = std::move(proj_or_agg.children[0]);
295
- // replace the redundant join
296
- proj_or_agg.children[0] = std::move(join.children[1 - delim_idx]);
297
- return true;
298
- }
187
+ join = std::move(replacement_op);
299
188
 
300
- static void GetDelimJoins(LogicalOperator &op, vector<LogicalOperator *> &delim_joins) {
301
- for (auto &child : op.children) {
302
- GetDelimJoins(*child, delim_joins);
303
- }
304
- if (op.type == LogicalOperatorType::LOGICAL_DELIM_JOIN) {
305
- delim_joins.push_back(&op);
306
- }
189
+ // TODO: Maybe go from delim join instead to save work
190
+ replacer.VisitOperator(*root);
191
+ return true;
307
192
  }
308
193
 
309
- static bool HasChild(LogicalOperator *haystack, LogicalOperator *needle, idx_t &side) {
310
- if (haystack == needle) {
311
- return true;
312
- }
313
- for (idx_t i = 0; i < haystack->children.size(); i++) {
314
- auto &child = haystack->children[i];
315
- idx_t dummy_side;
316
- if (HasChild(child.get(), needle, dummy_side)) {
317
- side = i;
318
- return true;
319
- }
320
- }
321
- return false;
194
+ static bool InequalityDelimJoinCanBeEliminated(JoinType &join_type) {
195
+ return join_type == JoinType::ANTI || join_type == JoinType::MARK || join_type == JoinType::SEMI ||
196
+ join_type == JoinType::SINGLE;
322
197
  }
323
198
 
324
- bool Deliminator::RemoveInequalityCandidate(unique_ptr<LogicalOperator> *plan, unique_ptr<LogicalOperator> *candidate,
325
- DeliminatorPlanUpdater &updater) {
326
- auto &proj_or_agg = **candidate;
327
- // first, we find a DelimJoin in "plan" that has only one DelimGet as a child, which is in "candidate"
328
- if (DelimGetCount(proj_or_agg) != 1) {
329
- // the candidate therefore must have only a single DelimGet in its children
330
- return false;
331
- }
332
-
333
- vector<LogicalOperator *> delim_joins;
334
- GetDelimJoins(**plan, delim_joins);
335
-
336
- LogicalOperator *parent = nullptr;
337
- idx_t parent_delim_get_side = 0;
338
- for (auto dj : delim_joins) {
339
- D_ASSERT(dj->type == LogicalOperatorType::LOGICAL_DELIM_JOIN);
340
- if (!HasChild(dj, &proj_or_agg, parent_delim_get_side)) {
341
- continue;
342
- }
343
- // we found a parent DelimJoin
344
- if (DelimGetCount(*dj) != 1) {
345
- // it has more than one DelimGet children
346
- continue;
199
+ bool FindAndReplaceBindings(vector<ColumnBinding> &traced_bindings, const vector<unique_ptr<Expression>> &expressions,
200
+ const vector<ColumnBinding> &current_bindings) {
201
+ for (auto &binding : traced_bindings) {
202
+ idx_t current_idx;
203
+ for (current_idx = 0; current_idx < expressions.size(); current_idx++) {
204
+ if (binding == current_bindings[current_idx]) {
205
+ break;
206
+ }
347
207
  }
348
208
 
349
- // we can only remove inequality join with a DelimGet if the parent DelimJoin has one of these join types
350
- auto &delim_join = dj->Cast<LogicalDelimJoin>();
351
- if (!InequalityDelimJoinCanBeEliminated(delim_join.join_type)) {
352
- continue;
209
+ if (current_idx == expressions.size() || expressions[current_idx]->type != ExpressionType::BOUND_COLUMN_REF) {
210
+ return false; // Didn't find / can't deal with non-colref
353
211
  }
354
212
 
355
- parent = dj;
356
- break;
357
- }
358
- if (!parent) {
359
- return false;
213
+ auto &colref = expressions[current_idx]->Cast<BoundColumnRefExpression>();
214
+ binding = colref.binding;
360
215
  }
216
+ return true;
217
+ }
361
218
 
362
- // we found the parent delim join, and we may be able to remove the child DelimGet join
363
- // but we need to make sure that their conditions refer to exactly the same columns
364
- auto &parent_delim_join = parent->Cast<LogicalDelimJoin>();
365
- auto &join = proj_or_agg.children[0]->Cast<LogicalComparisonJoin>();
366
- if (parent_delim_join.conditions.size() != join.conditions.size()) {
367
- // different number of conditions, can't replace
219
+ bool Deliminator::RemoveInequalityJoinWithDelimGet(LogicalDelimJoin &delim_join, const idx_t delim_get_count,
220
+ unique_ptr<LogicalOperator> &join,
221
+ const vector<ReplacementBinding> &replacement_bindings) {
222
+ auto &comparison_join = join->Cast<LogicalComparisonJoin>();
223
+ auto &delim_conditions = delim_join.conditions;
224
+ const auto &join_conditions = comparison_join.conditions;
225
+ if (delim_get_count != 1 || !InequalityDelimJoinCanBeEliminated(delim_join.join_type) ||
226
+ delim_conditions.size() != join_conditions.size()) {
368
227
  return false;
369
228
  }
370
229
 
371
- // we can only do this optimization under the following conditions:
372
- // 1. all join expressions coming from the DelimGet side are colrefs
373
- // 2. these expressions refer to colrefs coming from the proj/agg on top of the child DelimGet join
374
- // 3. the expression (before it was proj/agg) can be found in the conditions of the child DelimGet join
375
- for (auto &parent_cond : parent_delim_join.conditions) {
376
- auto &parent_expr = parent_delim_get_side == 0 ? parent_cond.left : parent_cond.right;
377
- if (parent_expr->type != ExpressionType::BOUND_COLUMN_REF) {
378
- // can only deal with colrefs
379
- return false;
230
+ // TODO: we cannot perform the optimization here because our pure inequality joins don't implement
231
+ // JoinType::SINGLE yet
232
+ if (delim_join.join_type == JoinType::SINGLE) {
233
+ bool has_one_equality = false;
234
+ for (auto &cond : join_conditions) {
235
+ has_one_equality = has_one_equality || IsEqualityJoinCondition(cond);
380
236
  }
381
- auto &parent_colref = parent_expr->Cast<BoundColumnRefExpression>();
382
- auto it = updater.reverse_proj_or_agg_map.find(parent_colref.binding);
383
- if (it == updater.reverse_proj_or_agg_map.end()) {
384
- // refers to a column that was not in the child DelimGet join
237
+ if (!has_one_equality) {
385
238
  return false;
386
239
  }
387
- // try to find the corresponding child condition
388
- // TODO: can be more flexible - allow CAST
389
- auto &child_expr = *it->second;
390
- bool found = false;
391
- for (auto &child_cond : join.conditions) {
392
- if (child_cond.left->Equals(child_expr) || child_cond.right->Equals(child_expr)) {
393
- found = true;
394
- break;
395
- }
396
- }
397
- if (!found) {
398
- // could not find the mapped expression in the child condition expressions
240
+ }
241
+
242
+ // We only support colref's
243
+ vector<ColumnBinding> traced_bindings;
244
+ for (const auto &cond : delim_conditions) {
245
+ if (cond.right->type != ExpressionType::BOUND_COLUMN_REF) {
399
246
  return false;
400
247
  }
248
+ auto &colref = cond.right->Cast<BoundColumnRefExpression>();
249
+ traced_bindings.emplace_back(colref.binding);
401
250
  }
402
251
 
403
- // TODO: we cannot perform the optimization here because our pure inequality joins don't implement
404
- // JoinType::SINGLE yet
405
- if (parent_delim_join.join_type == JoinType::SINGLE) {
406
- bool has_one_equality = false;
407
- for (auto &cond : join.conditions) {
408
- has_one_equality = has_one_equality || IsEqualityJoinCondition(cond);
252
+ // Now we trace down the bindings to the join (for now, we only trace it through a few operators)
253
+ reference<LogicalOperator> current_op = *delim_join.children[1];
254
+ while (&current_op.get() != join.get()) {
255
+ if (current_op.get().children.size() != 1) {
256
+ return false;
409
257
  }
410
- if (!has_one_equality) {
258
+
259
+ switch (current_op.get().type) {
260
+ case LogicalOperatorType::LOGICAL_PROJECTION:
261
+ FindAndReplaceBindings(traced_bindings, current_op.get().expressions, current_op.get().GetColumnBindings());
262
+ break;
263
+ case LogicalOperatorType::LOGICAL_FILTER:
264
+ break; // Doesn't change bindings
265
+ default:
411
266
  return false;
412
267
  }
268
+ current_op = *current_op.get().children[0];
413
269
  }
414
270
 
415
- // we are now sure that we can remove the child DelimGet join, so we basically do the same loop as above
416
- // this time without checks because we already did them, and replace the expressions
417
- for (auto &parent_cond : parent_delim_join.conditions) {
418
- auto &parent_expr = parent_delim_get_side == 0 ? parent_cond.left : parent_cond.right;
419
- auto &parent_colref = parent_expr->Cast<BoundColumnRefExpression>();
420
- auto it = updater.reverse_proj_or_agg_map.find(parent_colref.binding);
421
- auto &child_expr = *it->second;
422
- for (auto &child_cond : join.conditions) {
423
- if (!child_cond.left->Equals(child_expr) && !child_cond.right->Equals(child_expr)) {
424
- continue;
271
+ // Get the index (left or right) of the DelimGet side of the join
272
+ const idx_t delim_idx = OperatorIsDelimGet(*join->children[0]) ? 0 : 1;
273
+
274
+ bool found_all = true;
275
+ for (idx_t cond_idx = 0; cond_idx < delim_conditions.size(); cond_idx++) {
276
+ auto &delim_condition = delim_conditions[cond_idx];
277
+ const auto &traced_binding = traced_bindings[cond_idx];
278
+
279
+ bool found = false;
280
+ for (auto &join_condition : join_conditions) {
281
+ auto &delim_side = delim_idx == 0 ? *join_condition.left : *join_condition.right;
282
+ auto &colref = delim_side.Cast<BoundColumnRefExpression>();
283
+ if (colref.binding == traced_binding) {
284
+ delim_condition.comparison = FlipComparisonExpression(join_condition.comparison);
285
+ found = true;
286
+ break;
425
287
  }
426
- parent_expr = make_uniq<BoundColumnRefExpression>(parent_expr->alias, parent_expr->return_type, it->first);
427
- parent_cond.comparison =
428
- parent_delim_get_side == 0 ? child_cond.comparison : FlipComparisonExpression(child_cond.comparison);
429
- break;
430
288
  }
289
+ found_all = found_all && found;
431
290
  }
432
291
 
433
- // no longer needs to be a delim join
434
- parent_delim_join.duplicate_eliminated_columns.clear();
435
- parent_delim_join.type = LogicalOperatorType::LOGICAL_COMPARISON_JOIN;
436
-
437
- return true;
292
+ return found_all;
438
293
  }
439
294
 
440
295
  } // namespace duckdb
@@ -47,11 +47,20 @@ unique_ptr<LogicalOperator> FilterPushdown::Rewrite(unique_ptr<LogicalOperator>
47
47
  }
48
48
  }
49
49
 
50
+ ClientContext &FilterPushdown::GetContext() {
51
+ return optimizer.GetContext();
52
+ }
53
+
50
54
  unique_ptr<LogicalOperator> FilterPushdown::PushdownJoin(unique_ptr<LogicalOperator> op) {
51
55
  D_ASSERT(op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN ||
52
56
  op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN || op->type == LogicalOperatorType::LOGICAL_ANY_JOIN ||
53
57
  op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN);
54
58
  auto &join = op->Cast<LogicalJoin>();
59
+ if (!join.left_projection_map.empty() || !join.right_projection_map.empty()) {
60
+ // cannot push down further otherwise the projection maps won't be preserved
61
+ return FinishPushdown(std::move(op));
62
+ }
63
+
55
64
  unordered_set<idx_t> left_bindings, right_bindings;
56
65
  LogicalJoin::GetTableReferences(*op->children[0], left_bindings);
57
66
  LogicalJoin::GetTableReferences(*op->children[1], right_bindings);
@@ -14,10 +14,17 @@ idx_t EstimatedProperties::GetCardinality() const {
14
14
  return MinValue<double>(cardinality, max_idx_t);
15
15
  }
16
16
 
17
+ template <>
17
18
  double EstimatedProperties::GetCost() const {
18
19
  return cost;
19
20
  }
20
21
 
22
+ template <>
23
+ idx_t EstimatedProperties::GetCost() const {
24
+ auto max_idx_t = NumericLimits<idx_t>::Maximum() - 10000;
25
+ return MinValue<double>(cost, max_idx_t);
26
+ }
27
+
21
28
  void EstimatedProperties::SetCardinality(double new_card) {
22
29
  cardinality = new_card;
23
30
  }
@@ -23,7 +23,7 @@ unique_ptr<EstimatedProperties> EstimatedProperties::Copy() {
23
23
  }
24
24
 
25
25
  double JoinNode::GetCost() {
26
- return estimated_props->GetCost();
26
+ return estimated_props->GetCost<double>();
27
27
  }
28
28
 
29
29
  void JoinNode::SetCost(double cost) {
@@ -54,7 +54,7 @@ string JoinNode::ToString() {
54
54
  is_cartesian = (GetCardinality<double>() == left->GetCardinality<double>() * right->GetCardinality<double>());
55
55
  }
56
56
  result += "cartesian = " + to_string(is_cartesian) + "\n";
57
- result += "cost = " + to_string(estimated_props->GetCost()) + "\n";
57
+ result += "cost = " + to_string(estimated_props->GetCost<double>()) + "\n";
58
58
  result += "left = \n";
59
59
  if (left) {
60
60
  result += left->ToString();