duckdb 0.8.2-dev145.0 → 0.8.2-dev1493.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (476) hide show
  1. package/binding.gyp +15 -12
  2. package/binding.gyp.in +1 -1
  3. package/configure.py +1 -1
  4. package/duckdb_extension_config.cmake +10 -0
  5. package/package.json +1 -1
  6. package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
  7. package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
  8. package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
  9. package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
  10. package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
  11. package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
  12. package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
  13. package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
  14. package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
  15. package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
  16. package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
  17. package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
  18. package/src/duckdb/extension/json/include/json_common.hpp +47 -231
  19. package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
  20. package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
  21. package/src/duckdb/extension/json/json_common.cpp +272 -40
  22. package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
  23. package/src/duckdb/extension/json/json_functions/json_transform.cpp +17 -37
  24. package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
  25. package/src/duckdb/extension/json/json_functions.cpp +24 -24
  26. package/src/duckdb/extension/json/json_scan.cpp +3 -6
  27. package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
  28. package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
  29. package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
  30. package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
  31. package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
  32. package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
  33. package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
  34. package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
  35. package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
  36. package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
  37. package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
  38. package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
  39. package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
  40. package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
  41. package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
  42. package/src/duckdb/extension/parquet/parquet_extension.cpp +192 -20
  43. package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
  44. package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
  45. package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
  46. package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
  47. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
  48. package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
  49. package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
  50. package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
  51. package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
  52. package/src/duckdb/src/common/allocator.cpp +14 -2
  53. package/src/duckdb/src/common/arrow/arrow_appender.cpp +5 -11
  54. package/src/duckdb/src/common/assert.cpp +3 -0
  55. package/src/duckdb/src/common/enum_util.cpp +42 -5
  56. package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
  57. package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
  58. package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
  59. package/src/duckdb/src/common/exception.cpp +2 -2
  60. package/src/duckdb/src/common/file_system.cpp +19 -0
  61. package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
  62. package/src/duckdb/src/common/local_file_system.cpp +2 -2
  63. package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
  64. package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
  65. package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
  66. package/src/duckdb/src/common/re2_regex.cpp +1 -1
  67. package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
  68. package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
  69. package/src/duckdb/src/common/sort/partition_state.cpp +44 -11
  70. package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
  71. package/src/duckdb/src/common/types/bit.cpp +51 -0
  72. package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
  73. package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
  74. package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
  75. package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
  76. package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
  77. package/src/duckdb/src/common/types/date.cpp +9 -0
  78. package/src/duckdb/src/common/types/list_segment.cpp +24 -74
  79. package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
  80. package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
  81. package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
  82. package/src/duckdb/src/common/types/uuid.cpp +2 -2
  83. package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
  84. package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
  85. package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
  86. package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
  87. package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
  88. package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
  89. package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
  90. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
  91. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
  92. package/src/duckdb/src/core_functions/function_list.cpp +4 -2
  93. package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
  94. package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
  95. package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
  96. package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
  97. package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
  98. package/src/duckdb/src/execution/expression_executor.cpp +1 -1
  99. package/src/duckdb/src/execution/index/art/art.cpp +149 -139
  100. package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
  101. package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
  102. package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
  103. package/src/duckdb/src/execution/index/art/node.cpp +113 -120
  104. package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
  105. package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
  106. package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
  107. package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
  108. package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
  109. package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
  110. package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
  111. package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
  112. package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
  113. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
  114. package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
  115. package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
  116. package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +414 -283
  117. package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
  118. package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
  119. package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
  120. package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +22 -3
  121. package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +2 -2
  122. package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
  123. package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
  124. package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
  125. package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
  126. package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
  127. package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
  128. package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
  129. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
  130. package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
  131. package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
  132. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
  133. package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
  134. package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
  135. package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
  136. package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
  137. package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
  138. package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
  139. package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
  140. package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
  141. package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
  142. package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
  143. package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
  144. package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
  145. package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
  146. package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
  147. package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
  148. package/src/duckdb/src/function/function.cpp +3 -1
  149. package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
  150. package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
  151. package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
  152. package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
  153. package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
  154. package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
  155. package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
  156. package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
  157. package/src/duckdb/src/function/table/read_csv.cpp +100 -17
  158. package/src/duckdb/src/function/table/table_scan.cpp +9 -0
  159. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  160. package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
  161. package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
  162. package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
  163. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +8 -0
  164. package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
  165. package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
  166. package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
  167. package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
  168. package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
  169. package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
  170. package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
  171. package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
  172. package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
  173. package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
  174. package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
  175. package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
  176. package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
  177. package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
  178. package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
  179. package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
  180. package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +13 -0
  181. package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
  182. package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
  183. package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
  184. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
  185. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
  186. package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
  187. package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
  188. package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
  189. package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
  190. package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +0 -1
  191. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
  192. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
  193. package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
  194. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
  195. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
  196. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
  197. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
  198. package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
  199. package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
  200. package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
  201. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
  202. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
  203. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
  204. package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
  205. package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
  206. package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
  207. package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
  208. package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
  209. package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
  210. package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
  211. package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
  212. package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
  213. package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
  214. package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
  215. package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
  216. package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
  217. package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
  218. package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
  219. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
  220. package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
  221. package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
  222. package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
  223. package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
  224. package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
  225. package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
  226. package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
  227. package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
  228. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +2 -10
  229. package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
  230. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
  231. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
  232. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
  233. package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
  234. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
  235. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
  236. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
  237. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
  238. package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
  239. package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
  240. package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
  241. package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
  242. package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
  243. package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
  244. package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
  245. package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
  246. package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
  247. package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
  248. package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
  249. package/src/duckdb/src/include/duckdb/main/client_config.hpp +3 -0
  250. package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
  251. package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
  252. package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
  253. package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
  254. package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
  255. package/src/duckdb/src/include/duckdb/main/settings.hpp +30 -1
  256. package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
  257. package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
  258. package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
  259. package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
  260. package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
  261. package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
  262. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
  263. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
  264. package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
  265. package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
  266. package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
  267. package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
  268. package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
  269. package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
  270. package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
  271. package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
  272. package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
  273. package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
  274. package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
  275. package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
  276. package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
  277. package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
  278. package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
  279. package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
  280. package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
  281. package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
  282. package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
  283. package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
  284. package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
  285. package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
  286. package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
  287. package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
  288. package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
  289. package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
  290. package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
  291. package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
  292. package/src/duckdb/src/include/duckdb/planner/binder.hpp +8 -5
  293. package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
  294. package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
  295. package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
  296. package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
  297. package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
  298. package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
  299. package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
  300. package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
  301. package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
  302. package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
  303. package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
  304. package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
  305. package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
  306. package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
  307. package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
  308. package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
  309. package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
  310. package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
  311. package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
  312. package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
  313. package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
  314. package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
  315. package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
  316. package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
  317. package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
  318. package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
  319. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
  320. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
  321. package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
  322. package/src/duckdb/src/include/duckdb.h +28 -0
  323. package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
  324. package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
  325. package/src/duckdb/src/main/config.cpp +3 -0
  326. package/src/duckdb/src/main/database.cpp +1 -1
  327. package/src/duckdb/src/main/extension/extension_helper.cpp +96 -89
  328. package/src/duckdb/src/main/extension/extension_install.cpp +6 -0
  329. package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
  330. package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
  331. package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
  332. package/src/duckdb/src/main/relation.cpp +6 -5
  333. package/src/duckdb/src/main/settings/settings.cpp +64 -18
  334. package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
  335. package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
  336. package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
  337. package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
  338. package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
  339. package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
  340. package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
  341. package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
  342. package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
  343. package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
  344. package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
  345. package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
  346. package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
  347. package/src/duckdb/src/optimizer/optimizer.cpp +51 -14
  348. package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
  349. package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
  350. package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
  351. package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
  352. package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
  353. package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
  354. package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
  355. package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
  356. package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
  357. package/src/duckdb/src/parallel/executor.cpp +15 -0
  358. package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
  359. package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
  360. package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
  361. package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
  362. package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
  363. package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
  364. package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
  365. package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
  366. package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
  367. package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
  368. package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
  369. package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
  370. package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
  371. package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
  372. package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
  373. package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
  374. package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
  375. package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
  376. package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
  377. package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
  378. package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
  379. package/src/duckdb/src/parser/parser.cpp +8 -2
  380. package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
  381. package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
  382. package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
  383. package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
  384. package/src/duckdb/src/parser/query_node.cpp +15 -37
  385. package/src/duckdb/src/parser/result_modifier.cpp +0 -74
  386. package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
  387. package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
  388. package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
  389. package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
  390. package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
  391. package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
  392. package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
  393. package/src/duckdb/src/parser/tableref.cpp +0 -44
  394. package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
  395. package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
  396. package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
  397. package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
  398. package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
  399. package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
  400. package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
  401. package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
  402. package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
  403. package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
  404. package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
  405. package/src/duckdb/src/parser/transformer.cpp +15 -0
  406. package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
  407. package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
  408. package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
  409. package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
  410. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
  411. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
  412. package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
  413. package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
  414. package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
  415. package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
  416. package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +61 -26
  417. package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
  418. package/src/duckdb/src/planner/binder.cpp +5 -0
  419. package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
  420. package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
  421. package/src/duckdb/src/planner/expression_binder.cpp +3 -0
  422. package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
  423. package/src/duckdb/src/planner/logical_operator.cpp +5 -0
  424. package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
  425. package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
  426. package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
  427. package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
  428. package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
  429. package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
  430. package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
  431. package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
  432. package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
  433. package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
  434. package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
  435. package/src/duckdb/src/storage/compression/rle.cpp +0 -1
  436. package/src/duckdb/src/storage/data_table.cpp +1 -1
  437. package/src/duckdb/src/storage/local_storage.cpp +3 -3
  438. package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
  439. package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
  440. package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
  441. package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
  442. package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
  443. package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
  444. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  445. package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
  446. package/src/duckdb/src/storage/table/row_group.cpp +25 -9
  447. package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
  448. package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
  449. package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
  450. package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
  451. package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
  452. package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
  453. package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
  454. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
  455. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
  456. package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
  457. package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
  458. package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
  459. package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
  460. package/src/duckdb/ub_src_function_scalar.cpp +2 -0
  461. package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
  462. package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
  463. package/src/duckdb/ub_src_optimizer.cpp +6 -0
  464. package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
  465. package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
  466. package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
  467. package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
  468. package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
  469. package/src/duckdb/ub_src_planner_operator.cpp +4 -0
  470. package/src/duckdb/ub_src_storage_serialization.cpp +8 -0
  471. package/src/statement.cpp +10 -3
  472. package/test/test_all_types.test.ts +233 -0
  473. package/tsconfig.json +1 -0
  474. package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
  475. package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
  476. package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -2,18 +2,18 @@
2
2
 
3
3
  #include "duckdb/catalog/catalog_search_path.hpp"
4
4
  #include "duckdb/common/string_util.hpp"
5
+ #include "duckdb/main/attached_database.hpp"
5
6
  #include "duckdb/main/client_context.hpp"
6
7
  #include "duckdb/main/client_data.hpp"
7
8
  #include "duckdb/main/config.hpp"
9
+ #include "duckdb/main/database.hpp"
10
+ #include "duckdb/main/database_manager.hpp"
8
11
  #include "duckdb/main/query_profiler.hpp"
9
12
  #include "duckdb/parallel/task_scheduler.hpp"
10
13
  #include "duckdb/parser/parser.hpp"
11
14
  #include "duckdb/planner/expression_binder.hpp"
12
15
  #include "duckdb/storage/buffer_manager.hpp"
13
16
  #include "duckdb/storage/storage_manager.hpp"
14
- #include "duckdb/main/database_manager.hpp"
15
- #include "duckdb/main/attached_database.hpp"
16
- #include "duckdb/main/database.hpp"
17
17
 
18
18
  namespace duckdb {
19
19
 
@@ -112,7 +112,6 @@ Value DebugCheckpointAbort::GetSetting(ClientContext &context) {
112
112
  //===--------------------------------------------------------------------===//
113
113
  // Debug Force External
114
114
  //===--------------------------------------------------------------------===//
115
-
116
115
  void DebugForceExternal::ResetLocal(ClientContext &context) {
117
116
  ClientConfig::GetConfig(context).force_external = ClientConfig().force_external;
118
117
  }
@@ -128,7 +127,6 @@ Value DebugForceExternal::GetSetting(ClientContext &context) {
128
127
  //===--------------------------------------------------------------------===//
129
128
  // Debug Force NoCrossProduct
130
129
  //===--------------------------------------------------------------------===//
131
-
132
130
  void DebugForceNoCrossProduct::ResetLocal(ClientContext &context) {
133
131
  ClientConfig::GetConfig(context).force_no_cross_product = ClientConfig().force_no_cross_product;
134
132
  }
@@ -144,7 +142,6 @@ Value DebugForceNoCrossProduct::GetSetting(ClientContext &context) {
144
142
  //===--------------------------------------------------------------------===//
145
143
  // Ordered Aggregate Threshold
146
144
  //===--------------------------------------------------------------------===//
147
-
148
145
  void OrderedAggregateThreshold::ResetLocal(ClientContext &context) {
149
146
  ClientConfig::GetConfig(context).ordered_aggregate_threshold = ClientConfig().ordered_aggregate_threshold;
150
147
  }
@@ -303,6 +300,30 @@ Value DefaultNullOrderSetting::GetSetting(ClientContext &context) {
303
300
  }
304
301
  }
305
302
 
303
+ //===--------------------------------------------------------------------===//
304
+ // Disabled File Systems
305
+ //===--------------------------------------------------------------------===//
306
+ void DisabledFileSystemsSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
307
+ if (!db) {
308
+ throw InternalException("disabled_filesystems can only be set in an active database");
309
+ }
310
+ auto &fs = FileSystem::GetFileSystem(*db);
311
+ auto list = StringUtil::Split(input.ToString(), ",");
312
+ fs.SetDisabledFileSystems(list);
313
+ }
314
+
315
+ void DisabledFileSystemsSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
316
+ if (!db) {
317
+ throw InternalException("disabled_filesystems can only be set in an active database");
318
+ }
319
+ auto &fs = FileSystem::GetFileSystem(*db);
320
+ fs.SetDisabledFileSystems(vector<string>());
321
+ }
322
+
323
+ Value DisabledFileSystemsSetting::GetSetting(ClientContext &context) {
324
+ return Value("");
325
+ }
326
+
306
327
  //===--------------------------------------------------------------------===//
307
328
  // Disabled Optimizer
308
329
  //===--------------------------------------------------------------------===//
@@ -478,7 +499,6 @@ Value EnableProfilingSetting::GetSetting(ClientContext &context) {
478
499
  //===--------------------------------------------------------------------===//
479
500
  // Custom Extension Repository
480
501
  //===--------------------------------------------------------------------===//
481
-
482
502
  void CustomExtensionRepository::ResetLocal(ClientContext &context) {
483
503
  ClientConfig::GetConfig(context).custom_extension_repo = ClientConfig().custom_extension_repo;
484
504
  }
@@ -494,7 +514,6 @@ Value CustomExtensionRepository::GetSetting(ClientContext &context) {
494
514
  //===--------------------------------------------------------------------===//
495
515
  // Enable Progress Bar
496
516
  //===--------------------------------------------------------------------===//
497
-
498
517
  void EnableProgressBarSetting::ResetLocal(ClientContext &context) {
499
518
  auto &config = ClientConfig::GetConfig(context);
500
519
  ProgressBar::SystemOverrideCheck(config);
@@ -548,7 +567,6 @@ Value ExperimentalParallelCSVSetting::GetSetting(ClientContext &context) {
548
567
  //===--------------------------------------------------------------------===//
549
568
  // Explain Output
550
569
  //===--------------------------------------------------------------------===//
551
-
552
570
  void ExplainOutputSetting::ResetLocal(ClientContext &context) {
553
571
  ClientConfig::GetConfig(context).explain_output_type = ClientConfig().explain_output_type;
554
572
  }
@@ -686,7 +704,6 @@ Value ForceBitpackingModeSetting::GetSetting(ClientContext &context) {
686
704
  //===--------------------------------------------------------------------===//
687
705
  // Home Directory
688
706
  //===--------------------------------------------------------------------===//
689
-
690
707
  void HomeDirectorySetting::ResetLocal(ClientContext &context) {
691
708
  ClientConfig::GetConfig(context).home_directory = ClientConfig().home_directory;
692
709
  }
@@ -780,7 +797,6 @@ Value ImmediateTransactionModeSetting::GetSetting(ClientContext &context) {
780
797
  //===--------------------------------------------------------------------===//
781
798
  // Maximum Expression Depth
782
799
  //===--------------------------------------------------------------------===//
783
-
784
800
  void MaximumExpressionDepthSetting::ResetLocal(ClientContext &context) {
785
801
  ClientConfig::GetConfig(context).max_expression_depth = ClientConfig().max_expression_depth;
786
802
  }
@@ -830,7 +846,6 @@ Value PasswordSetting::GetSetting(ClientContext &context) {
830
846
  //===--------------------------------------------------------------------===//
831
847
  // Perfect Hash Threshold
832
848
  //===--------------------------------------------------------------------===//
833
-
834
849
  void PerfectHashThresholdSetting::ResetLocal(ClientContext &context) {
835
850
  ClientConfig::GetConfig(context).perfect_ht_threshold = ClientConfig().perfect_ht_threshold;
836
851
  }
@@ -848,9 +863,23 @@ Value PerfectHashThresholdSetting::GetSetting(ClientContext &context) {
848
863
  }
849
864
 
850
865
  //===--------------------------------------------------------------------===//
851
- // Pivot Limit
866
+ // Pivot Filter Threshold
852
867
  //===--------------------------------------------------------------------===//
868
+ void PivotFilterThreshold::ResetLocal(ClientContext &context) {
869
+ ClientConfig::GetConfig(context).pivot_filter_threshold = ClientConfig().pivot_filter_threshold;
870
+ }
871
+
872
+ void PivotFilterThreshold::SetLocal(ClientContext &context, const Value &input) {
873
+ ClientConfig::GetConfig(context).pivot_filter_threshold = input.GetValue<uint64_t>();
874
+ }
853
875
 
876
+ Value PivotFilterThreshold::GetSetting(ClientContext &context) {
877
+ return Value::BIGINT(ClientConfig::GetConfig(context).pivot_filter_threshold);
878
+ }
879
+
880
+ //===--------------------------------------------------------------------===//
881
+ // Pivot Limit
882
+ //===--------------------------------------------------------------------===//
854
883
  void PivotLimitSetting::ResetLocal(ClientContext &context) {
855
884
  ClientConfig::GetConfig(context).pivot_limit = ClientConfig().pivot_limit;
856
885
  }
@@ -866,7 +895,6 @@ Value PivotLimitSetting::GetSetting(ClientContext &context) {
866
895
  //===--------------------------------------------------------------------===//
867
896
  // PreserveIdentifierCase
868
897
  //===--------------------------------------------------------------------===//
869
-
870
898
  void PreserveIdentifierCase::ResetLocal(ClientContext &context) {
871
899
  ClientConfig::GetConfig(context).preserve_identifier_case = ClientConfig().preserve_identifier_case;
872
900
  }
@@ -917,7 +945,6 @@ Value ExportLargeBufferArrow::GetSetting(ClientContext &context) {
917
945
  //===--------------------------------------------------------------------===//
918
946
  // Profiler History Size
919
947
  //===--------------------------------------------------------------------===//
920
-
921
948
  void ProfilerHistorySize::ResetLocal(ClientContext &context) {
922
949
  auto &client_data = ClientData::Get(context);
923
950
  client_data.query_profiler_history->ResetProfilerHistorySize();
@@ -939,7 +966,6 @@ Value ProfilerHistorySize::GetSetting(ClientContext &context) {
939
966
  //===--------------------------------------------------------------------===//
940
967
  // Profile Output
941
968
  //===--------------------------------------------------------------------===//
942
-
943
969
  void ProfileOutputSetting::ResetLocal(ClientContext &context) {
944
970
  ClientConfig::GetConfig(context).profiler_save_location = ClientConfig().profiler_save_location;
945
971
  }
@@ -958,7 +984,6 @@ Value ProfileOutputSetting::GetSetting(ClientContext &context) {
958
984
  //===--------------------------------------------------------------------===//
959
985
  // Profiling Mode
960
986
  //===--------------------------------------------------------------------===//
961
-
962
987
  void ProfilingModeSetting::ResetLocal(ClientContext &context) {
963
988
  ClientConfig::GetConfig(context).enable_profiler = ClientConfig().enable_profiler;
964
989
  ClientConfig::GetConfig(context).enable_detailed_profiling = ClientConfig().enable_detailed_profiling;
@@ -992,7 +1017,6 @@ Value ProfilingModeSetting::GetSetting(ClientContext &context) {
992
1017
  //===--------------------------------------------------------------------===//
993
1018
  // Progress Bar Time
994
1019
  //===--------------------------------------------------------------------===//
995
-
996
1020
  void ProgressBarTimeSetting::ResetLocal(ClientContext &context) {
997
1021
  auto &config = ClientConfig::GetConfig(context);
998
1022
  ProgressBar::SystemOverrideCheck(config);
@@ -1112,4 +1136,26 @@ Value UsernameSetting::GetSetting(ClientContext &context) {
1112
1136
  return Value();
1113
1137
  }
1114
1138
 
1139
+ //===--------------------------------------------------------------------===//
1140
+ // Allocator Flush Threshold
1141
+ //===--------------------------------------------------------------------===//
1142
+ void FlushAllocatorSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
1143
+ config.options.allocator_flush_threshold = DBConfig::ParseMemoryLimit(input.ToString());
1144
+ if (db) {
1145
+ TaskScheduler::GetScheduler(*db).SetAllocatorFlushTreshold(config.options.allocator_flush_threshold);
1146
+ }
1147
+ }
1148
+
1149
+ void FlushAllocatorSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
1150
+ config.options.allocator_flush_threshold = DBConfig().options.allocator_flush_threshold;
1151
+ if (db) {
1152
+ TaskScheduler::GetScheduler(*db).SetAllocatorFlushTreshold(config.options.allocator_flush_threshold);
1153
+ }
1154
+ }
1155
+
1156
+ Value FlushAllocatorSetting::GetSetting(ClientContext &context) {
1157
+ auto &config = DBConfig::GetConfig(context);
1158
+ return Value(StringUtil::BytesToHumanReadableString(config.options.allocator_flush_threshold));
1159
+ }
1160
+
1115
1161
  } // namespace duckdb
@@ -0,0 +1,43 @@
1
+ #include "duckdb/optimizer/column_binding_replacer.hpp"
2
+
3
+ #include "duckdb/planner/expression/bound_columnref_expression.hpp"
4
+
5
+ namespace duckdb {
6
+
7
+ ReplacementBinding::ReplacementBinding(ColumnBinding old_binding, ColumnBinding new_binding)
8
+ : old_binding(old_binding), new_binding(new_binding), replace_type(false) {
9
+ }
10
+
11
+ ReplacementBinding::ReplacementBinding(ColumnBinding old_binding, ColumnBinding new_binding, LogicalType new_type)
12
+ : old_binding(old_binding), new_binding(new_binding), replace_type(true), new_type(std::move(new_type)) {
13
+ }
14
+
15
+ ColumnBindingReplacer::ColumnBindingReplacer() {
16
+ }
17
+
18
+ void ColumnBindingReplacer::VisitOperator(LogicalOperator &op) {
19
+ if (stop_operator && stop_operator.get() == &op) {
20
+ return;
21
+ }
22
+ VisitOperatorChildren(op);
23
+ VisitOperatorExpressions(op);
24
+ }
25
+
26
+ void ColumnBindingReplacer::VisitExpression(unique_ptr<Expression> *expression) {
27
+ auto &expr = *expression;
28
+ if (expr->expression_class == ExpressionClass::BOUND_COLUMN_REF) {
29
+ auto &bound_column_ref = expr->Cast<BoundColumnRefExpression>();
30
+ for (const auto &replace_binding : replacement_bindings) {
31
+ if (bound_column_ref.binding == replace_binding.old_binding) {
32
+ bound_column_ref.binding = replace_binding.new_binding;
33
+ if (replace_binding.replace_type) {
34
+ bound_column_ref.return_type = replace_binding.new_type;
35
+ }
36
+ }
37
+ }
38
+ }
39
+
40
+ VisitExpressionChildren(**expression);
41
+ }
42
+
43
+ } // namespace duckdb
@@ -1,7 +1,5 @@
1
1
  #include "duckdb/optimizer/column_lifetime_optimizer.hpp"
2
-
3
2
  #include "duckdb/planner/expression/bound_columnref_expression.hpp"
4
-
5
3
  #include "duckdb/planner/operator/logical_comparison_join.hpp"
6
4
  #include "duckdb/planner/operator/logical_delim_join.hpp"
7
5
  #include "duckdb/planner/operator/logical_filter.hpp"
@@ -20,6 +18,7 @@ void ColumnLifetimeAnalyzer::ExtractUnusedColumnBindings(vector<ColumnBinding> b
20
18
  void ColumnLifetimeAnalyzer::GenerateProjectionMap(vector<ColumnBinding> bindings,
21
19
  column_binding_set_t &unused_bindings,
22
20
  vector<idx_t> &projection_map) {
21
+ projection_map.clear();
23
22
  if (unused_bindings.empty()) {
24
23
  return;
25
24
  }
@@ -0,0 +1,140 @@
1
+ #include "duckdb/optimizer/compressed_materialization.hpp"
2
+ #include "duckdb/planner/expression/bound_aggregate_expression.hpp"
3
+ #include "duckdb/planner/expression/bound_columnref_expression.hpp"
4
+ #include "duckdb/planner/operator/logical_aggregate.hpp"
5
+
6
+ namespace duckdb {
7
+
8
+ void CompressedMaterialization::CompressAggregate(unique_ptr<LogicalOperator> &op) {
9
+ auto &aggregate = op->Cast<LogicalAggregate>();
10
+ auto &groups = aggregate.groups;
11
+ column_binding_set_t group_binding_set;
12
+ for (const auto &group : groups) {
13
+ if (group->type != ExpressionType::BOUND_COLUMN_REF) {
14
+ continue;
15
+ }
16
+ auto &colref = group->Cast<BoundColumnRefExpression>();
17
+ if (group_binding_set.find(colref.binding) != group_binding_set.end()) {
18
+ return; // Duplicate group - don't compress
19
+ }
20
+ group_binding_set.insert(colref.binding);
21
+ }
22
+ auto &group_stats = aggregate.group_stats;
23
+
24
+ // No need to compress if there are no groups/stats
25
+ if (groups.empty() || group_stats.empty()) {
26
+ return;
27
+ }
28
+ D_ASSERT(groups.size() == group_stats.size());
29
+
30
+ // Find all bindings referenced by non-colref expressions in the groups
31
+ // These are excluded from compression by projection
32
+ // But we can try to compress the expression directly
33
+ column_binding_set_t referenced_bindings;
34
+ vector<ColumnBinding> group_bindings(groups.size(), ColumnBinding());
35
+ vector<bool> needs_decompression(groups.size(), false);
36
+ vector<unique_ptr<BaseStatistics>> stored_group_stats;
37
+ stored_group_stats.resize(groups.size());
38
+ for (idx_t group_idx = 0; group_idx < groups.size(); group_idx++) {
39
+ auto &group_expr = *groups[group_idx];
40
+ if (group_expr.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) {
41
+ auto &colref = group_expr.Cast<BoundColumnRefExpression>();
42
+ group_bindings[group_idx] = colref.binding;
43
+ continue; // Will be compressed generically
44
+ }
45
+
46
+ // Mark the bindings referenced by the non-colref expression so they won't be modified
47
+ GetReferencedBindings(group_expr, referenced_bindings);
48
+
49
+ // The non-colref expression won't be compressed generically, so try to compress it here
50
+ if (!group_stats[group_idx]) {
51
+ continue; // Can't compress without stats
52
+ }
53
+
54
+ // Try to compress, if successful, replace the expression
55
+ auto compress_expr = GetCompressExpression(group_expr.Copy(), *group_stats[group_idx]);
56
+ if (compress_expr) {
57
+ needs_decompression[group_idx] = true;
58
+ stored_group_stats[group_idx] = std::move(group_stats[group_idx]);
59
+ groups[group_idx] = std::move(compress_expr->expression);
60
+ group_stats[group_idx] = std::move(compress_expr->stats);
61
+ }
62
+ }
63
+
64
+ // Anything referenced in the aggregate functions is also excluded
65
+ for (idx_t expr_idx = 0; expr_idx < aggregate.expressions.size(); expr_idx++) {
66
+ const auto &expr = *aggregate.expressions[expr_idx];
67
+ D_ASSERT(expr.type == ExpressionType::BOUND_AGGREGATE);
68
+ const auto &aggr_expr = expr.Cast<BoundAggregateExpression>();
69
+ for (const auto &child : aggr_expr.children) {
70
+ GetReferencedBindings(*child, referenced_bindings);
71
+ }
72
+ if (aggr_expr.filter) {
73
+ GetReferencedBindings(*aggr_expr.filter, referenced_bindings);
74
+ }
75
+ if (aggr_expr.order_bys) {
76
+ for (const auto &order : aggr_expr.order_bys->orders) {
77
+ const auto &order_expr = *order.expression;
78
+ if (order_expr.type != ExpressionType::BOUND_COLUMN_REF) {
79
+ GetReferencedBindings(order_expr, referenced_bindings);
80
+ }
81
+ }
82
+ }
83
+ }
84
+
85
+ // Create info for compression
86
+ CompressedMaterializationInfo info(*op, {0}, referenced_bindings);
87
+
88
+ // Create binding mapping
89
+ const auto bindings_out = aggregate.GetColumnBindings();
90
+ const auto &types = aggregate.types;
91
+ for (idx_t group_idx = 0; group_idx < groups.size(); group_idx++) {
92
+ // Aggregate changes bindings as it has a table idx
93
+ CMBindingInfo binding_info(bindings_out[group_idx], types[group_idx]);
94
+ binding_info.needs_decompression = needs_decompression[group_idx];
95
+ if (needs_decompression[group_idx]) {
96
+ // Compressed non-generically
97
+ auto entry = info.binding_map.emplace(bindings_out[group_idx], std::move(binding_info));
98
+ entry.first->second.stats = std::move(stored_group_stats[group_idx]);
99
+ } else if (group_bindings[group_idx] != ColumnBinding()) {
100
+ info.binding_map.emplace(group_bindings[group_idx], std::move(binding_info));
101
+ }
102
+ }
103
+
104
+ // Now try to compress
105
+ CreateProjections(op, info);
106
+
107
+ // Update aggregate statistics
108
+ UpdateAggregateStats(op);
109
+ }
110
+
111
+ void CompressedMaterialization::UpdateAggregateStats(unique_ptr<LogicalOperator> &op) {
112
+ if (op->type != LogicalOperatorType::LOGICAL_PROJECTION) {
113
+ return;
114
+ }
115
+
116
+ // Update aggregate group stats if compressed
117
+ auto &compressed_aggregate = op->children[0]->Cast<LogicalAggregate>();
118
+ auto &groups = compressed_aggregate.groups;
119
+ auto &group_stats = compressed_aggregate.group_stats;
120
+
121
+ for (idx_t group_idx = 0; group_idx < groups.size(); group_idx++) {
122
+ auto &group_expr = *groups[group_idx];
123
+ if (group_expr.GetExpressionType() != ExpressionType::BOUND_COLUMN_REF) {
124
+ continue;
125
+ }
126
+ auto &colref = group_expr.Cast<BoundColumnRefExpression>();
127
+ if (!group_stats[group_idx]) {
128
+ continue;
129
+ }
130
+ if (colref.return_type == group_stats[group_idx]->GetType()) {
131
+ continue;
132
+ }
133
+ auto it = statistics_map.find(colref.binding);
134
+ if (it != statistics_map.end() && it->second) {
135
+ group_stats[group_idx] = it->second->ToUnique();
136
+ }
137
+ }
138
+ }
139
+
140
+ } // namespace duckdb
@@ -0,0 +1,42 @@
1
+ #include "duckdb/optimizer/compressed_materialization.hpp"
2
+ #include "duckdb/planner/expression/bound_columnref_expression.hpp"
3
+ #include "duckdb/planner/operator/logical_distinct.hpp"
4
+
5
+ namespace duckdb {
6
+
7
+ void CompressedMaterialization::CompressDistinct(unique_ptr<LogicalOperator> &op) {
8
+ auto &distinct = op->Cast<LogicalDistinct>();
9
+ auto &distinct_targets = distinct.distinct_targets;
10
+
11
+ column_binding_set_t referenced_bindings;
12
+ for (auto &target : distinct_targets) {
13
+ if (target->type != ExpressionType::BOUND_COLUMN_REF) { // LCOV_EXCL_START
14
+ GetReferencedBindings(*target, referenced_bindings);
15
+ } // LCOV_EXCL_STOP
16
+ }
17
+
18
+ if (distinct.order_by) {
19
+ for (auto &order : distinct.order_by->orders) {
20
+ if (order.expression->type != ExpressionType::BOUND_COLUMN_REF) { // LCOV_EXCL_START
21
+ GetReferencedBindings(*order.expression, referenced_bindings);
22
+ } // LCOV_EXCL_STOP
23
+ }
24
+ }
25
+
26
+ // Create info for compression
27
+ CompressedMaterializationInfo info(*op, {0}, referenced_bindings);
28
+
29
+ // Create binding mapping
30
+ const auto bindings = distinct.GetColumnBindings();
31
+ const auto &types = distinct.types;
32
+ D_ASSERT(bindings.size() == types.size());
33
+ for (idx_t col_idx = 0; col_idx < bindings.size(); col_idx++) {
34
+ // Distinct does not change bindings, input binding is output binding
35
+ info.binding_map.emplace(bindings[col_idx], CMBindingInfo(bindings[col_idx], types[col_idx]));
36
+ }
37
+
38
+ // Now try to compress
39
+ CreateProjections(op, info);
40
+ }
41
+
42
+ } // namespace duckdb
@@ -0,0 +1,65 @@
1
+ #include "duckdb/optimizer/compressed_materialization.hpp"
2
+ #include "duckdb/planner/expression/bound_columnref_expression.hpp"
3
+ #include "duckdb/planner/operator/logical_order.hpp"
4
+
5
+ namespace duckdb {
6
+
7
+ void CompressedMaterialization::CompressOrder(unique_ptr<LogicalOperator> &op) {
8
+ auto &order = op->Cast<LogicalOrder>();
9
+
10
+ // Find all bindings referenced by non-colref expressions in the order nodes
11
+ // These are excluded from compression by projection
12
+ // But we can try to compress the expression directly
13
+ column_binding_set_t referenced_bindings;
14
+ for (idx_t order_node_idx = 0; order_node_idx < order.orders.size(); order_node_idx++) {
15
+ auto &bound_order = order.orders[order_node_idx];
16
+ auto &order_expression = *bound_order.expression;
17
+ if (order_expression.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) {
18
+ continue; // Will be compressed generically
19
+ }
20
+
21
+ // Mark the bindings referenced by the non-colref expression so they won't be modified
22
+ GetReferencedBindings(order_expression, referenced_bindings);
23
+ }
24
+
25
+ // Create info for compression
26
+ CompressedMaterializationInfo info(*op, {0}, referenced_bindings);
27
+
28
+ // Create binding mapping
29
+ const auto bindings = order.GetColumnBindings();
30
+ const auto &types = order.types;
31
+ D_ASSERT(bindings.size() == types.size());
32
+ for (idx_t col_idx = 0; col_idx < bindings.size(); col_idx++) {
33
+ // Order does not change bindings, input binding is output binding
34
+ info.binding_map.emplace(bindings[col_idx], CMBindingInfo(bindings[col_idx], types[col_idx]));
35
+ }
36
+
37
+ // Now try to compress
38
+ CreateProjections(op, info);
39
+
40
+ // Update order statistics
41
+ UpdateOrderStats(op);
42
+ }
43
+
44
+ void CompressedMaterialization::UpdateOrderStats(unique_ptr<LogicalOperator> &op) {
45
+ if (op->type != LogicalOperatorType::LOGICAL_PROJECTION) {
46
+ return;
47
+ }
48
+
49
+ // Update order stats if compressed
50
+ auto &compressed_order = op->children[0]->Cast<LogicalOrder>();
51
+ for (idx_t order_node_idx = 0; order_node_idx < compressed_order.orders.size(); order_node_idx++) {
52
+ auto &bound_order = compressed_order.orders[order_node_idx];
53
+ auto &order_expression = *bound_order.expression;
54
+ if (order_expression.GetExpressionType() != ExpressionType::BOUND_COLUMN_REF) {
55
+ continue;
56
+ }
57
+ auto &colref = order_expression.Cast<BoundColumnRefExpression>();
58
+ auto it = statistics_map.find(colref.binding);
59
+ if (it != statistics_map.end() && it->second) {
60
+ bound_order.stats = it->second->ToUnique();
61
+ }
62
+ }
63
+ }
64
+
65
+ } // namespace duckdb