duckdb 0.8.2-dev150.0 → 0.8.2-dev1549.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (489) hide show
  1. package/binding.gyp +15 -12
  2. package/binding.gyp.in +1 -1
  3. package/configure.py +1 -1
  4. package/duckdb_extension_config.cmake +10 -0
  5. package/package.json +1 -1
  6. package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
  7. package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
  8. package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
  9. package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
  10. package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
  11. package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
  12. package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
  13. package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
  14. package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
  15. package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
  16. package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
  17. package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
  18. package/src/duckdb/extension/json/include/json_common.hpp +47 -231
  19. package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
  20. package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
  21. package/src/duckdb/extension/json/json_common.cpp +272 -40
  22. package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
  23. package/src/duckdb/extension/json/json_functions/json_transform.cpp +17 -37
  24. package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
  25. package/src/duckdb/extension/json/json_functions.cpp +24 -24
  26. package/src/duckdb/extension/json/json_scan.cpp +3 -6
  27. package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
  28. package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
  29. package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
  30. package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
  31. package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
  32. package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
  33. package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
  34. package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
  35. package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
  36. package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
  37. package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
  38. package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
  39. package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
  40. package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
  41. package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
  42. package/src/duckdb/extension/parquet/parquet_extension.cpp +192 -20
  43. package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
  44. package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
  45. package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
  46. package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
  47. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
  48. package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
  49. package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
  50. package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
  51. package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
  52. package/src/duckdb/src/common/allocator.cpp +14 -2
  53. package/src/duckdb/src/common/arrow/arrow_appender.cpp +5 -11
  54. package/src/duckdb/src/common/assert.cpp +3 -0
  55. package/src/duckdb/src/common/enum_util.cpp +4619 -4446
  56. package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
  57. package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
  58. package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
  59. package/src/duckdb/src/common/exception.cpp +2 -2
  60. package/src/duckdb/src/common/extra_type_info.cpp +506 -0
  61. package/src/duckdb/src/common/file_system.cpp +19 -0
  62. package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
  63. package/src/duckdb/src/common/local_file_system.cpp +14 -14
  64. package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
  65. package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
  66. package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
  67. package/src/duckdb/src/common/re2_regex.cpp +1 -1
  68. package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
  69. package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
  70. package/src/duckdb/src/common/sort/partition_state.cpp +44 -11
  71. package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
  72. package/src/duckdb/src/common/types/bit.cpp +51 -0
  73. package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
  74. package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
  75. package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
  76. package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
  77. package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
  78. package/src/duckdb/src/common/types/date.cpp +9 -0
  79. package/src/duckdb/src/common/types/list_segment.cpp +24 -74
  80. package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
  81. package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
  82. package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
  83. package/src/duckdb/src/common/types/uuid.cpp +2 -2
  84. package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
  85. package/src/duckdb/src/common/types.cpp +8 -655
  86. package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
  87. package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
  88. package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
  89. package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
  90. package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
  91. package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
  92. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
  93. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
  94. package/src/duckdb/src/core_functions/function_list.cpp +4 -2
  95. package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
  96. package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
  97. package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
  98. package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
  99. package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
  100. package/src/duckdb/src/execution/expression_executor.cpp +1 -1
  101. package/src/duckdb/src/execution/index/art/art.cpp +149 -139
  102. package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
  103. package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
  104. package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
  105. package/src/duckdb/src/execution/index/art/node.cpp +113 -120
  106. package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
  107. package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
  108. package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
  109. package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
  110. package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
  111. package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
  112. package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
  113. package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
  114. package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
  115. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
  116. package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
  117. package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
  118. package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +414 -283
  119. package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
  120. package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
  121. package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +28 -12
  122. package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
  123. package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +23 -4
  124. package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +41 -5
  125. package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
  126. package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
  127. package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
  128. package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
  129. package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
  130. package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
  131. package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
  132. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
  133. package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
  134. package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
  135. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
  136. package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
  137. package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
  138. package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
  139. package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
  140. package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
  141. package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +13 -22
  142. package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +17 -13
  143. package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
  144. package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
  145. package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
  146. package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
  147. package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
  148. package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
  149. package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
  150. package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
  151. package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
  152. package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
  153. package/src/duckdb/src/function/function.cpp +3 -1
  154. package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
  155. package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
  156. package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
  157. package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
  158. package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
  159. package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
  160. package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
  161. package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
  162. package/src/duckdb/src/function/table/read_csv.cpp +100 -17
  163. package/src/duckdb/src/function/table/table_scan.cpp +9 -0
  164. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  165. package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
  166. package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
  167. package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
  168. package/src/duckdb/src/include/duckdb/common/dl.hpp +3 -1
  169. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +616 -584
  170. package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
  171. package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
  172. package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
  173. package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
  174. package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
  175. package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
  176. package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +219 -0
  177. package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
  178. package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
  179. package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
  180. package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
  181. package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
  182. package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
  183. package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
  184. package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
  185. package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
  186. package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
  187. package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +13 -0
  188. package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
  189. package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
  190. package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
  191. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
  192. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
  193. package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
  194. package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
  195. package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
  196. package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
  197. package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +0 -1
  198. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
  199. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
  200. package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
  201. package/src/duckdb/src/include/duckdb/common/types.hpp +1 -15
  202. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
  203. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
  204. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
  205. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
  206. package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
  207. package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
  208. package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
  209. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
  210. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
  211. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
  212. package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
  213. package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
  214. package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
  215. package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
  216. package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
  217. package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
  218. package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
  219. package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
  220. package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
  221. package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
  222. package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
  223. package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
  224. package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
  225. package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
  226. package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
  227. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
  228. package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
  229. package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
  230. package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
  231. package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
  232. package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
  233. package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
  234. package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
  235. package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
  236. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +2 -10
  237. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +1 -1
  238. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +1 -1
  239. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp +12 -1
  240. package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
  241. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
  242. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
  243. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
  244. package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
  245. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
  246. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
  247. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
  248. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
  249. package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
  250. package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
  251. package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
  252. package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
  253. package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
  254. package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
  255. package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
  256. package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
  257. package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
  258. package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
  259. package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
  260. package/src/duckdb/src/include/duckdb/main/client_config.hpp +5 -0
  261. package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
  262. package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
  263. package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
  264. package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
  265. package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
  266. package/src/duckdb/src/include/duckdb/main/settings.hpp +39 -1
  267. package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
  268. package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
  269. package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
  270. package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
  271. package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
  272. package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
  273. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
  274. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
  275. package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
  276. package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
  277. package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
  278. package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
  279. package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
  280. package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
  281. package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
  282. package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
  283. package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
  284. package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
  285. package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
  286. package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
  287. package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
  288. package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
  289. package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
  290. package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
  291. package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
  292. package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
  293. package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
  294. package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
  295. package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
  296. package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
  297. package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
  298. package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
  299. package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
  300. package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
  301. package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
  302. package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
  303. package/src/duckdb/src/include/duckdb/planner/binder.hpp +8 -5
  304. package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
  305. package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
  306. package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
  307. package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
  308. package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
  309. package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
  310. package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
  311. package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
  312. package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
  313. package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
  314. package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
  315. package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
  316. package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
  317. package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
  318. package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
  319. package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
  320. package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
  321. package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
  322. package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
  323. package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
  324. package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
  325. package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
  326. package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
  327. package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
  328. package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
  329. package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
  330. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
  331. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
  332. package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
  333. package/src/duckdb/src/include/duckdb.h +28 -0
  334. package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
  335. package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
  336. package/src/duckdb/src/main/config.cpp +4 -0
  337. package/src/duckdb/src/main/database.cpp +1 -1
  338. package/src/duckdb/src/main/extension/extension_helper.cpp +96 -89
  339. package/src/duckdb/src/main/extension/extension_install.cpp +6 -0
  340. package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
  341. package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
  342. package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
  343. package/src/duckdb/src/main/relation.cpp +6 -5
  344. package/src/duckdb/src/main/settings/settings.cpp +79 -18
  345. package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
  346. package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
  347. package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
  348. package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
  349. package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
  350. package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
  351. package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
  352. package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
  353. package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
  354. package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
  355. package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
  356. package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
  357. package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
  358. package/src/duckdb/src/optimizer/optimizer.cpp +51 -14
  359. package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
  360. package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
  361. package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
  362. package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
  363. package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
  364. package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
  365. package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
  366. package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
  367. package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
  368. package/src/duckdb/src/parallel/executor.cpp +15 -0
  369. package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
  370. package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
  371. package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
  372. package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
  373. package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
  374. package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
  375. package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
  376. package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
  377. package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
  378. package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
  379. package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
  380. package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
  381. package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
  382. package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
  383. package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
  384. package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
  385. package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
  386. package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
  387. package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
  388. package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
  389. package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
  390. package/src/duckdb/src/parser/parser.cpp +8 -2
  391. package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
  392. package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
  393. package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
  394. package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
  395. package/src/duckdb/src/parser/query_node.cpp +15 -37
  396. package/src/duckdb/src/parser/result_modifier.cpp +0 -74
  397. package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
  398. package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
  399. package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
  400. package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
  401. package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
  402. package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
  403. package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
  404. package/src/duckdb/src/parser/tableref.cpp +0 -44
  405. package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
  406. package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
  407. package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
  408. package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
  409. package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
  410. package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
  411. package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
  412. package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
  413. package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
  414. package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
  415. package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
  416. package/src/duckdb/src/parser/transformer.cpp +15 -0
  417. package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
  418. package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
  419. package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
  420. package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
  421. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
  422. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
  423. package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
  424. package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
  425. package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
  426. package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
  427. package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +61 -26
  428. package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
  429. package/src/duckdb/src/planner/binder.cpp +5 -0
  430. package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
  431. package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
  432. package/src/duckdb/src/planner/expression_binder.cpp +3 -0
  433. package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
  434. package/src/duckdb/src/planner/logical_operator.cpp +5 -0
  435. package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
  436. package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
  437. package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
  438. package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
  439. package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
  440. package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
  441. package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
  442. package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
  443. package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
  444. package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
  445. package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
  446. package/src/duckdb/src/storage/compression/rle.cpp +0 -1
  447. package/src/duckdb/src/storage/data_table.cpp +1 -1
  448. package/src/duckdb/src/storage/local_storage.cpp +3 -3
  449. package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
  450. package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
  451. package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
  452. package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
  453. package/src/duckdb/src/storage/serialization/serialize_types.cpp +127 -0
  454. package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
  455. package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
  456. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  457. package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
  458. package/src/duckdb/src/storage/table/row_group.cpp +25 -9
  459. package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
  460. package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
  461. package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
  462. package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
  463. package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
  464. package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
  465. package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
  466. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
  467. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
  468. package/src/duckdb/ub_src_common.cpp +2 -0
  469. package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
  470. package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
  471. package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
  472. package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
  473. package/src/duckdb/ub_src_function_scalar.cpp +2 -0
  474. package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
  475. package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
  476. package/src/duckdb/ub_src_optimizer.cpp +6 -0
  477. package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
  478. package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
  479. package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
  480. package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
  481. package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
  482. package/src/duckdb/ub_src_planner_operator.cpp +4 -0
  483. package/src/duckdb/ub_src_storage_serialization.cpp +10 -0
  484. package/src/statement.cpp +10 -3
  485. package/test/test_all_types.test.ts +233 -0
  486. package/tsconfig.json +1 -0
  487. package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
  488. package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
  489. package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -2,18 +2,18 @@
2
2
 
3
3
  #include "duckdb/catalog/catalog_search_path.hpp"
4
4
  #include "duckdb/common/string_util.hpp"
5
+ #include "duckdb/main/attached_database.hpp"
5
6
  #include "duckdb/main/client_context.hpp"
6
7
  #include "duckdb/main/client_data.hpp"
7
8
  #include "duckdb/main/config.hpp"
9
+ #include "duckdb/main/database.hpp"
10
+ #include "duckdb/main/database_manager.hpp"
8
11
  #include "duckdb/main/query_profiler.hpp"
9
12
  #include "duckdb/parallel/task_scheduler.hpp"
10
13
  #include "duckdb/parser/parser.hpp"
11
14
  #include "duckdb/planner/expression_binder.hpp"
12
15
  #include "duckdb/storage/buffer_manager.hpp"
13
16
  #include "duckdb/storage/storage_manager.hpp"
14
- #include "duckdb/main/database_manager.hpp"
15
- #include "duckdb/main/attached_database.hpp"
16
- #include "duckdb/main/database.hpp"
17
17
 
18
18
  namespace duckdb {
19
19
 
@@ -112,7 +112,6 @@ Value DebugCheckpointAbort::GetSetting(ClientContext &context) {
112
112
  //===--------------------------------------------------------------------===//
113
113
  // Debug Force External
114
114
  //===--------------------------------------------------------------------===//
115
-
116
115
  void DebugForceExternal::ResetLocal(ClientContext &context) {
117
116
  ClientConfig::GetConfig(context).force_external = ClientConfig().force_external;
118
117
  }
@@ -128,7 +127,6 @@ Value DebugForceExternal::GetSetting(ClientContext &context) {
128
127
  //===--------------------------------------------------------------------===//
129
128
  // Debug Force NoCrossProduct
130
129
  //===--------------------------------------------------------------------===//
131
-
132
130
  void DebugForceNoCrossProduct::ResetLocal(ClientContext &context) {
133
131
  ClientConfig::GetConfig(context).force_no_cross_product = ClientConfig().force_no_cross_product;
134
132
  }
@@ -144,7 +142,6 @@ Value DebugForceNoCrossProduct::GetSetting(ClientContext &context) {
144
142
  //===--------------------------------------------------------------------===//
145
143
  // Ordered Aggregate Threshold
146
144
  //===--------------------------------------------------------------------===//
147
-
148
145
  void OrderedAggregateThreshold::ResetLocal(ClientContext &context) {
149
146
  ClientConfig::GetConfig(context).ordered_aggregate_threshold = ClientConfig().ordered_aggregate_threshold;
150
147
  }
@@ -200,6 +197,21 @@ Value DebugAsOfIEJoin::GetSetting(ClientContext &context) {
200
197
  return Value::BOOLEAN(ClientConfig::GetConfig(context).force_asof_iejoin);
201
198
  }
202
199
 
200
+ //===--------------------------------------------------------------------===//
201
+ // Prefer Range Joins
202
+ //===--------------------------------------------------------------------===//
203
+ void PreferRangeJoins::ResetLocal(ClientContext &context) {
204
+ ClientConfig::GetConfig(context).prefer_range_joins = ClientConfig().prefer_range_joins;
205
+ }
206
+
207
+ void PreferRangeJoins::SetLocal(ClientContext &context, const Value &input) {
208
+ ClientConfig::GetConfig(context).prefer_range_joins = input.GetValue<bool>();
209
+ }
210
+
211
+ Value PreferRangeJoins::GetSetting(ClientContext &context) {
212
+ return Value::BOOLEAN(ClientConfig::GetConfig(context).prefer_range_joins);
213
+ }
214
+
203
215
  //===--------------------------------------------------------------------===//
204
216
  // Default Collation
205
217
  //===--------------------------------------------------------------------===//
@@ -303,6 +315,30 @@ Value DefaultNullOrderSetting::GetSetting(ClientContext &context) {
303
315
  }
304
316
  }
305
317
 
318
+ //===--------------------------------------------------------------------===//
319
+ // Disabled File Systems
320
+ //===--------------------------------------------------------------------===//
321
+ void DisabledFileSystemsSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
322
+ if (!db) {
323
+ throw InternalException("disabled_filesystems can only be set in an active database");
324
+ }
325
+ auto &fs = FileSystem::GetFileSystem(*db);
326
+ auto list = StringUtil::Split(input.ToString(), ",");
327
+ fs.SetDisabledFileSystems(list);
328
+ }
329
+
330
+ void DisabledFileSystemsSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
331
+ if (!db) {
332
+ throw InternalException("disabled_filesystems can only be set in an active database");
333
+ }
334
+ auto &fs = FileSystem::GetFileSystem(*db);
335
+ fs.SetDisabledFileSystems(vector<string>());
336
+ }
337
+
338
+ Value DisabledFileSystemsSetting::GetSetting(ClientContext &context) {
339
+ return Value("");
340
+ }
341
+
306
342
  //===--------------------------------------------------------------------===//
307
343
  // Disabled Optimizer
308
344
  //===--------------------------------------------------------------------===//
@@ -478,7 +514,6 @@ Value EnableProfilingSetting::GetSetting(ClientContext &context) {
478
514
  //===--------------------------------------------------------------------===//
479
515
  // Custom Extension Repository
480
516
  //===--------------------------------------------------------------------===//
481
-
482
517
  void CustomExtensionRepository::ResetLocal(ClientContext &context) {
483
518
  ClientConfig::GetConfig(context).custom_extension_repo = ClientConfig().custom_extension_repo;
484
519
  }
@@ -494,7 +529,6 @@ Value CustomExtensionRepository::GetSetting(ClientContext &context) {
494
529
  //===--------------------------------------------------------------------===//
495
530
  // Enable Progress Bar
496
531
  //===--------------------------------------------------------------------===//
497
-
498
532
  void EnableProgressBarSetting::ResetLocal(ClientContext &context) {
499
533
  auto &config = ClientConfig::GetConfig(context);
500
534
  ProgressBar::SystemOverrideCheck(config);
@@ -548,7 +582,6 @@ Value ExperimentalParallelCSVSetting::GetSetting(ClientContext &context) {
548
582
  //===--------------------------------------------------------------------===//
549
583
  // Explain Output
550
584
  //===--------------------------------------------------------------------===//
551
-
552
585
  void ExplainOutputSetting::ResetLocal(ClientContext &context) {
553
586
  ClientConfig::GetConfig(context).explain_output_type = ClientConfig().explain_output_type;
554
587
  }
@@ -686,7 +719,6 @@ Value ForceBitpackingModeSetting::GetSetting(ClientContext &context) {
686
719
  //===--------------------------------------------------------------------===//
687
720
  // Home Directory
688
721
  //===--------------------------------------------------------------------===//
689
-
690
722
  void HomeDirectorySetting::ResetLocal(ClientContext &context) {
691
723
  ClientConfig::GetConfig(context).home_directory = ClientConfig().home_directory;
692
724
  }
@@ -780,7 +812,6 @@ Value ImmediateTransactionModeSetting::GetSetting(ClientContext &context) {
780
812
  //===--------------------------------------------------------------------===//
781
813
  // Maximum Expression Depth
782
814
  //===--------------------------------------------------------------------===//
783
-
784
815
  void MaximumExpressionDepthSetting::ResetLocal(ClientContext &context) {
785
816
  ClientConfig::GetConfig(context).max_expression_depth = ClientConfig().max_expression_depth;
786
817
  }
@@ -830,7 +861,6 @@ Value PasswordSetting::GetSetting(ClientContext &context) {
830
861
  //===--------------------------------------------------------------------===//
831
862
  // Perfect Hash Threshold
832
863
  //===--------------------------------------------------------------------===//
833
-
834
864
  void PerfectHashThresholdSetting::ResetLocal(ClientContext &context) {
835
865
  ClientConfig::GetConfig(context).perfect_ht_threshold = ClientConfig().perfect_ht_threshold;
836
866
  }
@@ -848,9 +878,23 @@ Value PerfectHashThresholdSetting::GetSetting(ClientContext &context) {
848
878
  }
849
879
 
850
880
  //===--------------------------------------------------------------------===//
851
- // Pivot Limit
881
+ // Pivot Filter Threshold
852
882
  //===--------------------------------------------------------------------===//
883
+ void PivotFilterThreshold::ResetLocal(ClientContext &context) {
884
+ ClientConfig::GetConfig(context).pivot_filter_threshold = ClientConfig().pivot_filter_threshold;
885
+ }
886
+
887
+ void PivotFilterThreshold::SetLocal(ClientContext &context, const Value &input) {
888
+ ClientConfig::GetConfig(context).pivot_filter_threshold = input.GetValue<uint64_t>();
889
+ }
853
890
 
891
+ Value PivotFilterThreshold::GetSetting(ClientContext &context) {
892
+ return Value::BIGINT(ClientConfig::GetConfig(context).pivot_filter_threshold);
893
+ }
894
+
895
+ //===--------------------------------------------------------------------===//
896
+ // Pivot Limit
897
+ //===--------------------------------------------------------------------===//
854
898
  void PivotLimitSetting::ResetLocal(ClientContext &context) {
855
899
  ClientConfig::GetConfig(context).pivot_limit = ClientConfig().pivot_limit;
856
900
  }
@@ -866,7 +910,6 @@ Value PivotLimitSetting::GetSetting(ClientContext &context) {
866
910
  //===--------------------------------------------------------------------===//
867
911
  // PreserveIdentifierCase
868
912
  //===--------------------------------------------------------------------===//
869
-
870
913
  void PreserveIdentifierCase::ResetLocal(ClientContext &context) {
871
914
  ClientConfig::GetConfig(context).preserve_identifier_case = ClientConfig().preserve_identifier_case;
872
915
  }
@@ -917,7 +960,6 @@ Value ExportLargeBufferArrow::GetSetting(ClientContext &context) {
917
960
  //===--------------------------------------------------------------------===//
918
961
  // Profiler History Size
919
962
  //===--------------------------------------------------------------------===//
920
-
921
963
  void ProfilerHistorySize::ResetLocal(ClientContext &context) {
922
964
  auto &client_data = ClientData::Get(context);
923
965
  client_data.query_profiler_history->ResetProfilerHistorySize();
@@ -939,7 +981,6 @@ Value ProfilerHistorySize::GetSetting(ClientContext &context) {
939
981
  //===--------------------------------------------------------------------===//
940
982
  // Profile Output
941
983
  //===--------------------------------------------------------------------===//
942
-
943
984
  void ProfileOutputSetting::ResetLocal(ClientContext &context) {
944
985
  ClientConfig::GetConfig(context).profiler_save_location = ClientConfig().profiler_save_location;
945
986
  }
@@ -958,7 +999,6 @@ Value ProfileOutputSetting::GetSetting(ClientContext &context) {
958
999
  //===--------------------------------------------------------------------===//
959
1000
  // Profiling Mode
960
1001
  //===--------------------------------------------------------------------===//
961
-
962
1002
  void ProfilingModeSetting::ResetLocal(ClientContext &context) {
963
1003
  ClientConfig::GetConfig(context).enable_profiler = ClientConfig().enable_profiler;
964
1004
  ClientConfig::GetConfig(context).enable_detailed_profiling = ClientConfig().enable_detailed_profiling;
@@ -992,7 +1032,6 @@ Value ProfilingModeSetting::GetSetting(ClientContext &context) {
992
1032
  //===--------------------------------------------------------------------===//
993
1033
  // Progress Bar Time
994
1034
  //===--------------------------------------------------------------------===//
995
-
996
1035
  void ProgressBarTimeSetting::ResetLocal(ClientContext &context) {
997
1036
  auto &config = ClientConfig::GetConfig(context);
998
1037
  ProgressBar::SystemOverrideCheck(config);
@@ -1112,4 +1151,26 @@ Value UsernameSetting::GetSetting(ClientContext &context) {
1112
1151
  return Value();
1113
1152
  }
1114
1153
 
1154
+ //===--------------------------------------------------------------------===//
1155
+ // Allocator Flush Threshold
1156
+ //===--------------------------------------------------------------------===//
1157
+ void FlushAllocatorSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
1158
+ config.options.allocator_flush_threshold = DBConfig::ParseMemoryLimit(input.ToString());
1159
+ if (db) {
1160
+ TaskScheduler::GetScheduler(*db).SetAllocatorFlushTreshold(config.options.allocator_flush_threshold);
1161
+ }
1162
+ }
1163
+
1164
+ void FlushAllocatorSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
1165
+ config.options.allocator_flush_threshold = DBConfig().options.allocator_flush_threshold;
1166
+ if (db) {
1167
+ TaskScheduler::GetScheduler(*db).SetAllocatorFlushTreshold(config.options.allocator_flush_threshold);
1168
+ }
1169
+ }
1170
+
1171
+ Value FlushAllocatorSetting::GetSetting(ClientContext &context) {
1172
+ auto &config = DBConfig::GetConfig(context);
1173
+ return Value(StringUtil::BytesToHumanReadableString(config.options.allocator_flush_threshold));
1174
+ }
1175
+
1115
1176
  } // namespace duckdb
@@ -0,0 +1,43 @@
1
+ #include "duckdb/optimizer/column_binding_replacer.hpp"
2
+
3
+ #include "duckdb/planner/expression/bound_columnref_expression.hpp"
4
+
5
+ namespace duckdb {
6
+
7
+ ReplacementBinding::ReplacementBinding(ColumnBinding old_binding, ColumnBinding new_binding)
8
+ : old_binding(old_binding), new_binding(new_binding), replace_type(false) {
9
+ }
10
+
11
+ ReplacementBinding::ReplacementBinding(ColumnBinding old_binding, ColumnBinding new_binding, LogicalType new_type)
12
+ : old_binding(old_binding), new_binding(new_binding), replace_type(true), new_type(std::move(new_type)) {
13
+ }
14
+
15
+ ColumnBindingReplacer::ColumnBindingReplacer() {
16
+ }
17
+
18
+ void ColumnBindingReplacer::VisitOperator(LogicalOperator &op) {
19
+ if (stop_operator && stop_operator.get() == &op) {
20
+ return;
21
+ }
22
+ VisitOperatorChildren(op);
23
+ VisitOperatorExpressions(op);
24
+ }
25
+
26
+ void ColumnBindingReplacer::VisitExpression(unique_ptr<Expression> *expression) {
27
+ auto &expr = *expression;
28
+ if (expr->expression_class == ExpressionClass::BOUND_COLUMN_REF) {
29
+ auto &bound_column_ref = expr->Cast<BoundColumnRefExpression>();
30
+ for (const auto &replace_binding : replacement_bindings) {
31
+ if (bound_column_ref.binding == replace_binding.old_binding) {
32
+ bound_column_ref.binding = replace_binding.new_binding;
33
+ if (replace_binding.replace_type) {
34
+ bound_column_ref.return_type = replace_binding.new_type;
35
+ }
36
+ }
37
+ }
38
+ }
39
+
40
+ VisitExpressionChildren(**expression);
41
+ }
42
+
43
+ } // namespace duckdb
@@ -1,7 +1,5 @@
1
1
  #include "duckdb/optimizer/column_lifetime_optimizer.hpp"
2
-
3
2
  #include "duckdb/planner/expression/bound_columnref_expression.hpp"
4
-
5
3
  #include "duckdb/planner/operator/logical_comparison_join.hpp"
6
4
  #include "duckdb/planner/operator/logical_delim_join.hpp"
7
5
  #include "duckdb/planner/operator/logical_filter.hpp"
@@ -20,6 +18,7 @@ void ColumnLifetimeAnalyzer::ExtractUnusedColumnBindings(vector<ColumnBinding> b
20
18
  void ColumnLifetimeAnalyzer::GenerateProjectionMap(vector<ColumnBinding> bindings,
21
19
  column_binding_set_t &unused_bindings,
22
20
  vector<idx_t> &projection_map) {
21
+ projection_map.clear();
23
22
  if (unused_bindings.empty()) {
24
23
  return;
25
24
  }
@@ -0,0 +1,140 @@
1
+ #include "duckdb/optimizer/compressed_materialization.hpp"
2
+ #include "duckdb/planner/expression/bound_aggregate_expression.hpp"
3
+ #include "duckdb/planner/expression/bound_columnref_expression.hpp"
4
+ #include "duckdb/planner/operator/logical_aggregate.hpp"
5
+
6
+ namespace duckdb {
7
+
8
+ void CompressedMaterialization::CompressAggregate(unique_ptr<LogicalOperator> &op) {
9
+ auto &aggregate = op->Cast<LogicalAggregate>();
10
+ auto &groups = aggregate.groups;
11
+ column_binding_set_t group_binding_set;
12
+ for (const auto &group : groups) {
13
+ if (group->type != ExpressionType::BOUND_COLUMN_REF) {
14
+ continue;
15
+ }
16
+ auto &colref = group->Cast<BoundColumnRefExpression>();
17
+ if (group_binding_set.find(colref.binding) != group_binding_set.end()) {
18
+ return; // Duplicate group - don't compress
19
+ }
20
+ group_binding_set.insert(colref.binding);
21
+ }
22
+ auto &group_stats = aggregate.group_stats;
23
+
24
+ // No need to compress if there are no groups/stats
25
+ if (groups.empty() || group_stats.empty()) {
26
+ return;
27
+ }
28
+ D_ASSERT(groups.size() == group_stats.size());
29
+
30
+ // Find all bindings referenced by non-colref expressions in the groups
31
+ // These are excluded from compression by projection
32
+ // But we can try to compress the expression directly
33
+ column_binding_set_t referenced_bindings;
34
+ vector<ColumnBinding> group_bindings(groups.size(), ColumnBinding());
35
+ vector<bool> needs_decompression(groups.size(), false);
36
+ vector<unique_ptr<BaseStatistics>> stored_group_stats;
37
+ stored_group_stats.resize(groups.size());
38
+ for (idx_t group_idx = 0; group_idx < groups.size(); group_idx++) {
39
+ auto &group_expr = *groups[group_idx];
40
+ if (group_expr.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) {
41
+ auto &colref = group_expr.Cast<BoundColumnRefExpression>();
42
+ group_bindings[group_idx] = colref.binding;
43
+ continue; // Will be compressed generically
44
+ }
45
+
46
+ // Mark the bindings referenced by the non-colref expression so they won't be modified
47
+ GetReferencedBindings(group_expr, referenced_bindings);
48
+
49
+ // The non-colref expression won't be compressed generically, so try to compress it here
50
+ if (!group_stats[group_idx]) {
51
+ continue; // Can't compress without stats
52
+ }
53
+
54
+ // Try to compress, if successful, replace the expression
55
+ auto compress_expr = GetCompressExpression(group_expr.Copy(), *group_stats[group_idx]);
56
+ if (compress_expr) {
57
+ needs_decompression[group_idx] = true;
58
+ stored_group_stats[group_idx] = std::move(group_stats[group_idx]);
59
+ groups[group_idx] = std::move(compress_expr->expression);
60
+ group_stats[group_idx] = std::move(compress_expr->stats);
61
+ }
62
+ }
63
+
64
+ // Anything referenced in the aggregate functions is also excluded
65
+ for (idx_t expr_idx = 0; expr_idx < aggregate.expressions.size(); expr_idx++) {
66
+ const auto &expr = *aggregate.expressions[expr_idx];
67
+ D_ASSERT(expr.type == ExpressionType::BOUND_AGGREGATE);
68
+ const auto &aggr_expr = expr.Cast<BoundAggregateExpression>();
69
+ for (const auto &child : aggr_expr.children) {
70
+ GetReferencedBindings(*child, referenced_bindings);
71
+ }
72
+ if (aggr_expr.filter) {
73
+ GetReferencedBindings(*aggr_expr.filter, referenced_bindings);
74
+ }
75
+ if (aggr_expr.order_bys) {
76
+ for (const auto &order : aggr_expr.order_bys->orders) {
77
+ const auto &order_expr = *order.expression;
78
+ if (order_expr.type != ExpressionType::BOUND_COLUMN_REF) {
79
+ GetReferencedBindings(order_expr, referenced_bindings);
80
+ }
81
+ }
82
+ }
83
+ }
84
+
85
+ // Create info for compression
86
+ CompressedMaterializationInfo info(*op, {0}, referenced_bindings);
87
+
88
+ // Create binding mapping
89
+ const auto bindings_out = aggregate.GetColumnBindings();
90
+ const auto &types = aggregate.types;
91
+ for (idx_t group_idx = 0; group_idx < groups.size(); group_idx++) {
92
+ // Aggregate changes bindings as it has a table idx
93
+ CMBindingInfo binding_info(bindings_out[group_idx], types[group_idx]);
94
+ binding_info.needs_decompression = needs_decompression[group_idx];
95
+ if (needs_decompression[group_idx]) {
96
+ // Compressed non-generically
97
+ auto entry = info.binding_map.emplace(bindings_out[group_idx], std::move(binding_info));
98
+ entry.first->second.stats = std::move(stored_group_stats[group_idx]);
99
+ } else if (group_bindings[group_idx] != ColumnBinding()) {
100
+ info.binding_map.emplace(group_bindings[group_idx], std::move(binding_info));
101
+ }
102
+ }
103
+
104
+ // Now try to compress
105
+ CreateProjections(op, info);
106
+
107
+ // Update aggregate statistics
108
+ UpdateAggregateStats(op);
109
+ }
110
+
111
+ void CompressedMaterialization::UpdateAggregateStats(unique_ptr<LogicalOperator> &op) {
112
+ if (op->type != LogicalOperatorType::LOGICAL_PROJECTION) {
113
+ return;
114
+ }
115
+
116
+ // Update aggregate group stats if compressed
117
+ auto &compressed_aggregate = op->children[0]->Cast<LogicalAggregate>();
118
+ auto &groups = compressed_aggregate.groups;
119
+ auto &group_stats = compressed_aggregate.group_stats;
120
+
121
+ for (idx_t group_idx = 0; group_idx < groups.size(); group_idx++) {
122
+ auto &group_expr = *groups[group_idx];
123
+ if (group_expr.GetExpressionType() != ExpressionType::BOUND_COLUMN_REF) {
124
+ continue;
125
+ }
126
+ auto &colref = group_expr.Cast<BoundColumnRefExpression>();
127
+ if (!group_stats[group_idx]) {
128
+ continue;
129
+ }
130
+ if (colref.return_type == group_stats[group_idx]->GetType()) {
131
+ continue;
132
+ }
133
+ auto it = statistics_map.find(colref.binding);
134
+ if (it != statistics_map.end() && it->second) {
135
+ group_stats[group_idx] = it->second->ToUnique();
136
+ }
137
+ }
138
+ }
139
+
140
+ } // namespace duckdb
@@ -0,0 +1,42 @@
1
+ #include "duckdb/optimizer/compressed_materialization.hpp"
2
+ #include "duckdb/planner/expression/bound_columnref_expression.hpp"
3
+ #include "duckdb/planner/operator/logical_distinct.hpp"
4
+
5
+ namespace duckdb {
6
+
7
+ void CompressedMaterialization::CompressDistinct(unique_ptr<LogicalOperator> &op) {
8
+ auto &distinct = op->Cast<LogicalDistinct>();
9
+ auto &distinct_targets = distinct.distinct_targets;
10
+
11
+ column_binding_set_t referenced_bindings;
12
+ for (auto &target : distinct_targets) {
13
+ if (target->type != ExpressionType::BOUND_COLUMN_REF) { // LCOV_EXCL_START
14
+ GetReferencedBindings(*target, referenced_bindings);
15
+ } // LCOV_EXCL_STOP
16
+ }
17
+
18
+ if (distinct.order_by) {
19
+ for (auto &order : distinct.order_by->orders) {
20
+ if (order.expression->type != ExpressionType::BOUND_COLUMN_REF) { // LCOV_EXCL_START
21
+ GetReferencedBindings(*order.expression, referenced_bindings);
22
+ } // LCOV_EXCL_STOP
23
+ }
24
+ }
25
+
26
+ // Create info for compression
27
+ CompressedMaterializationInfo info(*op, {0}, referenced_bindings);
28
+
29
+ // Create binding mapping
30
+ const auto bindings = distinct.GetColumnBindings();
31
+ const auto &types = distinct.types;
32
+ D_ASSERT(bindings.size() == types.size());
33
+ for (idx_t col_idx = 0; col_idx < bindings.size(); col_idx++) {
34
+ // Distinct does not change bindings, input binding is output binding
35
+ info.binding_map.emplace(bindings[col_idx], CMBindingInfo(bindings[col_idx], types[col_idx]));
36
+ }
37
+
38
+ // Now try to compress
39
+ CreateProjections(op, info);
40
+ }
41
+
42
+ } // namespace duckdb
@@ -0,0 +1,65 @@
1
+ #include "duckdb/optimizer/compressed_materialization.hpp"
2
+ #include "duckdb/planner/expression/bound_columnref_expression.hpp"
3
+ #include "duckdb/planner/operator/logical_order.hpp"
4
+
5
+ namespace duckdb {
6
+
7
+ void CompressedMaterialization::CompressOrder(unique_ptr<LogicalOperator> &op) {
8
+ auto &order = op->Cast<LogicalOrder>();
9
+
10
+ // Find all bindings referenced by non-colref expressions in the order nodes
11
+ // These are excluded from compression by projection
12
+ // But we can try to compress the expression directly
13
+ column_binding_set_t referenced_bindings;
14
+ for (idx_t order_node_idx = 0; order_node_idx < order.orders.size(); order_node_idx++) {
15
+ auto &bound_order = order.orders[order_node_idx];
16
+ auto &order_expression = *bound_order.expression;
17
+ if (order_expression.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) {
18
+ continue; // Will be compressed generically
19
+ }
20
+
21
+ // Mark the bindings referenced by the non-colref expression so they won't be modified
22
+ GetReferencedBindings(order_expression, referenced_bindings);
23
+ }
24
+
25
+ // Create info for compression
26
+ CompressedMaterializationInfo info(*op, {0}, referenced_bindings);
27
+
28
+ // Create binding mapping
29
+ const auto bindings = order.GetColumnBindings();
30
+ const auto &types = order.types;
31
+ D_ASSERT(bindings.size() == types.size());
32
+ for (idx_t col_idx = 0; col_idx < bindings.size(); col_idx++) {
33
+ // Order does not change bindings, input binding is output binding
34
+ info.binding_map.emplace(bindings[col_idx], CMBindingInfo(bindings[col_idx], types[col_idx]));
35
+ }
36
+
37
+ // Now try to compress
38
+ CreateProjections(op, info);
39
+
40
+ // Update order statistics
41
+ UpdateOrderStats(op);
42
+ }
43
+
44
+ void CompressedMaterialization::UpdateOrderStats(unique_ptr<LogicalOperator> &op) {
45
+ if (op->type != LogicalOperatorType::LOGICAL_PROJECTION) {
46
+ return;
47
+ }
48
+
49
+ // Update order stats if compressed
50
+ auto &compressed_order = op->children[0]->Cast<LogicalOrder>();
51
+ for (idx_t order_node_idx = 0; order_node_idx < compressed_order.orders.size(); order_node_idx++) {
52
+ auto &bound_order = compressed_order.orders[order_node_idx];
53
+ auto &order_expression = *bound_order.expression;
54
+ if (order_expression.GetExpressionType() != ExpressionType::BOUND_COLUMN_REF) {
55
+ continue;
56
+ }
57
+ auto &colref = order_expression.Cast<BoundColumnRefExpression>();
58
+ auto it = statistics_map.find(colref.binding);
59
+ if (it != statistics_map.end() && it->second) {
60
+ bound_order.stats = it->second->ToUnique();
61
+ }
62
+ }
63
+ }
64
+
65
+ } // namespace duckdb