duckdb 0.8.2-dev161.0 → 0.8.2-dev1764.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (504) hide show
  1. package/binding.gyp +15 -12
  2. package/binding.gyp.in +1 -1
  3. package/configure.py +1 -1
  4. package/duckdb_extension_config.cmake +10 -0
  5. package/package.json +1 -1
  6. package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
  7. package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
  8. package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
  9. package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
  10. package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
  11. package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
  12. package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
  13. package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
  14. package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
  15. package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
  16. package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
  17. package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
  18. package/src/duckdb/extension/json/include/json_common.hpp +47 -231
  19. package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
  20. package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
  21. package/src/duckdb/extension/json/json_common.cpp +272 -40
  22. package/src/duckdb/extension/json/json_functions/json_create.cpp +21 -2
  23. package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
  24. package/src/duckdb/extension/json/json_functions/json_transform.cpp +91 -38
  25. package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
  26. package/src/duckdb/extension/json/json_functions.cpp +24 -24
  27. package/src/duckdb/extension/json/json_scan.cpp +3 -6
  28. package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
  29. package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
  30. package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
  31. package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
  32. package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
  33. package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
  34. package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
  35. package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
  36. package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
  37. package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
  38. package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
  39. package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
  40. package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
  41. package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
  42. package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
  43. package/src/duckdb/extension/parquet/parquet_extension.cpp +194 -20
  44. package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
  45. package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
  46. package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
  47. package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
  48. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
  49. package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
  50. package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
  51. package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
  52. package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
  53. package/src/duckdb/src/common/allocator.cpp +14 -2
  54. package/src/duckdb/src/common/arrow/arrow_appender.cpp +79 -12
  55. package/src/duckdb/src/common/arrow/arrow_converter.cpp +44 -19
  56. package/src/duckdb/src/common/assert.cpp +3 -0
  57. package/src/duckdb/src/common/enum_util.cpp +4619 -4446
  58. package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
  59. package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
  60. package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
  61. package/src/duckdb/src/common/exception.cpp +2 -2
  62. package/src/duckdb/src/common/extra_type_info.cpp +506 -0
  63. package/src/duckdb/src/common/file_system.cpp +19 -0
  64. package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
  65. package/src/duckdb/src/common/local_file_system.cpp +14 -14
  66. package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
  67. package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
  68. package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
  69. package/src/duckdb/src/common/re2_regex.cpp +1 -1
  70. package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
  71. package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
  72. package/src/duckdb/src/common/sort/partition_state.cpp +70 -50
  73. package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
  74. package/src/duckdb/src/common/types/bit.cpp +51 -0
  75. package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
  76. package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
  77. package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
  78. package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
  79. package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
  80. package/src/duckdb/src/common/types/date.cpp +9 -0
  81. package/src/duckdb/src/common/types/list_segment.cpp +24 -74
  82. package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
  83. package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
  84. package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
  85. package/src/duckdb/src/common/types/uuid.cpp +2 -2
  86. package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
  87. package/src/duckdb/src/common/types/value.cpp +11 -6
  88. package/src/duckdb/src/common/types.cpp +9 -656
  89. package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
  90. package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
  91. package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
  92. package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
  93. package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
  94. package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
  95. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
  96. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
  97. package/src/duckdb/src/core_functions/function_list.cpp +4 -2
  98. package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
  99. package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
  100. package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
  101. package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
  102. package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
  103. package/src/duckdb/src/execution/expression_executor.cpp +1 -1
  104. package/src/duckdb/src/execution/index/art/art.cpp +149 -139
  105. package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
  106. package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
  107. package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
  108. package/src/duckdb/src/execution/index/art/node.cpp +113 -120
  109. package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
  110. package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
  111. package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
  112. package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
  113. package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
  114. package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
  115. package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
  116. package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
  117. package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
  118. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
  119. package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
  120. package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
  121. package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +444 -284
  122. package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
  123. package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
  124. package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +28 -12
  125. package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
  126. package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +23 -4
  127. package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +41 -5
  128. package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
  129. package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
  130. package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
  131. package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
  132. package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
  133. package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
  134. package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
  135. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
  136. package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
  137. package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
  138. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
  139. package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
  140. package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
  141. package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
  142. package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
  143. package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
  144. package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +56 -33
  145. package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +17 -13
  146. package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
  147. package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
  148. package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
  149. package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
  150. package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
  151. package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
  152. package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
  153. package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
  154. package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
  155. package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
  156. package/src/duckdb/src/function/function.cpp +3 -1
  157. package/src/duckdb/src/function/pragma/pragma_queries.cpp +5 -0
  158. package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
  159. package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
  160. package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
  161. package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
  162. package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
  163. package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
  164. package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
  165. package/src/duckdb/src/function/table/arrow.cpp +19 -0
  166. package/src/duckdb/src/function/table/arrow_conversion.cpp +35 -1
  167. package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
  168. package/src/duckdb/src/function/table/read_csv.cpp +100 -17
  169. package/src/duckdb/src/function/table/system/test_all_types.cpp +7 -0
  170. package/src/duckdb/src/function/table/system_functions.cpp +1 -0
  171. package/src/duckdb/src/function/table/table_scan.cpp +9 -0
  172. package/src/duckdb/src/function/table/version/pragma_version.cpp +46 -2
  173. package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
  174. package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
  175. package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
  176. package/src/duckdb/src/include/duckdb/common/dl.hpp +3 -1
  177. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +616 -584
  178. package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
  179. package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
  180. package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
  181. package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
  182. package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
  183. package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
  184. package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +219 -0
  185. package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
  186. package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
  187. package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
  188. package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
  189. package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
  190. package/src/duckdb/src/include/duckdb/common/opener_file_system.hpp +2 -2
  191. package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
  192. package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
  193. package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
  194. package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
  195. package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
  196. package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +23 -8
  197. package/src/duckdb/src/include/duckdb/common/string_util.hpp +11 -0
  198. package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
  199. package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
  200. package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
  201. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
  202. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
  203. package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
  204. package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
  205. package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
  206. package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
  207. package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +5 -2
  208. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
  209. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
  210. package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
  211. package/src/duckdb/src/include/duckdb/common/types/value.hpp +1 -0
  212. package/src/duckdb/src/include/duckdb/common/types.hpp +1 -15
  213. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
  214. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
  215. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
  216. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
  217. package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
  218. package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
  219. package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
  220. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
  221. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
  222. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
  223. package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
  224. package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
  225. package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
  226. package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
  227. package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
  228. package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
  229. package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
  230. package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
  231. package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
  232. package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
  233. package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
  234. package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
  235. package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
  236. package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
  237. package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
  238. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
  239. package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
  240. package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
  241. package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
  242. package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
  243. package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
  244. package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
  245. package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
  246. package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
  247. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +3 -10
  248. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +1 -1
  249. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +1 -1
  250. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp +12 -1
  251. package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
  252. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
  253. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
  254. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
  255. package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
  256. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
  257. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
  258. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
  259. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
  260. package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
  261. package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
  262. package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
  263. package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
  264. package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
  265. package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
  266. package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
  267. package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
  268. package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
  269. package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
  270. package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
  271. package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +4 -0
  272. package/src/duckdb/src/include/duckdb/main/client_config.hpp +5 -0
  273. package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
  274. package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
  275. package/src/duckdb/src/include/duckdb/main/extension/generated_extension_loader.hpp +22 -0
  276. package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +2 -0
  277. package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
  278. package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
  279. package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
  280. package/src/duckdb/src/include/duckdb/main/settings.hpp +39 -1
  281. package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
  282. package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
  283. package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
  284. package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
  285. package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
  286. package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
  287. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
  288. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
  289. package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
  290. package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
  291. package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
  292. package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
  293. package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
  294. package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
  295. package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
  296. package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
  297. package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
  298. package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
  299. package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
  300. package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
  301. package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
  302. package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
  303. package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
  304. package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
  305. package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
  306. package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
  307. package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
  308. package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
  309. package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
  310. package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
  311. package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
  312. package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
  313. package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
  314. package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
  315. package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
  316. package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
  317. package/src/duckdb/src/include/duckdb/planner/binder.hpp +12 -5
  318. package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
  319. package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
  320. package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
  321. package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
  322. package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
  323. package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
  324. package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
  325. package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
  326. package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
  327. package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
  328. package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
  329. package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
  330. package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
  331. package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
  332. package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
  333. package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
  334. package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
  335. package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
  336. package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
  337. package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
  338. package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
  339. package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
  340. package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
  341. package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
  342. package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
  343. package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
  344. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
  345. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
  346. package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
  347. package/src/duckdb/src/include/duckdb.h +28 -0
  348. package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
  349. package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
  350. package/src/duckdb/src/main/config.cpp +4 -0
  351. package/src/duckdb/src/main/database.cpp +1 -1
  352. package/src/duckdb/src/main/extension/extension_helper.cpp +93 -88
  353. package/src/duckdb/src/main/extension/extension_install.cpp +9 -0
  354. package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
  355. package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
  356. package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
  357. package/src/duckdb/src/main/relation.cpp +6 -5
  358. package/src/duckdb/src/main/settings/settings.cpp +79 -18
  359. package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
  360. package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
  361. package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
  362. package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
  363. package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
  364. package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
  365. package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
  366. package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
  367. package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
  368. package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
  369. package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
  370. package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
  371. package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
  372. package/src/duckdb/src/optimizer/optimizer.cpp +49 -14
  373. package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
  374. package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
  375. package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
  376. package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
  377. package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
  378. package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
  379. package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
  380. package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
  381. package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
  382. package/src/duckdb/src/parallel/executor.cpp +15 -0
  383. package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
  384. package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
  385. package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
  386. package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
  387. package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
  388. package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
  389. package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
  390. package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
  391. package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
  392. package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
  393. package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
  394. package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
  395. package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
  396. package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
  397. package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
  398. package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
  399. package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
  400. package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
  401. package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
  402. package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
  403. package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
  404. package/src/duckdb/src/parser/parser.cpp +8 -2
  405. package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
  406. package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
  407. package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
  408. package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
  409. package/src/duckdb/src/parser/query_node.cpp +15 -37
  410. package/src/duckdb/src/parser/result_modifier.cpp +0 -74
  411. package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
  412. package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
  413. package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
  414. package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
  415. package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
  416. package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
  417. package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
  418. package/src/duckdb/src/parser/tableref.cpp +0 -44
  419. package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
  420. package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
  421. package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
  422. package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
  423. package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
  424. package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
  425. package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
  426. package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
  427. package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
  428. package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
  429. package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
  430. package/src/duckdb/src/parser/transformer.cpp +15 -0
  431. package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
  432. package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
  433. package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
  434. package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
  435. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
  436. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
  437. package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
  438. package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
  439. package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
  440. package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
  441. package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +64 -26
  442. package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
  443. package/src/duckdb/src/planner/binder.cpp +44 -31
  444. package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
  445. package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
  446. package/src/duckdb/src/planner/expression_binder.cpp +3 -0
  447. package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
  448. package/src/duckdb/src/planner/logical_operator.cpp +5 -0
  449. package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
  450. package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
  451. package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
  452. package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
  453. package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
  454. package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
  455. package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
  456. package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
  457. package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
  458. package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
  459. package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
  460. package/src/duckdb/src/storage/compression/rle.cpp +0 -1
  461. package/src/duckdb/src/storage/data_table.cpp +1 -1
  462. package/src/duckdb/src/storage/local_storage.cpp +3 -3
  463. package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
  464. package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
  465. package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
  466. package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
  467. package/src/duckdb/src/storage/serialization/serialize_types.cpp +127 -0
  468. package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
  469. package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
  470. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  471. package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
  472. package/src/duckdb/src/storage/table/row_group.cpp +25 -9
  473. package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
  474. package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
  475. package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
  476. package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
  477. package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
  478. package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
  479. package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
  480. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
  481. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
  482. package/src/duckdb/ub_src_common.cpp +2 -0
  483. package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
  484. package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
  485. package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
  486. package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
  487. package/src/duckdb/ub_src_function_scalar.cpp +2 -0
  488. package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
  489. package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
  490. package/src/duckdb/ub_src_optimizer.cpp +6 -0
  491. package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
  492. package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
  493. package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
  494. package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
  495. package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
  496. package/src/duckdb/ub_src_planner_operator.cpp +4 -0
  497. package/src/duckdb/ub_src_storage_serialization.cpp +10 -0
  498. package/src/statement.cpp +10 -3
  499. package/test/columns.test.ts +24 -1
  500. package/test/test_all_types.test.ts +234 -0
  501. package/tsconfig.json +1 -0
  502. package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
  503. package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
  504. package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -10,6 +10,7 @@
10
10
 
11
11
  #include "duckdb/common/file_system.hpp"
12
12
  #include "duckdb/common/map.hpp"
13
+ #include "duckdb/common/unordered_set.hpp"
13
14
 
14
15
  namespace duckdb {
15
16
 
@@ -22,120 +23,60 @@ public:
22
23
  FileCompressionType compression = FileCompressionType::UNCOMPRESSED,
23
24
  FileOpener *opener = nullptr) override;
24
25
 
25
- void Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override {
26
- handle.file_system.Read(handle, buffer, nr_bytes, location);
27
- };
26
+ void Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override;
27
+ void Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override;
28
28
 
29
- void Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override {
30
- handle.file_system.Write(handle, buffer, nr_bytes, location);
31
- }
29
+ int64_t Read(FileHandle &handle, void *buffer, int64_t nr_bytes) override;
32
30
 
33
- int64_t Read(FileHandle &handle, void *buffer, int64_t nr_bytes) override {
34
- return handle.file_system.Read(handle, buffer, nr_bytes);
35
- }
31
+ int64_t Write(FileHandle &handle, void *buffer, int64_t nr_bytes) override;
36
32
 
37
- int64_t Write(FileHandle &handle, void *buffer, int64_t nr_bytes) override {
38
- return handle.file_system.Write(handle, buffer, nr_bytes);
39
- }
33
+ int64_t GetFileSize(FileHandle &handle) override;
34
+ time_t GetLastModifiedTime(FileHandle &handle) override;
35
+ FileType GetFileType(FileHandle &handle) override;
40
36
 
41
- int64_t GetFileSize(FileHandle &handle) override {
42
- return handle.file_system.GetFileSize(handle);
43
- }
44
- time_t GetLastModifiedTime(FileHandle &handle) override {
45
- return handle.file_system.GetLastModifiedTime(handle);
46
- }
47
- FileType GetFileType(FileHandle &handle) override {
48
- return handle.file_system.GetFileType(handle);
49
- }
37
+ void Truncate(FileHandle &handle, int64_t new_size) override;
50
38
 
51
- void Truncate(FileHandle &handle, int64_t new_size) override {
52
- handle.file_system.Truncate(handle, new_size);
53
- }
54
-
55
- void FileSync(FileHandle &handle) override {
56
- handle.file_system.FileSync(handle);
57
- }
39
+ void FileSync(FileHandle &handle) override;
58
40
 
59
41
  // need to look up correct fs for this
60
- bool DirectoryExists(const string &directory) override {
61
- return FindFileSystem(directory)->DirectoryExists(directory);
62
- }
63
- void CreateDirectory(const string &directory) override {
64
- FindFileSystem(directory)->CreateDirectory(directory);
65
- }
42
+ bool DirectoryExists(const string &directory) override;
43
+ void CreateDirectory(const string &directory) override;
66
44
 
67
- void RemoveDirectory(const string &directory) override {
68
- FindFileSystem(directory)->RemoveDirectory(directory);
69
- }
45
+ void RemoveDirectory(const string &directory) override;
70
46
 
71
47
  bool ListFiles(const string &directory, const std::function<void(const string &, bool)> &callback,
72
- FileOpener *opener = nullptr) override {
73
- return FindFileSystem(directory)->ListFiles(directory, callback, opener);
74
- }
75
-
76
- void MoveFile(const string &source, const string &target) override {
77
- FindFileSystem(source)->MoveFile(source, target);
78
- }
79
-
80
- bool FileExists(const string &filename) override {
81
- return FindFileSystem(filename)->FileExists(filename);
82
- }
83
-
84
- bool IsPipe(const string &filename) override {
85
- return FindFileSystem(filename)->IsPipe(filename);
86
- }
87
- virtual void RemoveFile(const string &filename) override {
88
- FindFileSystem(filename)->RemoveFile(filename);
89
- }
90
-
91
- virtual vector<string> Glob(const string &path, FileOpener *opener = nullptr) override {
92
- return FindFileSystem(path)->Glob(path, opener);
93
- }
94
-
95
- void RegisterSubSystem(unique_ptr<FileSystem> fs) override {
96
- sub_systems.push_back(std::move(fs));
97
- }
98
-
99
- void UnregisterSubSystem(const string &name) override {
100
- for (auto sub_system = sub_systems.begin(); sub_system != sub_systems.end(); sub_system++) {
101
- if (sub_system->get()->GetName() == name) {
102
- sub_systems.erase(sub_system);
103
- return;
104
- }
105
- }
106
- throw InvalidInputException("Could not find filesystem with name %s", name);
107
- }
108
-
109
- void RegisterSubSystem(FileCompressionType compression_type, unique_ptr<FileSystem> fs) override {
110
- compressed_fs[compression_type] = std::move(fs);
111
- }
112
-
113
- vector<string> ListSubSystems() override {
114
- vector<string> names(sub_systems.size());
115
- for (idx_t i = 0; i < sub_systems.size(); i++) {
116
- names[i] = sub_systems[i]->GetName();
117
- }
118
- return names;
119
- }
120
-
121
- std::string GetName() const override {
122
- return "VirtualFileSystem";
123
- }
48
+ FileOpener *opener = nullptr) override;
49
+
50
+ void MoveFile(const string &source, const string &target) override;
51
+
52
+ bool FileExists(const string &filename) override;
53
+
54
+ bool IsPipe(const string &filename) override;
55
+ virtual void RemoveFile(const string &filename) override;
56
+
57
+ virtual vector<string> Glob(const string &path, FileOpener *opener = nullptr) override;
58
+
59
+ void RegisterSubSystem(unique_ptr<FileSystem> fs) override;
60
+
61
+ void UnregisterSubSystem(const string &name) override;
62
+
63
+ void RegisterSubSystem(FileCompressionType compression_type, unique_ptr<FileSystem> fs) override;
64
+
65
+ vector<string> ListSubSystems() override;
66
+
67
+ std::string GetName() const override;
68
+
69
+ void SetDisabledFileSystems(const vector<string> &names) override;
124
70
 
125
71
  private:
126
- FileSystem *FindFileSystem(const string &path) {
127
- for (auto &sub_system : sub_systems) {
128
- if (sub_system->CanHandleFile(path)) {
129
- return sub_system.get();
130
- }
131
- }
132
- return default_fs.get();
133
- }
72
+ FileSystem &FindFileSystem(const string &path);
73
+ FileSystem &FindFileSystemInternal(const string &path);
134
74
 
135
75
  private:
136
76
  vector<unique_ptr<FileSystem>> sub_systems;
137
77
  map<FileCompressionType, unique_ptr<FileSystem>> compressed_fs;
138
78
  const unique_ptr<FileSystem> default_fs;
79
+ unordered_set<string> disabled_file_systems;
139
80
  };
140
81
 
141
82
  } // namespace duckdb
@@ -31,10 +31,10 @@ struct CorrOperation {
31
31
  }
32
32
 
33
33
  template <class A_TYPE, class B_TYPE, class STATE, class OP>
34
- static void Operation(STATE &state, const A_TYPE &x_input, const B_TYPE &y_input, AggregateBinaryInput &idata) {
35
- CovarOperation::Operation<A_TYPE, B_TYPE, CovarState, OP>(state.cov_pop, x_input, y_input, idata);
36
- STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.dev_pop_x, x_input);
37
- STDDevBaseOperation::Execute<B_TYPE, StddevState>(state.dev_pop_y, y_input);
34
+ static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
35
+ CovarOperation::Operation<A_TYPE, B_TYPE, CovarState, OP>(state.cov_pop, y, x, idata);
36
+ STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.dev_pop_x, x);
37
+ STDDevBaseOperation::Execute<B_TYPE, StddevState>(state.dev_pop_y, y);
38
38
  }
39
39
 
40
40
  template <class STATE, class OP>
@@ -5,6 +5,7 @@
5
5
  //
6
6
  //
7
7
  //===----------------------------------------------------------------------===//
8
+ // COVAR_POP(y,x)
8
9
 
9
10
  #pragma once
10
11
 
@@ -29,7 +30,7 @@ struct CovarOperation {
29
30
  }
30
31
 
31
32
  template <class A_TYPE, class B_TYPE, class STATE, class OP>
32
- static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
33
+ static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
33
34
  // update running mean and d^2
34
35
  const uint64_t n = ++(state.count);
35
36
 
@@ -39,6 +40,7 @@ struct CovarOperation {
39
40
  const double dy = (y - state.meany);
40
41
  const double meany = state.meany + dy / n;
41
42
 
43
+ // Schubert and Gertz SSDBM 2018 (4.3)
42
44
  const double C = state.co_moment + dx * (y - meany);
43
45
 
44
46
  state.meanx = meanx;
@@ -5,7 +5,9 @@
5
5
  //
6
6
  //
7
7
  //===----------------------------------------------------------------------===//
8
- // This file is generated by scripts/generate_functions.py
8
+ // This file is automatically generated by scripts/generate_functions.py
9
+ // Do not edit this file manually, your changes will be overwritten
10
+ //===----------------------------------------------------------------------===//
9
11
 
10
12
  #pragma once
11
13
 
@@ -5,7 +5,9 @@
5
5
  //
6
6
  //
7
7
  //===----------------------------------------------------------------------===//
8
- // This file is generated by scripts/generate_functions.py
8
+ // This file is automatically generated by scripts/generate_functions.py
9
+ // Do not edit this file manually, your changes will be overwritten
10
+ //===----------------------------------------------------------------------===//
9
11
 
10
12
  #pragma once
11
13
 
@@ -5,7 +5,9 @@
5
5
  //
6
6
  //
7
7
  //===----------------------------------------------------------------------===//
8
- // This file is generated by scripts/generate_functions.py
8
+ // This file is automatically generated by scripts/generate_functions.py
9
+ // Do not edit this file manually, your changes will be overwritten
10
+ //===----------------------------------------------------------------------===//
9
11
 
10
12
  #pragma once
11
13
 
@@ -5,7 +5,9 @@
5
5
  //
6
6
  //
7
7
  //===----------------------------------------------------------------------===//
8
- // This file is generated by scripts/generate_functions.py
8
+ // This file is automatically generated by scripts/generate_functions.py
9
+ // Do not edit this file manually, your changes will be overwritten
10
+ //===----------------------------------------------------------------------===//
9
11
 
10
12
  #pragma once
11
13
 
@@ -5,6 +5,7 @@
5
5
  //
6
6
  //
7
7
  //===----------------------------------------------------------------------===//
8
+ // REGR_COUNT(y, x)
8
9
 
9
10
  #pragma once
10
11
 
@@ -25,9 +25,9 @@ struct RegrSlopeOperation {
25
25
  }
26
26
 
27
27
  template <class A_TYPE, class B_TYPE, class STATE, class OP>
28
- static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
29
- CovarOperation::Operation<A_TYPE, B_TYPE, CovarState, OP>(state.cov_pop, x, y, idata);
30
- STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop, y);
28
+ static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
29
+ CovarOperation::Operation<A_TYPE, B_TYPE, CovarState, OP>(state.cov_pop, y, x,idata);
30
+ STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop, x);
31
31
  }
32
32
 
33
33
  template <class STATE, class OP>
@@ -5,7 +5,9 @@
5
5
  //
6
6
  //
7
7
  //===----------------------------------------------------------------------===//
8
- // This file is generated by scripts/generate_functions.py
8
+ // This file is automatically generated by scripts/generate_functions.py
9
+ // Do not edit this file manually, your changes will be overwritten
10
+ //===----------------------------------------------------------------------===//
9
11
 
10
12
  #pragma once
11
13
 
@@ -5,7 +5,9 @@
5
5
  //
6
6
  //
7
7
  //===----------------------------------------------------------------------===//
8
- // This file is generated by scripts/generate_functions.py
8
+ // This file is automatically generated by scripts/generate_functions.py
9
+ // Do not edit this file manually, your changes will be overwritten
10
+ //===----------------------------------------------------------------------===//
9
11
 
10
12
  #pragma once
11
13
 
@@ -5,7 +5,9 @@
5
5
  //
6
6
  //
7
7
  //===----------------------------------------------------------------------===//
8
- // This file is generated by scripts/generate_functions.py
8
+ // This file is automatically generated by scripts/generate_functions.py
9
+ // Do not edit this file manually, your changes will be overwritten
10
+ //===----------------------------------------------------------------------===//
9
11
 
10
12
  #pragma once
11
13
 
@@ -5,7 +5,9 @@
5
5
  //
6
6
  //
7
7
  //===----------------------------------------------------------------------===//
8
- // This file is generated by scripts/generate_functions.py
8
+ // This file is automatically generated by scripts/generate_functions.py
9
+ // Do not edit this file manually, your changes will be overwritten
10
+ //===----------------------------------------------------------------------===//
9
11
 
10
12
  #pragma once
11
13
 
@@ -162,8 +164,8 @@ struct DecadeFun {
162
164
 
163
165
  struct EpochFun {
164
166
  static constexpr const char *Name = "epoch";
165
- static constexpr const char *Parameters = "ts";
166
- static constexpr const char *Description = "Extract the epoch component from a date or timestamp";
167
+ static constexpr const char *Parameters = "temporal";
168
+ static constexpr const char *Description = "Extract the epoch component from a temporal type";
167
169
  static constexpr const char *Example = "epoch(timestamp '2021-08-03 11:59:44.123456')";
168
170
 
169
171
  static ScalarFunctionSet GetFunctions();
@@ -171,11 +173,29 @@ struct EpochFun {
171
173
 
172
174
  struct EpochMsFun {
173
175
  static constexpr const char *Name = "epoch_ms";
174
- static constexpr const char *Parameters = "ms";
175
- static constexpr const char *Description = "Converts ms since epoch to a timestamp";
176
- static constexpr const char *Example = "epoch_ms(701222400000)";
176
+ static constexpr const char *Parameters = "temporal";
177
+ static constexpr const char *Description = "Extract the epoch component in milliseconds from a temporal type";
178
+ static constexpr const char *Example = "epoch_ms(timestamp '2021-08-03 11:59:44.123456')";
177
179
 
178
- static ScalarFunction GetFunction();
180
+ static ScalarFunctionSet GetFunctions();
181
+ };
182
+
183
+ struct EpochUsFun {
184
+ static constexpr const char *Name = "epoch_us";
185
+ static constexpr const char *Parameters = "temporal";
186
+ static constexpr const char *Description = "Extract the epoch component in microseconds from a temporal type";
187
+ static constexpr const char *Example = "epoch_us(timestamp '2021-08-03 11:59:44.123456')";
188
+
189
+ static ScalarFunctionSet GetFunctions();
190
+ };
191
+
192
+ struct EpochNsFun {
193
+ static constexpr const char *Name = "epoch_ns";
194
+ static constexpr const char *Parameters = "temporal";
195
+ static constexpr const char *Description = "Extract the epoch component in nanoseconds from a temporal type";
196
+ static constexpr const char *Example = "epoch_ns(timestamp '2021-08-03 11:59:44.123456')";
197
+
198
+ static ScalarFunctionSet GetFunctions();
179
199
  };
180
200
 
181
201
  struct EraFun {
@@ -277,7 +297,7 @@ struct MakeTimestampFun {
277
297
  static constexpr const char *Description = "The timestamp for the given parts";
278
298
  static constexpr const char *Example = "make_timestamp(1992, 9, 20, 13, 34, 27.123456)";
279
299
 
280
- static ScalarFunction GetFunction();
300
+ static ScalarFunctionSet GetFunctions();
281
301
  };
282
302
 
283
303
  struct MicrosecondsFun {
@@ -472,8 +492,8 @@ struct ToSecondsFun {
472
492
  struct ToTimestampFun {
473
493
  static constexpr const char *Name = "to_timestamp";
474
494
  static constexpr const char *Parameters = "sec";
475
- static constexpr const char *Description = "Converts sec since epoch to a timestamp";
476
- static constexpr const char *Example = "to_timestamp(701222400)";
495
+ static constexpr const char *Description = "Converts secs since epoch to a timestamp with time zone";
496
+ static constexpr const char *Example = "to_timestamp(1284352323.5)";
477
497
 
478
498
  static ScalarFunction GetFunction();
479
499
  };
@@ -490,7 +510,7 @@ struct ToYearsFun {
490
510
  struct TryStrpTimeFun {
491
511
  static constexpr const char *Name = "try_strptime";
492
512
  static constexpr const char *Parameters = "text,format";
493
- static constexpr const char *Description = "Converts string to timestamp with time zone according to the format string if %Z is specified. Returns NULL on failure.";
513
+ static constexpr const char *Description = "Converts string to timestamp using the format string (timestamp with time zone if %Z is specified). Returns NULL on failure.";
494
514
  static constexpr const char *Example = "try_strptime('Wed, 1 January 1992 - 08:38:40 PM', '%a, %-d %B %Y - %I:%M:%S %p')";
495
515
 
496
516
  static ScalarFunctionSet GetFunctions();
@@ -5,7 +5,9 @@
5
5
  //
6
6
  //
7
7
  //===----------------------------------------------------------------------===//
8
- // This file is generated by scripts/generate_functions.py
8
+ // This file is automatically generated by scripts/generate_functions.py
9
+ // Do not edit this file manually, your changes will be overwritten
10
+ //===----------------------------------------------------------------------===//
9
11
 
10
12
  #pragma once
11
13
 
@@ -5,7 +5,9 @@
5
5
  //
6
6
  //
7
7
  //===----------------------------------------------------------------------===//
8
- // This file is generated by scripts/generate_functions.py
8
+ // This file is automatically generated by scripts/generate_functions.py
9
+ // Do not edit this file manually, your changes will be overwritten
10
+ //===----------------------------------------------------------------------===//
9
11
 
10
12
  #pragma once
11
13
 
@@ -5,7 +5,9 @@
5
5
  //
6
6
  //
7
7
  //===----------------------------------------------------------------------===//
8
- // This file is generated by scripts/generate_functions.py
8
+ // This file is automatically generated by scripts/generate_functions.py
9
+ // Do not edit this file manually, your changes will be overwritten
10
+ //===----------------------------------------------------------------------===//
9
11
 
10
12
  #pragma once
11
13
 
@@ -5,7 +5,9 @@
5
5
  //
6
6
  //
7
7
  //===----------------------------------------------------------------------===//
8
- // This file is generated by scripts/generate_functions.py
8
+ // This file is automatically generated by scripts/generate_functions.py
9
+ // Do not edit this file manually, your changes will be overwritten
10
+ //===----------------------------------------------------------------------===//
9
11
 
10
12
  #pragma once
11
13
 
@@ -5,7 +5,9 @@
5
5
  //
6
6
  //
7
7
  //===----------------------------------------------------------------------===//
8
- // This file is generated by scripts/generate_functions.py
8
+ // This file is automatically generated by scripts/generate_functions.py
9
+ // Do not edit this file manually, your changes will be overwritten
10
+ //===----------------------------------------------------------------------===//
9
11
 
10
12
  #pragma once
11
13
 
@@ -5,7 +5,9 @@
5
5
  //
6
6
  //
7
7
  //===----------------------------------------------------------------------===//
8
- // This file is generated by scripts/generate_functions.py
8
+ // This file is automatically generated by scripts/generate_functions.py
9
+ // Do not edit this file manually, your changes will be overwritten
10
+ //===----------------------------------------------------------------------===//
9
11
 
10
12
  #pragma once
11
13
 
@@ -5,7 +5,9 @@
5
5
  //
6
6
  //
7
7
  //===----------------------------------------------------------------------===//
8
- // This file is generated by scripts/generate_functions.py
8
+ // This file is automatically generated by scripts/generate_functions.py
9
+ // Do not edit this file manually, your changes will be overwritten
10
+ //===----------------------------------------------------------------------===//
9
11
 
10
12
  #pragma once
11
13
 
@@ -5,7 +5,9 @@
5
5
  //
6
6
  //
7
7
  //===----------------------------------------------------------------------===//
8
- // This file is generated by scripts/generate_functions.py
8
+ // This file is automatically generated by scripts/generate_functions.py
9
+ // Do not edit this file manually, your changes will be overwritten
10
+ //===----------------------------------------------------------------------===//
9
11
 
10
12
  #pragma once
11
13
 
@@ -5,7 +5,9 @@
5
5
  //
6
6
  //
7
7
  //===----------------------------------------------------------------------===//
8
- // This file is generated by scripts/generate_functions.py
8
+ // This file is automatically generated by scripts/generate_functions.py
9
+ // Do not edit this file manually, your changes will be overwritten
10
+ //===----------------------------------------------------------------------===//
9
11
 
10
12
  #pragma once
11
13
 
@@ -5,7 +5,9 @@
5
5
  //
6
6
  //
7
7
  //===----------------------------------------------------------------------===//
8
- // This file is generated by scripts/generate_functions.py
8
+ // This file is automatically generated by scripts/generate_functions.py
9
+ // Do not edit this file manually, your changes will be overwritten
10
+ //===----------------------------------------------------------------------===//
9
11
 
10
12
  #pragma once
11
13
 
@@ -128,6 +128,9 @@ public:
128
128
  //! Executes the filter(if any) and update the aggregates
129
129
  void Combine(GroupedAggregateHashTable &other);
130
130
 
131
+ //! Appends the data in the other HT to this one
132
+ void Append(GroupedAggregateHashTable &other);
133
+
131
134
  TupleDataCollection &GetDataCollection() {
132
135
  return *data_collection;
133
136
  }
@@ -136,16 +139,29 @@ public:
136
139
  return data_collection->Count();
137
140
  }
138
141
 
142
+ idx_t DataSize() const {
143
+ return data_collection->SizeInBytes();
144
+ }
145
+
139
146
  static idx_t InitialCapacity();
140
147
  idx_t Capacity() {
141
148
  return capacity;
142
149
  }
143
150
 
151
+ static idx_t FirstPartSize(idx_t count, HtEntryType entry_type) {
152
+ idx_t entry_size = entry_type == HT_WIDTH_32 ? sizeof(aggr_ht_entry_32) : sizeof(aggr_ht_entry_64);
153
+ return NextPowerOfTwo(count * 2L) * entry_size;
154
+ }
155
+
156
+ idx_t TotalSize() const {
157
+ return DataSize() + FirstPartSize(Count(), entry_type);
158
+ }
159
+
144
160
  idx_t ResizeThreshold();
145
161
  idx_t MaxCapacity();
146
162
  static idx_t GetMaxCapacity(HtEntryType entry_type, idx_t tuple_size);
147
163
 
148
- void Partition(vector<GroupedAggregateHashTable *> &partition_hts, idx_t radix_bits);
164
+ void Partition(vector<GroupedAggregateHashTable *> &partition_hts, idx_t radix_bits, bool sink_done);
149
165
  void InitializeFirstPart();
150
166
 
151
167
  void Finalize();
@@ -165,7 +181,7 @@ private:
165
181
  vector<data_ptr_t> payload_hds_ptrs;
166
182
 
167
183
  //! The hashes of the HT
168
- BufferHandle hashes_hdl;
184
+ AllocatedData hashes_hdl;
169
185
  data_ptr_t hashes_hdl_ptr;
170
186
  idx_t hash_offset; // Offset into the layout of the hash column
171
187
 
@@ -178,8 +194,10 @@ private:
178
194
 
179
195
  vector<ExpressionType> predicates;
180
196
 
181
- //! The arena allocator used by the aggregates for their internal state
197
+ //! The active arena allocator used by the aggregates for their internal state
182
198
  shared_ptr<ArenaAllocator> aggregate_allocator;
199
+ //! Owning arena allocators that this HT has data from
200
+ vector<shared_ptr<ArenaAllocator>> stored_allocators;
183
201
 
184
202
  private:
185
203
  GroupedAggregateHashTable(const GroupedAggregateHashTable &) = delete;
@@ -88,6 +88,7 @@ public:
88
88
  void AddEvent(shared_ptr<Event> event);
89
89
 
90
90
  void AddRecursiveCTE(PhysicalOperator &rec_cte);
91
+ void AddMaterializedCTE(PhysicalOperator &mat_cte);
91
92
  void ReschedulePipelines(const vector<shared_ptr<MetaPipeline>> &pipelines, vector<shared_ptr<Event>> &events);
92
93
 
93
94
  //! Whether or not the root of the pipeline is a result collector object
@@ -129,6 +130,8 @@ private:
129
130
  vector<shared_ptr<Pipeline>> root_pipelines;
130
131
  //! The recursive CTE's in this query plan
131
132
  vector<reference<PhysicalOperator>> recursive_ctes;
133
+ //! The materialized CTE's in this query plan
134
+ vector<reference<PhysicalOperator>> materialized_ctes;
132
135
  //! The pipeline executor for the root pipeline
133
136
  unique_ptr<PipelineExecutor> root_executor;
134
137
  //! The current root pipeline index
@@ -112,14 +112,13 @@ private:
112
112
  void Erase(Node &node, const ARTKey &key, idx_t depth, const row_t &row_id);
113
113
 
114
114
  //! Returns all row IDs belonging to a key greater (or equal) than the search key
115
- bool SearchGreater(ARTIndexScanState &state, ARTKey &key, bool inclusive, idx_t max_count,
116
- vector<row_t> &result_ids);
115
+ bool SearchGreater(ARTIndexScanState &state, ARTKey &key, bool equal, idx_t max_count, vector<row_t> &result_ids);
117
116
  //! Returns all row IDs belonging to a key less (or equal) than the upper_bound
118
- bool SearchLess(ARTIndexScanState &state, ARTKey &upper_bound, bool inclusive, idx_t max_count,
117
+ bool SearchLess(ARTIndexScanState &state, ARTKey &upper_bound, bool equal, idx_t max_count,
119
118
  vector<row_t> &result_ids);
120
119
  //! Returns all row IDs belonging to a key within the range of lower_bound and upper_bound
121
- bool SearchCloseRange(ARTIndexScanState &state, ARTKey &lower_bound, ARTKey &upper_bound, bool left_inclusive,
122
- bool right_inclusive, idx_t max_count, vector<row_t> &result_ids);
120
+ bool SearchCloseRange(ARTIndexScanState &state, ARTKey &lower_bound, ARTKey &upper_bound, bool left_equal,
121
+ bool right_equal, idx_t max_count, vector<row_t> &result_ids);
123
122
 
124
123
  //! Initializes a merge operation by returning a set containing the buffer count of each fixed-size allocator
125
124
  void InitializeMerge(ARTFlags &flags);