duckdb 0.8.2-dev145.0 → 0.8.2-dev1493.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (476) hide show
  1. package/binding.gyp +15 -12
  2. package/binding.gyp.in +1 -1
  3. package/configure.py +1 -1
  4. package/duckdb_extension_config.cmake +10 -0
  5. package/package.json +1 -1
  6. package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
  7. package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
  8. package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
  9. package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
  10. package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
  11. package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
  12. package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
  13. package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
  14. package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
  15. package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
  16. package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
  17. package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
  18. package/src/duckdb/extension/json/include/json_common.hpp +47 -231
  19. package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
  20. package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
  21. package/src/duckdb/extension/json/json_common.cpp +272 -40
  22. package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
  23. package/src/duckdb/extension/json/json_functions/json_transform.cpp +17 -37
  24. package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
  25. package/src/duckdb/extension/json/json_functions.cpp +24 -24
  26. package/src/duckdb/extension/json/json_scan.cpp +3 -6
  27. package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
  28. package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
  29. package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
  30. package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
  31. package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
  32. package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
  33. package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
  34. package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
  35. package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
  36. package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
  37. package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
  38. package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
  39. package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
  40. package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
  41. package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
  42. package/src/duckdb/extension/parquet/parquet_extension.cpp +192 -20
  43. package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
  44. package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
  45. package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
  46. package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
  47. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
  48. package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
  49. package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
  50. package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
  51. package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
  52. package/src/duckdb/src/common/allocator.cpp +14 -2
  53. package/src/duckdb/src/common/arrow/arrow_appender.cpp +5 -11
  54. package/src/duckdb/src/common/assert.cpp +3 -0
  55. package/src/duckdb/src/common/enum_util.cpp +42 -5
  56. package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
  57. package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
  58. package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
  59. package/src/duckdb/src/common/exception.cpp +2 -2
  60. package/src/duckdb/src/common/file_system.cpp +19 -0
  61. package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
  62. package/src/duckdb/src/common/local_file_system.cpp +2 -2
  63. package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
  64. package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
  65. package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
  66. package/src/duckdb/src/common/re2_regex.cpp +1 -1
  67. package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
  68. package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
  69. package/src/duckdb/src/common/sort/partition_state.cpp +44 -11
  70. package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
  71. package/src/duckdb/src/common/types/bit.cpp +51 -0
  72. package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
  73. package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
  74. package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
  75. package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
  76. package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
  77. package/src/duckdb/src/common/types/date.cpp +9 -0
  78. package/src/duckdb/src/common/types/list_segment.cpp +24 -74
  79. package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
  80. package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
  81. package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
  82. package/src/duckdb/src/common/types/uuid.cpp +2 -2
  83. package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
  84. package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
  85. package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
  86. package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
  87. package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
  88. package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
  89. package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
  90. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
  91. package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
  92. package/src/duckdb/src/core_functions/function_list.cpp +4 -2
  93. package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
  94. package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
  95. package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
  96. package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
  97. package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
  98. package/src/duckdb/src/execution/expression_executor.cpp +1 -1
  99. package/src/duckdb/src/execution/index/art/art.cpp +149 -139
  100. package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
  101. package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
  102. package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
  103. package/src/duckdb/src/execution/index/art/node.cpp +113 -120
  104. package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
  105. package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
  106. package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
  107. package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
  108. package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
  109. package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
  110. package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
  111. package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
  112. package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
  113. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
  114. package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
  115. package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
  116. package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +414 -283
  117. package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
  118. package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
  119. package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
  120. package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +22 -3
  121. package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +2 -2
  122. package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
  123. package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
  124. package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
  125. package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
  126. package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
  127. package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
  128. package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
  129. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
  130. package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
  131. package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
  132. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
  133. package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
  134. package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
  135. package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
  136. package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
  137. package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
  138. package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
  139. package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
  140. package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
  141. package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
  142. package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
  143. package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
  144. package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
  145. package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
  146. package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
  147. package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
  148. package/src/duckdb/src/function/function.cpp +3 -1
  149. package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
  150. package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
  151. package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
  152. package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
  153. package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
  154. package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
  155. package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
  156. package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
  157. package/src/duckdb/src/function/table/read_csv.cpp +100 -17
  158. package/src/duckdb/src/function/table/table_scan.cpp +9 -0
  159. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  160. package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
  161. package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
  162. package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
  163. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +8 -0
  164. package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
  165. package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
  166. package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
  167. package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
  168. package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
  169. package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
  170. package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
  171. package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
  172. package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
  173. package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
  174. package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
  175. package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
  176. package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
  177. package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
  178. package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
  179. package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
  180. package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +13 -0
  181. package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
  182. package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
  183. package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
  184. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
  185. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
  186. package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
  187. package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
  188. package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
  189. package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
  190. package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +0 -1
  191. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
  192. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
  193. package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
  194. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
  195. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
  196. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
  197. package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
  198. package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
  199. package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
  200. package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
  201. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
  202. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
  203. package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
  204. package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
  205. package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
  206. package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
  207. package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
  208. package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
  209. package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
  210. package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
  211. package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
  212. package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
  213. package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
  214. package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
  215. package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
  216. package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
  217. package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
  218. package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
  219. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
  220. package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
  221. package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
  222. package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
  223. package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
  224. package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
  225. package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
  226. package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
  227. package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
  228. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +2 -10
  229. package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
  230. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
  231. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
  232. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
  233. package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
  234. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
  235. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
  236. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
  237. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
  238. package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
  239. package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
  240. package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
  241. package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
  242. package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
  243. package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
  244. package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
  245. package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
  246. package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
  247. package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
  248. package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
  249. package/src/duckdb/src/include/duckdb/main/client_config.hpp +3 -0
  250. package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
  251. package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
  252. package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
  253. package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
  254. package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
  255. package/src/duckdb/src/include/duckdb/main/settings.hpp +30 -1
  256. package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
  257. package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
  258. package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
  259. package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
  260. package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
  261. package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
  262. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
  263. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
  264. package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
  265. package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
  266. package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
  267. package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
  268. package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
  269. package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
  270. package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
  271. package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
  272. package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
  273. package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
  274. package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
  275. package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
  276. package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
  277. package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
  278. package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
  279. package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
  280. package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
  281. package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
  282. package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
  283. package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
  284. package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
  285. package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
  286. package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
  287. package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
  288. package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
  289. package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
  290. package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
  291. package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
  292. package/src/duckdb/src/include/duckdb/planner/binder.hpp +8 -5
  293. package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
  294. package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
  295. package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
  296. package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
  297. package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
  298. package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
  299. package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
  300. package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
  301. package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
  302. package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
  303. package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
  304. package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
  305. package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
  306. package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
  307. package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
  308. package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
  309. package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
  310. package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
  311. package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
  312. package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
  313. package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
  314. package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
  315. package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
  316. package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
  317. package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
  318. package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
  319. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
  320. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
  321. package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
  322. package/src/duckdb/src/include/duckdb.h +28 -0
  323. package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
  324. package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
  325. package/src/duckdb/src/main/config.cpp +3 -0
  326. package/src/duckdb/src/main/database.cpp +1 -1
  327. package/src/duckdb/src/main/extension/extension_helper.cpp +96 -89
  328. package/src/duckdb/src/main/extension/extension_install.cpp +6 -0
  329. package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
  330. package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
  331. package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
  332. package/src/duckdb/src/main/relation.cpp +6 -5
  333. package/src/duckdb/src/main/settings/settings.cpp +64 -18
  334. package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
  335. package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
  336. package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
  337. package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
  338. package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
  339. package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
  340. package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
  341. package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
  342. package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
  343. package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
  344. package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
  345. package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
  346. package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
  347. package/src/duckdb/src/optimizer/optimizer.cpp +51 -14
  348. package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
  349. package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
  350. package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
  351. package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
  352. package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
  353. package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
  354. package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
  355. package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
  356. package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
  357. package/src/duckdb/src/parallel/executor.cpp +15 -0
  358. package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
  359. package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
  360. package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
  361. package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
  362. package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
  363. package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
  364. package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
  365. package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
  366. package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
  367. package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
  368. package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
  369. package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
  370. package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
  371. package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
  372. package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
  373. package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
  374. package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
  375. package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
  376. package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
  377. package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
  378. package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
  379. package/src/duckdb/src/parser/parser.cpp +8 -2
  380. package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
  381. package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
  382. package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
  383. package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
  384. package/src/duckdb/src/parser/query_node.cpp +15 -37
  385. package/src/duckdb/src/parser/result_modifier.cpp +0 -74
  386. package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
  387. package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
  388. package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
  389. package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
  390. package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
  391. package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
  392. package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
  393. package/src/duckdb/src/parser/tableref.cpp +0 -44
  394. package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
  395. package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
  396. package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
  397. package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
  398. package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
  399. package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
  400. package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
  401. package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
  402. package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
  403. package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
  404. package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
  405. package/src/duckdb/src/parser/transformer.cpp +15 -0
  406. package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
  407. package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
  408. package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
  409. package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
  410. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
  411. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
  412. package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
  413. package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
  414. package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
  415. package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
  416. package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +61 -26
  417. package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
  418. package/src/duckdb/src/planner/binder.cpp +5 -0
  419. package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
  420. package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
  421. package/src/duckdb/src/planner/expression_binder.cpp +3 -0
  422. package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
  423. package/src/duckdb/src/planner/logical_operator.cpp +5 -0
  424. package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
  425. package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
  426. package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
  427. package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
  428. package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
  429. package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
  430. package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
  431. package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
  432. package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
  433. package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
  434. package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
  435. package/src/duckdb/src/storage/compression/rle.cpp +0 -1
  436. package/src/duckdb/src/storage/data_table.cpp +1 -1
  437. package/src/duckdb/src/storage/local_storage.cpp +3 -3
  438. package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
  439. package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
  440. package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
  441. package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
  442. package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
  443. package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
  444. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  445. package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
  446. package/src/duckdb/src/storage/table/row_group.cpp +25 -9
  447. package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
  448. package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
  449. package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
  450. package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
  451. package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
  452. package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
  453. package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
  454. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
  455. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
  456. package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
  457. package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
  458. package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
  459. package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
  460. package/src/duckdb/ub_src_function_scalar.cpp +2 -0
  461. package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
  462. package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
  463. package/src/duckdb/ub_src_optimizer.cpp +6 -0
  464. package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
  465. package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
  466. package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
  467. package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
  468. package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
  469. package/src/duckdb/ub_src_planner_operator.cpp +4 -0
  470. package/src/duckdb/ub_src_storage_serialization.cpp +8 -0
  471. package/src/statement.cpp +10 -3
  472. package/test/test_all_types.test.ts +233 -0
  473. package/tsconfig.json +1 -0
  474. package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
  475. package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
  476. package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -0,0 +1,166 @@
1
+ //===----------------------------------------------------------------------===//
2
+ // This file is automatically generated by scripts/generate_serialization.py
3
+ // Do not edit this file manually, your changes will be overwritten
4
+ //===----------------------------------------------------------------------===//
5
+
6
+ #include "duckdb/common/serializer/format_serializer.hpp"
7
+ #include "duckdb/common/serializer/format_deserializer.hpp"
8
+ #include "duckdb/parser/tableref/list.hpp"
9
+
10
+ namespace duckdb {
11
+
12
+ void TableRef::FormatSerialize(FormatSerializer &serializer) const {
13
+ serializer.WriteProperty("type", type);
14
+ serializer.WriteProperty("alias", alias);
15
+ serializer.WriteOptionalProperty("sample", sample);
16
+ }
17
+
18
+ unique_ptr<TableRef> TableRef::FormatDeserialize(FormatDeserializer &deserializer) {
19
+ auto type = deserializer.ReadProperty<TableReferenceType>("type");
20
+ auto alias = deserializer.ReadProperty<string>("alias");
21
+ auto sample = deserializer.ReadOptionalProperty<unique_ptr<SampleOptions>>("sample");
22
+ unique_ptr<TableRef> result;
23
+ switch (type) {
24
+ case TableReferenceType::BASE_TABLE:
25
+ result = BaseTableRef::FormatDeserialize(deserializer);
26
+ break;
27
+ case TableReferenceType::EMPTY:
28
+ result = EmptyTableRef::FormatDeserialize(deserializer);
29
+ break;
30
+ case TableReferenceType::EXPRESSION_LIST:
31
+ result = ExpressionListRef::FormatDeserialize(deserializer);
32
+ break;
33
+ case TableReferenceType::JOIN:
34
+ result = JoinRef::FormatDeserialize(deserializer);
35
+ break;
36
+ case TableReferenceType::PIVOT:
37
+ result = PivotRef::FormatDeserialize(deserializer);
38
+ break;
39
+ case TableReferenceType::SUBQUERY:
40
+ result = SubqueryRef::FormatDeserialize(deserializer);
41
+ break;
42
+ case TableReferenceType::TABLE_FUNCTION:
43
+ result = TableFunctionRef::FormatDeserialize(deserializer);
44
+ break;
45
+ default:
46
+ throw SerializationException("Unsupported type for deserialization of TableRef!");
47
+ }
48
+ result->alias = std::move(alias);
49
+ result->sample = std::move(sample);
50
+ return result;
51
+ }
52
+
53
+ void BaseTableRef::FormatSerialize(FormatSerializer &serializer) const {
54
+ TableRef::FormatSerialize(serializer);
55
+ serializer.WriteProperty("schema_name", schema_name);
56
+ serializer.WriteProperty("table_name", table_name);
57
+ serializer.WriteProperty("column_name_alias", column_name_alias);
58
+ serializer.WriteProperty("catalog_name", catalog_name);
59
+ }
60
+
61
+ unique_ptr<TableRef> BaseTableRef::FormatDeserialize(FormatDeserializer &deserializer) {
62
+ auto result = duckdb::unique_ptr<BaseTableRef>(new BaseTableRef());
63
+ deserializer.ReadProperty("schema_name", result->schema_name);
64
+ deserializer.ReadProperty("table_name", result->table_name);
65
+ deserializer.ReadProperty("column_name_alias", result->column_name_alias);
66
+ deserializer.ReadProperty("catalog_name", result->catalog_name);
67
+ return std::move(result);
68
+ }
69
+
70
+ void JoinRef::FormatSerialize(FormatSerializer &serializer) const {
71
+ TableRef::FormatSerialize(serializer);
72
+ serializer.WriteProperty("left", *left);
73
+ serializer.WriteProperty("right", *right);
74
+ serializer.WriteOptionalProperty("condition", condition);
75
+ serializer.WriteProperty("type", type);
76
+ serializer.WriteProperty("ref_type", ref_type);
77
+ serializer.WriteProperty("using_columns", using_columns);
78
+ }
79
+
80
+ unique_ptr<TableRef> JoinRef::FormatDeserialize(FormatDeserializer &deserializer) {
81
+ auto result = duckdb::unique_ptr<JoinRef>(new JoinRef());
82
+ deserializer.ReadProperty("left", result->left);
83
+ deserializer.ReadProperty("right", result->right);
84
+ deserializer.ReadOptionalProperty("condition", result->condition);
85
+ deserializer.ReadProperty("type", result->type);
86
+ deserializer.ReadProperty("ref_type", result->ref_type);
87
+ deserializer.ReadProperty("using_columns", result->using_columns);
88
+ return std::move(result);
89
+ }
90
+
91
+ void SubqueryRef::FormatSerialize(FormatSerializer &serializer) const {
92
+ TableRef::FormatSerialize(serializer);
93
+ serializer.WriteProperty("subquery", *subquery);
94
+ serializer.WriteProperty("column_name_alias", column_name_alias);
95
+ }
96
+
97
+ unique_ptr<TableRef> SubqueryRef::FormatDeserialize(FormatDeserializer &deserializer) {
98
+ auto result = duckdb::unique_ptr<SubqueryRef>(new SubqueryRef());
99
+ deserializer.ReadProperty("subquery", result->subquery);
100
+ deserializer.ReadProperty("column_name_alias", result->column_name_alias);
101
+ return std::move(result);
102
+ }
103
+
104
+ void TableFunctionRef::FormatSerialize(FormatSerializer &serializer) const {
105
+ TableRef::FormatSerialize(serializer);
106
+ serializer.WriteProperty("function", *function);
107
+ serializer.WriteProperty("alias", alias);
108
+ serializer.WriteProperty("column_name_alias", column_name_alias);
109
+ }
110
+
111
+ unique_ptr<TableRef> TableFunctionRef::FormatDeserialize(FormatDeserializer &deserializer) {
112
+ auto result = duckdb::unique_ptr<TableFunctionRef>(new TableFunctionRef());
113
+ deserializer.ReadProperty("function", result->function);
114
+ deserializer.ReadProperty("alias", result->alias);
115
+ deserializer.ReadProperty("column_name_alias", result->column_name_alias);
116
+ return std::move(result);
117
+ }
118
+
119
+ void EmptyTableRef::FormatSerialize(FormatSerializer &serializer) const {
120
+ TableRef::FormatSerialize(serializer);
121
+ }
122
+
123
+ unique_ptr<TableRef> EmptyTableRef::FormatDeserialize(FormatDeserializer &deserializer) {
124
+ auto result = duckdb::unique_ptr<EmptyTableRef>(new EmptyTableRef());
125
+ return std::move(result);
126
+ }
127
+
128
+ void ExpressionListRef::FormatSerialize(FormatSerializer &serializer) const {
129
+ TableRef::FormatSerialize(serializer);
130
+ serializer.WriteProperty("expected_names", expected_names);
131
+ serializer.WriteProperty("expected_types", expected_types);
132
+ serializer.WriteProperty("values", values);
133
+ }
134
+
135
+ unique_ptr<TableRef> ExpressionListRef::FormatDeserialize(FormatDeserializer &deserializer) {
136
+ auto result = duckdb::unique_ptr<ExpressionListRef>(new ExpressionListRef());
137
+ deserializer.ReadProperty("expected_names", result->expected_names);
138
+ deserializer.ReadProperty("expected_types", result->expected_types);
139
+ deserializer.ReadProperty("values", result->values);
140
+ return std::move(result);
141
+ }
142
+
143
+ void PivotRef::FormatSerialize(FormatSerializer &serializer) const {
144
+ TableRef::FormatSerialize(serializer);
145
+ serializer.WriteProperty("source", *source);
146
+ serializer.WriteProperty("aggregates", aggregates);
147
+ serializer.WriteProperty("unpivot_names", unpivot_names);
148
+ serializer.WriteProperty("pivots", pivots);
149
+ serializer.WriteProperty("groups", groups);
150
+ serializer.WriteProperty("column_name_alias", column_name_alias);
151
+ serializer.WriteProperty("include_nulls", include_nulls);
152
+ }
153
+
154
+ unique_ptr<TableRef> PivotRef::FormatDeserialize(FormatDeserializer &deserializer) {
155
+ auto result = duckdb::unique_ptr<PivotRef>(new PivotRef());
156
+ deserializer.ReadProperty("source", result->source);
157
+ deserializer.ReadProperty("aggregates", result->aggregates);
158
+ deserializer.ReadProperty("unpivot_names", result->unpivot_names);
159
+ deserializer.ReadProperty("pivots", result->pivots);
160
+ deserializer.ReadProperty("groups", result->groups);
161
+ deserializer.ReadProperty("column_name_alias", result->column_name_alias);
162
+ deserializer.ReadProperty("include_nulls", result->include_nulls);
163
+ return std::move(result);
164
+ }
165
+
166
+ } // namespace duckdb
@@ -368,6 +368,29 @@ void SingleFileBlockManager::Write(FileBuffer &buffer, block_id_t block_id) {
368
368
  ChecksumAndWrite(buffer, BLOCK_START + block_id * Storage::BLOCK_ALLOC_SIZE);
369
369
  }
370
370
 
371
+ void SingleFileBlockManager::Truncate() {
372
+ BlockManager::Truncate();
373
+ idx_t blocks_to_truncate = 0;
374
+ // reverse iterate over the free-list
375
+ for (auto entry = free_list.rbegin(); entry != free_list.rend(); entry++) {
376
+ auto block_id = *entry;
377
+ if (block_id + 1 != max_block) {
378
+ break;
379
+ }
380
+ blocks_to_truncate++;
381
+ max_block--;
382
+ }
383
+ if (blocks_to_truncate == 0) {
384
+ // nothing to truncate
385
+ return;
386
+ }
387
+ // truncate the file
388
+ for (idx_t i = 0; i < blocks_to_truncate; i++) {
389
+ free_list.erase(max_block + i);
390
+ }
391
+ handle->Truncate(BLOCK_START + max_block * Storage::BLOCK_ALLOC_SIZE);
392
+ }
393
+
371
394
  vector<block_id_t> SingleFileBlockManager::GetFreeListBlocks() {
372
395
  vector<block_id_t> free_list_blocks;
373
396
 
@@ -1,10 +1,11 @@
1
1
  #include "duckdb/storage/statistics/string_stats.hpp"
2
- #include "duckdb/storage/statistics/base_statistics.hpp"
2
+
3
3
  #include "duckdb/common/field_writer.hpp"
4
- #include "utf8proc_wrapper.hpp"
5
4
  #include "duckdb/common/string_util.hpp"
6
5
  #include "duckdb/common/types/vector.hpp"
7
6
  #include "duckdb/main/error_manager.hpp"
7
+ #include "duckdb/storage/statistics/base_statistics.hpp"
8
+ #include "utf8proc_wrapper.hpp"
8
9
 
9
10
  namespace duckdb {
10
11
 
@@ -67,6 +68,24 @@ bool StringStats::CanContainUnicode(const BaseStatistics &stats) {
67
68
  return StringStats::GetDataUnsafe(stats).has_unicode;
68
69
  }
69
70
 
71
+ string GetStringMinMaxValue(const data_t data[]) {
72
+ idx_t len;
73
+ for (len = 0; len < StringStatsData::MAX_STRING_MINMAX_SIZE; len++) {
74
+ if (!data[len]) {
75
+ break;
76
+ }
77
+ }
78
+ return string(const_char_ptr_cast(data), len);
79
+ }
80
+
81
+ string StringStats::Min(const BaseStatistics &stats) {
82
+ return GetStringMinMaxValue(StringStats::GetDataUnsafe(stats).min);
83
+ }
84
+
85
+ string StringStats::Max(const BaseStatistics &stats) {
86
+ return GetStringMinMaxValue(StringStats::GetDataUnsafe(stats).max);
87
+ }
88
+
70
89
  void StringStats::ResetMaxStringLength(BaseStatistics &stats) {
71
90
  StringStats::GetDataUnsafe(stats).has_max_string_length = false;
72
91
  }
@@ -2,7 +2,7 @@
2
2
 
3
3
  namespace duckdb {
4
4
 
5
- const uint64_t VERSION_NUMBER = 51;
5
+ const uint64_t VERSION_NUMBER = 52;
6
6
 
7
7
  struct StorageVersionInfo {
8
8
  const char *version_name;
@@ -89,6 +89,10 @@ void ChunkConstantInfo::Serialize(Serializer &serializer) {
89
89
  serializer.Write<idx_t>(start);
90
90
  }
91
91
 
92
+ idx_t ChunkConstantInfo::GetCommittedDeletedCount(idx_t max_count) {
93
+ return delete_id < TRANSACTION_ID_START ? max_count : 0;
94
+ }
95
+
92
96
  unique_ptr<ChunkInfo> ChunkConstantInfo::Deserialize(Deserializer &source) {
93
97
  auto start = source.Read<idx_t>();
94
98
 
@@ -244,6 +248,19 @@ void ChunkVectorInfo::Serialize(Serializer &serializer) {
244
248
  serializer.WriteData(data_ptr_cast(deleted_tuples), sizeof(bool) * STANDARD_VECTOR_SIZE);
245
249
  }
246
250
 
251
+ idx_t ChunkVectorInfo::GetCommittedDeletedCount(idx_t max_count) {
252
+ if (!any_deleted) {
253
+ return 0;
254
+ }
255
+ idx_t delete_count = 0;
256
+ for (idx_t i = 0; i < max_count; i++) {
257
+ if (deleted[i] < TRANSACTION_ID_START) {
258
+ delete_count++;
259
+ }
260
+ }
261
+ return delete_count;
262
+ }
263
+
247
264
  unique_ptr<ChunkInfo> ChunkVectorInfo::Deserialize(Deserializer &source) {
248
265
  auto start = source.Read<idx_t>();
249
266
 
@@ -67,6 +67,21 @@ void VersionNode::SetStart(idx_t start) {
67
67
  }
68
68
  }
69
69
 
70
+ idx_t VersionNode::GetCommittedDeletedCount(idx_t count) {
71
+ idx_t deleted_count = 0;
72
+ for (idx_t r = 0, i = 0; r < count; r += STANDARD_VECTOR_SIZE, i++) {
73
+ if (!info[i]) {
74
+ continue;
75
+ }
76
+ idx_t max_count = MinValue<idx_t>(STANDARD_VECTOR_SIZE, count - r);
77
+ if (max_count == 0) {
78
+ break;
79
+ }
80
+ deleted_count += info[i]->GetCommittedDeletedCount(max_count);
81
+ }
82
+ return deleted_count;
83
+ }
84
+
70
85
  RowGroup::~RowGroup() {
71
86
  }
72
87
 
@@ -112,10 +127,6 @@ ColumnData &RowGroup::GetColumn(storage_t c) {
112
127
  return *columns[c];
113
128
  }
114
129
 
115
- DatabaseInstance &RowGroup::GetDatabase() {
116
- return GetCollection().GetDatabase();
117
- }
118
-
119
130
  BlockManager &RowGroup::GetBlockManager() {
120
131
  return GetCollection().GetBlockManager();
121
132
  }
@@ -270,7 +281,7 @@ unique_ptr<RowGroup> RowGroup::AlterType(RowGroupCollection &new_collection, con
270
281
  }
271
282
 
272
283
  unique_ptr<RowGroup> RowGroup::AddColumn(RowGroupCollection &new_collection, ColumnDefinition &new_column,
273
- ExpressionExecutor &executor, Expression *default_value, Vector &result) {
284
+ ExpressionExecutor &executor, Expression &default_value, Vector &result) {
274
285
  Verify();
275
286
 
276
287
  // construct a new column data for the new column
@@ -285,10 +296,8 @@ unique_ptr<RowGroup> RowGroup::AddColumn(RowGroupCollection &new_collection, Col
285
296
  added_column->InitializeAppend(state);
286
297
  for (idx_t i = 0; i < rows_to_write; i += STANDARD_VECTOR_SIZE) {
287
298
  idx_t rows_in_this_vector = MinValue<idx_t>(rows_to_write - i, STANDARD_VECTOR_SIZE);
288
- if (default_value) {
289
- dummy_chunk.SetCardinality(rows_in_this_vector);
290
- executor.ExecuteExpression(dummy_chunk, result);
291
- }
299
+ dummy_chunk.SetCardinality(rows_in_this_vector);
300
+ executor.ExecuteExpression(dummy_chunk, result);
292
301
  added_column->Append(state, result, rows_in_this_vector);
293
302
  }
294
303
  }
@@ -796,6 +805,13 @@ RowGroupWriteData RowGroup::WriteToDisk(PartialBlockManager &manager,
796
805
  return result;
797
806
  }
798
807
 
808
+ bool RowGroup::AllDeleted() {
809
+ if (!version_info) {
810
+ return false;
811
+ }
812
+ return version_info->GetCommittedDeletedCount(count) == count;
813
+ }
814
+
799
815
  RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, TableStatistics &global_stats) {
800
816
  RowGroupPointer row_group_pointer;
801
817
 
@@ -67,10 +67,6 @@ AttachedDatabase &RowGroupCollection::GetAttached() {
67
67
  return GetTableInfo().db;
68
68
  }
69
69
 
70
- DatabaseInstance &RowGroupCollection::GetDatabase() {
71
- return GetAttached().GetDatabase();
72
- }
73
-
74
70
  //===--------------------------------------------------------------------===//
75
71
  // Initialize
76
72
  //===--------------------------------------------------------------------===//
@@ -97,10 +93,6 @@ RowGroup *RowGroupCollection::GetRowGroup(int64_t index) {
97
93
  return (RowGroup *)row_groups->GetSegmentByIndex(index);
98
94
  }
99
95
 
100
- idx_t RowGroupCollection::RowGroupCount() {
101
- return row_groups->GetSegmentCount();
102
- }
103
-
104
96
  void RowGroupCollection::Verify() {
105
97
  #ifdef DEBUG
106
98
  idx_t current_total_rows = 0;
@@ -600,11 +592,24 @@ void RowGroupCollection::UpdateColumn(TransactionData transaction, Vector &row_i
600
592
  // Checkpoint
601
593
  //===--------------------------------------------------------------------===//
602
594
  void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &global_stats) {
603
- for (auto &row_group : row_groups->Segments()) {
604
- auto rowg_writer = writer.GetRowGroupWriter(row_group);
605
- auto pointer = row_group.Checkpoint(*rowg_writer, global_stats);
606
- writer.AddRowGroup(std::move(pointer), std::move(rowg_writer));
595
+ bool can_vacuum_deletes = info->indexes.Empty();
596
+ idx_t start = this->row_start;
597
+ auto segments = row_groups->MoveSegments();
598
+ auto l = row_groups->Lock();
599
+ for (auto &entry : segments) {
600
+ auto &row_group = *entry.node;
601
+ if (can_vacuum_deletes && row_group.AllDeleted()) {
602
+ row_group.CommitDrop();
603
+ continue;
604
+ }
605
+ row_group.MoveToCollection(*this, start);
606
+ auto row_group_writer = writer.GetRowGroupWriter(row_group);
607
+ auto pointer = row_group.Checkpoint(*row_group_writer, global_stats);
608
+ writer.AddRowGroup(std::move(pointer), std::move(row_group_writer));
609
+ row_groups->AppendSegment(l, std::move(entry.node));
610
+ start += row_group.count;
607
611
  }
612
+ total_rows = start;
608
613
  }
609
614
 
610
615
  //===--------------------------------------------------------------------===//
@@ -637,7 +642,7 @@ vector<ColumnSegmentInfo> RowGroupCollection::GetColumnSegmentInfo() {
637
642
  // Alter
638
643
  //===--------------------------------------------------------------------===//
639
644
  shared_ptr<RowGroupCollection> RowGroupCollection::AddColumn(ClientContext &context, ColumnDefinition &new_column,
640
- Expression *default_value) {
645
+ Expression &default_value) {
641
646
  idx_t new_column_idx = types.size();
642
647
  auto new_types = types;
643
648
  new_types.push_back(new_column.GetType());
@@ -647,11 +652,7 @@ shared_ptr<RowGroupCollection> RowGroupCollection::AddColumn(ClientContext &cont
647
652
  ExpressionExecutor executor(context);
648
653
  DataChunk dummy_chunk;
649
654
  Vector default_vector(new_column.GetType());
650
- if (!default_value) {
651
- FlatVector::Validity(default_vector).SetAllInvalid(STANDARD_VECTOR_SIZE);
652
- } else {
653
- executor.AddExpression(*default_value);
654
- }
655
+ executor.AddExpression(default_value);
655
656
 
656
657
  result->stats.InitializeAddColumn(stats, new_column.GetType());
657
658
  auto &new_column_stats = result->stats.GetStats(new_column_idx);
@@ -80,7 +80,7 @@ namespace duckdb_moodycamel { namespace details {
80
80
  static const thread_id_t invalid_thread_id2 = 0xFFFFFFFFU; // Not technically guaranteed to be invalid, but is never used in practice. Note that all Win32 thread IDs are presently multiples of 4.
81
81
  static inline thread_id_t thread_id() { return static_cast<thread_id_t>(::GetCurrentThreadId()); }
82
82
  } }
83
- #elif defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) || (defined(__APPLE__) && TARGET_OS_IPHONE)
83
+ #elif defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) || (defined(__APPLE__) && TARGET_OS_IPHONE) || defined(__MVS__)
84
84
  namespace duckdb_moodycamel { namespace details {
85
85
  static_assert(sizeof(std::thread::id) == 4 || sizeof(std::thread::id) == 8, "std::thread::id is expected to be either 4 or 8 bytes");
86
86
 
@@ -194,7 +194,7 @@ namespace duckdb_moodycamel { namespace details {
194
194
  // VS2013 doesn't support `thread_local`, and MinGW-w64 w/ POSIX threading has a crippling bug: http://sourceforge.net/p/mingw-w64/bugs/445
195
195
  // g++ <=4.7 doesn't support thread_local either.
196
196
  // Finally, iOS/ARM doesn't have support for it either, and g++/ARM allows it to compile but it's unconfirmed to actually work
197
- #if (!defined(_MSC_VER) || _MSC_VER >= 1900) && (!defined(__MINGW32__) && !defined(__MINGW64__) || !defined(__WINPTHREADS_VERSION)) && (!defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && (!defined(__APPLE__) || !TARGET_OS_IPHONE) && !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__)
197
+ #if (!defined(_MSC_VER) || _MSC_VER >= 1900) && (!defined(__MINGW32__) && !defined(__MINGW64__) || !defined(__WINPTHREADS_VERSION)) && (!defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && (!defined(__APPLE__) || !TARGET_OS_IPHONE) && !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__) && !defined(__MVS__)
198
198
  // Assume `thread_local` is fully supported in all other C++11 compilers/platforms
199
199
  //#define MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED // always disabled for now since several users report having problems with it on
200
200
  #endif
@@ -27,6 +27,8 @@ extern "C" {
27
27
  #elif defined(__unix__)
28
28
  #include <semaphore.h>
29
29
  #include <chrono>
30
+ #elif defined(__MVS__)
31
+ #include <zos-semaphore.h>
30
32
  #endif
31
33
 
32
34
  namespace duckdb_moodycamel
@@ -254,6 +256,80 @@ public:
254
256
  }
255
257
  }
256
258
  };
259
+ #elif defined(__MVS__)
260
+ //---------------------------------------------------------
261
+ // Semaphore (MVS aka z/OS)
262
+ //---------------------------------------------------------
263
+ class Semaphore
264
+ {
265
+ private:
266
+ sem_t m_sema;
267
+
268
+ Semaphore(const Semaphore& other) MOODYCAMEL_DELETE_FUNCTION;
269
+ Semaphore& operator=(const Semaphore& other) MOODYCAMEL_DELETE_FUNCTION;
270
+
271
+ public:
272
+ Semaphore(int initialCount = 0)
273
+ {
274
+ assert(initialCount >= 0);
275
+ int rc = sem_init(&m_sema, 0, initialCount);
276
+ assert(rc == 0);
277
+ (void)rc;
278
+ }
279
+
280
+ ~Semaphore()
281
+ {
282
+ sem_destroy(&m_sema);
283
+ }
284
+
285
+ bool wait()
286
+ {
287
+ // http://stackoverflow.com/questions/2013181/gdb-causes-sem-wait-to-fail-with-eintr-error
288
+ int rc;
289
+ do {
290
+ rc = sem_wait(&m_sema);
291
+ } while (rc == -1 && errno == EINTR);
292
+ return rc == 0;
293
+ }
294
+
295
+ bool try_wait()
296
+ {
297
+ int rc;
298
+ do {
299
+ rc = sem_trywait(&m_sema);
300
+ } while (rc == -1 && errno == EINTR);
301
+ return rc == 0;
302
+ }
303
+
304
+ bool timed_wait(std::uint64_t usecs)
305
+ {
306
+ struct timespec ts;
307
+ const int usecs_in_1_sec = 1000000;
308
+ const int nsecs_in_1_sec = 1000000000;
309
+
310
+ ts.tv_sec = usecs / usecs_in_1_sec;
311
+ ts.tv_nsec = (usecs % usecs_in_1_sec) * 1000;
312
+
313
+ int rc;
314
+ do {
315
+ rc = sem_timedwait(&m_sema, &ts);
316
+ } while (rc == -1 && errno == EINTR);
317
+ return rc == 0;
318
+ }
319
+
320
+ void signal()
321
+ {
322
+ while (sem_post(&m_sema) == -1);
323
+ }
324
+
325
+ void signal(int count)
326
+ {
327
+ while (count-- > 0)
328
+ {
329
+ while (sem_post(&m_sema) == -1);
330
+ }
331
+ }
332
+ };
257
333
  #else
258
334
  #error Unsupported platform! (No semaphore wrapper available)
259
335
  #endif
@@ -130,6 +130,8 @@ from_chars_result from_chars(const char *first, const char *last,
130
130
  #include <machine/endian.h>
131
131
  #elif defined(sun) || defined(__sun)
132
132
  #include <sys/byteorder.h>
133
+ #elif defined(__MVS__)
134
+ #include <sys/endian.h>
133
135
  #else
134
136
  #include <endian.h>
135
137
  #endif
@@ -183,7 +183,16 @@ using socket_t = SOCKET;
183
183
 
184
184
  #include <arpa/inet.h>
185
185
  #include <cstring>
186
+ #ifndef __MVS__
186
187
  #include <ifaddrs.h>
188
+ #endif
189
+ #ifdef __MVS__
190
+ #include <net/if.h>
191
+ #include <strings.h>
192
+ #ifndef NI_MAXHOST
193
+ #define NI_MAXHOST 1025
194
+ #endif
195
+ #endif
187
196
  #include <netdb.h>
188
197
  #include <netinet/in.h>
189
198
  #ifdef __linux__
@@ -2668,7 +2677,7 @@ inline bool bind_ip_address(socket_t sock, const char *host) {
2668
2677
  return ret;
2669
2678
  }
2670
2679
 
2671
- #if !defined _WIN32 && !defined ANDROID
2680
+ #if !defined _WIN32 && !defined ANDROID && !defined __MVS__
2672
2681
  #define USE_IF2IP
2673
2682
  #endif
2674
2683
 
@@ -1058,10 +1058,19 @@ typedef struct PGOnConflictClause {
1058
1058
  *
1059
1059
  * We don't currently support the SEARCH or CYCLE clause.
1060
1060
  */
1061
+
1062
+ typedef enum PGCTEMaterialize
1063
+ {
1064
+ PGCTEMaterializeDefault, /* no option specified */
1065
+ PGCTEMaterializeAlways, /* MATERIALIZED */
1066
+ PGCTEMaterializeNever /* NOT MATERIALIZED */
1067
+ } PGCTEMaterialize;
1068
+
1061
1069
  typedef struct PGCommonTableExpr {
1062
1070
  PGNodeTag type;
1063
1071
  char *ctename; /* query name (never qualified) */
1064
1072
  PGList *aliascolnames; /* optional list of column names */
1073
+ PGCTEMaterialize ctematerialized; /* is this an optimization fence? */
1065
1074
  /* SelectStmt/InsertStmt/etc before parse analysis, PGQuery afterwards: */
1066
1075
  PGNode *ctequery; /* the CTE's subquery */
1067
1076
  int location; /* token location, or -1 if unknown */
@@ -1049,6 +1049,7 @@ typedef union YYSTYPE
1049
1049
  PGAlias *alias;
1050
1050
  PGRangeVar *range;
1051
1051
  PGIntoClause *into;
1052
+ PGCTEMaterialize ctematerialize;
1052
1053
  PGWithClause *with;
1053
1054
  PGInferClause *infer;
1054
1055
  PGOnConflictClause *onconflict;
@@ -1068,7 +1069,7 @@ typedef union YYSTYPE
1068
1069
  PGInsertColumnOrder bynameorposition;
1069
1070
  }
1070
1071
  /* Line 1529 of yacc.c. */
1071
- #line 1072 "third_party/libpg_query/grammar/grammar_out.hpp"
1072
+ #line 1073 "third_party/libpg_query/grammar/grammar_out.hpp"
1072
1073
  YYSTYPE;
1073
1074
  # define yystype YYSTYPE /* obsolescent; will be withdrawn */
1074
1075
  # define YYSTYPE_IS_DECLARED 1