duckdb 1.1.2-dev2.0 → 1.1.2-dev6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (289) hide show
  1. package/binding.gyp +4 -2
  2. package/package.json +1 -1
  3. package/src/duckdb/extension/icu/third_party/icu/common/putil.cpp +0 -5
  4. package/src/duckdb/extension/icu/third_party/icu/common/rbbiscan.cpp +1 -1
  5. package/src/duckdb/extension/icu/third_party/icu/common/rbbitblb.cpp +1 -1
  6. package/src/duckdb/extension/icu/third_party/icu/common/ucurr.cpp +1 -1
  7. package/src/duckdb/extension/icu/third_party/icu/common/uresbund.cpp +1 -1
  8. package/src/duckdb/extension/icu/third_party/icu/common/uresimp.h +31 -31
  9. package/src/duckdb/extension/icu/third_party/icu/common/ustring.cpp +1 -1
  10. package/src/duckdb/extension/icu/third_party/icu/common/uvector.cpp +1 -1
  11. package/src/duckdb/extension/icu/third_party/icu/i18n/coleitr.cpp +12 -12
  12. package/src/duckdb/extension/icu/third_party/icu/i18n/format.cpp +1 -1
  13. package/src/duckdb/extension/icu/third_party/icu/i18n/listformatter.cpp +4 -4
  14. package/src/duckdb/extension/icu/third_party/icu/i18n/number_decimalquantity.h +1 -1
  15. package/src/duckdb/extension/icu/third_party/icu/i18n/tzgnames.cpp +1 -1
  16. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/coleitr.h +28 -28
  17. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/format.h +7 -7
  18. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/ucol.h +1 -1
  19. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/ucoleitr.h +41 -41
  20. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/umsg.h +41 -41
  21. package/src/duckdb/extension/icu/third_party/icu/i18n/usrchimp.h +3 -3
  22. package/src/duckdb/extension/json/include/json_common.hpp +1 -1
  23. package/src/duckdb/extension/json/json_functions/json_structure.cpp +13 -7
  24. package/src/duckdb/extension/parquet/column_writer.cpp +2 -1
  25. package/src/duckdb/extension/parquet/geo_parquet.cpp +24 -9
  26. package/src/duckdb/extension/parquet/include/geo_parquet.hpp +3 -1
  27. package/src/duckdb/extension/parquet/include/parquet_reader.hpp +1 -0
  28. package/src/duckdb/extension/parquet/include/parquet_rle_bp_decoder.hpp +1 -1
  29. package/src/duckdb/extension/parquet/include/templated_column_reader.hpp +0 -4
  30. package/src/duckdb/extension/parquet/parquet_extension.cpp +20 -6
  31. package/src/duckdb/extension/parquet/parquet_reader.cpp +1 -2
  32. package/src/duckdb/extension/parquet/parquet_writer.cpp +1 -1
  33. package/src/duckdb/extension/parquet/serialize_parquet.cpp +0 -2
  34. package/src/duckdb/src/catalog/catalog_entry/duck_schema_entry.cpp +8 -1
  35. package/src/duckdb/src/catalog/default/default_functions.cpp +5 -5
  36. package/src/duckdb/src/common/allocator.cpp +3 -2
  37. package/src/duckdb/src/common/arrow/arrow_appender.cpp +1 -0
  38. package/src/duckdb/src/common/arrow/arrow_converter.cpp +11 -0
  39. package/src/duckdb/src/common/arrow/schema_metadata.cpp +6 -4
  40. package/src/duckdb/src/common/enum_util.cpp +33 -0
  41. package/src/duckdb/src/common/exception.cpp +3 -0
  42. package/src/duckdb/src/common/extra_type_info.cpp +1 -44
  43. package/src/duckdb/src/common/field_writer.cpp +97 -0
  44. package/src/duckdb/src/common/render_tree.cpp +7 -5
  45. package/src/duckdb/src/common/row_operations/row_match.cpp +359 -0
  46. package/src/duckdb/src/common/serializer/buffered_deserializer.cpp +27 -0
  47. package/src/duckdb/src/common/serializer/buffered_serializer.cpp +36 -0
  48. package/src/duckdb/src/common/serializer/format_serializer.cpp +15 -0
  49. package/src/duckdb/src/common/serializer.cpp +24 -0
  50. package/src/duckdb/src/common/sort/comparators.cpp +2 -2
  51. package/src/duckdb/src/common/types/bit.cpp +57 -34
  52. package/src/duckdb/src/common/types/data_chunk.cpp +32 -29
  53. package/src/duckdb/src/common/types/vector_cache.cpp +12 -6
  54. package/src/duckdb/src/common/vector_operations/comparison_operators.cpp +14 -0
  55. package/src/duckdb/src/core_functions/aggregate/distributive/bitstring_agg.cpp +20 -1
  56. package/src/duckdb/src/core_functions/aggregate/distributive/minmax.cpp +2 -2
  57. package/src/duckdb/src/core_functions/aggregate/holistic/approx_top_k.cpp +32 -7
  58. package/src/duckdb/src/core_functions/function_list.cpp +1 -2
  59. package/src/duckdb/src/core_functions/scalar/bit/bitstring.cpp +23 -5
  60. package/src/duckdb/src/core_functions/scalar/date/date_diff.cpp +12 -6
  61. package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +1 -1
  62. package/src/duckdb/src/execution/expression_executor/execute_between.cpp +4 -3
  63. package/src/duckdb/src/execution/expression_executor/execute_case.cpp +4 -3
  64. package/src/duckdb/src/execution/expression_executor/execute_cast.cpp +2 -1
  65. package/src/duckdb/src/execution/expression_executor/execute_comparison.cpp +3 -2
  66. package/src/duckdb/src/execution/expression_executor/execute_conjunction.cpp +2 -1
  67. package/src/duckdb/src/execution/expression_executor/execute_function.cpp +2 -1
  68. package/src/duckdb/src/execution/expression_executor/execute_operator.cpp +3 -2
  69. package/src/duckdb/src/execution/expression_executor/execute_reference.cpp +1 -1
  70. package/src/duckdb/src/execution/expression_executor.cpp +9 -3
  71. package/src/duckdb/src/execution/expression_executor_state.cpp +11 -9
  72. package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +238 -0
  73. package/src/duckdb/src/execution/index/art/plan_art.cpp +94 -0
  74. package/src/duckdb/src/execution/index/index_type_set.cpp +4 -1
  75. package/src/duckdb/src/execution/join_hashtable.cpp +7 -8
  76. package/src/duckdb/src/execution/operator/aggregate/physical_hash_aggregate.cpp +6 -4
  77. package/src/duckdb/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp +4 -4
  78. package/src/duckdb/src/execution/operator/csv_scanner/scanner/base_scanner.cpp +1 -1
  79. package/src/duckdb/src/execution/operator/csv_scanner/scanner/csv_schema.cpp +44 -5
  80. package/src/duckdb/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +28 -24
  81. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp +25 -26
  82. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp +5 -3
  83. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/header_detection.cpp +4 -4
  84. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_detection.cpp +2 -2
  85. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_refinement.cpp +1 -1
  86. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_replacement.cpp +1 -1
  87. package/src/duckdb/src/execution/operator/csv_scanner/state_machine/csv_state_machine.cpp +1 -1
  88. package/src/duckdb/src/execution/operator/csv_scanner/state_machine/csv_state_machine_cache.cpp +2 -2
  89. package/src/duckdb/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp +1 -1
  90. package/src/duckdb/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +1 -1
  91. package/src/duckdb/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +73 -27
  92. package/src/duckdb/src/execution/operator/helper/physical_buffered_collector.cpp +1 -1
  93. package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +695 -0
  94. package/src/duckdb/src/execution/operator/persistent/buffered_csv_reader.cpp +1487 -0
  95. package/src/duckdb/src/execution/operator/persistent/csv_buffer.cpp +72 -0
  96. package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +158 -0
  97. package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +280 -0
  98. package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +666 -0
  99. package/src/duckdb/src/execution/operator/persistent/physical_export.cpp +14 -4
  100. package/src/duckdb/src/execution/operator/schema/physical_create_index.cpp +207 -0
  101. package/src/duckdb/src/execution/partitionable_hashtable.cpp +207 -0
  102. package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +6 -1
  103. package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +0 -4
  104. package/src/duckdb/src/execution/physical_plan/plan_create_index.cpp +14 -87
  105. package/src/duckdb/src/execution/physical_plan/plan_export.cpp +1 -1
  106. package/src/duckdb/src/execution/physical_plan/plan_get.cpp +1 -1
  107. package/src/duckdb/src/execution/reservoir_sample.cpp +1 -1
  108. package/src/duckdb/src/execution/window_executor.cpp +3 -3
  109. package/src/duckdb/src/function/pragma/pragma_queries.cpp +1 -1
  110. package/src/duckdb/src/function/scalar/strftime_format.cpp +1 -2
  111. package/src/duckdb/src/function/scalar/string/concat.cpp +118 -151
  112. package/src/duckdb/src/function/table/arrow.cpp +13 -0
  113. package/src/duckdb/src/function/table/arrow_conversion.cpp +12 -7
  114. package/src/duckdb/src/function/table/copy_csv.cpp +1 -1
  115. package/src/duckdb/src/function/table/read_csv.cpp +2 -30
  116. package/src/duckdb/src/function/table/sniff_csv.cpp +2 -1
  117. package/src/duckdb/src/function/table/system/duckdb_secrets.cpp +15 -7
  118. package/src/duckdb/src/function/table/version/pragma_version.cpp +3 -3
  119. package/src/duckdb/src/include/duckdb/catalog/catalog_entry_retriever.hpp +1 -1
  120. package/src/duckdb/src/include/duckdb/common/atomic.hpp +13 -1
  121. package/src/duckdb/src/include/duckdb/common/bitpacking.hpp +3 -4
  122. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +8 -0
  123. package/src/duckdb/src/include/duckdb/common/enums/metric_type.hpp +2 -0
  124. package/src/duckdb/src/include/duckdb/common/exception.hpp +10 -0
  125. package/src/duckdb/src/include/duckdb/common/extra_type_info/enum_type_info.hpp +53 -0
  126. package/src/duckdb/src/include/duckdb/common/insertion_order_preserving_map.hpp +5 -5
  127. package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +5 -0
  128. package/src/duckdb/src/include/duckdb/common/types/bit.hpp +36 -33
  129. package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +10 -13
  130. package/src/duckdb/src/include/duckdb/common/types/uhugeint.hpp +1 -1
  131. package/src/duckdb/src/include/duckdb/common/types/vector_cache.hpp +7 -5
  132. package/src/duckdb/src/include/duckdb/common/windows_undefs.hpp +2 -1
  133. package/src/duckdb/src/include/duckdb/core_functions/aggregate/minmax_n_helpers.hpp +2 -0
  134. package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +1 -1
  135. package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +0 -6
  136. package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +1 -1
  137. package/src/duckdb/src/include/duckdb/execution/expression_executor_state.hpp +3 -2
  138. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +3 -0
  139. package/src/duckdb/src/include/duckdb/execution/index/index_type.hpp +16 -1
  140. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp +4 -4
  141. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp +4 -2
  142. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_schema.hpp +3 -2
  143. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/sniffer/csv_sniffer.hpp +91 -36
  144. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/sniffer/sniff_result.hpp +36 -0
  145. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +1 -1
  146. package/src/duckdb/src/include/duckdb/execution/operator/join/perfect_hash_join_executor.hpp +0 -1
  147. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_export.hpp +2 -5
  148. package/src/duckdb/src/include/duckdb/function/table_function.hpp +1 -1
  149. package/src/duckdb/src/include/duckdb/main/database.hpp +5 -0
  150. package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +1 -0
  151. package/src/duckdb/src/include/duckdb/main/profiling_info.hpp +20 -22
  152. package/src/duckdb/src/include/duckdb/main/query_profiler.hpp +7 -9
  153. package/src/duckdb/src/include/duckdb/main/secret/secret.hpp +8 -1
  154. package/src/duckdb/src/include/duckdb/main/table_description.hpp +14 -0
  155. package/src/duckdb/src/include/duckdb/optimizer/unnest_rewriter.hpp +5 -5
  156. package/src/duckdb/src/include/duckdb/parser/parsed_data/exported_table_data.hpp +15 -5
  157. package/src/duckdb/src/include/duckdb/parser/transformer.hpp +2 -0
  158. package/src/duckdb/src/include/duckdb/planner/expression_binder/order_binder.hpp +4 -0
  159. package/src/duckdb/src/include/duckdb/planner/operator/logical_export.hpp +10 -13
  160. package/src/duckdb/src/include/duckdb/planner/table_filter.hpp +1 -0
  161. package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp +2 -2
  162. package/src/duckdb/src/include/duckdb/storage/standard_buffer_manager.hpp +1 -1
  163. package/src/duckdb/src/include/duckdb/storage/statistics/distinct_statistics.hpp +0 -2
  164. package/src/duckdb/src/include/duckdb/storage/table/segment_tree.hpp +1 -0
  165. package/src/duckdb/src/include/duckdb/transaction/duck_transaction.hpp +5 -1
  166. package/src/duckdb/src/include/duckdb.h +2 -2
  167. package/src/duckdb/src/main/appender.cpp +3 -0
  168. package/src/duckdb/src/main/capi/profiling_info-c.cpp +5 -2
  169. package/src/duckdb/src/main/client_context.cpp +8 -2
  170. package/src/duckdb/src/main/connection.cpp +1 -1
  171. package/src/duckdb/src/main/database.cpp +13 -0
  172. package/src/duckdb/src/main/extension/extension_helper.cpp +1 -1
  173. package/src/duckdb/src/main/extension/extension_install.cpp +9 -1
  174. package/src/duckdb/src/main/extension/extension_load.cpp +3 -2
  175. package/src/duckdb/src/main/extension_install_info.cpp +1 -1
  176. package/src/duckdb/src/main/profiling_info.cpp +78 -58
  177. package/src/duckdb/src/main/query_profiler.cpp +79 -89
  178. package/src/duckdb/src/main/relation/read_csv_relation.cpp +1 -1
  179. package/src/duckdb/src/main/secret/secret.cpp +2 -1
  180. package/src/duckdb/src/main/secret/secret_manager.cpp +14 -0
  181. package/src/duckdb/src/optimizer/cte_filter_pusher.cpp +4 -2
  182. package/src/duckdb/src/optimizer/deliminator.cpp +0 -7
  183. package/src/duckdb/src/optimizer/in_clause_rewriter.cpp +7 -0
  184. package/src/duckdb/src/optimizer/pushdown/pushdown_left_join.cpp +4 -1
  185. package/src/duckdb/src/optimizer/unnest_rewriter.cpp +21 -21
  186. package/src/duckdb/src/parallel/task_scheduler.cpp +9 -0
  187. package/src/duckdb/src/parser/parsed_data/exported_table_data.cpp +22 -0
  188. package/src/duckdb/src/parser/parsed_expression_iterator.cpp +3 -0
  189. package/src/duckdb/src/parser/statement/insert_statement.cpp +7 -1
  190. package/src/duckdb/src/parser/transform/expression/transform_boolean_test.cpp +1 -1
  191. package/src/duckdb/src/parser/transform/helpers/transform_typename.cpp +89 -87
  192. package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +2 -2
  193. package/src/duckdb/src/planner/binder/expression/bind_macro_expression.cpp +4 -9
  194. package/src/duckdb/src/planner/binder/query_node/bind_select_node.cpp +4 -0
  195. package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +2 -2
  196. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +4 -1
  197. package/src/duckdb/src/planner/binder/statement/bind_export.cpp +4 -3
  198. package/src/duckdb/src/planner/expression_binder/order_binder.cpp +13 -3
  199. package/src/duckdb/src/planner/expression_binder.cpp +1 -1
  200. package/src/duckdb/src/planner/operator/logical_export.cpp +28 -0
  201. package/src/duckdb/src/planner/table_binding.cpp +1 -2
  202. package/src/duckdb/src/planner/table_filter.cpp +6 -2
  203. package/src/duckdb/src/storage/buffer/buffer_pool.cpp +2 -1
  204. package/src/duckdb/src/storage/checkpoint_manager.cpp +1 -1
  205. package/src/duckdb/src/storage/compression/bitpacking.cpp +7 -3
  206. package/src/duckdb/src/storage/compression/dictionary_compression.cpp +1 -1
  207. package/src/duckdb/src/storage/metadata/metadata_manager.cpp +2 -2
  208. package/src/duckdb/src/storage/serialization/serialize_logical_operator.cpp +16 -0
  209. package/src/duckdb/src/storage/serialization/serialize_nodes.cpp +29 -0
  210. package/src/duckdb/src/storage/serialization/serialize_parse_info.cpp +15 -0
  211. package/src/duckdb/src/storage/single_file_block_manager.cpp +2 -1
  212. package/src/duckdb/src/storage/statistics/distinct_statistics.cpp +3 -5
  213. package/src/duckdb/src/storage/storage_info.cpp +4 -4
  214. package/src/duckdb/src/storage/table/row_group_collection.cpp +1 -1
  215. package/src/duckdb/src/storage/table/row_version_manager.cpp +5 -1
  216. package/src/duckdb/src/storage/temporary_file_manager.cpp +1 -1
  217. package/src/duckdb/src/transaction/duck_transaction.cpp +15 -14
  218. package/src/duckdb/third_party/brotli/common/brotli_platform.h +1 -1
  219. package/src/duckdb/third_party/brotli/dec/decode.cpp +1 -1
  220. package/src/duckdb/third_party/brotli/enc/memory.cpp +4 -4
  221. package/src/duckdb/third_party/fsst/libfsst.cpp +1 -1
  222. package/src/duckdb/third_party/hyperloglog/sds.cpp +1 -1
  223. package/src/duckdb/third_party/hyperloglog/sds.hpp +1 -1
  224. package/src/duckdb/third_party/libpg_query/include/common/keywords.hpp +1 -1
  225. package/src/duckdb/third_party/libpg_query/include/datatype/timestamp.hpp +1 -1
  226. package/src/duckdb/third_party/libpg_query/include/mb/pg_wchar.hpp +1 -1
  227. package/src/duckdb/third_party/libpg_query/include/nodes/bitmapset.hpp +1 -1
  228. package/src/duckdb/third_party/libpg_query/include/nodes/lockoptions.hpp +1 -1
  229. package/src/duckdb/third_party/libpg_query/include/nodes/makefuncs.hpp +1 -1
  230. package/src/duckdb/third_party/libpg_query/include/nodes/pg_list.hpp +1 -1
  231. package/src/duckdb/third_party/libpg_query/include/nodes/value.hpp +1 -1
  232. package/src/duckdb/third_party/libpg_query/include/parser/gramparse.hpp +1 -1
  233. package/src/duckdb/third_party/libpg_query/include/parser/parser.hpp +1 -1
  234. package/src/duckdb/third_party/libpg_query/include/parser/scanner.hpp +1 -1
  235. package/src/duckdb/third_party/libpg_query/include/parser/scansup.hpp +1 -1
  236. package/src/duckdb/third_party/libpg_query/include/pg_functions.hpp +1 -1
  237. package/src/duckdb/third_party/libpg_query/pg_functions.cpp +1 -1
  238. package/src/duckdb/third_party/libpg_query/src_backend_nodes_list.cpp +1 -1
  239. package/src/duckdb/third_party/libpg_query/src_backend_nodes_makefuncs.cpp +1 -1
  240. package/src/duckdb/third_party/libpg_query/src_backend_nodes_value.cpp +1 -1
  241. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +1964 -1964
  242. package/src/duckdb/third_party/libpg_query/src_backend_parser_parser.cpp +1 -1
  243. package/src/duckdb/third_party/libpg_query/src_backend_parser_scansup.cpp +1 -1
  244. package/src/duckdb/third_party/libpg_query/src_common_keywords.cpp +1 -1
  245. package/src/duckdb/third_party/lz4/lz4.cpp +1 -1
  246. package/src/duckdb/third_party/mbedtls/include/des_alt.h +1 -1
  247. package/src/duckdb/third_party/mbedtls/include/mbedtls/aes_alt.h +1 -1
  248. package/src/duckdb/third_party/mbedtls/include/mbedtls/aria_alt.h +1 -1
  249. package/src/duckdb/third_party/mbedtls/include/mbedtls/asn1write.h +1 -1
  250. package/src/duckdb/third_party/mbedtls/include/mbedtls/camellia_alt.h +1 -1
  251. package/src/duckdb/third_party/mbedtls/include/mbedtls/ccm_alt.h +1 -1
  252. package/src/duckdb/third_party/mbedtls/include/mbedtls/chacha20.h +1 -1
  253. package/src/duckdb/third_party/mbedtls/include/mbedtls/chachapoly.h +1 -1
  254. package/src/duckdb/third_party/mbedtls/include/mbedtls/cmac.h +1 -1
  255. package/src/duckdb/third_party/mbedtls/include/mbedtls/config_psa.h +1 -1
  256. package/src/duckdb/third_party/mbedtls/include/mbedtls/ecdsa.h +1 -1
  257. package/src/duckdb/third_party/mbedtls/include/mbedtls/ecp.h +1 -1
  258. package/src/duckdb/third_party/mbedtls/include/mbedtls/gcm_alt.h +1 -1
  259. package/src/duckdb/third_party/mbedtls/include/mbedtls/md5.h +1 -1
  260. package/src/duckdb/third_party/mbedtls/include/mbedtls/nist_kw.h +1 -1
  261. package/src/duckdb/third_party/mbedtls/include/mbedtls/pkcs12.h +1 -1
  262. package/src/duckdb/third_party/mbedtls/include/mbedtls/pkcs5.h +1 -1
  263. package/src/duckdb/third_party/mbedtls/include/mbedtls/psa_util.h +1 -1
  264. package/src/duckdb/third_party/mbedtls/include/mbedtls/ripemd160.h +1 -1
  265. package/src/duckdb/third_party/mbedtls/include/mbedtls/threading.h +1 -1
  266. package/src/duckdb/third_party/mbedtls/include/mbedtls/timing.h +1 -1
  267. package/src/duckdb/third_party/mbedtls/include/platform_alt.h +1 -1
  268. package/src/duckdb/third_party/mbedtls/include/psa/crypto.h +1 -1
  269. package/src/duckdb/third_party/mbedtls/include/rsa_alt.h +1 -1
  270. package/src/duckdb/third_party/mbedtls/include/sha1_alt.h +1 -1
  271. package/src/duckdb/third_party/mbedtls/include/sha256_alt.h +1 -1
  272. package/src/duckdb/third_party/mbedtls/include/sha512_alt.h +1 -1
  273. package/src/duckdb/third_party/mbedtls/include/ssl_misc.h +1 -1
  274. package/src/duckdb/third_party/mbedtls/library/aesni.h +1 -1
  275. package/src/duckdb/third_party/mbedtls/library/padlock.h +1 -1
  276. package/src/duckdb/third_party/miniz/miniz.cpp +1 -1
  277. package/src/duckdb/third_party/parquet/parquet_types.cpp +1 -1
  278. package/src/duckdb/third_party/parquet/windows_compatibility.h +1 -1
  279. package/src/duckdb/third_party/pcg/pcg_extras.hpp +1 -1
  280. package/src/duckdb/third_party/pcg/pcg_uint128.hpp +1 -1
  281. package/src/duckdb/third_party/skiplist/Node.h +4 -4
  282. package/src/duckdb/third_party/snappy/snappy.cc +1 -1
  283. package/src/duckdb/third_party/snappy/snappy_version.hpp +1 -1
  284. package/src/duckdb/third_party/thrift/thrift/thrift-config.h +1 -1
  285. package/src/duckdb/third_party/zstd/decompress/zstd_decompress_block.cpp +1 -1
  286. package/src/duckdb/third_party/zstd/include/zstd_static.h +1 -1
  287. package/src/duckdb/ub_src_execution_index_art.cpp +2 -0
  288. package/src/duckdb/ub_src_parser_parsed_data.cpp +2 -0
  289. package/src/duckdb/ub_src_planner_operator.cpp +2 -0
@@ -28,6 +28,14 @@ struct DateDiff {
28
28
  });
29
29
  }
30
30
 
31
+ // We need to truncate down, not towards 0
32
+ static inline int64_t Truncate(int64_t value, int64_t units) {
33
+ return (value + (value < 0)) / units - (value < 0);
34
+ }
35
+ static inline int64_t Diff(int64_t start, int64_t end, int64_t units) {
36
+ return Truncate(end, units) - Truncate(start, units);
37
+ }
38
+
31
39
  struct YearOperator {
32
40
  template <class TA, class TB, class TR>
33
41
  static inline TR Operation(TA startdate, TB enddate) {
@@ -204,30 +212,28 @@ template <>
204
212
  int64_t DateDiff::MillisecondsOperator::Operation(timestamp_t startdate, timestamp_t enddate) {
205
213
  D_ASSERT(Timestamp::IsFinite(startdate));
206
214
  D_ASSERT(Timestamp::IsFinite(enddate));
207
- return Timestamp::GetEpochMs(enddate) - Timestamp::GetEpochMs(startdate);
215
+ return Diff(startdate.value, enddate.value, Interval::MICROS_PER_MSEC);
208
216
  }
209
217
 
210
218
  template <>
211
219
  int64_t DateDiff::SecondsOperator::Operation(timestamp_t startdate, timestamp_t enddate) {
212
220
  D_ASSERT(Timestamp::IsFinite(startdate));
213
221
  D_ASSERT(Timestamp::IsFinite(enddate));
214
- return Timestamp::GetEpochSeconds(enddate) - Timestamp::GetEpochSeconds(startdate);
222
+ return Diff(startdate.value, enddate.value, Interval::MICROS_PER_SEC);
215
223
  }
216
224
 
217
225
  template <>
218
226
  int64_t DateDiff::MinutesOperator::Operation(timestamp_t startdate, timestamp_t enddate) {
219
227
  D_ASSERT(Timestamp::IsFinite(startdate));
220
228
  D_ASSERT(Timestamp::IsFinite(enddate));
221
- return Timestamp::GetEpochSeconds(enddate) / Interval::SECS_PER_MINUTE -
222
- Timestamp::GetEpochSeconds(startdate) / Interval::SECS_PER_MINUTE;
229
+ return Diff(startdate.value, enddate.value, Interval::MICROS_PER_MINUTE);
223
230
  }
224
231
 
225
232
  template <>
226
233
  int64_t DateDiff::HoursOperator::Operation(timestamp_t startdate, timestamp_t enddate) {
227
234
  D_ASSERT(Timestamp::IsFinite(startdate));
228
235
  D_ASSERT(Timestamp::IsFinite(enddate));
229
- return Timestamp::GetEpochSeconds(enddate) / Interval::SECS_PER_HOUR -
230
- Timestamp::GetEpochSeconds(startdate) / Interval::SECS_PER_HOUR;
236
+ return Diff(startdate.value, enddate.value, Interval::MICROS_PER_HOUR);
231
237
  }
232
238
 
233
239
  // TIME specialisations
@@ -412,7 +412,7 @@ struct DatePart {
412
412
  D_ASSERT(input.ColumnCount() == 1);
413
413
 
414
414
  UnaryExecutor::Execute<int64_t, timestamp_t>(input.data[0], result, input.size(), [&](int64_t input) {
415
- // milisecond amounts provided to epoch_ms should never be considered infinite
415
+ // millisecond amounts provided to epoch_ms should never be considered infinite
416
416
  // instead such values will just throw when converted to microseconds
417
417
  return Timestamp::FromEpochMsPossiblyInfinite(input);
418
418
  });
@@ -89,9 +89,10 @@ static idx_t BetweenLoopTypeSwitch(Vector &input, Vector &lower, Vector &upper,
89
89
  unique_ptr<ExpressionState> ExpressionExecutor::InitializeState(const BoundBetweenExpression &expr,
90
90
  ExpressionExecutorState &root) {
91
91
  auto result = make_uniq<ExpressionState>(expr, root);
92
- result->AddChild(expr.input.get());
93
- result->AddChild(expr.lower.get());
94
- result->AddChild(expr.upper.get());
92
+ result->AddChild(*expr.input);
93
+ result->AddChild(*expr.lower);
94
+ result->AddChild(*expr.upper);
95
+
95
96
  result->Finalize();
96
97
  return result;
97
98
  }
@@ -18,10 +18,11 @@ unique_ptr<ExpressionState> ExpressionExecutor::InitializeState(const BoundCaseE
18
18
  ExpressionExecutorState &root) {
19
19
  auto result = make_uniq<CaseExpressionState>(expr, root);
20
20
  for (auto &case_check : expr.case_checks) {
21
- result->AddChild(case_check.when_expr.get());
22
- result->AddChild(case_check.then_expr.get());
21
+ result->AddChild(*case_check.when_expr);
22
+ result->AddChild(*case_check.then_expr);
23
23
  }
24
- result->AddChild(expr.else_expr.get());
24
+ result->AddChild(*expr.else_expr);
25
+
25
26
  result->Finalize();
26
27
  return std::move(result);
27
28
  }
@@ -8,8 +8,9 @@ namespace duckdb {
8
8
  unique_ptr<ExpressionState> ExpressionExecutor::InitializeState(const BoundCastExpression &expr,
9
9
  ExpressionExecutorState &root) {
10
10
  auto result = make_uniq<ExecuteFunctionState>(expr, root);
11
- result->AddChild(expr.child.get());
11
+ result->AddChild(*expr.child);
12
12
  result->Finalize();
13
+
13
14
  if (expr.bound_cast.init_local_state) {
14
15
  CastLocalStateParameters parameters(root.executor->GetContext(), expr.bound_cast.cast_data);
15
16
  result->local_state = expr.bound_cast.init_local_state(parameters);
@@ -12,8 +12,9 @@ namespace duckdb {
12
12
  unique_ptr<ExpressionState> ExpressionExecutor::InitializeState(const BoundComparisonExpression &expr,
13
13
  ExpressionExecutorState &root) {
14
14
  auto result = make_uniq<ExpressionState>(expr, root);
15
- result->AddChild(expr.left.get());
16
- result->AddChild(expr.right.get());
15
+ result->AddChild(*expr.left);
16
+ result->AddChild(*expr.right);
17
+
17
18
  result->Finalize();
18
19
  return result;
19
20
  }
@@ -18,8 +18,9 @@ unique_ptr<ExpressionState> ExpressionExecutor::InitializeState(const BoundConju
18
18
  ExpressionExecutorState &root) {
19
19
  auto result = make_uniq<ConjunctionState>(expr, root);
20
20
  for (auto &child : expr.children) {
21
- result->AddChild(child.get());
21
+ result->AddChild(*child);
22
22
  }
23
+
23
24
  result->Finalize();
24
25
  return std::move(result);
25
26
  }
@@ -14,8 +14,9 @@ unique_ptr<ExpressionState> ExpressionExecutor::InitializeState(const BoundFunct
14
14
  ExpressionExecutorState &root) {
15
15
  auto result = make_uniq<ExecuteFunctionState>(expr, root);
16
16
  for (auto &child : expr.children) {
17
- result->AddChild(child.get());
17
+ result->AddChild(*child);
18
18
  }
19
+
19
20
  result->Finalize();
20
21
  if (expr.function.init_local_state) {
21
22
  result->local_state = expr.function.init_local_state(*result, expr, expr.bind_info.get());
@@ -8,8 +8,9 @@ unique_ptr<ExpressionState> ExpressionExecutor::InitializeState(const BoundOpera
8
8
  ExpressionExecutorState &root) {
9
9
  auto result = make_uniq<ExpressionState>(expr, root);
10
10
  for (auto &child : expr.children) {
11
- result->AddChild(child.get());
11
+ result->AddChild(*child);
12
12
  }
13
+
13
14
  result->Finalize();
14
15
  return result;
15
16
  }
@@ -33,7 +34,7 @@ void ExpressionExecutor::Execute(const BoundOperatorExpression &expr, Expression
33
34
  intermediate.Reference(false_val);
34
35
 
35
36
  // in rhs is a list of constants
36
- // for every child, OR the result of the comparision with the left
37
+ // for every child, OR the result of the comparison with the left
37
38
  // to get the overall result.
38
39
  for (idx_t child = 1; child < expr.children.size(); child++) {
39
40
  Vector vector_to_check(expr.children[child]->return_type);
@@ -6,7 +6,7 @@ namespace duckdb {
6
6
  unique_ptr<ExpressionState> ExpressionExecutor::InitializeState(const BoundReferenceExpression &expr,
7
7
  ExpressionExecutorState &root) {
8
8
  auto result = make_uniq<ExpressionState>(expr, root);
9
- result->Finalize(true);
9
+ result->Finalize();
10
10
  return result;
11
11
  }
12
12
 
@@ -170,10 +170,16 @@ unique_ptr<ExpressionState> ExpressionExecutor::InitializeState(const Expression
170
170
  void ExpressionExecutor::Execute(const Expression &expr, ExpressionState *state, const SelectionVector *sel,
171
171
  idx_t count, Vector &result) {
172
172
  #ifdef DEBUG
173
- // the result vector has to be used for the first time or has to be reset
174
- // otherwise, the validity mask might contain previous (now incorrect) data
173
+ // The result vector must be used for the first time, or must be reset.
174
+ // Otherwise, the validity mask can contain previous (now incorrect) data.
175
175
  if (result.GetVectorType() == VectorType::FLAT_VECTOR) {
176
- D_ASSERT(FlatVector::Validity(result).CheckAllValid(count));
176
+
177
+ // We do not initialize vector caches for these expressions.
178
+ if (expr.GetExpressionClass() != ExpressionClass::BOUND_REF &&
179
+ expr.GetExpressionClass() != ExpressionClass::BOUND_CONSTANT &&
180
+ expr.GetExpressionClass() != ExpressionClass::BOUND_PARAMETER) {
181
+ D_ASSERT(FlatVector::Validity(result).CheckAllValid(count));
182
+ }
177
183
  }
178
184
  #endif
179
185
 
@@ -6,20 +6,22 @@
6
6
 
7
7
  namespace duckdb {
8
8
 
9
- void ExpressionState::AddChild(Expression *expr) {
10
- types.push_back(expr->return_type);
11
- child_states.push_back(ExpressionExecutor::InitializeState(*expr, root));
9
+ void ExpressionState::AddChild(Expression &child_expr) {
10
+ types.push_back(child_expr.return_type);
11
+ auto child_state = ExpressionExecutor::InitializeState(child_expr, root);
12
+ child_states.push_back(std::move(child_state));
13
+
14
+ auto expr_class = child_expr.GetExpressionClass();
15
+ auto initialize_child = expr_class != ExpressionClass::BOUND_REF && expr_class != ExpressionClass::BOUND_CONSTANT &&
16
+ expr_class != ExpressionClass::BOUND_PARAMETER;
17
+ initialize.push_back(initialize_child);
12
18
  }
13
19
 
14
- void ExpressionState::Finalize(bool empty) {
20
+ void ExpressionState::Finalize() {
15
21
  if (types.empty()) {
16
22
  return;
17
23
  }
18
- if (empty) {
19
- intermediate_chunk.InitializeEmpty(types);
20
- } else {
21
- intermediate_chunk.Initialize(GetAllocator(), types);
22
- }
24
+ intermediate_chunk.Initialize(GetAllocator(), types, initialize);
23
25
  }
24
26
 
25
27
  Allocator &ExpressionState::GetAllocator() {
@@ -0,0 +1,238 @@
1
+ #include "duckdb/execution/index/art/fixed_size_allocator.hpp"
2
+
3
+ namespace duckdb {
4
+
5
+ constexpr idx_t FixedSizeAllocator::BASE[];
6
+ constexpr uint8_t FixedSizeAllocator::SHIFT[];
7
+
8
+ FixedSizeAllocator::FixedSizeAllocator(const idx_t allocation_size, Allocator &allocator)
9
+ : allocation_size(allocation_size), total_allocations(0), allocator(allocator) {
10
+
11
+ // calculate how many allocations fit into one buffer
12
+
13
+ idx_t bits_per_value = sizeof(validity_t) * 8;
14
+ idx_t curr_alloc_size = 0;
15
+
16
+ bitmask_count = 0;
17
+ allocations_per_buffer = 0;
18
+
19
+ while (curr_alloc_size < BUFFER_ALLOC_SIZE) {
20
+ if (!bitmask_count || (bitmask_count * bits_per_value) % allocations_per_buffer == 0) {
21
+ bitmask_count++;
22
+ curr_alloc_size += sizeof(validity_t);
23
+ }
24
+
25
+ auto remaining_alloc_size = BUFFER_ALLOC_SIZE - curr_alloc_size;
26
+ auto remaining_allocations = MinValue(remaining_alloc_size / allocation_size, bits_per_value);
27
+
28
+ if (remaining_allocations == 0) {
29
+ break;
30
+ }
31
+
32
+ allocations_per_buffer += remaining_allocations;
33
+ curr_alloc_size += remaining_allocations * allocation_size;
34
+ }
35
+
36
+ allocation_offset = bitmask_count * sizeof(validity_t);
37
+ }
38
+
39
+ FixedSizeAllocator::~FixedSizeAllocator() {
40
+ for (auto &buffer : buffers) {
41
+ allocator.FreeData(buffer.ptr, BUFFER_ALLOC_SIZE);
42
+ }
43
+ }
44
+
45
+ Node FixedSizeAllocator::New() {
46
+
47
+ // no more free pointers
48
+ if (buffers_with_free_space.empty()) {
49
+
50
+ // add a new buffer
51
+ idx_t buffer_id = buffers.size();
52
+ D_ASSERT(buffer_id <= (uint32_t)DConstants::INVALID_INDEX);
53
+ auto buffer = allocator.AllocateData(BUFFER_ALLOC_SIZE);
54
+ buffers.emplace_back(buffer, 0);
55
+ buffers_with_free_space.insert(buffer_id);
56
+
57
+ // set the bitmask
58
+ ValidityMask mask(reinterpret_cast<validity_t *>(buffer));
59
+ mask.SetAllValid(allocations_per_buffer);
60
+ }
61
+
62
+ // return a pointer
63
+ D_ASSERT(!buffers_with_free_space.empty());
64
+ auto buffer_id = (uint32_t)*buffers_with_free_space.begin();
65
+
66
+ auto bitmask_ptr = reinterpret_cast<validity_t *>(buffers[buffer_id].ptr);
67
+ ValidityMask mask(bitmask_ptr);
68
+ auto offset = GetOffset(mask, buffers[buffer_id].allocation_count);
69
+
70
+ buffers[buffer_id].allocation_count++;
71
+ total_allocations++;
72
+ if (buffers[buffer_id].allocation_count == allocations_per_buffer) {
73
+ buffers_with_free_space.erase(buffer_id);
74
+ }
75
+
76
+ return Node(buffer_id, offset);
77
+ }
78
+
79
+ void FixedSizeAllocator::Free(const Node ptr) {
80
+ auto bitmask_ptr = reinterpret_cast<validity_t *>(buffers[ptr.GetBufferId()].ptr);
81
+ ValidityMask mask(bitmask_ptr);
82
+ D_ASSERT(!mask.RowIsValid(ptr.GetOffset()));
83
+ mask.SetValid(ptr.GetOffset());
84
+ buffers_with_free_space.insert(ptr.GetBufferId());
85
+
86
+ D_ASSERT(total_allocations > 0);
87
+ D_ASSERT(buffers[ptr.GetBufferId()].allocation_count > 0);
88
+ buffers[ptr.GetBufferId()].allocation_count--;
89
+ total_allocations--;
90
+ }
91
+
92
+ void FixedSizeAllocator::Reset() {
93
+
94
+ for (auto &buffer : buffers) {
95
+ allocator.FreeData(buffer.ptr, BUFFER_ALLOC_SIZE);
96
+ }
97
+ buffers.clear();
98
+ buffers_with_free_space.clear();
99
+ total_allocations = 0;
100
+ }
101
+
102
+ void FixedSizeAllocator::Merge(FixedSizeAllocator &other) {
103
+
104
+ D_ASSERT(allocation_size == other.allocation_size);
105
+
106
+ // remember the buffer count and merge the buffers
107
+ idx_t buffer_count = buffers.size();
108
+ for (auto &buffer : other.buffers) {
109
+ buffers.push_back(buffer);
110
+ }
111
+ other.buffers.clear();
112
+
113
+ // merge the buffers with free spaces
114
+ for (auto &buffer_id : other.buffers_with_free_space) {
115
+ buffers_with_free_space.insert(buffer_id + buffer_count);
116
+ }
117
+ other.buffers_with_free_space.clear();
118
+
119
+ // add the total allocations
120
+ total_allocations += other.total_allocations;
121
+ }
122
+
123
+ bool FixedSizeAllocator::InitializeVacuum() {
124
+
125
+ if (total_allocations == 0) {
126
+ Reset();
127
+ return false;
128
+ }
129
+
130
+ auto total_available_allocations = allocations_per_buffer * buffers.size();
131
+ D_ASSERT(total_available_allocations >= total_allocations);
132
+ auto total_free_positions = total_available_allocations - total_allocations;
133
+
134
+ // vacuum_count buffers can be freed
135
+ auto vacuum_count = total_free_positions / allocations_per_buffer;
136
+
137
+ // calculate the vacuum threshold adaptively
138
+ D_ASSERT(vacuum_count < buffers.size());
139
+ idx_t memory_usage = GetMemoryUsage();
140
+ idx_t excess_memory_usage = vacuum_count * BUFFER_ALLOC_SIZE;
141
+ auto excess_percentage = (double)excess_memory_usage / (double)memory_usage;
142
+ auto threshold = (double)VACUUM_THRESHOLD / 100.0;
143
+ if (excess_percentage < threshold) {
144
+ return false;
145
+ }
146
+
147
+ min_vacuum_buffer_id = buffers.size() - vacuum_count;
148
+
149
+ // remove all invalid buffers from the available buffer list to ensure that we do not reuse them
150
+ auto it = buffers_with_free_space.begin();
151
+ while (it != buffers_with_free_space.end()) {
152
+ if (*it >= min_vacuum_buffer_id) {
153
+ it = buffers_with_free_space.erase(it);
154
+ } else {
155
+ it++;
156
+ }
157
+ }
158
+
159
+ return true;
160
+ }
161
+
162
+ void FixedSizeAllocator::FinalizeVacuum() {
163
+
164
+ // free all (now unused) buffers
165
+ while (min_vacuum_buffer_id < buffers.size()) {
166
+ allocator.FreeData(buffers.back().ptr, BUFFER_ALLOC_SIZE);
167
+ buffers.pop_back();
168
+ }
169
+ }
170
+
171
+ Node FixedSizeAllocator::VacuumPointer(const Node ptr) {
172
+
173
+ // we do not need to adjust the bitmask of the old buffer, because we will free the entire
174
+ // buffer after the vacuum operation
175
+
176
+ auto new_ptr = New();
177
+
178
+ // new increases the allocation count
179
+ total_allocations--;
180
+
181
+ memcpy(Get(new_ptr), Get(ptr), allocation_size);
182
+ return new_ptr;
183
+ }
184
+
185
+ void FixedSizeAllocator::Verify() const {
186
+ #ifdef DEBUG
187
+ auto total_available_allocations = allocations_per_buffer * buffers.size();
188
+ D_ASSERT(total_available_allocations >= total_allocations);
189
+ D_ASSERT(buffers.size() >= buffers_with_free_space.size());
190
+ #endif
191
+ }
192
+
193
+ uint32_t FixedSizeAllocator::GetOffset(ValidityMask &mask, const idx_t allocation_count) {
194
+
195
+ auto data = mask.GetData();
196
+
197
+ // fills up a buffer sequentially before searching for free bits
198
+ if (mask.RowIsValid(allocation_count)) {
199
+ mask.SetInvalid(allocation_count);
200
+ return allocation_count;
201
+ }
202
+
203
+ // get an entry with free bits
204
+ for (idx_t entry_idx = 0; entry_idx < bitmask_count; entry_idx++) {
205
+ if (data[entry_idx] != 0) {
206
+
207
+ // find the position of the free bit
208
+ auto entry = data[entry_idx];
209
+ idx_t first_valid_bit = 0;
210
+
211
+ // this loop finds the position of the rightmost set bit in entry and stores it
212
+ // in first_valid_bit
213
+ for (idx_t i = 0; i < 6; i++) {
214
+ // set the left half of the bits of this level to zero and test if the entry is still not zero
215
+ if (entry & BASE[i]) {
216
+ // first valid bit is in the rightmost s[i] bits
217
+ // permanently set the left half of the bits to zero
218
+ entry &= BASE[i];
219
+ } else {
220
+ // first valid bit is in the leftmost s[i] bits
221
+ // shift by s[i] for the next iteration and add s[i] to the position of the rightmost set bit
222
+ entry >>= SHIFT[i];
223
+ first_valid_bit += SHIFT[i];
224
+ }
225
+ }
226
+ D_ASSERT(entry);
227
+
228
+ auto prev_bits = entry_idx * sizeof(validity_t) * 8;
229
+ D_ASSERT(mask.RowIsValid(prev_bits + first_valid_bit));
230
+ mask.SetInvalid(prev_bits + first_valid_bit);
231
+ return (prev_bits + first_valid_bit);
232
+ }
233
+ }
234
+
235
+ throw InternalException("Invalid bitmask of FixedSizeAllocator");
236
+ }
237
+
238
+ } // namespace duckdb
@@ -0,0 +1,94 @@
1
+
2
+ #include "duckdb/execution/operator/order/physical_order.hpp"
3
+ #include "duckdb/execution/operator/projection/physical_projection.hpp"
4
+ #include "duckdb/execution/operator/filter/physical_filter.hpp"
5
+ #include "duckdb/execution/operator/schema/physical_create_art_index.hpp"
6
+
7
+ #include "duckdb/planner/expression/bound_operator_expression.hpp"
8
+ #include "duckdb/planner/expression/bound_reference_expression.hpp"
9
+ #include "duckdb/planner/operator/logical_create_index.hpp"
10
+
11
+ #include "duckdb/execution/index/art/art.hpp"
12
+
13
+ namespace duckdb {
14
+
15
+ unique_ptr<PhysicalOperator> ART::CreatePlan(PlanIndexInput &input) {
16
+ // generate a physical plan for the parallel index creation which consists of the following operators
17
+ // table scan - projection (for expression execution) - filter (NOT NULL) - order (if applicable) - create index
18
+
19
+ auto &op = input.op;
20
+ auto &table_scan = input.table_scan;
21
+
22
+ vector<LogicalType> new_column_types;
23
+ vector<unique_ptr<Expression>> select_list;
24
+ for (idx_t i = 0; i < op.expressions.size(); i++) {
25
+ new_column_types.push_back(op.expressions[i]->return_type);
26
+ select_list.push_back(std::move(op.expressions[i]));
27
+ }
28
+ new_column_types.emplace_back(LogicalType::ROW_TYPE);
29
+ select_list.push_back(make_uniq<BoundReferenceExpression>(LogicalType::ROW_TYPE, op.info->scan_types.size() - 1));
30
+
31
+ auto projection = make_uniq<PhysicalProjection>(new_column_types, std::move(select_list), op.estimated_cardinality);
32
+ projection->children.push_back(std::move(table_scan));
33
+
34
+ // filter operator for IS_NOT_NULL on each key column
35
+
36
+ vector<LogicalType> filter_types;
37
+ vector<unique_ptr<Expression>> filter_select_list;
38
+
39
+ for (idx_t i = 0; i < new_column_types.size() - 1; i++) {
40
+ filter_types.push_back(new_column_types[i]);
41
+ auto is_not_null_expr =
42
+ make_uniq<BoundOperatorExpression>(ExpressionType::OPERATOR_IS_NOT_NULL, LogicalType::BOOLEAN);
43
+ auto bound_ref = make_uniq<BoundReferenceExpression>(new_column_types[i], i);
44
+ is_not_null_expr->children.push_back(std::move(bound_ref));
45
+ filter_select_list.push_back(std::move(is_not_null_expr));
46
+ }
47
+
48
+ auto null_filter =
49
+ make_uniq<PhysicalFilter>(std::move(filter_types), std::move(filter_select_list), op.estimated_cardinality);
50
+ null_filter->types.emplace_back(LogicalType::ROW_TYPE);
51
+ null_filter->children.push_back(std::move(projection));
52
+
53
+ // determine if we sort the data prior to index creation
54
+ // we don't sort, if either VARCHAR or compound key
55
+ auto perform_sorting = true;
56
+ if (op.unbound_expressions.size() > 1) {
57
+ perform_sorting = false;
58
+ } else if (op.unbound_expressions[0]->return_type.InternalType() == PhysicalType::VARCHAR) {
59
+ perform_sorting = false;
60
+ }
61
+
62
+ // actual physical create index operator
63
+
64
+ auto physical_create_index =
65
+ make_uniq<PhysicalCreateARTIndex>(op, op.table, op.info->column_ids, std::move(op.info),
66
+ std::move(op.unbound_expressions), op.estimated_cardinality, perform_sorting);
67
+
68
+ if (perform_sorting) {
69
+
70
+ // optional order operator
71
+ vector<BoundOrderByNode> orders;
72
+ vector<idx_t> projections;
73
+ for (idx_t i = 0; i < new_column_types.size() - 1; i++) {
74
+ auto col_expr = make_uniq_base<Expression, BoundReferenceExpression>(new_column_types[i], i);
75
+ orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_FIRST, std::move(col_expr));
76
+ projections.emplace_back(i);
77
+ }
78
+ projections.emplace_back(new_column_types.size() - 1);
79
+
80
+ auto physical_order = make_uniq<PhysicalOrder>(new_column_types, std::move(orders), std::move(projections),
81
+ op.estimated_cardinality);
82
+ physical_order->children.push_back(std::move(null_filter));
83
+
84
+ physical_create_index->children.push_back(std::move(physical_order));
85
+ } else {
86
+
87
+ // no ordering
88
+ physical_create_index->children.push_back(std::move(null_filter));
89
+ }
90
+
91
+ return std::move(physical_create_index);
92
+ }
93
+
94
+ } // namespace duckdb
@@ -5,10 +5,13 @@
5
5
  namespace duckdb {
6
6
 
7
7
  IndexTypeSet::IndexTypeSet() {
8
- // Register the ART index type
8
+
9
+ // Register the ART index type by default
9
10
  IndexType art_index_type;
10
11
  art_index_type.name = ART::TYPE_NAME;
11
12
  art_index_type.create_instance = ART::Create;
13
+ art_index_type.create_plan = ART::CreatePlan;
14
+
12
15
  RegisterIndexType(art_index_type);
13
16
  }
14
17
 
@@ -453,23 +453,21 @@ static inline data_ptr_t InsertRowToEntry(atomic<ht_entry_t> &entry, const data_
453
453
  // if we expect the entry to be empty, if the operation fails we need to cancel the whole operation as another
454
454
  // key might have been inserted in the meantime that does not match the current key
455
455
  if (EXPECT_EMPTY) {
456
-
457
456
  // add nullptr to the end of the list to mark the end
458
457
  StorePointer(nullptr, row_ptr_to_insert + pointer_offset);
459
458
 
460
459
  ht_entry_t new_empty_entry = ht_entry_t::GetDesiredEntry(row_ptr_to_insert, salt);
461
460
  ht_entry_t expected_empty_entry = ht_entry_t::GetEmptyEntry();
462
- std::atomic_compare_exchange_weak(&entry, &expected_empty_entry, new_empty_entry);
461
+ entry.compare_exchange_strong(expected_empty_entry, new_empty_entry, std::memory_order_acquire,
462
+ std::memory_order_relaxed);
463
463
 
464
464
  // if the expected empty entry actually was null, we can just return the pointer, and it will be a nullptr
465
465
  // if the expected entry was filled in the meantime, we need to cancel the operation and will return the
466
466
  // pointer to the next entry
467
467
  return expected_empty_entry.GetPointerOrNull();
468
- }
469
-
470
- // if we expect the entry to be full, we know that even if the insert fails the keys still match so we can
471
- // just keep trying until we succeed
472
- else {
468
+ } else {
469
+ // if we expect the entry to be full, we know that even if the insert fails the keys still match so we can
470
+ // just keep trying until we succeed
473
471
  ht_entry_t expected_current_entry = entry.load(std::memory_order_relaxed);
474
472
  ht_entry_t desired_new_entry = ht_entry_t::GetDesiredEntry(row_ptr_to_insert, salt);
475
473
  D_ASSERT(expected_current_entry.IsOccupied());
@@ -477,7 +475,8 @@ static inline data_ptr_t InsertRowToEntry(atomic<ht_entry_t> &entry, const data_
477
475
  do {
478
476
  data_ptr_t current_row_pointer = expected_current_entry.GetPointer();
479
477
  StorePointer(current_row_pointer, row_ptr_to_insert + pointer_offset);
480
- } while (!std::atomic_compare_exchange_weak(&entry, &expected_current_entry, desired_new_entry));
478
+ } while (!entry.compare_exchange_weak(expected_current_entry, desired_new_entry, std::memory_order_release,
479
+ std::memory_order_relaxed));
481
480
 
482
481
  return nullptr;
483
482
  }
@@ -319,15 +319,17 @@ void PhysicalHashAggregate::SinkDistinctGrouping(ExecutionContext &context, Data
319
319
  for (idx_t group_idx = 0; group_idx < grouped_aggregate_data.groups.size(); group_idx++) {
320
320
  auto &group = grouped_aggregate_data.groups[group_idx];
321
321
  auto &bound_ref = group->Cast<BoundReferenceExpression>();
322
- filtered_input.data[bound_ref.index].Reference(chunk.data[bound_ref.index]);
322
+ auto &col = filtered_input.data[bound_ref.index];
323
+ col.Reference(chunk.data[bound_ref.index]);
324
+ col.Slice(sel_vec, count);
323
325
  }
324
326
  for (idx_t child_idx = 0; child_idx < aggregate.children.size(); child_idx++) {
325
327
  auto &child = aggregate.children[child_idx];
326
328
  auto &bound_ref = child->Cast<BoundReferenceExpression>();
327
-
328
- filtered_input.data[bound_ref.index].Reference(chunk.data[bound_ref.index]);
329
+ auto &col = filtered_input.data[bound_ref.index];
330
+ col.Reference(chunk.data[bound_ref.index]);
331
+ col.Slice(sel_vec, count);
329
332
  }
330
- filtered_input.Slice(sel_vec, count);
331
333
  filtered_input.SetCardinality(count);
332
334
 
333
335
  radix_table.Sink(context, filtered_input, sink_input, empty_chunk, empty_filter);
@@ -119,15 +119,15 @@ void CSVBufferManager::ResetBuffer(const idx_t buffer_idx) {
119
119
  }
120
120
  }
121
121
 
122
- idx_t CSVBufferManager::GetBufferSize() {
122
+ idx_t CSVBufferManager::GetBufferSize() const {
123
123
  return buffer_size;
124
124
  }
125
125
 
126
- idx_t CSVBufferManager::BufferCount() {
126
+ idx_t CSVBufferManager::BufferCount() const {
127
127
  return cached_buffers.size();
128
128
  }
129
129
 
130
- bool CSVBufferManager::Done() {
130
+ bool CSVBufferManager::Done() const {
131
131
  return done;
132
132
  }
133
133
 
@@ -144,7 +144,7 @@ void CSVBufferManager::ResetBufferManager() {
144
144
  }
145
145
  }
146
146
 
147
- string CSVBufferManager::GetFilePath() {
147
+ string CSVBufferManager::GetFilePath() const {
148
148
  return file_path;
149
149
  }
150
150