duckdb 0.8.2-dev145.0 → 0.8.2-dev1493.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +15 -12
- package/binding.gyp.in +1 -1
- package/configure.py +1 -1
- package/duckdb_extension_config.cmake +10 -0
- package/package.json +1 -1
- package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
- package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
- package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
- package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
- package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
- package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
- package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
- package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
- package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
- package/src/duckdb/extension/json/include/json_common.hpp +47 -231
- package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
- package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
- package/src/duckdb/extension/json/json_common.cpp +272 -40
- package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
- package/src/duckdb/extension/json/json_functions/json_transform.cpp +17 -37
- package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
- package/src/duckdb/extension/json/json_functions.cpp +24 -24
- package/src/duckdb/extension/json/json_scan.cpp +3 -6
- package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
- package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
- package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
- package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
- package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
- package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
- package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
- package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
- package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
- package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
- package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
- package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
- package/src/duckdb/extension/parquet/parquet_extension.cpp +192 -20
- package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
- package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
- package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
- package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
- package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
- package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
- package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
- package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
- package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
- package/src/duckdb/src/common/allocator.cpp +14 -2
- package/src/duckdb/src/common/arrow/arrow_appender.cpp +5 -11
- package/src/duckdb/src/common/assert.cpp +3 -0
- package/src/duckdb/src/common/enum_util.cpp +42 -5
- package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
- package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
- package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
- package/src/duckdb/src/common/exception.cpp +2 -2
- package/src/duckdb/src/common/file_system.cpp +19 -0
- package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
- package/src/duckdb/src/common/local_file_system.cpp +2 -2
- package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
- package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
- package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
- package/src/duckdb/src/common/re2_regex.cpp +1 -1
- package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
- package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
- package/src/duckdb/src/common/sort/partition_state.cpp +44 -11
- package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
- package/src/duckdb/src/common/types/bit.cpp +51 -0
- package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
- package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
- package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
- package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
- package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
- package/src/duckdb/src/common/types/date.cpp +9 -0
- package/src/duckdb/src/common/types/list_segment.cpp +24 -74
- package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
- package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
- package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
- package/src/duckdb/src/common/types/uuid.cpp +2 -2
- package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
- package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
- package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
- package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
- package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
- package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
- package/src/duckdb/src/core_functions/function_list.cpp +4 -2
- package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
- package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
- package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
- package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
- package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
- package/src/duckdb/src/execution/expression_executor.cpp +1 -1
- package/src/duckdb/src/execution/index/art/art.cpp +149 -139
- package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
- package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
- package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
- package/src/duckdb/src/execution/index/art/node.cpp +113 -120
- package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
- package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
- package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
- package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
- package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
- package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
- package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
- package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
- package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
- package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
- package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
- package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
- package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +414 -283
- package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
- package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +22 -3
- package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +2 -2
- package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
- package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
- package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
- package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
- package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
- package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
- package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
- package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
- package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
- package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
- package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
- package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
- package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
- package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
- package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
- package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
- package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
- package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
- package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
- package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
- package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
- package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
- package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
- package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
- package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
- package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
- package/src/duckdb/src/function/function.cpp +3 -1
- package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
- package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
- package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
- package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
- package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
- package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
- package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
- package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
- package/src/duckdb/src/function/table/read_csv.cpp +100 -17
- package/src/duckdb/src/function/table/table_scan.cpp +9 -0
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
- package/src/duckdb/src/include/duckdb/common/enum_util.hpp +8 -0
- package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
- package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
- package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
- package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
- package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
- package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
- package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
- package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
- package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
- package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
- package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
- package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +13 -0
- package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
- package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
- package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +0 -1
- package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
- package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
- package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
- package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
- package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
- package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
- package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
- package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
- package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
- package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +2 -10
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
- package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
- package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
- package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
- package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
- package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
- package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
- package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
- package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
- package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
- package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
- package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
- package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
- package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
- package/src/duckdb/src/include/duckdb/main/client_config.hpp +3 -0
- package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
- package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
- package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
- package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
- package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
- package/src/duckdb/src/include/duckdb/main/settings.hpp +30 -1
- package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
- package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
- package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
- package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
- package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
- package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
- package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
- package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
- package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
- package/src/duckdb/src/include/duckdb/planner/binder.hpp +8 -5
- package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
- package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
- package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
- package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
- package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
- package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
- package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
- package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
- package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
- package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
- package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
- package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
- package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
- package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
- package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
- package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
- package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
- package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
- package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
- package/src/duckdb/src/include/duckdb.h +28 -0
- package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
- package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
- package/src/duckdb/src/main/config.cpp +3 -0
- package/src/duckdb/src/main/database.cpp +1 -1
- package/src/duckdb/src/main/extension/extension_helper.cpp +96 -89
- package/src/duckdb/src/main/extension/extension_install.cpp +6 -0
- package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
- package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
- package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
- package/src/duckdb/src/main/relation.cpp +6 -5
- package/src/duckdb/src/main/settings/settings.cpp +64 -18
- package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
- package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
- package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
- package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
- package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
- package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
- package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
- package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
- package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
- package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
- package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
- package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
- package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
- package/src/duckdb/src/optimizer/optimizer.cpp +51 -14
- package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
- package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
- package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
- package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
- package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
- package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
- package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
- package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
- package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
- package/src/duckdb/src/parallel/executor.cpp +15 -0
- package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
- package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
- package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
- package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
- package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
- package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
- package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
- package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
- package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
- package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
- package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
- package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
- package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
- package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
- package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
- package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
- package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
- package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
- package/src/duckdb/src/parser/parser.cpp +8 -2
- package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
- package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
- package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
- package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
- package/src/duckdb/src/parser/query_node.cpp +15 -37
- package/src/duckdb/src/parser/result_modifier.cpp +0 -74
- package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
- package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
- package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
- package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
- package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
- package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
- package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
- package/src/duckdb/src/parser/tableref.cpp +0 -44
- package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
- package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
- package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
- package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
- package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
- package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
- package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
- package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
- package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
- package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
- package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
- package/src/duckdb/src/parser/transformer.cpp +15 -0
- package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
- package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
- package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
- package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
- package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
- package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
- package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
- package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
- package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
- package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
- package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +61 -26
- package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
- package/src/duckdb/src/planner/binder.cpp +5 -0
- package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
- package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
- package/src/duckdb/src/planner/expression_binder.cpp +3 -0
- package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
- package/src/duckdb/src/planner/logical_operator.cpp +5 -0
- package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
- package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
- package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
- package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
- package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
- package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
- package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
- package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
- package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
- package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
- package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
- package/src/duckdb/src/storage/compression/rle.cpp +0 -1
- package/src/duckdb/src/storage/data_table.cpp +1 -1
- package/src/duckdb/src/storage/local_storage.cpp +3 -3
- package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
- package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
- package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
- package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
- package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
- package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
- package/src/duckdb/src/storage/storage_info.cpp +1 -1
- package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
- package/src/duckdb/src/storage/table/row_group.cpp +25 -9
- package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
- package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
- package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
- package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
- package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
- package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
- package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
- package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
- package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
- package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
- package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
- package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
- package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
- package/src/duckdb/ub_src_function_scalar.cpp +2 -0
- package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
- package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
- package/src/duckdb/ub_src_optimizer.cpp +6 -0
- package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
- package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
- package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
- package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
- package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
- package/src/duckdb/ub_src_planner_operator.cpp +4 -0
- package/src/duckdb/ub_src_storage_serialization.cpp +8 -0
- package/src/statement.cpp +10 -3
- package/test/test_all_types.test.ts +233 -0
- package/tsconfig.json +1 -0
- package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
- package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -87,7 +87,7 @@ PartitionGlobalSinkState::PartitionGlobalSinkState(ClientContext &context,
|
|
87
87
|
const vector<unique_ptr<BaseStatistics>> &partition_stats,
|
88
88
|
idx_t estimated_cardinality)
|
89
89
|
: context(context), buffer_manager(BufferManager::GetBufferManager(context)), allocator(Allocator::Get(context)),
|
90
|
-
payload_types(payload_types), memory_per_thread(0), count(0) {
|
90
|
+
fixed_bits(0), payload_types(payload_types), memory_per_thread(0), count(0) {
|
91
91
|
|
92
92
|
GenerateOrderings(partitions, orders, partition_bys, order_bys, partition_stats);
|
93
93
|
|
@@ -102,9 +102,19 @@ PartitionGlobalSinkState::PartitionGlobalSinkState(ClientContext &context,
|
|
102
102
|
}
|
103
103
|
}
|
104
104
|
|
105
|
+
void PartitionGlobalSinkState::SyncPartitioning(const PartitionGlobalSinkState &other) {
|
106
|
+
fixed_bits = other.grouping_data ? other.grouping_data->GetRadixBits() : 0;
|
107
|
+
|
108
|
+
const auto old_bits = grouping_data ? grouping_data->GetRadixBits() : 0;
|
109
|
+
if (fixed_bits != old_bits) {
|
110
|
+
const auto hash_col_idx = payload_types.size();
|
111
|
+
grouping_data = make_uniq<RadixPartitionedColumnData>(context, grouping_types, fixed_bits, hash_col_idx);
|
112
|
+
}
|
113
|
+
}
|
114
|
+
|
105
115
|
void PartitionGlobalSinkState::ResizeGroupingData(idx_t cardinality) {
|
106
116
|
// Have we started to combine? Then just live with it.
|
107
|
-
if (grouping_data && !grouping_data->GetPartitions().empty()) {
|
117
|
+
if (fixed_bits || (grouping_data && !grouping_data->GetPartitions().empty())) {
|
108
118
|
return;
|
109
119
|
}
|
110
120
|
// Is the average partition size too large?
|
@@ -186,9 +196,7 @@ void PartitionGlobalSinkState::CombineLocalPartition(GroupingPartition &local_pa
|
|
186
196
|
grouping_data->Combine(*local_partition);
|
187
197
|
}
|
188
198
|
|
189
|
-
void PartitionGlobalSinkState::BuildSortState(ColumnDataCollection &group_data,
|
190
|
-
auto &global_sort = *hash_group.global_sort;
|
191
|
-
|
199
|
+
void PartitionGlobalSinkState::BuildSortState(ColumnDataCollection &group_data, GlobalSortState &global_sort) const {
|
192
200
|
// Set up the sort expression computation.
|
193
201
|
vector<LogicalType> sort_types;
|
194
202
|
ExpressionExecutor executor(context);
|
@@ -234,6 +242,10 @@ void PartitionGlobalSinkState::BuildSortState(ColumnDataCollection &group_data,
|
|
234
242
|
}
|
235
243
|
|
236
244
|
global_sort.AddLocalState(local_sort);
|
245
|
+
}
|
246
|
+
|
247
|
+
void PartitionGlobalSinkState::BuildSortState(ColumnDataCollection &group_data, PartitionGlobalHashGroup &hash_group) {
|
248
|
+
BuildSortState(group_data, *hash_group.global_sort);
|
237
249
|
|
238
250
|
hash_group.count += group_data.Count();
|
239
251
|
}
|
@@ -482,18 +494,29 @@ public:
|
|
482
494
|
TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override;
|
483
495
|
|
484
496
|
private:
|
497
|
+
struct ExecutorCallback : public PartitionGlobalMergeStates::Callback {
|
498
|
+
explicit ExecutorCallback(Executor &executor) : executor(executor) {
|
499
|
+
}
|
500
|
+
|
501
|
+
bool HasError() const override {
|
502
|
+
return executor.HasError();
|
503
|
+
}
|
504
|
+
|
505
|
+
Executor &executor;
|
506
|
+
};
|
507
|
+
|
485
508
|
shared_ptr<Event> event;
|
486
509
|
PartitionLocalMergeState local_state;
|
487
510
|
PartitionGlobalMergeStates &hash_groups;
|
488
511
|
};
|
489
512
|
|
490
|
-
|
513
|
+
bool PartitionGlobalMergeStates::ExecuteTask(PartitionLocalMergeState &local_state, Callback &callback) {
|
491
514
|
// Loop until all hash groups are done
|
492
515
|
size_t sorted = 0;
|
493
|
-
while (sorted <
|
516
|
+
while (sorted < states.size()) {
|
494
517
|
// First check if there is an unfinished task for this thread
|
495
|
-
if (
|
496
|
-
return
|
518
|
+
if (callback.HasError()) {
|
519
|
+
return false;
|
497
520
|
}
|
498
521
|
if (!local_state.TaskFinished()) {
|
499
522
|
local_state.ExecuteTask();
|
@@ -501,8 +524,8 @@ TaskExecutionResult PartitionMergeTask::ExecuteTask(TaskExecutionMode mode) {
|
|
501
524
|
}
|
502
525
|
|
503
526
|
// Thread is done with its assigned task, try to fetch new work
|
504
|
-
for (auto group = sorted; group <
|
505
|
-
auto &global_state =
|
527
|
+
for (auto group = sorted; group < states.size(); ++group) {
|
528
|
+
auto &global_state = states[group];
|
506
529
|
if (global_state->IsSorted()) {
|
507
530
|
// This hash group is done
|
508
531
|
// Update the high water mark of densely completed groups
|
@@ -543,6 +566,16 @@ TaskExecutionResult PartitionMergeTask::ExecuteTask(TaskExecutionMode mode) {
|
|
543
566
|
}
|
544
567
|
}
|
545
568
|
|
569
|
+
return true;
|
570
|
+
}
|
571
|
+
|
572
|
+
TaskExecutionResult PartitionMergeTask::ExecuteTask(TaskExecutionMode mode) {
|
573
|
+
ExecutorCallback callback(executor);
|
574
|
+
|
575
|
+
if (!hash_groups.ExecuteTask(local_state, callback)) {
|
576
|
+
return TaskExecutionResult::TASK_ERROR;
|
577
|
+
}
|
578
|
+
|
546
579
|
event->FinishTask();
|
547
580
|
return TaskExecutionResult::TASK_FINISHED;
|
548
581
|
}
|
@@ -1,11 +1,14 @@
|
|
1
1
|
#include "duckdb/common/types/batched_data_collection.hpp"
|
2
|
+
|
3
|
+
#include "duckdb/common/optional_ptr.hpp"
|
2
4
|
#include "duckdb/common/printer.hpp"
|
3
5
|
#include "duckdb/storage/buffer_manager.hpp"
|
4
|
-
#include "duckdb/common/optional_ptr.hpp"
|
5
6
|
|
6
7
|
namespace duckdb {
|
7
8
|
|
8
|
-
BatchedDataCollection::BatchedDataCollection(vector<LogicalType> types_p
|
9
|
+
BatchedDataCollection::BatchedDataCollection(ClientContext &context_p, vector<LogicalType> types_p,
|
10
|
+
bool buffer_managed_p)
|
11
|
+
: context(context_p), types(std::move(types_p)), buffer_managed(buffer_managed_p) {
|
9
12
|
}
|
10
13
|
|
11
14
|
void BatchedDataCollection::Append(DataChunk &input, idx_t batch_index) {
|
@@ -20,6 +23,8 @@ void BatchedDataCollection::Append(DataChunk &input, idx_t batch_index) {
|
|
20
23
|
unique_ptr<ColumnDataCollection> new_collection;
|
21
24
|
if (last_collection.collection) {
|
22
25
|
new_collection = make_uniq<ColumnDataCollection>(*last_collection.collection);
|
26
|
+
} else if (buffer_managed) {
|
27
|
+
new_collection = make_uniq<ColumnDataCollection>(BufferManager::GetBufferManager(context), types);
|
23
28
|
} else {
|
24
29
|
new_collection = make_uniq<ColumnDataCollection>(Allocator::DefaultAllocator(), types);
|
25
30
|
}
|
@@ -1,4 +1,6 @@
|
|
1
|
+
#include "duckdb/common/assert.hpp"
|
1
2
|
#include "duckdb/common/operator/cast_operators.hpp"
|
3
|
+
#include "duckdb/common/typedefs.hpp"
|
2
4
|
#include "duckdb/common/types/bit.hpp"
|
3
5
|
#include "duckdb/common/types/string_type.hpp"
|
4
6
|
|
@@ -34,6 +36,13 @@ static inline idx_t GetBitSize(const string_t &str) {
|
|
34
36
|
return str_len;
|
35
37
|
}
|
36
38
|
|
39
|
+
uint8_t Bit::GetFirstByte(const string_t &str) {
|
40
|
+
D_ASSERT(str.GetSize() > 1);
|
41
|
+
|
42
|
+
auto data = const_data_ptr_cast(str.GetData());
|
43
|
+
return data[1] & ((1 << (8 - data[0])) - 1);
|
44
|
+
}
|
45
|
+
|
37
46
|
void Bit::Finalize(string_t &str) {
|
38
47
|
// bit strings require all padding bits to be set to 1
|
39
48
|
// this method sets all padding bits to 1
|
@@ -146,6 +155,48 @@ string Bit::ToBit(string_t str) {
|
|
146
155
|
return output_str.GetString();
|
147
156
|
}
|
148
157
|
|
158
|
+
void Bit::BlobToBit(string_t blob, string_t &output_str) {
|
159
|
+
auto data = const_data_ptr_cast(blob.GetData());
|
160
|
+
auto output = output_str.GetDataWriteable();
|
161
|
+
idx_t size = blob.GetSize();
|
162
|
+
|
163
|
+
*output = 0; // No padding
|
164
|
+
memcpy(output + 1, data, size);
|
165
|
+
}
|
166
|
+
|
167
|
+
string Bit::BlobToBit(string_t blob) {
|
168
|
+
auto buffer = make_unsafe_uniq_array<char>(blob.GetSize() + 1);
|
169
|
+
string_t output_str(buffer.get(), blob.GetSize() + 1);
|
170
|
+
Bit::BlobToBit(blob, output_str);
|
171
|
+
return output_str.GetString();
|
172
|
+
}
|
173
|
+
|
174
|
+
void Bit::BitToBlob(string_t bit, string_t &output_blob) {
|
175
|
+
D_ASSERT(bit.GetSize() == output_blob.GetSize() + 1);
|
176
|
+
|
177
|
+
auto data = const_data_ptr_cast(bit.GetData());
|
178
|
+
auto output = output_blob.GetDataWriteable();
|
179
|
+
idx_t size = output_blob.GetSize();
|
180
|
+
|
181
|
+
output[0] = GetFirstByte(bit);
|
182
|
+
if (size > 2) {
|
183
|
+
++output;
|
184
|
+
// First byte in bitstring contains amount of padded bits,
|
185
|
+
// second byte in bitstring is the padded byte,
|
186
|
+
// therefore the rest of the data starts at data + 2 (third byte)
|
187
|
+
memcpy(output, data + 2, size - 1);
|
188
|
+
}
|
189
|
+
}
|
190
|
+
|
191
|
+
string Bit::BitToBlob(string_t bit) {
|
192
|
+
D_ASSERT(bit.GetSize() > 1);
|
193
|
+
|
194
|
+
auto buffer = make_unsafe_uniq_array<char>(bit.GetSize() - 1);
|
195
|
+
string_t output_str(buffer.get(), bit.GetSize() - 1);
|
196
|
+
Bit::BitToBlob(bit, output_str);
|
197
|
+
return output_str.GetString();
|
198
|
+
}
|
199
|
+
|
149
200
|
// **** scalar functions ****
|
150
201
|
void Bit::BitString(const string_t &input, const idx_t &bit_length, string_t &result) {
|
151
202
|
char *res_buf = result.GetDataWriteable();
|
@@ -1,8 +1,8 @@
|
|
1
1
|
#include "duckdb/common/types/column/column_data_allocator.hpp"
|
2
2
|
|
3
3
|
#include "duckdb/common/types/column/column_data_collection_segment.hpp"
|
4
|
-
#include "duckdb/storage/buffer_manager.hpp"
|
5
4
|
#include "duckdb/storage/buffer/block_handle.hpp"
|
5
|
+
#include "duckdb/storage/buffer_manager.hpp"
|
6
6
|
|
7
7
|
namespace duckdb {
|
8
8
|
|
@@ -19,6 +19,7 @@ ColumnDataAllocator::ColumnDataAllocator(ClientContext &context, ColumnDataAlloc
|
|
19
19
|
: type(allocator_type) {
|
20
20
|
switch (type) {
|
21
21
|
case ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR:
|
22
|
+
case ColumnDataAllocatorType::HYBRID:
|
22
23
|
alloc.buffer_manager = &BufferManager::GetBufferManager(context);
|
23
24
|
break;
|
24
25
|
case ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR:
|
@@ -33,6 +34,7 @@ ColumnDataAllocator::ColumnDataAllocator(ColumnDataAllocator &other) {
|
|
33
34
|
type = other.GetType();
|
34
35
|
switch (type) {
|
35
36
|
case ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR:
|
37
|
+
case ColumnDataAllocatorType::HYBRID:
|
36
38
|
alloc.allocator = other.alloc.allocator;
|
37
39
|
break;
|
38
40
|
case ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR:
|
@@ -44,7 +46,7 @@ ColumnDataAllocator::ColumnDataAllocator(ColumnDataAllocator &other) {
|
|
44
46
|
}
|
45
47
|
|
46
48
|
BufferHandle ColumnDataAllocator::Pin(uint32_t block_id) {
|
47
|
-
D_ASSERT(type == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR);
|
49
|
+
D_ASSERT(type == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR || type == ColumnDataAllocatorType::HYBRID);
|
48
50
|
shared_ptr<BlockHandle> handle;
|
49
51
|
if (shared) {
|
50
52
|
// we only need to grab the lock when accessing the vector, because vector access is not thread-safe:
|
@@ -58,7 +60,7 @@ BufferHandle ColumnDataAllocator::Pin(uint32_t block_id) {
|
|
58
60
|
}
|
59
61
|
|
60
62
|
BufferHandle ColumnDataAllocator::AllocateBlock(idx_t size) {
|
61
|
-
D_ASSERT(type == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR);
|
63
|
+
D_ASSERT(type == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR || type == ColumnDataAllocatorType::HYBRID);
|
62
64
|
auto block_size = MaxValue<idx_t>(size, Storage::BLOCK_SIZE);
|
63
65
|
BlockMetaData data;
|
64
66
|
data.size = 0;
|
@@ -136,6 +138,7 @@ void ColumnDataAllocator::AllocateData(idx_t size, uint32_t &block_id, uint32_t
|
|
136
138
|
ChunkManagementState *chunk_state) {
|
137
139
|
switch (type) {
|
138
140
|
case ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR:
|
141
|
+
case ColumnDataAllocatorType::HYBRID:
|
139
142
|
if (shared) {
|
140
143
|
lock_guard<mutex> guard(lock);
|
141
144
|
AllocateBuffer(size, block_id, offset, chunk_state);
|
@@ -174,8 +177,8 @@ data_ptr_t ColumnDataAllocator::GetDataPointer(ChunkManagementState &state, uint
|
|
174
177
|
return state.handles[block_id].Ptr() + offset;
|
175
178
|
}
|
176
179
|
|
177
|
-
void ColumnDataAllocator::UnswizzlePointers(ChunkManagementState &state, Vector &result,
|
178
|
-
|
180
|
+
void ColumnDataAllocator::UnswizzlePointers(ChunkManagementState &state, Vector &result, idx_t v_offset, uint16_t count,
|
181
|
+
uint32_t block_id, uint32_t offset) {
|
179
182
|
D_ASSERT(result.GetType().InternalType() == PhysicalType::VARCHAR);
|
180
183
|
lock_guard<mutex> guard(lock);
|
181
184
|
|
@@ -225,7 +228,7 @@ Allocator &ColumnDataAllocator::GetAllocator() {
|
|
225
228
|
}
|
226
229
|
|
227
230
|
void ColumnDataAllocator::InitializeChunkState(ChunkManagementState &state, ChunkMetaData &chunk) {
|
228
|
-
if (type != ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR) {
|
231
|
+
if (type != ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR && type != ColumnDataAllocatorType::HYBRID) {
|
229
232
|
// nothing to pin
|
230
233
|
return;
|
231
234
|
}
|
@@ -333,7 +333,7 @@ struct StandardValueCopy : public BaseValueCopy<T> {
|
|
333
333
|
|
334
334
|
struct StringValueCopy : public BaseValueCopy<string_t> {
|
335
335
|
static string_t Operation(ColumnDataMetaData &meta_data, string_t input) {
|
336
|
-
return input.IsInlined() ? input : meta_data.segment.heap
|
336
|
+
return input.IsInlined() ? input : meta_data.segment.heap->AddBlob(input);
|
337
337
|
}
|
338
338
|
};
|
339
339
|
|
@@ -423,7 +423,8 @@ void ColumnDataCopy<string_t>(ColumnDataMetaData &meta_data, const UnifiedVector
|
|
423
423
|
idx_t offset, idx_t copy_count) {
|
424
424
|
|
425
425
|
const auto &allocator_type = meta_data.segment.allocator->GetType();
|
426
|
-
if (allocator_type == ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR
|
426
|
+
if (allocator_type == ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR ||
|
427
|
+
allocator_type == ColumnDataAllocatorType::HYBRID) {
|
427
428
|
// strings cannot be spilled to disk - use StringHeap
|
428
429
|
TemplatedColumnDataCopy<StringValueCopy>(meta_data, source_data, source, offset, copy_count);
|
429
430
|
return;
|
@@ -930,6 +931,7 @@ void ColumnDataCollection::Verify() {
|
|
930
931
|
#endif
|
931
932
|
}
|
932
933
|
|
934
|
+
// LCOV_EXCL_START
|
933
935
|
string ColumnDataCollection::ToString() const {
|
934
936
|
DataChunk chunk;
|
935
937
|
InitializeScanChunk(chunk);
|
@@ -950,6 +952,7 @@ string ColumnDataCollection::ToString() const {
|
|
950
952
|
|
951
953
|
return result;
|
952
954
|
}
|
955
|
+
// LCOV_EXCL_STOP
|
953
956
|
|
954
957
|
void ColumnDataCollection::Print() const {
|
955
958
|
Printer::Print(ToString());
|
@@ -1030,6 +1033,18 @@ bool ColumnDataCollection::ResultEquals(const ColumnDataCollection &left, const
|
|
1030
1033
|
return true;
|
1031
1034
|
}
|
1032
1035
|
|
1036
|
+
vector<shared_ptr<StringHeap>> ColumnDataCollection::GetHeapReferences() {
|
1037
|
+
vector<shared_ptr<StringHeap>> result(segments.size(), nullptr);
|
1038
|
+
for (idx_t segment_idx = 0; segment_idx < segments.size(); segment_idx++) {
|
1039
|
+
result[segment_idx] = segments[segment_idx]->heap;
|
1040
|
+
}
|
1041
|
+
return result;
|
1042
|
+
}
|
1043
|
+
|
1044
|
+
ColumnDataAllocatorType ColumnDataCollection::GetAllocatorType() const {
|
1045
|
+
return allocator->GetType();
|
1046
|
+
}
|
1047
|
+
|
1033
1048
|
const vector<unique_ptr<ColumnDataCollectionSegment>> &ColumnDataCollection::GetSegments() const {
|
1034
1049
|
return segments;
|
1035
1050
|
}
|
@@ -6,7 +6,8 @@ namespace duckdb {
|
|
6
6
|
|
7
7
|
ColumnDataCollectionSegment::ColumnDataCollectionSegment(shared_ptr<ColumnDataAllocator> allocator_p,
|
8
8
|
vector<LogicalType> types_p)
|
9
|
-
: allocator(std::move(allocator_p)), types(std::move(types_p)), count(0),
|
9
|
+
: allocator(std::move(allocator_p)), types(std::move(types_p)), count(0),
|
10
|
+
heap(make_shared<StringHeap>(allocator->GetAllocator())) {
|
10
11
|
}
|
11
12
|
|
12
13
|
idx_t ColumnDataCollectionSegment::GetDataSize(idx_t type_size) {
|
@@ -26,7 +27,8 @@ VectorDataIndex ColumnDataCollectionSegment::AllocateVectorInternal(const Logica
|
|
26
27
|
auto type_size = internal_type == PhysicalType::STRUCT ? 0 : GetTypeIdSize(internal_type);
|
27
28
|
allocator->AllocateData(GetDataSize(type_size) + ValidityMask::STANDARD_MASK_SIZE, meta_data.block_id,
|
28
29
|
meta_data.offset, chunk_state);
|
29
|
-
if (allocator->GetType() == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR
|
30
|
+
if (allocator->GetType() == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR ||
|
31
|
+
allocator->GetType() == ColumnDataAllocatorType::HYBRID) {
|
30
32
|
chunk_meta.block_ids.insert(meta_data.block_id);
|
31
33
|
}
|
32
34
|
|
@@ -203,10 +205,17 @@ idx_t ColumnDataCollectionSegment::ReadVector(ChunkManagementState &state, Vecto
|
|
203
205
|
}
|
204
206
|
} else if (internal_type == PhysicalType::VARCHAR) {
|
205
207
|
if (allocator->GetType() == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR) {
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
208
|
+
auto next_index = vector_index;
|
209
|
+
idx_t offset = 0;
|
210
|
+
while (next_index.IsValid()) {
|
211
|
+
auto ¤t_vdata = GetVectorData(next_index);
|
212
|
+
for (auto &swizzle_segment : current_vdata.swizzle_data) {
|
213
|
+
auto &string_heap_segment = GetVectorData(swizzle_segment.child_index);
|
214
|
+
allocator->UnswizzlePointers(state, result, offset + swizzle_segment.offset, swizzle_segment.count,
|
215
|
+
string_heap_segment.block_id, string_heap_segment.offset);
|
216
|
+
}
|
217
|
+
offset += current_vdata.count;
|
218
|
+
next_index = current_vdata.next_data;
|
210
219
|
}
|
211
220
|
}
|
212
221
|
if (state.properties == ColumnDataScanProperties::DISALLOW_ZERO_COPY) {
|
@@ -32,13 +32,13 @@ PartitionedColumnData::~PartitionedColumnData() {
|
|
32
32
|
|
33
33
|
void PartitionedColumnData::InitializeAppendState(PartitionedColumnDataAppendState &state) const {
|
34
34
|
state.partition_sel.Initialize();
|
35
|
-
state.slice_chunk.Initialize(context, types);
|
35
|
+
state.slice_chunk.Initialize(BufferAllocator::Get(context), types);
|
36
36
|
InitializeAppendStateInternal(state);
|
37
37
|
}
|
38
38
|
|
39
39
|
unique_ptr<DataChunk> PartitionedColumnData::CreatePartitionBuffer() const {
|
40
40
|
auto result = make_uniq<DataChunk>();
|
41
|
-
result->Initialize(
|
41
|
+
result->Initialize(BufferAllocator::Get(context), types, BufferSize());
|
42
42
|
return result;
|
43
43
|
}
|
44
44
|
|
@@ -309,7 +309,7 @@ void DataChunk::Hash(Vector &result) {
|
|
309
309
|
|
310
310
|
void DataChunk::Hash(vector<idx_t> &column_ids, Vector &result) {
|
311
311
|
D_ASSERT(result.GetType().id() == LogicalType::HASH);
|
312
|
-
D_ASSERT(column_ids.
|
312
|
+
D_ASSERT(!column_ids.empty());
|
313
313
|
|
314
314
|
VectorOperations::Hash(data[column_ids[0]], result, size());
|
315
315
|
for (idx_t i = 1; i < column_ids.size(); i++) {
|
@@ -327,7 +327,7 @@ void DataChunk::Verify() {
|
|
327
327
|
#endif
|
328
328
|
}
|
329
329
|
|
330
|
-
void DataChunk::Print() {
|
330
|
+
void DataChunk::Print() const {
|
331
331
|
Printer::Print(ToString());
|
332
332
|
}
|
333
333
|
|
@@ -441,6 +441,15 @@ int64_t Date::EpochMicroseconds(date_t date) {
|
|
441
441
|
return result;
|
442
442
|
}
|
443
443
|
|
444
|
+
int64_t Date::EpochMilliseconds(date_t date) {
|
445
|
+
int64_t result;
|
446
|
+
const auto MILLIS_PER_DAY = Interval::MICROS_PER_DAY / Interval::MICROS_PER_MSEC;
|
447
|
+
if (!TryMultiplyOperator::Operation<int64_t, int64_t, int64_t>(date.days, MILLIS_PER_DAY, result)) {
|
448
|
+
throw ConversionException("Could not convert DATE (%s) to milliseconds", Date::ToString(date));
|
449
|
+
}
|
450
|
+
return result;
|
451
|
+
}
|
452
|
+
|
444
453
|
int32_t Date::ExtractYear(date_t d, int32_t *last_year) {
|
445
454
|
auto n = d.days;
|
446
455
|
// cached look up: check if year of this date is the same as the last one we looked up
|
@@ -12,8 +12,8 @@ static idx_t GetAllocationSize(uint16_t capacity) {
|
|
12
12
|
}
|
13
13
|
|
14
14
|
template <class T>
|
15
|
-
static data_ptr_t AllocatePrimitiveData(
|
16
|
-
return allocator.
|
15
|
+
static data_ptr_t AllocatePrimitiveData(ArenaAllocator &allocator, uint16_t capacity) {
|
16
|
+
return allocator.Allocate(GetAllocationSize<T>(capacity));
|
17
17
|
}
|
18
18
|
|
19
19
|
template <class T>
|
@@ -34,8 +34,8 @@ static idx_t GetAllocationSizeList(uint16_t capacity) {
|
|
34
34
|
return AlignValue(sizeof(ListSegment) + capacity * (sizeof(bool) + sizeof(uint64_t)) + sizeof(LinkedList));
|
35
35
|
}
|
36
36
|
|
37
|
-
static data_ptr_t AllocateListData(
|
38
|
-
return allocator.
|
37
|
+
static data_ptr_t AllocateListData(ArenaAllocator &allocator, uint16_t capacity) {
|
38
|
+
return allocator.Allocate(GetAllocationSizeList(capacity));
|
39
39
|
}
|
40
40
|
|
41
41
|
static uint64_t *GetListLengthData(ListSegment *segment) {
|
@@ -65,8 +65,8 @@ static idx_t GetAllocationSizeStruct(uint16_t capacity, idx_t child_count) {
|
|
65
65
|
return AlignValue(sizeof(ListSegment) + capacity * sizeof(bool) + child_count * sizeof(ListSegment *));
|
66
66
|
}
|
67
67
|
|
68
|
-
static data_ptr_t AllocateStructData(
|
69
|
-
return allocator.
|
68
|
+
static data_ptr_t AllocateStructData(ArenaAllocator &allocator, uint16_t capacity, idx_t child_count) {
|
69
|
+
return allocator.Allocate(GetAllocationSizeStruct(capacity, child_count));
|
70
70
|
}
|
71
71
|
|
72
72
|
static ListSegment **GetStructData(ListSegment *segment) {
|
@@ -98,20 +98,8 @@ static uint16_t GetCapacityForNewSegment(uint16_t capacity) {
|
|
98
98
|
//===--------------------------------------------------------------------===//
|
99
99
|
// Create & Destroy
|
100
100
|
//===--------------------------------------------------------------------===//
|
101
|
-
static void DestroyLinkedList(const ListSegmentFunctions &functions, Allocator &allocator, LinkedList &list) {
|
102
|
-
auto segment = list.first_segment;
|
103
|
-
while (segment) {
|
104
|
-
auto next_segment = segment->next;
|
105
|
-
functions.destroy(functions, segment, allocator);
|
106
|
-
segment = next_segment;
|
107
|
-
}
|
108
|
-
list.first_segment = nullptr;
|
109
|
-
list.last_segment = nullptr;
|
110
|
-
list.total_capacity = 0;
|
111
|
-
}
|
112
|
-
|
113
101
|
template <class T>
|
114
|
-
static ListSegment *CreatePrimitiveSegment(const ListSegmentFunctions &,
|
102
|
+
static ListSegment *CreatePrimitiveSegment(const ListSegmentFunctions &, ArenaAllocator &allocator, uint16_t capacity) {
|
115
103
|
// allocate data and set the header
|
116
104
|
auto segment = (ListSegment *)AllocatePrimitiveData<T>(allocator, capacity);
|
117
105
|
segment->capacity = capacity;
|
@@ -120,13 +108,7 @@ static ListSegment *CreatePrimitiveSegment(const ListSegmentFunctions &, Allocat
|
|
120
108
|
return segment;
|
121
109
|
}
|
122
110
|
|
123
|
-
|
124
|
-
void DestroyPrimitiveSegment(const ListSegmentFunctions &, ListSegment *segment, Allocator &allocator) {
|
125
|
-
D_ASSERT(segment);
|
126
|
-
allocator.FreeData(data_ptr_cast(segment), GetAllocationSize<T>(segment->capacity));
|
127
|
-
}
|
128
|
-
|
129
|
-
static ListSegment *CreateListSegment(const ListSegmentFunctions &, Allocator &allocator, uint16_t capacity) {
|
111
|
+
static ListSegment *CreateListSegment(const ListSegmentFunctions &, ArenaAllocator &allocator, uint16_t capacity) {
|
130
112
|
// allocate data and set the header
|
131
113
|
auto segment = reinterpret_cast<ListSegment *>(AllocateListData(allocator, capacity));
|
132
114
|
segment->capacity = capacity;
|
@@ -141,16 +123,7 @@ static ListSegment *CreateListSegment(const ListSegmentFunctions &, Allocator &a
|
|
141
123
|
return segment;
|
142
124
|
}
|
143
125
|
|
144
|
-
|
145
|
-
// destroy the child list
|
146
|
-
auto linked_child_list = Load<LinkedList>(data_ptr_cast(GetListChildData(segment)));
|
147
|
-
DestroyLinkedList(functions.child_functions[0], allocator, linked_child_list);
|
148
|
-
|
149
|
-
// destroy the list segment itself
|
150
|
-
allocator.FreeData(data_ptr_cast(segment), GetAllocationSizeList(segment->capacity));
|
151
|
-
}
|
152
|
-
|
153
|
-
static ListSegment *CreateStructSegment(const ListSegmentFunctions &functions, Allocator &allocator,
|
126
|
+
static ListSegment *CreateStructSegment(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
|
154
127
|
uint16_t capacity) {
|
155
128
|
// allocate data and set header
|
156
129
|
auto segment =
|
@@ -170,21 +143,8 @@ static ListSegment *CreateStructSegment(const ListSegmentFunctions &functions, A
|
|
170
143
|
return segment;
|
171
144
|
}
|
172
145
|
|
173
|
-
|
174
|
-
|
175
|
-
auto child_segments = GetStructData(segment);
|
176
|
-
for (idx_t i = 0; i < functions.child_functions.size(); i++) {
|
177
|
-
auto child_function = functions.child_functions[i];
|
178
|
-
auto child_segment = Load<ListSegment *>(data_ptr_cast(child_segments + i));
|
179
|
-
child_function.destroy(child_function, child_segment, allocator);
|
180
|
-
}
|
181
|
-
|
182
|
-
// destroy the struct segment itself
|
183
|
-
allocator.FreeData(data_ptr_cast(segment),
|
184
|
-
GetAllocationSizeStruct(segment->capacity, functions.child_functions.size()));
|
185
|
-
}
|
186
|
-
|
187
|
-
static ListSegment *GetSegment(const ListSegmentFunctions &functions, Allocator &allocator, LinkedList &linked_list) {
|
146
|
+
static ListSegment *GetSegment(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
|
147
|
+
LinkedList &linked_list) {
|
188
148
|
ListSegment *segment;
|
189
149
|
|
190
150
|
// determine segment
|
@@ -214,7 +174,7 @@ static ListSegment *GetSegment(const ListSegmentFunctions &functions, Allocator
|
|
214
174
|
// Append
|
215
175
|
//===--------------------------------------------------------------------===//
|
216
176
|
template <class T>
|
217
|
-
static void WriteDataToPrimitiveSegment(const ListSegmentFunctions &functions,
|
177
|
+
static void WriteDataToPrimitiveSegment(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
|
218
178
|
ListSegment *segment, Vector &input, idx_t &entry_idx, idx_t &count) {
|
219
179
|
|
220
180
|
// get the vector data and the source index of the entry that we want to write
|
@@ -232,8 +192,8 @@ static void WriteDataToPrimitiveSegment(const ListSegmentFunctions &functions, A
|
|
232
192
|
}
|
233
193
|
}
|
234
194
|
|
235
|
-
static void WriteDataToVarcharSegment(const ListSegmentFunctions &functions,
|
236
|
-
Vector &input, idx_t &entry_idx, idx_t &count) {
|
195
|
+
static void WriteDataToVarcharSegment(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
|
196
|
+
ListSegment *segment, Vector &input, idx_t &entry_idx, idx_t &count) {
|
237
197
|
|
238
198
|
// get the vector data and the source index of the entry that we want to write
|
239
199
|
auto input_data = FlatVector::GetData<string_t>(input);
|
@@ -275,8 +235,8 @@ static void WriteDataToVarcharSegment(const ListSegmentFunctions &functions, All
|
|
275
235
|
Store<LinkedList>(child_segments, data_ptr_cast(GetListChildData(segment)));
|
276
236
|
}
|
277
237
|
|
278
|
-
static void WriteDataToListSegment(const ListSegmentFunctions &functions,
|
279
|
-
Vector &input, idx_t &entry_idx, idx_t &count) {
|
238
|
+
static void WriteDataToListSegment(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
|
239
|
+
ListSegment *segment, Vector &input, idx_t &entry_idx, idx_t &count) {
|
280
240
|
|
281
241
|
// get the vector data and the source index of the entry that we want to write
|
282
242
|
auto input_data = FlatVector::GetData<list_entry_t>(input);
|
@@ -315,8 +275,8 @@ static void WriteDataToListSegment(const ListSegmentFunctions &functions, Alloca
|
|
315
275
|
Store<uint64_t>(list_length, data_ptr_cast(list_length_data + segment->count));
|
316
276
|
}
|
317
277
|
|
318
|
-
static void WriteDataToStructSegment(const ListSegmentFunctions &functions,
|
319
|
-
Vector &input, idx_t &entry_idx, idx_t &count) {
|
278
|
+
static void WriteDataToStructSegment(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
|
279
|
+
ListSegment *segment, Vector &input, idx_t &entry_idx, idx_t &count) {
|
320
280
|
|
321
281
|
// write null validity
|
322
282
|
auto null_mask = GetNullMask(segment);
|
@@ -338,8 +298,8 @@ static void WriteDataToStructSegment(const ListSegmentFunctions &functions, Allo
|
|
338
298
|
}
|
339
299
|
}
|
340
300
|
|
341
|
-
void ListSegmentFunctions::AppendRow(
|
342
|
-
idx_t &count) const {
|
301
|
+
void ListSegmentFunctions::AppendRow(ArenaAllocator &allocator, LinkedList &linked_list, Vector &input,
|
302
|
+
idx_t &entry_idx, idx_t &count) const {
|
343
303
|
|
344
304
|
D_ASSERT(input.GetVectorType() == VectorType::FLAT_VECTOR);
|
345
305
|
auto &write_data_to_segment = *this;
|
@@ -503,7 +463,7 @@ void ListSegmentFunctions::BuildListVector(const LinkedList &linked_list, Vector
|
|
503
463
|
//===--------------------------------------------------------------------===//
|
504
464
|
template <class T>
|
505
465
|
static ListSegment *CopyDataFromPrimitiveSegment(const ListSegmentFunctions &, const ListSegment *source,
|
506
|
-
|
466
|
+
ArenaAllocator &allocator) {
|
507
467
|
|
508
468
|
auto target = (ListSegment *)AllocatePrimitiveData<T>(allocator, source->capacity);
|
509
469
|
memcpy(target, source, sizeof(ListSegment) + source->capacity * (sizeof(bool) + sizeof(T)));
|
@@ -512,7 +472,7 @@ static ListSegment *CopyDataFromPrimitiveSegment(const ListSegmentFunctions &, c
|
|
512
472
|
}
|
513
473
|
|
514
474
|
static ListSegment *CopyDataFromListSegment(const ListSegmentFunctions &functions, const ListSegment *source,
|
515
|
-
|
475
|
+
ArenaAllocator &allocator) {
|
516
476
|
|
517
477
|
// create an empty linked list for the child vector of target
|
518
478
|
auto source_linked_child_list = Load<LinkedList>(const_data_ptr_cast(GetListChildData(source)));
|
@@ -538,7 +498,7 @@ static ListSegment *CopyDataFromListSegment(const ListSegmentFunctions &function
|
|
538
498
|
}
|
539
499
|
|
540
500
|
static ListSegment *CopyDataFromStructSegment(const ListSegmentFunctions &functions, const ListSegment *source,
|
541
|
-
|
501
|
+
ArenaAllocator &allocator) {
|
542
502
|
|
543
503
|
auto source_child_count = functions.child_functions.size();
|
544
504
|
auto target = reinterpret_cast<ListSegment *>(AllocateStructData(allocator, source->capacity, source_child_count));
|
@@ -560,7 +520,7 @@ static ListSegment *CopyDataFromStructSegment(const ListSegmentFunctions &functi
|
|
560
520
|
}
|
561
521
|
|
562
522
|
void ListSegmentFunctions::CopyLinkedList(const LinkedList &source_list, LinkedList &target_list,
|
563
|
-
|
523
|
+
ArenaAllocator &allocator) const {
|
564
524
|
auto ©_data_from_segment = *this;
|
565
525
|
auto source_segment = source_list.first_segment;
|
566
526
|
|
@@ -578,12 +538,6 @@ void ListSegmentFunctions::CopyLinkedList(const LinkedList &source_list, LinkedL
|
|
578
538
|
}
|
579
539
|
}
|
580
540
|
|
581
|
-
//===--------------------------------------------------------------------===//
|
582
|
-
// Destroy
|
583
|
-
//===--------------------------------------------------------------------===//
|
584
|
-
void ListSegmentFunctions::Destroy(Allocator &allocator, LinkedList &linked_list) const {
|
585
|
-
DestroyLinkedList(*this, allocator, linked_list);
|
586
|
-
}
|
587
541
|
//===--------------------------------------------------------------------===//
|
588
542
|
// Functions
|
589
543
|
//===--------------------------------------------------------------------===//
|
@@ -593,7 +547,6 @@ void SegmentPrimitiveFunction(ListSegmentFunctions &functions) {
|
|
593
547
|
functions.write_data = WriteDataToPrimitiveSegment<T>;
|
594
548
|
functions.read_data = ReadDataFromPrimitiveSegment<T>;
|
595
549
|
functions.copy_data = CopyDataFromPrimitiveSegment<T>;
|
596
|
-
functions.destroy = DestroyPrimitiveSegment<T>;
|
597
550
|
}
|
598
551
|
|
599
552
|
void GetSegmentDataFunctions(ListSegmentFunctions &functions, const LogicalType &type) {
|
@@ -645,7 +598,6 @@ void GetSegmentDataFunctions(ListSegmentFunctions &functions, const LogicalType
|
|
645
598
|
functions.write_data = WriteDataToVarcharSegment;
|
646
599
|
functions.read_data = ReadDataFromVarcharSegment;
|
647
600
|
functions.copy_data = CopyDataFromListSegment;
|
648
|
-
functions.destroy = DestroyListSegment;
|
649
601
|
|
650
602
|
functions.child_functions.emplace_back();
|
651
603
|
SegmentPrimitiveFunction<char>(functions.child_functions.back());
|
@@ -656,7 +608,6 @@ void GetSegmentDataFunctions(ListSegmentFunctions &functions, const LogicalType
|
|
656
608
|
functions.write_data = WriteDataToListSegment;
|
657
609
|
functions.read_data = ReadDataFromListSegment;
|
658
610
|
functions.copy_data = CopyDataFromListSegment;
|
659
|
-
functions.destroy = DestroyListSegment;
|
660
611
|
|
661
612
|
// recurse
|
662
613
|
functions.child_functions.emplace_back();
|
@@ -668,7 +619,6 @@ void GetSegmentDataFunctions(ListSegmentFunctions &functions, const LogicalType
|
|
668
619
|
functions.write_data = WriteDataToStructSegment;
|
669
620
|
functions.read_data = ReadDataFromStructSegment;
|
670
621
|
functions.copy_data = CopyDataFromStructSegment;
|
671
|
-
functions.destroy = DestroyStructSegment;
|
672
622
|
|
673
623
|
// recurse
|
674
624
|
auto child_types = StructType::GetChildTypes(type);
|