duckdb 1.1.4-dev9.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/binding.gyp +1 -0
- package/package.json +2 -2
- package/src/duckdb/extension/core_functions/function_list.cpp +1 -0
- package/src/duckdb/extension/core_functions/include/core_functions/scalar/map_functions.hpp +9 -0
- package/src/duckdb/extension/core_functions/scalar/date/current.cpp +1 -0
- package/src/duckdb/extension/core_functions/scalar/generic/can_implicitly_cast.cpp +2 -2
- package/src/duckdb/extension/core_functions/scalar/generic/typeof.cpp +1 -1
- package/src/duckdb/extension/core_functions/scalar/list/flatten.cpp +91 -61
- package/src/duckdb/extension/core_functions/scalar/map/map_extract.cpp +89 -8
- package/src/duckdb/extension/icu/icu-current.cpp +63 -0
- package/src/duckdb/extension/icu/icu-makedate.cpp +43 -39
- package/src/duckdb/extension/icu/icu-timezone.cpp +63 -63
- package/src/duckdb/extension/icu/icu_extension.cpp +2 -0
- package/src/duckdb/extension/icu/include/icu-casts.hpp +39 -0
- package/src/duckdb/extension/icu/include/icu-current.hpp +17 -0
- package/src/duckdb/extension/icu/third_party/icu/stubdata/stubdata.cpp +1 -1
- package/src/duckdb/extension/json/json_functions/json_structure.cpp +3 -1
- package/src/duckdb/extension/parquet/column_writer.cpp +26 -18
- package/src/duckdb/extension/parquet/include/parquet_reader.hpp +0 -6
- package/src/duckdb/extension/parquet/include/parquet_writer.hpp +15 -1
- package/src/duckdb/extension/parquet/include/resizable_buffer.hpp +1 -0
- package/src/duckdb/extension/parquet/parquet_extension.cpp +67 -15
- package/src/duckdb/extension/parquet/parquet_reader.cpp +5 -3
- package/src/duckdb/extension/parquet/parquet_writer.cpp +5 -6
- package/src/duckdb/src/catalog/catalog.cpp +21 -8
- package/src/duckdb/src/catalog/catalog_search_path.cpp +17 -1
- package/src/duckdb/src/catalog/catalog_set.cpp +1 -1
- package/src/duckdb/src/catalog/default/default_functions.cpp +0 -3
- package/src/duckdb/src/catalog/dependency_list.cpp +7 -0
- package/src/duckdb/src/common/adbc/adbc.cpp +1 -56
- package/src/duckdb/src/common/arrow/arrow_converter.cpp +3 -2
- package/src/duckdb/src/common/arrow/arrow_type_extension.cpp +58 -28
- package/src/duckdb/src/common/arrow/schema_metadata.cpp +1 -1
- package/src/duckdb/src/common/compressed_file_system.cpp +6 -2
- package/src/duckdb/src/common/enum_util.cpp +26 -22
- package/src/duckdb/src/common/error_data.cpp +3 -2
- package/src/duckdb/src/common/gzip_file_system.cpp +8 -8
- package/src/duckdb/src/common/local_file_system.cpp +2 -2
- package/src/duckdb/src/common/multi_file_reader.cpp +1 -1
- package/src/duckdb/src/common/random_engine.cpp +4 -1
- package/src/duckdb/src/common/serializer/memory_stream.cpp +23 -19
- package/src/duckdb/src/common/serializer/serializer.cpp +1 -1
- package/src/duckdb/src/common/types/bit.cpp +1 -1
- package/src/duckdb/src/common/types/column/column_data_allocator.cpp +0 -5
- package/src/duckdb/src/common/types/column/column_data_collection.cpp +4 -1
- package/src/duckdb/src/common/types/data_chunk.cpp +2 -1
- package/src/duckdb/src/common/types/row/tuple_data_segment.cpp +0 -4
- package/src/duckdb/src/common/types.cpp +1 -1
- package/src/duckdb/src/execution/index/art/art.cpp +52 -42
- package/src/duckdb/src/execution/index/art/leaf.cpp +4 -9
- package/src/duckdb/src/execution/index/art/node.cpp +13 -13
- package/src/duckdb/src/execution/index/art/prefix.cpp +21 -16
- package/src/duckdb/src/execution/index/bound_index.cpp +6 -8
- package/src/duckdb/src/execution/index/fixed_size_allocator.cpp +39 -34
- package/src/duckdb/src/execution/index/fixed_size_buffer.cpp +2 -1
- package/src/duckdb/src/execution/index/unbound_index.cpp +10 -0
- package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +62 -44
- package/src/duckdb/src/execution/operator/csv_scanner/scanner/column_count_scanner.cpp +26 -0
- package/src/duckdb/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +69 -40
- package/src/duckdb/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp +3 -7
- package/src/duckdb/src/execution/operator/csv_scanner/sniffer/header_detection.cpp +11 -5
- package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_detection.cpp +4 -0
- package/src/duckdb/src/execution/operator/csv_scanner/state_machine/csv_state_machine_cache.cpp +8 -8
- package/src/duckdb/src/execution/operator/csv_scanner/util/csv_error.cpp +36 -12
- package/src/duckdb/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +12 -9
- package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +0 -1
- package/src/duckdb/src/execution/operator/persistent/physical_copy_database.cpp +29 -1
- package/src/duckdb/src/execution/operator/persistent/physical_delete.cpp +58 -10
- package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +58 -35
- package/src/duckdb/src/execution/operator/schema/physical_create_art_index.cpp +2 -1
- package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +9 -4
- package/src/duckdb/src/execution/sample/reservoir_sample.cpp +7 -6
- package/src/duckdb/src/function/compression_config.cpp +4 -0
- package/src/duckdb/src/function/function_binder.cpp +1 -1
- package/src/duckdb/src/function/scalar/system/write_log.cpp +2 -2
- package/src/duckdb/src/function/table/arrow/arrow_duck_schema.cpp +15 -2
- package/src/duckdb/src/function/table/arrow_conversion.cpp +10 -10
- package/src/duckdb/src/function/table/copy_csv.cpp +8 -5
- package/src/duckdb/src/function/table/read_csv.cpp +21 -4
- package/src/duckdb/src/function/table/sniff_csv.cpp +7 -0
- package/src/duckdb/src/function/table/system/duckdb_extensions.cpp +4 -0
- package/src/duckdb/src/function/table/system/duckdb_secret_types.cpp +71 -0
- package/src/duckdb/src/function/table/system_functions.cpp +1 -0
- package/src/duckdb/src/function/table/table_scan.cpp +120 -36
- package/src/duckdb/src/function/table/version/pragma_version.cpp +4 -4
- package/src/duckdb/src/function/window/window_aggregate_function.cpp +6 -1
- package/src/duckdb/src/function/window/window_boundaries_state.cpp +135 -11
- package/src/duckdb/src/function/window/window_segment_tree.cpp +50 -22
- package/src/duckdb/src/function/window/window_token_tree.cpp +4 -3
- package/src/duckdb/src/include/duckdb/catalog/catalog.hpp +4 -0
- package/src/duckdb/src/include/duckdb/catalog/catalog_search_path.hpp +2 -0
- package/src/duckdb/src/include/duckdb/catalog/dependency_list.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/arrow/arrow_type_extension.hpp +4 -2
- package/src/duckdb/src/include/duckdb/common/enum_util.hpp +8 -8
- package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +0 -2
- package/src/duckdb/src/include/duckdb/common/serializer/deserializer.hpp +8 -3
- package/src/duckdb/src/include/duckdb/common/serializer/memory_stream.hpp +6 -1
- package/src/duckdb/src/include/duckdb/common/serializer/serialization_data.hpp +25 -0
- package/src/duckdb/src/include/duckdb/common/serializer/serializer.hpp +9 -3
- package/src/duckdb/src/include/duckdb/common/types/selection_vector.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +11 -14
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +5 -4
- package/src/duckdb/src/include/duckdb/execution/index/bound_index.hpp +21 -10
- package/src/duckdb/src/include/duckdb/execution/index/fixed_size_allocator.hpp +6 -5
- package/src/duckdb/src/include/duckdb/execution/index/fixed_size_buffer.hpp +37 -32
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/base_scanner.hpp +36 -1
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/column_count_scanner.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/sniffer/csv_sniffer.hpp +2 -0
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/state_machine_options.hpp +5 -5
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +5 -30
- package/src/duckdb/src/include/duckdb/execution/reservoir_sample.hpp +7 -1
- package/src/duckdb/src/include/duckdb/function/scalar_function.hpp +3 -3
- package/src/duckdb/src/include/duckdb/function/table/arrow/arrow_duck_schema.hpp +1 -0
- package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +4 -0
- package/src/duckdb/src/include/duckdb/function/window/window_boundaries_state.hpp +2 -2
- package/src/duckdb/src/include/duckdb/logging/logger.hpp +40 -119
- package/src/duckdb/src/include/duckdb/logging/logging.hpp +0 -2
- package/src/duckdb/src/include/duckdb/main/config.hpp +5 -0
- package/src/duckdb/src/include/duckdb/main/connection.hpp +0 -8
- package/src/duckdb/src/include/duckdb/main/connection_manager.hpp +2 -1
- package/src/duckdb/src/include/duckdb/main/extension.hpp +1 -0
- package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +11 -7
- package/src/duckdb/src/include/duckdb/main/extension_helper.hpp +1 -0
- package/src/duckdb/src/include/duckdb/main/secret/secret.hpp +2 -0
- package/src/duckdb/src/include/duckdb/main/secret/secret_manager.hpp +3 -0
- package/src/duckdb/src/include/duckdb/main/settings.hpp +10 -0
- package/src/duckdb/src/include/duckdb/parser/constraint.hpp +9 -0
- package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +36 -9
- package/src/duckdb/src/include/duckdb/parser/parsed_data/create_view_info.hpp +2 -1
- package/src/duckdb/src/include/duckdb/parser/query_node/set_operation_node.hpp +8 -2
- package/src/duckdb/src/include/duckdb/planner/binder.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/expression/bound_parameter_data.hpp +9 -1
- package/src/duckdb/src/include/duckdb/planner/filter/constant_filter.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/filter/in_filter.hpp +0 -2
- package/src/duckdb/src/include/duckdb/planner/filter/optional_filter.hpp +4 -4
- package/src/duckdb/src/include/duckdb/planner/table_filter.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/data_table.hpp +14 -10
- package/src/duckdb/src/include/duckdb/storage/index_storage_info.hpp +4 -0
- package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +6 -1
- package/src/duckdb/src/include/duckdb/storage/storage_info.hpp +7 -2
- package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp +9 -0
- package/src/duckdb/src/include/duckdb/storage/storage_options.hpp +2 -0
- package/src/duckdb/src/include/duckdb/storage/string_uncompressed.hpp +4 -3
- package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp +2 -0
- package/src/duckdb/src/include/duckdb/storage/table/table_index_list.hpp +6 -4
- package/src/duckdb/src/include/duckdb/storage/table/table_statistics.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp +2 -0
- package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -0
- package/src/duckdb/src/include/duckdb/transaction/meta_transaction.hpp +1 -1
- package/src/duckdb/src/logging/logger.cpp +8 -66
- package/src/duckdb/src/main/attached_database.cpp +3 -1
- package/src/duckdb/src/main/client_context.cpp +4 -2
- package/src/duckdb/src/main/config.cpp +20 -2
- package/src/duckdb/src/main/connection.cpp +2 -29
- package/src/duckdb/src/main/connection_manager.cpp +5 -3
- package/src/duckdb/src/main/database.cpp +2 -2
- package/src/duckdb/src/main/extension/extension_helper.cpp +4 -5
- package/src/duckdb/src/main/extension/extension_install.cpp +23 -10
- package/src/duckdb/src/main/extension/extension_load.cpp +6 -7
- package/src/duckdb/src/main/extension.cpp +27 -9
- package/src/duckdb/src/main/secret/secret_manager.cpp +11 -0
- package/src/duckdb/src/main/settings/custom_settings.cpp +44 -0
- package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +6 -0
- package/src/duckdb/src/optimizer/filter_combiner.cpp +13 -3
- package/src/duckdb/src/optimizer/filter_pushdown.cpp +33 -6
- package/src/duckdb/src/optimizer/late_materialization.cpp +14 -3
- package/src/duckdb/src/optimizer/remove_unused_columns.cpp +0 -3
- package/src/duckdb/src/parser/parsed_data/attach_info.cpp +5 -1
- package/src/duckdb/src/parser/parsed_data/create_view_info.cpp +6 -3
- package/src/duckdb/src/parser/query_node/set_operation_node.cpp +49 -0
- package/src/duckdb/src/parser/transform/expression/transform_columnref.cpp +1 -0
- package/src/duckdb/src/parser/transform/expression/transform_function.cpp +50 -12
- package/src/duckdb/src/planner/binder/expression/bind_columnref_expression.cpp +7 -5
- package/src/duckdb/src/planner/binder/expression/bind_comparison_expression.cpp +1 -0
- package/src/duckdb/src/planner/binder/expression/bind_operator_expression.cpp +2 -2
- package/src/duckdb/src/planner/binder/expression/bind_star_expression.cpp +12 -2
- package/src/duckdb/src/planner/binder/statement/bind_copy_database.cpp +0 -1
- package/src/duckdb/src/planner/binder/statement/bind_create.cpp +55 -39
- package/src/duckdb/src/planner/binder/statement/bind_execute.cpp +2 -1
- package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +15 -7
- package/src/duckdb/src/planner/binder/tableref/bind_showref.cpp +13 -8
- package/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp +8 -3
- package/src/duckdb/src/planner/expression/bound_function_expression.cpp +17 -1
- package/src/duckdb/src/planner/expression_binder/index_binder.cpp +1 -0
- package/src/duckdb/src/planner/filter/conjunction_filter.cpp +1 -0
- package/src/duckdb/src/planner/filter/constant_filter.cpp +21 -0
- package/src/duckdb/src/planner/filter/in_filter.cpp +4 -7
- package/src/duckdb/src/planner/logical_operator.cpp +5 -3
- package/src/duckdb/src/planner/planner.cpp +1 -1
- package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +2 -0
- package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +3 -4
- package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -5
- package/src/duckdb/src/storage/compression/dictionary/decompression.cpp +4 -4
- package/src/duckdb/src/storage/compression/fsst.cpp +2 -2
- package/src/duckdb/src/storage/compression/roaring/common.cpp +10 -1
- package/src/duckdb/src/storage/compression/string_uncompressed.cpp +11 -6
- package/src/duckdb/src/storage/compression/validity_uncompressed.cpp +4 -0
- package/src/duckdb/src/storage/compression/zstd.cpp +6 -0
- package/src/duckdb/src/storage/data_table.cpp +104 -109
- package/src/duckdb/src/storage/local_storage.cpp +8 -6
- package/src/duckdb/src/storage/magic_bytes.cpp +1 -1
- package/src/duckdb/src/storage/serialization/serialize_dependency.cpp +3 -3
- package/src/duckdb/src/storage/serialization/serialize_nodes.cpp +3 -3
- package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +7 -5
- package/src/duckdb/src/storage/single_file_block_manager.cpp +95 -28
- package/src/duckdb/src/storage/storage_info.cpp +38 -0
- package/src/duckdb/src/storage/storage_manager.cpp +11 -0
- package/src/duckdb/src/storage/table/column_data.cpp +4 -0
- package/src/duckdb/src/storage/table/column_data_checkpointer.cpp +3 -3
- package/src/duckdb/src/storage/table/row_group_collection.cpp +67 -68
- package/src/duckdb/src/storage/table/table_statistics.cpp +4 -4
- package/src/duckdb/src/storage/table_index_list.cpp +41 -15
- package/src/duckdb/src/storage/wal_replay.cpp +3 -1
- package/src/duckdb/src/storage/write_ahead_log.cpp +11 -4
- package/src/duckdb/src/transaction/meta_transaction.cpp +1 -1
- package/src/duckdb/src/verification/deserialized_statement_verifier.cpp +2 -1
- package/src/duckdb/third_party/httplib/httplib.hpp +0 -1
- package/src/duckdb/third_party/re2/util/logging.h +10 -10
- package/src/duckdb/ub_src_function_table_system.cpp +2 -0
@@ -79,17 +79,26 @@ public:
|
|
79
79
|
|
80
80
|
void Combine(WindowSegmentTreePart &other, idx_t count);
|
81
81
|
|
82
|
-
void Evaluate(const WindowSegmentTreeGlobalState &tree, const idx_t *begins, const idx_t *ends,
|
83
|
-
idx_t count, idx_t row_idx, FramePart frame_part);
|
82
|
+
void Evaluate(const WindowSegmentTreeGlobalState &tree, const idx_t *begins, const idx_t *ends, const idx_t *bounds,
|
83
|
+
Vector &result, idx_t count, idx_t row_idx, FramePart frame_part);
|
84
84
|
|
85
85
|
protected:
|
86
86
|
//! Initialises the accumulation state vector (statef)
|
87
87
|
void Initialize(idx_t count);
|
88
88
|
//! Accumulate upper tree levels
|
89
89
|
void EvaluateUpperLevels(const WindowSegmentTreeGlobalState &tree, const idx_t *begins, const idx_t *ends,
|
90
|
-
idx_t count, idx_t row_idx, FramePart frame_part);
|
91
|
-
void EvaluateLeaves(const WindowSegmentTreeGlobalState &tree, const idx_t *begins, const idx_t *ends,
|
92
|
-
idx_t row_idx, FramePart frame_part, FramePart leaf_part);
|
90
|
+
const idx_t *bounds, idx_t count, idx_t row_idx, FramePart frame_part);
|
91
|
+
void EvaluateLeaves(const WindowSegmentTreeGlobalState &tree, const idx_t *begins, const idx_t *ends,
|
92
|
+
const idx_t *bounds, idx_t count, idx_t row_idx, FramePart frame_part, FramePart leaf_part);
|
93
|
+
|
94
|
+
static inline const idx_t *FrameBegins(const idx_t *begins, const idx_t *ends, const idx_t *bounds,
|
95
|
+
FramePart frame_part) {
|
96
|
+
return frame_part == FramePart::RIGHT ? bounds : begins;
|
97
|
+
}
|
98
|
+
static inline const idx_t *FrameEnds(const idx_t *begins, const idx_t *ends, const idx_t *bounds,
|
99
|
+
FramePart frame_part) {
|
100
|
+
return frame_part == FramePart::LEFT ? bounds : ends;
|
101
|
+
}
|
93
102
|
|
94
103
|
public:
|
95
104
|
//! Allocator for aggregates
|
@@ -400,8 +409,12 @@ void WindowSegmentTreeState::Evaluate(const WindowSegmentTreeGlobalState >stat
|
|
400
409
|
}
|
401
410
|
|
402
411
|
if (gtstate.aggregator.exclude_mode != WindowExcludeMode::NO_OTHER) {
|
412
|
+
// If we exclude the current row, then both left and right need to contain it.
|
413
|
+
const bool exclude_current = gtstate.aggregator.exclude_mode == WindowExcludeMode::CURRENT_ROW;
|
414
|
+
|
403
415
|
// 1. evaluate the tree left of the excluded part
|
404
|
-
|
416
|
+
auto middle = exclude_current ? peer_end : peer_begin;
|
417
|
+
part->Evaluate(gtstate, window_begin, middle, window_end, result, count, row_idx, WindowSegmentTreePart::LEFT);
|
405
418
|
|
406
419
|
// 2. set up a second state for the right of the excluded part
|
407
420
|
if (!right_part) {
|
@@ -409,32 +422,35 @@ void WindowSegmentTreeState::Evaluate(const WindowSegmentTreeGlobalState >stat
|
|
409
422
|
}
|
410
423
|
|
411
424
|
// 3. evaluate the tree right of the excluded part
|
412
|
-
|
425
|
+
middle = exclude_current ? peer_begin : peer_end;
|
426
|
+
right_part->Evaluate(gtstate, middle, window_end, window_begin, result, count, row_idx,
|
427
|
+
WindowSegmentTreePart::RIGHT);
|
413
428
|
|
414
429
|
// 4. combine the buffer state into the Segment Tree State
|
415
430
|
part->Combine(*right_part, count);
|
416
431
|
} else {
|
417
|
-
part->Evaluate(gtstate, window_begin, window_end, result, count, row_idx, WindowSegmentTreePart::FULL);
|
432
|
+
part->Evaluate(gtstate, window_begin, window_end, nullptr, result, count, row_idx, WindowSegmentTreePart::FULL);
|
418
433
|
}
|
419
434
|
|
420
435
|
part->Finalize(result, count);
|
421
436
|
}
|
422
437
|
|
423
438
|
void WindowSegmentTreePart::Evaluate(const WindowSegmentTreeGlobalState &tree, const idx_t *begins, const idx_t *ends,
|
424
|
-
Vector &result, idx_t count, idx_t row_idx,
|
439
|
+
const idx_t *bounds, Vector &result, idx_t count, idx_t row_idx,
|
440
|
+
FramePart frame_part) {
|
425
441
|
Initialize(count);
|
426
442
|
|
427
443
|
if (order_insensitive) {
|
428
444
|
// First pass: aggregate the segment tree nodes with sharing
|
429
|
-
EvaluateUpperLevels(tree, begins, ends, count, row_idx, frame_part);
|
445
|
+
EvaluateUpperLevels(tree, begins, ends, bounds, count, row_idx, frame_part);
|
430
446
|
|
431
447
|
// Second pass: aggregate the ragged leaves
|
432
|
-
EvaluateLeaves(tree, begins, ends, count, row_idx, frame_part, FramePart::FULL);
|
448
|
+
EvaluateLeaves(tree, begins, ends, bounds, count, row_idx, frame_part, FramePart::FULL);
|
433
449
|
} else {
|
434
450
|
// Evaluate leaves in order
|
435
|
-
EvaluateLeaves(tree, begins, ends, count, row_idx, frame_part, FramePart::LEFT);
|
436
|
-
EvaluateUpperLevels(tree, begins, ends, count, row_idx, frame_part);
|
437
|
-
EvaluateLeaves(tree, begins, ends, count, row_idx, frame_part, FramePart::RIGHT);
|
451
|
+
EvaluateLeaves(tree, begins, ends, bounds, count, row_idx, frame_part, FramePart::LEFT);
|
452
|
+
EvaluateUpperLevels(tree, begins, ends, bounds, count, row_idx, frame_part);
|
453
|
+
EvaluateLeaves(tree, begins, ends, bounds, count, row_idx, frame_part, FramePart::RIGHT);
|
438
454
|
}
|
439
455
|
}
|
440
456
|
|
@@ -447,13 +463,18 @@ void WindowSegmentTreePart::Initialize(idx_t count) {
|
|
447
463
|
}
|
448
464
|
|
449
465
|
void WindowSegmentTreePart::EvaluateUpperLevels(const WindowSegmentTreeGlobalState &tree, const idx_t *begins,
|
450
|
-
const idx_t *ends, idx_t count, idx_t row_idx,
|
466
|
+
const idx_t *ends, const idx_t *bounds, idx_t count, idx_t row_idx,
|
467
|
+
FramePart frame_part) {
|
451
468
|
auto fdata = FlatVector::GetData<data_ptr_t>(statef);
|
452
469
|
|
453
470
|
const auto exclude_mode = tree.tree.exclude_mode;
|
454
471
|
const bool begin_on_curr_row = frame_part == FramePart::RIGHT && exclude_mode == WindowExcludeMode::CURRENT_ROW;
|
455
472
|
const bool end_on_curr_row = frame_part == FramePart::LEFT && exclude_mode == WindowExcludeMode::CURRENT_ROW;
|
456
473
|
|
474
|
+
// We need the full range of the frame to clamp
|
475
|
+
auto frame_begins = FrameBegins(begins, ends, bounds, frame_part);
|
476
|
+
auto frame_ends = FrameEnds(begins, ends, bounds, frame_part);
|
477
|
+
|
457
478
|
const auto max_level = tree.levels_flat_start.size() + 1;
|
458
479
|
right_stack.resize(max_level, {0, 0});
|
459
480
|
|
@@ -467,8 +488,8 @@ void WindowSegmentTreePart::EvaluateUpperLevels(const WindowSegmentTreeGlobalSta
|
|
467
488
|
for (idx_t rid = 0, cur_row = row_idx; rid < count; ++rid, ++cur_row) {
|
468
489
|
auto state_ptr = fdata[rid];
|
469
490
|
|
470
|
-
auto begin = begin_on_curr_row ? cur_row + 1 : begins[rid];
|
471
|
-
auto end = end_on_curr_row ? cur_row : ends[rid];
|
491
|
+
auto begin = MaxValue(begin_on_curr_row ? cur_row + 1 : begins[rid], frame_begins[rid]);
|
492
|
+
auto end = MinValue(end_on_curr_row ? cur_row : ends[rid], frame_ends[rid]);
|
472
493
|
if (begin >= end) {
|
473
494
|
continue;
|
474
495
|
}
|
@@ -541,8 +562,8 @@ void WindowSegmentTreePart::EvaluateUpperLevels(const WindowSegmentTreeGlobalSta
|
|
541
562
|
}
|
542
563
|
|
543
564
|
void WindowSegmentTreePart::EvaluateLeaves(const WindowSegmentTreeGlobalState &tree, const idx_t *begins,
|
544
|
-
const idx_t *ends, idx_t
|
545
|
-
FramePart leaf_part) {
|
565
|
+
const idx_t *ends, const idx_t *bounds, idx_t count, idx_t row_idx,
|
566
|
+
FramePart frame_part, FramePart leaf_part) {
|
546
567
|
|
547
568
|
auto fdata = FlatVector::GetData<data_ptr_t>(statef);
|
548
569
|
|
@@ -558,12 +579,19 @@ void WindowSegmentTreePart::EvaluateLeaves(const WindowSegmentTreeGlobalState &t
|
|
558
579
|
// current row
|
559
580
|
const bool add_curr_row = compute_left && frame_part == FramePart::RIGHT && exclude_mode == WindowExcludeMode::TIES;
|
560
581
|
|
582
|
+
// We need the full range of the frame to clamp
|
583
|
+
auto frame_begins = FrameBegins(begins, ends, bounds, frame_part);
|
584
|
+
auto frame_ends = FrameEnds(begins, ends, bounds, frame_part);
|
585
|
+
|
561
586
|
for (idx_t rid = 0, cur_row = row_idx; rid < count; ++rid, ++cur_row) {
|
562
587
|
auto state_ptr = fdata[rid];
|
563
588
|
|
564
|
-
const auto
|
565
|
-
|
566
|
-
|
589
|
+
const auto frame_begin = frame_begins[rid];
|
590
|
+
auto begin = MaxValue(begin_on_curr_row ? cur_row + 1 : begins[rid], frame_begin);
|
591
|
+
|
592
|
+
const auto frame_end = frame_ends[rid];
|
593
|
+
auto end = MinValue(end_on_curr_row ? cur_row : ends[rid], frame_end);
|
594
|
+
if (add_curr_row && frame_begin <= cur_row && cur_row < frame_end) {
|
567
595
|
WindowSegmentValue(tree, 0, cur_row, cur_row + 1, state_ptr);
|
568
596
|
}
|
569
597
|
if (begin >= end) {
|
@@ -122,13 +122,14 @@ idx_t WindowTokenTree::Rank(const idx_t lower, const idx_t upper, const idx_t ro
|
|
122
122
|
|
123
123
|
template <typename TREE>
|
124
124
|
static idx_t NextPeer(const TREE &tree, const idx_t lower, const idx_t upper, const idx_t row_idx) {
|
125
|
-
|
125
|
+
// We return an index, not a relative position
|
126
|
+
idx_t idx = lower;
|
126
127
|
// Because tokens are dense, we can find the next peer by adding 1 to the probed token value
|
127
128
|
const auto needle = tree.LowestLevel()[row_idx] + 1;
|
128
129
|
tree.AggregateLowerBound(lower, upper, needle, [&](idx_t level, const idx_t run_begin, const idx_t run_pos) {
|
129
|
-
|
130
|
+
idx += run_pos - run_begin;
|
130
131
|
});
|
131
|
-
return
|
132
|
+
return idx;
|
132
133
|
}
|
133
134
|
|
134
135
|
idx_t WindowTokenTree::PeerEnd(const idx_t lower, const idx_t upper, const idx_t row_idx) const {
|
@@ -315,7 +315,11 @@ public:
|
|
315
315
|
return CatalogLookupBehavior::STANDARD;
|
316
316
|
}
|
317
317
|
|
318
|
+
//! Returns the default schema of the catalog
|
319
|
+
virtual string GetDefaultSchema() const;
|
320
|
+
|
318
321
|
//! The default table is used for `SELECT * FROM <catalog_name>;`
|
322
|
+
//! FIXME: these should be virtual methods
|
319
323
|
DUCKDB_API bool HasDefaultTable() const;
|
320
324
|
DUCKDB_API void SetDefaultTable(const string &schema, const string &name);
|
321
325
|
DUCKDB_API string GetDefaultTable() const;
|
@@ -53,7 +53,9 @@ public:
|
|
53
53
|
return set_paths;
|
54
54
|
}
|
55
55
|
DUCKDB_API const CatalogSearchEntry &GetDefault();
|
56
|
+
//! FIXME: this method is deprecated
|
56
57
|
DUCKDB_API string GetDefaultSchema(const string &catalog);
|
58
|
+
DUCKDB_API string GetDefaultSchema(ClientContext &context, const string &catalog);
|
57
59
|
DUCKDB_API string GetDefaultCatalog(const string &schema);
|
58
60
|
|
59
61
|
DUCKDB_API vector<string> GetSchemasForCatalog(const string &catalog);
|
@@ -31,6 +31,7 @@ public:
|
|
31
31
|
public:
|
32
32
|
explicit LogicalDependency(CatalogEntry &entry);
|
33
33
|
LogicalDependency();
|
34
|
+
LogicalDependency(optional_ptr<Catalog> catalog, CatalogEntryInfo entry, string catalog_str);
|
34
35
|
bool operator==(const LogicalDependency &other) const;
|
35
36
|
|
36
37
|
public:
|
@@ -65,11 +65,13 @@ typedef void (*populate_arrow_schema_t)(DuckDBArrowSchemaHolder &root_holder, Ar
|
|
65
65
|
const LogicalType &type, ClientContext &context,
|
66
66
|
const ArrowTypeExtension &extension);
|
67
67
|
|
68
|
-
typedef
|
68
|
+
typedef unique_ptr<ArrowType> (*get_type_t)(const ArrowSchema &schema, const ArrowSchemaMetadata &schema_metadata);
|
69
69
|
|
70
70
|
class ArrowTypeExtension {
|
71
71
|
public:
|
72
72
|
ArrowTypeExtension() {};
|
73
|
+
//! This type is not registered, so we just use whatever is the format and hope for the best
|
74
|
+
explicit ArrowTypeExtension(ArrowExtensionMetadata &extension_metadata, unique_ptr<ArrowType> type);
|
73
75
|
//! We either have simple extensions where we only return one type
|
74
76
|
ArrowTypeExtension(string extension_name, string arrow_format, shared_ptr<ArrowTypeExtensionData> type);
|
75
77
|
ArrowTypeExtension(string vendor_name, string type_name, string arrow_format,
|
@@ -84,7 +86,7 @@ public:
|
|
84
86
|
|
85
87
|
ArrowExtensionMetadata GetInfo() const;
|
86
88
|
|
87
|
-
|
89
|
+
unique_ptr<ArrowType> GetType(const ArrowSchema &schema, const ArrowSchemaMetadata &schema_metadata) const;
|
88
90
|
|
89
91
|
shared_ptr<ArrowTypeExtensionData> GetTypeExtension() const;
|
90
92
|
|
@@ -32,8 +32,6 @@ struct EnumUtil {
|
|
32
32
|
static string ToString(T value) { return string(ToChars<T>(value)); }
|
33
33
|
};
|
34
34
|
|
35
|
-
enum class ARTAppendMode : uint8_t;
|
36
|
-
|
37
35
|
enum class ARTConflictType : uint8_t;
|
38
36
|
|
39
37
|
enum class AccessMode : uint8_t;
|
@@ -188,6 +186,8 @@ enum class GateStatus : uint8_t;
|
|
188
186
|
|
189
187
|
enum class HLLStorageType : uint8_t;
|
190
188
|
|
189
|
+
enum class IndexAppendMode : uint8_t;
|
190
|
+
|
191
191
|
enum class IndexConstraintType : uint8_t;
|
192
192
|
|
193
193
|
enum class InsertColumnOrder : uint8_t;
|
@@ -391,9 +391,6 @@ enum class WindowBoundary : uint8_t;
|
|
391
391
|
enum class WindowExcludeMode : uint8_t;
|
392
392
|
|
393
393
|
|
394
|
-
template<>
|
395
|
-
const char* EnumUtil::ToChars<ARTAppendMode>(ARTAppendMode value);
|
396
|
-
|
397
394
|
template<>
|
398
395
|
const char* EnumUtil::ToChars<ARTConflictType>(ARTConflictType value);
|
399
396
|
|
@@ -625,6 +622,9 @@ const char* EnumUtil::ToChars<GateStatus>(GateStatus value);
|
|
625
622
|
template<>
|
626
623
|
const char* EnumUtil::ToChars<HLLStorageType>(HLLStorageType value);
|
627
624
|
|
625
|
+
template<>
|
626
|
+
const char* EnumUtil::ToChars<IndexAppendMode>(IndexAppendMode value);
|
627
|
+
|
628
628
|
template<>
|
629
629
|
const char* EnumUtil::ToChars<IndexConstraintType>(IndexConstraintType value);
|
630
630
|
|
@@ -929,9 +929,6 @@ template<>
|
|
929
929
|
const char* EnumUtil::ToChars<WindowExcludeMode>(WindowExcludeMode value);
|
930
930
|
|
931
931
|
|
932
|
-
template<>
|
933
|
-
ARTAppendMode EnumUtil::FromString<ARTAppendMode>(const char *value);
|
934
|
-
|
935
932
|
template<>
|
936
933
|
ARTConflictType EnumUtil::FromString<ARTConflictType>(const char *value);
|
937
934
|
|
@@ -1163,6 +1160,9 @@ GateStatus EnumUtil::FromString<GateStatus>(const char *value);
|
|
1163
1160
|
template<>
|
1164
1161
|
HLLStorageType EnumUtil::FromString<HLLStorageType>(const char *value);
|
1165
1162
|
|
1163
|
+
template<>
|
1164
|
+
IndexAppendMode EnumUtil::FromString<IndexAppendMode>(const char *value);
|
1165
|
+
|
1166
1166
|
template<>
|
1167
1167
|
IndexConstraintType EnumUtil::FromString<IndexConstraintType>(const char *value);
|
1168
1168
|
|
@@ -47,7 +47,6 @@ public:
|
|
47
47
|
identifier(other.identifier) {
|
48
48
|
}
|
49
49
|
|
50
|
-
MultiFileReaderColumnDefinition(MultiFileReaderColumnDefinition &&other) noexcept = default;
|
51
50
|
MultiFileReaderColumnDefinition &operator=(const MultiFileReaderColumnDefinition &other) {
|
52
51
|
if (this != &other) {
|
53
52
|
name = other.name;
|
@@ -58,7 +57,6 @@ public:
|
|
58
57
|
}
|
59
58
|
return *this;
|
60
59
|
}
|
61
|
-
MultiFileReaderColumnDefinition &operator=(MultiFileReaderColumnDefinition &&other) noexcept = default;
|
62
60
|
|
63
61
|
public:
|
64
62
|
static vector<MultiFileReaderColumnDefinition> ColumnsFromNamesAndTypes(const vector<string> &names,
|
@@ -81,7 +81,7 @@ public:
|
|
81
81
|
}
|
82
82
|
|
83
83
|
template <typename T>
|
84
|
-
inline T ReadPropertyWithExplicitDefault(const field_id_t field_id, const char *tag, T
|
84
|
+
inline T ReadPropertyWithExplicitDefault(const field_id_t field_id, const char *tag, T default_value) {
|
85
85
|
if (!OnOptionalPropertyBegin(field_id, tag)) {
|
86
86
|
OnOptionalPropertyEnd(false);
|
87
87
|
return std::forward<T>(default_value);
|
@@ -104,7 +104,7 @@ public:
|
|
104
104
|
}
|
105
105
|
|
106
106
|
template <typename T>
|
107
|
-
inline void ReadPropertyWithExplicitDefault(const field_id_t field_id, const char *tag, T &ret, T
|
107
|
+
inline void ReadPropertyWithExplicitDefault(const field_id_t field_id, const char *tag, T &ret, T default_value) {
|
108
108
|
if (!OnOptionalPropertyBegin(field_id, tag)) {
|
109
109
|
ret = std::forward<T>(default_value);
|
110
110
|
OnOptionalPropertyEnd(false);
|
@@ -116,7 +116,7 @@ public:
|
|
116
116
|
|
117
117
|
template <typename T>
|
118
118
|
inline void ReadPropertyWithExplicitDefault(const field_id_t field_id, const char *tag, CSVOption<T> &ret,
|
119
|
-
T
|
119
|
+
T default_value) {
|
120
120
|
if (!OnOptionalPropertyBegin(field_id, tag)) {
|
121
121
|
ret = std::forward<T>(default_value);
|
122
122
|
OnOptionalPropertyEnd(false);
|
@@ -159,6 +159,11 @@ public:
|
|
159
159
|
return data.Get<T>();
|
160
160
|
}
|
161
161
|
|
162
|
+
template <class T>
|
163
|
+
optional_ptr<T> TryGet() {
|
164
|
+
return data.TryGet<T>();
|
165
|
+
}
|
166
|
+
|
162
167
|
//! Unset a serialization property
|
163
168
|
template <class T>
|
164
169
|
void Unset() {
|
@@ -13,17 +13,22 @@
|
|
13
13
|
#include "duckdb/common/typedefs.hpp"
|
14
14
|
|
15
15
|
namespace duckdb {
|
16
|
+
class Allocator;
|
16
17
|
|
17
18
|
class MemoryStream : public WriteStream, public ReadStream {
|
18
19
|
private:
|
20
|
+
optional_ptr<Allocator> allocator;
|
19
21
|
idx_t position;
|
20
22
|
idx_t capacity;
|
21
|
-
bool owns_data;
|
22
23
|
data_ptr_t data;
|
23
24
|
|
24
25
|
public:
|
25
26
|
static constexpr idx_t DEFAULT_INITIAL_CAPACITY = 512;
|
26
27
|
|
28
|
+
// Create a new owning MemoryStream with an internal backing buffer with the specified capacity. The stream will
|
29
|
+
// own the backing buffer, resize it when needed and free its memory when the stream is destroyed
|
30
|
+
explicit MemoryStream(Allocator &allocator, idx_t capacity = DEFAULT_INITIAL_CAPACITY);
|
31
|
+
|
27
32
|
// Create a new owning MemoryStream with an internal backing buffer with the specified capacity. The stream will
|
28
33
|
// own the backing buffer, resize it when needed and free its memory when the stream is destroyed
|
29
34
|
explicit MemoryStream(idx_t capacity = DEFAULT_INITIAL_CAPACITY);
|
@@ -30,6 +30,7 @@ struct SerializationData {
|
|
30
30
|
|
31
31
|
stack<reference<ClientContext>> contexts;
|
32
32
|
stack<reference<DatabaseInstance>> databases;
|
33
|
+
stack<reference<Catalog>> catalogs;
|
33
34
|
stack<idx_t> enums;
|
34
35
|
stack<reference<bound_parameter_map_t>> parameter_data;
|
35
36
|
stack<const_reference<LogicalType>> types;
|
@@ -42,6 +43,9 @@ struct SerializationData {
|
|
42
43
|
template <class T>
|
43
44
|
T Get() = delete;
|
44
45
|
|
46
|
+
template <class T>
|
47
|
+
optional_ptr<T> TryGet() = delete;
|
48
|
+
|
45
49
|
template <class T>
|
46
50
|
void Unset() = delete;
|
47
51
|
|
@@ -163,6 +167,27 @@ inline void SerializationData::Unset<ClientContext>() {
|
|
163
167
|
contexts.pop();
|
164
168
|
}
|
165
169
|
|
170
|
+
template <>
|
171
|
+
inline void SerializationData::Set(Catalog &catalog) {
|
172
|
+
catalogs.emplace(catalog);
|
173
|
+
}
|
174
|
+
|
175
|
+
template <>
|
176
|
+
inline Catalog &SerializationData::Get() {
|
177
|
+
AssertNotEmpty(catalogs);
|
178
|
+
return catalogs.top();
|
179
|
+
}
|
180
|
+
|
181
|
+
template <>
|
182
|
+
inline optional_ptr<Catalog> SerializationData::TryGet() {
|
183
|
+
return catalogs.empty() ? nullptr : &catalogs.top().get();
|
184
|
+
}
|
185
|
+
|
186
|
+
template <>
|
187
|
+
inline void SerializationData::Unset<Catalog>() {
|
188
|
+
AssertNotEmpty(catalogs);
|
189
|
+
catalogs.pop();
|
190
|
+
}
|
166
191
|
template <>
|
167
192
|
inline void SerializationData::Set(DatabaseInstance &db) {
|
168
193
|
databases.emplace(db);
|
@@ -28,6 +28,9 @@ namespace duckdb {
|
|
28
28
|
|
29
29
|
class SerializationOptions {
|
30
30
|
public:
|
31
|
+
SerializationOptions() = default;
|
32
|
+
explicit SerializationOptions(AttachedDatabase &db);
|
33
|
+
|
31
34
|
bool serialize_enum_as_string = false;
|
32
35
|
bool serialize_default_values = false;
|
33
36
|
SerializationCompatibility serialization_compatibility = SerializationCompatibility::Default();
|
@@ -68,6 +71,9 @@ public:
|
|
68
71
|
};
|
69
72
|
|
70
73
|
public:
|
74
|
+
SerializationOptions GetOptions() {
|
75
|
+
return options;
|
76
|
+
}
|
71
77
|
SerializationData &GetSerializationData() {
|
72
78
|
return data;
|
73
79
|
}
|
@@ -99,7 +105,7 @@ public:
|
|
99
105
|
}
|
100
106
|
|
101
107
|
template <class T>
|
102
|
-
void WritePropertyWithDefault(const field_id_t field_id, const char *tag, const T &value, const T
|
108
|
+
void WritePropertyWithDefault(const field_id_t field_id, const char *tag, const T &value, const T &default_value) {
|
103
109
|
// If current value is default, don't write it
|
104
110
|
if (!options.serialize_default_values && (value == default_value)) {
|
105
111
|
OnOptionalPropertyBegin(field_id, tag, false);
|
@@ -114,7 +120,7 @@ public:
|
|
114
120
|
// Specialization for Value (default Value comparison throws when comparing nulls)
|
115
121
|
template <class T>
|
116
122
|
void WritePropertyWithDefault(const field_id_t field_id, const char *tag, const CSVOption<T> &value,
|
117
|
-
const T
|
123
|
+
const T &default_value) {
|
118
124
|
// If current value is default, don't write it
|
119
125
|
if (!options.serialize_default_values && (value == default_value)) {
|
120
126
|
OnOptionalPropertyBegin(field_id, tag, false);
|
@@ -377,7 +383,7 @@ void Serializer::WriteValue(const vector<bool> &vec);
|
|
377
383
|
// Specialization for Value (default Value comparison throws when comparing nulls)
|
378
384
|
template <>
|
379
385
|
void Serializer::WritePropertyWithDefault<Value>(const field_id_t field_id, const char *tag, const Value &value,
|
380
|
-
const Value
|
386
|
+
const Value &default_value);
|
381
387
|
|
382
388
|
// List Impl
|
383
389
|
template <class FUNC>
|
@@ -32,7 +32,7 @@ struct SelectionVector {
|
|
32
32
|
Initialize(count);
|
33
33
|
}
|
34
34
|
SelectionVector(idx_t start, idx_t count) {
|
35
|
-
Initialize(STANDARD_VECTOR_SIZE);
|
35
|
+
Initialize(MaxValue<idx_t>(count, STANDARD_VECTOR_SIZE));
|
36
36
|
for (idx_t i = 0; i < count; i++) {
|
37
37
|
set_index(i, start + i);
|
38
38
|
}
|
@@ -16,7 +16,6 @@ namespace duckdb {
|
|
16
16
|
|
17
17
|
enum class VerifyExistenceType : uint8_t { APPEND = 0, APPEND_FK = 1, DELETE_FK = 2 };
|
18
18
|
enum class ARTConflictType : uint8_t { NO_CONFLICT = 0, CONSTRAINT = 1, TRANSACTION = 2 };
|
19
|
-
enum class ARTAppendMode : uint8_t { DEFAULT = 0, IGNORE_DUPLICATES = 1, INSERT_DUPLICATES = 2 };
|
20
19
|
|
21
20
|
class ConflictManager;
|
22
21
|
class ARTKey;
|
@@ -62,8 +61,6 @@ public:
|
|
62
61
|
bool owns_data;
|
63
62
|
//! The number of bytes fitting in the prefix.
|
64
63
|
uint8_t prefix_count;
|
65
|
-
//! The append mode.
|
66
|
-
ARTAppendMode append_mode;
|
67
64
|
|
68
65
|
public:
|
69
66
|
//! Try to initialize a scan on the ART with the given expression and filter.
|
@@ -74,21 +71,19 @@ public:
|
|
74
71
|
|
75
72
|
//! Appends data to the locked index.
|
76
73
|
ErrorData Append(IndexLock &l, DataChunk &chunk, Vector &row_ids) override;
|
77
|
-
//! Appends data to the locked index and verifies constraint violations
|
78
|
-
ErrorData
|
79
|
-
optional_ptr<BoundIndex> delete_index) override;
|
74
|
+
//! Appends data to the locked index and verifies constraint violations.
|
75
|
+
ErrorData Append(IndexLock &l, DataChunk &chunk, Vector &row_ids, IndexAppendInfo &info) override;
|
80
76
|
|
81
77
|
//! Internally inserts a chunk.
|
82
78
|
ARTConflictType Insert(Node &node, const ARTKey &key, idx_t depth, const ARTKey &row_id, const GateStatus status,
|
83
|
-
optional_ptr<ART> delete_art);
|
79
|
+
optional_ptr<ART> delete_art, const IndexAppendMode append_mode);
|
84
80
|
//! Insert a chunk.
|
85
81
|
ErrorData Insert(IndexLock &l, DataChunk &chunk, Vector &row_ids) override;
|
86
|
-
//! Insert a chunk and verifies constraint violations
|
87
|
-
ErrorData Insert(IndexLock &l, DataChunk &data, Vector &row_ids,
|
82
|
+
//! Insert a chunk and verifies constraint violations.
|
83
|
+
ErrorData Insert(IndexLock &l, DataChunk &data, Vector &row_ids, IndexAppendInfo &info) override;
|
88
84
|
|
89
85
|
//! Verify that data can be appended to the index without a constraint violation.
|
90
|
-
void VerifyAppend(DataChunk &chunk, optional_ptr<
|
91
|
-
optional_ptr<ConflictManager> manager) override;
|
86
|
+
void VerifyAppend(DataChunk &chunk, IndexAppendInfo &info, optional_ptr<ConflictManager> manager) override;
|
92
87
|
|
93
88
|
//! Delete a chunk from the ART.
|
94
89
|
void Delete(IndexLock &lock, DataChunk &entries, Vector &row_ids) override;
|
@@ -131,15 +126,17 @@ private:
|
|
131
126
|
void InsertIntoEmpty(Node &node, const ARTKey &key, const idx_t depth, const ARTKey &row_id,
|
132
127
|
const GateStatus status);
|
133
128
|
ARTConflictType InsertIntoInlined(Node &node, const ARTKey &key, const idx_t depth, const ARTKey &row_id,
|
134
|
-
const GateStatus status, optional_ptr<ART> delete_art
|
129
|
+
const GateStatus status, optional_ptr<ART> delete_art,
|
130
|
+
const IndexAppendMode append_mode);
|
135
131
|
ARTConflictType InsertIntoNode(Node &node, const ARTKey &key, const idx_t depth, const ARTKey &row_id,
|
136
|
-
const GateStatus status, optional_ptr<ART> delete_art
|
132
|
+
const GateStatus status, optional_ptr<ART> delete_art,
|
133
|
+
const IndexAppendMode append_mode);
|
137
134
|
|
138
135
|
string GenerateErrorKeyName(DataChunk &input, idx_t row);
|
139
136
|
string GenerateConstraintErrorMessage(VerifyExistenceType verify_type, const string &key_name);
|
140
137
|
void VerifyLeaf(const Node &leaf, const ARTKey &key, optional_ptr<ART> delete_art, ConflictManager &manager,
|
141
138
|
optional_idx &conflict_idx, idx_t i);
|
142
|
-
void VerifyConstraint(DataChunk &chunk,
|
139
|
+
void VerifyConstraint(DataChunk &chunk, IndexAppendInfo &info, ConflictManager &manager) override;
|
143
140
|
string GetConstraintViolationMessage(VerifyExistenceType verify_type, idx_t failed_index,
|
144
141
|
DataChunk &input) override;
|
145
142
|
|
@@ -41,7 +41,7 @@ public:
|
|
41
41
|
return art.prefix_count;
|
42
42
|
}
|
43
43
|
static idx_t GetMismatchWithOther(const Prefix &l_prefix, const Prefix &r_prefix, const idx_t max_count);
|
44
|
-
static
|
44
|
+
static optional_idx GetMismatchWithKey(ART &art, const Node &node, const ARTKey &key, idx_t &depth);
|
45
45
|
static uint8_t GetByte(const ART &art, const Node &node, const uint8_t pos);
|
46
46
|
|
47
47
|
public:
|
@@ -64,8 +64,8 @@ public:
|
|
64
64
|
//! 1. a non-prefix node.
|
65
65
|
//! 2. a mismatching byte.
|
66
66
|
//! Early-out, if the next prefix is a gate node.
|
67
|
-
static
|
68
|
-
static
|
67
|
+
static optional_idx Traverse(ART &art, reference<const Node> &node, const ARTKey &key, idx_t &depth);
|
68
|
+
static optional_idx TraverseMutable(ART &art, reference<Node> &node, const ARTKey &key, idx_t &depth);
|
69
69
|
|
70
70
|
//! Traverse two prefixes to find
|
71
71
|
//! 1. that they match.
|
@@ -85,7 +85,8 @@ public:
|
|
85
85
|
|
86
86
|
//! Insert a key into a prefix.
|
87
87
|
static ARTConflictType Insert(ART &art, Node &node, const ARTKey &key, idx_t depth, const ARTKey &row_id,
|
88
|
-
const GateStatus status, optional_ptr<ART> delete_art
|
88
|
+
const GateStatus status, optional_ptr<ART> delete_art,
|
89
|
+
const IndexAppendMode append_mode);
|
89
90
|
|
90
91
|
//! Returns the string representation of the node, or only traverses and verifies the node and its subtree
|
91
92
|
static string VerifyAndToString(ART &art, const Node &node, const bool only_verify);
|
@@ -28,6 +28,19 @@ class ConflictManager;
|
|
28
28
|
struct IndexLock;
|
29
29
|
struct IndexScanState;
|
30
30
|
|
31
|
+
enum class IndexAppendMode : uint8_t { DEFAULT = 0, IGNORE_DUPLICATES = 1, INSERT_DUPLICATES = 2 };
|
32
|
+
|
33
|
+
class IndexAppendInfo {
|
34
|
+
public:
|
35
|
+
IndexAppendInfo() : append_mode(IndexAppendMode::DEFAULT), delete_index(nullptr) {};
|
36
|
+
IndexAppendInfo(const IndexAppendMode append_mode, const optional_ptr<BoundIndex> delete_index)
|
37
|
+
: append_mode(append_mode), delete_index(delete_index) {};
|
38
|
+
|
39
|
+
public:
|
40
|
+
IndexAppendMode append_mode;
|
41
|
+
optional_ptr<BoundIndex> delete_index;
|
42
|
+
};
|
43
|
+
|
31
44
|
//! The index is an abstract base class that serves as the basis for indexes
|
32
45
|
class BoundIndex : public Index {
|
33
46
|
public:
|
@@ -71,17 +84,15 @@ public:
|
|
71
84
|
virtual ErrorData Append(IndexLock &l, DataChunk &chunk, Vector &row_ids) = 0;
|
72
85
|
//! Obtains a lock and calls Append while holding that lock.
|
73
86
|
ErrorData Append(DataChunk &chunk, Vector &row_ids);
|
74
|
-
//! Appends data to the locked index and verifies constraint violations
|
75
|
-
virtual ErrorData
|
76
|
-
|
77
|
-
|
78
|
-
ErrorData AppendWithDeleteIndex(DataChunk &chunk, Vector &row_ids, optional_ptr<BoundIndex> delete_index);
|
87
|
+
//! Appends data to the locked index and verifies constraint violations.
|
88
|
+
virtual ErrorData Append(IndexLock &l, DataChunk &chunk, Vector &row_ids, IndexAppendInfo &info);
|
89
|
+
//! Obtains a lock and calls Append while holding that lock.
|
90
|
+
ErrorData Append(DataChunk &chunk, Vector &row_ids, IndexAppendInfo &info);
|
79
91
|
|
80
92
|
//! Verify that data can be appended to the index without a constraint violation.
|
81
|
-
virtual void VerifyAppend(DataChunk &chunk, optional_ptr<
|
82
|
-
optional_ptr<ConflictManager> manager);
|
93
|
+
virtual void VerifyAppend(DataChunk &chunk, IndexAppendInfo &info, optional_ptr<ConflictManager> manager);
|
83
94
|
//! Verifies the constraint for a chunk of data.
|
84
|
-
virtual void VerifyConstraint(DataChunk &chunk,
|
95
|
+
virtual void VerifyConstraint(DataChunk &chunk, IndexAppendInfo &info, ConflictManager &manager);
|
85
96
|
|
86
97
|
//! Deletes all data from the index. The lock obtained from InitializeLock must be held
|
87
98
|
virtual void CommitDrop(IndexLock &index_lock) = 0;
|
@@ -94,8 +105,8 @@ public:
|
|
94
105
|
|
95
106
|
//! Insert a chunk.
|
96
107
|
virtual ErrorData Insert(IndexLock &l, DataChunk &chunk, Vector &row_ids) = 0;
|
97
|
-
//! Insert a chunk and verifies constraint violations
|
98
|
-
virtual ErrorData Insert(IndexLock &l, DataChunk &chunk, Vector &row_ids,
|
108
|
+
//! Insert a chunk and verifies constraint violations.
|
109
|
+
virtual ErrorData Insert(IndexLock &l, DataChunk &chunk, Vector &row_ids, IndexAppendInfo &info);
|
99
110
|
|
100
111
|
//! Merge another index into this index. The lock obtained from InitializeLock must be held, and the other
|
101
112
|
//! index must also be locked during the merge
|
@@ -54,8 +54,9 @@ public:
|
|
54
54
|
D_ASSERT(ptr.GetOffset() < available_segments_per_buffer);
|
55
55
|
D_ASSERT(buffers.find(ptr.GetBufferId()) != buffers.end());
|
56
56
|
|
57
|
-
auto
|
58
|
-
|
57
|
+
auto buffer_it = buffers.find(ptr.GetBufferId());
|
58
|
+
D_ASSERT(buffer_it != buffers.end());
|
59
|
+
auto buffer_ptr = buffer_it->second->Get(dirty);
|
59
60
|
return buffer_ptr + ptr.GetOffset() * segment_size + bitmask_offset;
|
60
61
|
}
|
61
62
|
|
@@ -71,11 +72,11 @@ public:
|
|
71
72
|
D_ASSERT(buffers.find(ptr.GetBufferId()) != buffers.end());
|
72
73
|
|
73
74
|
auto &buffer = buffers.find(ptr.GetBufferId())->second;
|
74
|
-
if (!buffer
|
75
|
+
if (!buffer->InMemory()) {
|
75
76
|
return nullptr;
|
76
77
|
}
|
77
78
|
|
78
|
-
auto buffer_ptr = buffer
|
79
|
+
auto buffer_ptr = buffer->Get();
|
79
80
|
auto raw_ptr = buffer_ptr + ptr.GetOffset() * segment_size + bitmask_offset;
|
80
81
|
return raw_ptr;
|
81
82
|
}
|
@@ -152,7 +153,7 @@ private:
|
|
152
153
|
idx_t total_segment_count;
|
153
154
|
|
154
155
|
//! Buffers containing the segments
|
155
|
-
unordered_map<idx_t, FixedSizeBuffer
|
156
|
+
unordered_map<idx_t, unique_ptr<FixedSizeBuffer>> buffers;
|
156
157
|
//! Buffers with free space
|
157
158
|
unordered_set<idx_t> buffers_with_free_space;
|
158
159
|
//! Buffers qualifying for a vacuum (helper field to allow for fast NeedsVacuum checks)
|