duckdb 1.1.4-dev9.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/binding.gyp +1 -0
- package/package.json +2 -2
- package/src/duckdb/extension/core_functions/function_list.cpp +1 -0
- package/src/duckdb/extension/core_functions/include/core_functions/scalar/map_functions.hpp +9 -0
- package/src/duckdb/extension/core_functions/scalar/date/current.cpp +1 -0
- package/src/duckdb/extension/core_functions/scalar/generic/can_implicitly_cast.cpp +2 -2
- package/src/duckdb/extension/core_functions/scalar/generic/typeof.cpp +1 -1
- package/src/duckdb/extension/core_functions/scalar/list/flatten.cpp +91 -61
- package/src/duckdb/extension/core_functions/scalar/map/map_extract.cpp +89 -8
- package/src/duckdb/extension/icu/icu-current.cpp +63 -0
- package/src/duckdb/extension/icu/icu-makedate.cpp +43 -39
- package/src/duckdb/extension/icu/icu-timezone.cpp +63 -63
- package/src/duckdb/extension/icu/icu_extension.cpp +2 -0
- package/src/duckdb/extension/icu/include/icu-casts.hpp +39 -0
- package/src/duckdb/extension/icu/include/icu-current.hpp +17 -0
- package/src/duckdb/extension/icu/third_party/icu/stubdata/stubdata.cpp +1 -1
- package/src/duckdb/extension/json/json_functions/json_structure.cpp +3 -1
- package/src/duckdb/extension/parquet/column_writer.cpp +26 -18
- package/src/duckdb/extension/parquet/include/parquet_reader.hpp +0 -6
- package/src/duckdb/extension/parquet/include/parquet_writer.hpp +15 -1
- package/src/duckdb/extension/parquet/include/resizable_buffer.hpp +1 -0
- package/src/duckdb/extension/parquet/parquet_extension.cpp +67 -15
- package/src/duckdb/extension/parquet/parquet_reader.cpp +5 -3
- package/src/duckdb/extension/parquet/parquet_writer.cpp +5 -6
- package/src/duckdb/src/catalog/catalog.cpp +21 -8
- package/src/duckdb/src/catalog/catalog_search_path.cpp +17 -1
- package/src/duckdb/src/catalog/catalog_set.cpp +1 -1
- package/src/duckdb/src/catalog/default/default_functions.cpp +0 -3
- package/src/duckdb/src/catalog/dependency_list.cpp +7 -0
- package/src/duckdb/src/common/adbc/adbc.cpp +1 -56
- package/src/duckdb/src/common/arrow/arrow_converter.cpp +3 -2
- package/src/duckdb/src/common/arrow/arrow_type_extension.cpp +58 -28
- package/src/duckdb/src/common/arrow/schema_metadata.cpp +1 -1
- package/src/duckdb/src/common/compressed_file_system.cpp +6 -2
- package/src/duckdb/src/common/enum_util.cpp +26 -22
- package/src/duckdb/src/common/error_data.cpp +3 -2
- package/src/duckdb/src/common/gzip_file_system.cpp +8 -8
- package/src/duckdb/src/common/local_file_system.cpp +2 -2
- package/src/duckdb/src/common/multi_file_reader.cpp +1 -1
- package/src/duckdb/src/common/random_engine.cpp +4 -1
- package/src/duckdb/src/common/serializer/memory_stream.cpp +23 -19
- package/src/duckdb/src/common/serializer/serializer.cpp +1 -1
- package/src/duckdb/src/common/types/bit.cpp +1 -1
- package/src/duckdb/src/common/types/column/column_data_allocator.cpp +0 -5
- package/src/duckdb/src/common/types/column/column_data_collection.cpp +4 -1
- package/src/duckdb/src/common/types/data_chunk.cpp +2 -1
- package/src/duckdb/src/common/types/row/tuple_data_segment.cpp +0 -4
- package/src/duckdb/src/common/types.cpp +1 -1
- package/src/duckdb/src/execution/index/art/art.cpp +52 -42
- package/src/duckdb/src/execution/index/art/leaf.cpp +4 -9
- package/src/duckdb/src/execution/index/art/node.cpp +13 -13
- package/src/duckdb/src/execution/index/art/prefix.cpp +21 -16
- package/src/duckdb/src/execution/index/bound_index.cpp +6 -8
- package/src/duckdb/src/execution/index/fixed_size_allocator.cpp +39 -34
- package/src/duckdb/src/execution/index/fixed_size_buffer.cpp +2 -1
- package/src/duckdb/src/execution/index/unbound_index.cpp +10 -0
- package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +62 -44
- package/src/duckdb/src/execution/operator/csv_scanner/scanner/column_count_scanner.cpp +26 -0
- package/src/duckdb/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +69 -40
- package/src/duckdb/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp +3 -7
- package/src/duckdb/src/execution/operator/csv_scanner/sniffer/header_detection.cpp +11 -5
- package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_detection.cpp +4 -0
- package/src/duckdb/src/execution/operator/csv_scanner/state_machine/csv_state_machine_cache.cpp +8 -8
- package/src/duckdb/src/execution/operator/csv_scanner/util/csv_error.cpp +36 -12
- package/src/duckdb/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +12 -9
- package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +0 -1
- package/src/duckdb/src/execution/operator/persistent/physical_copy_database.cpp +29 -1
- package/src/duckdb/src/execution/operator/persistent/physical_delete.cpp +58 -10
- package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +58 -35
- package/src/duckdb/src/execution/operator/schema/physical_create_art_index.cpp +2 -1
- package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +9 -4
- package/src/duckdb/src/execution/sample/reservoir_sample.cpp +7 -6
- package/src/duckdb/src/function/compression_config.cpp +4 -0
- package/src/duckdb/src/function/function_binder.cpp +1 -1
- package/src/duckdb/src/function/scalar/system/write_log.cpp +2 -2
- package/src/duckdb/src/function/table/arrow/arrow_duck_schema.cpp +15 -2
- package/src/duckdb/src/function/table/arrow_conversion.cpp +10 -10
- package/src/duckdb/src/function/table/copy_csv.cpp +8 -5
- package/src/duckdb/src/function/table/read_csv.cpp +21 -4
- package/src/duckdb/src/function/table/sniff_csv.cpp +7 -0
- package/src/duckdb/src/function/table/system/duckdb_extensions.cpp +4 -0
- package/src/duckdb/src/function/table/system/duckdb_secret_types.cpp +71 -0
- package/src/duckdb/src/function/table/system_functions.cpp +1 -0
- package/src/duckdb/src/function/table/table_scan.cpp +120 -36
- package/src/duckdb/src/function/table/version/pragma_version.cpp +4 -4
- package/src/duckdb/src/function/window/window_aggregate_function.cpp +6 -1
- package/src/duckdb/src/function/window/window_boundaries_state.cpp +135 -11
- package/src/duckdb/src/function/window/window_segment_tree.cpp +50 -22
- package/src/duckdb/src/function/window/window_token_tree.cpp +4 -3
- package/src/duckdb/src/include/duckdb/catalog/catalog.hpp +4 -0
- package/src/duckdb/src/include/duckdb/catalog/catalog_search_path.hpp +2 -0
- package/src/duckdb/src/include/duckdb/catalog/dependency_list.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/arrow/arrow_type_extension.hpp +4 -2
- package/src/duckdb/src/include/duckdb/common/enum_util.hpp +8 -8
- package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +0 -2
- package/src/duckdb/src/include/duckdb/common/serializer/deserializer.hpp +8 -3
- package/src/duckdb/src/include/duckdb/common/serializer/memory_stream.hpp +6 -1
- package/src/duckdb/src/include/duckdb/common/serializer/serialization_data.hpp +25 -0
- package/src/duckdb/src/include/duckdb/common/serializer/serializer.hpp +9 -3
- package/src/duckdb/src/include/duckdb/common/types/selection_vector.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +11 -14
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +5 -4
- package/src/duckdb/src/include/duckdb/execution/index/bound_index.hpp +21 -10
- package/src/duckdb/src/include/duckdb/execution/index/fixed_size_allocator.hpp +6 -5
- package/src/duckdb/src/include/duckdb/execution/index/fixed_size_buffer.hpp +37 -32
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/base_scanner.hpp +36 -1
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/column_count_scanner.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/sniffer/csv_sniffer.hpp +2 -0
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/state_machine_options.hpp +5 -5
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +5 -30
- package/src/duckdb/src/include/duckdb/execution/reservoir_sample.hpp +7 -1
- package/src/duckdb/src/include/duckdb/function/scalar_function.hpp +3 -3
- package/src/duckdb/src/include/duckdb/function/table/arrow/arrow_duck_schema.hpp +1 -0
- package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +4 -0
- package/src/duckdb/src/include/duckdb/function/window/window_boundaries_state.hpp +2 -2
- package/src/duckdb/src/include/duckdb/logging/logger.hpp +40 -119
- package/src/duckdb/src/include/duckdb/logging/logging.hpp +0 -2
- package/src/duckdb/src/include/duckdb/main/config.hpp +5 -0
- package/src/duckdb/src/include/duckdb/main/connection.hpp +0 -8
- package/src/duckdb/src/include/duckdb/main/connection_manager.hpp +2 -1
- package/src/duckdb/src/include/duckdb/main/extension.hpp +1 -0
- package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +11 -7
- package/src/duckdb/src/include/duckdb/main/extension_helper.hpp +1 -0
- package/src/duckdb/src/include/duckdb/main/secret/secret.hpp +2 -0
- package/src/duckdb/src/include/duckdb/main/secret/secret_manager.hpp +3 -0
- package/src/duckdb/src/include/duckdb/main/settings.hpp +10 -0
- package/src/duckdb/src/include/duckdb/parser/constraint.hpp +9 -0
- package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +36 -9
- package/src/duckdb/src/include/duckdb/parser/parsed_data/create_view_info.hpp +2 -1
- package/src/duckdb/src/include/duckdb/parser/query_node/set_operation_node.hpp +8 -2
- package/src/duckdb/src/include/duckdb/planner/binder.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/expression/bound_parameter_data.hpp +9 -1
- package/src/duckdb/src/include/duckdb/planner/filter/constant_filter.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/filter/in_filter.hpp +0 -2
- package/src/duckdb/src/include/duckdb/planner/filter/optional_filter.hpp +4 -4
- package/src/duckdb/src/include/duckdb/planner/table_filter.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/data_table.hpp +14 -10
- package/src/duckdb/src/include/duckdb/storage/index_storage_info.hpp +4 -0
- package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +6 -1
- package/src/duckdb/src/include/duckdb/storage/storage_info.hpp +7 -2
- package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp +9 -0
- package/src/duckdb/src/include/duckdb/storage/storage_options.hpp +2 -0
- package/src/duckdb/src/include/duckdb/storage/string_uncompressed.hpp +4 -3
- package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp +2 -0
- package/src/duckdb/src/include/duckdb/storage/table/table_index_list.hpp +6 -4
- package/src/duckdb/src/include/duckdb/storage/table/table_statistics.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp +2 -0
- package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -0
- package/src/duckdb/src/include/duckdb/transaction/meta_transaction.hpp +1 -1
- package/src/duckdb/src/logging/logger.cpp +8 -66
- package/src/duckdb/src/main/attached_database.cpp +3 -1
- package/src/duckdb/src/main/client_context.cpp +4 -2
- package/src/duckdb/src/main/config.cpp +20 -2
- package/src/duckdb/src/main/connection.cpp +2 -29
- package/src/duckdb/src/main/connection_manager.cpp +5 -3
- package/src/duckdb/src/main/database.cpp +2 -2
- package/src/duckdb/src/main/extension/extension_helper.cpp +4 -5
- package/src/duckdb/src/main/extension/extension_install.cpp +23 -10
- package/src/duckdb/src/main/extension/extension_load.cpp +6 -7
- package/src/duckdb/src/main/extension.cpp +27 -9
- package/src/duckdb/src/main/secret/secret_manager.cpp +11 -0
- package/src/duckdb/src/main/settings/custom_settings.cpp +44 -0
- package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +6 -0
- package/src/duckdb/src/optimizer/filter_combiner.cpp +13 -3
- package/src/duckdb/src/optimizer/filter_pushdown.cpp +33 -6
- package/src/duckdb/src/optimizer/late_materialization.cpp +14 -3
- package/src/duckdb/src/optimizer/remove_unused_columns.cpp +0 -3
- package/src/duckdb/src/parser/parsed_data/attach_info.cpp +5 -1
- package/src/duckdb/src/parser/parsed_data/create_view_info.cpp +6 -3
- package/src/duckdb/src/parser/query_node/set_operation_node.cpp +49 -0
- package/src/duckdb/src/parser/transform/expression/transform_columnref.cpp +1 -0
- package/src/duckdb/src/parser/transform/expression/transform_function.cpp +50 -12
- package/src/duckdb/src/planner/binder/expression/bind_columnref_expression.cpp +7 -5
- package/src/duckdb/src/planner/binder/expression/bind_comparison_expression.cpp +1 -0
- package/src/duckdb/src/planner/binder/expression/bind_operator_expression.cpp +2 -2
- package/src/duckdb/src/planner/binder/expression/bind_star_expression.cpp +12 -2
- package/src/duckdb/src/planner/binder/statement/bind_copy_database.cpp +0 -1
- package/src/duckdb/src/planner/binder/statement/bind_create.cpp +55 -39
- package/src/duckdb/src/planner/binder/statement/bind_execute.cpp +2 -1
- package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +15 -7
- package/src/duckdb/src/planner/binder/tableref/bind_showref.cpp +13 -8
- package/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp +8 -3
- package/src/duckdb/src/planner/expression/bound_function_expression.cpp +17 -1
- package/src/duckdb/src/planner/expression_binder/index_binder.cpp +1 -0
- package/src/duckdb/src/planner/filter/conjunction_filter.cpp +1 -0
- package/src/duckdb/src/planner/filter/constant_filter.cpp +21 -0
- package/src/duckdb/src/planner/filter/in_filter.cpp +4 -7
- package/src/duckdb/src/planner/logical_operator.cpp +5 -3
- package/src/duckdb/src/planner/planner.cpp +1 -1
- package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +2 -0
- package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +3 -4
- package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -5
- package/src/duckdb/src/storage/compression/dictionary/decompression.cpp +4 -4
- package/src/duckdb/src/storage/compression/fsst.cpp +2 -2
- package/src/duckdb/src/storage/compression/roaring/common.cpp +10 -1
- package/src/duckdb/src/storage/compression/string_uncompressed.cpp +11 -6
- package/src/duckdb/src/storage/compression/validity_uncompressed.cpp +4 -0
- package/src/duckdb/src/storage/compression/zstd.cpp +6 -0
- package/src/duckdb/src/storage/data_table.cpp +104 -109
- package/src/duckdb/src/storage/local_storage.cpp +8 -6
- package/src/duckdb/src/storage/magic_bytes.cpp +1 -1
- package/src/duckdb/src/storage/serialization/serialize_dependency.cpp +3 -3
- package/src/duckdb/src/storage/serialization/serialize_nodes.cpp +3 -3
- package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +7 -5
- package/src/duckdb/src/storage/single_file_block_manager.cpp +95 -28
- package/src/duckdb/src/storage/storage_info.cpp +38 -0
- package/src/duckdb/src/storage/storage_manager.cpp +11 -0
- package/src/duckdb/src/storage/table/column_data.cpp +4 -0
- package/src/duckdb/src/storage/table/column_data_checkpointer.cpp +3 -3
- package/src/duckdb/src/storage/table/row_group_collection.cpp +67 -68
- package/src/duckdb/src/storage/table/table_statistics.cpp +4 -4
- package/src/duckdb/src/storage/table_index_list.cpp +41 -15
- package/src/duckdb/src/storage/wal_replay.cpp +3 -1
- package/src/duckdb/src/storage/write_ahead_log.cpp +11 -4
- package/src/duckdb/src/transaction/meta_transaction.cpp +1 -1
- package/src/duckdb/src/verification/deserialized_statement_verifier.cpp +2 -1
- package/src/duckdb/third_party/httplib/httplib.hpp +0 -1
- package/src/duckdb/third_party/re2/util/logging.h +10 -10
- package/src/duckdb/ub_src_function_table_system.cpp +2 -0
@@ -49,14 +49,13 @@ IndexPointer FixedSizeAllocator::New() {
|
|
49
49
|
|
50
50
|
// add a new buffer
|
51
51
|
auto buffer_id = GetAvailableBufferId();
|
52
|
-
FixedSizeBuffer
|
53
|
-
buffers.insert(make_pair(buffer_id, std::move(new_buffer)));
|
52
|
+
buffers[buffer_id] = make_uniq<FixedSizeBuffer>(block_manager);
|
54
53
|
buffers_with_free_space.insert(buffer_id);
|
55
54
|
|
56
55
|
// set the bitmask
|
57
56
|
D_ASSERT(buffers.find(buffer_id) != buffers.end());
|
58
57
|
auto &buffer = buffers.find(buffer_id)->second;
|
59
|
-
ValidityMask mask(reinterpret_cast<validity_t *>(buffer
|
58
|
+
ValidityMask mask(reinterpret_cast<validity_t *>(buffer->Get()), available_segments_per_buffer);
|
60
59
|
|
61
60
|
// zero-initialize the bitmask to avoid leaking memory to disk
|
62
61
|
auto data = mask.GetData();
|
@@ -74,16 +73,16 @@ IndexPointer FixedSizeAllocator::New() {
|
|
74
73
|
|
75
74
|
D_ASSERT(buffers.find(buffer_id) != buffers.end());
|
76
75
|
auto &buffer = buffers.find(buffer_id)->second;
|
77
|
-
auto offset = buffer
|
76
|
+
auto offset = buffer->GetOffset(bitmask_count, available_segments_per_buffer);
|
78
77
|
|
79
78
|
total_segment_count++;
|
80
|
-
buffer
|
81
|
-
if (buffer
|
79
|
+
buffer->segment_count++;
|
80
|
+
if (buffer->segment_count == available_segments_per_buffer) {
|
82
81
|
buffers_with_free_space.erase(buffer_id);
|
83
82
|
}
|
84
83
|
|
85
84
|
// zero-initialize that segment
|
86
|
-
auto buffer_ptr = buffer
|
85
|
+
auto buffer_ptr = buffer->Get();
|
87
86
|
auto offset_in_buffer = buffer_ptr + offset * segment_size + bitmask_offset;
|
88
87
|
memset(offset_in_buffer, 0, segment_size);
|
89
88
|
|
@@ -98,24 +97,21 @@ void FixedSizeAllocator::Free(const IndexPointer ptr) {
|
|
98
97
|
D_ASSERT(buffers.find(buffer_id) != buffers.end());
|
99
98
|
auto &buffer = buffers.find(buffer_id)->second;
|
100
99
|
|
101
|
-
auto bitmask_ptr = reinterpret_cast<validity_t *>(buffer
|
100
|
+
auto bitmask_ptr = reinterpret_cast<validity_t *>(buffer->Get());
|
102
101
|
ValidityMask mask(bitmask_ptr, offset + 1); // FIXME
|
103
102
|
D_ASSERT(!mask.RowIsValid(offset));
|
104
103
|
mask.SetValid(offset);
|
105
104
|
|
106
105
|
D_ASSERT(total_segment_count > 0);
|
107
|
-
D_ASSERT(buffer
|
106
|
+
D_ASSERT(buffer->segment_count > 0);
|
108
107
|
|
109
108
|
// adjust the allocator fields
|
110
109
|
buffers_with_free_space.insert(buffer_id);
|
111
110
|
total_segment_count--;
|
112
|
-
buffer
|
111
|
+
buffer->segment_count--;
|
113
112
|
}
|
114
113
|
|
115
114
|
void FixedSizeAllocator::Reset() {
|
116
|
-
for (auto &buffer : buffers) {
|
117
|
-
buffer.second.Destroy();
|
118
|
-
}
|
119
115
|
buffers.clear();
|
120
116
|
buffers_with_free_space.clear();
|
121
117
|
total_segment_count = 0;
|
@@ -124,7 +120,7 @@ void FixedSizeAllocator::Reset() {
|
|
124
120
|
idx_t FixedSizeAllocator::GetInMemorySize() const {
|
125
121
|
idx_t memory_usage = 0;
|
126
122
|
for (auto &buffer : buffers) {
|
127
|
-
if (buffer.second
|
123
|
+
if (buffer.second->InMemory()) {
|
128
124
|
memory_usage += block_manager.GetBlockSize();
|
129
125
|
}
|
130
126
|
}
|
@@ -179,9 +175,9 @@ bool FixedSizeAllocator::InitializeVacuum() {
|
|
179
175
|
idx_t available_segments_in_memory = 0;
|
180
176
|
|
181
177
|
for (auto &buffer : buffers) {
|
182
|
-
buffer.second
|
183
|
-
if (buffer.second
|
184
|
-
auto available_segments_in_buffer = available_segments_per_buffer - buffer.second
|
178
|
+
buffer.second->vacuum = false;
|
179
|
+
if (buffer.second->InMemory()) {
|
180
|
+
auto available_segments_in_buffer = available_segments_per_buffer - buffer.second->segment_count;
|
185
181
|
available_segments_in_memory += available_segments_in_buffer;
|
186
182
|
temporary_vacuum_buffers.emplace(available_segments_in_buffer, buffer.first);
|
187
183
|
}
|
@@ -216,7 +212,7 @@ bool FixedSizeAllocator::InitializeVacuum() {
|
|
216
212
|
for (auto &vacuum_buffer : temporary_vacuum_buffers) {
|
217
213
|
auto buffer_id = vacuum_buffer.second;
|
218
214
|
D_ASSERT(buffers.find(buffer_id) != buffers.end());
|
219
|
-
buffers.find(buffer_id)->second
|
215
|
+
buffers.find(buffer_id)->second->vacuum = true;
|
220
216
|
buffers_with_free_space.erase(buffer_id);
|
221
217
|
}
|
222
218
|
|
@@ -231,9 +227,7 @@ void FixedSizeAllocator::FinalizeVacuum() {
|
|
231
227
|
|
232
228
|
for (auto &buffer_id : vacuum_buffers) {
|
233
229
|
D_ASSERT(buffers.find(buffer_id) != buffers.end());
|
234
|
-
|
235
|
-
D_ASSERT(buffer.InMemory());
|
236
|
-
buffer.Destroy();
|
230
|
+
D_ASSERT(buffers.find(buffer_id)->second->InMemory());
|
237
231
|
buffers.erase(buffer_id);
|
238
232
|
}
|
239
233
|
vacuum_buffers.clear();
|
@@ -259,9 +253,15 @@ FixedSizeAllocatorInfo FixedSizeAllocator::GetInfo() const {
|
|
259
253
|
|
260
254
|
for (const auto &buffer : buffers) {
|
261
255
|
info.buffer_ids.push_back(buffer.first);
|
262
|
-
|
263
|
-
|
264
|
-
|
256
|
+
|
257
|
+
// Memory safety check.
|
258
|
+
if (buffer.first > idx_t(MAX_ROW_ID)) {
|
259
|
+
throw InternalException("Initializing invalid buffer ID in FixedSizeAllocator::GetInfo");
|
260
|
+
}
|
261
|
+
|
262
|
+
info.block_pointers.push_back(buffer.second->block_pointer);
|
263
|
+
info.segment_counts.push_back(buffer.second->segment_count);
|
264
|
+
info.allocation_sizes.push_back(buffer.second->allocation_size);
|
265
265
|
}
|
266
266
|
|
267
267
|
for (auto &buffer_id : buffers_with_free_space) {
|
@@ -273,7 +273,7 @@ FixedSizeAllocatorInfo FixedSizeAllocator::GetInfo() const {
|
|
273
273
|
|
274
274
|
void FixedSizeAllocator::SerializeBuffers(PartialBlockManager &partial_block_manager) {
|
275
275
|
for (auto &buffer : buffers) {
|
276
|
-
buffer.second
|
276
|
+
buffer.second->Serialize(partial_block_manager, available_segments_per_buffer, segment_size, bitmask_offset);
|
277
277
|
}
|
278
278
|
}
|
279
279
|
|
@@ -281,8 +281,8 @@ vector<IndexBufferInfo> FixedSizeAllocator::InitSerializationToWAL() {
|
|
281
281
|
|
282
282
|
vector<IndexBufferInfo> buffer_infos;
|
283
283
|
for (auto &buffer : buffers) {
|
284
|
-
buffer.second
|
285
|
-
buffer_infos.emplace_back(buffer.second
|
284
|
+
buffer.second->SetAllocationSize(available_segments_per_buffer, segment_size, bitmask_offset);
|
285
|
+
buffer_infos.emplace_back(buffer.second->Get(), buffer.second->allocation_size);
|
286
286
|
}
|
287
287
|
return buffer_infos;
|
288
288
|
}
|
@@ -295,13 +295,19 @@ void FixedSizeAllocator::Init(const FixedSizeAllocatorInfo &info) {
|
|
295
295
|
|
296
296
|
// read all FixedSizeBuffer data
|
297
297
|
auto buffer_id = info.buffer_ids[i];
|
298
|
+
|
299
|
+
// Memory safety check.
|
300
|
+
if (buffer_id > idx_t(MAX_ROW_ID)) {
|
301
|
+
throw InternalException("Initializing invalid buffer ID in FixedSizeAllocator::Init");
|
302
|
+
}
|
303
|
+
|
298
304
|
auto buffer_block_pointer = info.block_pointers[i];
|
299
305
|
auto segment_count = info.segment_counts[i];
|
300
306
|
auto allocation_size = info.allocation_sizes[i];
|
301
307
|
|
302
308
|
// create the FixedSizeBuffer
|
303
|
-
|
304
|
-
|
309
|
+
buffers[buffer_id] =
|
310
|
+
make_uniq<FixedSizeBuffer>(block_manager, segment_count, allocation_size, buffer_block_pointer);
|
305
311
|
total_segment_count += segment_count;
|
306
312
|
}
|
307
313
|
|
@@ -324,8 +330,8 @@ void FixedSizeAllocator::Deserialize(MetadataManager &metadata_manager, const Bl
|
|
324
330
|
auto buffer_block_pointer = reader.Read<BlockPointer>();
|
325
331
|
auto segment_count = reader.Read<idx_t>();
|
326
332
|
auto allocation_size = reader.Read<idx_t>();
|
327
|
-
|
328
|
-
|
333
|
+
buffers[buffer_id] =
|
334
|
+
make_uniq<FixedSizeBuffer>(block_manager, segment_count, allocation_size, buffer_block_pointer);
|
329
335
|
total_segment_count += segment_count;
|
330
336
|
}
|
331
337
|
for (idx_t i = 0; i < buffers_with_free_space_count; i++) {
|
@@ -346,13 +352,12 @@ void FixedSizeAllocator::RemoveEmptyBuffers() {
|
|
346
352
|
|
347
353
|
auto buffer_it = buffers.begin();
|
348
354
|
while (buffer_it != buffers.end()) {
|
349
|
-
if (buffer_it->second
|
350
|
-
buffer_it
|
355
|
+
if (buffer_it->second->segment_count != 0) {
|
356
|
+
++buffer_it;
|
351
357
|
continue;
|
352
358
|
}
|
353
359
|
|
354
360
|
buffers_with_free_space.erase(buffer_it->first);
|
355
|
-
buffer_it->second.Destroy();
|
356
361
|
buffer_it = buffers.erase(buffer_it);
|
357
362
|
}
|
358
363
|
}
|
@@ -54,7 +54,8 @@ FixedSizeBuffer::FixedSizeBuffer(BlockManager &block_manager, const idx_t segmen
|
|
54
54
|
D_ASSERT(block_handle->BlockId() < MAXIMUM_BLOCK);
|
55
55
|
}
|
56
56
|
|
57
|
-
|
57
|
+
FixedSizeBuffer::~FixedSizeBuffer() {
|
58
|
+
lock_guard<mutex> l(lock);
|
58
59
|
if (InMemory()) {
|
59
60
|
// we can have multiple readers on a pinned block, and unpinning the buffer handle
|
60
61
|
// decrements the reader count on the underlying block handle (Destroy() unpins)
|
@@ -14,6 +14,16 @@ UnboundIndex::UnboundIndex(unique_ptr<CreateInfo> create_info, IndexStorageInfo
|
|
14
14
|
TableIOManager &table_io_manager, AttachedDatabase &db)
|
15
15
|
: Index(create_info->Cast<CreateIndexInfo>().column_ids, table_io_manager, db), create_info(std::move(create_info)),
|
16
16
|
storage_info(std::move(storage_info_p)) {
|
17
|
+
|
18
|
+
// Memory safety check.
|
19
|
+
for (idx_t info_idx = 0; info_idx < storage_info.allocator_infos.size(); info_idx++) {
|
20
|
+
auto &info = storage_info.allocator_infos[info_idx];
|
21
|
+
for (idx_t buffer_idx = 0; buffer_idx < info.buffer_ids.size(); buffer_idx++) {
|
22
|
+
if (info.buffer_ids[buffer_idx] > idx_t(MAX_ROW_ID)) {
|
23
|
+
throw InternalException("Found invalid buffer ID in UnboundIndex constructor");
|
24
|
+
}
|
25
|
+
}
|
26
|
+
}
|
17
27
|
}
|
18
28
|
|
19
29
|
void UnboundIndex::CommitDrop() {
|
@@ -153,8 +153,6 @@ public:
|
|
153
153
|
ComputeOffset(context, wexpr, offset);
|
154
154
|
ComputeDefault(context, wexpr, dflt);
|
155
155
|
|
156
|
-
curr_chunk.Initialize(context, {wexpr.return_type});
|
157
|
-
|
158
156
|
buffered = idx_t(std::abs(offset));
|
159
157
|
prev.Reference(dflt);
|
160
158
|
prev.Flatten(buffered);
|
@@ -162,6 +160,10 @@ public:
|
|
162
160
|
}
|
163
161
|
|
164
162
|
void Execute(ExecutionContext &context, DataChunk &input, DataChunk &delayed, Vector &result) {
|
163
|
+
if (!curr_chunk.ColumnCount()) {
|
164
|
+
curr_chunk.Initialize(context.client, {result.GetType()}, delayed.GetCapacity());
|
165
|
+
}
|
166
|
+
|
165
167
|
if (offset >= 0) {
|
166
168
|
ExecuteLag(context, input, result);
|
167
169
|
} else {
|
@@ -212,7 +214,7 @@ public:
|
|
212
214
|
idx_t pos = 0;
|
213
215
|
idx_t unified_offset = buffered;
|
214
216
|
if (unified_offset < count) {
|
215
|
-
|
217
|
+
Reset(curr_chunk);
|
216
218
|
executor.Execute(input, curr_chunk);
|
217
219
|
VectorOperations::Copy(curr, result, count, unified_offset, pos);
|
218
220
|
pos += count - unified_offset;
|
@@ -221,7 +223,7 @@ public:
|
|
221
223
|
// Copy unified[unified_offset:] => result[pos:]
|
222
224
|
idx_t unified_count = count + delayed.size();
|
223
225
|
if (unified_offset < unified_count) {
|
224
|
-
|
226
|
+
Reset(curr_chunk);
|
225
227
|
executor.Execute(delayed, curr_chunk);
|
226
228
|
idx_t delayed_offset = unified_offset - count;
|
227
229
|
// Only copy as many values as we need
|
@@ -312,6 +314,13 @@ public:
|
|
312
314
|
initialized = true;
|
313
315
|
}
|
314
316
|
|
317
|
+
static inline void Reset(DataChunk &chunk) {
|
318
|
+
// Reset trashes the capacity...
|
319
|
+
const auto capacity = chunk.GetCapacity();
|
320
|
+
chunk.Reset();
|
321
|
+
chunk.SetCapacity(capacity);
|
322
|
+
}
|
323
|
+
|
315
324
|
public:
|
316
325
|
//! We can't initialise until we have an input chunk
|
317
326
|
bool initialized;
|
@@ -470,34 +479,34 @@ void StreamingWindowState::AggregateState::Execute(ExecutionContext &context, Da
|
|
470
479
|
}
|
471
480
|
}
|
472
481
|
|
473
|
-
void PhysicalStreamingWindow::ExecuteFunctions(ExecutionContext &context, DataChunk &
|
482
|
+
void PhysicalStreamingWindow::ExecuteFunctions(ExecutionContext &context, DataChunk &output, DataChunk &delayed,
|
474
483
|
GlobalOperatorState &gstate_p, OperatorState &state_p) const {
|
475
484
|
auto &gstate = gstate_p.Cast<StreamingWindowGlobalState>();
|
476
485
|
auto &state = state_p.Cast<StreamingWindowState>();
|
477
486
|
|
478
487
|
// Compute window functions
|
479
|
-
const idx_t count =
|
488
|
+
const idx_t count = output.size();
|
480
489
|
const column_t input_width = children[0]->GetTypes().size();
|
481
490
|
for (column_t expr_idx = 0; expr_idx < select_list.size(); expr_idx++) {
|
482
491
|
column_t col_idx = input_width + expr_idx;
|
483
492
|
auto &expr = *select_list[expr_idx];
|
484
|
-
auto &result =
|
493
|
+
auto &result = output.data[col_idx];
|
485
494
|
switch (expr.GetExpressionType()) {
|
486
495
|
case ExpressionType::WINDOW_AGGREGATE:
|
487
|
-
state.aggregate_states[expr_idx]->Execute(context,
|
496
|
+
state.aggregate_states[expr_idx]->Execute(context, output, result);
|
488
497
|
break;
|
489
498
|
case ExpressionType::WINDOW_FIRST_VALUE:
|
490
499
|
case ExpressionType::WINDOW_PERCENT_RANK:
|
491
500
|
case ExpressionType::WINDOW_RANK:
|
492
501
|
case ExpressionType::WINDOW_RANK_DENSE: {
|
493
502
|
// Reference constant vector
|
494
|
-
|
503
|
+
output.data[col_idx].Reference(*state.const_vectors[expr_idx]);
|
495
504
|
break;
|
496
505
|
}
|
497
506
|
case ExpressionType::WINDOW_ROW_NUMBER: {
|
498
507
|
// Set row numbers
|
499
508
|
int64_t start_row = gstate.row_number;
|
500
|
-
auto rdata = FlatVector::GetData<int64_t>(
|
509
|
+
auto rdata = FlatVector::GetData<int64_t>(output.data[col_idx]);
|
501
510
|
for (idx_t i = 0; i < count; i++) {
|
502
511
|
rdata[i] = NumericCast<int64_t>(start_row + NumericCast<int64_t>(i));
|
503
512
|
}
|
@@ -505,7 +514,7 @@ void PhysicalStreamingWindow::ExecuteFunctions(ExecutionContext &context, DataCh
|
|
505
514
|
}
|
506
515
|
case ExpressionType::WINDOW_LAG:
|
507
516
|
case ExpressionType::WINDOW_LEAD:
|
508
|
-
state.lead_lag_states[expr_idx]->Execute(context,
|
517
|
+
state.lead_lag_states[expr_idx]->Execute(context, output, delayed, result);
|
509
518
|
break;
|
510
519
|
default:
|
511
520
|
throw NotImplementedException("%s for StreamingWindow", ExpressionTypeToString(expr.GetExpressionType()));
|
@@ -515,13 +524,13 @@ void PhysicalStreamingWindow::ExecuteFunctions(ExecutionContext &context, DataCh
|
|
515
524
|
}
|
516
525
|
|
517
526
|
void PhysicalStreamingWindow::ExecuteInput(ExecutionContext &context, DataChunk &delayed, DataChunk &input,
|
518
|
-
DataChunk &
|
527
|
+
DataChunk &output, GlobalOperatorState &gstate_p,
|
519
528
|
OperatorState &state_p) const {
|
520
529
|
auto &state = state_p.Cast<StreamingWindowState>();
|
521
530
|
|
522
531
|
// Put payload columns in place
|
523
532
|
for (idx_t col_idx = 0; col_idx < input.data.size(); col_idx++) {
|
524
|
-
|
533
|
+
output.data[col_idx].Reference(input.data[col_idx]);
|
525
534
|
}
|
526
535
|
idx_t count = input.size();
|
527
536
|
|
@@ -531,51 +540,53 @@ void PhysicalStreamingWindow::ExecuteInput(ExecutionContext &context, DataChunk
|
|
531
540
|
count -= state.lead_count;
|
532
541
|
input.Copy(delayed, count);
|
533
542
|
}
|
534
|
-
|
543
|
+
output.SetCardinality(count);
|
535
544
|
|
536
|
-
ExecuteFunctions(context,
|
545
|
+
ExecuteFunctions(context, output, state.delayed, gstate_p, state_p);
|
537
546
|
}
|
538
547
|
|
539
548
|
void PhysicalStreamingWindow::ExecuteShifted(ExecutionContext &context, DataChunk &delayed, DataChunk &input,
|
540
|
-
DataChunk &
|
549
|
+
DataChunk &output, GlobalOperatorState &gstate_p,
|
541
550
|
OperatorState &state_p) const {
|
542
551
|
auto &state = state_p.Cast<StreamingWindowState>();
|
543
552
|
auto &shifted = state.shifted;
|
544
553
|
|
545
|
-
idx_t
|
546
|
-
idx_t
|
547
|
-
|
554
|
+
idx_t out = output.size();
|
555
|
+
idx_t in = input.size();
|
556
|
+
idx_t delay = delayed.size();
|
557
|
+
D_ASSERT(out <= delay);
|
558
|
+
|
559
|
+
state.Reset(shifted);
|
548
560
|
// shifted = delayed
|
549
561
|
delayed.Copy(shifted);
|
550
|
-
|
562
|
+
state.Reset(delayed);
|
551
563
|
for (idx_t col_idx = 0; col_idx < delayed.data.size(); ++col_idx) {
|
552
|
-
//
|
553
|
-
|
554
|
-
// delayed[0:
|
555
|
-
VectorOperations::Copy(shifted.data[col_idx], delayed.data[col_idx],
|
556
|
-
// delayed[
|
557
|
-
VectorOperations::Copy(input.data[col_idx], delayed.data[col_idx],
|
564
|
+
// output[0:out] = delayed[0:out]
|
565
|
+
output.data[col_idx].Reference(shifted.data[col_idx]);
|
566
|
+
// delayed[0:out] = delayed[out:delay-out]
|
567
|
+
VectorOperations::Copy(shifted.data[col_idx], delayed.data[col_idx], delay, out, 0);
|
568
|
+
// delayed[delay-out:delay-out+in] = input[0:in]
|
569
|
+
VectorOperations::Copy(input.data[col_idx], delayed.data[col_idx], in, 0, delay - out);
|
558
570
|
}
|
559
|
-
|
560
|
-
delayed.SetCardinality(d);
|
571
|
+
delayed.SetCardinality(delay - out + in);
|
561
572
|
|
562
|
-
ExecuteFunctions(context,
|
573
|
+
ExecuteFunctions(context, output, delayed, gstate_p, state_p);
|
563
574
|
}
|
564
575
|
|
565
576
|
void PhysicalStreamingWindow::ExecuteDelayed(ExecutionContext &context, DataChunk &delayed, DataChunk &input,
|
566
|
-
DataChunk &
|
577
|
+
DataChunk &output, GlobalOperatorState &gstate_p,
|
567
578
|
OperatorState &state_p) const {
|
568
579
|
// Put payload columns in place
|
569
580
|
for (idx_t col_idx = 0; col_idx < delayed.data.size(); col_idx++) {
|
570
|
-
|
581
|
+
output.data[col_idx].Reference(delayed.data[col_idx]);
|
571
582
|
}
|
572
583
|
idx_t count = delayed.size();
|
573
|
-
|
584
|
+
output.SetCardinality(count);
|
574
585
|
|
575
|
-
ExecuteFunctions(context,
|
586
|
+
ExecuteFunctions(context, output, input, gstate_p, state_p);
|
576
587
|
}
|
577
588
|
|
578
|
-
OperatorResultType PhysicalStreamingWindow::Execute(ExecutionContext &context, DataChunk &input, DataChunk &
|
589
|
+
OperatorResultType PhysicalStreamingWindow::Execute(ExecutionContext &context, DataChunk &input, DataChunk &output,
|
579
590
|
GlobalOperatorState &gstate_p, OperatorState &state_p) const {
|
580
591
|
auto &state = state_p.Cast<StreamingWindowState>();
|
581
592
|
if (!state.initialized) {
|
@@ -585,37 +596,37 @@ OperatorResultType PhysicalStreamingWindow::Execute(ExecutionContext &context, D
|
|
585
596
|
auto &delayed = state.delayed;
|
586
597
|
// We can Reset delayed now that no one can be referencing it.
|
587
598
|
if (!delayed.size()) {
|
588
|
-
|
599
|
+
state.Reset(delayed);
|
589
600
|
}
|
590
|
-
|
591
|
-
if (available <= state.lead_count) {
|
601
|
+
if (delayed.size() < state.lead_count) {
|
592
602
|
// If we don't have enough to produce a single row,
|
593
603
|
// then just delay more rows, return nothing
|
594
604
|
// and ask for more data.
|
595
605
|
delayed.Append(input);
|
596
|
-
|
606
|
+
output.SetCardinality(0);
|
597
607
|
return OperatorResultType::NEED_MORE_INPUT;
|
598
608
|
} else if (input.size() < delayed.size()) {
|
599
609
|
// If we can't consume all of the delayed values,
|
600
610
|
// we need to split them instead of referencing them all
|
601
|
-
|
611
|
+
output.SetCardinality(input.size());
|
612
|
+
ExecuteShifted(context, delayed, input, output, gstate_p, state_p);
|
602
613
|
// We delayed the unused input so ask for more
|
603
614
|
return OperatorResultType::NEED_MORE_INPUT;
|
604
615
|
} else if (delayed.size()) {
|
605
616
|
// We have enough delayed rows so flush them
|
606
|
-
ExecuteDelayed(context, delayed, input,
|
617
|
+
ExecuteDelayed(context, delayed, input, output, gstate_p, state_p);
|
607
618
|
// Defer resetting delayed as it may be referenced.
|
608
619
|
delayed.SetCardinality(0);
|
609
620
|
// Come back to process the input
|
610
621
|
return OperatorResultType::HAVE_MORE_OUTPUT;
|
611
622
|
} else {
|
612
623
|
// No delayed rows, so emit what we can and delay the rest.
|
613
|
-
ExecuteInput(context, delayed, input,
|
624
|
+
ExecuteInput(context, delayed, input, output, gstate_p, state_p);
|
614
625
|
return OperatorResultType::NEED_MORE_INPUT;
|
615
626
|
}
|
616
627
|
}
|
617
628
|
|
618
|
-
OperatorFinalizeResultType PhysicalStreamingWindow::FinalExecute(ExecutionContext &context, DataChunk &
|
629
|
+
OperatorFinalizeResultType PhysicalStreamingWindow::FinalExecute(ExecutionContext &context, DataChunk &output,
|
619
630
|
GlobalOperatorState &gstate_p,
|
620
631
|
OperatorState &state_p) const {
|
621
632
|
auto &state = state_p.Cast<StreamingWindowState>();
|
@@ -624,8 +635,15 @@ OperatorFinalizeResultType PhysicalStreamingWindow::FinalExecute(ExecutionContex
|
|
624
635
|
auto &delayed = state.delayed;
|
625
636
|
// There are no more input rows
|
626
637
|
auto &input = state.shifted;
|
627
|
-
|
628
|
-
|
638
|
+
state.Reset(input);
|
639
|
+
|
640
|
+
if (output.GetCapacity() < delayed.size()) {
|
641
|
+
// More than one output buffer was delayed, so shift in what we can
|
642
|
+
output.SetCardinality(output.GetCapacity());
|
643
|
+
ExecuteShifted(context, delayed, input, output, gstate_p, state_p);
|
644
|
+
return OperatorFinalizeResultType::HAVE_MORE_OUTPUT;
|
645
|
+
}
|
646
|
+
ExecuteDelayed(context, delayed, input, output, gstate_p, state_p);
|
629
647
|
}
|
630
648
|
|
631
649
|
return OperatorFinalizeResultType::FINISHED;
|
@@ -39,7 +39,13 @@ idx_t ColumnCountResult::GetMostFrequentColumnCount() const {
|
|
39
39
|
}
|
40
40
|
|
41
41
|
bool ColumnCountResult::AddRow(ColumnCountResult &result, idx_t buffer_pos) {
|
42
|
+
const LinePosition cur_position(result.cur_buffer_idx, buffer_pos + 1, result.current_buffer_size);
|
43
|
+
if (cur_position - result.last_position > result.state_machine.options.maximum_line_size.GetValue() &&
|
44
|
+
buffer_pos != NumericLimits<idx_t>::Maximum()) {
|
45
|
+
result.error = true;
|
46
|
+
}
|
42
47
|
result.InternalAddRow();
|
48
|
+
result.last_position = cur_position;
|
43
49
|
if (!result.states.EmptyLastValue()) {
|
44
50
|
idx_t col_count_idx = result.result_position;
|
45
51
|
for (idx_t i = 0; i < result.result_position + 1; i++) {
|
@@ -99,6 +105,13 @@ ColumnCountScanner::ColumnCountScanner(shared_ptr<CSVBufferManager> buffer_manag
|
|
99
105
|
: BaseScanner(std::move(buffer_manager), state_machine, std::move(error_handler), true, nullptr, iterator),
|
100
106
|
result(states, *state_machine, result_size_p), column_count(1), result_size(result_size_p) {
|
101
107
|
sniffing = true;
|
108
|
+
idx_t actual_size = 0;
|
109
|
+
if (cur_buffer_handle) {
|
110
|
+
actual_size = cur_buffer_handle->actual_size;
|
111
|
+
}
|
112
|
+
result.last_position = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, actual_size};
|
113
|
+
result.current_buffer_size = actual_size;
|
114
|
+
result.cur_buffer_idx = iterator.pos.buffer_idx;
|
102
115
|
}
|
103
116
|
|
104
117
|
unique_ptr<StringValueScanner> ColumnCountScanner::UpgradeToStringValueScanner() {
|
@@ -117,6 +130,9 @@ unique_ptr<StringValueScanner> ColumnCountScanner::UpgradeToStringValueScanner()
|
|
117
130
|
ColumnCountResult &ColumnCountScanner::ParseChunk() {
|
118
131
|
result.result_position = 0;
|
119
132
|
column_count = 1;
|
133
|
+
if (cur_buffer_handle) {
|
134
|
+
result.current_buffer_size = cur_buffer_handle->actual_size;
|
135
|
+
}
|
120
136
|
ParseChunkInternal(result);
|
121
137
|
return result;
|
122
138
|
}
|
@@ -139,6 +155,7 @@ void ColumnCountScanner::FinalizeChunkProcess() {
|
|
139
155
|
if (iterator.pos.buffer_pos == cur_buffer_handle->actual_size) {
|
140
156
|
// Move to next buffer
|
141
157
|
cur_buffer_handle = buffer_manager->GetBuffer(++iterator.pos.buffer_idx);
|
158
|
+
|
142
159
|
if (!cur_buffer_handle) {
|
143
160
|
buffer_handle_ptr = nullptr;
|
144
161
|
if (states.IsQuotedCurrent() && !states.IsUnquoted()) {
|
@@ -158,6 +175,15 @@ void ColumnCountScanner::FinalizeChunkProcess() {
|
|
158
175
|
result.AddRow(result, NumericLimits<idx_t>::Maximum());
|
159
176
|
}
|
160
177
|
return;
|
178
|
+
} else {
|
179
|
+
result.cur_buffer_idx = iterator.pos.buffer_idx;
|
180
|
+
result.current_buffer_size = cur_buffer_handle->actual_size;
|
181
|
+
// Do a quick check that the line is still sane
|
182
|
+
const LinePosition cur_position(result.cur_buffer_idx, 0, result.current_buffer_size);
|
183
|
+
if (cur_position - result.last_position > result.state_machine.options.maximum_line_size.GetValue()) {
|
184
|
+
result.error = true;
|
185
|
+
return;
|
186
|
+
}
|
161
187
|
}
|
162
188
|
iterator.pos.buffer_pos = 0;
|
163
189
|
buffer_handle_ptr = cur_buffer_handle->Ptr();
|