duckdb 1.1.4-dev9.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/binding.gyp +1 -0
- package/package.json +2 -2
- package/src/duckdb/extension/core_functions/function_list.cpp +1 -0
- package/src/duckdb/extension/core_functions/include/core_functions/scalar/map_functions.hpp +9 -0
- package/src/duckdb/extension/core_functions/scalar/date/current.cpp +1 -0
- package/src/duckdb/extension/core_functions/scalar/generic/can_implicitly_cast.cpp +2 -2
- package/src/duckdb/extension/core_functions/scalar/generic/typeof.cpp +1 -1
- package/src/duckdb/extension/core_functions/scalar/list/flatten.cpp +91 -61
- package/src/duckdb/extension/core_functions/scalar/map/map_extract.cpp +89 -8
- package/src/duckdb/extension/icu/icu-current.cpp +63 -0
- package/src/duckdb/extension/icu/icu-makedate.cpp +43 -39
- package/src/duckdb/extension/icu/icu-timezone.cpp +63 -63
- package/src/duckdb/extension/icu/icu_extension.cpp +2 -0
- package/src/duckdb/extension/icu/include/icu-casts.hpp +39 -0
- package/src/duckdb/extension/icu/include/icu-current.hpp +17 -0
- package/src/duckdb/extension/icu/third_party/icu/stubdata/stubdata.cpp +1 -1
- package/src/duckdb/extension/json/json_functions/json_structure.cpp +3 -1
- package/src/duckdb/extension/parquet/column_writer.cpp +26 -18
- package/src/duckdb/extension/parquet/include/parquet_reader.hpp +0 -6
- package/src/duckdb/extension/parquet/include/parquet_writer.hpp +15 -1
- package/src/duckdb/extension/parquet/include/resizable_buffer.hpp +1 -0
- package/src/duckdb/extension/parquet/parquet_extension.cpp +67 -15
- package/src/duckdb/extension/parquet/parquet_reader.cpp +5 -3
- package/src/duckdb/extension/parquet/parquet_writer.cpp +5 -6
- package/src/duckdb/src/catalog/catalog.cpp +21 -8
- package/src/duckdb/src/catalog/catalog_search_path.cpp +17 -1
- package/src/duckdb/src/catalog/catalog_set.cpp +1 -1
- package/src/duckdb/src/catalog/default/default_functions.cpp +0 -3
- package/src/duckdb/src/catalog/dependency_list.cpp +7 -0
- package/src/duckdb/src/common/adbc/adbc.cpp +1 -56
- package/src/duckdb/src/common/arrow/arrow_converter.cpp +3 -2
- package/src/duckdb/src/common/arrow/arrow_type_extension.cpp +58 -28
- package/src/duckdb/src/common/arrow/schema_metadata.cpp +1 -1
- package/src/duckdb/src/common/compressed_file_system.cpp +6 -2
- package/src/duckdb/src/common/enum_util.cpp +26 -22
- package/src/duckdb/src/common/error_data.cpp +3 -2
- package/src/duckdb/src/common/gzip_file_system.cpp +8 -8
- package/src/duckdb/src/common/local_file_system.cpp +2 -2
- package/src/duckdb/src/common/multi_file_reader.cpp +1 -1
- package/src/duckdb/src/common/random_engine.cpp +4 -1
- package/src/duckdb/src/common/serializer/memory_stream.cpp +23 -19
- package/src/duckdb/src/common/serializer/serializer.cpp +1 -1
- package/src/duckdb/src/common/types/bit.cpp +1 -1
- package/src/duckdb/src/common/types/column/column_data_allocator.cpp +0 -5
- package/src/duckdb/src/common/types/column/column_data_collection.cpp +4 -1
- package/src/duckdb/src/common/types/data_chunk.cpp +2 -1
- package/src/duckdb/src/common/types/row/tuple_data_segment.cpp +0 -4
- package/src/duckdb/src/common/types.cpp +1 -1
- package/src/duckdb/src/execution/index/art/art.cpp +52 -42
- package/src/duckdb/src/execution/index/art/leaf.cpp +4 -9
- package/src/duckdb/src/execution/index/art/node.cpp +13 -13
- package/src/duckdb/src/execution/index/art/prefix.cpp +21 -16
- package/src/duckdb/src/execution/index/bound_index.cpp +6 -8
- package/src/duckdb/src/execution/index/fixed_size_allocator.cpp +39 -34
- package/src/duckdb/src/execution/index/fixed_size_buffer.cpp +2 -1
- package/src/duckdb/src/execution/index/unbound_index.cpp +10 -0
- package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +62 -44
- package/src/duckdb/src/execution/operator/csv_scanner/scanner/column_count_scanner.cpp +26 -0
- package/src/duckdb/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +69 -40
- package/src/duckdb/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp +3 -7
- package/src/duckdb/src/execution/operator/csv_scanner/sniffer/header_detection.cpp +11 -5
- package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_detection.cpp +4 -0
- package/src/duckdb/src/execution/operator/csv_scanner/state_machine/csv_state_machine_cache.cpp +8 -8
- package/src/duckdb/src/execution/operator/csv_scanner/util/csv_error.cpp +36 -12
- package/src/duckdb/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +12 -9
- package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +0 -1
- package/src/duckdb/src/execution/operator/persistent/physical_copy_database.cpp +29 -1
- package/src/duckdb/src/execution/operator/persistent/physical_delete.cpp +58 -10
- package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +58 -35
- package/src/duckdb/src/execution/operator/schema/physical_create_art_index.cpp +2 -1
- package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +9 -4
- package/src/duckdb/src/execution/sample/reservoir_sample.cpp +7 -6
- package/src/duckdb/src/function/compression_config.cpp +4 -0
- package/src/duckdb/src/function/function_binder.cpp +1 -1
- package/src/duckdb/src/function/scalar/system/write_log.cpp +2 -2
- package/src/duckdb/src/function/table/arrow/arrow_duck_schema.cpp +15 -2
- package/src/duckdb/src/function/table/arrow_conversion.cpp +10 -10
- package/src/duckdb/src/function/table/copy_csv.cpp +8 -5
- package/src/duckdb/src/function/table/read_csv.cpp +21 -4
- package/src/duckdb/src/function/table/sniff_csv.cpp +7 -0
- package/src/duckdb/src/function/table/system/duckdb_extensions.cpp +4 -0
- package/src/duckdb/src/function/table/system/duckdb_secret_types.cpp +71 -0
- package/src/duckdb/src/function/table/system_functions.cpp +1 -0
- package/src/duckdb/src/function/table/table_scan.cpp +120 -36
- package/src/duckdb/src/function/table/version/pragma_version.cpp +4 -4
- package/src/duckdb/src/function/window/window_aggregate_function.cpp +6 -1
- package/src/duckdb/src/function/window/window_boundaries_state.cpp +135 -11
- package/src/duckdb/src/function/window/window_segment_tree.cpp +50 -22
- package/src/duckdb/src/function/window/window_token_tree.cpp +4 -3
- package/src/duckdb/src/include/duckdb/catalog/catalog.hpp +4 -0
- package/src/duckdb/src/include/duckdb/catalog/catalog_search_path.hpp +2 -0
- package/src/duckdb/src/include/duckdb/catalog/dependency_list.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/arrow/arrow_type_extension.hpp +4 -2
- package/src/duckdb/src/include/duckdb/common/enum_util.hpp +8 -8
- package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +0 -2
- package/src/duckdb/src/include/duckdb/common/serializer/deserializer.hpp +8 -3
- package/src/duckdb/src/include/duckdb/common/serializer/memory_stream.hpp +6 -1
- package/src/duckdb/src/include/duckdb/common/serializer/serialization_data.hpp +25 -0
- package/src/duckdb/src/include/duckdb/common/serializer/serializer.hpp +9 -3
- package/src/duckdb/src/include/duckdb/common/types/selection_vector.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +11 -14
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +5 -4
- package/src/duckdb/src/include/duckdb/execution/index/bound_index.hpp +21 -10
- package/src/duckdb/src/include/duckdb/execution/index/fixed_size_allocator.hpp +6 -5
- package/src/duckdb/src/include/duckdb/execution/index/fixed_size_buffer.hpp +37 -32
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/base_scanner.hpp +36 -1
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/column_count_scanner.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/sniffer/csv_sniffer.hpp +2 -0
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/state_machine_options.hpp +5 -5
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +5 -30
- package/src/duckdb/src/include/duckdb/execution/reservoir_sample.hpp +7 -1
- package/src/duckdb/src/include/duckdb/function/scalar_function.hpp +3 -3
- package/src/duckdb/src/include/duckdb/function/table/arrow/arrow_duck_schema.hpp +1 -0
- package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +4 -0
- package/src/duckdb/src/include/duckdb/function/window/window_boundaries_state.hpp +2 -2
- package/src/duckdb/src/include/duckdb/logging/logger.hpp +40 -119
- package/src/duckdb/src/include/duckdb/logging/logging.hpp +0 -2
- package/src/duckdb/src/include/duckdb/main/config.hpp +5 -0
- package/src/duckdb/src/include/duckdb/main/connection.hpp +0 -8
- package/src/duckdb/src/include/duckdb/main/connection_manager.hpp +2 -1
- package/src/duckdb/src/include/duckdb/main/extension.hpp +1 -0
- package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +11 -7
- package/src/duckdb/src/include/duckdb/main/extension_helper.hpp +1 -0
- package/src/duckdb/src/include/duckdb/main/secret/secret.hpp +2 -0
- package/src/duckdb/src/include/duckdb/main/secret/secret_manager.hpp +3 -0
- package/src/duckdb/src/include/duckdb/main/settings.hpp +10 -0
- package/src/duckdb/src/include/duckdb/parser/constraint.hpp +9 -0
- package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +36 -9
- package/src/duckdb/src/include/duckdb/parser/parsed_data/create_view_info.hpp +2 -1
- package/src/duckdb/src/include/duckdb/parser/query_node/set_operation_node.hpp +8 -2
- package/src/duckdb/src/include/duckdb/planner/binder.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/expression/bound_parameter_data.hpp +9 -1
- package/src/duckdb/src/include/duckdb/planner/filter/constant_filter.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/filter/in_filter.hpp +0 -2
- package/src/duckdb/src/include/duckdb/planner/filter/optional_filter.hpp +4 -4
- package/src/duckdb/src/include/duckdb/planner/table_filter.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/data_table.hpp +14 -10
- package/src/duckdb/src/include/duckdb/storage/index_storage_info.hpp +4 -0
- package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +6 -1
- package/src/duckdb/src/include/duckdb/storage/storage_info.hpp +7 -2
- package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp +9 -0
- package/src/duckdb/src/include/duckdb/storage/storage_options.hpp +2 -0
- package/src/duckdb/src/include/duckdb/storage/string_uncompressed.hpp +4 -3
- package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp +2 -0
- package/src/duckdb/src/include/duckdb/storage/table/table_index_list.hpp +6 -4
- package/src/duckdb/src/include/duckdb/storage/table/table_statistics.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp +2 -0
- package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -0
- package/src/duckdb/src/include/duckdb/transaction/meta_transaction.hpp +1 -1
- package/src/duckdb/src/logging/logger.cpp +8 -66
- package/src/duckdb/src/main/attached_database.cpp +3 -1
- package/src/duckdb/src/main/client_context.cpp +4 -2
- package/src/duckdb/src/main/config.cpp +20 -2
- package/src/duckdb/src/main/connection.cpp +2 -29
- package/src/duckdb/src/main/connection_manager.cpp +5 -3
- package/src/duckdb/src/main/database.cpp +2 -2
- package/src/duckdb/src/main/extension/extension_helper.cpp +4 -5
- package/src/duckdb/src/main/extension/extension_install.cpp +23 -10
- package/src/duckdb/src/main/extension/extension_load.cpp +6 -7
- package/src/duckdb/src/main/extension.cpp +27 -9
- package/src/duckdb/src/main/secret/secret_manager.cpp +11 -0
- package/src/duckdb/src/main/settings/custom_settings.cpp +44 -0
- package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +6 -0
- package/src/duckdb/src/optimizer/filter_combiner.cpp +13 -3
- package/src/duckdb/src/optimizer/filter_pushdown.cpp +33 -6
- package/src/duckdb/src/optimizer/late_materialization.cpp +14 -3
- package/src/duckdb/src/optimizer/remove_unused_columns.cpp +0 -3
- package/src/duckdb/src/parser/parsed_data/attach_info.cpp +5 -1
- package/src/duckdb/src/parser/parsed_data/create_view_info.cpp +6 -3
- package/src/duckdb/src/parser/query_node/set_operation_node.cpp +49 -0
- package/src/duckdb/src/parser/transform/expression/transform_columnref.cpp +1 -0
- package/src/duckdb/src/parser/transform/expression/transform_function.cpp +50 -12
- package/src/duckdb/src/planner/binder/expression/bind_columnref_expression.cpp +7 -5
- package/src/duckdb/src/planner/binder/expression/bind_comparison_expression.cpp +1 -0
- package/src/duckdb/src/planner/binder/expression/bind_operator_expression.cpp +2 -2
- package/src/duckdb/src/planner/binder/expression/bind_star_expression.cpp +12 -2
- package/src/duckdb/src/planner/binder/statement/bind_copy_database.cpp +0 -1
- package/src/duckdb/src/planner/binder/statement/bind_create.cpp +55 -39
- package/src/duckdb/src/planner/binder/statement/bind_execute.cpp +2 -1
- package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +15 -7
- package/src/duckdb/src/planner/binder/tableref/bind_showref.cpp +13 -8
- package/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp +8 -3
- package/src/duckdb/src/planner/expression/bound_function_expression.cpp +17 -1
- package/src/duckdb/src/planner/expression_binder/index_binder.cpp +1 -0
- package/src/duckdb/src/planner/filter/conjunction_filter.cpp +1 -0
- package/src/duckdb/src/planner/filter/constant_filter.cpp +21 -0
- package/src/duckdb/src/planner/filter/in_filter.cpp +4 -7
- package/src/duckdb/src/planner/logical_operator.cpp +5 -3
- package/src/duckdb/src/planner/planner.cpp +1 -1
- package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +2 -0
- package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +3 -4
- package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -5
- package/src/duckdb/src/storage/compression/dictionary/decompression.cpp +4 -4
- package/src/duckdb/src/storage/compression/fsst.cpp +2 -2
- package/src/duckdb/src/storage/compression/roaring/common.cpp +10 -1
- package/src/duckdb/src/storage/compression/string_uncompressed.cpp +11 -6
- package/src/duckdb/src/storage/compression/validity_uncompressed.cpp +4 -0
- package/src/duckdb/src/storage/compression/zstd.cpp +6 -0
- package/src/duckdb/src/storage/data_table.cpp +104 -109
- package/src/duckdb/src/storage/local_storage.cpp +8 -6
- package/src/duckdb/src/storage/magic_bytes.cpp +1 -1
- package/src/duckdb/src/storage/serialization/serialize_dependency.cpp +3 -3
- package/src/duckdb/src/storage/serialization/serialize_nodes.cpp +3 -3
- package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +7 -5
- package/src/duckdb/src/storage/single_file_block_manager.cpp +95 -28
- package/src/duckdb/src/storage/storage_info.cpp +38 -0
- package/src/duckdb/src/storage/storage_manager.cpp +11 -0
- package/src/duckdb/src/storage/table/column_data.cpp +4 -0
- package/src/duckdb/src/storage/table/column_data_checkpointer.cpp +3 -3
- package/src/duckdb/src/storage/table/row_group_collection.cpp +67 -68
- package/src/duckdb/src/storage/table/table_statistics.cpp +4 -4
- package/src/duckdb/src/storage/table_index_list.cpp +41 -15
- package/src/duckdb/src/storage/wal_replay.cpp +3 -1
- package/src/duckdb/src/storage/write_ahead_log.cpp +11 -4
- package/src/duckdb/src/transaction/meta_transaction.cpp +1 -1
- package/src/duckdb/src/verification/deserialized_statement_verifier.cpp +2 -1
- package/src/duckdb/third_party/httplib/httplib.hpp +0 -1
- package/src/duckdb/third_party/re2/util/logging.h +10 -10
- package/src/duckdb/ub_src_function_table_system.cpp +2 -0
@@ -34,6 +34,8 @@ public:
|
|
34
34
|
//! yet in memory, and it only serializes dirty and non-written buffers to disk during
|
35
35
|
//! serialization.
|
36
36
|
class FixedSizeBuffer {
|
37
|
+
friend class FixedSizeAllocator;
|
38
|
+
|
37
39
|
public:
|
38
40
|
//! Constants for fast offset calculations in the bitmask
|
39
41
|
static constexpr idx_t BASE[] = {0x00000000FFFFFFFF, 0x0000FFFF, 0x00FF, 0x0F, 0x3, 0x1};
|
@@ -46,33 +48,12 @@ public:
|
|
46
48
|
FixedSizeBuffer(BlockManager &block_manager, const idx_t segment_count, const idx_t allocation_size,
|
47
49
|
const BlockPointer &block_pointer);
|
48
50
|
|
49
|
-
|
50
|
-
BlockManager &block_manager;
|
51
|
-
|
52
|
-
//! The number of allocated segments
|
53
|
-
idx_t segment_count;
|
54
|
-
//! The size of allocated memory in this buffer (necessary for copying while pinning)
|
55
|
-
idx_t allocation_size;
|
56
|
-
|
57
|
-
//! True: the in-memory buffer is no longer consistent with a (possibly existing) copy on disk
|
58
|
-
bool dirty;
|
59
|
-
//! True: can be vacuumed after the vacuum operation
|
60
|
-
bool vacuum;
|
61
|
-
|
62
|
-
//! Partial block id and offset
|
63
|
-
BlockPointer block_pointer;
|
51
|
+
~FixedSizeBuffer();
|
64
52
|
|
65
|
-
|
66
|
-
//! Returns true, if the buffer is in-memory
|
67
|
-
inline bool InMemory() const {
|
68
|
-
return buffer_handle.IsValid();
|
69
|
-
}
|
70
|
-
//! Returns true, if the block is on-disk
|
71
|
-
inline bool OnDisk() const {
|
72
|
-
return block_pointer.IsValid();
|
73
|
-
}
|
53
|
+
private:
|
74
54
|
//! Returns a pointer to the buffer in memory, and calls Deserialize, if the buffer is not in memory
|
75
|
-
|
55
|
+
data_ptr_t Get(const bool dirty_p = true) {
|
56
|
+
lock_guard<mutex> l(lock);
|
76
57
|
if (!InMemory()) {
|
77
58
|
Pin();
|
78
59
|
}
|
@@ -81,8 +62,17 @@ public:
|
|
81
62
|
}
|
82
63
|
return buffer_handle.Ptr();
|
83
64
|
}
|
84
|
-
|
85
|
-
|
65
|
+
|
66
|
+
//! Returns true, if the buffer is in-memory
|
67
|
+
bool InMemory() const {
|
68
|
+
return buffer_handle.IsValid();
|
69
|
+
}
|
70
|
+
|
71
|
+
//! Returns true, if the block is on-disk
|
72
|
+
bool OnDisk() const {
|
73
|
+
return block_pointer.IsValid();
|
74
|
+
}
|
75
|
+
|
86
76
|
//! Serializes a buffer (if dirty or not on disk)
|
87
77
|
void Serialize(PartialBlockManager &partial_block_manager, const idx_t available_segments, const idx_t segment_size,
|
88
78
|
const idx_t bitmask_offset);
|
@@ -92,17 +82,32 @@ public:
|
|
92
82
|
uint32_t GetOffset(const idx_t bitmask_count, const idx_t available_segments);
|
93
83
|
//! Sets the allocation size, if dirty
|
94
84
|
void SetAllocationSize(const idx_t available_segments, const idx_t segment_size, const idx_t bitmask_offset);
|
85
|
+
//! Sets all uninitialized regions of a buffer in the respective partial block allocation
|
86
|
+
void SetUninitializedRegions(PartialBlockForIndex &p_block_for_index, const idx_t segment_size, const idx_t offset,
|
87
|
+
const idx_t bitmask_offset, const idx_t available_segments);
|
95
88
|
|
96
89
|
private:
|
90
|
+
//! Block manager of the database instance
|
91
|
+
BlockManager &block_manager;
|
92
|
+
|
93
|
+
//! The number of allocated segments
|
94
|
+
idx_t segment_count;
|
95
|
+
//! The size of allocated memory in this buffer (necessary for copying while pinning)
|
96
|
+
idx_t allocation_size;
|
97
|
+
|
98
|
+
//! True: the in-memory buffer is no longer consistent with a (possibly existing) copy on disk
|
99
|
+
bool dirty;
|
100
|
+
//! True: can be vacuumed after the vacuum operation
|
101
|
+
bool vacuum;
|
102
|
+
|
103
|
+
//! Partial block id and offset
|
104
|
+
BlockPointer block_pointer;
|
97
105
|
//! The buffer handle of the in-memory buffer
|
98
106
|
BufferHandle buffer_handle;
|
99
107
|
//! The block handle of the on-disk buffer
|
100
108
|
shared_ptr<BlockHandle> block_handle;
|
101
|
-
|
102
|
-
|
103
|
-
//! Sets all uninitialized regions of a buffer in the respective partial block allocation
|
104
|
-
void SetUninitializedRegions(PartialBlockForIndex &p_block_for_index, const idx_t segment_size, const idx_t offset,
|
105
|
-
const idx_t bitmask_offset, const idx_t available_segments);
|
109
|
+
//! The lock for this fixed size buffer handle
|
110
|
+
mutex lock;
|
106
111
|
};
|
107
112
|
|
108
113
|
} // namespace duckdb
|
@@ -17,6 +17,35 @@
|
|
17
17
|
namespace duckdb {
|
18
18
|
|
19
19
|
class CSVFileScan;
|
20
|
+
|
21
|
+
//! Class that keeps track of line starts, used for line size verification
|
22
|
+
class LinePosition {
|
23
|
+
public:
|
24
|
+
LinePosition() {
|
25
|
+
}
|
26
|
+
LinePosition(idx_t buffer_idx_p, idx_t buffer_pos_p, idx_t buffer_size_p)
|
27
|
+
: buffer_pos(buffer_pos_p), buffer_size(buffer_size_p), buffer_idx(buffer_idx_p) {
|
28
|
+
}
|
29
|
+
|
30
|
+
idx_t operator-(const LinePosition &other) const {
|
31
|
+
if (other.buffer_idx == buffer_idx) {
|
32
|
+
return buffer_pos - other.buffer_pos;
|
33
|
+
}
|
34
|
+
return other.buffer_size - other.buffer_pos + buffer_pos;
|
35
|
+
}
|
36
|
+
|
37
|
+
bool operator==(const LinePosition &other) const {
|
38
|
+
return buffer_pos == other.buffer_pos && buffer_idx == other.buffer_idx && buffer_size == other.buffer_size;
|
39
|
+
}
|
40
|
+
|
41
|
+
idx_t GetGlobalPosition(idx_t requested_buffer_size, bool first_char_nl = false) const {
|
42
|
+
return requested_buffer_size * buffer_idx + buffer_pos + first_char_nl;
|
43
|
+
}
|
44
|
+
idx_t buffer_pos = 0;
|
45
|
+
idx_t buffer_size = 0;
|
46
|
+
idx_t buffer_idx = 0;
|
47
|
+
};
|
48
|
+
|
20
49
|
class ScannerResult {
|
21
50
|
public:
|
22
51
|
ScannerResult(CSVStates &states, CSVStateMachine &state_machine, idx_t result_size);
|
@@ -52,6 +81,10 @@ public:
|
|
52
81
|
return result.comment == true;
|
53
82
|
}
|
54
83
|
|
84
|
+
inline bool IsStateCurrent(CSVState state) const {
|
85
|
+
return states.states[1] == state;
|
86
|
+
}
|
87
|
+
|
55
88
|
//! Variable to keep information regarding quoted and escaped values
|
56
89
|
bool quoted = false;
|
57
90
|
//! If the current quoted value is unquoted
|
@@ -62,6 +95,8 @@ public:
|
|
62
95
|
bool comment = false;
|
63
96
|
idx_t quoted_position = 0;
|
64
97
|
|
98
|
+
LinePosition last_position;
|
99
|
+
|
65
100
|
//! Size of the result
|
66
101
|
const idx_t result_size;
|
67
102
|
|
@@ -88,7 +123,7 @@ public:
|
|
88
123
|
//! Returns true if the scanner is finished
|
89
124
|
bool FinishedFile() const;
|
90
125
|
|
91
|
-
//! Parses data into
|
126
|
+
//! Parses data into an output_chunk
|
92
127
|
virtual ScannerResult &ParseChunk();
|
93
128
|
|
94
129
|
//! Returns the result from the last Parse call. Shouts at you if you call it wrong
|
package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/column_count_scanner.hpp
CHANGED
@@ -41,6 +41,9 @@ public:
|
|
41
41
|
bool error = false;
|
42
42
|
idx_t result_position = 0;
|
43
43
|
bool cur_line_starts_as_comment = false;
|
44
|
+
|
45
|
+
idx_t cur_buffer_idx = 0;
|
46
|
+
idx_t current_buffer_size = 0;
|
44
47
|
//! How many rows fit a given column count
|
45
48
|
map<idx_t, idx_t> rows_per_column_count;
|
46
49
|
//! Adds a Value to the result
|
package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/state_machine_options.hpp
CHANGED
@@ -16,9 +16,9 @@ namespace duckdb {
|
|
16
16
|
struct CSVStateMachineOptions {
|
17
17
|
CSVStateMachineOptions() {};
|
18
18
|
CSVStateMachineOptions(string delimiter_p, char quote_p, char escape_p, char comment_p,
|
19
|
-
NewLineIdentifier new_line_p, bool
|
19
|
+
NewLineIdentifier new_line_p, bool strict_mode_p)
|
20
20
|
: delimiter(std::move(delimiter_p)), quote(quote_p), escape(escape_p), comment(comment_p), new_line(new_line_p),
|
21
|
-
|
21
|
+
strict_mode(strict_mode_p) {};
|
22
22
|
|
23
23
|
//! Delimiter to separate columns within each line
|
24
24
|
CSVOption<string> delimiter {","};
|
@@ -30,12 +30,12 @@ struct CSVStateMachineOptions {
|
|
30
30
|
CSVOption<char> comment = '\0';
|
31
31
|
//! New Line separator
|
32
32
|
CSVOption<NewLineIdentifier> new_line = NewLineIdentifier::NOT_SET;
|
33
|
-
//!
|
34
|
-
CSVOption<bool>
|
33
|
+
//! How Strict the parser should be
|
34
|
+
CSVOption<bool> strict_mode = true;
|
35
35
|
|
36
36
|
bool operator==(const CSVStateMachineOptions &other) const {
|
37
37
|
return delimiter == other.delimiter && quote == other.quote && escape == other.escape &&
|
38
|
-
new_line == other.new_line && comment == other.comment &&
|
38
|
+
new_line == other.new_line && comment == other.comment && strict_mode == other.strict_mode;
|
39
39
|
}
|
40
40
|
};
|
41
41
|
} // namespace duckdb
|
package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp
CHANGED
@@ -27,34 +27,6 @@ struct CSVBufferUsage {
|
|
27
27
|
idx_t buffer_idx;
|
28
28
|
};
|
29
29
|
|
30
|
-
//! Class that keeps track of line starts, used for line size verification
|
31
|
-
class LinePosition {
|
32
|
-
public:
|
33
|
-
LinePosition() {
|
34
|
-
}
|
35
|
-
LinePosition(idx_t buffer_idx_p, idx_t buffer_pos_p, idx_t buffer_size_p)
|
36
|
-
: buffer_pos(buffer_pos_p), buffer_size(buffer_size_p), buffer_idx(buffer_idx_p) {
|
37
|
-
}
|
38
|
-
|
39
|
-
idx_t operator-(const LinePosition &other) const {
|
40
|
-
if (other.buffer_idx == buffer_idx) {
|
41
|
-
return buffer_pos - other.buffer_pos;
|
42
|
-
}
|
43
|
-
return other.buffer_size - other.buffer_pos + buffer_pos;
|
44
|
-
}
|
45
|
-
|
46
|
-
bool operator==(const LinePosition &other) const {
|
47
|
-
return buffer_pos == other.buffer_pos && buffer_idx == other.buffer_idx && buffer_size == other.buffer_size;
|
48
|
-
}
|
49
|
-
|
50
|
-
idx_t GetGlobalPosition(idx_t requested_buffer_size, bool first_char_nl = false) const {
|
51
|
-
return requested_buffer_size * buffer_idx + buffer_pos + first_char_nl;
|
52
|
-
}
|
53
|
-
idx_t buffer_pos = 0;
|
54
|
-
idx_t buffer_size = 0;
|
55
|
-
idx_t buffer_idx = 0;
|
56
|
-
};
|
57
|
-
|
58
30
|
//! Keeps track of start and end of line positions in regard to the CSV file
|
59
31
|
class FullLinePosition {
|
60
32
|
public:
|
@@ -181,7 +153,7 @@ public:
|
|
181
153
|
unsafe_vector<ValidityMask *> validity_mask;
|
182
154
|
|
183
155
|
//! Variables to iterate over the CSV buffers
|
184
|
-
|
156
|
+
|
185
157
|
char *buffer_ptr;
|
186
158
|
idx_t buffer_size;
|
187
159
|
idx_t position_before_comment;
|
@@ -245,6 +217,8 @@ public:
|
|
245
217
|
//! (i.e., non-comment) line.
|
246
218
|
bool first_line_is_comment = false;
|
247
219
|
|
220
|
+
bool ignore_empty_values = true;
|
221
|
+
|
248
222
|
//! Specialized code for quoted values, makes sure to remove quotes and escapes
|
249
223
|
static inline void AddQuotedValue(StringValueResult &result, const idx_t buffer_pos);
|
250
224
|
//! Specialized code for possibly escaped values, makes sure to remove escapes
|
@@ -322,7 +296,8 @@ public:
|
|
322
296
|
bool FinishedIterator() const;
|
323
297
|
|
324
298
|
//! Creates a new string with all escaped values removed
|
325
|
-
static string_t RemoveEscape(const char *str_ptr, idx_t end, char escape, char quote,
|
299
|
+
static string_t RemoveEscape(const char *str_ptr, idx_t end, char escape, char quote, bool strict_mode,
|
300
|
+
Vector &vector);
|
326
301
|
|
327
302
|
//! If we can directly cast the type when consuming the CSV file, or we have to do it later
|
328
303
|
static bool CanDirectlyCast(const LogicalType &type, bool icu_loaded);
|
@@ -172,7 +172,13 @@ public:
|
|
172
172
|
static constexpr const SampleType TYPE = SampleType::RESERVOIR_SAMPLE;
|
173
173
|
|
174
174
|
constexpr static idx_t FIXED_SAMPLE_SIZE_MULTIPLIER = 10;
|
175
|
-
|
175
|
+
// size is small enough, then the threshold to switch
|
176
|
+
// MinValue between std vec size and fixed sample size.
|
177
|
+
// During 'fast' sampling, we want every new vector to have the potential
|
178
|
+
// to add to the sample. If the threshold is too far below the standard vector size, then
|
179
|
+
// samples in the sample have a higher weight than new samples coming in.
|
180
|
+
// i.e during vector_size=2, 2 new samples will not be significant compared 2048 samples from 204800 tuples.
|
181
|
+
constexpr static idx_t FAST_TO_SLOW_THRESHOLD = MinValue<idx_t>(STANDARD_VECTOR_SIZE, 60);
|
176
182
|
|
177
183
|
// If the table has less than 204800 rows, this is the percentage
|
178
184
|
// of values we save when serializing/returning a sample.
|
@@ -78,13 +78,13 @@ struct FunctionModifiedDatabasesInput {
|
|
78
78
|
|
79
79
|
struct FunctionBindExpressionInput {
|
80
80
|
FunctionBindExpressionInput(ClientContext &context_p, optional_ptr<FunctionData> bind_data_p,
|
81
|
-
|
82
|
-
: context(context_p), bind_data(bind_data_p),
|
81
|
+
vector<unique_ptr<Expression>> &children_p)
|
82
|
+
: context(context_p), bind_data(bind_data_p), children(children_p) {
|
83
83
|
}
|
84
84
|
|
85
85
|
ClientContext &context;
|
86
86
|
optional_ptr<FunctionData> bind_data;
|
87
|
-
|
87
|
+
vector<unique_ptr<Expression>> &children;
|
88
88
|
};
|
89
89
|
|
90
90
|
struct ScalarFunctionBindInput {
|
@@ -84,6 +84,7 @@ public:
|
|
84
84
|
}
|
85
85
|
void ThrowIfInvalid() const;
|
86
86
|
|
87
|
+
static unique_ptr<ArrowType> GetTypeFromFormat(string &format);
|
87
88
|
static unique_ptr<ArrowType> GetTypeFromFormat(DBConfig &config, ArrowSchema &schema, string &format);
|
88
89
|
|
89
90
|
static unique_ptr<ArrowType> GetTypeFromSchema(DBConfig &config, ArrowSchema &schema);
|
@@ -103,6 +103,10 @@ struct DuckDBOptimizersFun {
|
|
103
103
|
static void RegisterFunction(BuiltinFunctions &set);
|
104
104
|
};
|
105
105
|
|
106
|
+
struct DuckDBSecretTypesFun {
|
107
|
+
static void RegisterFunction(BuiltinFunctions &set);
|
108
|
+
};
|
109
|
+
|
106
110
|
struct DuckDBSequencesFun {
|
107
111
|
static void RegisterFunction(BuiltinFunctions &set);
|
108
112
|
};
|
@@ -113,9 +113,9 @@ struct WindowBoundariesState {
|
|
113
113
|
void ValidEnd(DataChunk &bounds, idx_t row_idx, const idx_t count, bool is_jump, const ValidityMask &partition_mask,
|
114
114
|
const ValidityMask &order_mask, optional_ptr<WindowCursor> range);
|
115
115
|
void FrameBegin(DataChunk &bounds, idx_t row_idx, const idx_t count, WindowInputExpression &boundary_begin,
|
116
|
-
optional_ptr<WindowCursor> range);
|
116
|
+
const ValidityMask &order_mask, optional_ptr<WindowCursor> range);
|
117
117
|
void FrameEnd(DataChunk &bounds, idx_t row_idx, const idx_t count, WindowInputExpression &boundary_end,
|
118
|
-
optional_ptr<WindowCursor> range);
|
118
|
+
const ValidityMask &order_mask, optional_ptr<WindowCursor> range);
|
119
119
|
|
120
120
|
static void ClampFrame(const idx_t count, idx_t *values, const idx_t *begin, const idx_t *end) {
|
121
121
|
for (idx_t i = 0; i < count; ++i) {
|
@@ -25,6 +25,34 @@ class FileOpener;
|
|
25
25
|
class LogStorage;
|
26
26
|
class ExecutionContext;
|
27
27
|
|
28
|
+
//! Logger Macro's are preferred method of calling logger
|
29
|
+
#define DUCKDB_LOG(SOURCE, TYPE, LEVEL, ...) \
|
30
|
+
{ \
|
31
|
+
auto &logger = Logger::Get(SOURCE); \
|
32
|
+
if (logger.ShouldLog(TYPE, LEVEL)) { \
|
33
|
+
logger.WriteLog(TYPE, LEVEL, __VA_ARGS__); \
|
34
|
+
} \
|
35
|
+
}
|
36
|
+
|
37
|
+
//! Use below macros to write to logger.
|
38
|
+
// Parameters:
|
39
|
+
// SOURCE: the context to fetch the logger from, e.g. the ClientContext, or the DatabaseInstance, see Logger::Get
|
40
|
+
// TYPE : a string describing the type of this log entry. Preferred format: `<duckdb/extension name>(.<sometype>)
|
41
|
+
// e.g. `duckdb.Extensions.ExtensionAutoloaded`, `my_extension` or `my_extension.some_type`
|
42
|
+
// PARAMS: Either a string-like type such as `const char *`, `string` or `string_t` or a format string plus the
|
43
|
+
// string parameters
|
44
|
+
//
|
45
|
+
// Examples:
|
46
|
+
// DUCKDB_LOG_TRACE(client_context, "duckdb", "Something happened");
|
47
|
+
// DUCKDB_LOG_INFO(database_instance, "duckdb", CallFunctionThatReturnsString());
|
48
|
+
//
|
49
|
+
#define DUCKDB_LOG_TRACE(SOURCE, TYPE, ...) DUCKDB_LOG(SOURCE, TYPE, LogLevel::LOG_TRACE, __VA_ARGS__)
|
50
|
+
#define DUCKDB_LOG_DEBUG(SOURCE, TYPE, ...) DUCKDB_LOG(SOURCE, TYPE, LogLevel::LOG_DEBUG, __VA_ARGS__)
|
51
|
+
#define DUCKDB_LOG_INFO(SOURCE, TYPE, ...) DUCKDB_LOG(SOURCE, TYPE, LogLevel::LOG_INFO, __VA_ARGS__)
|
52
|
+
#define DUCKDB_LOG_WARN(SOURCE, TYPE, ...) DUCKDB_LOG(SOURCE, TYPE, LogLevel::LOG_WARN, __VA_ARGS__)
|
53
|
+
#define DUCKDB_LOG_ERROR(SOURCE, TYPE, ...) DUCKDB_LOG(SOURCE, TYPE, LogLevel::LOG_ERROR, __VA_ARGS__)
|
54
|
+
#define DUCKDB_LOG_FATAL(SOURCE, TYPE, ...) DUCKDB_LOG(SOURCE, TYPE, LogLevel::LOG_FATAL, __VA_ARGS__)
|
55
|
+
|
28
56
|
//! Main logging interface
|
29
57
|
class Logger {
|
30
58
|
public:
|
@@ -33,19 +61,20 @@ public:
|
|
33
61
|
|
34
62
|
DUCKDB_API virtual ~Logger() = default;
|
35
63
|
|
36
|
-
|
37
|
-
DUCKDB_API void Log(const char *log_type, LogLevel log_level, const char *log_message);
|
38
|
-
DUCKDB_API void Log(LogLevel log_level, const char *log_message);
|
39
|
-
DUCKDB_API void Log(const char *log_type, LogLevel log_level, const string_t &log_message);
|
40
|
-
DUCKDB_API void Log(LogLevel log_level, const string_t &log_message);
|
41
|
-
DUCKDB_API void Log(const char *log_type, LogLevel log_level, std::function<string()> callback);
|
42
|
-
DUCKDB_API void Log(LogLevel log_level, std::function<string()> callback);
|
43
|
-
|
44
|
-
// Main interface for subclasses
|
64
|
+
// Main Logging interface. In most cases the macros above should be used instead of calling these directly
|
45
65
|
DUCKDB_API virtual bool ShouldLog(const char *log_type, LogLevel log_level) = 0;
|
46
|
-
DUCKDB_API virtual bool ShouldLog(LogLevel log_level) = 0;
|
47
66
|
DUCKDB_API virtual void WriteLog(const char *log_type, LogLevel log_level, const char *message) = 0;
|
48
|
-
|
67
|
+
|
68
|
+
// Some more string types for easy logging
|
69
|
+
DUCKDB_API void WriteLog(const char *log_type, LogLevel log_level, const string &message);
|
70
|
+
DUCKDB_API void WriteLog(const char *log_type, LogLevel log_level, const string_t &message);
|
71
|
+
|
72
|
+
// Syntactic sugar for formatted strings
|
73
|
+
template <typename... ARGS>
|
74
|
+
void WriteLog(const char *log_type, LogLevel log_level, const char *format_string, ARGS... params) {
|
75
|
+
auto formatted_string = StringUtil::Format(format_string, params...);
|
76
|
+
WriteLog(log_type, log_level, formatted_string.c_str());
|
77
|
+
}
|
49
78
|
|
50
79
|
DUCKDB_API virtual void Flush() = 0;
|
51
80
|
|
@@ -58,104 +87,6 @@ public:
|
|
58
87
|
DUCKDB_API static Logger &Get(const FileOpener &opener);
|
59
88
|
DUCKDB_API static Logger &Get(const DatabaseInstance &db);
|
60
89
|
|
61
|
-
//! Logger::Log with raw C-String
|
62
|
-
template <class T>
|
63
|
-
static void Log(const char *log_type, T &log_context_source, LogLevel log_level, const char *log_message) {
|
64
|
-
Logger::Get(log_context_source).Log(log_type, log_level, log_message);
|
65
|
-
}
|
66
|
-
template <class T>
|
67
|
-
static void Log(const char *log_type, T &log_context_source, LogLevel log_level, const string &log_message) {
|
68
|
-
Logger::Get(log_context_source).Log(log_type, log_level, log_message.c_str());
|
69
|
-
}
|
70
|
-
template <class T>
|
71
|
-
static void Log(const char *log_type, T &log_context_source, LogLevel log_level, const string_t &log_message) {
|
72
|
-
Logger::Get(log_context_source).Log(log_type, log_level, log_message);
|
73
|
-
}
|
74
|
-
template <class T>
|
75
|
-
static void Log(T &log_context_source, LogLevel log_level, const char *log_message) {
|
76
|
-
Logger::Get(log_context_source).Log(log_level, log_message);
|
77
|
-
}
|
78
|
-
template <class T>
|
79
|
-
static void Log(T &log_context_source, LogLevel log_level, const string &log_message) {
|
80
|
-
Logger::Get(log_context_source).Log(log_level, log_message.c_str());
|
81
|
-
}
|
82
|
-
template <class T>
|
83
|
-
static void Log(T &log_context_source, LogLevel log_level, const string_t &log_message) {
|
84
|
-
Logger::Get(log_context_source).Log(log_level, log_message);
|
85
|
-
}
|
86
|
-
|
87
|
-
//! Logger::Log with callback
|
88
|
-
template <class T>
|
89
|
-
static void Log(const char *log_type, T &log_context_source, LogLevel log_level, std::function<string()> callback) {
|
90
|
-
Logger::Get(log_context_source).Log(log_type, log_level, std::move(callback));
|
91
|
-
}
|
92
|
-
template <class T>
|
93
|
-
static void Log(T &log_context_source, LogLevel log_level, std::function<string()> callback) {
|
94
|
-
Logger::Get(log_context_source).Log(log_level, std::move(callback));
|
95
|
-
}
|
96
|
-
//! Logger::Log with StringUtil::Format
|
97
|
-
template <class T, typename... ARGS>
|
98
|
-
static void Log(const char *log_type, T &log_context_source, LogLevel log_level, const char *format_string,
|
99
|
-
ARGS... params) {
|
100
|
-
Logger::Get(log_context_source).Log(log_type, log_level, [&]() {
|
101
|
-
return StringUtil::Format(format_string, params...);
|
102
|
-
});
|
103
|
-
}
|
104
|
-
template <class T, typename... ARGS>
|
105
|
-
static void Log(T &log_context_source, LogLevel log_level, const char *format_string, ARGS... params) {
|
106
|
-
Logger::Get(log_context_source).Log(log_level, [&]() { return StringUtil::Format(format_string, params...); });
|
107
|
-
}
|
108
|
-
|
109
|
-
//! Templates wrapping Logging::Log(..., LoggingLevel, ....)
|
110
|
-
template <class T, typename... ARGS>
|
111
|
-
static void Trace(T &log_context_source, ARGS... params) {
|
112
|
-
Log(log_context_source, LogLevel::LOG_TRACE, params...);
|
113
|
-
}
|
114
|
-
template <class T, typename... ARGS>
|
115
|
-
static void Trace(const char *log_type, T &log_context_source, ARGS... params) {
|
116
|
-
Log(log_type, log_context_source, LogLevel::LOG_TRACE, params...);
|
117
|
-
}
|
118
|
-
template <class T, typename... ARGS>
|
119
|
-
static void Debug(T &log_context_source, ARGS... params) {
|
120
|
-
Log(log_context_source, LogLevel::LOG_DEBUG, params...);
|
121
|
-
}
|
122
|
-
template <class T, typename... ARGS>
|
123
|
-
static void Debug(const char *log_type, T &log_context_source, ARGS... params) {
|
124
|
-
Log(log_type, log_context_source, LogLevel::LOG_DEBUG, params...);
|
125
|
-
}
|
126
|
-
template <class T, typename... ARGS>
|
127
|
-
static void Info(T &log_context_source, ARGS... params) {
|
128
|
-
Log(log_context_source, LogLevel::LOG_INFO, params...);
|
129
|
-
}
|
130
|
-
template <class T, typename... ARGS>
|
131
|
-
static void Info(const char *log_type, T &log_context_source, ARGS... params) {
|
132
|
-
Log(log_type, log_context_source, LogLevel::LOG_INFO, params...);
|
133
|
-
}
|
134
|
-
template <class T, typename... ARGS>
|
135
|
-
static void Warn(T &log_context_source, ARGS... params) {
|
136
|
-
Log(log_context_source, LogLevel::LOG_WARN, params...);
|
137
|
-
}
|
138
|
-
template <class T, typename... ARGS>
|
139
|
-
static void Warn(const char *log_type, T &log_context_source, ARGS... params) {
|
140
|
-
Log(log_type, log_context_source, LogLevel::LOG_WARN, params...);
|
141
|
-
}
|
142
|
-
template <class T, typename... ARGS>
|
143
|
-
static void Error(T &log_context_source, ARGS... params) {
|
144
|
-
Log(log_context_source, LogLevel::LOG_ERROR, params...);
|
145
|
-
}
|
146
|
-
template <class T, typename... ARGS>
|
147
|
-
static void Error(const char *log_type, T &log_context_source, ARGS... params) {
|
148
|
-
Log(log_type, log_context_source, LogLevel::LOG_ERROR, params...);
|
149
|
-
}
|
150
|
-
template <class T, typename... ARGS>
|
151
|
-
static void Fatal(T &log_context_source, ARGS... params) {
|
152
|
-
Log(log_context_source, LogLevel::LOG_FATAL, params...);
|
153
|
-
}
|
154
|
-
template <class T, typename... ARGS>
|
155
|
-
static void Fatal(const char *log_type, T &log_context_source, ARGS... params) {
|
156
|
-
Log(log_type, log_context_source, LogLevel::LOG_FATAL, params...);
|
157
|
-
}
|
158
|
-
|
159
90
|
template <class T>
|
160
91
|
static void Flush(T &log_context_source) {
|
161
92
|
Get(log_context_source).Flush();
|
@@ -182,9 +113,7 @@ public:
|
|
182
113
|
|
183
114
|
// Main Logger API
|
184
115
|
bool ShouldLog(const char *log_type, LogLevel log_level) override;
|
185
|
-
bool ShouldLog(LogLevel log_level) override;
|
186
116
|
void WriteLog(const char *log_type, LogLevel log_level, const char *message) override;
|
187
|
-
void WriteLog(LogLevel log_level, const char *message) override;
|
188
117
|
|
189
118
|
void Flush() override;
|
190
119
|
bool IsThreadSafe() override {
|
@@ -209,9 +138,7 @@ public:
|
|
209
138
|
|
210
139
|
// Main Logger API
|
211
140
|
bool ShouldLog(const char *log_type, LogLevel log_level) override;
|
212
|
-
bool ShouldLog(LogLevel log_level) override;
|
213
141
|
void WriteLog(const char *log_type, LogLevel log_level, const char *message) override;
|
214
|
-
void WriteLog(LogLevel log_level, const char *message) override;
|
215
142
|
void Flush() override;
|
216
143
|
|
217
144
|
bool IsThreadSafe() override {
|
@@ -234,9 +161,7 @@ public:
|
|
234
161
|
|
235
162
|
// Main Logger API
|
236
163
|
bool ShouldLog(const char *log_type, LogLevel log_level) override;
|
237
|
-
bool ShouldLog(LogLevel log_level) override;
|
238
164
|
void WriteLog(const char *log_type, LogLevel log_level, const char *message) override;
|
239
|
-
void WriteLog(LogLevel log_level, const char *message) override;
|
240
165
|
|
241
166
|
void Flush() override;
|
242
167
|
bool IsThreadSafe() override {
|
@@ -269,11 +194,7 @@ public:
|
|
269
194
|
bool ShouldLog(const char *log_type, LogLevel log_level) override {
|
270
195
|
return false;
|
271
196
|
}
|
272
|
-
bool ShouldLog(LogLevel log_level) override {
|
273
|
-
return false;
|
274
|
-
};
|
275
197
|
void WriteLog(const char *log_type, LogLevel log_level, const char *message) override {};
|
276
|
-
void WriteLog(LogLevel log_level, const char *message) override {};
|
277
198
|
void Flush() override {
|
278
199
|
}
|
279
200
|
bool IsThreadSafe() override {
|
@@ -101,6 +101,8 @@ struct ExtensionOption {
|
|
101
101
|
|
102
102
|
class SerializationCompatibility {
|
103
103
|
public:
|
104
|
+
static SerializationCompatibility FromDatabase(AttachedDatabase &db);
|
105
|
+
static SerializationCompatibility FromIndex(idx_t serialization_version);
|
104
106
|
static SerializationCompatibility FromString(const string &input);
|
105
107
|
static SerializationCompatibility Default();
|
106
108
|
static SerializationCompatibility Latest();
|
@@ -202,6 +204,8 @@ struct DBConfigOptions {
|
|
202
204
|
uint64_t zstd_min_string_length = 4096;
|
203
205
|
//! Force a specific compression method to be used when checkpointing (if available)
|
204
206
|
CompressionType force_compression = CompressionType::COMPRESSION_AUTO;
|
207
|
+
//! The set of disabled compression methods (default empty)
|
208
|
+
set<CompressionType> disabled_compression_methods;
|
205
209
|
//! Force a specific bitpacking mode to be used when using the bitpacking compression method
|
206
210
|
BitpackingMode force_bitpacking_mode = BitpackingMode::AUTO;
|
207
211
|
//! Debug setting for window aggregation mode: (window, combine, separate)
|
@@ -380,6 +384,7 @@ public:
|
|
380
384
|
DUCKDB_API ArrowTypeExtension GetArrowExtension(ArrowExtensionMetadata info) const;
|
381
385
|
DUCKDB_API ArrowTypeExtension GetArrowExtension(const LogicalType &type) const;
|
382
386
|
DUCKDB_API bool HasArrowExtension(const LogicalType &type) const;
|
387
|
+
DUCKDB_API bool HasArrowExtension(ArrowExtensionMetadata info) const;
|
383
388
|
DUCKDB_API void RegisterArrowExtension(const ArrowTypeExtension &extension) const;
|
384
389
|
|
385
390
|
bool operator==(const DBConfig &other);
|
@@ -166,14 +166,6 @@ public:
|
|
166
166
|
DUCKDB_API shared_ptr<Relation> RelationFromQuery(unique_ptr<SelectStatement> select_stmt,
|
167
167
|
const string &alias = "queryrelation", const string &query = "");
|
168
168
|
|
169
|
-
//! Returns a substrait BLOB from a valid query
|
170
|
-
DUCKDB_API string GetSubstrait(const string &query);
|
171
|
-
//! Returns a Query Result from a substrait blob
|
172
|
-
DUCKDB_API unique_ptr<QueryResult> FromSubstrait(const string &proto);
|
173
|
-
//! Returns a substrait BLOB from a valid query
|
174
|
-
DUCKDB_API string GetSubstraitJSON(const string &query);
|
175
|
-
//! Returns a Query Result from a substrait JSON
|
176
|
-
DUCKDB_API unique_ptr<QueryResult> FromSubstraitJSON(const string &json);
|
177
169
|
DUCKDB_API void BeginTransaction();
|
178
170
|
DUCKDB_API void Commit();
|
179
171
|
DUCKDB_API void Rollback();
|
@@ -34,8 +34,9 @@ public:
|
|
34
34
|
static ConnectionManager &Get(ClientContext &context);
|
35
35
|
|
36
36
|
private:
|
37
|
-
|
37
|
+
mutex connections_lock;
|
38
38
|
reference_map_t<ClientContext, weak_ptr<ClientContext>> connections;
|
39
|
+
atomic<idx_t> connection_count;
|
39
40
|
};
|
40
41
|
|
41
42
|
} // namespace duckdb
|