duckdb 1.2.1-dev6.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/duckdb/extension/core_functions/aggregate/distributive/string_agg.cpp +14 -22
- package/src/duckdb/extension/core_functions/aggregate/nested/list.cpp +0 -1
- package/src/duckdb/extension/core_functions/lambda_functions.cpp +0 -11
- package/src/duckdb/extension/core_functions/scalar/list/list_aggregates.cpp +18 -6
- package/src/duckdb/extension/icu/icu-datefunc.cpp +9 -2
- package/src/duckdb/extension/icu/icu-strptime.cpp +7 -11
- package/src/duckdb/extension/icu/include/icu-datefunc.hpp +3 -1
- package/src/duckdb/extension/json/buffered_json_reader.cpp +18 -31
- package/src/duckdb/extension/json/json_extension.cpp +8 -3
- package/src/duckdb/extension/parquet/column_reader.cpp +4 -6
- package/src/duckdb/extension/parquet/column_writer.cpp +33 -12
- package/src/duckdb/extension/parquet/include/column_reader.hpp +0 -2
- package/src/duckdb/extension/parquet/include/parquet_bss_encoder.hpp +0 -1
- package/src/duckdb/extension/parquet/include/parquet_dlba_encoder.hpp +1 -2
- package/src/duckdb/src/catalog/catalog.cpp +12 -0
- package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
- package/src/duckdb/src/catalog/catalog_entry_retriever.cpp +1 -1
- package/src/duckdb/src/catalog/catalog_search_path.cpp +8 -8
- package/src/duckdb/src/common/bind_helpers.cpp +3 -0
- package/src/duckdb/src/common/compressed_file_system.cpp +2 -0
- package/src/duckdb/src/common/hive_partitioning.cpp +1 -1
- package/src/duckdb/src/common/multi_file_reader.cpp +3 -3
- package/src/duckdb/src/execution/aggregate_hashtable.cpp +1 -1
- package/src/duckdb/src/execution/index/art/art.cpp +19 -6
- package/src/duckdb/src/execution/index/art/iterator.cpp +7 -3
- package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +11 -4
- package/src/duckdb/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp +2 -2
- package/src/duckdb/src/execution/operator/csv_scanner/encode/csv_encoder.cpp +5 -1
- package/src/duckdb/src/execution/operator/csv_scanner/scanner/base_scanner.cpp +3 -2
- package/src/duckdb/src/execution/operator/csv_scanner/scanner/csv_schema.cpp +2 -2
- package/src/duckdb/src/execution/operator/csv_scanner/scanner/scanner_boundary.cpp +1 -1
- package/src/duckdb/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +20 -12
- package/src/duckdb/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp +19 -22
- package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_refinement.cpp +1 -1
- package/src/duckdb/src/execution/operator/csv_scanner/util/csv_error.cpp +1 -0
- package/src/duckdb/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +16 -0
- package/src/duckdb/src/execution/operator/helper/physical_reservoir_sample.cpp +1 -0
- package/src/duckdb/src/execution/operator/helper/physical_streaming_sample.cpp +16 -7
- package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp +3 -1
- package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +11 -1
- package/src/duckdb/src/execution/operator/schema/physical_create_art_index.cpp +5 -7
- package/src/duckdb/src/execution/physical_plan/plan_create_index.cpp +11 -0
- package/src/duckdb/src/execution/physical_plan/plan_sample.cpp +1 -3
- package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +14 -5
- package/src/duckdb/src/execution/sample/reservoir_sample.cpp +24 -12
- package/src/duckdb/src/function/scalar/generic/getvariable.cpp +3 -3
- package/src/duckdb/src/function/table/version/pragma_version.cpp +3 -3
- package/src/duckdb/src/function/window/window_aggregate_states.cpp +3 -0
- package/src/duckdb/src/function/window/window_boundaries_state.cpp +108 -48
- package/src/duckdb/src/function/window/window_constant_aggregator.cpp +5 -5
- package/src/duckdb/src/function/window/window_distinct_aggregator.cpp +6 -0
- package/src/duckdb/src/include/duckdb/catalog/catalog_entry_retriever.hpp +1 -1
- package/src/duckdb/src/include/duckdb/catalog/catalog_search_path.hpp +10 -9
- package/src/duckdb/src/include/duckdb/common/adbc/adbc-init.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +2 -0
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/base_scanner.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_buffer.hpp +5 -4
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_option.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_schema.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/encode/csv_encoder.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/sniffer/csv_sniffer.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_streaming_sample.hpp +3 -7
- package/src/duckdb/src/include/duckdb/execution/reservoir_sample.hpp +2 -1
- package/src/duckdb/src/include/duckdb/function/lambda_functions.hpp +11 -3
- package/src/duckdb/src/include/duckdb/function/window/window_boundaries_state.hpp +4 -0
- package/src/duckdb/src/include/duckdb/main/client_context_state.hpp +4 -0
- package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +25 -7
- package/src/duckdb/src/include/duckdb/main/pending_query_result.hpp +2 -0
- package/src/duckdb/src/include/duckdb/main/query_profiler.hpp +7 -0
- package/src/duckdb/src/include/duckdb/optimizer/filter_combiner.hpp +2 -2
- package/src/duckdb/src/include/duckdb/optimizer/late_materialization.hpp +2 -1
- package/src/duckdb/src/include/duckdb/optimizer/optimizer_extension.hpp +11 -5
- package/src/duckdb/src/include/duckdb/parallel/executor_task.hpp +4 -1
- package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +0 -1
- package/src/duckdb/src/include/duckdb/parallel/task_executor.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parallel/task_notifier.hpp +27 -0
- package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/expression/bound_subquery_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +1 -0
- package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp +3 -1
- package/src/duckdb/src/include/duckdb/storage/checkpoint_manager.hpp +7 -1
- package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp +3 -2
- package/src/duckdb/src/include/duckdb.h +495 -480
- package/src/duckdb/src/main/attached_database.cpp +1 -1
- package/src/duckdb/src/main/capi/duckdb-c.cpp +5 -1
- package/src/duckdb/src/main/capi/helper-c.cpp +8 -0
- package/src/duckdb/src/main/config.cpp +7 -1
- package/src/duckdb/src/main/database.cpp +8 -8
- package/src/duckdb/src/main/extension/extension_helper.cpp +3 -1
- package/src/duckdb/src/main/extension/extension_load.cpp +12 -12
- package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -0
- package/src/duckdb/src/optimizer/join_order/query_graph_manager.cpp +2 -2
- package/src/duckdb/src/optimizer/late_materialization.cpp +26 -5
- package/src/duckdb/src/optimizer/optimizer.cpp +12 -1
- package/src/duckdb/src/parallel/executor_task.cpp +10 -6
- package/src/duckdb/src/parallel/task_executor.cpp +4 -1
- package/src/duckdb/src/parallel/task_notifier.cpp +23 -0
- package/src/duckdb/src/parallel/task_scheduler.cpp +33 -0
- package/src/duckdb/src/parser/transform/expression/transform_subquery.cpp +4 -1
- package/src/duckdb/src/planner/binder/expression/bind_subquery_expression.cpp +1 -1
- package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +4 -2
- package/src/duckdb/src/planner/binder/statement/bind_create.cpp +7 -2
- package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +6 -5
- package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +4 -2
- package/src/duckdb/src/storage/checkpoint_manager.cpp +4 -3
- package/src/duckdb/src/storage/compression/string_uncompressed.cpp +21 -10
- package/src/duckdb/src/storage/storage_info.cpp +2 -0
- package/src/duckdb/src/storage/storage_manager.cpp +2 -2
- package/src/duckdb/src/storage/table/row_group.cpp +5 -6
- package/src/duckdb/src/storage/table/scan_state.cpp +6 -0
- package/src/duckdb/src/transaction/duck_transaction.cpp +11 -3
- package/src/duckdb/src/transaction/duck_transaction_manager.cpp +2 -2
- package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +17 -0
- package/src/duckdb/ub_src_parallel.cpp +2 -0
@@ -48,20 +48,21 @@ public:
|
|
48
48
|
DUCKDB_API void Set(vector<CatalogSearchEntry> new_paths, CatalogSetPathType set_type);
|
49
49
|
DUCKDB_API void Reset();
|
50
50
|
|
51
|
-
DUCKDB_API const vector<CatalogSearchEntry> &Get();
|
52
|
-
const vector<CatalogSearchEntry> &GetSetPaths() {
|
51
|
+
DUCKDB_API const vector<CatalogSearchEntry> &Get() const;
|
52
|
+
const vector<CatalogSearchEntry> &GetSetPaths() const {
|
53
53
|
return set_paths;
|
54
54
|
}
|
55
|
-
DUCKDB_API const CatalogSearchEntry &GetDefault();
|
55
|
+
DUCKDB_API const CatalogSearchEntry &GetDefault() const;
|
56
56
|
//! FIXME: this method is deprecated
|
57
|
-
DUCKDB_API string GetDefaultSchema(const string &catalog);
|
58
|
-
DUCKDB_API string GetDefaultSchema(ClientContext &context, const string &catalog);
|
59
|
-
DUCKDB_API string GetDefaultCatalog(const string &schema);
|
57
|
+
DUCKDB_API string GetDefaultSchema(const string &catalog) const;
|
58
|
+
DUCKDB_API string GetDefaultSchema(ClientContext &context, const string &catalog) const;
|
59
|
+
DUCKDB_API string GetDefaultCatalog(const string &schema) const;
|
60
60
|
|
61
|
-
DUCKDB_API vector<string> GetSchemasForCatalog(const string &catalog);
|
62
|
-
DUCKDB_API vector<string> GetCatalogsForSchema(const string &schema);
|
61
|
+
DUCKDB_API vector<string> GetSchemasForCatalog(const string &catalog) const;
|
62
|
+
DUCKDB_API vector<string> GetCatalogsForSchema(const string &schema) const;
|
63
63
|
|
64
|
-
DUCKDB_API bool SchemaInSearchPath(ClientContext &context, const string &catalog_name,
|
64
|
+
DUCKDB_API bool SchemaInSearchPath(ClientContext &context, const string &catalog_name,
|
65
|
+
const string &schema_name) const;
|
65
66
|
|
66
67
|
private:
|
67
68
|
//! Set paths without checking if they exist
|
@@ -30,7 +30,7 @@ extern "C" {
|
|
30
30
|
typedef uint8_t AdbcStatusCode;
|
31
31
|
|
32
32
|
//! We gotta leak the symbols of the init function
|
33
|
-
|
33
|
+
DUCKDB_C_API AdbcStatusCode duckdb_adbc_init(int version, void *driver, struct AdbcError *error);
|
34
34
|
|
35
35
|
#ifdef __cplusplus
|
36
36
|
}
|
@@ -168,7 +168,7 @@ struct MultiFileFilterEntry {
|
|
168
168
|
struct MultiFileConstantEntry {
|
169
169
|
MultiFileConstantEntry(idx_t column_id, Value value_p) : column_id(column_id), value(std::move(value_p)) {
|
170
170
|
}
|
171
|
-
//! The column id to apply the constant value to
|
171
|
+
//! The (global) column id to apply the constant value to
|
172
172
|
idx_t column_id;
|
173
173
|
//! The constant value
|
174
174
|
Value value;
|
@@ -273,7 +273,7 @@ struct MultiFileReader {
|
|
273
273
|
const string &initial_file, const MultiFileReaderBindData &options,
|
274
274
|
optional_ptr<MultiFileReaderGlobalState> global_state);
|
275
275
|
//! Populated the filter_map
|
276
|
-
DUCKDB_API virtual void CreateFilterMap(const vector<
|
276
|
+
DUCKDB_API virtual void CreateFilterMap(const vector<ColumnIndex> &global_column_ids,
|
277
277
|
optional_ptr<TableFilterSet> filters, MultiFileReaderData &reader_data,
|
278
278
|
optional_ptr<MultiFileReaderGlobalState> global_state);
|
279
279
|
|
@@ -90,6 +90,8 @@ private:
|
|
90
90
|
GateStatus status;
|
91
91
|
//! Depth in a nested leaf.
|
92
92
|
uint8_t nested_depth = 0;
|
93
|
+
//! True, if we entered a nested leaf to retrieve the next node.
|
94
|
+
bool entered_nested_leaf = false;
|
93
95
|
|
94
96
|
private:
|
95
97
|
//! Goes to the next leaf in the ART and sets it as last_leaf,
|
@@ -116,7 +116,7 @@ class BaseScanner {
|
|
116
116
|
public:
|
117
117
|
explicit BaseScanner(shared_ptr<CSVBufferManager> buffer_manager, shared_ptr<CSVStateMachine> state_machine,
|
118
118
|
shared_ptr<CSVErrorHandler> error_handler, bool sniffing = false,
|
119
|
-
shared_ptr<CSVFileScan> csv_file_scan = nullptr, CSVIterator iterator = {});
|
119
|
+
shared_ptr<CSVFileScan> csv_file_scan = nullptr, const CSVIterator &iterator = {});
|
120
120
|
|
121
121
|
virtual ~BaseScanner() = default;
|
122
122
|
|
@@ -45,14 +45,15 @@ class CSVBuffer {
|
|
45
45
|
public:
|
46
46
|
//! Constructor for Initial Buffer
|
47
47
|
CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle &file_handle,
|
48
|
-
idx_t &global_csv_current_position, idx_t file_number);
|
48
|
+
const idx_t &global_csv_current_position, idx_t file_number);
|
49
49
|
|
50
50
|
//! Constructor for `Next()` Buffers
|
51
51
|
CSVBuffer(CSVFileHandle &file_handle, ClientContext &context, idx_t buffer_size, idx_t global_csv_current_position,
|
52
52
|
idx_t file_number_p, idx_t buffer_idx);
|
53
53
|
|
54
54
|
//! Creates a new buffer with the next part of the CSV File
|
55
|
-
shared_ptr<CSVBuffer> Next(CSVFileHandle &file_handle, idx_t buffer_size, idx_t file_number,
|
55
|
+
shared_ptr<CSVBuffer> Next(CSVFileHandle &file_handle, idx_t buffer_size, idx_t file_number,
|
56
|
+
bool &has_seaked) const;
|
56
57
|
|
57
58
|
//! Gets the buffer actual size
|
58
59
|
idx_t GetBufferSize() const;
|
@@ -67,12 +68,12 @@ public:
|
|
67
68
|
//! Wrapper for the Pin Function, if it can seek, it means that the buffer might have been destroyed, hence we must
|
68
69
|
//! Scan it from the disk file again.
|
69
70
|
shared_ptr<CSVBufferHandle> Pin(CSVFileHandle &file_handle, bool &has_seeked);
|
70
|
-
//! Wrapper for
|
71
|
+
//! Wrapper for unpin
|
71
72
|
void Unpin();
|
72
73
|
char *Ptr() {
|
73
74
|
return char_ptr_cast(handle.Ptr());
|
74
75
|
}
|
75
|
-
bool IsUnloaded() {
|
76
|
+
bool IsUnloaded() const {
|
76
77
|
return block->IsUnloaded();
|
77
78
|
}
|
78
79
|
|
@@ -30,7 +30,7 @@ struct CSVOption { // NOLINT: work-around bug in clang-tidy
|
|
30
30
|
public:
|
31
31
|
CSVOption(T value_p) : value(value_p) { // NOLINT: allow implicit conversion from value
|
32
32
|
}
|
33
|
-
CSVOption(T value_p, bool set_by_user_p) :
|
33
|
+
CSVOption(T value_p, bool set_by_user_p) : set_by_user(set_by_user_p), value(value_p) {
|
34
34
|
}
|
35
35
|
|
36
36
|
CSVOption() {};
|
@@ -14,7 +14,7 @@
|
|
14
14
|
namespace duckdb {
|
15
15
|
//! Basic CSV Column Info
|
16
16
|
struct CSVColumnInfo {
|
17
|
-
CSVColumnInfo(string &name_p, LogicalType &type_p) : name(name_p), type(type_p) {
|
17
|
+
CSVColumnInfo(const string &name_p, const LogicalType &type_p) : name(name_p), type(type_p) {
|
18
18
|
}
|
19
19
|
string name;
|
20
20
|
LogicalType type;
|
@@ -25,7 +25,7 @@ struct CSVSchema {
|
|
25
25
|
explicit CSVSchema(const bool empty = false) : empty(empty) {
|
26
26
|
}
|
27
27
|
|
28
|
-
CSVSchema(vector<string> &names, vector<LogicalType> &types, const string &file_path, idx_t rows_read,
|
28
|
+
CSVSchema(const vector<string> &names, const vector<LogicalType> &types, const string &file_path, idx_t rows_read,
|
29
29
|
const bool empty = false);
|
30
30
|
|
31
31
|
//! Initializes the schema based on names and types
|
@@ -46,7 +46,7 @@ private:
|
|
46
46
|
class CSVEncoder {
|
47
47
|
public:
|
48
48
|
//! Constructor, basically takes an encoding and the output buffer size
|
49
|
-
CSVEncoder(DBConfig &config, const string &encoding_name, idx_t buffer_size);
|
49
|
+
CSVEncoder(const DBConfig &config, const string &encoding_name, idx_t buffer_size);
|
50
50
|
//! Main encode function, it reads the file into an encoded buffer and converts it to the output buffer
|
51
51
|
idx_t Encode(FileHandle &file_handle_input, char *output_buffer, const idx_t decoded_buffer_size);
|
52
52
|
string encoding_name;
|
package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/sniffer/csv_sniffer.hpp
CHANGED
@@ -222,7 +222,7 @@ private:
|
|
222
222
|
//! ------------------ Type Refinement ------------------ //
|
223
223
|
//! ------------------------------------------------------//
|
224
224
|
void RefineTypes();
|
225
|
-
bool TryCastVector(Vector &parse_chunk_col, idx_t size, const LogicalType &sql_type);
|
225
|
+
bool TryCastVector(Vector &parse_chunk_col, idx_t size, const LogicalType &sql_type) const;
|
226
226
|
vector<LogicalType> detected_types;
|
227
227
|
//! If when finding a SQLNULL type in type detection we default it to varchar
|
228
228
|
const bool default_null_to_varchar;
|
package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp
CHANGED
@@ -121,8 +121,8 @@ private:
|
|
121
121
|
};
|
122
122
|
|
123
123
|
struct ParseTypeInfo {
|
124
|
-
ParseTypeInfo() {};
|
125
|
-
ParseTypeInfo(const LogicalType &type, bool validate_utf_8_p) : validate_utf8(validate_utf_8_p) {
|
124
|
+
ParseTypeInfo() : validate_utf8(false), type_id(), internal_type(), scale(0), width(0) {};
|
125
|
+
ParseTypeInfo(const LogicalType &type, const bool validate_utf_8_p) : validate_utf8(validate_utf_8_p) {
|
126
126
|
type_id = type.id();
|
127
127
|
internal_type = type.InternalType();
|
128
128
|
if (type.id() == LogicalTypeId::DECIMAL) {
|
package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_streaming_sample.hpp
CHANGED
@@ -19,12 +19,10 @@ public:
|
|
19
19
|
static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::STREAMING_SAMPLE;
|
20
20
|
|
21
21
|
public:
|
22
|
-
PhysicalStreamingSample(vector<LogicalType> types,
|
23
|
-
idx_t estimated_cardinality);
|
22
|
+
PhysicalStreamingSample(vector<LogicalType> types, unique_ptr<SampleOptions> options, idx_t estimated_cardinality);
|
24
23
|
|
25
|
-
|
24
|
+
unique_ptr<SampleOptions> sample_options;
|
26
25
|
double percentage;
|
27
|
-
int64_t seed;
|
28
26
|
|
29
27
|
public:
|
30
28
|
// Operator interface
|
@@ -32,9 +30,7 @@ public:
|
|
32
30
|
OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
|
33
31
|
GlobalOperatorState &gstate, OperatorState &state) const override;
|
34
32
|
|
35
|
-
bool ParallelOperator() const override
|
36
|
-
return true;
|
37
|
-
}
|
33
|
+
bool ParallelOperator() const override;
|
38
34
|
|
39
35
|
InsertionOrderPreservingMap<string> ParamsToString() const override;
|
40
36
|
|
@@ -201,7 +201,8 @@ public:
|
|
201
201
|
void ConvertToReservoirSample();
|
202
202
|
|
203
203
|
//! Get the capactiy of the data chunk reserved for storing samples
|
204
|
-
|
204
|
+
template <typename T>
|
205
|
+
T GetReservoirChunkCapacity() const;
|
205
206
|
|
206
207
|
//! If for_serialization=true then the sample_chunk is not padded with extra spaces for
|
207
208
|
//! future sampling values
|
@@ -17,7 +17,7 @@
|
|
17
17
|
|
18
18
|
namespace duckdb {
|
19
19
|
|
20
|
-
struct ListLambdaBindData : public FunctionData {
|
20
|
+
struct ListLambdaBindData final : public FunctionData {
|
21
21
|
public:
|
22
22
|
ListLambdaBindData(const LogicalType &return_type, unique_ptr<Expression> lambda_expr, const bool has_index = false)
|
23
23
|
: return_type(return_type), lambda_expr(std::move(lambda_expr)), has_index(has_index) {};
|
@@ -30,8 +30,16 @@ public:
|
|
30
30
|
bool has_index;
|
31
31
|
|
32
32
|
public:
|
33
|
-
|
34
|
-
|
33
|
+
unique_ptr<FunctionData> Copy() const override {
|
34
|
+
auto lambda_expr_copy = lambda_expr ? lambda_expr->Copy() : nullptr;
|
35
|
+
return make_uniq<ListLambdaBindData>(return_type, std::move(lambda_expr_copy), has_index);
|
36
|
+
}
|
37
|
+
|
38
|
+
bool Equals(const FunctionData &other_p) const override {
|
39
|
+
auto &other = other_p.Cast<ListLambdaBindData>();
|
40
|
+
return Expression::Equals(lambda_expr, other.lambda_expr) && return_type == other.return_type &&
|
41
|
+
has_index == other.has_index;
|
42
|
+
}
|
35
43
|
|
36
44
|
//! Serializes a lambda function's bind data
|
37
45
|
static void Serialize(Serializer &serializer, const optional_ptr<FunctionData> bind_data_p,
|
@@ -489,6 +489,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
|
|
489
489
|
{"read_xlsx", "excel", CatalogType::TABLE_FUNCTION_ENTRY},
|
490
490
|
{"reduce", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
|
491
491
|
{"reduce_sql_statement", "sqlsmith", CatalogType::TABLE_FUNCTION_ENTRY},
|
492
|
+
{"register_geoarrow_extensions", "spatial", CatalogType::TABLE_FUNCTION_ENTRY},
|
492
493
|
{"regr_avgx", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
|
493
494
|
{"regr_avgy", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
|
494
495
|
{"regr_count", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
|
@@ -538,6 +539,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
|
|
538
539
|
{"st_centroid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
539
540
|
{"st_collect", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
540
541
|
{"st_collectionextract", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
542
|
+
{"st_concavehull", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
541
543
|
{"st_contains", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
542
544
|
{"st_containsproperly", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
543
545
|
{"st_convexhull", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
@@ -588,14 +590,17 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
|
|
588
590
|
{"st_isvalid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
589
591
|
{"st_length", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
590
592
|
{"st_length_spheroid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
593
|
+
{"st_lineinterpolatepoint", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
594
|
+
{"st_lineinterpolatepoints", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
591
595
|
{"st_linemerge", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
592
596
|
{"st_linestring2dfromwkb", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
593
|
-
{"
|
597
|
+
{"st_linesubstring", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
594
598
|
{"st_m", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
595
599
|
{"st_makeenvelope", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
596
600
|
{"st_makeline", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
597
601
|
{"st_makepolygon", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
598
602
|
{"st_makevalid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
603
|
+
{"st_minimumrotatedrectangle", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
599
604
|
{"st_mmax", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
600
605
|
{"st_mmin", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
601
606
|
{"st_multi", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
@@ -634,6 +639,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
|
|
634
639
|
{"st_transform", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
635
640
|
{"st_union", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
636
641
|
{"st_union_agg", "spatial", CatalogType::AGGREGATE_FUNCTION_ENTRY},
|
642
|
+
{"st_voronoidiagram", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
637
643
|
{"st_within", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
638
644
|
{"st_x", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
639
645
|
{"st_xmax", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
@@ -645,13 +651,15 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
|
|
645
651
|
{"st_zmax", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
646
652
|
{"st_zmflag", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
647
653
|
{"st_zmin", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
|
648
|
-
{"start_ui", "
|
654
|
+
{"start_ui", "ui", CatalogType::TABLE_FUNCTION_ENTRY},
|
655
|
+
{"start_ui_server", "ui", CatalogType::TABLE_FUNCTION_ENTRY},
|
649
656
|
{"starts_with", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
|
650
657
|
{"stats", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
|
651
658
|
{"stddev", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
|
652
659
|
{"stddev_pop", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
|
653
660
|
{"stddev_samp", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
|
654
661
|
{"stem", "fts", CatalogType::SCALAR_FUNCTION_ENTRY},
|
662
|
+
{"stop_ui_server", "ui", CatalogType::TABLE_FUNCTION_ENTRY},
|
655
663
|
{"string_agg", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
|
656
664
|
{"strpos", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
|
657
665
|
{"struct_insert", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
|
@@ -695,6 +703,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
|
|
695
703
|
{"trunc", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
|
696
704
|
{"txid_current", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
|
697
705
|
{"typeof", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
|
706
|
+
{"ui_is_started", "ui", CatalogType::TABLE_FUNCTION_ENTRY},
|
698
707
|
{"unbin", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
|
699
708
|
{"unhex", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
|
700
709
|
{"unicode", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
|
@@ -983,6 +992,9 @@ static constexpr ExtensionEntry EXTENSION_SETTINGS[] = {
|
|
983
992
|
{"sqlite_all_varchar", "sqlite_scanner"},
|
984
993
|
{"sqlite_debug_show_queries", "sqlite_scanner"},
|
985
994
|
{"timezone", "icu"},
|
995
|
+
{"ui_local_port", "ui"},
|
996
|
+
{"ui_polling_interval", "ui"},
|
997
|
+
{"ui_remote_url", "ui"},
|
986
998
|
{"unsafe_enable_version_guessing", "iceberg"},
|
987
999
|
}; // END_OF_EXTENSION_SETTINGS
|
988
1000
|
|
@@ -1070,10 +1082,16 @@ static constexpr ExtensionEntry EXTENSION_SECRET_PROVIDERS[] = {
|
|
1070
1082
|
{"mysql/config", "mysql_scanner"},
|
1071
1083
|
{"postgres/config", "postgres_scanner"}}; // EXTENSION_SECRET_PROVIDERS
|
1072
1084
|
|
1073
|
-
static constexpr const char *AUTOLOADABLE_EXTENSIONS[] = {
|
1074
|
-
|
1075
|
-
|
1076
|
-
|
1077
|
-
|
1085
|
+
static constexpr const char *AUTOLOADABLE_EXTENSIONS[] = {"aws", "azure",
|
1086
|
+
"autocomplete", "core_functions",
|
1087
|
+
"delta", "excel",
|
1088
|
+
"fts", "httpfs",
|
1089
|
+
"iceberg", "inet",
|
1090
|
+
"icu", "json",
|
1091
|
+
"motherduck", "mysql_scanner",
|
1092
|
+
"parquet", "sqlite_scanner",
|
1093
|
+
"sqlsmith", "postgres_scanner",
|
1094
|
+
"tpcds", "tpch",
|
1095
|
+
"uc_catalog", "ui"}; // END_OF_AUTOLOADABLE_EXTENSIONS
|
1078
1096
|
|
1079
1097
|
} // namespace duckdb
|
@@ -29,6 +29,8 @@ public:
|
|
29
29
|
DUCKDB_API explicit PendingQueryResult(ErrorData error_message);
|
30
30
|
DUCKDB_API ~PendingQueryResult() override;
|
31
31
|
DUCKDB_API bool AllowStreamResult() const;
|
32
|
+
PendingQueryResult(const PendingQueryResult &) = delete;
|
33
|
+
PendingQueryResult &operator=(const PendingQueryResult &) = delete;
|
32
34
|
|
33
35
|
public:
|
34
36
|
//! Executes a single task within the query, returning whether or not the query is ready.
|
@@ -166,6 +166,13 @@ public:
|
|
166
166
|
return root.get();
|
167
167
|
}
|
168
168
|
|
169
|
+
//! Provides access to the root of the query tree, but ensures there are no concurrent modifications
|
170
|
+
//! This can be useful when implementing continuous profiling or making customizations
|
171
|
+
DUCKDB_API void GetRootUnderLock(const std::function<void(optional_ptr<ProfilingNode>)> &callback) {
|
172
|
+
lock_guard<std::mutex> guard(lock);
|
173
|
+
callback(GetRoot());
|
174
|
+
}
|
175
|
+
|
169
176
|
private:
|
170
177
|
ClientContext &context;
|
171
178
|
|
@@ -103,8 +103,8 @@ private:
|
|
103
103
|
|
104
104
|
expression_map_t<unique_ptr<Expression>> stored_expressions;
|
105
105
|
expression_map_t<idx_t> equivalence_set_map;
|
106
|
-
|
107
|
-
|
106
|
+
map<idx_t, vector<ExpressionValueInformation>> constant_values;
|
107
|
+
map<idx_t, vector<reference<Expression>>> equivalence_map;
|
108
108
|
idx_t set_index = 0;
|
109
109
|
//
|
110
110
|
// //! Structures used for OR Filters
|
@@ -14,6 +14,7 @@
|
|
14
14
|
namespace duckdb {
|
15
15
|
class LogicalOperator;
|
16
16
|
class LogicalGet;
|
17
|
+
class LogicalLimit;
|
17
18
|
class Optimizer;
|
18
19
|
|
19
20
|
//! Transform
|
@@ -34,7 +35,7 @@ private:
|
|
34
35
|
void ReplaceTableReferences(Expression &expr, idx_t new_table_index);
|
35
36
|
unique_ptr<Expression> GetExpression(LogicalOperator &op, idx_t column_index);
|
36
37
|
void ReplaceExpressionReferences(LogicalOperator &next_op, unique_ptr<Expression> &expr);
|
37
|
-
bool OptimizeLargeLimit(
|
38
|
+
bool OptimizeLargeLimit(LogicalLimit &limit, idx_t limit_val, bool has_offset);
|
38
39
|
|
39
40
|
private:
|
40
41
|
Optimizer &optimizer;
|
@@ -29,14 +29,20 @@ struct OptimizerExtensionInput {
|
|
29
29
|
};
|
30
30
|
|
31
31
|
typedef void (*optimize_function_t)(OptimizerExtensionInput &input, unique_ptr<LogicalOperator> &plan);
|
32
|
+
typedef void (*pre_optimize_function_t)(OptimizerExtensionInput &input, unique_ptr<LogicalOperator> &plan);
|
32
33
|
|
33
34
|
class OptimizerExtension {
|
34
35
|
public:
|
35
|
-
//! The
|
36
|
-
//! Takes a query
|
37
|
-
|
38
|
-
|
39
|
-
//!
|
36
|
+
//! The optimize function of the optimizer extension.
|
37
|
+
//! Takes a logical query plan as an input, which it can modify in place
|
38
|
+
//! This runs, after the DuckDB optimizers have run
|
39
|
+
optimize_function_t optimize_function = nullptr;
|
40
|
+
//! The pre-optimize function of the optimizer extension.
|
41
|
+
//! Takes a logical query plan as an input, which it can modify in place
|
42
|
+
//! This runs, before the DuckDB optimizers have run
|
43
|
+
pre_optimize_function_t pre_optimize_function = nullptr;
|
44
|
+
|
45
|
+
//! Additional optimizer info passed to the optimize functions
|
40
46
|
shared_ptr<OptimizerExtensionInfo> optimizer_info;
|
41
47
|
};
|
42
48
|
|
@@ -9,10 +9,10 @@
|
|
9
9
|
#pragma once
|
10
10
|
|
11
11
|
#include "duckdb/parallel/task.hpp"
|
12
|
-
#include "duckdb/parallel/event.hpp"
|
13
12
|
#include "duckdb/common/optional_ptr.hpp"
|
14
13
|
|
15
14
|
namespace duckdb {
|
15
|
+
class Event;
|
16
16
|
class PhysicalOperator;
|
17
17
|
class ThreadContext;
|
18
18
|
|
@@ -34,6 +34,9 @@ public:
|
|
34
34
|
unique_ptr<ThreadContext> thread_context;
|
35
35
|
optional_ptr<const PhysicalOperator> op;
|
36
36
|
|
37
|
+
private:
|
38
|
+
ClientContext &context;
|
39
|
+
|
37
40
|
public:
|
38
41
|
virtual TaskExecutionResult ExecuteTask(TaskExecutionMode mode) = 0;
|
39
42
|
TaskExecutionResult Execute(TaskExecutionMode mode) override;
|
@@ -10,6 +10,7 @@
|
|
10
10
|
|
11
11
|
#include "duckdb/common/common.hpp"
|
12
12
|
#include "duckdb/common/atomic.hpp"
|
13
|
+
#include "duckdb/common/optional_ptr.hpp"
|
13
14
|
#include "duckdb/parallel/task.hpp"
|
14
15
|
#include "duckdb/execution/task_error_manager.hpp"
|
15
16
|
|
@@ -47,6 +48,8 @@ private:
|
|
47
48
|
unique_ptr<ProducerToken> token;
|
48
49
|
atomic<idx_t> completed_tasks;
|
49
50
|
atomic<idx_t> total_tasks;
|
51
|
+
friend class BaseExecutorTask;
|
52
|
+
optional_ptr<ClientContext> context;
|
50
53
|
};
|
51
54
|
|
52
55
|
class BaseExecutorTask : public Task {
|
@@ -0,0 +1,27 @@
|
|
1
|
+
//===----------------------------------------------------------------------===//
|
2
|
+
// DuckDB
|
3
|
+
//
|
4
|
+
// duckdb/parallel/task_notifier.hpp
|
5
|
+
//
|
6
|
+
//
|
7
|
+
//===----------------------------------------------------------------------===//
|
8
|
+
|
9
|
+
#pragma once
|
10
|
+
|
11
|
+
#include "duckdb/common/optional_ptr.hpp"
|
12
|
+
|
13
|
+
namespace duckdb {
|
14
|
+
class ClientContext;
|
15
|
+
|
16
|
+
//! The TaskNotifier notifies ClientContextState listener about started / stopped tasks
|
17
|
+
class TaskNotifier {
|
18
|
+
public:
|
19
|
+
explicit TaskNotifier(optional_ptr<ClientContext> context_p);
|
20
|
+
|
21
|
+
~TaskNotifier();
|
22
|
+
|
23
|
+
private:
|
24
|
+
optional_ptr<ClientContext> context;
|
25
|
+
};
|
26
|
+
|
27
|
+
} // namespace duckdb
|
@@ -68,6 +68,10 @@ public:
|
|
68
68
|
//! Returns the number of threads
|
69
69
|
DUCKDB_API int32_t NumberOfThreads();
|
70
70
|
|
71
|
+
idx_t GetNumberOfTasks() const;
|
72
|
+
idx_t GetProducerCount() const;
|
73
|
+
idx_t GetTaskCountForProducer(ProducerToken &token) const;
|
74
|
+
|
71
75
|
//! Send signals to n threads, signalling for them to wake up and attempt to execute a task
|
72
76
|
void Signal(idx_t n);
|
73
77
|
|
@@ -40,7 +40,7 @@ public:
|
|
40
40
|
vector<LogicalType> child_types;
|
41
41
|
//! The target LogicalType of the subquery result (i.e. to which type it should be casted, if child_type <>
|
42
42
|
//! child_target). Only used for ANY expressions.
|
43
|
-
LogicalType
|
43
|
+
vector<LogicalType> child_targets;
|
44
44
|
|
45
45
|
public:
|
46
46
|
bool HasSubquery() const override {
|
@@ -23,7 +23,7 @@ class TableStatistics;
|
|
23
23
|
//! Abstraction will support, for example: tiering, versioning, or splitting into multiple block managers.
|
24
24
|
class TableDataWriter {
|
25
25
|
public:
|
26
|
-
explicit TableDataWriter(TableCatalogEntry &table);
|
26
|
+
explicit TableDataWriter(TableCatalogEntry &table, optional_ptr<ClientContext> client_context);
|
27
27
|
virtual ~TableDataWriter();
|
28
28
|
|
29
29
|
public:
|
@@ -39,9 +39,11 @@ public:
|
|
39
39
|
|
40
40
|
TaskScheduler &GetScheduler();
|
41
41
|
DatabaseInstance &GetDatabase();
|
42
|
+
optional_ptr<ClientContext> GetClientContext();
|
42
43
|
|
43
44
|
protected:
|
44
45
|
DuckTableEntry &table;
|
46
|
+
optional_ptr<ClientContext> client_context;
|
45
47
|
//! Pointers to the start of each row group.
|
46
48
|
vector<RowGroupPointer> row_group_pointers;
|
47
49
|
};
|
@@ -97,7 +97,8 @@ class SingleFileCheckpointWriter final : public CheckpointWriter {
|
|
97
97
|
friend class SingleFileTableDataWriter;
|
98
98
|
|
99
99
|
public:
|
100
|
-
SingleFileCheckpointWriter(
|
100
|
+
SingleFileCheckpointWriter(optional_ptr<ClientContext> client_context, AttachedDatabase &db,
|
101
|
+
BlockManager &block_manager, CheckpointType checkpoint_type);
|
101
102
|
|
102
103
|
//! Checkpoint the current state of the WAL and flush it to the main storage. This should be called BEFORE any
|
103
104
|
//! connection is available because right now the checkpointing cannot be done online. (TODO)
|
@@ -112,10 +113,15 @@ public:
|
|
112
113
|
return checkpoint_type;
|
113
114
|
}
|
114
115
|
|
116
|
+
optional_ptr<ClientContext> GetClientContext() const {
|
117
|
+
return client_context;
|
118
|
+
}
|
119
|
+
|
115
120
|
public:
|
116
121
|
void WriteTable(TableCatalogEntry &table, Serializer &serializer) override;
|
117
122
|
|
118
123
|
private:
|
124
|
+
optional_ptr<ClientContext> client_context;
|
119
125
|
//! The metadata writer is responsible for writing schema information
|
120
126
|
unique_ptr<MetadataWriter> metadata_writer;
|
121
127
|
//! The table data writer is responsible for writing the DataPointers used by the table chunks
|
@@ -100,7 +100,8 @@ public:
|
|
100
100
|
virtual bool AutomaticCheckpoint(idx_t estimated_wal_bytes) = 0;
|
101
101
|
virtual unique_ptr<StorageCommitState> GenStorageCommitState(WriteAheadLog &wal) = 0;
|
102
102
|
virtual bool IsCheckpointClean(MetaBlockPointer checkpoint_id) = 0;
|
103
|
-
virtual void CreateCheckpoint(
|
103
|
+
virtual void CreateCheckpoint(optional_ptr<ClientContext> client_context,
|
104
|
+
CheckpointOptions options = CheckpointOptions()) = 0;
|
104
105
|
virtual DatabaseSize GetDatabaseSize() = 0;
|
105
106
|
virtual vector<MetadataBlockInfo> GetMetadataInfo() = 0;
|
106
107
|
virtual shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) = 0;
|
@@ -159,7 +160,7 @@ public:
|
|
159
160
|
bool AutomaticCheckpoint(idx_t estimated_wal_bytes) override;
|
160
161
|
unique_ptr<StorageCommitState> GenStorageCommitState(WriteAheadLog &wal) override;
|
161
162
|
bool IsCheckpointClean(MetaBlockPointer checkpoint_id) override;
|
162
|
-
void CreateCheckpoint(CheckpointOptions options) override;
|
163
|
+
void CreateCheckpoint(optional_ptr<ClientContext> client_context, CheckpointOptions options) override;
|
163
164
|
DatabaseSize GetDatabaseSize() override;
|
164
165
|
vector<MetadataBlockInfo> GetMetadataInfo() override;
|
165
166
|
shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) override;
|