duckdb 0.8.2-dev2700.0 → 0.8.2-dev2842.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/duckdb/extension/icu/icu-makedate.cpp +12 -6
- package/src/duckdb/extension/json/include/json_deserializer.hpp +1 -1
- package/src/duckdb/extension/json/include/json_serializer.hpp +1 -1
- package/src/duckdb/extension/json/json_deserializer.cpp +10 -10
- package/src/duckdb/extension/json/json_scan.cpp +2 -2
- package/src/duckdb/extension/json/json_serializer.cpp +11 -10
- package/src/duckdb/extension/json/serialize_json.cpp +44 -44
- package/src/duckdb/extension/parquet/parquet_extension.cpp +11 -10
- package/src/duckdb/extension/parquet/serialize_parquet.cpp +6 -6
- package/src/duckdb/src/common/adbc/adbc.cpp +52 -21
- package/src/duckdb/src/common/adbc/driver_manager.cpp +12 -2
- package/src/duckdb/src/common/enum_util.cpp +5 -0
- package/src/duckdb/src/common/extra_type_info.cpp +2 -2
- package/src/duckdb/src/common/serializer/binary_deserializer.cpp +5 -3
- package/src/duckdb/src/common/serializer/binary_serializer.cpp +10 -5
- package/src/duckdb/src/common/types/column/column_data_collection.cpp +4 -4
- package/src/duckdb/src/common/types/row/row_data_collection_scanner.cpp +35 -5
- package/src/duckdb/src/common/types/value.cpp +33 -33
- package/src/duckdb/src/common/types/vector.cpp +20 -20
- package/src/duckdb/src/core_functions/aggregate/holistic/approximate_quantile.cpp +2 -2
- package/src/duckdb/src/core_functions/aggregate/holistic/quantile.cpp +6 -6
- package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +4 -4
- package/src/duckdb/src/core_functions/scalar/list/list_lambdas.cpp +4 -4
- package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +283 -91
- package/src/duckdb/src/execution/operator/filter/physical_filter.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -2
- package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +1 -1
- package/src/duckdb/src/execution/physical_plan_generator.cpp +1 -6
- package/src/duckdb/src/execution/window_executor.cpp +10 -1
- package/src/duckdb/src/function/table/read_csv.cpp +4 -4
- package/src/duckdb/src/function/table/table_scan.cpp +14 -14
- package/src/duckdb/src/function/table/version/pragma_version.cpp +5 -2
- package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/enums/pending_execution_result.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/index_vector.hpp +2 -2
- package/src/duckdb/src/include/duckdb/common/serializer/binary_deserializer.hpp +7 -3
- package/src/duckdb/src/include/duckdb/common/serializer/binary_serializer.hpp +2 -1
- package/src/duckdb/src/include/duckdb/common/serializer/format_deserializer.hpp +18 -17
- package/src/duckdb/src/include/duckdb/common/serializer/format_serializer.hpp +10 -9
- package/src/duckdb/src/include/duckdb/common/serializer/serialization_traits.hpp +4 -0
- package/src/duckdb/src/include/duckdb/common/types/row/row_data_collection_scanner.hpp +5 -1
- package/src/duckdb/src/include/duckdb/execution/physical_operator.hpp +0 -2
- package/src/duckdb/src/include/duckdb/function/function_serialization.hpp +10 -10
- package/src/duckdb/src/include/duckdb/main/pending_query_result.hpp +5 -0
- package/src/duckdb/src/include/duckdb/main/relation/aggregate_relation.hpp +4 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +37 -63
- package/src/duckdb/src/include/duckdb/optimizer/join_order/cost_model.hpp +37 -0
- package/src/duckdb/src/include/duckdb/optimizer/join_order/join_node.hpp +14 -29
- package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +7 -21
- package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +0 -11
- package/src/duckdb/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp +89 -0
- package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +17 -31
- package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph_manager.hpp +113 -0
- package/src/duckdb/src/include/duckdb/optimizer/join_order/relation_manager.hpp +73 -0
- package/src/duckdb/src/include/duckdb/optimizer/join_order/relation_statistics_helper.hpp +73 -0
- package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +4 -1
- package/src/duckdb/src/include/duckdb/parser/group_by_node.hpp +11 -0
- package/src/duckdb/src/include/duckdb/parser/parser.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/logical_operator.hpp +0 -2
- package/src/duckdb/src/include/duckdb.h +11 -1
- package/src/duckdb/src/main/capi/pending-c.cpp +17 -0
- package/src/duckdb/src/main/pending_query_result.cpp +9 -1
- package/src/duckdb/src/main/relation/aggregate_relation.cpp +20 -10
- package/src/duckdb/src/main/relation.cpp +4 -4
- package/src/duckdb/src/optimizer/join_order/cardinality_estimator.cpp +79 -325
- package/src/duckdb/src/optimizer/join_order/cost_model.cpp +19 -0
- package/src/duckdb/src/optimizer/join_order/join_node.cpp +5 -37
- package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +48 -1078
- package/src/duckdb/src/optimizer/join_order/plan_enumerator.cpp +552 -0
- package/src/duckdb/src/optimizer/join_order/query_graph.cpp +32 -29
- package/src/duckdb/src/optimizer/join_order/query_graph_manager.cpp +409 -0
- package/src/duckdb/src/optimizer/join_order/relation_manager.cpp +356 -0
- package/src/duckdb/src/optimizer/join_order/relation_statistics_helper.cpp +351 -0
- package/src/duckdb/src/parallel/executor.cpp +6 -0
- package/src/duckdb/src/parallel/task_scheduler.cpp +7 -0
- package/src/duckdb/src/parser/parser.cpp +18 -3
- package/src/duckdb/src/parser/tableref/pivotref.cpp +6 -6
- package/src/duckdb/src/planner/binder/statement/bind_execute.cpp +1 -1
- package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +10 -10
- package/src/duckdb/src/planner/expression/bound_function_expression.cpp +6 -6
- package/src/duckdb/src/planner/expression/bound_window_expression.cpp +24 -24
- package/src/duckdb/src/planner/operator/logical_extension_operator.cpp +2 -2
- package/src/duckdb/src/planner/operator/logical_get.cpp +26 -22
- package/src/duckdb/src/storage/serialization/serialize_constraint.cpp +26 -26
- package/src/duckdb/src/storage/serialization/serialize_create_info.cpp +66 -66
- package/src/duckdb/src/storage/serialization/serialize_expression.cpp +78 -78
- package/src/duckdb/src/storage/serialization/serialize_logical_operator.cpp +250 -250
- package/src/duckdb/src/storage/serialization/serialize_macro_function.cpp +10 -10
- package/src/duckdb/src/storage/serialization/serialize_nodes.cpp +206 -206
- package/src/duckdb/src/storage/serialization/serialize_parse_info.cpp +116 -116
- package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +110 -110
- package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +48 -48
- package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +16 -16
- package/src/duckdb/src/storage/serialization/serialize_statement.cpp +2 -2
- package/src/duckdb/src/storage/serialization/serialize_table_filter.cpp +10 -10
- package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +54 -54
- package/src/duckdb/src/storage/serialization/serialize_types.cpp +22 -22
- package/src/duckdb/src/storage/table/update_segment.cpp +1 -1
- package/src/duckdb/ub_src_optimizer_join_order.cpp +10 -0
@@ -1,8 +1,8 @@
|
|
1
1
|
#ifndef DUCKDB_VERSION
|
2
|
-
#define DUCKDB_VERSION "0.8.2-
|
2
|
+
#define DUCKDB_VERSION "0.8.2-dev2842"
|
3
3
|
#endif
|
4
4
|
#ifndef DUCKDB_SOURCE_ID
|
5
|
-
#define DUCKDB_SOURCE_ID "
|
5
|
+
#define DUCKDB_SOURCE_ID "6421a36e94"
|
6
6
|
#endif
|
7
7
|
#include "duckdb/function/table/system_functions.hpp"
|
8
8
|
#include "duckdb/main/database.hpp"
|
@@ -95,6 +95,9 @@ string DuckDB::Platform() {
|
|
95
95
|
postfix = "_gcc4";
|
96
96
|
}
|
97
97
|
#endif
|
98
|
+
#if defined(__ANDROID__)
|
99
|
+
postfix += "_android"; // using + because it may also be gcc4
|
100
|
+
#endif
|
98
101
|
#ifdef __MINGW32__
|
99
102
|
postfix = "_mingw";
|
100
103
|
#endif
|
@@ -12,6 +12,6 @@
|
|
12
12
|
|
13
13
|
namespace duckdb {
|
14
14
|
|
15
|
-
enum class PendingExecutionResult : uint8_t { RESULT_READY, RESULT_NOT_READY, EXECUTION_ERROR };
|
15
|
+
enum class PendingExecutionResult : uint8_t { RESULT_READY, RESULT_NOT_READY, EXECUTION_ERROR, NO_TASKS_AVAILABLE };
|
16
16
|
|
17
17
|
} // namespace duckdb
|
@@ -62,12 +62,12 @@ public:
|
|
62
62
|
}
|
63
63
|
|
64
64
|
void FormatSerialize(FormatSerializer &serializer) const {
|
65
|
-
serializer.WriteProperty("internal_vector", internal_vector);
|
65
|
+
serializer.WriteProperty(100, "internal_vector", internal_vector);
|
66
66
|
}
|
67
67
|
|
68
68
|
static IndexVector<T, INDEX_TYPE> FormatDeserialize(FormatDeserializer &deserializer) {
|
69
69
|
IndexVector<T, INDEX_TYPE> result;
|
70
|
-
deserializer.ReadProperty("internal_vector", result.internal_vector);
|
70
|
+
deserializer.ReadProperty(100, "internal_vector", result.internal_vector);
|
71
71
|
return result;
|
72
72
|
}
|
73
73
|
|
@@ -45,13 +45,17 @@ private:
|
|
45
45
|
struct State {
|
46
46
|
uint32_t expected_field_count;
|
47
47
|
idx_t expected_size;
|
48
|
+
field_id_t expected_field_id;
|
48
49
|
uint32_t read_field_count;
|
49
|
-
|
50
|
-
|
50
|
+
|
51
|
+
State(uint32_t expected_field_count, idx_t expected_size, field_id_t expected_field_id)
|
52
|
+
: expected_field_count(expected_field_count), expected_size(expected_size),
|
53
|
+
expected_field_id(expected_field_id), read_field_count(0) {
|
51
54
|
}
|
52
55
|
};
|
53
56
|
|
54
57
|
const char *current_tag = nullptr;
|
58
|
+
field_id_t current_field_id = 0;
|
55
59
|
data_ptr_t ptr;
|
56
60
|
data_ptr_t end_ptr;
|
57
61
|
vector<State> stack;
|
@@ -72,7 +76,7 @@ private:
|
|
72
76
|
}
|
73
77
|
|
74
78
|
// Set the 'tag' of the property to read
|
75
|
-
void SetTag(const char *tag) final;
|
79
|
+
void SetTag(const field_id_t field_id, const char *tag) final;
|
76
80
|
|
77
81
|
//===--------------------------------------------------------------------===//
|
78
82
|
// Nested Types Hooks
|
@@ -24,6 +24,7 @@ private:
|
|
24
24
|
};
|
25
25
|
|
26
26
|
const char *current_tag;
|
27
|
+
field_id_t current_field_id = 0;
|
27
28
|
|
28
29
|
vector<data_t> data;
|
29
30
|
vector<State> stack;
|
@@ -55,7 +56,7 @@ public:
|
|
55
56
|
return std::move(serializer.data);
|
56
57
|
}
|
57
58
|
|
58
|
-
void SetTag(const char *tag) final;
|
59
|
+
void SetTag(const field_id_t field_id, const char *tag) final;
|
59
60
|
|
60
61
|
//===--------------------------------------------------------------------===//
|
61
62
|
// Nested Types Hooks
|
@@ -30,22 +30,22 @@ protected:
|
|
30
30
|
public:
|
31
31
|
// Read into an existing value
|
32
32
|
template <typename T>
|
33
|
-
inline void ReadProperty(const char *tag, T &ret) {
|
34
|
-
SetTag(tag);
|
33
|
+
inline void ReadProperty(const field_id_t field_id, const char *tag, T &ret) {
|
34
|
+
SetTag(field_id, tag);
|
35
35
|
ret = Read<T>();
|
36
36
|
}
|
37
37
|
|
38
38
|
// Read and return a value
|
39
39
|
template <typename T>
|
40
|
-
inline T ReadProperty(const char *tag) {
|
41
|
-
SetTag(tag);
|
40
|
+
inline T ReadProperty(const field_id_t field_id, const char *tag) {
|
41
|
+
SetTag(field_id, tag);
|
42
42
|
return Read<T>();
|
43
43
|
}
|
44
44
|
|
45
45
|
// Read optional property and return a value, or forward a default value
|
46
46
|
template <typename T>
|
47
|
-
inline T ReadOptionalPropertyOrDefault(const char *tag, T &&default_value) {
|
48
|
-
SetTag(tag);
|
47
|
+
inline T ReadOptionalPropertyOrDefault(const field_id_t field_id, const char *tag, T &&default_value) {
|
48
|
+
SetTag(field_id, tag);
|
49
49
|
auto present = OnOptionalBegin();
|
50
50
|
if (present) {
|
51
51
|
auto item = Read<T>();
|
@@ -59,8 +59,8 @@ public:
|
|
59
59
|
|
60
60
|
// Read optional property into an existing value, or use a default value
|
61
61
|
template <typename T>
|
62
|
-
inline void ReadOptionalPropertyOrDefault(const char *tag, T &ret, T &&default_value) {
|
63
|
-
SetTag(tag);
|
62
|
+
inline void ReadOptionalPropertyOrDefault(const field_id_t field_id, const char *tag, T &ret, T &&default_value) {
|
63
|
+
SetTag(field_id, tag);
|
64
64
|
auto present = OnOptionalBegin();
|
65
65
|
if (present) {
|
66
66
|
ret = Read<T>();
|
@@ -74,8 +74,8 @@ public:
|
|
74
74
|
// Read optional property and return a value, or default construct it
|
75
75
|
template <typename T>
|
76
76
|
inline typename std::enable_if<std::is_default_constructible<T>::value, T>::type
|
77
|
-
ReadOptionalProperty(const char *tag) {
|
78
|
-
SetTag(tag);
|
77
|
+
ReadOptionalProperty(const field_id_t field_id, const char *tag) {
|
78
|
+
SetTag(field_id, tag);
|
79
79
|
auto present = OnOptionalBegin();
|
80
80
|
if (present) {
|
81
81
|
auto item = Read<T>();
|
@@ -90,8 +90,8 @@ public:
|
|
90
90
|
// Read optional property into an existing value, or default construct it
|
91
91
|
template <typename T>
|
92
92
|
inline typename std::enable_if<std::is_default_constructible<T>::value, void>::type
|
93
|
-
ReadOptionalProperty(const char *tag, T &ret) {
|
94
|
-
SetTag(tag);
|
93
|
+
ReadOptionalProperty(const field_id_t field_id, const char *tag, T &ret) {
|
94
|
+
SetTag(field_id, tag);
|
95
95
|
auto present = OnOptionalBegin();
|
96
96
|
if (present) {
|
97
97
|
ret = Read<T>();
|
@@ -104,8 +104,8 @@ public:
|
|
104
104
|
|
105
105
|
// Special case:
|
106
106
|
// Read into an existing data_ptr_t
|
107
|
-
inline void ReadProperty(const char *tag, data_ptr_t ret, idx_t count) {
|
108
|
-
SetTag(tag);
|
107
|
+
inline void ReadProperty(const field_id_t field_id, const char *tag, data_ptr_t ret, idx_t count) {
|
108
|
+
SetTag(field_id, tag);
|
109
109
|
ReadDataPtr(ret, count);
|
110
110
|
}
|
111
111
|
|
@@ -128,8 +128,8 @@ public:
|
|
128
128
|
}
|
129
129
|
|
130
130
|
// Manually begin an object - should be followed by EndObject
|
131
|
-
void BeginObject(const char *tag) {
|
132
|
-
SetTag(tag);
|
131
|
+
void BeginObject(const field_id_t field_id, const char *tag) {
|
132
|
+
SetTag(field_id, tag);
|
133
133
|
OnObjectBegin();
|
134
134
|
}
|
135
135
|
|
@@ -392,7 +392,8 @@ private:
|
|
392
392
|
}
|
393
393
|
|
394
394
|
protected:
|
395
|
-
virtual void SetTag(const char *tag) {
|
395
|
+
virtual void SetTag(const field_id_t field_id, const char *tag) {
|
396
|
+
(void)field_id;
|
396
397
|
(void)tag;
|
397
398
|
}
|
398
399
|
|
@@ -28,15 +28,15 @@ protected:
|
|
28
28
|
public:
|
29
29
|
// Serialize a value
|
30
30
|
template <class T>
|
31
|
-
void WriteProperty(const char *tag, const T &value) {
|
32
|
-
SetTag(tag);
|
31
|
+
void WriteProperty(const field_id_t field_id, const char *tag, const T &value) {
|
32
|
+
SetTag(field_id, tag);
|
33
33
|
WriteValue(value);
|
34
34
|
}
|
35
35
|
|
36
36
|
// Optional pointer
|
37
37
|
template <class POINTER>
|
38
|
-
void WriteOptionalProperty(const char *tag, POINTER &&ptr) {
|
39
|
-
SetTag(tag);
|
38
|
+
void WriteOptionalProperty(const field_id_t field_id, const char *tag, POINTER &&ptr) {
|
39
|
+
SetTag(field_id, tag);
|
40
40
|
if (ptr == nullptr) {
|
41
41
|
OnOptionalBegin(false);
|
42
42
|
OnOptionalEnd(false);
|
@@ -48,14 +48,14 @@ public:
|
|
48
48
|
}
|
49
49
|
|
50
50
|
// Special case: data_ptr_T
|
51
|
-
void WriteProperty(const char *tag, const_data_ptr_t ptr, idx_t count) {
|
52
|
-
SetTag(tag);
|
51
|
+
void WriteProperty(const field_id_t field_id, const char *tag, const_data_ptr_t ptr, idx_t count) {
|
52
|
+
SetTag(field_id, tag);
|
53
53
|
WriteDataPtr(ptr, count);
|
54
54
|
}
|
55
55
|
|
56
56
|
// Manually begin an object - should be followed by EndObject
|
57
|
-
void BeginObject(const char *tag) {
|
58
|
-
SetTag(tag);
|
57
|
+
void BeginObject(const field_id_t field_id, const char *tag) {
|
58
|
+
SetTag(field_id, tag);
|
59
59
|
OnObjectBegin();
|
60
60
|
}
|
61
61
|
|
@@ -196,7 +196,8 @@ protected:
|
|
196
196
|
}
|
197
197
|
|
198
198
|
// Handle setting a "tag" (optional)
|
199
|
-
virtual void SetTag(const char *tag) {
|
199
|
+
virtual void SetTag(const field_id_t field_id, const char *tag) {
|
200
|
+
(void)field_id;
|
200
201
|
(void)tag;
|
201
202
|
}
|
202
203
|
|
@@ -1,5 +1,7 @@
|
|
1
1
|
#pragma once
|
2
2
|
#include <type_traits>
|
3
|
+
#include <cstdint>
|
4
|
+
|
3
5
|
#include "duckdb/common/vector.hpp"
|
4
6
|
#include "duckdb/common/unordered_map.hpp"
|
5
7
|
#include "duckdb/common/unordered_set.hpp"
|
@@ -10,6 +12,8 @@ namespace duckdb {
|
|
10
12
|
class FormatSerializer; // Forward declare
|
11
13
|
class FormatDeserializer; // Forward declare
|
12
14
|
|
15
|
+
typedef uint32_t field_id_t;
|
16
|
+
|
13
17
|
// Backport to c++11
|
14
18
|
template <class...>
|
15
19
|
using void_t = void;
|
@@ -50,6 +50,10 @@ public:
|
|
50
50
|
RowDataCollectionScanner(RowDataCollection &rows, RowDataCollection &heap, const RowLayout &layout, bool external,
|
51
51
|
bool flush = true);
|
52
52
|
|
53
|
+
// Single block scan
|
54
|
+
RowDataCollectionScanner(RowDataCollection &rows, RowDataCollection &heap, const RowLayout &layout, bool external,
|
55
|
+
idx_t block_idx, bool flush);
|
56
|
+
|
53
57
|
//! The type layout of the payload
|
54
58
|
inline const vector<LogicalType> &GetTypes() const {
|
55
59
|
return layout.GetTypes();
|
@@ -93,7 +97,7 @@ private:
|
|
93
97
|
//! Read state
|
94
98
|
ScanState read_state;
|
95
99
|
//! The total count of sorted_data
|
96
|
-
|
100
|
+
idx_t total_count;
|
97
101
|
//! The number of rows scanned so far
|
98
102
|
idx_t total_scanned;
|
99
103
|
//! Addresses used to gather from the sorted data
|
@@ -36,7 +36,6 @@ public:
|
|
36
36
|
public:
|
37
37
|
PhysicalOperator(PhysicalOperatorType type, vector<LogicalType> types, idx_t estimated_cardinality)
|
38
38
|
: type(type), types(std::move(types)), estimated_cardinality(estimated_cardinality) {
|
39
|
-
estimated_props = make_uniq<EstimatedProperties>(estimated_cardinality, 0);
|
40
39
|
}
|
41
40
|
|
42
41
|
virtual ~PhysicalOperator() {
|
@@ -50,7 +49,6 @@ public:
|
|
50
49
|
vector<LogicalType> types;
|
51
50
|
//! The estimated cardinality of this physical operator
|
52
51
|
idx_t estimated_cardinality;
|
53
|
-
unique_ptr<EstimatedProperties> estimated_props;
|
54
52
|
|
55
53
|
//! The global sink state of this operator
|
56
54
|
unique_ptr<GlobalSinkState> sink_state;
|
@@ -104,13 +104,13 @@ public:
|
|
104
104
|
static void FormatSerialize(FormatSerializer &serializer, const FUNC &function,
|
105
105
|
optional_ptr<FunctionData> bind_info) {
|
106
106
|
D_ASSERT(!function.name.empty());
|
107
|
-
serializer.WriteProperty("name", function.name);
|
108
|
-
serializer.WriteProperty("arguments", function.arguments);
|
109
|
-
serializer.WriteProperty("original_arguments", function.original_arguments);
|
107
|
+
serializer.WriteProperty(500, "name", function.name);
|
108
|
+
serializer.WriteProperty(501, "arguments", function.arguments);
|
109
|
+
serializer.WriteProperty(502, "original_arguments", function.original_arguments);
|
110
110
|
bool has_serialize = function.format_serialize;
|
111
|
-
serializer.WriteProperty("has_serialize", has_serialize);
|
111
|
+
serializer.WriteProperty(503, "has_serialize", has_serialize);
|
112
112
|
if (has_serialize) {
|
113
|
-
serializer.BeginObject("function_data");
|
113
|
+
serializer.BeginObject(504, "function_data");
|
114
114
|
function.format_serialize(serializer, bind_info, function);
|
115
115
|
serializer.EndObject();
|
116
116
|
D_ASSERT(function.format_deserialize);
|
@@ -135,12 +135,12 @@ public:
|
|
135
135
|
template <class FUNC, class CATALOG_ENTRY>
|
136
136
|
static pair<FUNC, bool> FormatDeserializeBase(FormatDeserializer &deserializer, CatalogType catalog_type) {
|
137
137
|
auto &context = deserializer.Get<ClientContext &>();
|
138
|
-
auto name = deserializer.ReadProperty<string>("name");
|
139
|
-
auto arguments = deserializer.ReadProperty<vector<LogicalType>>("arguments");
|
140
|
-
auto original_arguments = deserializer.ReadProperty<vector<LogicalType>>("original_arguments");
|
138
|
+
auto name = deserializer.ReadProperty<string>(500, "name");
|
139
|
+
auto arguments = deserializer.ReadProperty<vector<LogicalType>>(501, "arguments");
|
140
|
+
auto original_arguments = deserializer.ReadProperty<vector<LogicalType>>(502, "original_arguments");
|
141
141
|
auto function = DeserializeFunction<FUNC, CATALOG_ENTRY>(context, catalog_type, name, std::move(arguments),
|
142
142
|
std::move(original_arguments));
|
143
|
-
auto has_serialize = deserializer.ReadProperty<bool>("has_serialize");
|
143
|
+
auto has_serialize = deserializer.ReadProperty<bool>(503, "has_serialize");
|
144
144
|
return make_pair(std::move(function), has_serialize);
|
145
145
|
}
|
146
146
|
|
@@ -150,7 +150,7 @@ public:
|
|
150
150
|
throw SerializationException("Function requires deserialization but no deserialization function for %s",
|
151
151
|
function.name);
|
152
152
|
}
|
153
|
-
deserializer.BeginObject("function_data");
|
153
|
+
deserializer.BeginObject(504, "function_data");
|
154
154
|
auto result = function.format_deserialize(deserializer, function);
|
155
155
|
deserializer.EndObject();
|
156
156
|
return result;
|
@@ -34,6 +34,8 @@ public:
|
|
34
34
|
//! If this returns RESULT_READY, the Execute function can be called to obtain a pointer to the result.
|
35
35
|
//! If this returns RESULT_NOT_READY, the ExecuteTask function should be called again.
|
36
36
|
//! If this returns EXECUTION_ERROR, an error occurred during execution.
|
37
|
+
//! If this returns NO_TASKS_AVAILABLE, this means currently no meaningful work can be done by the current executor,
|
38
|
+
//! but tasks may become available in the future.
|
37
39
|
//! The error message can be obtained by calling GetError() on the PendingQueryResult.
|
38
40
|
DUCKDB_API PendingExecutionResult ExecuteTask();
|
39
41
|
|
@@ -43,6 +45,9 @@ public:
|
|
43
45
|
|
44
46
|
DUCKDB_API void Close();
|
45
47
|
|
48
|
+
//! Function to determine whether execution is considered finished
|
49
|
+
DUCKDB_API static bool IsFinished(PendingExecutionResult result);
|
50
|
+
|
46
51
|
private:
|
47
52
|
shared_ptr<ClientContext> context;
|
48
53
|
bool allow_stream_result;
|
@@ -10,17 +10,20 @@
|
|
10
10
|
|
11
11
|
#include "duckdb/main/relation.hpp"
|
12
12
|
#include "duckdb/parser/parsed_expression.hpp"
|
13
|
+
#include "duckdb/parser/group_by_node.hpp"
|
13
14
|
|
14
15
|
namespace duckdb {
|
15
16
|
|
16
17
|
class AggregateRelation : public Relation {
|
17
18
|
public:
|
18
19
|
DUCKDB_API AggregateRelation(shared_ptr<Relation> child, vector<unique_ptr<ParsedExpression>> expressions);
|
20
|
+
DUCKDB_API AggregateRelation(shared_ptr<Relation> child, vector<unique_ptr<ParsedExpression>> expressions,
|
21
|
+
GroupByNode groups);
|
19
22
|
DUCKDB_API AggregateRelation(shared_ptr<Relation> child, vector<unique_ptr<ParsedExpression>> expressions,
|
20
23
|
vector<unique_ptr<ParsedExpression>> groups);
|
21
24
|
|
22
25
|
vector<unique_ptr<ParsedExpression>> expressions;
|
23
|
-
|
26
|
+
GroupByNode groups;
|
24
27
|
vector<ColumnDefinition> columns;
|
25
28
|
shared_ptr<Relation> child;
|
26
29
|
|
@@ -7,21 +7,14 @@
|
|
7
7
|
//===----------------------------------------------------------------------===//
|
8
8
|
#pragma once
|
9
9
|
|
10
|
-
#include "duckdb/optimizer/join_order/join_node.hpp"
|
11
|
-
#include "duckdb/planner/column_binding.hpp"
|
12
10
|
#include "duckdb/planner/column_binding_map.hpp"
|
13
|
-
#include "duckdb/
|
14
|
-
|
11
|
+
#include "duckdb/optimizer/join_order/query_graph.hpp"
|
12
|
+
|
13
|
+
#include "duckdb/optimizer/join_order/relation_statistics_helper.hpp"
|
15
14
|
|
16
15
|
namespace duckdb {
|
17
16
|
|
18
|
-
struct
|
19
|
-
string original_name;
|
20
|
-
// the relation columns used in join filters
|
21
|
-
// Needed when iterating over columns and initializing total domain values.
|
22
|
-
unordered_set<idx_t> columns;
|
23
|
-
double cardinality;
|
24
|
-
};
|
17
|
+
struct FilterInfo;
|
25
18
|
|
26
19
|
struct RelationsToTDom {
|
27
20
|
//! column binding sets that are equivalent in a join plan.
|
@@ -33,19 +26,13 @@ struct RelationsToTDom {
|
|
33
26
|
idx_t tdom_no_hll;
|
34
27
|
bool has_tdom_hll;
|
35
28
|
vector<FilterInfo *> filters;
|
29
|
+
vector<string> column_names;
|
36
30
|
|
37
31
|
RelationsToTDom(const column_binding_set_t &column_binding_set)
|
38
32
|
: equivalent_relations(column_binding_set), tdom_hll(0), tdom_no_hll(NumericLimits<idx_t>::Maximum()),
|
39
33
|
has_tdom_hll(false) {};
|
40
34
|
};
|
41
35
|
|
42
|
-
struct NodeOp {
|
43
|
-
unique_ptr<JoinNode> node;
|
44
|
-
LogicalOperator &op;
|
45
|
-
|
46
|
-
NodeOp(unique_ptr<JoinNode> node, LogicalOperator &op) : node(std::move(node)), op(op) {};
|
47
|
-
};
|
48
|
-
|
49
36
|
struct Subgraph2Denominator {
|
50
37
|
unordered_set<idx_t> relations;
|
51
38
|
double denom;
|
@@ -53,69 +40,56 @@ struct Subgraph2Denominator {
|
|
53
40
|
Subgraph2Denominator() : relations(), denom(1) {};
|
54
41
|
};
|
55
42
|
|
56
|
-
class
|
43
|
+
class CardinalityHelper {
|
57
44
|
public:
|
58
|
-
|
45
|
+
CardinalityHelper() {
|
59
46
|
}
|
47
|
+
CardinalityHelper(double cardinality_before_filters, double filter_string)
|
48
|
+
: cardinality_before_filters(cardinality_before_filters), filter_strength(filter_string) {};
|
60
49
|
|
61
|
-
|
62
|
-
|
50
|
+
public:
|
51
|
+
double cardinality_before_filters;
|
52
|
+
double filter_strength;
|
53
|
+
|
54
|
+
vector<string> table_names_joined;
|
55
|
+
vector<string> column_names;
|
56
|
+
};
|
63
57
|
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
column_binding_map_t<ColumnBinding> relation_column_to_original_column;
|
58
|
+
class CardinalityEstimator {
|
59
|
+
public:
|
60
|
+
explicit CardinalityEstimator() {};
|
68
61
|
|
62
|
+
private:
|
69
63
|
vector<RelationsToTDom> relations_to_tdoms;
|
64
|
+
unordered_map<string, CardinalityHelper> relation_set_2_cardinality;
|
65
|
+
JoinRelationSetManager set_manager;
|
66
|
+
vector<RelationStats> relation_stats;
|
70
67
|
|
71
68
|
public:
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
//!
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
void
|
85
|
-
void
|
86
|
-
void AddRelationColumnMapping(LogicalGet &get, idx_t relation_id);
|
87
|
-
|
88
|
-
void InitTotalDomains();
|
89
|
-
void UpdateTotalDomains(JoinNode &node, LogicalOperator &op);
|
90
|
-
void InitEquivalentRelations(vector<unique_ptr<FilterInfo>> &filter_infos);
|
91
|
-
|
92
|
-
void InitCardinalityEstimatorProps(vector<NodeOp> &node_ops, vector<unique_ptr<FilterInfo>> &filter_infos);
|
93
|
-
double EstimateCardinalityWithSet(JoinRelationSet &new_set);
|
94
|
-
void EstimateBaseTableCardinality(JoinNode &node, LogicalOperator &op);
|
95
|
-
double EstimateCrossProduct(const JoinNode &left, const JoinNode &right);
|
96
|
-
static double ComputeCost(JoinNode &left, JoinNode &right, double expected_cardinality);
|
69
|
+
void RemoveEmptyTotalDomains();
|
70
|
+
void UpdateTotalDomains(optional_ptr<JoinRelationSet> set, RelationStats &stats);
|
71
|
+
void InitEquivalentRelations(const vector<unique_ptr<FilterInfo>> &filter_infos);
|
72
|
+
|
73
|
+
void InitCardinalityEstimatorProps(optional_ptr<JoinRelationSet> set, RelationStats &stats);
|
74
|
+
|
75
|
+
//! cost model needs estimated cardinalities to the fraction since the formula captures
|
76
|
+
//! distinct count selectivities and multiplicities. Hence the template
|
77
|
+
template <class T>
|
78
|
+
T EstimateCardinalityWithSet(JoinRelationSet &new_set);
|
79
|
+
|
80
|
+
//! used for debugging.
|
81
|
+
void AddRelationNamesToTdoms(vector<RelationStats> &stats);
|
82
|
+
void PrintRelationToTdomInfo();
|
97
83
|
|
98
84
|
private:
|
99
85
|
bool SingleColumnFilter(FilterInfo &filter_info);
|
100
|
-
//! Filter & bindings -> list of indexes into the equivalent_relations array.
|
101
|
-
// The column binding set at each index is an equivalence set.
|
102
86
|
vector<idx_t> DetermineMatchingEquivalentSets(FilterInfo *filter_info);
|
103
|
-
|
104
87
|
//! Given a filter, add the column bindings to the matching equivalent set at the index
|
105
88
|
//! given in matching equivalent sets.
|
106
89
|
//! If there are multiple equivalence sets, they are merged.
|
107
90
|
void AddToEquivalenceSets(FilterInfo *filter_info, vector<idx_t> matching_equivalent_sets);
|
108
|
-
|
109
|
-
optional_ptr<TableFilterSet> GetTableFilters(LogicalOperator &op, idx_t table_index);
|
110
|
-
|
111
91
|
void AddRelationTdom(FilterInfo &filter_info);
|
112
92
|
bool EmptyFilter(FilterInfo &filter_info);
|
113
|
-
|
114
|
-
idx_t InspectConjunctionAND(idx_t cardinality, idx_t column_index, ConjunctionAndFilter &fil,
|
115
|
-
unique_ptr<BaseStatistics> base_stats);
|
116
|
-
idx_t InspectConjunctionOR(idx_t cardinality, idx_t column_index, ConjunctionOrFilter &fil,
|
117
|
-
unique_ptr<BaseStatistics> base_stats);
|
118
|
-
idx_t InspectTableFilters(idx_t cardinality, LogicalOperator &op, TableFilterSet &table_filters, idx_t table_index);
|
119
93
|
};
|
120
94
|
|
121
95
|
} // namespace duckdb
|
@@ -0,0 +1,37 @@
|
|
1
|
+
//===----------------------------------------------------------------------===//
|
2
|
+
// DuckDB
|
3
|
+
//
|
4
|
+
// duckdb/optimizer/join_order/cost_model.hpp
|
5
|
+
//
|
6
|
+
//
|
7
|
+
//===----------------------------------------------------------------------===//
|
8
|
+
#pragma once
|
9
|
+
|
10
|
+
#include "duckdb/optimizer/join_order/join_node.hpp"
|
11
|
+
#include "duckdb/optimizer/join_order/cardinality_estimator.hpp"
|
12
|
+
|
13
|
+
namespace duckdb {
|
14
|
+
|
15
|
+
class QueryGraphManager;
|
16
|
+
|
17
|
+
class CostModel {
|
18
|
+
public:
|
19
|
+
CostModel(QueryGraphManager &query_graph_manager);
|
20
|
+
|
21
|
+
private:
|
22
|
+
//! query graph storing relation manager information
|
23
|
+
QueryGraphManager &query_graph_manager;
|
24
|
+
|
25
|
+
public:
|
26
|
+
void InitCostModel();
|
27
|
+
|
28
|
+
//! Compute cost of a join relation set
|
29
|
+
double ComputeCost(JoinNode &left, JoinNode &right);
|
30
|
+
|
31
|
+
//! Cardinality Estimator used to calculate cost
|
32
|
+
CardinalityEstimator cardinality_estimator;
|
33
|
+
|
34
|
+
private:
|
35
|
+
};
|
36
|
+
|
37
|
+
} // namespace duckdb
|
@@ -5,62 +5,47 @@
|
|
5
5
|
//
|
6
6
|
//
|
7
7
|
//===----------------------------------------------------------------------===//
|
8
|
-
|
9
8
|
#pragma once
|
10
9
|
|
11
|
-
#include "duckdb/common/unordered_map.hpp"
|
12
|
-
#include "duckdb/common/unordered_set.hpp"
|
13
|
-
#include "duckdb/optimizer/join_order/estimated_properties.hpp"
|
14
10
|
#include "duckdb/optimizer/join_order/join_relation.hpp"
|
15
11
|
#include "duckdb/optimizer/join_order/query_graph.hpp"
|
16
|
-
#include "duckdb/parser/expression_map.hpp"
|
17
|
-
#include "duckdb/planner/logical_operator_visitor.hpp"
|
18
|
-
#include "duckdb/planner/table_filter.hpp"
|
19
|
-
#include "duckdb/storage/statistics/distinct_statistics.hpp"
|
20
12
|
|
21
13
|
namespace duckdb {
|
22
14
|
|
23
|
-
|
15
|
+
struct NeighborInfo;
|
24
16
|
|
25
17
|
class JoinNode {
|
26
18
|
public:
|
27
19
|
//! Represents a node in the join plan
|
28
20
|
JoinRelationSet &set;
|
21
|
+
//! information on how left and right are connected
|
29
22
|
optional_ptr<NeighborInfo> info;
|
30
|
-
//!
|
31
|
-
//! estimated_props.cardinality will be the cardinality after filters. With no filters, the two are equal
|
32
|
-
bool has_filter;
|
23
|
+
//! left and right plans
|
33
24
|
optional_ptr<JoinNode> left;
|
34
25
|
optional_ptr<JoinNode> right;
|
35
26
|
|
36
|
-
|
27
|
+
//! The cost of the join node. The cost is stored here so that the cost of
|
28
|
+
//! a join node stays in sync with how the join node is constructed. Storing the cost in an unordered_set
|
29
|
+
//! in the cost model is error prone. If the plan enumerator join node is updated and not the cost model
|
30
|
+
//! the whole Join Order Optimizer can start exhibiting undesired behavior.
|
31
|
+
double cost;
|
32
|
+
//! used only to populate logical operators with estimated caridnalities after the best join plan has been found.
|
33
|
+
idx_t cardinality;
|
34
|
+
|
35
|
+
//! Create an intermediate node in the join tree. base_cardinality = estimated_props.cardinality
|
36
|
+
JoinNode(JoinRelationSet &set, optional_ptr<NeighborInfo> info, JoinNode &left, JoinNode &right, double cost);
|
37
37
|
|
38
38
|
//! Create a leaf node in the join tree
|
39
39
|
//! set cost to 0 for leaf nodes
|
40
40
|
//! cost will be the cost to *produce* an intermediate table
|
41
|
-
JoinNode(JoinRelationSet &set
|
42
|
-
|
43
|
-
//! Create an intermediate node in the join tree. base_cardinality = estimated_props.cardinality
|
44
|
-
JoinNode(JoinRelationSet &set, optional_ptr<NeighborInfo> info, JoinNode &left, JoinNode &right,
|
45
|
-
const double base_cardinality, double cost);
|
41
|
+
JoinNode(JoinRelationSet &set);
|
46
42
|
|
47
43
|
bool operator==(const JoinNode &other) {
|
48
44
|
return other.set.ToString().compare(set.ToString()) == 0;
|
49
45
|
}
|
50
46
|
|
51
47
|
private:
|
52
|
-
double base_cardinality;
|
53
|
-
|
54
48
|
public:
|
55
|
-
template <class CARDINALITY_TYPE>
|
56
|
-
CARDINALITY_TYPE GetCardinality() const {
|
57
|
-
return estimated_props->GetCardinality<CARDINALITY_TYPE>();
|
58
|
-
}
|
59
|
-
double GetCost();
|
60
|
-
void SetCost(double cost);
|
61
|
-
double GetBaseTableCardinality();
|
62
|
-
void SetBaseTableCardinality(double base_card);
|
63
|
-
void SetEstimatedCardinality(double estimated_card);
|
64
49
|
void PrintJoinNode();
|
65
50
|
string ToString();
|
66
51
|
};
|