duckdb 0.7.2-dev921.0 → 0.7.2-dev982.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/duckdb/src/common/enums/logical_operator_type.cpp +2 -0
- package/src/duckdb/src/common/serializer/enum_serializer.cpp +4 -0
- package/src/duckdb/src/common/types/value.cpp +46 -0
- package/src/duckdb/src/execution/column_binding_resolver.cpp +15 -5
- package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +2 -0
- package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +3 -3
- package/src/duckdb/src/execution/operator/persistent/buffered_csv_reader.cpp +5 -13
- package/src/duckdb/src/execution/operator/projection/physical_projection.cpp +34 -0
- package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +97 -0
- package/src/duckdb/src/execution/physical_plan_generator.cpp +3 -0
- package/src/duckdb/src/function/scalar/math/numeric.cpp +87 -0
- package/src/duckdb/src/function/scalar/math_functions.cpp +3 -0
- package/src/duckdb/src/function/scalar/string/hex.cpp +201 -0
- package/src/duckdb/src/function/scalar/string_functions.cpp +1 -0
- package/src/duckdb/src/function/table/read_csv.cpp +46 -0
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +5 -4
- package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/string_util.hpp +13 -0
- package/src/duckdb/src/include/duckdb/common/types/value.hpp +11 -7
- package/src/duckdb/src/include/duckdb/common/vector_operations/unary_executor.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +6 -0
- package/src/duckdb/src/include/duckdb/execution/operator/projection/physical_projection.hpp +5 -0
- package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +1 -0
- package/src/duckdb/src/include/duckdb/function/scalar/math_functions.hpp +8 -0
- package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_asof_join.hpp +22 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -2
- package/src/duckdb/src/include/duckdb.h +1 -1
- package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -0
- package/src/duckdb/src/optimizer/filter_pullup.cpp +3 -1
- package/src/duckdb/src/optimizer/filter_pushdown.cpp +3 -1
- package/src/duckdb/src/optimizer/join_order/cardinality_estimator.cpp +4 -0
- package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +8 -4
- package/src/duckdb/src/optimizer/pullup/pullup_from_left.cpp +2 -2
- package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +1 -1
- package/src/duckdb/src/optimizer/pushdown/pushdown_inner_join.cpp +3 -0
- package/src/duckdb/src/optimizer/pushdown/pushdown_left_join.cpp +4 -2
- package/src/duckdb/src/optimizer/pushdown/pushdown_mark_join.cpp +1 -1
- package/src/duckdb/src/optimizer/remove_unused_columns.cpp +1 -0
- package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +1 -0
- package/src/duckdb/src/optimizer/statistics_propagator.cpp +1 -0
- package/src/duckdb/src/parser/tableref/joinref.cpp +4 -0
- package/src/duckdb/src/parser/transform/tableref/transform_join.cpp +8 -1
- package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +10 -3
- package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +60 -12
- package/src/duckdb/src/planner/logical_operator.cpp +3 -0
- package/src/duckdb/src/planner/logical_operator_visitor.cpp +1 -0
- package/src/duckdb/src/planner/operator/logical_asof_join.cpp +8 -0
- package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +3 -1
- package/src/duckdb/third_party/libpg_query/include/nodes/nodes.hpp +32 -0
- package/src/duckdb/third_party/libpg_query/include/nodes/primnodes.hpp +3 -3
- package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +915 -913
- package/src/duckdb/third_party/libpg_query/include/parser/kwlist.hpp +1 -0
- package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +17371 -17306
- package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
- package/src/duckdb/ub_src_function_scalar_string.cpp +2 -0
- package/src/duckdb/ub_src_planner_operator.cpp +2 -0
@@ -53,6 +53,25 @@ void ReadCSVData::FinalizeRead(ClientContext &context) {
|
|
53
53
|
}
|
54
54
|
}
|
55
55
|
|
56
|
+
uint8_t GetCandidateSpecificity(const LogicalType &candidate_type) {
|
57
|
+
//! Const ht with accepted auto_types and their weights in specificity
|
58
|
+
const duckdb::unordered_map<uint8_t, uint8_t> auto_type_candidates_specificity {
|
59
|
+
{(uint8_t)LogicalTypeId::VARCHAR, 0}, {(uint8_t)LogicalTypeId::TIMESTAMP, 1},
|
60
|
+
{(uint8_t)LogicalTypeId::DATE, 2}, {(uint8_t)LogicalTypeId::TIME, 3},
|
61
|
+
{(uint8_t)LogicalTypeId::DOUBLE, 4}, {(uint8_t)LogicalTypeId::FLOAT, 5},
|
62
|
+
{(uint8_t)LogicalTypeId::BIGINT, 6}, {(uint8_t)LogicalTypeId::INTEGER, 7},
|
63
|
+
{(uint8_t)LogicalTypeId::SMALLINT, 8}, {(uint8_t)LogicalTypeId::TINYINT, 9},
|
64
|
+
{(uint8_t)LogicalTypeId::BOOLEAN, 10}, {(uint8_t)LogicalTypeId::SQLNULL, 11}};
|
65
|
+
|
66
|
+
auto id = (uint8_t)candidate_type.id();
|
67
|
+
auto it = auto_type_candidates_specificity.find(id);
|
68
|
+
if (it == auto_type_candidates_specificity.end()) {
|
69
|
+
throw BinderException("Auto Type Candidate of type %s is not accepted as a valid input",
|
70
|
+
LogicalTypeIdToString(candidate_type.id()));
|
71
|
+
}
|
72
|
+
return it->second;
|
73
|
+
}
|
74
|
+
|
56
75
|
static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, TableFunctionBindInput &input,
|
57
76
|
vector<LogicalType> &return_types, vector<string> &names) {
|
58
77
|
auto &config = DBConfig::GetConfig(context);
|
@@ -105,6 +124,32 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, TableFunctio
|
|
105
124
|
if (names.empty()) {
|
106
125
|
throw BinderException("read_csv requires at least a single column as input!");
|
107
126
|
}
|
127
|
+
} else if (loption == "auto_type_candidates") {
|
128
|
+
options.auto_type_candidates.clear();
|
129
|
+
map<uint8_t, LogicalType> candidate_types;
|
130
|
+
// We always have the extremes of Null and Varchar, so we can default to varchar if the
|
131
|
+
// sniffer is not able to confidently detect that column type
|
132
|
+
candidate_types[GetCandidateSpecificity(LogicalType::VARCHAR)] = LogicalType::VARCHAR;
|
133
|
+
candidate_types[GetCandidateSpecificity(LogicalType::SQLNULL)] = LogicalType::SQLNULL;
|
134
|
+
|
135
|
+
auto &child_type = kv.second.type();
|
136
|
+
if (child_type.id() != LogicalTypeId::LIST) {
|
137
|
+
throw BinderException("read_csv auto_types requires a list as input");
|
138
|
+
}
|
139
|
+
auto &list_children = ListValue::GetChildren(kv.second);
|
140
|
+
if (list_children.empty()) {
|
141
|
+
throw BinderException("auto_type_candidates requires at least one type");
|
142
|
+
}
|
143
|
+
for (auto &child : list_children) {
|
144
|
+
if (child.type().id() != LogicalTypeId::VARCHAR) {
|
145
|
+
throw BinderException("auto_type_candidates requires a type specification as string");
|
146
|
+
}
|
147
|
+
auto candidate_type = TransformStringToLogicalType(StringValue::Get(child), context);
|
148
|
+
candidate_types[GetCandidateSpecificity(candidate_type)] = candidate_type;
|
149
|
+
}
|
150
|
+
for (auto &candidate_type : candidate_types) {
|
151
|
+
options.auto_type_candidates.emplace_back(candidate_type.second);
|
152
|
+
}
|
108
153
|
} else if (loption == "column_names" || loption == "names") {
|
109
154
|
if (!options.name_list.empty()) {
|
110
155
|
throw BinderException("read_csv_auto column_names/names can only be supplied once");
|
@@ -795,6 +840,7 @@ static void ReadCSVAddNamedParameters(TableFunction &table_function) {
|
|
795
840
|
table_function.named_parameters["escape"] = LogicalType::VARCHAR;
|
796
841
|
table_function.named_parameters["nullstr"] = LogicalType::VARCHAR;
|
797
842
|
table_function.named_parameters["columns"] = LogicalType::ANY;
|
843
|
+
table_function.named_parameters["auto_type_candidates"] = LogicalType::ANY;
|
798
844
|
table_function.named_parameters["header"] = LogicalType::BOOLEAN;
|
799
845
|
table_function.named_parameters["auto_detect"] = LogicalType::BOOLEAN;
|
800
846
|
table_function.named_parameters["sample_size"] = LogicalType::BIGINT;
|
@@ -1,8 +1,8 @@
|
|
1
1
|
#ifndef DUCKDB_VERSION
|
2
|
-
#define DUCKDB_VERSION "0.7.2-
|
2
|
+
#define DUCKDB_VERSION "0.7.2-dev982"
|
3
3
|
#endif
|
4
4
|
#ifndef DUCKDB_SOURCE_ID
|
5
|
-
#define DUCKDB_SOURCE_ID "
|
5
|
+
#define DUCKDB_SOURCE_ID "d43e34e8ba"
|
6
6
|
#endif
|
7
7
|
#include "duckdb/function/table/system_functions.hpp"
|
8
8
|
#include "duckdb/main/database.hpp"
|
@@ -16,10 +16,11 @@ namespace duckdb {
|
|
16
16
|
// Join Reference Types
|
17
17
|
//===--------------------------------------------------------------------===//
|
18
18
|
enum class JoinRefType : uint8_t {
|
19
|
-
REGULAR,
|
20
|
-
NATURAL,
|
21
|
-
CROSS,
|
22
|
-
POSITIONAL // Positional condition
|
19
|
+
REGULAR, // Explicit conditions
|
20
|
+
NATURAL, // Implied conditions
|
21
|
+
CROSS, // No condition
|
22
|
+
POSITIONAL, // Positional condition
|
23
|
+
ASOF // AsOf conditions
|
23
24
|
};
|
24
25
|
|
25
26
|
const char *ToString(JoinRefType value);
|
@@ -21,6 +21,19 @@ namespace duckdb {
|
|
21
21
|
*/
|
22
22
|
class StringUtil {
|
23
23
|
public:
|
24
|
+
static uint8_t GetHexValue(char c) {
|
25
|
+
if (c >= '0' && c <= '9') {
|
26
|
+
return c - '0';
|
27
|
+
}
|
28
|
+
if (c >= 'a' && c <= 'f') {
|
29
|
+
return c - 'a' + 10;
|
30
|
+
}
|
31
|
+
if (c >= 'A' && c <= 'F') {
|
32
|
+
return c - 'A' + 10;
|
33
|
+
}
|
34
|
+
throw InvalidInputException("Invalid input for hex digit: %s", string(c, 1));
|
35
|
+
}
|
36
|
+
|
24
37
|
DUCKDB_API static bool CharacterIsSpace(char c) {
|
25
38
|
return c == ' ' || c == '\t' || c == '\n' || c == '\v' || c == '\f' || c == '\r';
|
26
39
|
}
|
@@ -67,7 +67,7 @@ public:
|
|
67
67
|
inline LogicalType &GetTypeMutable() {
|
68
68
|
return type_;
|
69
69
|
}
|
70
|
-
inline const LogicalType &type() const {
|
70
|
+
inline const LogicalType &type() const { // NOLINT
|
71
71
|
return type_;
|
72
72
|
}
|
73
73
|
inline bool IsNull() const {
|
@@ -78,6 +78,10 @@ public:
|
|
78
78
|
DUCKDB_API static Value MinimumValue(const LogicalType &type);
|
79
79
|
//! Create the highest possible value of a given type (numeric only)
|
80
80
|
DUCKDB_API static Value MaximumValue(const LogicalType &type);
|
81
|
+
//! Create the negative infinite value of a given type (numeric only)
|
82
|
+
DUCKDB_API static Value NegativeInfinity(const LogicalType &type);
|
83
|
+
//! Create the positive infinite value of a given type (numeric only)
|
84
|
+
DUCKDB_API static Value Infinity(const LogicalType &type);
|
81
85
|
//! Create a Numeric value of the specified type with the specified value
|
82
86
|
DUCKDB_API static Value Numeric(const LogicalType &type, int64_t value);
|
83
87
|
DUCKDB_API static Value Numeric(const LogicalType &type, hugeint_t value);
|
@@ -161,7 +165,7 @@ public:
|
|
161
165
|
|
162
166
|
//! Create a blob Value from a data pointer and a length: no bytes are interpreted
|
163
167
|
DUCKDB_API static Value BLOB(const_data_ptr_t data, idx_t len);
|
164
|
-
DUCKDB_API static Value BLOB_RAW(const string &data) {
|
168
|
+
DUCKDB_API static Value BLOB_RAW(const string &data) { // NOLINT
|
165
169
|
return Value::BLOB((const_data_ptr_t)data.c_str(), data.size());
|
166
170
|
}
|
167
171
|
//! Creates a blob by casting a specified string to a blob (i.e. interpreting \x characters)
|
@@ -280,7 +284,7 @@ public:
|
|
280
284
|
|
281
285
|
private:
|
282
286
|
//! The logical of the value
|
283
|
-
LogicalType type_;
|
287
|
+
LogicalType type_; // NOLINT
|
284
288
|
|
285
289
|
//! Whether or not the value is NULL
|
286
290
|
bool is_null;
|
@@ -297,17 +301,17 @@ private:
|
|
297
301
|
uint32_t uinteger;
|
298
302
|
uint64_t ubigint;
|
299
303
|
hugeint_t hugeint;
|
300
|
-
float float_;
|
301
|
-
double double_;
|
304
|
+
float float_; // NOLINT
|
305
|
+
double double_; // NOLINT
|
302
306
|
uintptr_t pointer;
|
303
307
|
uint64_t hash;
|
304
308
|
date_t date;
|
305
309
|
dtime_t time;
|
306
310
|
timestamp_t timestamp;
|
307
311
|
interval_t interval;
|
308
|
-
} value_;
|
312
|
+
} value_; // NOLINT
|
309
313
|
|
310
|
-
shared_ptr<ExtraValueInfo> value_info_;
|
314
|
+
shared_ptr<ExtraValueInfo> value_info_; // NOLINT
|
311
315
|
|
312
316
|
private:
|
313
317
|
template <class T>
|
@@ -209,8 +209,8 @@ public:
|
|
209
209
|
|
210
210
|
template <class INPUT_TYPE, class RESULT_TYPE, class OP>
|
211
211
|
static void ExecuteString(Vector &input, Vector &result, idx_t count) {
|
212
|
-
UnaryExecutor::GenericExecute<
|
213
|
-
|
212
|
+
UnaryExecutor::GenericExecute<INPUT_TYPE, RESULT_TYPE, UnaryStringOperator<OP>>(input, result, count,
|
213
|
+
(void *)&result);
|
214
214
|
}
|
215
215
|
};
|
216
216
|
|
@@ -14,6 +14,7 @@
|
|
14
14
|
#include "duckdb/common/types/value.hpp"
|
15
15
|
#include "duckdb/common/field_writer.hpp"
|
16
16
|
#include "duckdb/common/case_insensitive_map.hpp"
|
17
|
+
#include "duckdb/common/types.hpp"
|
17
18
|
|
18
19
|
namespace duckdb {
|
19
20
|
|
@@ -77,6 +78,11 @@ struct BufferedCSVReaderOptions {
|
|
77
78
|
vector<LogicalType> sql_type_list;
|
78
79
|
//! User-defined name list
|
79
80
|
vector<string> name_list;
|
81
|
+
//! Types considered as candidates for auto detection ordered by descending specificity (~ from high to low)
|
82
|
+
vector<LogicalType> auto_type_candidates = {LogicalType::VARCHAR, LogicalType::TIMESTAMP, LogicalType::DATE,
|
83
|
+
LogicalType::TIME, LogicalType::DOUBLE, LogicalType::BIGINT,
|
84
|
+
LogicalType::BOOLEAN, LogicalType::SQLNULL};
|
85
|
+
|
80
86
|
//===--------------------------------------------------------------------===//
|
81
87
|
// ReadCSVOptions
|
82
88
|
//===--------------------------------------------------------------------===//
|
@@ -30,6 +30,11 @@ public:
|
|
30
30
|
}
|
31
31
|
|
32
32
|
string ParamsToString() const override;
|
33
|
+
|
34
|
+
static unique_ptr<PhysicalOperator>
|
35
|
+
CreateJoinProjection(vector<LogicalType> proj_types, const vector<LogicalType> &lhs_types,
|
36
|
+
const vector<LogicalType> &rhs_types, const vector<idx_t> &left_projection_map,
|
37
|
+
const vector<idx_t> &right_projection_map, const idx_t estimated_cardinality);
|
33
38
|
};
|
34
39
|
|
35
40
|
} // namespace duckdb
|
@@ -48,6 +48,7 @@ protected:
|
|
48
48
|
|
49
49
|
unique_ptr<PhysicalOperator> CreatePlan(LogicalAggregate &op);
|
50
50
|
unique_ptr<PhysicalOperator> CreatePlan(LogicalAnyJoin &op);
|
51
|
+
unique_ptr<PhysicalOperator> CreatePlan(LogicalAsOfJoin &op);
|
51
52
|
unique_ptr<PhysicalOperator> CreatePlan(LogicalColumnDataGet &op);
|
52
53
|
unique_ptr<PhysicalOperator> CreatePlan(LogicalComparisonJoin &op);
|
53
54
|
unique_ptr<PhysicalOperator> CreatePlan(LogicalCreate &op);
|
@@ -122,4 +122,12 @@ struct IsFiniteFun {
|
|
122
122
|
static void RegisterFunction(BuiltinFunctions &set);
|
123
123
|
};
|
124
124
|
|
125
|
+
struct GreatestCommonDivisorFun {
|
126
|
+
static void RegisterFunction(BuiltinFunctions &set);
|
127
|
+
};
|
128
|
+
|
129
|
+
struct LeastCommonMultipleFun {
|
130
|
+
static void RegisterFunction(BuiltinFunctions &set);
|
131
|
+
};
|
132
|
+
|
125
133
|
} // namespace duckdb
|
@@ -1,5 +1,6 @@
|
|
1
1
|
#include "duckdb/planner/operator/logical_aggregate.hpp"
|
2
2
|
#include "duckdb/planner/operator/logical_any_join.hpp"
|
3
|
+
#include "duckdb/planner/operator/logical_asof_join.hpp"
|
3
4
|
#include "duckdb/planner/operator/logical_column_data_get.hpp"
|
4
5
|
#include "duckdb/planner/operator/logical_comparison_join.hpp"
|
5
6
|
#include "duckdb/planner/operator/logical_copy_to_file.hpp"
|
@@ -0,0 +1,22 @@
|
|
1
|
+
//===----------------------------------------------------------------------===//
|
2
|
+
// DuckDB
|
3
|
+
//
|
4
|
+
// duckdb/planner/operator/logical_asof_join.hpp
|
5
|
+
//
|
6
|
+
//
|
7
|
+
//===----------------------------------------------------------------------===//
|
8
|
+
|
9
|
+
#pragma once
|
10
|
+
|
11
|
+
#include "duckdb/planner/operator/logical_comparison_join.hpp"
|
12
|
+
|
13
|
+
namespace duckdb {
|
14
|
+
|
15
|
+
//! LogicalAsOfJoin represents a temporal-style join with one less-than inequality.
|
16
|
+
//! This inequality matches the greatest value on the right that satisfies the condition.
|
17
|
+
class LogicalAsOfJoin : public LogicalComparisonJoin {
|
18
|
+
public:
|
19
|
+
explicit LogicalAsOfJoin(JoinType type);
|
20
|
+
};
|
21
|
+
|
22
|
+
} // namespace duckdb
|
@@ -9,6 +9,7 @@
|
|
9
9
|
#pragma once
|
10
10
|
|
11
11
|
#include "duckdb/common/constants.hpp"
|
12
|
+
#include "duckdb/common/enums/joinref_type.hpp"
|
12
13
|
#include "duckdb/common/unordered_set.hpp"
|
13
14
|
#include "duckdb/planner/joinside.hpp"
|
14
15
|
#include "duckdb/planner/operator/logical_join.hpp"
|
@@ -34,10 +35,12 @@ public:
|
|
34
35
|
FieldReader &reader);
|
35
36
|
|
36
37
|
public:
|
37
|
-
static unique_ptr<LogicalOperator> CreateJoin(JoinType type,
|
38
|
+
static unique_ptr<LogicalOperator> CreateJoin(JoinType type, JoinRefType ref_type,
|
39
|
+
unique_ptr<LogicalOperator> left_child,
|
38
40
|
unique_ptr<LogicalOperator> right_child,
|
39
41
|
unique_ptr<Expression> condition);
|
40
|
-
static unique_ptr<LogicalOperator> CreateJoin(JoinType type,
|
42
|
+
static unique_ptr<LogicalOperator> CreateJoin(JoinType type, JoinRefType ref_type,
|
43
|
+
unique_ptr<LogicalOperator> left_child,
|
41
44
|
unique_ptr<LogicalOperator> right_child,
|
42
45
|
vector<JoinCondition> conditions,
|
43
46
|
vector<unique_ptr<Expression>> arbitrary_expressions);
|
@@ -585,7 +585,7 @@ DUCKDB_API bool duckdb_result_is_streaming(duckdb_result result);
|
|
585
585
|
Returns the number of data chunks present in the result.
|
586
586
|
|
587
587
|
* result: The result object
|
588
|
-
* returns:
|
588
|
+
* returns: Number of data chunks present in the result.
|
589
589
|
*/
|
590
590
|
DUCKDB_API idx_t duckdb_result_chunk_count(duckdb_result result);
|
591
591
|
|
@@ -57,6 +57,7 @@ void ColumnLifetimeAnalyzer::VisitOperator(LogicalOperator &op) {
|
|
57
57
|
analyzer.VisitOperator(*op.children[0]);
|
58
58
|
return;
|
59
59
|
}
|
60
|
+
case LogicalOperatorType::LOGICAL_ASOF_JOIN:
|
60
61
|
case LogicalOperatorType::LOGICAL_DELIM_JOIN:
|
61
62
|
case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: {
|
62
63
|
if (everything_referenced) {
|
@@ -14,6 +14,7 @@ unique_ptr<LogicalOperator> FilterPullup::Rewrite(unique_ptr<LogicalOperator> op
|
|
14
14
|
case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
|
15
15
|
case LogicalOperatorType::LOGICAL_ANY_JOIN:
|
16
16
|
case LogicalOperatorType::LOGICAL_DELIM_JOIN:
|
17
|
+
case LogicalOperatorType::LOGICAL_ASOF_JOIN:
|
17
18
|
return PullupJoin(std::move(op));
|
18
19
|
case LogicalOperatorType::LOGICAL_INTERSECT:
|
19
20
|
case LogicalOperatorType::LOGICAL_EXCEPT:
|
@@ -31,7 +32,8 @@ unique_ptr<LogicalOperator> FilterPullup::Rewrite(unique_ptr<LogicalOperator> op
|
|
31
32
|
|
32
33
|
unique_ptr<LogicalOperator> FilterPullup::PullupJoin(unique_ptr<LogicalOperator> op) {
|
33
34
|
D_ASSERT(op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN ||
|
34
|
-
op->type == LogicalOperatorType::
|
35
|
+
op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN || op->type == LogicalOperatorType::LOGICAL_ANY_JOIN ||
|
36
|
+
op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN);
|
35
37
|
auto &join = (LogicalJoin &)*op;
|
36
38
|
|
37
39
|
switch (join.join_type) {
|
@@ -23,6 +23,7 @@ unique_ptr<LogicalOperator> FilterPushdown::Rewrite(unique_ptr<LogicalOperator>
|
|
23
23
|
return PushdownCrossProduct(std::move(op));
|
24
24
|
case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
|
25
25
|
case LogicalOperatorType::LOGICAL_ANY_JOIN:
|
26
|
+
case LogicalOperatorType::LOGICAL_ASOF_JOIN:
|
26
27
|
case LogicalOperatorType::LOGICAL_DELIM_JOIN:
|
27
28
|
return PushdownJoin(std::move(op));
|
28
29
|
case LogicalOperatorType::LOGICAL_PROJECTION:
|
@@ -48,7 +49,8 @@ unique_ptr<LogicalOperator> FilterPushdown::Rewrite(unique_ptr<LogicalOperator>
|
|
48
49
|
|
49
50
|
unique_ptr<LogicalOperator> FilterPushdown::PushdownJoin(unique_ptr<LogicalOperator> op) {
|
50
51
|
D_ASSERT(op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN ||
|
51
|
-
op->type == LogicalOperatorType::
|
52
|
+
op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN || op->type == LogicalOperatorType::LOGICAL_ANY_JOIN ||
|
53
|
+
op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN);
|
52
54
|
auto &join = (LogicalJoin &)*op;
|
53
55
|
unordered_set<idx_t> left_bindings, right_bindings;
|
54
56
|
LogicalJoin::GetTableReferences(*op->children[0], left_bindings);
|
@@ -309,6 +309,7 @@ static LogicalGet *GetLogicalGet(LogicalOperator *op, idx_t table_index = DConst
|
|
309
309
|
case LogicalOperatorType::LOGICAL_PROJECTION:
|
310
310
|
get = GetLogicalGet(op->children.at(0).get(), table_index);
|
311
311
|
break;
|
312
|
+
case LogicalOperatorType::LOGICAL_ASOF_JOIN:
|
312
313
|
case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: {
|
313
314
|
LogicalComparisonJoin *join = (LogicalComparisonJoin *)op;
|
314
315
|
// We should never be calling GetLogicalGet without a valid table_index.
|
@@ -383,6 +384,9 @@ void CardinalityEstimator::InitCardinalityEstimatorProps(vector<NodeOp> *node_op
|
|
383
384
|
// less than the base table cardinality.
|
384
385
|
join_node->SetCost(join_node->GetBaseTableCardinality());
|
385
386
|
}
|
387
|
+
} else if (op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN) {
|
388
|
+
// AsOf joins have the cardinality of the LHS
|
389
|
+
join_node->SetCost(join_node->GetBaseTableCardinality());
|
386
390
|
}
|
387
391
|
// Total domains can be affected by filters. So we update base table cardinality first
|
388
392
|
EstimateBaseTableCardinality(join_node, op);
|
@@ -113,7 +113,7 @@ bool JoinOrderOptimizer::ExtractJoinRelations(LogicalOperator &input_op, vector<
|
|
113
113
|
bool non_reorderable_operation = false;
|
114
114
|
if (op->type == LogicalOperatorType::LOGICAL_UNION || op->type == LogicalOperatorType::LOGICAL_EXCEPT ||
|
115
115
|
op->type == LogicalOperatorType::LOGICAL_INTERSECT || op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN ||
|
116
|
-
op->type == LogicalOperatorType::LOGICAL_ANY_JOIN) {
|
116
|
+
op->type == LogicalOperatorType::LOGICAL_ANY_JOIN || op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN) {
|
117
117
|
// set operation, optimize separately in children
|
118
118
|
non_reorderable_operation = true;
|
119
119
|
}
|
@@ -184,6 +184,7 @@ bool JoinOrderOptimizer::ExtractJoinRelations(LogicalOperator &input_op, vector<
|
|
184
184
|
}
|
185
185
|
|
186
186
|
switch (op->type) {
|
187
|
+
case LogicalOperatorType::LOGICAL_ASOF_JOIN:
|
187
188
|
case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
|
188
189
|
case LogicalOperatorType::LOGICAL_CROSS_PRODUCT: {
|
189
190
|
// inner join or cross product
|
@@ -866,7 +867,8 @@ JoinOrderOptimizer::GenerateJoins(vector<unique_ptr<LogicalOperator>> &extracted
|
|
866
867
|
result_operator->children[0] = std::move(comp_join);
|
867
868
|
}
|
868
869
|
} else {
|
869
|
-
D_ASSERT(node->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN
|
870
|
+
D_ASSERT(node->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN ||
|
871
|
+
node->type == LogicalOperatorType::LOGICAL_ASOF_JOIN);
|
870
872
|
auto &comp_join = (LogicalComparisonJoin &)*node;
|
871
873
|
comp_join.conditions.push_back(std::move(cond));
|
872
874
|
}
|
@@ -908,7 +910,8 @@ unique_ptr<LogicalOperator> JoinOrderOptimizer::RewritePlan(unique_ptr<LogicalOp
|
|
908
910
|
auto op = plan.get();
|
909
911
|
auto parent = plan.get();
|
910
912
|
while (op->type != LogicalOperatorType::LOGICAL_CROSS_PRODUCT &&
|
911
|
-
op->type != LogicalOperatorType::LOGICAL_COMPARISON_JOIN
|
913
|
+
op->type != LogicalOperatorType::LOGICAL_COMPARISON_JOIN &&
|
914
|
+
op->type != LogicalOperatorType::LOGICAL_ASOF_JOIN) {
|
912
915
|
D_ASSERT(op->children.size() == 1);
|
913
916
|
parent = op;
|
914
917
|
op = op->children[0].get();
|
@@ -943,7 +946,8 @@ unique_ptr<LogicalOperator> JoinOrderOptimizer::Optimize(unique_ptr<LogicalOpera
|
|
943
946
|
// filters in the process
|
944
947
|
expression_set_t filter_set;
|
945
948
|
for (auto &f_op : filter_operators) {
|
946
|
-
if (f_op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN
|
949
|
+
if (f_op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN ||
|
950
|
+
f_op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN) {
|
947
951
|
auto &join = (LogicalComparisonJoin &)*f_op;
|
948
952
|
D_ASSERT(join.join_type == JoinType::INNER);
|
949
953
|
D_ASSERT(join.expressions.empty());
|
@@ -6,8 +6,8 @@ namespace duckdb {
|
|
6
6
|
|
7
7
|
unique_ptr<LogicalOperator> FilterPullup::PullupFromLeft(unique_ptr<LogicalOperator> op) {
|
8
8
|
D_ASSERT(op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN ||
|
9
|
-
op->type == LogicalOperatorType::
|
10
|
-
op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN);
|
9
|
+
op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN || op->type == LogicalOperatorType::LOGICAL_ANY_JOIN ||
|
10
|
+
op->type == LogicalOperatorType::LOGICAL_EXCEPT || op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN);
|
11
11
|
|
12
12
|
FilterPullup left_pullup(true, can_add_column);
|
13
13
|
FilterPullup right_pullup(false, can_add_column);
|
@@ -46,7 +46,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownCrossProduct(unique_ptr<Logi
|
|
46
46
|
right_bindings, join_expressions, conditions,
|
47
47
|
arbitrary_expressions);
|
48
48
|
// create the join from the join conditions
|
49
|
-
return LogicalComparisonJoin::CreateJoin(JoinType::INNER, std::move(op->children[0]),
|
49
|
+
return LogicalComparisonJoin::CreateJoin(JoinType::INNER, JoinRefType::REGULAR, std::move(op->children[0]),
|
50
50
|
std::move(op->children[1]), std::move(conditions),
|
51
51
|
std::move(arbitrary_expressions));
|
52
52
|
} else {
|
@@ -24,6 +24,9 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownInnerJoin(unique_ptr<Logical
|
|
24
24
|
// filter statically evaluates to false, strip tree
|
25
25
|
return make_unique<LogicalEmptyResult>(std::move(op));
|
26
26
|
}
|
27
|
+
} else if (op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN) {
|
28
|
+
// Don't mess with non-standard condition interpretations
|
29
|
+
return FinishPushdown(std::move(op));
|
27
30
|
} else {
|
28
31
|
// comparison join
|
29
32
|
D_ASSERT(op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN);
|
@@ -68,7 +68,9 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownLeftJoin(unique_ptr<LogicalO
|
|
68
68
|
// for a comparison join we create a FilterCombiner that checks if we can push conditions on LHS join conditions
|
69
69
|
// into the RHS of the join
|
70
70
|
FilterCombiner filter_combiner(optimizer);
|
71
|
-
|
71
|
+
const auto isComparison = (op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN ||
|
72
|
+
op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN);
|
73
|
+
if (isComparison) {
|
72
74
|
// add all comparison conditions
|
73
75
|
auto &comparison_join = (LogicalComparisonJoin &)*op;
|
74
76
|
for (auto &cond : comparison_join.conditions) {
|
@@ -82,7 +84,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownLeftJoin(unique_ptr<LogicalO
|
|
82
84
|
if (side == JoinSide::LEFT) {
|
83
85
|
// bindings match left side
|
84
86
|
// we can push the filter into the left side
|
85
|
-
if (
|
87
|
+
if (isComparison) {
|
86
88
|
// we MIGHT be able to push it down the RHS as well, but only if it is a comparison that matches the
|
87
89
|
// join predicates we use the FilterCombiner to figure this out add the expression to the FilterCombiner
|
88
90
|
filter_combiner.AddFilter(filters[i]->filter->Copy());
|
@@ -13,7 +13,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownMarkJoin(unique_ptr<LogicalO
|
|
13
13
|
auto &comp_join = (LogicalComparisonJoin &)*op;
|
14
14
|
D_ASSERT(join.join_type == JoinType::MARK);
|
15
15
|
D_ASSERT(op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN ||
|
16
|
-
op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN);
|
16
|
+
op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN || op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN);
|
17
17
|
|
18
18
|
right_bindings.insert(comp_join.mark_index);
|
19
19
|
FilterPushdown left_pushdown(optimizer), right_pushdown(optimizer);
|
@@ -71,6 +71,7 @@ void RemoveUnusedColumns::VisitOperator(LogicalOperator &op) {
|
|
71
71
|
remove.VisitOperator(*op.children[0]);
|
72
72
|
return;
|
73
73
|
}
|
74
|
+
case LogicalOperatorType::LOGICAL_ASOF_JOIN:
|
74
75
|
case LogicalOperatorType::LOGICAL_DELIM_JOIN:
|
75
76
|
case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: {
|
76
77
|
if (!everything_referenced) {
|
@@ -195,6 +195,7 @@ unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalJoin
|
|
195
195
|
switch (join.type) {
|
196
196
|
case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
|
197
197
|
case LogicalOperatorType::LOGICAL_DELIM_JOIN:
|
198
|
+
case LogicalOperatorType::LOGICAL_ASOF_JOIN:
|
198
199
|
PropagateStatistics((LogicalComparisonJoin &)join, node_ptr);
|
199
200
|
break;
|
200
201
|
case LogicalOperatorType::LOGICAL_ANY_JOIN:
|
@@ -36,6 +36,7 @@ unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalOper
|
|
36
36
|
case LogicalOperatorType::LOGICAL_PROJECTION:
|
37
37
|
return PropagateStatistics((LogicalProjection &)node, node_ptr);
|
38
38
|
case LogicalOperatorType::LOGICAL_ANY_JOIN:
|
39
|
+
case LogicalOperatorType::LOGICAL_ASOF_JOIN:
|
39
40
|
case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
|
40
41
|
case LogicalOperatorType::LOGICAL_JOIN:
|
41
42
|
case LogicalOperatorType::LOGICAL_DELIM_JOIN:
|
@@ -18,6 +18,10 @@ string JoinRef::ToString() const {
|
|
18
18
|
result += "NATURAL ";
|
19
19
|
result += JoinTypeToString(type) + " JOIN ";
|
20
20
|
break;
|
21
|
+
case JoinRefType::ASOF:
|
22
|
+
result += "ASOF ";
|
23
|
+
result += JoinTypeToString(type) + " JOIN ";
|
24
|
+
break;
|
21
25
|
case JoinRefType::CROSS:
|
22
26
|
result += ", ";
|
23
27
|
break;
|
@@ -44,8 +44,15 @@ unique_ptr<TableRef> Transformer::TransformJoin(duckdb_libpgquery::PGJoinExpr *r
|
|
44
44
|
// Check the type of left arg and right arg before transform
|
45
45
|
result->left = TransformTableRefNode(root->larg);
|
46
46
|
result->right = TransformTableRefNode(root->rarg);
|
47
|
-
|
47
|
+
switch (root->joinreftype) {
|
48
|
+
case duckdb_libpgquery::PG_JOIN_NATURAL:
|
48
49
|
result->ref_type = JoinRefType::NATURAL;
|
50
|
+
break;
|
51
|
+
case duckdb_libpgquery::PG_JOIN_ASOF:
|
52
|
+
result->ref_type = JoinRefType::ASOF;
|
53
|
+
break;
|
54
|
+
default:
|
55
|
+
break;
|
49
56
|
}
|
50
57
|
result->query_location = root->location;
|
51
58
|
|
@@ -25,11 +25,11 @@ static unique_ptr<ParsedExpression> BindColumn(Binder &binder, ClientContext &co
|
|
25
25
|
|
26
26
|
static unique_ptr<ParsedExpression> AddCondition(ClientContext &context, Binder &left_binder, Binder &right_binder,
|
27
27
|
const string &left_alias, const string &right_alias,
|
28
|
-
const string &column_name) {
|
28
|
+
const string &column_name, ExpressionType type) {
|
29
29
|
ExpressionBinder expr_binder(left_binder, context);
|
30
30
|
auto left = BindColumn(left_binder, context, left_alias, column_name);
|
31
31
|
auto right = BindColumn(right_binder, context, right_alias, column_name);
|
32
|
-
return make_unique<ComparisonExpression>(
|
32
|
+
return make_unique<ComparisonExpression>(type, std::move(left), std::move(right));
|
33
33
|
}
|
34
34
|
|
35
35
|
bool Binder::TryFindBinding(const string &using_column, const string &join_side, string &result) {
|
@@ -198,12 +198,14 @@ unique_ptr<BoundTableRef> Binder::Bind(JoinRef &ref) {
|
|
198
198
|
break;
|
199
199
|
}
|
200
200
|
case JoinRefType::REGULAR:
|
201
|
+
case JoinRefType::ASOF:
|
201
202
|
if (!ref.using_columns.empty()) {
|
202
203
|
// USING columns
|
203
204
|
D_ASSERT(!result->condition);
|
204
205
|
extra_using_columns = ref.using_columns;
|
205
206
|
}
|
206
207
|
break;
|
208
|
+
|
207
209
|
case JoinRefType::CROSS:
|
208
210
|
case JoinRefType::POSITIONAL:
|
209
211
|
break;
|
@@ -241,8 +243,13 @@ unique_ptr<BoundTableRef> Binder::Bind(JoinRef &ref) {
|
|
241
243
|
left_binding = RetrieveUsingBinding(left_binder, left_using_binding, using_column, "left", set.get());
|
242
244
|
right_binding = RetrieveUsingBinding(right_binder, right_using_binding, using_column, "right", set.get());
|
243
245
|
|
246
|
+
// Last column of ASOF JOIN ... USING is >=
|
247
|
+
const auto type = (ref.ref_type == JoinRefType::ASOF && i == extra_using_columns.size() - 1)
|
248
|
+
? ExpressionType::COMPARE_GREATERTHANOREQUALTO
|
249
|
+
: ExpressionType::COMPARE_EQUAL;
|
250
|
+
|
244
251
|
extra_conditions.push_back(
|
245
|
-
AddCondition(context, left_binder, right_binder, left_binding, right_binding, using_column));
|
252
|
+
AddCondition(context, left_binder, right_binder, left_binding, right_binding, using_column, type));
|
246
253
|
|
247
254
|
AddUsingBindings(*set, left_using_binding, left_binding);
|
248
255
|
AddUsingBindings(*set, right_using_binding, right_binding);
|