duckdb 0.7.2-dev333.0 → 0.7.2-dev457.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +1 -0
- package/lib/duckdb.d.ts +42 -0
- package/package.json +1 -1
- package/src/connection.cpp +1 -2
- package/src/database.cpp +1 -1
- package/src/duckdb/extension/icu/icu-extension.cpp +2 -0
- package/src/duckdb/extension/icu/icu-list-range.cpp +207 -0
- package/src/duckdb/extension/icu/include/icu-list-range.hpp +17 -0
- package/src/duckdb/extension/json/json_functions/read_json.cpp +6 -5
- package/src/duckdb/extension/parquet/include/parquet_timestamp.hpp +0 -1
- package/src/duckdb/extension/parquet/parquet_timestamp.cpp +8 -6
- package/src/duckdb/src/common/exception.cpp +15 -1
- package/src/duckdb/src/common/preserved_error.cpp +7 -5
- package/src/duckdb/src/execution/operator/scan/physical_positional_scan.cpp +20 -5
- package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +9 -1
- package/src/duckdb/src/execution/physical_plan/plan_distinct.cpp +5 -8
- package/src/duckdb/src/execution/physical_plan/plan_positional_join.cpp +14 -5
- package/src/duckdb/src/function/aggregate/distributive/bool.cpp +2 -0
- package/src/duckdb/src/function/aggregate/distributive/count.cpp +1 -0
- package/src/duckdb/src/function/aggregate/distributive/minmax.cpp +2 -0
- package/src/duckdb/src/function/aggregate/distributive/sum.cpp +8 -0
- package/src/duckdb/src/function/aggregate/holistic/quantile.cpp +15 -0
- package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +42 -11
- package/src/duckdb/src/function/cast/time_casts.cpp +2 -2
- package/src/duckdb/src/function/function_binder.cpp +1 -8
- package/src/duckdb/src/function/scalar/date/current.cpp +0 -2
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/common/exception.hpp +38 -2
- package/src/duckdb/src/include/duckdb/common/preserved_error.hpp +3 -1
- package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +0 -3
- package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +6 -3
- package/src/duckdb/src/include/duckdb/function/function_binder.hpp +3 -6
- package/src/duckdb/src/include/duckdb/main/prepared_statement.hpp +2 -0
- package/src/duckdb/src/include/duckdb/main/relation/explain_relation.hpp +2 -1
- package/src/duckdb/src/include/duckdb/main/relation.hpp +2 -1
- package/src/duckdb/src/include/duckdb/optimizer/rule/list.hpp +1 -0
- package/src/duckdb/src/include/duckdb/optimizer/rule/ordered_aggregate_optimizer.hpp +24 -0
- package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +2 -2
- package/src/duckdb/src/include/duckdb/parser/expression/star_expression.hpp +2 -2
- package/src/duckdb/src/include/duckdb/parser/transformer.hpp +2 -0
- package/src/duckdb/src/include/duckdb/planner/binder.hpp +4 -3
- package/src/duckdb/src/include/duckdb/planner/bound_result_modifier.hpp +3 -0
- package/src/duckdb/src/include/duckdb/planner/expression/bound_aggregate_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/planner/expression_binder/order_binder.hpp +4 -1
- package/src/duckdb/src/include/duckdb/planner/operator/logical_distinct.hpp +3 -0
- package/src/duckdb/src/main/extension/extension_install.cpp +2 -2
- package/src/duckdb/src/main/prepared_statement.cpp +4 -0
- package/src/duckdb/src/main/relation/explain_relation.cpp +3 -3
- package/src/duckdb/src/main/relation.cpp +3 -2
- package/src/duckdb/src/optimizer/optimizer.cpp +1 -0
- package/src/duckdb/src/optimizer/rule/ordered_aggregate_optimizer.cpp +30 -0
- package/src/duckdb/src/parser/expression/star_expression.cpp +6 -6
- package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -1
- package/src/duckdb/src/parser/transform/expression/transform_columnref.cpp +17 -2
- package/src/duckdb/src/parser/transform/expression/transform_function.cpp +45 -40
- package/src/duckdb/src/parser/transform/helpers/transform_groupby.cpp +7 -0
- package/src/duckdb/src/parser/transform/helpers/transform_orderby.cpp +0 -7
- package/src/duckdb/src/planner/bind_context.cpp +2 -25
- package/src/duckdb/src/planner/binder/expression/bind_aggregate_expression.cpp +6 -4
- package/src/duckdb/src/planner/binder/expression/bind_lambda.cpp +3 -2
- package/src/duckdb/src/planner/binder/expression/bind_star_expression.cpp +176 -0
- package/src/duckdb/src/planner/binder/query_node/bind_select_node.cpp +57 -82
- package/src/duckdb/src/planner/binder/query_node/plan_query_node.cpp +11 -0
- package/src/duckdb/src/planner/binder/statement/bind_delete.cpp +1 -1
- package/src/duckdb/src/planner/binder/statement/bind_insert.cpp +2 -2
- package/src/duckdb/src/planner/binder/statement/bind_update.cpp +1 -1
- package/src/duckdb/src/planner/binder.cpp +12 -23
- package/src/duckdb/src/planner/bound_result_modifier.cpp +26 -0
- package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +9 -2
- package/src/duckdb/src/planner/expression_iterator.cpp +5 -0
- package/src/duckdb/src/planner/logical_operator_visitor.cpp +5 -0
- package/src/duckdb/src/planner/operator/logical_distinct.cpp +3 -0
- package/src/duckdb/src/storage/storage_info.cpp +1 -1
- package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +1 -1
- package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +8141 -8313
- package/src/duckdb/ub_src_optimizer_rule.cpp +2 -0
- package/src/duckdb/ub_src_planner_binder_expression.cpp +2 -0
- package/src/duckdb_node.hpp +2 -1
- package/src/statement.cpp +5 -5
- package/src/utils.cpp +15 -2
- package/test/syntax_error.test.ts +3 -1
package/binding.gyp
CHANGED
@@ -230,6 +230,7 @@
|
|
230
230
|
"src/duckdb/extension/icu/./icu-datefunc.cpp",
|
231
231
|
"src/duckdb/extension/icu/./icu-extension.cpp",
|
232
232
|
"src/duckdb/extension/icu/./icu-makedate.cpp",
|
233
|
+
"src/duckdb/extension/icu/./icu-list-range.cpp",
|
233
234
|
"src/duckdb/extension/icu/./icu-timezone.cpp",
|
234
235
|
"src/duckdb/extension/icu/./icu-datesub.cpp",
|
235
236
|
"src/duckdb/extension/icu/./icu-timebucket.cpp",
|
package/lib/duckdb.d.ts
CHANGED
@@ -4,12 +4,54 @@
|
|
4
4
|
* on Node.JS API
|
5
5
|
*/
|
6
6
|
|
7
|
+
export type ExceptionType =
|
8
|
+
| "Invalid" // invalid type
|
9
|
+
| "Out of Range" // value out of range error
|
10
|
+
| "Conversion" // conversion/casting error
|
11
|
+
| "Unknown Type" // unknown type
|
12
|
+
| "Decimal" // decimal related
|
13
|
+
| "Mismatch Type" // type mismatch
|
14
|
+
| "Divide by Zero" // divide by 0
|
15
|
+
| "Object Size" // object size exceeded
|
16
|
+
| "Invalid type" // incompatible for operation
|
17
|
+
| "Serialization" // serialization
|
18
|
+
| "TransactionContext" // transaction management
|
19
|
+
| "Not implemented" // method not implemented
|
20
|
+
| "Expression" // expression parsing
|
21
|
+
| "Catalog" // catalog related
|
22
|
+
| "Parser" // parser related
|
23
|
+
| "Binder" // binder related
|
24
|
+
| "Planner" // planner related
|
25
|
+
| "Scheduler" // scheduler related
|
26
|
+
| "Executor" // executor related
|
27
|
+
| "Constraint" // constraint related
|
28
|
+
| "Index" // index related
|
29
|
+
| "Stat" // stat related
|
30
|
+
| "Connection" // connection related
|
31
|
+
| "Syntax" // syntax related
|
32
|
+
| "Settings" // settings related
|
33
|
+
| "Optimizer" // optimizer related
|
34
|
+
| "NullPointer" // nullptr exception
|
35
|
+
| "IO" // IO exception
|
36
|
+
| "INTERRUPT" // interrupt
|
37
|
+
| "FATAL" // Fatal exceptions are non-recoverable and render the entire DB in an unusable state
|
38
|
+
| "INTERNAL" // Internal exceptions indicate something went wrong internally (i.e. bug in the code base)
|
39
|
+
| "Invalid Input" // Input or arguments error
|
40
|
+
| "Out of Memory" // out of memory
|
41
|
+
| "Permission" // insufficient permissions
|
42
|
+
| "Parameter Not Resolved" // parameter types could not be resolved
|
43
|
+
| "Parameter Not Allowed" // parameter types not allowed
|
44
|
+
| "Dependency" // dependency
|
45
|
+
| "Unknown"
|
46
|
+
;
|
47
|
+
|
7
48
|
/**
|
8
49
|
* Standard error shape for DuckDB errors
|
9
50
|
*/
|
10
51
|
export interface DuckDbError extends Error {
|
11
52
|
errno: -1; // value of ERROR
|
12
53
|
code: 'DUCKDB_NODEJS_ERROR';
|
54
|
+
errorType: ExceptionType;
|
13
55
|
}
|
14
56
|
|
15
57
|
type Callback<T> = (err: DuckDbError | null, res: T) => void;
|
package/package.json
CHANGED
package/src/connection.cpp
CHANGED
@@ -388,8 +388,7 @@ struct ExecTask : public Task {
|
|
388
388
|
void Callback() override {
|
389
389
|
auto env = object.Env();
|
390
390
|
Napi::HandleScope scope(env);
|
391
|
-
callback.Value().MakeCallback(object.Value(),
|
392
|
-
{success ? env.Null() : Utils::CreateError(env, error.Message())});
|
391
|
+
callback.Value().MakeCallback(object.Value(), {success ? env.Null() : Utils::CreateError(env, error)});
|
393
392
|
};
|
394
393
|
|
395
394
|
std::string sql;
|
package/src/database.cpp
CHANGED
@@ -13,6 +13,7 @@
|
|
13
13
|
#include "include/icu-datesub.hpp"
|
14
14
|
#include "include/icu-datetrunc.hpp"
|
15
15
|
#include "include/icu-makedate.hpp"
|
16
|
+
#include "include/icu-list-range.hpp"
|
16
17
|
#include "include/icu-table-range.hpp"
|
17
18
|
#include "include/icu-strptime.hpp"
|
18
19
|
#include "include/icu-timebucket.hpp"
|
@@ -268,6 +269,7 @@ void ICUExtension::Load(DuckDB &db) {
|
|
268
269
|
RegisterICUDateTruncFunctions(*con.context);
|
269
270
|
RegisterICUMakeDateFunctions(*con.context);
|
270
271
|
RegisterICUTableRangeFunctions(*con.context);
|
272
|
+
RegisterICUListRangeFunctions(*con.context);
|
271
273
|
RegisterICUStrptimeFunctions(*con.context);
|
272
274
|
RegisterICUTimeBucketFunctions(*con.context);
|
273
275
|
RegisterICUTimeZoneFunctions(*con.context);
|
@@ -0,0 +1,207 @@
|
|
1
|
+
#include "duckdb/common/exception.hpp"
|
2
|
+
#include "duckdb/common/types/interval.hpp"
|
3
|
+
#include "duckdb/common/types/timestamp.hpp"
|
4
|
+
#include "duckdb/common/types/vector.hpp"
|
5
|
+
#include "duckdb/function/function_set.hpp"
|
6
|
+
#include "duckdb/function/scalar_function.hpp"
|
7
|
+
#include "duckdb/main/client_context.hpp"
|
8
|
+
#include "duckdb/parser/parsed_data/create_scalar_function_info.hpp"
|
9
|
+
#include "include/icu-datefunc.hpp"
|
10
|
+
|
11
|
+
namespace duckdb {
|
12
|
+
|
13
|
+
struct ICUListRange : public ICUDateFunc {
|
14
|
+
template <bool INCLUSIVE_BOUND>
|
15
|
+
class RangeInfoStruct {
|
16
|
+
public:
|
17
|
+
explicit RangeInfoStruct(DataChunk &args_p) : args(args_p) {
|
18
|
+
if (args.ColumnCount() == 3) {
|
19
|
+
args.data[0].ToUnifiedFormat(args.size(), vdata[0]);
|
20
|
+
args.data[1].ToUnifiedFormat(args.size(), vdata[1]);
|
21
|
+
args.data[2].ToUnifiedFormat(args.size(), vdata[2]);
|
22
|
+
} else {
|
23
|
+
throw InternalException("Unsupported number of parameters for range");
|
24
|
+
}
|
25
|
+
}
|
26
|
+
|
27
|
+
bool RowIsValid(idx_t row_idx) {
|
28
|
+
for (idx_t i = 0; i < args.ColumnCount(); i++) {
|
29
|
+
auto idx = vdata[i].sel->get_index(row_idx);
|
30
|
+
if (!vdata[i].validity.RowIsValid(idx)) {
|
31
|
+
return false;
|
32
|
+
}
|
33
|
+
}
|
34
|
+
return true;
|
35
|
+
}
|
36
|
+
|
37
|
+
timestamp_t StartListValue(idx_t row_idx) {
|
38
|
+
auto data = (timestamp_t *)vdata[0].data;
|
39
|
+
auto idx = vdata[0].sel->get_index(row_idx);
|
40
|
+
return data[idx];
|
41
|
+
}
|
42
|
+
|
43
|
+
timestamp_t EndListValue(idx_t row_idx) {
|
44
|
+
auto data = (timestamp_t *)vdata[1].data;
|
45
|
+
auto idx = vdata[1].sel->get_index(row_idx);
|
46
|
+
return data[idx];
|
47
|
+
}
|
48
|
+
|
49
|
+
interval_t ListIncrementValue(idx_t row_idx) {
|
50
|
+
auto data = (interval_t *)vdata[2].data;
|
51
|
+
auto idx = vdata[2].sel->get_index(row_idx);
|
52
|
+
return data[idx];
|
53
|
+
}
|
54
|
+
|
55
|
+
void GetListValues(idx_t row_idx, timestamp_t &start_value, timestamp_t &end_value,
|
56
|
+
interval_t &increment_value) {
|
57
|
+
start_value = StartListValue(row_idx);
|
58
|
+
end_value = EndListValue(row_idx);
|
59
|
+
increment_value = ListIncrementValue(row_idx);
|
60
|
+
}
|
61
|
+
|
62
|
+
uint64_t ListLength(idx_t row_idx, icu::Calendar *calendar) {
|
63
|
+
timestamp_t start_value;
|
64
|
+
timestamp_t end_value;
|
65
|
+
interval_t increment_value;
|
66
|
+
GetListValues(row_idx, start_value, end_value, increment_value);
|
67
|
+
return ListLength(start_value, end_value, increment_value, INCLUSIVE_BOUND, calendar);
|
68
|
+
}
|
69
|
+
|
70
|
+
void Increment(timestamp_t &input, interval_t increment, icu::Calendar *calendar) {
|
71
|
+
input = Add(calendar, input, increment);
|
72
|
+
}
|
73
|
+
|
74
|
+
private:
|
75
|
+
DataChunk &args;
|
76
|
+
UnifiedVectorFormat vdata[3];
|
77
|
+
|
78
|
+
uint64_t ListLength(timestamp_t start_value, timestamp_t end_value, interval_t increment_value,
|
79
|
+
bool inclusive_bound, icu::Calendar *calendar) {
|
80
|
+
bool is_positive = increment_value.months > 0 || increment_value.days > 0 || increment_value.micros > 0;
|
81
|
+
bool is_negative = increment_value.months < 0 || increment_value.days < 0 || increment_value.micros < 0;
|
82
|
+
if (!is_negative && !is_positive) {
|
83
|
+
// interval is 0: no result
|
84
|
+
return 0;
|
85
|
+
}
|
86
|
+
// We don't allow infinite bounds because they generate errors or infinite loops
|
87
|
+
if (!Timestamp::IsFinite(start_value) || !Timestamp::IsFinite(end_value)) {
|
88
|
+
throw InvalidInputException("Interval infinite bounds not supported");
|
89
|
+
}
|
90
|
+
|
91
|
+
if (is_negative && is_positive) {
|
92
|
+
// we don't allow a mix of
|
93
|
+
throw InvalidInputException("Interval with mix of negative/positive entries not supported");
|
94
|
+
}
|
95
|
+
if (start_value > end_value && is_positive) {
|
96
|
+
return 0;
|
97
|
+
}
|
98
|
+
if (start_value < end_value && is_negative) {
|
99
|
+
return 0;
|
100
|
+
}
|
101
|
+
int64_t total_values = 0;
|
102
|
+
if (is_negative) {
|
103
|
+
// negative interval, start_value is going down
|
104
|
+
while (inclusive_bound ? start_value >= end_value : start_value > end_value) {
|
105
|
+
start_value = Add(calendar, start_value, increment_value);
|
106
|
+
total_values++;
|
107
|
+
if (total_values > NumericLimits<uint32_t>::Maximum()) {
|
108
|
+
throw InvalidInputException("Lists larger than 2^32 elements are not supported");
|
109
|
+
}
|
110
|
+
}
|
111
|
+
} else {
|
112
|
+
// positive interval, start_value is going up
|
113
|
+
while (inclusive_bound ? start_value <= end_value : start_value < end_value) {
|
114
|
+
start_value = Add(calendar, start_value, increment_value);
|
115
|
+
total_values++;
|
116
|
+
if (total_values > NumericLimits<uint32_t>::Maximum()) {
|
117
|
+
throw InvalidInputException("Lists larger than 2^32 elements are not supported");
|
118
|
+
}
|
119
|
+
}
|
120
|
+
}
|
121
|
+
return total_values;
|
122
|
+
}
|
123
|
+
};
|
124
|
+
|
125
|
+
template <bool INCLUSIVE_BOUND>
|
126
|
+
static void ICUListRangeFunction(DataChunk &args, ExpressionState &state, Vector &result) {
|
127
|
+
D_ASSERT(result.GetType().id() == LogicalTypeId::LIST);
|
128
|
+
D_ASSERT(args.ColumnCount() == 3);
|
129
|
+
|
130
|
+
auto &func_expr = (BoundFunctionExpression &)state.expr;
|
131
|
+
auto &bind_info = (BindData &)*func_expr.bind_info;
|
132
|
+
CalendarPtr calendar_ptr(bind_info.calendar->clone());
|
133
|
+
auto calendar = calendar_ptr.get();
|
134
|
+
|
135
|
+
RangeInfoStruct<INCLUSIVE_BOUND> info(args);
|
136
|
+
idx_t args_size = 1;
|
137
|
+
auto result_type = VectorType::CONSTANT_VECTOR;
|
138
|
+
for (idx_t i = 0; i < args.ColumnCount(); i++) {
|
139
|
+
if (args.data[i].GetVectorType() != VectorType::CONSTANT_VECTOR) {
|
140
|
+
args_size = args.size();
|
141
|
+
result_type = VectorType::FLAT_VECTOR;
|
142
|
+
break;
|
143
|
+
}
|
144
|
+
}
|
145
|
+
auto list_data = FlatVector::GetData<list_entry_t>(result);
|
146
|
+
auto &result_validity = FlatVector::Validity(result);
|
147
|
+
int64_t total_size = 0;
|
148
|
+
for (idx_t i = 0; i < args_size; i++) {
|
149
|
+
if (!info.RowIsValid(i)) {
|
150
|
+
result_validity.SetInvalid(i);
|
151
|
+
list_data[i].offset = total_size;
|
152
|
+
list_data[i].length = 0;
|
153
|
+
} else {
|
154
|
+
list_data[i].offset = total_size;
|
155
|
+
list_data[i].length = info.ListLength(i, calendar);
|
156
|
+
total_size += list_data[i].length;
|
157
|
+
}
|
158
|
+
}
|
159
|
+
|
160
|
+
// now construct the child vector of the list
|
161
|
+
ListVector::Reserve(result, total_size);
|
162
|
+
auto range_data = FlatVector::GetData<timestamp_t>(ListVector::GetEntry(result));
|
163
|
+
idx_t total_idx = 0;
|
164
|
+
for (idx_t i = 0; i < args_size; i++) {
|
165
|
+
timestamp_t start_value = info.StartListValue(i);
|
166
|
+
interval_t increment = info.ListIncrementValue(i);
|
167
|
+
|
168
|
+
timestamp_t range_value = start_value;
|
169
|
+
for (idx_t range_idx = 0; range_idx < list_data[i].length; range_idx++) {
|
170
|
+
if (range_idx > 0) {
|
171
|
+
info.Increment(range_value, increment, calendar);
|
172
|
+
}
|
173
|
+
range_data[total_idx++] = range_value;
|
174
|
+
}
|
175
|
+
}
|
176
|
+
|
177
|
+
ListVector::SetListSize(result, total_size);
|
178
|
+
result.SetVectorType(result_type);
|
179
|
+
|
180
|
+
result.Verify(args.size());
|
181
|
+
}
|
182
|
+
|
183
|
+
static void AddICUListRangeFunction(ClientContext &context) {
|
184
|
+
auto &catalog = Catalog::GetSystemCatalog(context);
|
185
|
+
|
186
|
+
ScalarFunctionSet range("range");
|
187
|
+
range.AddFunction(ScalarFunction({LogicalType::TIMESTAMP_TZ, LogicalType::TIMESTAMP_TZ, LogicalType::INTERVAL},
|
188
|
+
LogicalType::LIST(LogicalType::TIMESTAMP_TZ), ICUListRangeFunction<false>,
|
189
|
+
Bind));
|
190
|
+
CreateScalarFunctionInfo range_func_info(range);
|
191
|
+
catalog.AddFunction(context, &range_func_info);
|
192
|
+
|
193
|
+
// generate_series: similar to range, but inclusive instead of exclusive bounds on the RHS
|
194
|
+
ScalarFunctionSet generate_series("generate_series");
|
195
|
+
generate_series.AddFunction(
|
196
|
+
ScalarFunction({LogicalType::TIMESTAMP_TZ, LogicalType::TIMESTAMP_TZ, LogicalType::INTERVAL},
|
197
|
+
LogicalType::LIST(LogicalType::TIMESTAMP_TZ), ICUListRangeFunction<true>, Bind));
|
198
|
+
CreateScalarFunctionInfo generate_series_func_info(generate_series);
|
199
|
+
catalog.AddFunction(context, &generate_series_func_info);
|
200
|
+
}
|
201
|
+
};
|
202
|
+
|
203
|
+
void RegisterICUListRangeFunctions(ClientContext &context) {
|
204
|
+
ICUListRange::AddICUListRangeFunction(context);
|
205
|
+
}
|
206
|
+
|
207
|
+
} // namespace duckdb
|
@@ -0,0 +1,17 @@
|
|
1
|
+
//===----------------------------------------------------------------------===//
|
2
|
+
// DuckDB
|
3
|
+
//
|
4
|
+
// icu-list-range.hpp
|
5
|
+
//
|
6
|
+
//
|
7
|
+
//===----------------------------------------------------------------------===//
|
8
|
+
|
9
|
+
#pragma once
|
10
|
+
|
11
|
+
#include "duckdb.hpp"
|
12
|
+
|
13
|
+
namespace duckdb {
|
14
|
+
|
15
|
+
void RegisterICUListRangeFunctions(ClientContext &context);
|
16
|
+
|
17
|
+
} // namespace duckdb
|
@@ -20,6 +20,12 @@ void JSONScan::AutoDetect(ClientContext &context, JSONScanData &bind_data, vecto
|
|
20
20
|
idx_t remaining = bind_data.sample_size;
|
21
21
|
while (remaining != 0) {
|
22
22
|
allocator.Reset();
|
23
|
+
|
24
|
+
if (gstate.file_index >= 10) {
|
25
|
+
// We really shouldn't open more than 10 files when sampling
|
26
|
+
break;
|
27
|
+
}
|
28
|
+
|
23
29
|
auto read_count = lstate.ReadNext(gstate);
|
24
30
|
if (lstate.scan_count > 1) {
|
25
31
|
more_than_one = true;
|
@@ -46,11 +52,6 @@ void JSONScan::AutoDetect(ClientContext &context, JSONScanData &bind_data, vecto
|
|
46
52
|
node.InitializeCandidateTypes(bind_data.max_depth);
|
47
53
|
node.RefineCandidateTypes(values, next, string_vector, allocator, bind_data.date_format_map);
|
48
54
|
remaining -= next;
|
49
|
-
|
50
|
-
if (gstate.file_index == 10) {
|
51
|
-
// We really shouldn't open more than 10 files when sampling
|
52
|
-
break;
|
53
|
-
}
|
54
55
|
}
|
55
56
|
bind_data.type = original_scan_type;
|
56
57
|
|
@@ -16,7 +16,6 @@ struct Int96 {
|
|
16
16
|
uint32_t value[3];
|
17
17
|
};
|
18
18
|
|
19
|
-
int64_t ImpalaTimestampToNanoseconds(const Int96 &impala_timestamp);
|
20
19
|
timestamp_t ImpalaTimestampToTimestamp(const Int96 &raw_ts);
|
21
20
|
Int96 TimestampToImpalaTimestamp(timestamp_t &ts);
|
22
21
|
timestamp_t ParquetTimestampMicrosToTimestamp(const int64_t &raw_ts);
|
@@ -12,24 +12,26 @@ namespace duckdb {
|
|
12
12
|
// surely they are joking
|
13
13
|
static constexpr int64_t JULIAN_TO_UNIX_EPOCH_DAYS = 2440588LL;
|
14
14
|
static constexpr int64_t MILLISECONDS_PER_DAY = 86400000LL;
|
15
|
-
static constexpr int64_t
|
15
|
+
static constexpr int64_t MICROSECONDS_PER_DAY = MILLISECONDS_PER_DAY * 1000LL;
|
16
|
+
static constexpr int64_t NANOSECONDS_PER_MICRO = 1000LL;
|
16
17
|
|
17
|
-
int64_t
|
18
|
+
static int64_t ImpalaTimestampToMicroseconds(const Int96 &impala_timestamp) {
|
18
19
|
int64_t days_since_epoch = impala_timestamp.value[2] - JULIAN_TO_UNIX_EPOCH_DAYS;
|
19
20
|
auto nanoseconds = Load<int64_t>((data_ptr_t)impala_timestamp.value);
|
20
|
-
|
21
|
+
auto microseconds = nanoseconds / NANOSECONDS_PER_MICRO;
|
22
|
+
return days_since_epoch * MICROSECONDS_PER_DAY + microseconds;
|
21
23
|
}
|
22
24
|
|
23
25
|
timestamp_t ImpalaTimestampToTimestamp(const Int96 &raw_ts) {
|
24
|
-
auto
|
25
|
-
return Timestamp::
|
26
|
+
auto impala_us = ImpalaTimestampToMicroseconds(raw_ts);
|
27
|
+
return Timestamp::FromEpochMicroSeconds(impala_us);
|
26
28
|
}
|
27
29
|
|
28
30
|
Int96 TimestampToImpalaTimestamp(timestamp_t &ts) {
|
29
31
|
int32_t hour, min, sec, msec;
|
30
32
|
Time::Convert(Timestamp::GetTime(ts), hour, min, sec, msec);
|
31
33
|
uint64_t ms_since_midnight = hour * 60 * 60 * 1000 + min * 60 * 1000 + sec * 1000 + msec;
|
32
|
-
auto days_since_epoch = Date::Epoch(Timestamp::GetDate(ts)) / (24 * 60 * 60);
|
34
|
+
auto days_since_epoch = Date::Epoch(Timestamp::GetDate(ts)) / int64_t(24 * 60 * 60);
|
33
35
|
// first two uint32 in Int96 are nanoseconds since midnights
|
34
36
|
// last uint32 is number of days since year 4713 BC ("Julian date")
|
35
37
|
Int96 impala_ts;
|
@@ -138,12 +138,22 @@ string Exception::ExceptionTypeToString(ExceptionType type) {
|
|
138
138
|
return "Parameter Not Allowed";
|
139
139
|
case ExceptionType::DEPENDENCY:
|
140
140
|
return "Dependency";
|
141
|
+
case ExceptionType::HTTP:
|
142
|
+
return "HTTP";
|
141
143
|
default:
|
142
144
|
return "Unknown";
|
143
145
|
}
|
144
146
|
}
|
145
147
|
|
146
|
-
|
148
|
+
const HTTPException &Exception::AsHTTPException() const {
|
149
|
+
D_ASSERT(type == ExceptionType::HTTP);
|
150
|
+
const auto &e = static_cast<const HTTPException *>(this);
|
151
|
+
D_ASSERT(e->GetStatusCode() != 0);
|
152
|
+
return *e;
|
153
|
+
}
|
154
|
+
|
155
|
+
void Exception::ThrowAsTypeWithMessage(ExceptionType type, const string &message,
|
156
|
+
const std::shared_ptr<Exception> &original) {
|
147
157
|
switch (type) {
|
148
158
|
case ExceptionType::OUT_OF_RANGE:
|
149
159
|
throw OutOfRangeException(message);
|
@@ -191,6 +201,10 @@ void Exception::ThrowAsTypeWithMessage(ExceptionType type, const string &message
|
|
191
201
|
throw FatalException(message);
|
192
202
|
case ExceptionType::DEPENDENCY:
|
193
203
|
throw DependencyException(message);
|
204
|
+
case ExceptionType::HTTP: {
|
205
|
+
auto exc = original->AsHTTPException();
|
206
|
+
throw HTTPException(exc.GetStatusCode(), exc.GetResponse(), exc.what());
|
207
|
+
}
|
194
208
|
default:
|
195
209
|
throw Exception(type, message);
|
196
210
|
}
|
@@ -7,15 +7,17 @@
|
|
7
7
|
|
8
8
|
namespace duckdb {
|
9
9
|
|
10
|
-
PreservedError::PreservedError() : initialized(false) {
|
10
|
+
PreservedError::PreservedError() : initialized(false), exception_instance(nullptr) {
|
11
11
|
}
|
12
12
|
|
13
13
|
PreservedError::PreservedError(const Exception &exception)
|
14
|
-
: initialized(true), type(exception.type), raw_message(SanitizeErrorMessage(exception.RawMessage()))
|
14
|
+
: initialized(true), type(exception.type), raw_message(SanitizeErrorMessage(exception.RawMessage())),
|
15
|
+
exception_instance(exception.Copy()) {
|
15
16
|
}
|
16
17
|
|
17
18
|
PreservedError::PreservedError(const string &message)
|
18
|
-
: initialized(true), type(ExceptionType::INVALID), raw_message(SanitizeErrorMessage(message))
|
19
|
+
: initialized(true), type(ExceptionType::INVALID), raw_message(SanitizeErrorMessage(message)),
|
20
|
+
exception_instance(nullptr) {
|
19
21
|
}
|
20
22
|
|
21
23
|
const string &PreservedError::Message() {
|
@@ -33,9 +35,9 @@ void PreservedError::Throw(const string &prepended_message) const {
|
|
33
35
|
D_ASSERT(initialized);
|
34
36
|
if (!prepended_message.empty()) {
|
35
37
|
string new_message = prepended_message + raw_message;
|
36
|
-
Exception::ThrowAsTypeWithMessage(type, new_message);
|
38
|
+
Exception::ThrowAsTypeWithMessage(type, new_message, exception_instance);
|
37
39
|
}
|
38
|
-
Exception::ThrowAsTypeWithMessage(type, raw_message);
|
40
|
+
Exception::ThrowAsTypeWithMessage(type, raw_message, exception_instance);
|
39
41
|
}
|
40
42
|
|
41
43
|
const ExceptionType &PreservedError::Type() const {
|
@@ -12,13 +12,28 @@ namespace duckdb {
|
|
12
12
|
PhysicalPositionalScan::PhysicalPositionalScan(vector<LogicalType> types, unique_ptr<PhysicalOperator> left,
|
13
13
|
unique_ptr<PhysicalOperator> right)
|
14
14
|
: PhysicalOperator(PhysicalOperatorType::POSITIONAL_SCAN, std::move(types),
|
15
|
-
|
15
|
+
MaxValue(left->estimated_cardinality, right->estimated_cardinality)) {
|
16
16
|
|
17
17
|
// Manage the children ourselves
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
18
|
+
if (left->type == PhysicalOperatorType::TABLE_SCAN) {
|
19
|
+
child_tables.emplace_back(std::move(left));
|
20
|
+
} else if (left->type == PhysicalOperatorType::POSITIONAL_SCAN) {
|
21
|
+
auto &left_scan = (PhysicalPositionalScan &)*left;
|
22
|
+
child_tables = std::move(left_scan.child_tables);
|
23
|
+
} else {
|
24
|
+
throw InternalException("Invalid left input for PhysicalPositionalScan");
|
25
|
+
}
|
26
|
+
|
27
|
+
if (right->type == PhysicalOperatorType::TABLE_SCAN) {
|
28
|
+
child_tables.emplace_back(std::move(right));
|
29
|
+
} else if (right->type == PhysicalOperatorType::POSITIONAL_SCAN) {
|
30
|
+
auto &right_scan = (PhysicalPositionalScan &)*right;
|
31
|
+
auto &right_tables = right_scan.child_tables;
|
32
|
+
child_tables.reserve(child_tables.size() + right_tables.size());
|
33
|
+
std::move(right_tables.begin(), right_tables.end(), std::back_inserter(child_tables));
|
34
|
+
} else {
|
35
|
+
throw InternalException("Invalid right input for PhysicalPositionalScan");
|
36
|
+
}
|
22
37
|
}
|
23
38
|
|
24
39
|
class PositionalScanGlobalSourceState : public GlobalSourceState {
|
@@ -9,6 +9,7 @@
|
|
9
9
|
#include "duckdb/parser/expression/comparison_expression.hpp"
|
10
10
|
#include "duckdb/planner/expression/bound_aggregate_expression.hpp"
|
11
11
|
#include "duckdb/planner/operator/logical_aggregate.hpp"
|
12
|
+
#include "duckdb/function/function_binder.hpp"
|
12
13
|
|
13
14
|
namespace duckdb {
|
14
15
|
|
@@ -169,13 +170,20 @@ PhysicalPlanGenerator::ExtractAggregateExpressions(unique_ptr<PhysicalOperator>
|
|
169
170
|
vector<unique_ptr<Expression>> expressions;
|
170
171
|
vector<LogicalType> types;
|
171
172
|
|
173
|
+
// bind sorted aggregates
|
174
|
+
for (auto &aggr : aggregates) {
|
175
|
+
auto &bound_aggr = (BoundAggregateExpression &)*aggr;
|
176
|
+
if (bound_aggr.order_bys) {
|
177
|
+
// sorted aggregate!
|
178
|
+
FunctionBinder::BindSortedAggregate(context, bound_aggr, groups);
|
179
|
+
}
|
180
|
+
}
|
172
181
|
for (auto &group : groups) {
|
173
182
|
auto ref = make_unique<BoundReferenceExpression>(group->return_type, expressions.size());
|
174
183
|
types.push_back(group->return_type);
|
175
184
|
expressions.push_back(std::move(group));
|
176
185
|
group = std::move(ref);
|
177
186
|
}
|
178
|
-
|
179
187
|
for (auto &aggr : aggregates) {
|
180
188
|
auto &bound_aggr = (BoundAggregateExpression &)*aggr;
|
181
189
|
for (auto &child : bound_aggr.children) {
|
@@ -9,8 +9,10 @@
|
|
9
9
|
|
10
10
|
namespace duckdb {
|
11
11
|
|
12
|
-
unique_ptr<PhysicalOperator> PhysicalPlanGenerator::
|
13
|
-
|
12
|
+
unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalDistinct &op) {
|
13
|
+
D_ASSERT(op.children.size() == 1);
|
14
|
+
auto child = CreatePlan(*op.children[0]);
|
15
|
+
auto &distinct_targets = op.distinct_targets;
|
14
16
|
D_ASSERT(child);
|
15
17
|
D_ASSERT(!distinct_targets.empty());
|
16
18
|
|
@@ -55,6 +57,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreateDistinctOn(unique_ptr<
|
|
55
57
|
FunctionBinder function_binder(context);
|
56
58
|
auto first_aggregate = function_binder.BindAggregateFunction(
|
57
59
|
FirstFun::GetFunction(logical_type), std::move(first_children), nullptr, AggregateType::NON_DISTINCT);
|
60
|
+
first_aggregate->order_bys = op.order_by ? op.order_by->Copy() : nullptr;
|
58
61
|
// add the projection
|
59
62
|
projections.push_back(make_unique<BoundReferenceExpression>(logical_type, group_count + aggregates.size()));
|
60
63
|
// push it to the list of aggregates
|
@@ -81,10 +84,4 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreateDistinctOn(unique_ptr<
|
|
81
84
|
return std::move(aggr_projection);
|
82
85
|
}
|
83
86
|
|
84
|
-
unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalDistinct &op) {
|
85
|
-
D_ASSERT(op.children.size() == 1);
|
86
|
-
auto plan = CreatePlan(*op.children[0]);
|
87
|
-
return CreateDistinctOn(std::move(plan), std::move(op.distinct_targets));
|
88
|
-
}
|
89
|
-
|
90
87
|
} // namespace duckdb
|
@@ -10,12 +10,21 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalPositional
|
|
10
10
|
|
11
11
|
auto left = CreatePlan(*op.children[0]);
|
12
12
|
auto right = CreatePlan(*op.children[1]);
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
13
|
+
switch (left->type) {
|
14
|
+
case PhysicalOperatorType::TABLE_SCAN:
|
15
|
+
case PhysicalOperatorType::POSITIONAL_SCAN:
|
16
|
+
switch (right->type) {
|
17
|
+
case PhysicalOperatorType::TABLE_SCAN:
|
18
|
+
case PhysicalOperatorType::POSITIONAL_SCAN:
|
19
|
+
return make_unique<PhysicalPositionalScan>(op.types, std::move(left), std::move(right));
|
20
|
+
default:
|
21
|
+
break;
|
22
|
+
}
|
23
|
+
default:
|
24
|
+
break;
|
18
25
|
}
|
26
|
+
|
27
|
+
return make_unique<PhysicalPositionalJoin>(op.types, std::move(left), std::move(right), op.estimated_cardinality);
|
19
28
|
}
|
20
29
|
|
21
30
|
} // namespace duckdb
|
@@ -95,6 +95,7 @@ AggregateFunction BoolOrFun::GetFunction() {
|
|
95
95
|
auto fun = AggregateFunction::UnaryAggregate<BoolState, bool, bool, BoolOrFunFunction>(
|
96
96
|
LogicalType(LogicalTypeId::BOOLEAN), LogicalType::BOOLEAN);
|
97
97
|
fun.name = "bool_or";
|
98
|
+
fun.order_dependent = AggregateOrderDependent::NOT_ORDER_DEPENDENT;
|
98
99
|
return fun;
|
99
100
|
}
|
100
101
|
|
@@ -102,6 +103,7 @@ AggregateFunction BoolAndFun::GetFunction() {
|
|
102
103
|
auto fun = AggregateFunction::UnaryAggregate<BoolState, bool, bool, BoolAndFunFunction>(
|
103
104
|
LogicalType(LogicalTypeId::BOOLEAN), LogicalType::BOOLEAN);
|
104
105
|
fun.name = "bool_and";
|
106
|
+
fun.order_dependent = AggregateOrderDependent::NOT_ORDER_DEPENDENT;
|
105
107
|
return fun;
|
106
108
|
}
|
107
109
|
|
@@ -76,6 +76,7 @@ AggregateFunction CountFun::GetFunction() {
|
|
76
76
|
LogicalType(LogicalTypeId::ANY), LogicalType::BIGINT);
|
77
77
|
fun.name = "count";
|
78
78
|
fun.null_handling = FunctionNullHandling::SPECIAL_HANDLING;
|
79
|
+
fun.order_dependent = AggregateOrderDependent::NOT_ORDER_DEPENDENT;
|
79
80
|
return fun;
|
80
81
|
}
|
81
82
|
|
@@ -514,6 +514,7 @@ unique_ptr<FunctionData> BindDecimalMinMax(ClientContext &context, AggregateFunc
|
|
514
514
|
function.name = std::move(name);
|
515
515
|
function.arguments[0] = decimal_type;
|
516
516
|
function.return_type = decimal_type;
|
517
|
+
function.order_dependent = AggregateOrderDependent::NOT_ORDER_DEPENDENT;
|
517
518
|
return nullptr;
|
518
519
|
}
|
519
520
|
|
@@ -545,6 +546,7 @@ unique_ptr<FunctionData> BindMinMax(ClientContext &context, AggregateFunction &f
|
|
545
546
|
auto name = std::move(function.name);
|
546
547
|
function = GetMinMaxOperator<OP, OP_STRING, OP_VECTOR>(input_type);
|
547
548
|
function.name = std::move(name);
|
549
|
+
function.order_dependent = AggregateOrderDependent::NOT_ORDER_DEPENDENT;
|
548
550
|
if (function.bind) {
|
549
551
|
return function.bind(context, function, arguments);
|
550
552
|
} else {
|