duckdb 0.8.2-dev1859.0 → 0.8.2-dev1968.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +9 -9
- package/package.json +1 -1
- package/src/duckdb/src/common/enum_util.cpp +28 -0
- package/src/duckdb/src/common/types/hugeint.cpp +40 -0
- package/src/duckdb/src/core_functions/function_list.cpp +1 -0
- package/src/duckdb/src/core_functions/scalar/string/to_base.cpp +66 -0
- package/src/duckdb/src/execution/operator/aggregate/physical_hash_aggregate.cpp +14 -11
- package/src/duckdb/src/execution/operator/aggregate/physical_perfecthash_aggregate.cpp +6 -4
- package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +14 -12
- package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +6 -4
- package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +8 -6
- package/src/duckdb/src/execution/operator/helper/physical_explain_analyze.cpp +2 -2
- package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -3
- package/src/duckdb/src/execution/operator/helper/physical_materialized_collector.cpp +7 -5
- package/src/duckdb/src/execution/operator/helper/physical_vacuum.cpp +7 -5
- package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +5 -4
- package/src/duckdb/src/execution/operator/join/physical_blockwise_nl_join.cpp +2 -2
- package/src/duckdb/src/execution/operator/join/physical_delim_join.cpp +13 -6
- package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +7 -5
- package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +7 -5
- package/src/duckdb/src/execution/operator/join/physical_nested_loop_join.cpp +7 -4
- package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +8 -6
- package/src/duckdb/src/execution/operator/order/physical_order.cpp +7 -5
- package/src/duckdb/src/execution/operator/order/physical_top_n.cpp +7 -5
- package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +8 -6
- package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp +8 -7
- package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp +8 -6
- package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +11 -9
- package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +10 -10
- package/src/duckdb/src/execution/operator/persistent/physical_update.cpp +4 -2
- package/src/duckdb/src/execution/operator/schema/physical_create_index.cpp +7 -6
- package/src/duckdb/src/execution/physical_operator.cpp +3 -2
- package/src/duckdb/src/function/table/read_csv.cpp +4 -3
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/common/bitpacking.hpp +70 -55
- package/src/duckdb/src/include/duckdb/common/enum_util.hpp +8 -0
- package/src/duckdb/src/include/duckdb/common/enums/operator_result_type.hpp +5 -1
- package/src/duckdb/src/include/duckdb/common/hugeint.hpp +15 -0
- package/src/duckdb/src/include/duckdb/common/limits.hpp +52 -149
- package/src/duckdb/src/include/duckdb/common/numeric_utils.hpp +48 -0
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +1 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +1 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +9 -0
- package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_hash_aggregate.hpp +3 -3
- package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_perfecthash_aggregate.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_ungrouped_aggregate.hpp +3 -3
- package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_window.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_batch_collector.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_explain_analyze.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_limit.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_materialized_collector.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_vacuum.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_blockwise_nl_join.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_delim_join.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_hash_join.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_nested_loop_join.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/order/physical_order.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/order/physical_top_n.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_copy_to_file.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_insert.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_copy_to_file.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_fixed_batch_copy.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_insert.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_update.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_index.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/physical_operator.hpp +3 -3
- package/src/duckdb/src/include/duckdb/execution/physical_operator_states.hpp +11 -0
- package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +0 -3
- package/src/duckdb/src/include/duckdb/parallel/pipeline_executor.hpp +3 -2
- package/src/duckdb/src/main/relation/join_relation.cpp +1 -1
- package/src/duckdb/src/parallel/pipeline.cpp +0 -17
- package/src/duckdb/src/parallel/pipeline_executor.cpp +26 -7
- package/src/duckdb/src/parallel/pipeline_finish_event.cpp +55 -1
- package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +13 -3
- package/src/duckdb/src/storage/compression/bitpacking.cpp +87 -63
- package/src/duckdb/src/storage/compression/bitpacking_hugeint.cpp +295 -0
- package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
- package/src/duckdb/ub_src_core_functions_scalar_string.cpp +2 -0
- package/src/duckdb/ub_src_storage_compression.cpp +2 -0
package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_ungrouped_aggregate.hpp
CHANGED
@@ -44,9 +44,9 @@ public:
|
|
44
44
|
public:
|
45
45
|
// Sink interface
|
46
46
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
47
|
-
|
47
|
+
SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
|
48
48
|
SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
49
|
-
|
49
|
+
OperatorSinkFinalizeInput &input) const override;
|
50
50
|
|
51
51
|
unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
|
52
52
|
unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
|
@@ -68,7 +68,7 @@ private:
|
|
68
68
|
SinkFinalizeType FinalizeDistinct(Pipeline &pipeline, Event &event, ClientContext &context,
|
69
69
|
GlobalSinkState &gstate) const;
|
70
70
|
//! Combine the distinct aggregates
|
71
|
-
void CombineDistinct(ExecutionContext &context,
|
71
|
+
void CombineDistinct(ExecutionContext &context, OperatorSinkCombineInput &input) const;
|
72
72
|
//! Sink the distinct aggregates
|
73
73
|
void SinkDistinct(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const;
|
74
74
|
};
|
@@ -51,9 +51,9 @@ public:
|
|
51
51
|
public:
|
52
52
|
// Sink interface
|
53
53
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
54
|
-
|
54
|
+
SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
|
55
55
|
SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
56
|
-
|
56
|
+
OperatorSinkFinalizeInput &input) const override;
|
57
57
|
|
58
58
|
unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
|
59
59
|
unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
|
package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_batch_collector.hpp
CHANGED
@@ -22,9 +22,9 @@ public:
|
|
22
22
|
public:
|
23
23
|
// Sink interface
|
24
24
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
25
|
-
|
25
|
+
SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
|
26
26
|
SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
27
|
-
|
27
|
+
OperatorSinkFinalizeInput &input) const override;
|
28
28
|
|
29
29
|
unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
|
30
30
|
unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
|
package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_explain_analyze.hpp
CHANGED
@@ -34,7 +34,7 @@ public:
|
|
34
34
|
// Sink Interface
|
35
35
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
36
36
|
SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
37
|
-
|
37
|
+
OperatorSinkFinalizeInput &input) const override;
|
38
38
|
|
39
39
|
unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
|
40
40
|
|
@@ -44,7 +44,7 @@ public:
|
|
44
44
|
public:
|
45
45
|
// Sink Interface
|
46
46
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
47
|
-
|
47
|
+
SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
|
48
48
|
unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
|
49
49
|
unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
|
50
50
|
|
package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_materialized_collector.hpp
CHANGED
@@ -24,7 +24,7 @@ public:
|
|
24
24
|
public:
|
25
25
|
// Sink interface
|
26
26
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
27
|
-
|
27
|
+
SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
|
28
28
|
|
29
29
|
unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
|
30
30
|
unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
|
@@ -38,9 +38,9 @@ public:
|
|
38
38
|
unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
|
39
39
|
|
40
40
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
41
|
-
|
41
|
+
SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
|
42
42
|
SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
43
|
-
|
43
|
+
OperatorSinkFinalizeInput &input) const override;
|
44
44
|
|
45
45
|
bool IsSink() const override {
|
46
46
|
return info->has_table;
|
@@ -69,9 +69,9 @@ public:
|
|
69
69
|
unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
|
70
70
|
unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
|
71
71
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
72
|
-
|
72
|
+
SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
|
73
73
|
SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
74
|
-
|
74
|
+
OperatorSinkFinalizeInput &input) const override;
|
75
75
|
|
76
76
|
bool IsSink() const override {
|
77
77
|
return true;
|
package/src/duckdb/src/include/duckdb/execution/operator/join/physical_blockwise_nl_join.hpp
CHANGED
@@ -59,7 +59,7 @@ public:
|
|
59
59
|
unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
|
60
60
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
61
61
|
SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
62
|
-
|
62
|
+
OperatorSinkFinalizeInput &input) const override;
|
63
63
|
|
64
64
|
bool IsSink() const override {
|
65
65
|
return true;
|
@@ -35,9 +35,9 @@ public:
|
|
35
35
|
unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
|
36
36
|
unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
|
37
37
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
38
|
-
|
38
|
+
SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
|
39
39
|
SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
40
|
-
|
40
|
+
OperatorSinkFinalizeInput &input) const override;
|
41
41
|
|
42
42
|
bool IsSink() const override {
|
43
43
|
return true;
|
@@ -79,9 +79,9 @@ public:
|
|
79
79
|
|
80
80
|
unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
|
81
81
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
82
|
-
|
82
|
+
SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
|
83
83
|
SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
84
|
-
|
84
|
+
OperatorSinkFinalizeInput &input) const override;
|
85
85
|
|
86
86
|
bool IsSink() const override {
|
87
87
|
return true;
|
@@ -51,9 +51,9 @@ public:
|
|
51
51
|
unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
|
52
52
|
unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
|
53
53
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
54
|
-
|
54
|
+
SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
|
55
55
|
SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
56
|
-
|
56
|
+
OperatorSinkFinalizeInput &input) const override;
|
57
57
|
|
58
58
|
bool IsSink() const override {
|
59
59
|
return true;
|
@@ -54,9 +54,9 @@ public:
|
|
54
54
|
unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
|
55
55
|
unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
|
56
56
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
57
|
-
|
57
|
+
SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
|
58
58
|
SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
59
|
-
|
59
|
+
OperatorSinkFinalizeInput &input) const override;
|
60
60
|
|
61
61
|
bool IsSink() const override {
|
62
62
|
return true;
|
package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp
CHANGED
@@ -60,9 +60,9 @@ public:
|
|
60
60
|
unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
|
61
61
|
unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
|
62
62
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
63
|
-
|
63
|
+
SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
|
64
64
|
SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
65
|
-
|
65
|
+
OperatorSinkFinalizeInput &input) const override;
|
66
66
|
|
67
67
|
bool IsSink() const override {
|
68
68
|
return true;
|
@@ -60,9 +60,9 @@ public:
|
|
60
60
|
unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
|
61
61
|
unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
|
62
62
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
63
|
-
|
63
|
+
SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
|
64
64
|
SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
65
|
-
|
65
|
+
OperatorSinkFinalizeInput &input) const override;
|
66
66
|
|
67
67
|
bool IsSink() const override {
|
68
68
|
return true;
|
@@ -42,9 +42,9 @@ public:
|
|
42
42
|
|
43
43
|
public:
|
44
44
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
45
|
-
|
45
|
+
SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
|
46
46
|
SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
47
|
-
|
47
|
+
OperatorSinkFinalizeInput &input) const override;
|
48
48
|
unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
|
49
49
|
unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
|
50
50
|
|
package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_copy_to_file.hpp
CHANGED
@@ -41,9 +41,9 @@ public:
|
|
41
41
|
public:
|
42
42
|
// Sink interface
|
43
43
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
44
|
-
|
44
|
+
SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
|
45
45
|
SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
46
|
-
|
46
|
+
OperatorSinkFinalizeInput &input) const override;
|
47
47
|
unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
|
48
48
|
unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
|
49
49
|
void NextBatch(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate_p) const override;
|
package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_insert.hpp
CHANGED
@@ -54,9 +54,9 @@ public:
|
|
54
54
|
unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
|
55
55
|
void NextBatch(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate_p) const override;
|
56
56
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
57
|
-
|
57
|
+
SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
|
58
58
|
SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
59
|
-
|
59
|
+
OperatorSinkFinalizeInput &input) const override;
|
60
60
|
|
61
61
|
bool RequiresBatchIndex() const override {
|
62
62
|
return true;
|
package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_copy_to_file.hpp
CHANGED
@@ -50,9 +50,9 @@ public:
|
|
50
50
|
public:
|
51
51
|
// Sink interface
|
52
52
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
53
|
-
|
53
|
+
SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
|
54
54
|
SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
55
|
-
|
55
|
+
OperatorSinkFinalizeInput &input) const override;
|
56
56
|
unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
|
57
57
|
unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
|
58
58
|
|
package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_fixed_batch_copy.hpp
CHANGED
@@ -40,9 +40,9 @@ public:
|
|
40
40
|
public:
|
41
41
|
// Sink interface
|
42
42
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
43
|
-
|
43
|
+
SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
|
44
44
|
SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
45
|
-
|
45
|
+
OperatorSinkFinalizeInput &input) const override;
|
46
46
|
unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
|
47
47
|
unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
|
48
48
|
void NextBatch(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate_p) const override;
|
@@ -88,9 +88,9 @@ public:
|
|
88
88
|
unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
|
89
89
|
unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
|
90
90
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
91
|
-
|
91
|
+
SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
|
92
92
|
SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
93
|
-
|
93
|
+
OperatorSinkFinalizeInput &input) const override;
|
94
94
|
|
95
95
|
bool IsSink() const override {
|
96
96
|
return true;
|
@@ -47,7 +47,7 @@ public:
|
|
47
47
|
unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
|
48
48
|
unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
|
49
49
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
50
|
-
|
50
|
+
SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
|
51
51
|
|
52
52
|
bool IsSink() const override {
|
53
53
|
return true;
|
@@ -53,9 +53,9 @@ public:
|
|
53
53
|
unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
|
54
54
|
|
55
55
|
SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
|
56
|
-
|
56
|
+
SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
|
57
57
|
SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
58
|
-
|
58
|
+
OperatorSinkFinalizeInput &input) const override;
|
59
59
|
|
60
60
|
bool IsSink() const override {
|
61
61
|
return true;
|
@@ -140,14 +140,14 @@ public:
|
|
140
140
|
// The combine is called when a single thread has completed execution of its part of the pipeline, it is the final
|
141
141
|
// time that a specific LocalSinkState is accessible. This method can be called in parallel while other Sink() or
|
142
142
|
// Combine() calls are active on the same GlobalSinkState.
|
143
|
-
virtual
|
143
|
+
virtual SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const;
|
144
144
|
//! The finalize is called when ALL threads are finished execution. It is called only once per pipeline, and is
|
145
145
|
//! entirely single threaded.
|
146
146
|
//! If Finalize returns SinkResultType::FINISHED, the sink is marked as finished
|
147
147
|
virtual SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
148
|
-
|
148
|
+
OperatorSinkFinalizeInput &input) const;
|
149
149
|
//! For sinks with RequiresBatchIndex set to true, when a new batch starts being processed this method is called
|
150
|
-
//! This allows flushing of the current batch (e.g. to disk)
|
150
|
+
//! This allows flushing of the current batch (e.g. to disk) TODO: should this be able to block too?
|
151
151
|
virtual void NextBatch(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate_p) const;
|
152
152
|
|
153
153
|
virtual unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const;
|
@@ -165,6 +165,17 @@ struct OperatorSourceInput {
|
|
165
165
|
InterruptState &interrupt_state;
|
166
166
|
};
|
167
167
|
|
168
|
+
struct OperatorSinkCombineInput {
|
169
|
+
GlobalSinkState &global_state;
|
170
|
+
LocalSinkState &local_state;
|
171
|
+
InterruptState &interrupt_state;
|
172
|
+
};
|
173
|
+
|
174
|
+
struct OperatorSinkFinalizeInput {
|
175
|
+
GlobalSinkState &global_state;
|
176
|
+
InterruptState &interrupt_state;
|
177
|
+
};
|
178
|
+
|
168
179
|
// LCOV_EXCL_STOP
|
169
180
|
|
170
181
|
} // namespace duckdb
|
@@ -32,6 +32,9 @@ static constexpr ExtensionEntry EXTENSION_FUNCTIONS[] = {{"->>", "json"},
|
|
32
32
|
{"from_substrait_json", "substrait"},
|
33
33
|
{"get_substrait", "substrait"},
|
34
34
|
{"get_substrait_json", "substrait"},
|
35
|
+
{"iceberg_scan", "iceberg"},
|
36
|
+
{"iceberg_metadata", "iceberg"},
|
37
|
+
{"iceberg_snapshots", "iceberg"},
|
35
38
|
{"icu_calendar_names", "icu"},
|
36
39
|
{"icu_sort_key", "icu"},
|
37
40
|
{"json", "json"},
|
@@ -48,8 +48,8 @@ public:
|
|
48
48
|
//! If OperatorResultType::FINISHED is returned, more input will not change the result anymore
|
49
49
|
OperatorResultType ExecutePush(DataChunk &input);
|
50
50
|
//! Called after depleting the source: finalizes the execution of this pipeline executor
|
51
|
-
//! This should only be called once per PipelineExecutor
|
52
|
-
|
51
|
+
//! This should only be called once per PipelineExecutor.
|
52
|
+
PipelineExecuteResult PushFinalize();
|
53
53
|
|
54
54
|
//! Initializes a chunk with the types that will flow out of ExecutePull
|
55
55
|
void InitializeChunk(DataChunk &chunk);
|
@@ -142,6 +142,7 @@ private:
|
|
142
142
|
//! Debugging state: number of times blocked
|
143
143
|
int debug_blocked_sink_count = 0;
|
144
144
|
int debug_blocked_source_count = 0;
|
145
|
+
int debug_blocked_combine_count = 0;
|
145
146
|
//! Number of times the Sink/Source will block before actually returning data
|
146
147
|
int debug_blocked_target_count = 1;
|
147
148
|
#endif
|
@@ -52,7 +52,7 @@ const vector<ColumnDefinition> &JoinRelation::Columns() {
|
|
52
52
|
|
53
53
|
string JoinRelation::ToString(idx_t depth) {
|
54
54
|
string str = RenderWhitespace(depth);
|
55
|
-
str += "Join " + EnumUtil::ToString(join_type);
|
55
|
+
str += "Join " + EnumUtil::ToString(join_ref_type) + " " + EnumUtil::ToString(join_type);
|
56
56
|
if (condition) {
|
57
57
|
str += " " + condition->GetName();
|
58
58
|
}
|
@@ -218,23 +218,6 @@ void Pipeline::Ready() {
|
|
218
218
|
std::reverse(operators.begin(), operators.end());
|
219
219
|
}
|
220
220
|
|
221
|
-
void Pipeline::Finalize(Event &event) {
|
222
|
-
if (executor.HasError()) {
|
223
|
-
return;
|
224
|
-
}
|
225
|
-
D_ASSERT(ready);
|
226
|
-
try {
|
227
|
-
auto sink_state = sink->Finalize(*this, event, executor.context, *sink->sink_state);
|
228
|
-
sink->sink_state->state = sink_state;
|
229
|
-
} catch (Exception &ex) { // LCOV_EXCL_START
|
230
|
-
executor.PushError(PreservedError(ex));
|
231
|
-
} catch (std::exception &ex) {
|
232
|
-
executor.PushError(PreservedError(ex));
|
233
|
-
} catch (...) {
|
234
|
-
executor.PushError(PreservedError("Unknown exception in Finalize!"));
|
235
|
-
} // LCOV_EXCL_STOP
|
236
|
-
}
|
237
|
-
|
238
221
|
void Pipeline::AddDependency(shared_ptr<Pipeline> &pipeline) {
|
239
222
|
D_ASSERT(pipeline);
|
240
223
|
dependencies.push_back(weak_ptr<Pipeline>(pipeline));
|
@@ -171,9 +171,7 @@ PipelineExecuteResult PipelineExecutor::Execute(idx_t max_chunks) {
|
|
171
171
|
return PipelineExecuteResult::NOT_FINISHED;
|
172
172
|
}
|
173
173
|
|
174
|
-
PushFinalize();
|
175
|
-
|
176
|
-
return PipelineExecuteResult::FINISHED;
|
174
|
+
return PushFinalize();
|
177
175
|
}
|
178
176
|
|
179
177
|
PipelineExecuteResult PipelineExecutor::Execute() {
|
@@ -238,24 +236,45 @@ OperatorResultType PipelineExecutor::ExecutePushInternal(DataChunk &input, idx_t
|
|
238
236
|
}
|
239
237
|
}
|
240
238
|
|
241
|
-
|
239
|
+
PipelineExecuteResult PipelineExecutor::PushFinalize() {
|
242
240
|
if (finalized) {
|
243
241
|
throw InternalException("Calling PushFinalize on a pipeline that has been finalized already");
|
244
242
|
}
|
245
243
|
|
246
244
|
D_ASSERT(local_sink_state);
|
247
245
|
|
248
|
-
|
246
|
+
// Run the combine for the sink
|
247
|
+
OperatorSinkCombineInput combine_input {*pipeline.sink->sink_state, *local_sink_state, interrupt_state};
|
248
|
+
|
249
|
+
#ifdef DUCKDB_DEBUG_ASYNC_SINK_SOURCE
|
250
|
+
if (debug_blocked_combine_count < debug_blocked_target_count) {
|
251
|
+
debug_blocked_combine_count++;
|
252
|
+
|
253
|
+
auto &callback_state = combine_input.interrupt_state;
|
254
|
+
std::thread rewake_thread([callback_state] {
|
255
|
+
std::this_thread::sleep_for(std::chrono::milliseconds(1));
|
256
|
+
callback_state.Callback();
|
257
|
+
});
|
258
|
+
rewake_thread.detach();
|
249
259
|
|
250
|
-
|
251
|
-
|
260
|
+
return PipelineExecuteResult::INTERRUPTED;
|
261
|
+
}
|
262
|
+
#endif
|
263
|
+
auto result = pipeline.sink->Combine(context, combine_input);
|
252
264
|
|
265
|
+
if (result == SinkCombineResultType::BLOCKED) {
|
266
|
+
return PipelineExecuteResult::INTERRUPTED;
|
267
|
+
}
|
268
|
+
|
269
|
+
finalized = true;
|
253
270
|
// flush all query profiler info
|
254
271
|
for (idx_t i = 0; i < intermediate_states.size(); i++) {
|
255
272
|
intermediate_states[i]->Finalize(pipeline.operators[i].get(), context);
|
256
273
|
}
|
257
274
|
pipeline.executor.Flush(thread);
|
258
275
|
local_sink_state.reset();
|
276
|
+
|
277
|
+
return PipelineExecuteResult::FINISHED;
|
259
278
|
}
|
260
279
|
|
261
280
|
// TODO: Refactoring the StreamingQueryResult to use Push-based execution should eliminate the need for this code
|
@@ -1,16 +1,70 @@
|
|
1
1
|
#include "duckdb/parallel/pipeline_finish_event.hpp"
|
2
2
|
#include "duckdb/execution/executor.hpp"
|
3
|
+
#include "duckdb/parallel/interrupt.hpp"
|
3
4
|
|
4
5
|
namespace duckdb {
|
5
6
|
|
7
|
+
//! The PipelineFinishTask calls Finalize on the sink. Note that this is a single-threaded operation, but is executed
|
8
|
+
//! in a task to allow the Finalize call to block (e.g. for async I/O)
|
9
|
+
class PipelineFinishTask : public ExecutorTask {
|
10
|
+
public:
|
11
|
+
explicit PipelineFinishTask(Pipeline &pipeline_p, shared_ptr<Event> event_p)
|
12
|
+
: ExecutorTask(pipeline_p.executor), pipeline(pipeline_p), event(std::move(event_p)) {
|
13
|
+
}
|
14
|
+
|
15
|
+
Pipeline &pipeline;
|
16
|
+
shared_ptr<Event> event;
|
17
|
+
|
18
|
+
public:
|
19
|
+
TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override {
|
20
|
+
auto sink = pipeline.GetSink();
|
21
|
+
InterruptState interrupt_state(shared_from_this());
|
22
|
+
OperatorSinkFinalizeInput finalize_input {*sink->sink_state, interrupt_state};
|
23
|
+
|
24
|
+
#ifdef DUCKDB_DEBUG_ASYNC_SINK_SOURCE
|
25
|
+
if (debug_blocked_count < debug_blocked_target_count) {
|
26
|
+
debug_blocked_count++;
|
27
|
+
|
28
|
+
auto &callback_state = interrupt_state;
|
29
|
+
std::thread rewake_thread([callback_state] {
|
30
|
+
std::this_thread::sleep_for(std::chrono::milliseconds(1));
|
31
|
+
callback_state.Callback();
|
32
|
+
});
|
33
|
+
rewake_thread.detach();
|
34
|
+
|
35
|
+
return TaskExecutionResult::TASK_BLOCKED;
|
36
|
+
}
|
37
|
+
#endif
|
38
|
+
auto sink_state = sink->Finalize(pipeline, *event, executor.context, finalize_input);
|
39
|
+
|
40
|
+
if (sink_state == SinkFinalizeType::BLOCKED) {
|
41
|
+
return TaskExecutionResult::TASK_BLOCKED;
|
42
|
+
}
|
43
|
+
|
44
|
+
sink->sink_state->state = sink_state;
|
45
|
+
event->FinishTask();
|
46
|
+
return TaskExecutionResult::TASK_FINISHED;
|
47
|
+
}
|
48
|
+
|
49
|
+
private:
|
50
|
+
#ifdef DUCKDB_DEBUG_ASYNC_SINK_SOURCE
|
51
|
+
//! Debugging state: number of times blocked
|
52
|
+
int debug_blocked_count = 0;
|
53
|
+
//! Number of times the Finalize will block before actually returning data
|
54
|
+
int debug_blocked_target_count = 1;
|
55
|
+
#endif
|
56
|
+
};
|
57
|
+
|
6
58
|
PipelineFinishEvent::PipelineFinishEvent(shared_ptr<Pipeline> pipeline_p) : BasePipelineEvent(std::move(pipeline_p)) {
|
7
59
|
}
|
8
60
|
|
9
61
|
void PipelineFinishEvent::Schedule() {
|
62
|
+
vector<shared_ptr<Task>> tasks;
|
63
|
+
tasks.push_back(make_uniq<PipelineFinishTask>(*pipeline, shared_from_this()));
|
64
|
+
SetTasks(std::move(tasks));
|
10
65
|
}
|
11
66
|
|
12
67
|
void PipelineFinishEvent::FinishEvent() {
|
13
|
-
pipeline->Finalize(*this);
|
14
68
|
}
|
15
69
|
|
16
70
|
} // namespace duckdb
|
@@ -21,15 +21,25 @@ unique_ptr<BoundTableRef> Binder::Bind(BaseTableRef &ref) {
|
|
21
21
|
QueryErrorContext error_context(root_statement, ref.query_location);
|
22
22
|
// CTEs and views are also referred to using BaseTableRefs, hence need to distinguish here
|
23
23
|
// check if the table name refers to a CTE
|
24
|
-
|
24
|
+
|
25
|
+
// CTE name should never be qualified (i.e. schema_name should be empty)
|
26
|
+
optional_ptr<CommonTableExpressionInfo> found_cte = nullptr;
|
27
|
+
if (ref.schema_name.empty()) {
|
28
|
+
found_cte = FindCTE(ref.table_name, ref.table_name == alias);
|
29
|
+
}
|
30
|
+
|
25
31
|
if (found_cte) {
|
26
32
|
// Check if there is a CTE binding in the BindContext
|
27
33
|
auto &cte = *found_cte;
|
28
34
|
auto ctebinding = bind_context.GetCTEBinding(ref.table_name);
|
29
35
|
if (!ctebinding) {
|
30
36
|
if (CTEIsAlreadyBound(cte)) {
|
31
|
-
throw BinderException(
|
32
|
-
|
37
|
+
throw BinderException(
|
38
|
+
"Circular reference to CTE \"%s\", There are two possible solutions. \n1. use WITH RECURSIVE to "
|
39
|
+
"use recursive CTEs. \n2. If "
|
40
|
+
"you want to use the TABLE name \"%s\" the same as the CTE name, please explicitly add "
|
41
|
+
"\"SCHEMA\" before table name. You can try \"main.%s\" (main is the duckdb default schema)",
|
42
|
+
ref.table_name, ref.table_name, ref.table_name);
|
33
43
|
}
|
34
44
|
// Move CTE to subquery and bind recursively
|
35
45
|
SubqueryRef subquery(unique_ptr_cast<SQLStatement, SelectStatement>(cte.query->Copy()));
|