duckdb 0.8.2-dev1859.0 → 0.8.2-dev1968.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +9 -9
- package/package.json +1 -1
- package/src/duckdb/src/common/enum_util.cpp +28 -0
- package/src/duckdb/src/common/types/hugeint.cpp +40 -0
- package/src/duckdb/src/core_functions/function_list.cpp +1 -0
- package/src/duckdb/src/core_functions/scalar/string/to_base.cpp +66 -0
- package/src/duckdb/src/execution/operator/aggregate/physical_hash_aggregate.cpp +14 -11
- package/src/duckdb/src/execution/operator/aggregate/physical_perfecthash_aggregate.cpp +6 -4
- package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +14 -12
- package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +6 -4
- package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +8 -6
- package/src/duckdb/src/execution/operator/helper/physical_explain_analyze.cpp +2 -2
- package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -3
- package/src/duckdb/src/execution/operator/helper/physical_materialized_collector.cpp +7 -5
- package/src/duckdb/src/execution/operator/helper/physical_vacuum.cpp +7 -5
- package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +5 -4
- package/src/duckdb/src/execution/operator/join/physical_blockwise_nl_join.cpp +2 -2
- package/src/duckdb/src/execution/operator/join/physical_delim_join.cpp +13 -6
- package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +7 -5
- package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +7 -5
- package/src/duckdb/src/execution/operator/join/physical_nested_loop_join.cpp +7 -4
- package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +8 -6
- package/src/duckdb/src/execution/operator/order/physical_order.cpp +7 -5
- package/src/duckdb/src/execution/operator/order/physical_top_n.cpp +7 -5
- package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +8 -6
- package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp +8 -7
- package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp +8 -6
- package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +11 -9
- package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +10 -10
- package/src/duckdb/src/execution/operator/persistent/physical_update.cpp +4 -2
- package/src/duckdb/src/execution/operator/schema/physical_create_index.cpp +7 -6
- package/src/duckdb/src/execution/physical_operator.cpp +3 -2
- package/src/duckdb/src/function/table/read_csv.cpp +4 -3
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/common/bitpacking.hpp +70 -55
- package/src/duckdb/src/include/duckdb/common/enum_util.hpp +8 -0
- package/src/duckdb/src/include/duckdb/common/enums/operator_result_type.hpp +5 -1
- package/src/duckdb/src/include/duckdb/common/hugeint.hpp +15 -0
- package/src/duckdb/src/include/duckdb/common/limits.hpp +52 -149
- package/src/duckdb/src/include/duckdb/common/numeric_utils.hpp +48 -0
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +1 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +1 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +9 -0
- package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_hash_aggregate.hpp +3 -3
- package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_perfecthash_aggregate.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_ungrouped_aggregate.hpp +3 -3
- package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_window.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_batch_collector.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_explain_analyze.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_limit.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_materialized_collector.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_vacuum.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_blockwise_nl_join.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_delim_join.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_hash_join.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_nested_loop_join.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/order/physical_order.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/order/physical_top_n.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_copy_to_file.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_insert.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_copy_to_file.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_fixed_batch_copy.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_insert.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_update.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_index.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/physical_operator.hpp +3 -3
- package/src/duckdb/src/include/duckdb/execution/physical_operator_states.hpp +11 -0
- package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +0 -3
- package/src/duckdb/src/include/duckdb/parallel/pipeline_executor.hpp +3 -2
- package/src/duckdb/src/main/relation/join_relation.cpp +1 -1
- package/src/duckdb/src/parallel/pipeline.cpp +0 -17
- package/src/duckdb/src/parallel/pipeline_executor.cpp +26 -7
- package/src/duckdb/src/parallel/pipeline_finish_event.cpp +55 -1
- package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +13 -3
- package/src/duckdb/src/storage/compression/bitpacking.cpp +87 -63
- package/src/duckdb/src/storage/compression/bitpacking_hugeint.cpp +295 -0
- package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
- package/src/duckdb/ub_src_core_functions_scalar_string.cpp +2 -0
- package/src/duckdb/ub_src_storage_compression.cpp +2 -0
@@ -191,17 +191,20 @@ SinkResultType PhysicalNestedLoopJoin::Sink(ExecutionContext &context, DataChunk
|
|
191
191
|
return SinkResultType::NEED_MORE_INPUT;
|
192
192
|
}
|
193
193
|
|
194
|
-
|
195
|
-
|
194
|
+
SinkCombineResultType PhysicalNestedLoopJoin::Combine(ExecutionContext &context,
|
195
|
+
OperatorSinkCombineInput &input) const {
|
196
|
+
auto &state = input.local_state.Cast<NestedLoopJoinLocalState>();
|
196
197
|
auto &client_profiler = QueryProfiler::Get(context.client);
|
197
198
|
|
198
199
|
context.thread.profiler.Flush(*this, state.rhs_executor, "rhs_executor", 1);
|
199
200
|
client_profiler.Flush(context.thread.profiler);
|
201
|
+
|
202
|
+
return SinkCombineResultType::FINISHED;
|
200
203
|
}
|
201
204
|
|
202
205
|
SinkFinalizeType PhysicalNestedLoopJoin::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
203
|
-
|
204
|
-
auto &gstate =
|
206
|
+
OperatorSinkFinalizeInput &input) const {
|
207
|
+
auto &gstate = input.global_state.Cast<NestedLoopJoinGlobalState>();
|
205
208
|
gstate.right_outer.Initialize(gstate.right_payload_data.Count());
|
206
209
|
if (gstate.right_payload_data.Count() == 0 && EmptyResultIfRHSIsEmpty()) {
|
207
210
|
return SinkFinalizeType::NO_OUTPUT_POSSIBLE;
|
@@ -118,23 +118,25 @@ SinkResultType PhysicalPiecewiseMergeJoin::Sink(ExecutionContext &context, DataC
|
|
118
118
|
return SinkResultType::NEED_MORE_INPUT;
|
119
119
|
}
|
120
120
|
|
121
|
-
|
122
|
-
|
123
|
-
auto &gstate =
|
124
|
-
auto &lstate =
|
121
|
+
SinkCombineResultType PhysicalPiecewiseMergeJoin::Combine(ExecutionContext &context,
|
122
|
+
OperatorSinkCombineInput &input) const {
|
123
|
+
auto &gstate = input.global_state.Cast<MergeJoinGlobalState>();
|
124
|
+
auto &lstate = input.local_state.Cast<MergeJoinLocalState>();
|
125
125
|
gstate.table->Combine(lstate.table);
|
126
126
|
auto &client_profiler = QueryProfiler::Get(context.client);
|
127
127
|
|
128
128
|
context.thread.profiler.Flush(*this, lstate.table.executor, "rhs_executor", 1);
|
129
129
|
client_profiler.Flush(context.thread.profiler);
|
130
|
+
|
131
|
+
return SinkCombineResultType::FINISHED;
|
130
132
|
}
|
131
133
|
|
132
134
|
//===--------------------------------------------------------------------===//
|
133
135
|
// Finalize
|
134
136
|
//===--------------------------------------------------------------------===//
|
135
137
|
SinkFinalizeType PhysicalPiecewiseMergeJoin::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
136
|
-
|
137
|
-
auto &gstate =
|
138
|
+
OperatorSinkFinalizeInput &input) const {
|
139
|
+
auto &gstate = input.global_state.Cast<MergeJoinGlobalState>();
|
138
140
|
auto &global_sort_state = gstate.table->global_sort_state;
|
139
141
|
|
140
142
|
if (IsRightOuterJoin(join_type)) {
|
@@ -101,10 +101,12 @@ SinkResultType PhysicalOrder::Sink(ExecutionContext &context, DataChunk &chunk,
|
|
101
101
|
return SinkResultType::NEED_MORE_INPUT;
|
102
102
|
}
|
103
103
|
|
104
|
-
|
105
|
-
auto &gstate =
|
106
|
-
auto &lstate =
|
104
|
+
SinkCombineResultType PhysicalOrder::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
|
105
|
+
auto &gstate = input.global_state.Cast<OrderGlobalSinkState>();
|
106
|
+
auto &lstate = input.local_state.Cast<OrderLocalSinkState>();
|
107
107
|
gstate.global_sort_state.AddLocalState(lstate.local_sort_state);
|
108
|
+
|
109
|
+
return SinkCombineResultType::FINISHED;
|
108
110
|
}
|
109
111
|
|
110
112
|
class PhysicalOrderMergeTask : public ExecutorTask {
|
@@ -163,8 +165,8 @@ public:
|
|
163
165
|
};
|
164
166
|
|
165
167
|
SinkFinalizeType PhysicalOrder::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
166
|
-
|
167
|
-
auto &state =
|
168
|
+
OperatorSinkFinalizeInput &input) const {
|
169
|
+
auto &state = input.global_state.Cast<OrderGlobalSinkState>();
|
168
170
|
auto &global_sort_state = state.global_sort_state;
|
169
171
|
|
170
172
|
if (global_sort_state.sorted_blocks.empty()) {
|
@@ -446,21 +446,23 @@ SinkResultType PhysicalTopN::Sink(ExecutionContext &context, DataChunk &chunk, O
|
|
446
446
|
//===--------------------------------------------------------------------===//
|
447
447
|
// Combine
|
448
448
|
//===--------------------------------------------------------------------===//
|
449
|
-
|
450
|
-
auto &gstate =
|
451
|
-
auto &lstate =
|
449
|
+
SinkCombineResultType PhysicalTopN::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
|
450
|
+
auto &gstate = input.global_state.Cast<TopNGlobalState>();
|
451
|
+
auto &lstate = input.local_state.Cast<TopNLocalState>();
|
452
452
|
|
453
453
|
// scan the local top N and append it to the global heap
|
454
454
|
lock_guard<mutex> glock(gstate.lock);
|
455
455
|
gstate.heap.Combine(lstate.heap);
|
456
|
+
|
457
|
+
return SinkCombineResultType::FINISHED;
|
456
458
|
}
|
457
459
|
|
458
460
|
//===--------------------------------------------------------------------===//
|
459
461
|
// Finalize
|
460
462
|
//===--------------------------------------------------------------------===//
|
461
463
|
SinkFinalizeType PhysicalTopN::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
462
|
-
|
463
|
-
auto &gstate =
|
464
|
+
OperatorSinkFinalizeInput &input) const {
|
465
|
+
auto &gstate = input.global_state.Cast<TopNGlobalState>();
|
464
466
|
// global finalize: compute the final top N
|
465
467
|
gstate.heap.Finalize();
|
466
468
|
return SinkFinalizeType::READY;
|
@@ -89,11 +89,13 @@ SinkResultType PhysicalBatchCopyToFile::Sink(ExecutionContext &context, DataChun
|
|
89
89
|
return SinkResultType::NEED_MORE_INPUT;
|
90
90
|
}
|
91
91
|
|
92
|
-
|
93
|
-
|
94
|
-
auto &state =
|
95
|
-
auto &gstate =
|
92
|
+
SinkCombineResultType PhysicalBatchCopyToFile::Combine(ExecutionContext &context,
|
93
|
+
OperatorSinkCombineInput &input) const {
|
94
|
+
auto &state = input.local_state.Cast<BatchCopyToLocalState>();
|
95
|
+
auto &gstate = input.global_state.Cast<BatchCopyToGlobalState>();
|
96
96
|
gstate.rows_copied += state.rows_copied;
|
97
|
+
|
98
|
+
return SinkCombineResultType::FINISHED;
|
97
99
|
}
|
98
100
|
|
99
101
|
//===--------------------------------------------------------------------===//
|
@@ -114,8 +116,8 @@ SinkFinalizeType PhysicalBatchCopyToFile::FinalFlush(ClientContext &context, Glo
|
|
114
116
|
}
|
115
117
|
|
116
118
|
SinkFinalizeType PhysicalBatchCopyToFile::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
117
|
-
|
118
|
-
FinalFlush(context,
|
119
|
+
OperatorSinkFinalizeInput &input) const {
|
120
|
+
FinalFlush(context, input.global_state);
|
119
121
|
return SinkFinalizeType::READY;
|
120
122
|
}
|
121
123
|
|
@@ -344,16 +344,15 @@ SinkResultType PhysicalBatchInsert::Sink(ExecutionContext &context, DataChunk &c
|
|
344
344
|
return SinkResultType::NEED_MORE_INPUT;
|
345
345
|
}
|
346
346
|
|
347
|
-
|
348
|
-
|
349
|
-
auto &
|
350
|
-
auto &lstate = lstate_p.Cast<BatchInsertLocalState>();
|
347
|
+
SinkCombineResultType PhysicalBatchInsert::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
|
348
|
+
auto &gstate = input.global_state.Cast<BatchInsertGlobalState>();
|
349
|
+
auto &lstate = input.local_state.Cast<BatchInsertLocalState>();
|
351
350
|
auto &client_profiler = QueryProfiler::Get(context.client);
|
352
351
|
context.thread.profiler.Flush(*this, lstate.default_executor, "default_executor", 1);
|
353
352
|
client_profiler.Flush(context.thread.profiler);
|
354
353
|
|
355
354
|
if (!lstate.current_collection) {
|
356
|
-
return;
|
355
|
+
return SinkCombineResultType::FINISHED;
|
357
356
|
}
|
358
357
|
|
359
358
|
if (lstate.current_collection->GetTotalRows() > 0) {
|
@@ -366,11 +365,13 @@ void PhysicalBatchInsert::Combine(ExecutionContext &context, GlobalSinkState &gs
|
|
366
365
|
lock_guard<mutex> l(gstate.lock);
|
367
366
|
gstate.table.GetStorage().FinalizeOptimisticWriter(context.client, *lstate.writer);
|
368
367
|
}
|
368
|
+
|
369
|
+
return SinkCombineResultType::FINISHED;
|
369
370
|
}
|
370
371
|
|
371
372
|
SinkFinalizeType PhysicalBatchInsert::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
372
|
-
|
373
|
-
auto &gstate =
|
373
|
+
OperatorSinkFinalizeInput &input) const {
|
374
|
+
auto &gstate = input.global_state.Cast<BatchInsertGlobalState>();
|
374
375
|
|
375
376
|
// in the finalize, do a final pass over all of the collections we created and try to merge smaller collections
|
376
377
|
// together
|
@@ -97,9 +97,9 @@ static string CreateDirRecursive(const vector<idx_t> &cols, const vector<string>
|
|
97
97
|
return path;
|
98
98
|
}
|
99
99
|
|
100
|
-
|
101
|
-
auto &g =
|
102
|
-
auto &l =
|
100
|
+
SinkCombineResultType PhysicalCopyToFile::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
|
101
|
+
auto &g = input.global_state.Cast<CopyToFunctionGlobalState>();
|
102
|
+
auto &l = input.local_state.Cast<CopyToFunctionLocalState>();
|
103
103
|
|
104
104
|
if (partition_output) {
|
105
105
|
auto &fs = FileSystem::GetFileSystem(context.client);
|
@@ -130,7 +130,7 @@ void PhysicalCopyToFile::Combine(ExecutionContext &context, GlobalSinkState &gst
|
|
130
130
|
function.copy_to_finalize(context.client, *bind_data, *fun_data_global);
|
131
131
|
}
|
132
132
|
|
133
|
-
return;
|
133
|
+
return SinkCombineResultType::FINISHED;
|
134
134
|
}
|
135
135
|
|
136
136
|
if (function.copy_to_combine) {
|
@@ -141,11 +141,13 @@ void PhysicalCopyToFile::Combine(ExecutionContext &context, GlobalSinkState &gst
|
|
141
141
|
function.copy_to_finalize(context.client, *bind_data, *l.global_state);
|
142
142
|
}
|
143
143
|
}
|
144
|
+
|
145
|
+
return SinkCombineResultType::FINISHED;
|
144
146
|
}
|
145
147
|
|
146
148
|
SinkFinalizeType PhysicalCopyToFile::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
147
|
-
|
148
|
-
auto &gstate =
|
149
|
+
OperatorSinkFinalizeInput &input) const {
|
150
|
+
auto &gstate = input.global_state.Cast<CopyToFunctionGlobalState>();
|
149
151
|
if (per_thread_output || partition_output) {
|
150
152
|
// already happened in combine
|
151
153
|
return SinkFinalizeType::READY;
|
@@ -136,10 +136,10 @@ SinkResultType PhysicalFixedBatchCopy::Sink(ExecutionContext &context, DataChunk
|
|
136
136
|
return SinkResultType::NEED_MORE_INPUT;
|
137
137
|
}
|
138
138
|
|
139
|
-
|
140
|
-
|
141
|
-
auto &state =
|
142
|
-
auto &gstate =
|
139
|
+
SinkCombineResultType PhysicalFixedBatchCopy::Combine(ExecutionContext &context,
|
140
|
+
OperatorSinkCombineInput &input) const {
|
141
|
+
auto &state = input.local_state.Cast<FixedBatchCopyLocalState>();
|
142
|
+
auto &gstate = input.global_state.Cast<FixedBatchCopyGlobalState>();
|
143
143
|
gstate.rows_copied += state.rows_copied;
|
144
144
|
if (!gstate.any_finished) {
|
145
145
|
// signal that this thread is finished processing batches and that we should move on to Finalize
|
@@ -147,6 +147,8 @@ void PhysicalFixedBatchCopy::Combine(ExecutionContext &context, GlobalSinkState
|
|
147
147
|
gstate.any_finished = true;
|
148
148
|
}
|
149
149
|
ExecuteTasks(context.client, gstate);
|
150
|
+
|
151
|
+
return SinkCombineResultType::FINISHED;
|
150
152
|
}
|
151
153
|
|
152
154
|
//===--------------------------------------------------------------------===//
|
@@ -225,16 +227,16 @@ SinkFinalizeType PhysicalFixedBatchCopy::FinalFlush(ClientContext &context, Glob
|
|
225
227
|
}
|
226
228
|
|
227
229
|
SinkFinalizeType PhysicalFixedBatchCopy::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
228
|
-
|
229
|
-
auto &gstate =
|
230
|
+
OperatorSinkFinalizeInput &input) const {
|
231
|
+
auto &gstate = input.global_state.Cast<FixedBatchCopyGlobalState>();
|
230
232
|
idx_t min_batch_index = idx_t(NumericLimits<int64_t>::Maximum());
|
231
233
|
// repartition any remaining batches
|
232
|
-
RepartitionBatches(context,
|
234
|
+
RepartitionBatches(context, input.global_state, min_batch_index, true);
|
233
235
|
// check if we have multiple tasks to execute
|
234
236
|
if (gstate.TaskCount() <= 1) {
|
235
237
|
// we don't - just execute the remaining task and finish flushing to disk
|
236
|
-
ExecuteTasks(context,
|
237
|
-
FinalFlush(context,
|
238
|
+
ExecuteTasks(context, input.global_state);
|
239
|
+
FinalFlush(context, input.global_state);
|
238
240
|
return SinkFinalizeType::READY;
|
239
241
|
}
|
240
242
|
// we have multiple tasks remaining - launch an event to execute the tasks in parallel
|
@@ -463,19 +463,17 @@ SinkResultType PhysicalInsert::Sink(ExecutionContext &context, DataChunk &chunk,
|
|
463
463
|
return SinkResultType::NEED_MORE_INPUT;
|
464
464
|
}
|
465
465
|
|
466
|
-
|
467
|
-
auto &gstate =
|
468
|
-
auto &lstate =
|
466
|
+
SinkCombineResultType PhysicalInsert::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
|
467
|
+
auto &gstate = input.global_state.Cast<InsertGlobalState>();
|
468
|
+
auto &lstate = input.local_state.Cast<InsertLocalState>();
|
469
469
|
auto &client_profiler = QueryProfiler::Get(context.client);
|
470
470
|
context.thread.profiler.Flush(*this, lstate.default_executor, "default_executor", 1);
|
471
471
|
client_profiler.Flush(context.thread.profiler);
|
472
472
|
|
473
|
-
if (!parallel) {
|
474
|
-
return;
|
475
|
-
}
|
476
|
-
if (!lstate.local_collection) {
|
477
|
-
return;
|
473
|
+
if (!parallel || !lstate.local_collection) {
|
474
|
+
return SinkCombineResultType::FINISHED;
|
478
475
|
}
|
476
|
+
|
479
477
|
// parallel append: finalize the append
|
480
478
|
TransactionData tdata(0, 0);
|
481
479
|
lstate.local_collection->FinalizeAppend(tdata, lstate.local_append_state);
|
@@ -500,11 +498,13 @@ void PhysicalInsert::Combine(ExecutionContext &context, GlobalSinkState &gstate_
|
|
500
498
|
gstate.table.GetStorage().FinalizeOptimisticWriter(context.client, *lstate.writer);
|
501
499
|
gstate.table.GetStorage().LocalMerge(context.client, *lstate.local_collection);
|
502
500
|
}
|
501
|
+
|
502
|
+
return SinkCombineResultType::FINISHED;
|
503
503
|
}
|
504
504
|
|
505
505
|
SinkFinalizeType PhysicalInsert::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
506
|
-
|
507
|
-
auto &gstate =
|
506
|
+
OperatorSinkFinalizeInput &input) const {
|
507
|
+
auto &gstate = input.global_state.Cast<InsertGlobalState>();
|
508
508
|
if (!parallel && gstate.initialized) {
|
509
509
|
auto &table = gstate.table;
|
510
510
|
auto &storage = table.GetStorage();
|
@@ -140,11 +140,13 @@ unique_ptr<LocalSinkState> PhysicalUpdate::GetLocalSinkState(ExecutionContext &c
|
|
140
140
|
return make_uniq<UpdateLocalState>(context.client, expressions, table.GetTypes(), bound_defaults);
|
141
141
|
}
|
142
142
|
|
143
|
-
|
144
|
-
auto &state =
|
143
|
+
SinkCombineResultType PhysicalUpdate::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
|
144
|
+
auto &state = input.local_state.Cast<UpdateLocalState>();
|
145
145
|
auto &client_profiler = QueryProfiler::Get(context.client);
|
146
146
|
context.thread.profiler.Flush(*this, state.default_executor, "default_executor", 1);
|
147
147
|
client_profiler.Flush(context.thread.profiler);
|
148
|
+
|
149
|
+
return SinkCombineResultType::FINISHED;
|
148
150
|
}
|
149
151
|
|
150
152
|
//===--------------------------------------------------------------------===//
|
@@ -123,11 +123,10 @@ SinkResultType PhysicalCreateIndex::Sink(ExecutionContext &context, DataChunk &c
|
|
123
123
|
return SinkResultType::NEED_MORE_INPUT;
|
124
124
|
}
|
125
125
|
|
126
|
-
|
127
|
-
LocalSinkState &lstate_p) const {
|
126
|
+
SinkCombineResultType PhysicalCreateIndex::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
|
128
127
|
|
129
|
-
auto &gstate =
|
130
|
-
auto &lstate =
|
128
|
+
auto &gstate = input.global_state.Cast<CreateIndexGlobalSinkState>();
|
129
|
+
auto &lstate = input.local_state.Cast<CreateIndexLocalSinkState>();
|
131
130
|
|
132
131
|
// merge the local index into the global index
|
133
132
|
if (!gstate.global_index->MergeIndexes(*lstate.local_index)) {
|
@@ -136,14 +135,16 @@ void PhysicalCreateIndex::Combine(ExecutionContext &context, GlobalSinkState &gs
|
|
136
135
|
|
137
136
|
// vacuum excess memory
|
138
137
|
gstate.global_index->Vacuum();
|
138
|
+
|
139
|
+
return SinkCombineResultType::FINISHED;
|
139
140
|
}
|
140
141
|
|
141
142
|
SinkFinalizeType PhysicalCreateIndex::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
142
|
-
|
143
|
+
OperatorSinkFinalizeInput &input) const {
|
143
144
|
|
144
145
|
// here, we just set the resulting global index as the newly created index of the table
|
145
146
|
|
146
|
-
auto &state =
|
147
|
+
auto &state = input.global_state.Cast<CreateIndexGlobalSinkState>();
|
147
148
|
D_ASSERT(!state.global_index->VerifyAndToString(true).empty());
|
148
149
|
|
149
150
|
auto &storage = table.GetStorage();
|
@@ -97,11 +97,12 @@ SinkResultType PhysicalOperator::Sink(ExecutionContext &context, DataChunk &chun
|
|
97
97
|
|
98
98
|
// LCOV_EXCL_STOP
|
99
99
|
|
100
|
-
|
100
|
+
SinkCombineResultType PhysicalOperator::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
|
101
|
+
return SinkCombineResultType::FINISHED;
|
101
102
|
}
|
102
103
|
|
103
104
|
SinkFinalizeType PhysicalOperator::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
|
104
|
-
|
105
|
+
OperatorSinkFinalizeInput &input) const {
|
105
106
|
return SinkFinalizeType::READY;
|
106
107
|
}
|
107
108
|
|
@@ -343,7 +343,8 @@ public:
|
|
343
343
|
line_info.lines_read[0][0]++;
|
344
344
|
}
|
345
345
|
}
|
346
|
-
ParallelCSVGlobalState(
|
346
|
+
explicit ParallelCSVGlobalState(idx_t system_threads_p)
|
347
|
+
: system_threads(system_threads_p), line_info(main_mutex, batch_to_tuple_end, tuple_start, tuple_end) {
|
347
348
|
running_threads = MaxThreads();
|
348
349
|
}
|
349
350
|
|
@@ -405,7 +406,7 @@ private:
|
|
405
406
|
//! How many bytes we should execute per local state
|
406
407
|
idx_t bytes_per_local_state;
|
407
408
|
//! Size of first file
|
408
|
-
idx_t first_file_size;
|
409
|
+
idx_t first_file_size = 0;
|
409
410
|
//! Whether or not this is an on-disk file
|
410
411
|
bool on_disk_file = true;
|
411
412
|
//! Basically max number of threads in DuckDB
|
@@ -691,7 +692,7 @@ static unique_ptr<GlobalTableFunctionState> ParallelCSVInitGlobal(ClientContext
|
|
691
692
|
auto &bind_data = input.bind_data->CastNoConst<ReadCSVData>();
|
692
693
|
if (bind_data.files.empty()) {
|
693
694
|
// This can happen when a filename based filter pushdown has eliminated all possible files for this scan.
|
694
|
-
return make_uniq<ParallelCSVGlobalState>();
|
695
|
+
return make_uniq<ParallelCSVGlobalState>(context.db->NumberOfThreads());
|
695
696
|
}
|
696
697
|
unique_ptr<CSVFileHandle> file_handle;
|
697
698
|
|
@@ -1,8 +1,8 @@
|
|
1
1
|
#ifndef DUCKDB_VERSION
|
2
|
-
#define DUCKDB_VERSION "0.8.2-
|
2
|
+
#define DUCKDB_VERSION "0.8.2-dev1968"
|
3
3
|
#endif
|
4
4
|
#ifndef DUCKDB_SOURCE_ID
|
5
|
-
#define DUCKDB_SOURCE_ID "
|
5
|
+
#define DUCKDB_SOURCE_ID "b1d5e20f31"
|
6
6
|
#endif
|
7
7
|
#include "duckdb/function/table/system_functions.hpp"
|
8
8
|
#include "duckdb/main/database.hpp"
|