duckdb 0.8.2-dev77.0 → 0.9.0

package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp

@@ -0,0 +1,48 @@
+#include "duckdb/main/appender.hpp"
+#include "duckdb/parser/parsed_data/create_table_info.hpp"
+#include "duckdb/function/table/read_csv.hpp"
+#include "duckdb/execution/operator/persistent/csv_rejects_table.hpp"
+#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
+
+namespace duckdb {
+
+TableCatalogEntry &CSVRejectsTable::GetTable(ClientContext &context) {
+	auto &temp_catalog = Catalog::GetCatalog(context, TEMP_CATALOG);
+	auto &table_entry = temp_catalog.GetEntry<TableCatalogEntry>(context, TEMP_CATALOG, DEFAULT_SCHEMA, name);
+	return table_entry;
+}
+
+shared_ptr<CSVRejectsTable> CSVRejectsTable::GetOrCreate(ClientContext &context, const string &name) {
+	auto key = "CSV_REJECTS_TABLE_CACHE_ENTRY_" + StringUtil::Upper(name);
+	auto &cache = ObjectCache::GetObjectCache(context);
+	return cache.GetOrCreate<CSVRejectsTable>(key, name);
+}
+
+void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData &data) {
+	// (Re)Create the temporary rejects table
+	auto &catalog = Catalog::GetCatalog(context, TEMP_CATALOG);
+	auto info = make_uniq<CreateTableInfo>(TEMP_CATALOG, DEFAULT_SCHEMA, name);
+	info->temporary = true;
+	info->on_conflict = OnCreateConflict::ERROR_ON_CONFLICT;
+	info->columns.AddColumn(ColumnDefinition("file", LogicalType::VARCHAR));
+	info->columns.AddColumn(ColumnDefinition("line", LogicalType::BIGINT));
+	info->columns.AddColumn(ColumnDefinition("column", LogicalType::BIGINT));
+	info->columns.AddColumn(ColumnDefinition("column_name", LogicalType::VARCHAR));
+	info->columns.AddColumn(ColumnDefinition("parsed_value", LogicalType::VARCHAR));
+
+	if (!data.options.rejects_recovery_columns.empty()) {
+		child_list_t<LogicalType> recovery_key_components;
+		for (auto &col_name : data.options.rejects_recovery_columns) {
+			recovery_key_components.emplace_back(col_name, LogicalType::VARCHAR);
+		}
+		info->columns.AddColumn(ColumnDefinition("recovery_columns", LogicalType::STRUCT(recovery_key_components)));
+	}
+
+	info->columns.AddColumn(ColumnDefinition("error", LogicalType::VARCHAR));
+
+	catalog.CreateTable(context, std::move(info));
+
+	count = 0;
+}
+
+} // namespace duckdb

package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp

@@ -1,9 +1,11 @@
 #include "duckdb/execution/operator/persistent/physical_batch_copy_to_file.hpp"
+
+#include "duckdb/common/allocator.hpp"
+#include "duckdb/common/types/batched_data_collection.hpp"
+#include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/execution/operator/persistent/physical_copy_to_file.hpp"
 #include "duckdb/parallel/base_pipeline_event.hpp"
-#include "duckdb/common/vector_operations/vector_operations.hpp"
-#include "duckdb/common/types/batched_data_collection.hpp"
-#include "duckdb/common/allocator.hpp"
+
 #include <algorithm>
 
 namespace duckdb {
@@ -67,7 +69,7 @@ public:
 	optional_idx batch_index;
 
 	void InitializeCollection(ClientContext &context, const PhysicalOperator &op) {
-		collection = make_uniq<ColumnDataCollection>(Allocator::Get(context), op.children[0]->types);
+		collection = make_uniq<ColumnDataCollection>(BufferAllocator::Get(context), op.children[0]->types);
 		collection->InitializeAppend(append_state);
 	}
 };
@@ -87,11 +89,13 @@ SinkResultType PhysicalBatchCopyToFile::Sink(ExecutionContext &context, DataChun
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
-void PhysicalBatchCopyToFile::Combine(ExecutionContext &context, GlobalSinkState &gstate_p,
-                                      LocalSinkState &lstate) const {
-	auto &state = lstate.Cast<BatchCopyToLocalState>();
-	auto &gstate = gstate_p.Cast<BatchCopyToGlobalState>();
+SinkCombineResultType PhysicalBatchCopyToFile::Combine(ExecutionContext &context,
+                                                       OperatorSinkCombineInput &input) const {
+	auto &state = input.local_state.Cast<BatchCopyToLocalState>();
+	auto &gstate = input.global_state.Cast<BatchCopyToGlobalState>();
 	gstate.rows_copied += state.rows_copied;
+
+	return SinkCombineResultType::FINISHED;
 }
 
 //===--------------------------------------------------------------------===//
@@ -112,8 +116,8 @@ SinkFinalizeType PhysicalBatchCopyToFile::FinalFlush(ClientContext &context, Glo
 }
 
 SinkFinalizeType PhysicalBatchCopyToFile::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
-                                                   GlobalSinkState &gstate_p) const {
-	FinalFlush(context, gstate_p);
+                                                   OperatorSinkFinalizeInput &input) const {
+	FinalFlush(context, input.global_state);
 	return SinkFinalizeType::READY;
 }
 

package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp

@@ -6,6 +6,7 @@
 #include "duckdb/storage/table_io_manager.hpp"
 #include "duckdb/transaction/local_storage.hpp"
 #include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
+#include "duckdb/transaction/duck_transaction.hpp"
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
 
@@ -119,6 +120,7 @@ public:
 	idx_t insert_count;
 	vector<RowGroupBatchEntry> collections;
 	idx_t next_start = 0;
+	bool optimistically_written = false;
 
 	void FindMergeCollections(idx_t min_batch_index, optional_idx &merged_batch_index,
 	                          vector<unique_ptr<RowGroupCollection>> &result) {
@@ -176,10 +178,12 @@ public:
 	unique_ptr<RowGroupCollection> MergeCollections(ClientContext &context,
 	                                                vector<unique_ptr<RowGroupCollection>> merge_collections,
 	                                                OptimisticDataWriter &writer) {
+		D_ASSERT(!merge_collections.empty());
 		CollectionMerger merger(context);
 		for (auto &collection : merge_collections) {
 			merger.AddCollection(std::move(collection));
 		}
+		optimistically_written = true;
 		return merger.Flush(writer);
 	}
 
@@ -194,7 +198,7 @@ public:
 		}
 		auto new_count = current_collection->GetTotalRows();
 		auto batch_type =
-		    new_count < RowGroup::ROW_GROUP_SIZE ? RowGroupBatchType::NOT_FLUSHED : RowGroupBatchType::FLUSHED;
+		    new_count < Storage::ROW_GROUP_SIZE ? RowGroupBatchType::NOT_FLUSHED : RowGroupBatchType::FLUSHED;
 		if (batch_type == RowGroupBatchType::FLUSHED && writer) {
 			writer->WriteLastRowGroup(*current_collection);
 		}
@@ -344,16 +348,15 @@ SinkResultType PhysicalBatchInsert::Sink(ExecutionContext &context, DataChunk &c
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
-void PhysicalBatchInsert::Combine(ExecutionContext &context, GlobalSinkState &gstate_p,
-                                  LocalSinkState &lstate_p) const {
-	auto &gstate = gstate_p.Cast<BatchInsertGlobalState>();
-	auto &lstate = lstate_p.Cast<BatchInsertLocalState>();
+SinkCombineResultType PhysicalBatchInsert::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
+	auto &gstate = input.global_state.Cast<BatchInsertGlobalState>();
+	auto &lstate = input.local_state.Cast<BatchInsertLocalState>();
 	auto &client_profiler = QueryProfiler::Get(context.client);
 	context.thread.profiler.Flush(*this, lstate.default_executor, "default_executor", 1);
 	client_profiler.Flush(context.thread.profiler);
 
 	if (!lstate.current_collection) {
-		return;
+		return SinkCombineResultType::FINISHED;
 	}
 
 	if (lstate.current_collection->GetTotalRows() > 0) {
@@ -366,54 +369,73 @@ void PhysicalBatchInsert::Combine(ExecutionContext &context, GlobalSinkState &gs
 		lock_guard<mutex> l(gstate.lock);
 		gstate.table.GetStorage().FinalizeOptimisticWriter(context.client, *lstate.writer);
 	}
+
+	return SinkCombineResultType::FINISHED;
 }
 
 SinkFinalizeType PhysicalBatchInsert::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
-                                               GlobalSinkState &gstate_p) const {
-	auto &gstate = gstate_p.Cast<BatchInsertGlobalState>();
-
-	// in the finalize, do a final pass over all of the collections we created and try to merge smaller collections
-	// together
-	vector<unique_ptr<CollectionMerger>> mergers;
-	unique_ptr<CollectionMerger> current_merger;
-
-	auto &storage = gstate.table.GetStorage();
-	for (auto &entry : gstate.collections) {
-		if (entry.type == RowGroupBatchType::NOT_FLUSHED) {
-			// this collection has not been flushed: add it to the merge set
-			if (!current_merger) {
-				current_merger = make_uniq<CollectionMerger>(context);
-			}
-			current_merger->AddCollection(std::move(entry.collection));
-		} else {
-			// this collection has been flushed: it does not need to be merged
-			// create a separate collection merger only for this entry
-			if (current_merger) {
-				// we have small collections remaining: flush them
-				mergers.push_back(std::move(current_merger));
-				current_merger.reset();
+                                               OperatorSinkFinalizeInput &input) const {
+	auto &gstate = input.global_state.Cast<BatchInsertGlobalState>();
+
+	if (gstate.optimistically_written || gstate.insert_count >= LocalStorage::MERGE_THRESHOLD) {
+		// we have written data to disk optimistically or are inserting a large amount of data
+		// perform a final pass over all of the row groups and merge them together
+		vector<unique_ptr<CollectionMerger>> mergers;
+		unique_ptr<CollectionMerger> current_merger;
+
+		auto &storage = gstate.table.GetStorage();
+		for (auto &entry : gstate.collections) {
+			if (entry.type == RowGroupBatchType::NOT_FLUSHED) {
+				// this collection has not been flushed: add it to the merge set
+				if (!current_merger) {
+					current_merger = make_uniq<CollectionMerger>(context);
+				}
+				current_merger->AddCollection(std::move(entry.collection));
+			} else {
+				// this collection has been flushed: it does not need to be merged
+				// create a separate collection merger only for this entry
+				if (current_merger) {
+					// we have small collections remaining: flush them
+					mergers.push_back(std::move(current_merger));
+					current_merger.reset();
+				}
+				auto larger_merger = make_uniq<CollectionMerger>(context);
+				larger_merger->AddCollection(std::move(entry.collection));
+				mergers.push_back(std::move(larger_merger));
 			}
-			auto larger_merger = make_uniq<CollectionMerger>(context);
-			larger_merger->AddCollection(std::move(entry.collection));
-			mergers.push_back(std::move(larger_merger));
 		}
-	}
-	if (current_merger) {
-		mergers.push_back(std::move(current_merger));
-	}
+		if (current_merger) {
+			mergers.push_back(std::move(current_merger));
+		}
 
-	// now that we have created all of the mergers, perform the actual merging
-	vector<unique_ptr<RowGroupCollection>> final_collections;
-	final_collections.reserve(mergers.size());
-	auto &writer = storage.CreateOptimisticWriter(context);
-	for (auto &merger : mergers) {
-		final_collections.push_back(merger->Flush(writer));
-	}
-	storage.FinalizeOptimisticWriter(context, writer);
+		// now that we have created all of the mergers, perform the actual merging
+		vector<unique_ptr<RowGroupCollection>> final_collections;
+		final_collections.reserve(mergers.size());
+		auto &writer = storage.CreateOptimisticWriter(context);
+		for (auto &merger : mergers) {
+			final_collections.push_back(merger->Flush(writer));
+		}
+		storage.FinalizeOptimisticWriter(context, writer);
 
-	// finally, merge the row groups into the local storage
-	for (auto &collection : final_collections) {
-		storage.LocalMerge(context, *collection);
+		// finally, merge the row groups into the local storage
+		for (auto &collection : final_collections) {
+			storage.LocalMerge(context, *collection);
+		}
+	} else {
+		// we are writing a small amount of data to disk
+		// append directly to transaction local storage
+		auto &table = gstate.table;
+		auto &storage = table.GetStorage();
+		LocalAppendState append_state;
+		storage.InitializeLocalAppend(append_state, context);
+		auto &transaction = DuckTransaction::Get(context, table.catalog);
+		for (auto &entry : gstate.collections) {
+			entry.collection->Scan(transaction, [&](DataChunk &insert_chunk) {
+				storage.LocalAppend(append_state, table, context, insert_chunk);
+				return true;
+			});
+		}
+		storage.FinalizeLocalAppend(append_state);
 	}
 	return SinkFinalizeType::READY;
 }

package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp

@@ -97,9 +97,9 @@ static string CreateDirRecursive(const vector<idx_t> &cols, const vector<string>
 	return path;
 }
 
-void PhysicalCopyToFile::Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const {
-	auto &g = gstate.Cast<CopyToFunctionGlobalState>();
-	auto &l = lstate.Cast<CopyToFunctionLocalState>();
+SinkCombineResultType PhysicalCopyToFile::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
+	auto &g = input.global_state.Cast<CopyToFunctionGlobalState>();
+	auto &l = input.local_state.Cast<CopyToFunctionLocalState>();
 
 	if (partition_output) {
 		auto &fs = FileSystem::GetFileSystem(context.client);
@@ -108,7 +108,7 @@ void PhysicalCopyToFile::Combine(ExecutionContext &context, GlobalSinkState &gst
 		auto partition_key_map = l.part_buffer->GetReverseMap();
 
 		string trimmed_path = file_path;
-		StringUtil::RTrim(trimmed_path, fs.PathSeparator());
+		StringUtil::RTrim(trimmed_path, fs.PathSeparator(trimmed_path));
 
 		for (idx_t i = 0; i < partitions.size(); i++) {
 			string hive_path =
@@ -130,7 +130,7 @@ void PhysicalCopyToFile::Combine(ExecutionContext &context, GlobalSinkState &gst
 			function.copy_to_finalize(context.client, *bind_data, *fun_data_global);
 		}
 
-		return;
+		return SinkCombineResultType::FINISHED;
 	}
 
 	if (function.copy_to_combine) {
@@ -141,11 +141,13 @@ void PhysicalCopyToFile::Combine(ExecutionContext &context, GlobalSinkState &gst
 			function.copy_to_finalize(context.client, *bind_data, *l.global_state);
 		}
 	}
+
+	return SinkCombineResultType::FINISHED;
 }
 
 SinkFinalizeType PhysicalCopyToFile::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
-                                              GlobalSinkState &gstate_p) const {
-	auto &gstate = gstate_p.Cast<CopyToFunctionGlobalState>();
+                                              OperatorSinkFinalizeInput &input) const {
+	auto &gstate = input.global_state.Cast<CopyToFunctionGlobalState>();
 	if (per_thread_output || partition_output) {
 		// already happened in combine
 		return SinkFinalizeType::READY;

package/src/duckdb/src/execution/operator/persistent/physical_export.cpp

@@ -59,7 +59,7 @@ static void WriteCopyStatement(FileSystem &fs, stringstream &ss, CopyInfo &info,
 	if (info.format == "csv") {
 		// insert default csv options, if not specified
 		if (info.options.find("header") == info.options.end()) {
-			info.options["header"].push_back(Value::INTEGER(0));
+			info.options["header"].push_back(Value::INTEGER(1));
 		}
 		if (info.options.find("delimiter") == info.options.end() && info.options.find("sep") == info.options.end() &&
 		    info.options.find("delim") == info.options.end()) {

package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp

@@ -116,7 +116,7 @@ public:
 	optional_idx batch_index;
 
 	void InitializeCollection(ClientContext &context, const PhysicalOperator &op) {
-		collection = make_uniq<ColumnDataCollection>(Allocator::Get(context), op.children[0]->types);
+		collection = make_uniq<ColumnDataCollection>(BufferAllocator::Get(context), op.children[0]->types);
 		collection->InitializeAppend(append_state);
 	}
 };
@@ -136,10 +136,10 @@ SinkResultType PhysicalFixedBatchCopy::Sink(ExecutionContext &context, DataChunk
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
-void PhysicalFixedBatchCopy::Combine(ExecutionContext &context, GlobalSinkState &gstate_p,
-                                     LocalSinkState &lstate) const {
-	auto &state = lstate.Cast<FixedBatchCopyLocalState>();
-	auto &gstate = gstate_p.Cast<FixedBatchCopyGlobalState>();
+SinkCombineResultType PhysicalFixedBatchCopy::Combine(ExecutionContext &context,
+                                                      OperatorSinkCombineInput &input) const {
+	auto &state = input.local_state.Cast<FixedBatchCopyLocalState>();
+	auto &gstate = input.global_state.Cast<FixedBatchCopyGlobalState>();
 	gstate.rows_copied += state.rows_copied;
 	if (!gstate.any_finished) {
 		// signal that this thread is finished processing batches and that we should move on to Finalize
@@ -147,6 +147,8 @@ void PhysicalFixedBatchCopy::Combine(ExecutionContext &context, GlobalSinkState
 		gstate.any_finished = true;
 	}
 	ExecuteTasks(context.client, gstate);
+
+	return SinkCombineResultType::FINISHED;
 }
 
 //===--------------------------------------------------------------------===//
@@ -225,16 +227,16 @@ SinkFinalizeType PhysicalFixedBatchCopy::FinalFlush(ClientContext &context, Glob
 }
 
 SinkFinalizeType PhysicalFixedBatchCopy::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
-                                                  GlobalSinkState &gstate_p) const {
-	auto &gstate = gstate_p.Cast<FixedBatchCopyGlobalState>();
+                                                  OperatorSinkFinalizeInput &input) const {
+	auto &gstate = input.global_state.Cast<FixedBatchCopyGlobalState>();
 	idx_t min_batch_index = idx_t(NumericLimits<int64_t>::Maximum());
 	// repartition any remaining batches
-	RepartitionBatches(context, gstate_p, min_batch_index, true);
+	RepartitionBatches(context, input.global_state, min_batch_index, true);
 	// check if we have multiple tasks to execute
 	if (gstate.TaskCount() <= 1) {
 		// we don't - just execute the remaining task and finish flushing to disk
-		ExecuteTasks(context, gstate_p);
-		FinalFlush(context, gstate_p);
+		ExecuteTasks(context, input.global_state);
+		FinalFlush(context, input.global_state);
 		return SinkFinalizeType::READY;
 	}
 	// we have multiple tasks remaining - launch an event to execute the tasks in parallel
@@ -353,7 +355,7 @@ void PhysicalFixedBatchCopy::RepartitionBatches(ClientContext &context, GlobalSi
 			} else {
 				// the collection is too large for a batch - we need to repartition
 				// create an empty collection
-				current_collection = make_uniq<ColumnDataCollection>(Allocator::Get(context), children[0]->types);
+				current_collection = make_uniq<ColumnDataCollection>(BufferAllocator::Get(context), children[0]->types);
 			}
 			if (current_collection) {
 				current_collection->InitializeAppend(append_state);
@@ -373,7 +375,7 @@ void PhysicalFixedBatchCopy::RepartitionBatches(ClientContext &context, GlobalSi
 			}
 			// the collection is full - move it to the result and create a new one
 			gstate.AddTask(make_uniq<PrepareBatchTask>(gstate.scheduled_batch_index++, std::move(current_collection)));
-			current_collection = make_uniq<ColumnDataCollection>(Allocator::Get(context), children[0]->types);
+			current_collection = make_uniq<ColumnDataCollection>(BufferAllocator::Get(context), children[0]->types);
 			current_collection->InitializeAppend(append_state);
 		}
 	}

package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp

@@ -41,7 +41,7 @@ PhysicalInsert::PhysicalInsert(vector<LogicalType> types_p, TableCatalogEntry &t
 		return;
 	}
 
-	D_ASSERT(set_expressions.size() == set_columns.size());
+	D_ASSERT(this->set_expressions.size() == this->set_columns.size());
 
 	// One or more columns are referenced from the existing table,
 	// we use the 'insert_types' to figure out which types these columns have
@@ -463,19 +463,17 @@ SinkResultType PhysicalInsert::Sink(ExecutionContext &context, DataChunk &chunk,
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
-void PhysicalInsert::Combine(ExecutionContext &context, GlobalSinkState &gstate_p, LocalSinkState &lstate_p) const {
-	auto &gstate = gstate_p.Cast<InsertGlobalState>();
-	auto &lstate = lstate_p.Cast<InsertLocalState>();
+SinkCombineResultType PhysicalInsert::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
+	auto &gstate = input.global_state.Cast<InsertGlobalState>();
+	auto &lstate = input.local_state.Cast<InsertLocalState>();
 	auto &client_profiler = QueryProfiler::Get(context.client);
 	context.thread.profiler.Flush(*this, lstate.default_executor, "default_executor", 1);
 	client_profiler.Flush(context.thread.profiler);
 
-	if (!parallel) {
-		return;
-	}
-	if (!lstate.local_collection) {
-		return;
+	if (!parallel || !lstate.local_collection) {
+		return SinkCombineResultType::FINISHED;
 	}
+
 	// parallel append: finalize the append
 	TransactionData tdata(0, 0);
 	lstate.local_collection->FinalizeAppend(tdata, lstate.local_append_state);
@@ -484,7 +482,7 @@ void PhysicalInsert::Combine(ExecutionContext &context, GlobalSinkState &gstate_
 
 	lock_guard<mutex> lock(gstate.lock);
 	gstate.insert_count += append_count;
-	if (append_count < RowGroup::ROW_GROUP_SIZE) {
+	if (append_count < Storage::ROW_GROUP_SIZE) {
 		// we have few rows - append to the local storage directly
 		auto &table = gstate.table;
 		auto &storage = table.GetStorage();
@@ -500,11 +498,13 @@ void PhysicalInsert::Combine(ExecutionContext &context, GlobalSinkState &gstate_
 		gstate.table.GetStorage().FinalizeOptimisticWriter(context.client, *lstate.writer);
 		gstate.table.GetStorage().LocalMerge(context.client, *lstate.local_collection);
 	}
+
+	return SinkCombineResultType::FINISHED;
 }
 
 SinkFinalizeType PhysicalInsert::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
-                                          GlobalSinkState &state) const {
-	auto &gstate = state.Cast<InsertGlobalState>();
+                                          OperatorSinkFinalizeInput &input) const {
+	auto &gstate = input.global_state.Cast<InsertGlobalState>();
 	if (!parallel && gstate.initialized) {
 		auto &table = gstate.table;
 		auto &storage = table.GetStorage();

package/src/duckdb/src/execution/operator/persistent/physical_update.cpp

@@ -140,11 +140,13 @@ unique_ptr<LocalSinkState> PhysicalUpdate::GetLocalSinkState(ExecutionContext &c
 	return make_uniq<UpdateLocalState>(context.client, expressions, table.GetTypes(), bound_defaults);
 }
 
-void PhysicalUpdate::Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const {
-	auto &state = lstate.Cast<UpdateLocalState>();
+SinkCombineResultType PhysicalUpdate::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
+	auto &state = input.local_state.Cast<UpdateLocalState>();
 	auto &client_profiler = QueryProfiler::Get(context.client);
 	context.thread.profiler.Flush(*this, state.default_executor, "default_executor", 1);
 	client_profiler.Flush(context.thread.profiler);
+
+	return SinkCombineResultType::FINISHED;
 }
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp

@@ -16,6 +16,7 @@ PhysicalPivot::PhysicalPivot(vector<LogicalType> types_p, unique_ptr<PhysicalOpe
 		pivot_map[bound_pivot.pivot_values[p]] = bound_pivot.group_count + p;
 	}
 	// extract the empty aggregate expressions
+	ArenaAllocator allocator(Allocator::DefaultAllocator());
 	for (auto &aggr_expr : bound_pivot.aggregates) {
 		auto &aggr = aggr_expr->Cast<BoundAggregateExpression>();
 		// for each aggregate, initialize an empty aggregate state and finalize it immediately
@@ -23,7 +24,7 @@ PhysicalPivot::PhysicalPivot(vector<LogicalType> types_p, unique_ptr<PhysicalOpe
 		aggr.function.initialize(state.get());
 		Vector state_vector(Value::POINTER(CastPointerToValue(state.get())));
 		Vector result_vector(aggr_expr->return_type);
-		AggregateInputData aggr_input_data(aggr.bind_info.get(), Allocator::DefaultAllocator());
+		AggregateInputData aggr_input_data(aggr.bind_info.get(), allocator);
 		aggr.function.finalize(state_vector, aggr_input_data, result_vector, 1, 0);
 		empty_aggregates.push_back(result_vector.GetValue(0));
 	}

package/src/duckdb/src/execution/operator/projection/physical_unnest.cpp

@@ -308,40 +308,37 @@ OperatorResultType PhysicalUnnest::ExecuteInternal(ExecutionContext &context, Da
 				// UNNEST(NULL)
 				chunk.SetCardinality(0);
 				break;
+			}
 
-			} else {
-
-				auto &vector_data = state.list_vector_data[col_idx];
-				auto current_idx = vector_data.sel->get_index(state.current_row);
-
-				if (!vector_data.validity.RowIsValid(current_idx)) {
-					UnnestNull(0, this_chunk_len, result_vector);
+			auto &vector_data = state.list_vector_data[col_idx];
+			auto current_idx = vector_data.sel->get_index(state.current_row);
 
-				} else {
+			if (!vector_data.validity.RowIsValid(current_idx)) {
+				UnnestNull(0, this_chunk_len, result_vector);
+				continue;
+			}
 
-					auto list_data = UnifiedVectorFormat::GetData<list_entry_t>(vector_data);
-					auto list_entry = list_data[current_idx];
+			auto list_data = UnifiedVectorFormat::GetData<list_entry_t>(vector_data);
+			auto list_entry = list_data[current_idx];
 
-					idx_t list_count = 0;
-					if (state.list_position < list_entry.length) {
-						// there are still list_count elements to unnest
-						list_count = MinValue<idx_t>(this_chunk_len, list_entry.length - state.list_position);
+			idx_t list_count = 0;
+			if (state.list_position < list_entry.length) {
+				// there are still list_count elements to unnest
+				list_count = MinValue<idx_t>(this_chunk_len, list_entry.length - state.list_position);
 
-						auto &list_vector = state.list_data.data[col_idx];
-						auto &child_vector = ListVector::GetEntry(list_vector);
-						auto list_size = ListVector::GetListSize(list_vector);
-						auto &child_vector_data = state.list_child_data[col_idx];
+				auto &list_vector = state.list_data.data[col_idx];
+				auto &child_vector = ListVector::GetEntry(list_vector);
+				auto list_size = ListVector::GetListSize(list_vector);
+				auto &child_vector_data = state.list_child_data[col_idx];
 
-						auto base_offset = list_entry.offset + state.list_position;
-						UnnestVector(child_vector_data, child_vector, list_size, base_offset, base_offset + list_count,
-						             result_vector);
-					}
+				auto base_offset = list_entry.offset + state.list_position;
+				UnnestVector(child_vector_data, child_vector, list_size, base_offset, base_offset + list_count,
+				             result_vector);
+			}
 
-					// fill the rest with NULLs
-					if (list_count != this_chunk_len) {
-						UnnestNull(list_count, this_chunk_len, result_vector);
-					}
-				}
+			// fill the rest with NULLs
+			if (list_count != this_chunk_len) {
+				UnnestNull(list_count, this_chunk_len, result_vector);
 			}
 		}
 

package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp

@@ -64,6 +64,9 @@ void PhysicalColumnDataScan::BuildPipelines(Pipeline &current, MetaPipeline &met
 		state.SetPipelineSource(current, delim_join.distinct->Cast<PhysicalOperator>());
 		return;
 	}
+	case PhysicalOperatorType::CTE_SCAN: {
+		break;
+	}
 	case PhysicalOperatorType::RECURSIVE_CTE_SCAN:
 		if (!meta_pipeline.HasRecursiveCTE()) {
 			throw InternalException("Recursive CTE scan found without recursive CTE node");
@@ -76,4 +79,20 @@ void PhysicalColumnDataScan::BuildPipelines(Pipeline &current, MetaPipeline &met
 	state.SetPipelineSource(current, *this);
 }
 
+string PhysicalColumnDataScan::ParamsToString() const {
+	string result = "";
+	switch (type) {
+	case PhysicalOperatorType::CTE_SCAN:
+	case PhysicalOperatorType::RECURSIVE_CTE_SCAN: {
+		result += "\n[INFOSEPARATOR]\n";
+		result += StringUtil::Format("idx: %llu", cte_index);
+		break;
+	}
+	default:
+		break;
+	}
+
+	return result;
+}
+
 } // namespace duckdb

package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp

@@ -9,24 +9,15 @@
 
 namespace duckdb {
 
-PhysicalTableScan::PhysicalTableScan(vector<LogicalType> types, TableFunction function_p,
-                                     unique_ptr<FunctionData> bind_data_p, vector<column_t> column_ids_p,
-                                     vector<string> names_p, unique_ptr<TableFilterSet> table_filters_p,
-                                     idx_t estimated_cardinality)
-    : PhysicalOperator(PhysicalOperatorType::TABLE_SCAN, std::move(types), estimated_cardinality),
-      function(std::move(function_p)), bind_data(std::move(bind_data_p)), column_ids(std::move(column_ids_p)),
-      names(std::move(names_p)), table_filters(std::move(table_filters_p)) {
-}
-
 PhysicalTableScan::PhysicalTableScan(vector<LogicalType> types, TableFunction function_p,
                                      unique_ptr<FunctionData> bind_data_p, vector<LogicalType> returned_types_p,
                                      vector<column_t> column_ids_p, vector<idx_t> projection_ids_p,
                                      vector<string> names_p, unique_ptr<TableFilterSet> table_filters_p,
-                                     idx_t estimated_cardinality)
+                                     idx_t estimated_cardinality, ExtraOperatorInfo extra_info)
     : PhysicalOperator(PhysicalOperatorType::TABLE_SCAN, std::move(types), estimated_cardinality),
       function(std::move(function_p)), bind_data(std::move(bind_data_p)), returned_types(std::move(returned_types_p)),
       column_ids(std::move(column_ids_p)), projection_ids(std::move(projection_ids_p)), names(std::move(names_p)),
-      table_filters(std::move(table_filters_p)) {
+      table_filters(std::move(table_filters_p)), extra_info(extra_info) {
 }
 
 class TableScanGlobalSourceState : public GlobalSourceState {
@@ -149,8 +140,12 @@ string PhysicalTableScan::ParamsToString() const {
 			}
 		}
 	}
+	if (!extra_info.file_filters.empty()) {
+		result += "\n[INFOSEPARATOR]\n";
+		result += "File Filters: " + extra_info.file_filters;
+	}
 	result += "\n[INFOSEPARATOR]\n";
-	result += StringUtil::Format("EC: %llu", estimated_props->GetCardinality<idx_t>());
+	result += StringUtil::Format("EC: %llu", estimated_cardinality);
 	return result;
 }