duckdb 0.8.2-dev33.0 → 0.8.2-dev3334.0

package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp

@@ -1,5 +1,4 @@
 #include "duckdb/execution/operator/persistent/base_csv_reader.hpp"
-
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/common/file_system.hpp"
 #include "duckdb/common/string_util.hpp"
@@ -10,6 +9,7 @@
 #include "duckdb/common/vector_operations/unary_executor.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/function/scalar/strftime_format.hpp"
+#include "duckdb/main/appender.hpp"
 #include "duckdb/main/database.hpp"
 #include "duckdb/parser/column_definition.hpp"
 #include "duckdb/storage/data_table.hpp"
@@ -18,7 +18,8 @@
 #include "duckdb/parser/keyword_helper.hpp"
 #include "duckdb/main/error_manager.hpp"
 #include "duckdb/execution/operator/persistent/parallel_csv_reader.hpp"
-
+#include "duckdb/execution/operator/persistent/csv_rejects_table.hpp"
+#include "duckdb/main/client_data.hpp"
 #include <algorithm>
 #include <cctype>
 #include <cstring>
@@ -448,6 +449,17 @@ bool TryCastFloatingVectorCommaSeparated(BufferedCSVReaderOptions &options, Vect
 	}
 }
 
+// Location of erroneous value in the current parse chunk
+struct ErrorLocation {
+	idx_t row_idx;
+	idx_t col_idx;
+	idx_t row_line;
+
+	ErrorLocation(idx_t row_idx, idx_t col_idx, idx_t row_line)
+	    : row_idx(row_idx), col_idx(col_idx), row_line(row_line) {
+	}
+};
+
 bool BaseCSVReader::Flush(DataChunk &insert_chunk, idx_t buffer_idx, bool try_add_line) {
 	if (parse_chunk.size() == 0) {
 		return true;
@@ -506,10 +518,7 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, idx_t buffer_idx, bool try_ad
 			if (try_add_line) {
 				return false;
 			}
-			if (options.ignore_errors) {
-				conversion_error_ignored = true;
-				continue;
-			}
+
 			string col_name = to_string(col_idx);
 			if (col_idx < names.size()) {
 				col_name = "\"" + names[col_idx] + "\"";
@@ -527,16 +536,18 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, idx_t buffer_idx, bool try_ad
 				}
 			}
 
-			idx_t error_line;
 			// The line_error must be summed with linenr (All lines emmited from this batch)
 			// But subtracted from the parse_chunk
 			D_ASSERT(line_error + linenr >= parse_chunk.size());
 			line_error += linenr;
 			line_error -= parse_chunk.size();
 
-			error_line = GetLineError(line_error, buffer_idx);
+			auto error_line = GetLineError(line_error, buffer_idx);
+
+			if (options.ignore_errors) {
+				conversion_error_ignored = true;
 
-			if (options.auto_detect) {
+			} else if (options.auto_detect) {
 				throw InvalidInputException("%s in column %s, at line %llu.\n\nParser "
 				                            "options:\n%s.\n\nConsider either increasing the sample size "
 				                            "(SAMPLE_SIZE=X [X rows] or SAMPLE_SIZE=-1 [all rows]), "
@@ -550,11 +561,19 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, idx_t buffer_idx, bool try_ad
 	}
 	if (conversion_error_ignored) {
 		D_ASSERT(options.ignore_errors);
+
 		SelectionVector succesful_rows(parse_chunk.size());
 		idx_t sel_size = 0;
 
+		// Keep track of failed cells
+		vector<ErrorLocation> failed_cells;
+
 		for (idx_t row_idx = 0; row_idx < parse_chunk.size(); row_idx++) {
-			bool failed = false;
+
+			auto global_row_idx = row_idx + linenr - parse_chunk.size();
+			auto row_line = GetLineError(global_row_idx, buffer_idx, false);
+
+			bool row_failed = false;
 			for (idx_t c = 0; c < reader_data.column_ids.size(); c++) {
 				auto col_idx = reader_data.column_ids[c];
 				auto result_idx = reader_data.column_mapping[c];
@@ -564,14 +583,82 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, idx_t buffer_idx, bool try_ad
 
 				bool was_already_null = FlatVector::IsNull(parse_vector, row_idx);
 				if (!was_already_null && FlatVector::IsNull(result_vector, row_idx)) {
-					failed = true;
-					break;
+					row_failed = true;
+					failed_cells.emplace_back(row_idx, col_idx, row_line);
 				}
 			}
-			if (!failed) {
+			if (!row_failed) {
 				succesful_rows.set_index(sel_size++, row_idx);
 			}
 		}
+
+		// Now do a second pass to produce the reject table entries
+		if (!failed_cells.empty() && !options.rejects_table_name.empty()) {
+			auto limit = options.rejects_limit;
+
+			auto rejects = CSVRejectsTable::GetOrCreate(context, options.rejects_table_name);
+			lock_guard<mutex> lock(rejects->write_lock);
+
+			// short circuit if we already have too many rejects
+			if (limit == 0 || rejects->count < limit) {
+				auto &table = rejects->GetTable(context);
+				InternalAppender appender(context, table);
+				auto file_name = GetFileName();
+
+				for (auto &cell : failed_cells) {
+					if (limit != 0 && rejects->count >= limit) {
+						break;
+					}
+					rejects->count++;
+
+					auto row_idx = cell.row_idx;
+					auto col_idx = cell.col_idx;
+					auto row_line = cell.row_line;
+
+					auto col_name = to_string(col_idx);
+					if (col_idx < names.size()) {
+						col_name = "\"" + names[col_idx] + "\"";
+					}
+
+					auto &parse_vector = parse_chunk.data[col_idx];
+					auto parsed_str = FlatVector::GetData<string_t>(parse_vector)[row_idx];
+					auto &type = insert_chunk.data[col_idx].GetType();
+					auto row_error_msg = StringUtil::Format("Could not convert string '%s' to '%s'",
+					                                        parsed_str.GetString(), type.ToString());
+
+					// Add the row to the rejects table
+					appender.BeginRow();
+					appender.Append(string_t(file_name));
+					appender.Append(row_line);
+					appender.Append(col_idx);
+					appender.Append(string_t(col_name));
+					appender.Append(parsed_str);
+
+					if (!options.rejects_recovery_columns.empty()) {
+						child_list_t<Value> recovery_key;
+						for (auto &key_idx : options.rejects_recovery_column_ids) {
+							// Figure out if the recovery key is valid.
+							// If not, error out for real.
+							auto &component_vector = parse_chunk.data[key_idx];
+							if (FlatVector::IsNull(component_vector, row_idx)) {
+								throw InvalidInputException("%s at line %llu in column %s. Parser options:\n%s ",
+								                            "Could not parse recovery column", row_line, col_name,
+								                            options.ToString());
+							}
+							auto component = Value(FlatVector::GetData<string_t>(component_vector)[row_idx]);
+							recovery_key.emplace_back(names[key_idx], component);
+						}
+						appender.Append(Value::STRUCT(recovery_key));
+					}
+
+					appender.Append(string_t(row_error_msg));
+					appender.EndRow();
+				}
+				appender.Close();
+			}
+		}
+
+		// Now slice the insert chunk to only include the succesful rows
 		insert_chunk.Slice(succesful_rows, sel_size);
 	}
 	parse_chunk.Reset();

package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp

@@ -13,7 +13,7 @@ CSVFileHandle::CSVFileHandle(FileSystem &fs, Allocator &allocator, unique_ptr<Fi
 
 unique_ptr<FileHandle> CSVFileHandle::OpenFileHandle(FileSystem &fs, Allocator &allocator, const string &path,
                                                      FileCompressionType compression) {
-	auto file_handle = fs.OpenFile(path.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK, compression);
+	auto file_handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK, compression);
 	if (file_handle->CanSeek()) {
 		file_handle->Reset();
 	}

package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp

@@ -179,6 +179,26 @@ void BufferedCSVReaderOptions::SetReadOption(const string &loption, const Value
 		allow_quoted_nulls = ParseBoolean(value, loption);
 	} else if (loption == "parallel") {
 		parallel_mode = ParseBoolean(value, loption) ? ParallelMode::PARALLEL : ParallelMode::SINGLE_THREADED;
+	} else if (loption == "rejects_table") {
+		// skip, handled in SetRejectsOptions
+		auto table_name = ParseString(value, loption);
+		if (table_name.empty()) {
+			throw BinderException("REJECTS_TABLE option cannot be empty");
+		}
+		rejects_table_name = table_name;
+	} else if (loption == "rejects_recovery_columns") {
+		// Get the list of columns to use as a recovery key
+		auto &children = ListValue::GetChildren(value);
+		for (auto &child : children) {
+			auto col_name = child.GetValue<string>();
+			rejects_recovery_columns.push_back(col_name);
+		}
+	} else if (loption == "rejects_limit") {
+		int64_t limit = ParseInteger(value, loption);
+		if (limit < 0) {
+			throw BinderException("Unsupported parameter for REJECTS_LIMIT: cannot be negative");
+		}
+		rejects_limit = limit;
 	} else {
 		throw BinderException("Unrecognized option for CSV reader \"%s\"", loption);
 	}

package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp

@@ -0,0 +1,48 @@
+#include "duckdb/main/appender.hpp"
+#include "duckdb/parser/parsed_data/create_table_info.hpp"
+#include "duckdb/function/table/read_csv.hpp"
+#include "duckdb/execution/operator/persistent/csv_rejects_table.hpp"
+#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
+
+namespace duckdb {
+
+TableCatalogEntry &CSVRejectsTable::GetTable(ClientContext &context) {
+	auto &temp_catalog = Catalog::GetCatalog(context, TEMP_CATALOG);
+	auto &table_entry = temp_catalog.GetEntry<TableCatalogEntry>(context, TEMP_CATALOG, DEFAULT_SCHEMA, name);
+	return table_entry;
+}
+
+shared_ptr<CSVRejectsTable> CSVRejectsTable::GetOrCreate(ClientContext &context, const string &name) {
+	auto key = "CSV_REJECTS_TABLE_CACHE_ENTRY_" + StringUtil::Upper(name);
+	auto &cache = ObjectCache::GetObjectCache(context);
+	return cache.GetOrCreate<CSVRejectsTable>(key, name);
+}
+
+void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData &data) {
+	// (Re)Create the temporary rejects table
+	auto &catalog = Catalog::GetCatalog(context, TEMP_CATALOG);
+	auto info = make_uniq<CreateTableInfo>(TEMP_CATALOG, DEFAULT_SCHEMA, name);
+	info->temporary = true;
+	info->on_conflict = OnCreateConflict::ERROR_ON_CONFLICT;
+	info->columns.AddColumn(ColumnDefinition("file", LogicalType::VARCHAR));
+	info->columns.AddColumn(ColumnDefinition("line", LogicalType::BIGINT));
+	info->columns.AddColumn(ColumnDefinition("column", LogicalType::BIGINT));
+	info->columns.AddColumn(ColumnDefinition("column_name", LogicalType::VARCHAR));
+	info->columns.AddColumn(ColumnDefinition("parsed_value", LogicalType::VARCHAR));
+
+	if (!data.options.rejects_recovery_columns.empty()) {
+		child_list_t<LogicalType> recovery_key_components;
+		for (auto &col_name : data.options.rejects_recovery_columns) {
+			recovery_key_components.emplace_back(col_name, LogicalType::VARCHAR);
+		}
+		info->columns.AddColumn(ColumnDefinition("recovery_columns", LogicalType::STRUCT(recovery_key_components)));
+	}
+
+	info->columns.AddColumn(ColumnDefinition("error", LogicalType::VARCHAR));
+
+	catalog.CreateTable(context, std::move(info));
+
+	count = 0;
+}
+
+} // namespace duckdb

package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp

@@ -635,12 +635,11 @@ void ParallelCSVReader::ParseCSV(DataChunk &insert_chunk) {
 	}
 }
 
-idx_t ParallelCSVReader::GetLineError(idx_t line_error, idx_t buffer_idx) {
+idx_t ParallelCSVReader::GetLineError(idx_t line_error, idx_t buffer_idx, bool stop_at_first) {
 	while (true) {
 		if (buffer->line_info->CanItGetLine(file_idx, buffer_idx)) {
 			auto cur_start = verification_positions.beginning_of_first_line + buffer->buffer->GetCSVGlobalStart();
-			// line errors are 1-indexed
-			return buffer->line_info->GetLine(buffer_idx, line_error, file_idx, cur_start, false);
+			return buffer->line_info->GetLine(buffer_idx, line_error, file_idx, cur_start, false, stop_at_first);
 		}
 	}
 }

package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp

@@ -1,9 +1,11 @@
 #include "duckdb/execution/operator/persistent/physical_batch_copy_to_file.hpp"
+
+#include "duckdb/common/allocator.hpp"
+#include "duckdb/common/types/batched_data_collection.hpp"
+#include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/execution/operator/persistent/physical_copy_to_file.hpp"
 #include "duckdb/parallel/base_pipeline_event.hpp"
-#include "duckdb/common/vector_operations/vector_operations.hpp"
-#include "duckdb/common/types/batched_data_collection.hpp"
-#include "duckdb/common/allocator.hpp"
+
 #include <algorithm>
 
 namespace duckdb {
@@ -67,7 +69,7 @@ public:
 	optional_idx batch_index;
 
 	void InitializeCollection(ClientContext &context, const PhysicalOperator &op) {
-		collection = make_uniq<ColumnDataCollection>(Allocator::Get(context), op.children[0]->types);
+		collection = make_uniq<ColumnDataCollection>(BufferAllocator::Get(context), op.children[0]->types);
 		collection->InitializeAppend(append_state);
 	}
 };
@@ -87,11 +89,13 @@ SinkResultType PhysicalBatchCopyToFile::Sink(ExecutionContext &context, DataChun
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
-void PhysicalBatchCopyToFile::Combine(ExecutionContext &context, GlobalSinkState &gstate_p,
-                                      LocalSinkState &lstate) const {
-	auto &state = lstate.Cast<BatchCopyToLocalState>();
-	auto &gstate = gstate_p.Cast<BatchCopyToGlobalState>();
+SinkCombineResultType PhysicalBatchCopyToFile::Combine(ExecutionContext &context,
+                                                       OperatorSinkCombineInput &input) const {
+	auto &state = input.local_state.Cast<BatchCopyToLocalState>();
+	auto &gstate = input.global_state.Cast<BatchCopyToGlobalState>();
 	gstate.rows_copied += state.rows_copied;
+
+	return SinkCombineResultType::FINISHED;
 }
 
 //===--------------------------------------------------------------------===//
@@ -112,8 +116,8 @@ SinkFinalizeType PhysicalBatchCopyToFile::FinalFlush(ClientContext &context, Glo
 }
 
 SinkFinalizeType PhysicalBatchCopyToFile::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
-                                                   GlobalSinkState &gstate_p) const {
-	FinalFlush(context, gstate_p);
+                                                   OperatorSinkFinalizeInput &input) const {
+	FinalFlush(context, input.global_state);
 	return SinkFinalizeType::READY;
 }
 

package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp

@@ -327,10 +327,11 @@ SinkResultType PhysicalBatchInsert::Sink(ExecutionContext &context, DataChunk &c
 		// no collection yet: create a new one
 		lstate.CreateNewCollection(table, insert_types);
 		lstate.writer = &table.GetStorage().CreateOptimisticWriter(context.client);
-	} else if (lstate.current_index != batch_index) {
+	}
+
+	if (lstate.current_index != batch_index) {
 		throw InternalException("Current batch differs from batch - but NextBatch was not called!?");
 	}
-	lstate.current_index = batch_index;
 
 	table.GetStorage().VerifyAppendConstraints(table, context.client, lstate.insert_chunk);
 
@@ -343,16 +344,15 @@ SinkResultType PhysicalBatchInsert::Sink(ExecutionContext &context, DataChunk &c
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
-void PhysicalBatchInsert::Combine(ExecutionContext &context, GlobalSinkState &gstate_p,
-                                  LocalSinkState &lstate_p) const {
-	auto &gstate = gstate_p.Cast<BatchInsertGlobalState>();
-	auto &lstate = lstate_p.Cast<BatchInsertLocalState>();
+SinkCombineResultType PhysicalBatchInsert::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
+	auto &gstate = input.global_state.Cast<BatchInsertGlobalState>();
+	auto &lstate = input.local_state.Cast<BatchInsertLocalState>();
 	auto &client_profiler = QueryProfiler::Get(context.client);
 	context.thread.profiler.Flush(*this, lstate.default_executor, "default_executor", 1);
 	client_profiler.Flush(context.thread.profiler);
 
 	if (!lstate.current_collection) {
-		return;
+		return SinkCombineResultType::FINISHED;
 	}
 
 	if (lstate.current_collection->GetTotalRows() > 0) {
@@ -365,11 +365,13 @@ void PhysicalBatchInsert::Combine(ExecutionContext &context, GlobalSinkState &gs
 		lock_guard<mutex> l(gstate.lock);
 		gstate.table.GetStorage().FinalizeOptimisticWriter(context.client, *lstate.writer);
 	}
+
+	return SinkCombineResultType::FINISHED;
 }
 
 SinkFinalizeType PhysicalBatchInsert::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
-                                               GlobalSinkState &gstate_p) const {
-	auto &gstate = gstate_p.Cast<BatchInsertGlobalState>();
+                                               OperatorSinkFinalizeInput &input) const {
+	auto &gstate = input.global_state.Cast<BatchInsertGlobalState>();
 
 	// in the finalize, do a final pass over all of the collections we created and try to merge smaller collections
 	// together

package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp

@@ -97,9 +97,9 @@ static string CreateDirRecursive(const vector<idx_t> &cols, const vector<string>
 	return path;
 }
 
-void PhysicalCopyToFile::Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const {
-	auto &g = gstate.Cast<CopyToFunctionGlobalState>();
-	auto &l = lstate.Cast<CopyToFunctionLocalState>();
+SinkCombineResultType PhysicalCopyToFile::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
+	auto &g = input.global_state.Cast<CopyToFunctionGlobalState>();
+	auto &l = input.local_state.Cast<CopyToFunctionLocalState>();
 
 	if (partition_output) {
 		auto &fs = FileSystem::GetFileSystem(context.client);
@@ -108,7 +108,7 @@ void PhysicalCopyToFile::Combine(ExecutionContext &context, GlobalSinkState &gst
 		auto partition_key_map = l.part_buffer->GetReverseMap();
 
 		string trimmed_path = file_path;
-		StringUtil::RTrim(trimmed_path, fs.PathSeparator());
+		StringUtil::RTrim(trimmed_path, fs.PathSeparator(trimmed_path));
 
 		for (idx_t i = 0; i < partitions.size(); i++) {
 			string hive_path =
@@ -130,7 +130,7 @@ void PhysicalCopyToFile::Combine(ExecutionContext &context, GlobalSinkState &gst
 			function.copy_to_finalize(context.client, *bind_data, *fun_data_global);
 		}
 
-		return;
+		return SinkCombineResultType::FINISHED;
 	}
 
 	if (function.copy_to_combine) {
@@ -141,11 +141,13 @@ void PhysicalCopyToFile::Combine(ExecutionContext &context, GlobalSinkState &gst
 			function.copy_to_finalize(context.client, *bind_data, *l.global_state);
 		}
 	}
+
+	return SinkCombineResultType::FINISHED;
 }
 
 SinkFinalizeType PhysicalCopyToFile::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
-                                              GlobalSinkState &gstate_p) const {
-	auto &gstate = gstate_p.Cast<CopyToFunctionGlobalState>();
+                                              OperatorSinkFinalizeInput &input) const {
+	auto &gstate = input.global_state.Cast<CopyToFunctionGlobalState>();
 	if (per_thread_output || partition_output) {
 		// already happened in combine
 		return SinkFinalizeType::READY;

package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp

@@ -116,7 +116,7 @@ public:
 	optional_idx batch_index;
 
 	void InitializeCollection(ClientContext &context, const PhysicalOperator &op) {
-		collection = make_uniq<ColumnDataCollection>(Allocator::Get(context), op.children[0]->types);
+		collection = make_uniq<ColumnDataCollection>(BufferAllocator::Get(context), op.children[0]->types);
 		collection->InitializeAppend(append_state);
 	}
 };
@@ -136,10 +136,10 @@ SinkResultType PhysicalFixedBatchCopy::Sink(ExecutionContext &context, DataChunk
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
-void PhysicalFixedBatchCopy::Combine(ExecutionContext &context, GlobalSinkState &gstate_p,
-                                     LocalSinkState &lstate) const {
-	auto &state = lstate.Cast<FixedBatchCopyLocalState>();
-	auto &gstate = gstate_p.Cast<FixedBatchCopyGlobalState>();
+SinkCombineResultType PhysicalFixedBatchCopy::Combine(ExecutionContext &context,
+                                                      OperatorSinkCombineInput &input) const {
+	auto &state = input.local_state.Cast<FixedBatchCopyLocalState>();
+	auto &gstate = input.global_state.Cast<FixedBatchCopyGlobalState>();
 	gstate.rows_copied += state.rows_copied;
 	if (!gstate.any_finished) {
 		// signal that this thread is finished processing batches and that we should move on to Finalize
@@ -147,6 +147,8 @@ void PhysicalFixedBatchCopy::Combine(ExecutionContext &context, GlobalSinkState
 		gstate.any_finished = true;
 	}
 	ExecuteTasks(context.client, gstate);
+
+	return SinkCombineResultType::FINISHED;
 }
 
 //===--------------------------------------------------------------------===//
@@ -225,16 +227,16 @@ SinkFinalizeType PhysicalFixedBatchCopy::FinalFlush(ClientContext &context, Glob
 }
 
 SinkFinalizeType PhysicalFixedBatchCopy::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
-                                                  GlobalSinkState &gstate_p) const {
-	auto &gstate = gstate_p.Cast<FixedBatchCopyGlobalState>();
+                                                  OperatorSinkFinalizeInput &input) const {
+	auto &gstate = input.global_state.Cast<FixedBatchCopyGlobalState>();
 	idx_t min_batch_index = idx_t(NumericLimits<int64_t>::Maximum());
 	// repartition any remaining batches
-	RepartitionBatches(context, gstate_p, min_batch_index, true);
+	RepartitionBatches(context, input.global_state, min_batch_index, true);
 	// check if we have multiple tasks to execute
 	if (gstate.TaskCount() <= 1) {
 		// we don't - just execute the remaining task and finish flushing to disk
-		ExecuteTasks(context, gstate_p);
-		FinalFlush(context, gstate_p);
+		ExecuteTasks(context, input.global_state);
+		FinalFlush(context, input.global_state);
 		return SinkFinalizeType::READY;
 	}
 	// we have multiple tasks remaining - launch an event to execute the tasks in parallel
@@ -353,7 +355,7 @@ void PhysicalFixedBatchCopy::RepartitionBatches(ClientContext &context, GlobalSi
 			} else {
 				// the collection is too large for a batch - we need to repartition
 				// create an empty collection
-				current_collection = make_uniq<ColumnDataCollection>(Allocator::Get(context), children[0]->types);
+				current_collection = make_uniq<ColumnDataCollection>(BufferAllocator::Get(context), children[0]->types);
 			}
 			if (current_collection) {
 				current_collection->InitializeAppend(append_state);
@@ -373,7 +375,7 @@ void PhysicalFixedBatchCopy::RepartitionBatches(ClientContext &context, GlobalSi
 			}
 			// the collection is full - move it to the result and create a new one
 			gstate.AddTask(make_uniq<PrepareBatchTask>(gstate.scheduled_batch_index++, std::move(current_collection)));
-			current_collection = make_uniq<ColumnDataCollection>(Allocator::Get(context), children[0]->types);
+			current_collection = make_uniq<ColumnDataCollection>(BufferAllocator::Get(context), children[0]->types);
 			current_collection->InitializeAppend(append_state);
 		}
 	}

package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp

@@ -41,7 +41,7 @@ PhysicalInsert::PhysicalInsert(vector<LogicalType> types_p, TableCatalogEntry &t
 		return;
 	}
 
-	D_ASSERT(set_expressions.size() == set_columns.size());
+	D_ASSERT(this->set_expressions.size() == this->set_columns.size());
 
 	// One or more columns are referenced from the existing table,
 	// we use the 'insert_types' to figure out which types these columns have
@@ -463,19 +463,17 @@ SinkResultType PhysicalInsert::Sink(ExecutionContext &context, DataChunk &chunk,
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
-void PhysicalInsert::Combine(ExecutionContext &context, GlobalSinkState &gstate_p, LocalSinkState &lstate_p) const {
-	auto &gstate = gstate_p.Cast<InsertGlobalState>();
-	auto &lstate = lstate_p.Cast<InsertLocalState>();
+SinkCombineResultType PhysicalInsert::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
+	auto &gstate = input.global_state.Cast<InsertGlobalState>();
+	auto &lstate = input.local_state.Cast<InsertLocalState>();
 	auto &client_profiler = QueryProfiler::Get(context.client);
 	context.thread.profiler.Flush(*this, lstate.default_executor, "default_executor", 1);
 	client_profiler.Flush(context.thread.profiler);
 
-	if (!parallel) {
-		return;
-	}
-	if (!lstate.local_collection) {
-		return;
+	if (!parallel || !lstate.local_collection) {
+		return SinkCombineResultType::FINISHED;
 	}
+
 	// parallel append: finalize the append
 	TransactionData tdata(0, 0);
 	lstate.local_collection->FinalizeAppend(tdata, lstate.local_append_state);
@@ -500,11 +498,13 @@ void PhysicalInsert::Combine(ExecutionContext &context, GlobalSinkState &gstate_
 		gstate.table.GetStorage().FinalizeOptimisticWriter(context.client, *lstate.writer);
 		gstate.table.GetStorage().LocalMerge(context.client, *lstate.local_collection);
 	}
+
+	return SinkCombineResultType::FINISHED;
 }
 
 SinkFinalizeType PhysicalInsert::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
-                                          GlobalSinkState &state) const {
-	auto &gstate = state.Cast<InsertGlobalState>();
+                                          OperatorSinkFinalizeInput &input) const {
+	auto &gstate = input.global_state.Cast<InsertGlobalState>();
 	if (!parallel && gstate.initialized) {
 		auto &table = gstate.table;
 		auto &storage = table.GetStorage();

package/src/duckdb/src/execution/operator/persistent/physical_update.cpp

@@ -140,11 +140,13 @@ unique_ptr<LocalSinkState> PhysicalUpdate::GetLocalSinkState(ExecutionContext &c
 	return make_uniq<UpdateLocalState>(context.client, expressions, table.GetTypes(), bound_defaults);
 }
 
-void PhysicalUpdate::Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const {
-	auto &state = lstate.Cast<UpdateLocalState>();
+SinkCombineResultType PhysicalUpdate::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
+	auto &state = input.local_state.Cast<UpdateLocalState>();
 	auto &client_profiler = QueryProfiler::Get(context.client);
 	context.thread.profiler.Flush(*this, state.default_executor, "default_executor", 1);
 	client_profiler.Flush(context.thread.profiler);
+
+	return SinkCombineResultType::FINISHED;
 }
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp

@@ -16,6 +16,7 @@ PhysicalPivot::PhysicalPivot(vector<LogicalType> types_p, unique_ptr<PhysicalOpe
 		pivot_map[bound_pivot.pivot_values[p]] = bound_pivot.group_count + p;
 	}
 	// extract the empty aggregate expressions
+	ArenaAllocator allocator(Allocator::DefaultAllocator());
 	for (auto &aggr_expr : bound_pivot.aggregates) {
 		auto &aggr = aggr_expr->Cast<BoundAggregateExpression>();
 		// for each aggregate, initialize an empty aggregate state and finalize it immediately
@@ -23,7 +24,7 @@ PhysicalPivot::PhysicalPivot(vector<LogicalType> types_p, unique_ptr<PhysicalOpe
 		aggr.function.initialize(state.get());
 		Vector state_vector(Value::POINTER(CastPointerToValue(state.get())));
 		Vector result_vector(aggr_expr->return_type);
-		AggregateInputData aggr_input_data(aggr.bind_info.get(), Allocator::DefaultAllocator());
+		AggregateInputData aggr_input_data(aggr.bind_info.get(), allocator);
 		aggr.function.finalize(state_vector, aggr_input_data, result_vector, 1, 0);
 		empty_aggregates.push_back(result_vector.GetValue(0));
 	}

package/src/duckdb/src/execution/operator/projection/physical_unnest.cpp

@@ -308,40 +308,37 @@ OperatorResultType PhysicalUnnest::ExecuteInternal(ExecutionContext &context, Da
 				// UNNEST(NULL)
 				chunk.SetCardinality(0);
 				break;
+			}
 
-			} else {
-
-				auto &vector_data = state.list_vector_data[col_idx];
-				auto current_idx = vector_data.sel->get_index(state.current_row);
-
-				if (!vector_data.validity.RowIsValid(current_idx)) {
-					UnnestNull(0, this_chunk_len, result_vector);
+			auto &vector_data = state.list_vector_data[col_idx];
+			auto current_idx = vector_data.sel->get_index(state.current_row);
 
-				} else {
+			if (!vector_data.validity.RowIsValid(current_idx)) {
+				UnnestNull(0, this_chunk_len, result_vector);
+				continue;
+			}
 
-					auto list_data = UnifiedVectorFormat::GetData<list_entry_t>(vector_data);
-					auto list_entry = list_data[current_idx];
+			auto list_data = UnifiedVectorFormat::GetData<list_entry_t>(vector_data);
+			auto list_entry = list_data[current_idx];
 
-					idx_t list_count = 0;
-					if (state.list_position < list_entry.length) {
-						// there are still list_count elements to unnest
-						list_count = MinValue<idx_t>(this_chunk_len, list_entry.length - state.list_position);
+			idx_t list_count = 0;
+			if (state.list_position < list_entry.length) {
+				// there are still list_count elements to unnest
+				list_count = MinValue<idx_t>(this_chunk_len, list_entry.length - state.list_position);
 
-						auto &list_vector = state.list_data.data[col_idx];
-						auto &child_vector = ListVector::GetEntry(list_vector);
-						auto list_size = ListVector::GetListSize(list_vector);
-						auto &child_vector_data = state.list_child_data[col_idx];
+				auto &list_vector = state.list_data.data[col_idx];
+				auto &child_vector = ListVector::GetEntry(list_vector);
+				auto list_size = ListVector::GetListSize(list_vector);
+				auto &child_vector_data = state.list_child_data[col_idx];
 
-						auto base_offset = list_entry.offset + state.list_position;
-						UnnestVector(child_vector_data, child_vector, list_size, base_offset, base_offset + list_count,
-						             result_vector);
-					}
+				auto base_offset = list_entry.offset + state.list_position;
+				UnnestVector(child_vector_data, child_vector, list_size, base_offset, base_offset + list_count,
+				             result_vector);
+			}
 
-					// fill the rest with NULLs
-					if (list_count != this_chunk_len) {
-						UnnestNull(list_count, this_chunk_len, result_vector);
-					}
-				}
+			// fill the rest with NULLs
+			if (list_count != this_chunk_len) {
+				UnnestNull(list_count, this_chunk_len, result_vector);
 			}
 		}