duckdb 1.3.3-dev4.0 → 1.3.3

package/src/duckdb/src/execution/operator/persistent/physical_export.cpp

@@ -21,10 +21,9 @@ void ReorderTableEntries(catalog_entry_vector_t &tables);
 
 using duckdb::stringstream;
 
-PhysicalExport::PhysicalExport(PhysicalPlan &physical_plan, vector<LogicalType> types, CopyFunction function,
-                               unique_ptr<CopyInfo> info, idx_t estimated_cardinality,
-                               unique_ptr<BoundExportData> exported_tables)
-    : PhysicalOperator(physical_plan, PhysicalOperatorType::EXPORT, std::move(types), estimated_cardinality),
+PhysicalExport::PhysicalExport(vector<LogicalType> types, CopyFunction function, unique_ptr<CopyInfo> info,
+                               idx_t estimated_cardinality, unique_ptr<BoundExportData> exported_tables)
+    : PhysicalOperator(PhysicalOperatorType::EXPORT, std::move(types), estimated_cardinality),
       function(std::move(function)), info(std::move(info)), exported_tables(std::move(exported_tables)) {
 }
 
@@ -55,7 +54,8 @@ static void WriteStringStreamToFile(FileSystem &fs, stringstream &ss, const stri
 	handle.reset();
 }
 
-static void WriteCopyStatement(FileSystem &fs, stringstream &ss, CopyInfo &info, ExportedTableData &exported_table) {
+static void WriteCopyStatement(FileSystem &fs, stringstream &ss, CopyInfo &info, ExportedTableData &exported_table,
+                               CopyFunction const &function) {
 	ss << "COPY ";
 
 	//! NOTE: The catalog is explicitly not set here
@@ -68,12 +68,26 @@ static void WriteCopyStatement(FileSystem &fs, stringstream &ss, CopyInfo &info,
 	// write the copy options
 	ss << "FORMAT '" << info.format << "'";
 	if (info.format == "csv") {
+		// insert default csv options, if not specified
+		if (info.options.find("header") == info.options.end()) {
+			info.options["header"].push_back(Value::INTEGER(1));
+		}
+		if (info.options.find("delimiter") == info.options.end() && info.options.find("sep") == info.options.end() &&
+		    info.options.find("delim") == info.options.end()) {
+			info.options["delimiter"].push_back(Value(","));
+		}
+		if (info.options.find("quote") == info.options.end()) {
+			info.options["quote"].push_back(Value("\""));
+		}
 		info.options.erase("force_not_null");
 		for (auto &not_null_column : exported_table.not_null_columns) {
 			info.options["force_not_null"].push_back(not_null_column);
 		}
 	}
 	for (auto &copy_option : info.options) {
+		if (copy_option.first == "force_quote") {
+			continue;
+		}
 		if (copy_option.second.empty()) {
 			// empty options are interpreted as TRUE
 			copy_option.second.push_back(true);
@@ -234,7 +248,7 @@ SourceResultType PhysicalExport::GetData(ExecutionContext &context, DataChunk &c
 	stringstream load_ss;
 	for (idx_t i = 0; i < exported_tables->data.size(); i++) {
 		auto exported_table_info = exported_tables->data[i].table_data;
-		WriteCopyStatement(fs, load_ss, *info, exported_table_info);
+		WriteCopyStatement(fs, load_ss, *info, exported_table_info, function);
 	}
 	WriteStringStreamToFile(fs, load_ss, fs.JoinPath(info->file_path, "load.sql"));
 	state.finished = true;

package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp

@@ -22,7 +22,7 @@
 
 namespace duckdb {
 
-PhysicalInsert::PhysicalInsert(PhysicalPlan &physical_plan, vector<LogicalType> types_p, TableCatalogEntry &table,
+PhysicalInsert::PhysicalInsert(vector<LogicalType> types_p, TableCatalogEntry &table,
                                vector<unique_ptr<BoundConstraint>> bound_constraints_p,
                                vector<unique_ptr<Expression>> set_expressions, vector<PhysicalIndex> set_columns,
                                vector<LogicalType> set_types, idx_t estimated_cardinality, bool return_chunk,
@@ -30,10 +30,10 @@ PhysicalInsert::PhysicalInsert(PhysicalPlan &physical_plan, vector<LogicalType>
                                unique_ptr<Expression> on_conflict_condition_p,
                                unique_ptr<Expression> do_update_condition_p, unordered_set<column_t> conflict_target_p,
                                vector<column_t> columns_to_fetch_p, bool update_is_del_and_insert)
-    : PhysicalOperator(physical_plan, PhysicalOperatorType::INSERT, std::move(types_p), estimated_cardinality),
-      insert_table(&table), insert_types(table.GetTypes()), bound_constraints(std::move(bound_constraints_p)),
-      return_chunk(return_chunk), parallel(parallel), action_type(action_type),
-      set_expressions(std::move(set_expressions)), set_columns(std::move(set_columns)), set_types(std::move(set_types)),
+    : PhysicalOperator(PhysicalOperatorType::INSERT, std::move(types_p), estimated_cardinality), insert_table(&table),
+      insert_types(table.GetTypes()), bound_constraints(std::move(bound_constraints_p)), return_chunk(return_chunk),
+      parallel(parallel), action_type(action_type), set_expressions(std::move(set_expressions)),
+      set_columns(std::move(set_columns)), set_types(std::move(set_types)),
       on_conflict_condition(std::move(on_conflict_condition_p)), do_update_condition(std::move(do_update_condition_p)),
       conflict_target(std::move(conflict_target_p)), update_is_del_and_insert(update_is_del_and_insert) {
 
@@ -54,10 +54,10 @@ PhysicalInsert::PhysicalInsert(PhysicalPlan &physical_plan, vector<LogicalType>
 	}
 }
 
-PhysicalInsert::PhysicalInsert(PhysicalPlan &physical_plan, LogicalOperator &op, SchemaCatalogEntry &schema,
-                               unique_ptr<BoundCreateTableInfo> info_p, idx_t estimated_cardinality, bool parallel)
-    : PhysicalOperator(physical_plan, PhysicalOperatorType::CREATE_TABLE_AS, op.types, estimated_cardinality),
-      insert_table(nullptr), return_chunk(false), schema(&schema), info(std::move(info_p)), parallel(parallel),
+PhysicalInsert::PhysicalInsert(LogicalOperator &op, SchemaCatalogEntry &schema, unique_ptr<BoundCreateTableInfo> info_p,
+                               idx_t estimated_cardinality, bool parallel)
+    : PhysicalOperator(PhysicalOperatorType::CREATE_TABLE_AS, op.types, estimated_cardinality), insert_table(nullptr),
+      return_chunk(false), schema(&schema), info(std::move(info_p)), parallel(parallel),
       action_type(OnConflictAction::THROW), update_is_del_and_insert(false) {
 	GetInsertInfo(*info, insert_types);
 }
@@ -76,7 +76,6 @@ void PhysicalInsert::GetInsertInfo(const BoundCreateTableInfo &info, vector<Logi
 InsertGlobalState::InsertGlobalState(ClientContext &context, const vector<LogicalType> &return_types,
                                      DuckTableEntry &table)
     : table(table), insert_count(0), return_collection(context, return_types) {
-	table.GetStorage().BindIndexes(context);
 }
 
 InsertLocalState::InsertLocalState(ClientContext &context, const vector<LogicalType> &types,
@@ -203,22 +202,21 @@ static void CreateUpdateChunk(ExecutionContext &context, DataChunk &chunk, Table
 		do_update_filter_result.SetCardinality(chunk.size());
 		do_update_filter_result.Flatten();
 
-		SelectionVector sel(chunk.size());
-		idx_t count = 0;
+		ManagedSelection selection(chunk.size());
 
 		auto where_data = FlatVector::GetData<bool>(do_update_filter_result.data[0]);
 		for (idx_t i = 0; i < chunk.size(); i++) {
 			if (where_data[i]) {
-				sel.set_index(count, i);
-				count++;
+				selection.Append(i);
 			}
 		}
-		if (count != chunk.size()) {
-			// Filter any conflicts not meeting the condition.
-			chunk.Slice(sel, count);
-			chunk.SetCardinality(count);
-			row_ids.Slice(sel, count);
-			row_ids.Flatten(count);
+		if (selection.Count() != selection.Size()) {
+			// Not all conflicts met the condition, need to filter out the ones that don't
+			chunk.Slice(selection.Selection(), selection.Count());
+			chunk.SetCardinality(selection.Count());
+			// Also apply this Slice to the to-update row_ids
+			row_ids.Slice(selection.Selection(), selection.Count());
+			row_ids.Flatten(selection.Count());
 		}
 	}
 
@@ -359,7 +357,7 @@ static void PrepareSortKeys(DataChunk &input, unordered_map<column_t, unique_ptr
 }
 
 static map<idx_t, vector<idx_t>> CheckDistinctness(DataChunk &input, ConflictInfo &info,
-                                                   reference_set_t<Index> &matched_indexes) {
+                                                   unordered_set<BoundIndex *> &matched_indexes) {
 	map<idx_t, vector<idx_t>> conflicts;
 	unordered_map<idx_t, unique_ptr<Vector>> sort_keys;
 	//! Register which rows have already caused a conflict
@@ -368,7 +366,7 @@ static map<idx_t, vector<idx_t>> CheckDistinctness(DataChunk &input, ConflictInf
 	auto &column_ids = info.column_ids;
 	if (column_ids.empty()) {
 		for (auto index : matched_indexes) {
-			auto &index_column_ids = index.get().GetColumnIdSet();
+			auto &index_column_ids = index->GetColumnIdSet();
 			PrepareSortKeys(input, sort_keys, index_column_ids);
 			vector<reference<Vector>> columns;
 			for (auto &idx : index_column_ids) {
@@ -402,19 +400,16 @@ static void VerifyOnConflictCondition(ExecutionContext &context, DataChunk &comb
 		return;
 	}
 
-	// We need to throw.
-	// Filter any passing tuples and verify again with those violating the constraint.
-	SelectionVector sel(combined_chunk.size());
-	idx_t count = 0;
+	// We need to throw. Filter all tuples that passed, and verify again with those that violate the constraint.
+	ManagedSelection sel(combined_chunk.size());
 	auto data = FlatVector::GetData<bool>(conflict_condition_result.data[0]);
 	for (idx_t i = 0; i < combined_chunk.size(); i++) {
 		if (!data[i]) {
-			// The tuple does not meet the condition.
-			sel.set_index(count, i);
-			count++;
+			// This tuple did not meet the condition.
+			sel.Append(i);
 		}
 	}
-	combined_chunk.Slice(sel, count);
+	combined_chunk.Slice(sel.Selection(), sel.Count());
 
 	// Verify and throw.
 	if (GLOBAL) {
@@ -424,7 +419,7 @@ static void VerifyOnConflictCondition(ExecutionContext &context, DataChunk &comb
 
 	auto &indexes = local_storage.GetIndexes(context.client, data_table);
 	auto storage = local_storage.GetStorage(data_table);
-	data_table.VerifyUniqueIndexes(indexes, storage, tuples, nullptr);
+	DataTable::VerifyUniqueIndexes(indexes, storage, tuples, nullptr);
 	throw InternalException("VerifyUniqueIndexes was expected to throw but didn't");
 }
 
@@ -436,6 +431,7 @@ static idx_t HandleInsertConflicts(TableCatalogEntry &table, ExecutionContext &c
 	auto &conflict_target = op.conflict_target;
 	auto &columns_to_fetch = op.columns_to_fetch;
 	auto &data_table = table.GetStorage();
+
 	auto &local_storage = LocalStorage::Get(context.client, data_table.db);
 
 	ConflictInfo conflict_info(conflict_target);
@@ -446,53 +442,46 @@ static idx_t HandleInsertConflicts(TableCatalogEntry &table, ExecutionContext &c
 		data_table.VerifyAppendConstraints(constraint_state, context.client, tuples, storage, &conflict_manager);
 	} else {
 		auto &indexes = local_storage.GetIndexes(context.client, data_table);
-		data_table.VerifyUniqueIndexes(indexes, storage, tuples, &conflict_manager);
+		DataTable::VerifyUniqueIndexes(indexes, storage, tuples, &conflict_manager);
 	}
 
-	if (!conflict_manager.HasConflicts()) {
-		// No conflicts, i.e., no updates.
+	conflict_manager.Finalize();
+	if (conflict_manager.ConflictCount() == 0) {
+		// No conflicts found, 0 updates performed
 		return 0;
 	}
+	idx_t affected_tuples = 0;
 
-	if (GLOBAL) {
-		auto &transaction = DuckTransaction::Get(context.client, table.catalog);
-		conflict_manager.FinalizeGlobal(transaction, data_table);
-	} else {
-		conflict_manager.FinalizeLocal(data_table, local_storage);
-	}
-	auto &row_ids = conflict_manager.GetRowIds();
-	auto conflict_count = conflict_manager.ConflictCount();
+	auto &conflicts = conflict_manager.Conflicts();
+	auto &row_ids = conflict_manager.RowIds();
 
-	// Contains the original values causing the conflicts.
-	DataChunk scan_chunk;
-	// ColumnFetchState pins the fetched rows.
-	unique_ptr<ColumnFetchState> fetch_state;
+	DataChunk conflict_chunk; // contains only the conflicting values
+	DataChunk scan_chunk;     // contains the original values, that caused the conflict
+	DataChunk combined_chunk; // contains conflict_chunk + scan_chunk (wide)
+
+	// Filter out everything but the conflicting rows
+	conflict_chunk.Initialize(context.client, tuples.GetTypes());
+	conflict_chunk.Reference(tuples);
+	conflict_chunk.Slice(conflicts.Selection(), conflicts.Count());
+	conflict_chunk.SetCardinality(conflicts.Count());
 
+	// Holds the pins for the fetched rows
+	unique_ptr<ColumnFetchState> fetch_state;
 	if (!types_to_fetch.empty()) {
 		D_ASSERT(scan_chunk.size() == 0);
-		// We scan the existing table for the conflicting tuples, if we
-		// need them for the conditions, or SET expressions.
+		// When these values are required for the conditions or the SET expressions,
+		// then we scan the existing table for the conflicting tuples, using the rowids
 		scan_chunk.Initialize(context.client, types_to_fetch);
 		fetch_state = make_uniq<ColumnFetchState>();
-
 		if (GLOBAL) {
 			auto &transaction = DuckTransaction::Get(context.client, table.catalog);
-			data_table.Fetch(transaction, scan_chunk, columns_to_fetch, row_ids, conflict_count, *fetch_state);
+			data_table.Fetch(transaction, scan_chunk, columns_to_fetch, row_ids, conflicts.Count(), *fetch_state);
 		} else {
-			local_storage.FetchChunk(data_table, row_ids, conflict_count, columns_to_fetch, scan_chunk, *fetch_state);
+			local_storage.FetchChunk(data_table, row_ids, conflicts.Count(), columns_to_fetch, scan_chunk,
+			                         *fetch_state);
 		}
 	}
 
-	// Only contains the conflicting values.
-	DataChunk conflict_chunk;
-	conflict_chunk.InitializeEmpty(tuples.GetTypes());
-	conflict_chunk.Reference(tuples);
-	conflict_chunk.Slice(conflict_manager.GetInvertedSel(), conflict_count);
-	conflict_chunk.SetCardinality(conflict_count);
-
-	// Contains the conflict chunk and the scanned chunk (wide).
-	DataChunk combined_chunk;
-
 	// Splice the Input chunk and the fetched chunk together
 	CombineExistingAndInsertTuples(combined_chunk, scan_chunk, conflict_chunk, context.client, op);
 
@@ -502,18 +491,16 @@ static idx_t HandleInsertConflicts(TableCatalogEntry &table, ExecutionContext &c
 
 	if (&tuples == &lstate.update_chunk) {
 		// Allow updating duplicate rows for the 'update_chunk'
-		RegisterUpdatedRows(lstate, row_ids, conflict_count);
+		RegisterUpdatedRows(lstate, row_ids, combined_chunk.size());
 	}
-	auto affected_tuples = PerformOnConflictAction<GLOBAL>(lstate, gstate, context, combined_chunk, table, row_ids, op);
+
+	affected_tuples += PerformOnConflictAction<GLOBAL>(lstate, gstate, context, combined_chunk, table, row_ids, op);
 
 	// Remove the conflicting tuples from the insert chunk
-	// We can use only the primay data because the secondary data has the same indexes in the chunk.
 	SelectionVector sel_vec(tuples.size());
-	auto &inverted_sel = conflict_manager.GetInvertedSel();
-	auto new_size = SelectionVector::Inverted(inverted_sel, sel_vec, conflict_count, tuples.size());
+	idx_t new_size = SelectionVector::Inverted(conflicts.Selection(), sel_vec, conflicts.Count(), tuples.size());
 	tuples.Slice(sel_vec, new_size);
 	tuples.SetCardinality(new_size);
-
 	return affected_tuples;
 }
 
@@ -530,42 +517,42 @@ idx_t PhysicalInsert::OnConflictHandling(TableCatalogEntry &table, ExecutionCont
 	}
 
 	ConflictInfo conflict_info(conflict_target);
-	reference_set_t<Index> matching_indexes;
 
+	auto &global_indexes = data_table.GetDataTableInfo()->GetIndexes();
+	auto &local_indexes = local_storage.GetIndexes(context.client, data_table);
+
+	unordered_set<BoundIndex *> matched_indexes;
 	if (conflict_info.column_ids.empty()) {
-		auto &global_indexes = data_table.GetDataTableInfo()->GetIndexes();
 		// We care about every index that applies to the table if no ON CONFLICT (...) target is given
 		global_indexes.Scan([&](Index &index) {
 			if (!index.IsUnique()) {
 				return false;
 			}
-			D_ASSERT(index.IsBound());
 			if (conflict_info.ConflictTargetMatches(index)) {
-				matching_indexes.insert(index);
+				D_ASSERT(index.IsBound());
+				auto &bound_index = index.Cast<BoundIndex>();
+				matched_indexes.insert(&bound_index);
 			}
 			return false;
 		});
-		auto &local_indexes = local_storage.GetIndexes(context.client, data_table);
 		local_indexes.Scan([&](Index &index) {
 			if (!index.IsUnique()) {
 				return false;
 			}
-			D_ASSERT(index.IsBound());
 			if (conflict_info.ConflictTargetMatches(index)) {
+				D_ASSERT(index.IsBound());
 				auto &bound_index = index.Cast<BoundIndex>();
-				matching_indexes.insert(bound_index);
+				matched_indexes.insert(&bound_index);
 			}
 			return false;
 		});
 	}
 
-	auto inner_conflicts = CheckDistinctness(insert_chunk, conflict_info, matching_indexes);
+	auto inner_conflicts = CheckDistinctness(insert_chunk, conflict_info, matched_indexes);
 	idx_t count = insert_chunk.size();
 	if (!inner_conflicts.empty()) {
-		// We have at least one inner conflict to filter out.
-		SelectionVector sel(count);
-		idx_t sel_count = 0;
-
+		// We have at least one inner conflict, filter it out
+		ManagedSelection sel_vec(count);
 		ValidityMask not_a_conflict(count);
 		set<idx_t> last_occurrences_of_conflict;
 		for (idx_t i = 0; i < count; i++) {
@@ -583,8 +570,7 @@ idx_t PhysicalInsert::OnConflictHandling(TableCatalogEntry &table, ExecutionCont
 				}
 			}
 			if (not_a_conflict.RowIsValid(i)) {
-				sel.set_index(sel_count, i);
-				sel_count++;
+				sel_vec.Append(i);
 			}
 		}
 		if (action_type == OnConflictAction::UPDATE) {
@@ -593,21 +579,18 @@ idx_t PhysicalInsert::OnConflictHandling(TableCatalogEntry &table, ExecutionCont
 				throw NotImplementedException("Inner conflicts detected with a conditional DO UPDATE on-conflict "
 				                              "action, not fully implemented yet");
 			}
-
-			SelectionVector last_occurrences(last_occurrences_of_conflict.size());
-			idx_t last_occurrences_count = 0;
+			ManagedSelection last_occurrences(last_occurrences_of_conflict.size());
 			for (auto &idx : last_occurrences_of_conflict) {
-				last_occurrences.set_index(last_occurrences_count, idx);
-				last_occurrences_count++;
+				last_occurrences.Append(idx);
 			}
 
 			lstate.update_chunk.Reference(insert_chunk);
-			lstate.update_chunk.Slice(last_occurrences, last_occurrences_count);
-			lstate.update_chunk.SetCardinality(last_occurrences_count);
+			lstate.update_chunk.Slice(last_occurrences.Selection(), last_occurrences.Count());
+			lstate.update_chunk.SetCardinality(last_occurrences.Count());
 		}
 
-		insert_chunk.Slice(sel, sel_count);
-		insert_chunk.SetCardinality(sel_count);
+		insert_chunk.Slice(sel_vec.Selection(), sel_vec.Count());
+		insert_chunk.SetCardinality(sel_vec.Count());
 	}
 
 	// Check whether any conflicts arise, and if they all meet the conflict_target + condition
@@ -661,7 +644,7 @@ SinkResultType PhysicalInsert::Sink(ExecutionContext &context, DataChunk &insert
 		collection->InitializeAppend(lstate.local_append_state);
 
 		lock_guard<mutex> l(gstate.lock);
-		lstate.optimistic_writer = make_uniq<OptimisticDataWriter>(context.client, data_table);
+		lstate.optimistic_writer = make_uniq<OptimisticDataWriter>(data_table);
 		lstate.collection_index = data_table.CreateOptimisticCollection(context.client, std::move(collection));
 	}
 

package/src/duckdb/src/execution/operator/persistent/physical_update.cpp

@@ -15,14 +15,13 @@
 
 namespace duckdb {
 
-PhysicalUpdate::PhysicalUpdate(PhysicalPlan &physical_plan, vector<LogicalType> types, TableCatalogEntry &tableref,
-                               DataTable &table, vector<PhysicalIndex> columns,
-                               vector<unique_ptr<Expression>> expressions,
+PhysicalUpdate::PhysicalUpdate(vector<LogicalType> types, TableCatalogEntry &tableref, DataTable &table,
+                               vector<PhysicalIndex> columns, vector<unique_ptr<Expression>> expressions,
                                vector<unique_ptr<Expression>> bound_defaults,
                                vector<unique_ptr<BoundConstraint>> bound_constraints, idx_t estimated_cardinality,
                                bool return_chunk)
-    : PhysicalOperator(physical_plan, PhysicalOperatorType::UPDATE, std::move(types), estimated_cardinality),
-      tableref(tableref), table(table), columns(std::move(columns)), expressions(std::move(expressions)),
+    : PhysicalOperator(PhysicalOperatorType::UPDATE, std::move(types), estimated_cardinality), tableref(tableref),
+      table(table), columns(std::move(columns)), expressions(std::move(expressions)),
       bound_defaults(std::move(bound_defaults)), bound_constraints(std::move(bound_constraints)),
       return_chunk(return_chunk), index_update(false) {
 
@@ -56,7 +55,7 @@ public:
 	}
 
 	mutex lock;
-	atomic<idx_t> updated_count;
+	idx_t updated_count;
 	unordered_set<row_t> updated_rows;
 	ColumnDataCollection return_collection;
 };
@@ -128,6 +127,7 @@ SinkResultType PhysicalUpdate::Sink(ExecutionContext &context, DataChunk &chunk,
 		update_chunk.data[i].Reference(chunk.data[binding.index]);
 	}
 
+	lock_guard<mutex> glock(g_state.lock);
 	auto &row_ids = chunk.data[chunk.ColumnCount() - 1];
 	DataChunk &mock_chunk = l_state.mock_chunk;
 
@@ -143,7 +143,6 @@ SinkResultType PhysicalUpdate::Sink(ExecutionContext &context, DataChunk &chunk,
 		table.Update(update_state, context.client, row_ids, columns, update_chunk);
 
 		if (return_chunk) {
-			lock_guard<mutex> glock(g_state.lock);
 			g_state.return_collection.Append(mock_chunk);
 		}
 		g_state.updated_count += chunk.size();
@@ -159,11 +158,10 @@ SinkResultType PhysicalUpdate::Sink(ExecutionContext &context, DataChunk &chunk,
 	idx_t update_count = 0;
 	auto row_id_data = FlatVector::GetData<row_t>(row_ids);
 
-	lock_guard<mutex> glock(g_state.lock);
 	for (idx_t i = 0; i < update_chunk.size(); i++) {
 		auto row_id = row_id_data[i];
-		const auto is_new = g_state.updated_rows.insert(row_id).second;
-		if (is_new) {
+		if (g_state.updated_rows.find(row_id) == g_state.updated_rows.end()) {
+			g_state.updated_rows.insert(row_id);
 			sel.set_index(update_count++, i);
 		}
 	}
@@ -250,7 +248,7 @@ SourceResultType PhysicalUpdate::GetData(ExecutionContext &context, DataChunk &c
 	auto &g = sink_state->Cast<UpdateGlobalState>();
 	if (!return_chunk) {
 		chunk.SetCardinality(1);
-		chunk.SetValue(0, 0, Value::BIGINT(NumericCast<int64_t>(g.updated_count.load())));
+		chunk.SetValue(0, 0, Value::BIGINT(NumericCast<int64_t>(g.updated_count)));
 		return SourceResultType::FINISHED;
 	}
 

package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp

@@ -1,15 +1,11 @@
 #include "duckdb/execution/operator/projection/physical_pivot.hpp"
-
-#include "duckdb/execution/physical_plan_generator.hpp"
 #include "duckdb/planner/expression/bound_aggregate_expression.hpp"
 
 namespace duckdb {
 
-PhysicalPivot::PhysicalPivot(PhysicalPlan &physical_plan, vector<LogicalType> types_p, PhysicalOperator &child,
-                             BoundPivotInfo bound_pivot_p)
-    : PhysicalOperator(physical_plan, PhysicalOperatorType::PIVOT, std::move(types_p), child.estimated_cardinality),
+PhysicalPivot::PhysicalPivot(vector<LogicalType> types_p, PhysicalOperator &child, BoundPivotInfo bound_pivot_p)
+    : PhysicalOperator(PhysicalOperatorType::PIVOT, std::move(types_p), child.estimated_cardinality),
       bound_pivot(std::move(bound_pivot_p)) {
-
 	children.push_back(child);
 	for (idx_t p = 0; p < bound_pivot.pivot_values.size(); p++) {
 		auto entry = pivot_map.find(bound_pivot.pivot_values[p]);
@@ -19,6 +15,7 @@ PhysicalPivot::PhysicalPivot(PhysicalPlan &physical_plan, vector<LogicalType> ty
 		pivot_map[bound_pivot.pivot_values[p]] = bound_pivot.group_count + p;
 	}
 	// extract the empty aggregate expressions
+	ArenaAllocator allocator(Allocator::DefaultAllocator());
 	for (auto &aggr_expr : bound_pivot.aggregates) {
 		auto &aggr = aggr_expr->Cast<BoundAggregateExpression>();
 		// for each aggregate, initialize an empty aggregate state and finalize it immediately
@@ -26,7 +23,7 @@ PhysicalPivot::PhysicalPivot(PhysicalPlan &physical_plan, vector<LogicalType> ty
 		aggr.function.initialize(aggr.function, state.get());
 		Vector state_vector(Value::POINTER(CastPointerToValue(state.get())));
 		Vector result_vector(aggr_expr->return_type);
-		AggregateInputData aggr_input_data(aggr.bind_info.get(), physical_plan.ArenaRef());
+		AggregateInputData aggr_input_data(aggr.bind_info.get(), allocator);
 		aggr.function.finalize(state_vector, aggr_input_data, result_vector, 1, 0);
 		empty_aggregates.push_back(result_vector.GetValue(0));
 	}

package/src/duckdb/src/execution/operator/projection/physical_projection.cpp

@@ -19,9 +19,9 @@ public:
 	}
 };
 
-PhysicalProjection::PhysicalProjection(PhysicalPlan &physical_plan, vector<LogicalType> types,
-                                       vector<unique_ptr<Expression>> select_list, idx_t estimated_cardinality)
-    : PhysicalOperator(physical_plan, PhysicalOperatorType::PROJECTION, std::move(types), estimated_cardinality),
+PhysicalProjection::PhysicalProjection(vector<LogicalType> types, vector<unique_ptr<Expression>> select_list,
+                                       idx_t estimated_cardinality)
+    : PhysicalOperator(PhysicalOperatorType::PROJECTION, std::move(types), estimated_cardinality),
       select_list(std::move(select_list)) {
 }
 
@@ -36,6 +36,39 @@ unique_ptr<OperatorState> PhysicalProjection::GetOperatorState(ExecutionContext
 	return make_uniq<ProjectionState>(context, select_list);
 }
 
+unique_ptr<PhysicalOperator>
+PhysicalProjection::CreateJoinProjection(vector<LogicalType> proj_types, const vector<LogicalType> &lhs_types,
+                                         const vector<LogicalType> &rhs_types, const vector<idx_t> &left_projection_map,
+                                         const vector<idx_t> &right_projection_map, const idx_t estimated_cardinality) {
+
+	vector<unique_ptr<Expression>> proj_selects;
+	proj_selects.reserve(proj_types.size());
+
+	if (left_projection_map.empty()) {
+		for (storage_t i = 0; i < lhs_types.size(); ++i) {
+			proj_selects.emplace_back(make_uniq<BoundReferenceExpression>(lhs_types[i], i));
+		}
+	} else {
+		for (auto i : left_projection_map) {
+			proj_selects.emplace_back(make_uniq<BoundReferenceExpression>(lhs_types[i], i));
+		}
+	}
+	const auto left_cols = lhs_types.size();
+
+	if (right_projection_map.empty()) {
+		for (storage_t i = 0; i < rhs_types.size(); ++i) {
+			proj_selects.emplace_back(make_uniq<BoundReferenceExpression>(rhs_types[i], left_cols + i));
+		}
+
+	} else {
+		for (auto i : right_projection_map) {
+			proj_selects.emplace_back(make_uniq<BoundReferenceExpression>(rhs_types[i], left_cols + i));
+		}
+	}
+
+	return make_uniq<PhysicalProjection>(std::move(proj_types), std::move(proj_selects), estimated_cardinality);
+}
+
 InsertionOrderPreservingMap<string> PhysicalProjection::ParamsToString() const {
 	InsertionOrderPreservingMap<string> result;
 	string projections;

package/src/duckdb/src/execution/operator/projection/physical_tableinout_function.cpp

@@ -11,7 +11,6 @@ public:
 	idx_t row_index;
 	bool new_row;
 	DataChunk input_chunk;
-	idx_t current_ordinality_idx = 1;
 };
 
 class TableInOutGlobalState : public GlobalOperatorState {
@@ -22,11 +21,11 @@ public:
 	unique_ptr<GlobalTableFunctionState> global_state;
 };
 
-PhysicalTableInOutFunction::PhysicalTableInOutFunction(PhysicalPlan &physical_plan, vector<LogicalType> types,
-                                                       TableFunction function_p, unique_ptr<FunctionData> bind_data_p,
+PhysicalTableInOutFunction::PhysicalTableInOutFunction(vector<LogicalType> types, TableFunction function_p,
+                                                       unique_ptr<FunctionData> bind_data_p,
                                                        vector<ColumnIndex> column_ids_p, idx_t estimated_cardinality,
                                                        vector<column_t> project_input_p)
-    : PhysicalOperator(physical_plan, PhysicalOperatorType::INOUT_FUNCTION, std::move(types), estimated_cardinality),
+    : PhysicalOperator(PhysicalOperatorType::INOUT_FUNCTION, std::move(types), estimated_cardinality),
       function(std::move(function_p)), bind_data(std::move(bind_data_p)), column_ids(std::move(column_ids_p)),
       projected_input(std::move(project_input_p)) {
 }
@@ -62,15 +61,6 @@ unique_ptr<GlobalOperatorState> PhysicalTableInOutFunction::GetGlobalOperatorSta
 	return std::move(result);
 }
 
-void PhysicalTableInOutFunction::SetOrdinality(DataChunk &chunk, const optional_idx &ordinality_column_idx,
-                                               const idx_t &ordinality_idx, const idx_t &ordinality) {
-	D_ASSERT(ordinality_column_idx.IsValid());
-	if (ordinality > 0) {
-		constexpr idx_t step = 1;
-		chunk.data[ordinality_column_idx.GetIndex()].Sequence(static_cast<int64_t>(ordinality_idx), step, ordinality);
-	}
-}
-
 OperatorResultType PhysicalTableInOutFunction::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
                                                        GlobalOperatorState &gstate_p, OperatorState &state_p) const {
 	auto &gstate = gstate_p.Cast<TableInOutGlobalState>();
@@ -78,13 +68,7 @@ OperatorResultType PhysicalTableInOutFunction::Execute(ExecutionContext &context
 	TableFunctionInput data(bind_data.get(), state.local_state.get(), gstate.global_state.get());
 	if (projected_input.empty()) {
 		// straightforward case - no need to project input
-		auto result = function.in_out_function(context, data, input, chunk);
-		if (this->ordinality_idx.IsValid()) {
-			const idx_t ordinality = chunk.size();
-			SetOrdinality(chunk, this->ordinality_idx, state.current_ordinality_idx, ordinality);
-			state.current_ordinality_idx += ordinality;
-		}
-		return result;
+		return function.in_out_function(context, data, input, chunk);
 	}
 	// when project_input is set we execute the input function row-by-row
 	if (state.new_row) {
@@ -103,7 +87,6 @@ OperatorResultType PhysicalTableInOutFunction::Execute(ExecutionContext &context
 		state.input_chunk.SetCardinality(1);
 		state.row_index++;
 		state.new_row = false;
-		state.current_ordinality_idx = 1;
 	}
 	// set up the output data in "chunk"
 	D_ASSERT(chunk.ColumnCount() > projected_input.size());
@@ -115,11 +98,6 @@ OperatorResultType PhysicalTableInOutFunction::Execute(ExecutionContext &context
 		ConstantVector::Reference(chunk.data[target_idx], input.data[source_idx], state.row_index - 1, 1);
 	}
 	auto result = function.in_out_function(context, data, state.input_chunk, chunk);
-	if (this->ordinality_idx.IsValid()) {
-		const idx_t ordinality = chunk.size();
-		SetOrdinality(chunk, this->ordinality_idx, state.current_ordinality_idx, ordinality);
-		state.current_ordinality_idx += ordinality;
-	}
 	if (result == OperatorResultType::FINISHED) {
 		return result;
 	}

package/src/duckdb/src/execution/operator/projection/physical_unnest.cpp

@@ -62,11 +62,9 @@ void UnnestOperatorState::Reset() {
 	first_fetch = true;
 }
 
-PhysicalUnnest::PhysicalUnnest(PhysicalPlan &physical_plan, vector<LogicalType> types,
-                               vector<unique_ptr<Expression>> select_list, idx_t estimated_cardinality,
-                               PhysicalOperatorType type)
-    : PhysicalOperator(physical_plan, type, std::move(types), estimated_cardinality),
-      select_list(std::move(select_list)) {
+PhysicalUnnest::PhysicalUnnest(vector<LogicalType> types, vector<unique_ptr<Expression>> select_list,
+                               idx_t estimated_cardinality, PhysicalOperatorType type)
+    : PhysicalOperator(type, std::move(types), estimated_cardinality), select_list(std::move(select_list)) {
 	D_ASSERT(!this->select_list.empty());
 }