duckdb 1.3.4 → 1.4.0

package/src/duckdb/src/execution/operator/join/physical_right_delim_join.cpp

@@ -10,12 +10,13 @@
 
 namespace duckdb {
 
-PhysicalRightDelimJoin::PhysicalRightDelimJoin(PhysicalPlanGenerator &planner, vector<LogicalType> types,
-                                               PhysicalOperator &original_join, PhysicalOperator &distinct,
+PhysicalRightDelimJoin::PhysicalRightDelimJoin(PhysicalPlan &physical_plan, PhysicalPlanGenerator &planner,
+                                               vector<LogicalType> types, PhysicalOperator &original_join,
+                                               PhysicalOperator &distinct,
                                                const vector<const_reference<PhysicalOperator>> &delim_scans,
                                                idx_t estimated_cardinality, optional_idx delim_idx)
-    : PhysicalDelimJoin(PhysicalOperatorType::RIGHT_DELIM_JOIN, std::move(types), original_join, distinct, delim_scans,
-                        estimated_cardinality, delim_idx) {
+    : PhysicalDelimJoin(physical_plan, PhysicalOperatorType::RIGHT_DELIM_JOIN, std::move(types), original_join,
+                        distinct, delim_scans, estimated_cardinality, delim_idx) {
 	D_ASSERT(join.children.size() == 2);
 	// now for the original join
 	// we take its right child, this is the side that we will duplicate eliminate

package/src/duckdb/src/execution/operator/order/physical_order.cpp

@@ -1,19 +1,13 @@
 #include "duckdb/execution/operator/order/physical_order.hpp"
 
-#include "duckdb/common/sort/sort.hpp"
-#include "duckdb/execution/expression_executor.hpp"
-#include "duckdb/main/client_context.hpp"
-#include "duckdb/parallel/base_pipeline_event.hpp"
-#include "duckdb/parallel/executor_task.hpp"
-#include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/common/shared_ptr.hpp"
+#include "duckdb/common/sorting/sort.hpp"
 
 namespace duckdb {
 
-PhysicalOrder::PhysicalOrder(vector<LogicalType> types, vector<BoundOrderByNode> orders, vector<idx_t> projections,
-                             idx_t estimated_cardinality)
-    : PhysicalOperator(PhysicalOperatorType::ORDER_BY, std::move(types), estimated_cardinality),
-      orders(std::move(orders)), projections(std::move(projections)) {
+PhysicalOrder::PhysicalOrder(PhysicalPlan &physical_plan, vector<LogicalType> types, vector<BoundOrderByNode> orders,
+                             vector<idx_t> projections, idx_t estimated_cardinality, bool is_index_sort_p)
+    : PhysicalOperator(physical_plan, PhysicalOperatorType::ORDER_BY, std::move(types), estimated_cardinality),
+      orders(std::move(orders)), projections(std::move(projections)), is_index_sort(is_index_sort_p) {
 }
 
 //===--------------------------------------------------------------------===//
@@ -21,261 +15,126 @@ PhysicalOrder::PhysicalOrder(vector<LogicalType> types, vector<BoundOrderByNode>
 //===--------------------------------------------------------------------===//
 class OrderGlobalSinkState : public GlobalSinkState {
 public:
-	OrderGlobalSinkState(ClientContext &context, const PhysicalOrder &order, RowLayout &payload_layout)
-	    : order(order), global_sort_state(context, order.orders, payload_layout) {
+	OrderGlobalSinkState(const PhysicalOrder &op, ClientContext &context)
+	    : sort(context, op.orders, op.children[0].get().types, op.projections, op.is_index_sort),
+	      state(sort.GetGlobalSinkState(context)) {
 	}
 
-	const PhysicalOrder &order;
-	//! Global sort state
-	GlobalSortState global_sort_state;
-	//! Memory usage per thread
-	idx_t memory_per_thread;
+public:
+	Sort sort;
+	unique_ptr<GlobalSinkState> state;
 };
 
 class OrderLocalSinkState : public LocalSinkState {
 public:
-	OrderLocalSinkState(ClientContext &context, const PhysicalOrder &op) : key_executor(context) {
-		// Initialize order clause expression executor and DataChunk
-		vector<LogicalType> key_types;
-		for (auto &order : op.orders) {
-			key_types.push_back(order.expression->return_type);
-			key_executor.AddExpression(*order.expression);
-		}
-		auto &allocator = Allocator::Get(context);
-		keys.Initialize(allocator, key_types);
-		payload.Initialize(allocator, op.types);
+	OrderLocalSinkState() {
 	}
 
 public:
-	//! The local sort state
-	LocalSortState local_sort_state;
-	//! Key expression executor, and chunk to hold the vectors
-	ExpressionExecutor key_executor;
-	DataChunk keys;
-	//! Payload chunk to hold the vectors
-	DataChunk payload;
+	unique_ptr<LocalSinkState> state;
 };
 
-unique_ptr<GlobalSinkState> PhysicalOrder::GetGlobalSinkState(ClientContext &context) const {
-	// Get the payload layout from the return types
-	RowLayout payload_layout;
-	payload_layout.Initialize(types);
-	auto state = make_uniq<OrderGlobalSinkState>(context, *this, payload_layout);
-	// Set external (can be force with the PRAGMA)
-	state->global_sort_state.external = ClientConfig::GetConfig(context).force_external;
-	state->memory_per_thread = GetMaxThreadMemory(context);
-	return std::move(state);
+unique_ptr<LocalSinkState> PhysicalOrder::GetLocalSinkState(ExecutionContext &) const {
+	return make_uniq<OrderLocalSinkState>();
 }
 
-unique_ptr<LocalSinkState> PhysicalOrder::GetLocalSinkState(ExecutionContext &context) const {
-	return make_uniq<OrderLocalSinkState>(context.client, *this);
+unique_ptr<GlobalSinkState> PhysicalOrder::GetGlobalSinkState(ClientContext &context) const {
+	return make_uniq<OrderGlobalSinkState>(*this, context);
 }
 
 SinkResultType PhysicalOrder::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const {
 	auto &gstate = input.global_state.Cast<OrderGlobalSinkState>();
 	auto &lstate = input.local_state.Cast<OrderLocalSinkState>();
-
-	auto &global_sort_state = gstate.global_sort_state;
-	auto &local_sort_state = lstate.local_sort_state;
-
-	// Initialize local state (if necessary)
-	if (!local_sort_state.initialized) {
-		local_sort_state.Initialize(global_sort_state, BufferManager::GetBufferManager(context.client));
-	}
-
-	// Obtain sorting columns
-	auto &keys = lstate.keys;
-	keys.Reset();
-	lstate.key_executor.Execute(chunk, keys);
-
-	auto &payload = lstate.payload;
-	payload.ReferenceColumns(chunk, projections);
-
-	// Sink the data into the local sort state
-	keys.Verify();
-	chunk.Verify();
-	local_sort_state.SinkChunk(keys, payload);
-
-	// When sorting data reaches a certain size, we sort it
-	if (local_sort_state.SizeInBytes() >= gstate.memory_per_thread) {
-		local_sort_state.Sort(global_sort_state, true);
+	if (!lstate.state) {
+		lstate.state = gstate.sort.GetLocalSinkState(context);
 	}
-	return SinkResultType::NEED_MORE_INPUT;
+	OperatorSinkInput sort_input {*gstate.state, *lstate.state, input.interrupt_state};
+	return gstate.sort.Sink(context, chunk, sort_input);
 }
 
 SinkCombineResultType PhysicalOrder::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
 	auto &gstate = input.global_state.Cast<OrderGlobalSinkState>();
 	auto &lstate = input.local_state.Cast<OrderLocalSinkState>();
-	gstate.global_sort_state.AddLocalState(lstate.local_sort_state);
-
-	return SinkCombineResultType::FINISHED;
-}
-
-class PhysicalOrderMergeTask : public ExecutorTask {
-public:
-	PhysicalOrderMergeTask(shared_ptr<Event> event_p, ClientContext &context, OrderGlobalSinkState &state,
-	                       const PhysicalOperator &op_p)
-	    : ExecutorTask(context, std::move(event_p), op_p), context(context), state(state) {
-	}
-
-	TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override {
-		// Initialize merge sorted and iterate until done
-		auto &global_sort_state = state.global_sort_state;
-		MergeSorter merge_sorter(global_sort_state, BufferManager::GetBufferManager(context));
-		merge_sorter.PerformInMergeRound();
-		event->FinishTask();
-		return TaskExecutionResult::TASK_FINISHED;
-	}
-
-	string TaskType() const override {
-		return "PhysicalOrderMergeTask";
-	}
-
-private:
-	ClientContext &context;
-	OrderGlobalSinkState &state;
-};
-
-class OrderMergeEvent : public BasePipelineEvent {
-public:
-	OrderMergeEvent(OrderGlobalSinkState &gstate_p, Pipeline &pipeline_p, const PhysicalOperator &op_p)
-	    : BasePipelineEvent(pipeline_p), gstate(gstate_p), op(op_p) {
-	}
-
-	OrderGlobalSinkState &gstate;
-	const PhysicalOperator &op;
-
-public:
-	void Schedule() override {
-		auto &context = pipeline->GetClientContext();
-
-		// Schedule tasks equal to the number of threads, which will each merge multiple partitions
-		auto &ts = TaskScheduler::GetScheduler(context);
-		auto num_threads = NumericCast<idx_t>(ts.NumberOfThreads());
-
-		vector<shared_ptr<Task>> merge_tasks;
-		for (idx_t tnum = 0; tnum < num_threads; tnum++) {
-			merge_tasks.push_back(make_uniq<PhysicalOrderMergeTask>(shared_from_this(), context, gstate, op));
-		}
-		SetTasks(std::move(merge_tasks));
+	if (!lstate.state) {
+		return SinkCombineResultType::FINISHED;
 	}
-
-	void FinishEvent() override {
-		auto &global_sort_state = gstate.global_sort_state;
-
-		global_sort_state.CompleteMergeRound();
-		if (global_sort_state.sorted_blocks.size() > 1) {
-			// Multiple blocks remaining: Schedule the next round
-			PhysicalOrder::ScheduleMergeTasks(*pipeline, *this, gstate);
-		}
-	}
-};
+	OperatorSinkCombineInput sort_input {*gstate.state, *lstate.state, input.interrupt_state};
+	return gstate.sort.Combine(context, sort_input);
+}
 
 SinkFinalizeType PhysicalOrder::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
                                          OperatorSinkFinalizeInput &input) const {
-	auto &state = input.global_state.Cast<OrderGlobalSinkState>();
-	auto &global_sort_state = state.global_sort_state;
-
-	if (global_sort_state.sorted_blocks.empty()) {
-		// Empty input!
-		return SinkFinalizeType::NO_OUTPUT_POSSIBLE;
-	}
-
-	// Prepare for merge sort phase
-	global_sort_state.PrepareMergePhase();
-
-	// Start the merge phase or finish if a merge is not necessary
-	if (global_sort_state.sorted_blocks.size() > 1) {
-		PhysicalOrder::ScheduleMergeTasks(pipeline, event, state);
-	}
-	return SinkFinalizeType::READY;
+	auto &gstate = input.global_state.Cast<OrderGlobalSinkState>();
+	OperatorSinkFinalizeInput sort_input {*gstate.state, input.interrupt_state};
+	return gstate.sort.Finalize(context, sort_input);
 }
 
-void PhysicalOrder::ScheduleMergeTasks(Pipeline &pipeline, Event &event, OrderGlobalSinkState &state) {
-	// Initialize global sort state for a round of merging
-	state.global_sort_state.InitializeMergeRound();
-	auto new_event = make_shared_ptr<OrderMergeEvent>(state, pipeline, state.order);
-	event.InsertEvent(std::move(new_event));
+ProgressData PhysicalOrder::GetSinkProgress(ClientContext &context, GlobalSinkState &gstate_p,
+                                            const ProgressData source_progress) const {
+	auto &gstate = gstate_p.Cast<OrderGlobalSinkState>();
+	return gstate.sort.GetSinkProgress(context, *gstate.state, source_progress);
 }
 
 //===--------------------------------------------------------------------===//
 // Source
 //===--------------------------------------------------------------------===//
-class PhysicalOrderGlobalSourceState : public GlobalSourceState {
+class OrderGlobalSourceState : public GlobalSourceState {
 public:
-	explicit PhysicalOrderGlobalSourceState(OrderGlobalSinkState &sink) : next_batch_index(0) {
-		auto &global_sort_state = sink.global_sort_state;
-		if (global_sort_state.sorted_blocks.empty()) {
-			total_batches = 0;
-		} else {
-			D_ASSERT(global_sort_state.sorted_blocks.size() == 1);
-			total_batches = global_sort_state.sorted_blocks[0]->payload_data->data_blocks.size();
-		}
+	explicit OrderGlobalSourceState(ClientContext &context, OrderGlobalSinkState &sink)
+	    : sort(sink.sort), state(sort.GetGlobalSourceState(context, *sink.state)) {
 	}
 
+public:
 	idx_t MaxThreads() override {
-		return total_batches;
+		return state->MaxThreads();
 	}
 
 public:
-	atomic<idx_t> next_batch_index;
-	idx_t total_batches;
+	Sort &sort;
+	unique_ptr<GlobalSourceState> state;
 };
 
-unique_ptr<GlobalSourceState> PhysicalOrder::GetGlobalSourceState(ClientContext &context) const {
-	auto &sink = this->sink_state->Cast<OrderGlobalSinkState>();
-	return make_uniq<PhysicalOrderGlobalSourceState>(sink);
-}
-
-class PhysicalOrderLocalSourceState : public LocalSourceState {
+class OrderLocalSourceState : public LocalSourceState {
 public:
-	explicit PhysicalOrderLocalSourceState(PhysicalOrderGlobalSourceState &gstate)
-	    : batch_index(gstate.next_batch_index++) {
+	explicit OrderLocalSourceState(ExecutionContext &context, OrderGlobalSourceState &gstate)
+	    : state(gstate.sort.GetLocalSourceState(context, *gstate.state)) {
 	}
 
 public:
-	idx_t batch_index;
-	unique_ptr<PayloadScanner> scanner;
+	unique_ptr<LocalSourceState> state;
 };
 
 unique_ptr<LocalSourceState> PhysicalOrder::GetLocalSourceState(ExecutionContext &context,
                                                                 GlobalSourceState &gstate_p) const {
-	auto &gstate = gstate_p.Cast<PhysicalOrderGlobalSourceState>();
-	return make_uniq<PhysicalOrderLocalSourceState>(gstate);
+	auto &gstate = gstate_p.Cast<OrderGlobalSourceState>();
+	return make_uniq<OrderLocalSourceState>(context, gstate);
 }
 
-SourceResultType PhysicalOrder::GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const {
-	auto &gstate = input.global_state.Cast<PhysicalOrderGlobalSourceState>();
-	auto &lstate = input.local_state.Cast<PhysicalOrderLocalSourceState>();
-
-	if (lstate.scanner && lstate.scanner->Remaining() == 0) {
-		lstate.batch_index = gstate.next_batch_index++;
-		lstate.scanner = nullptr;
-	}
-
-	if (lstate.batch_index >= gstate.total_batches) {
-		return SourceResultType::FINISHED;
-	}
-
-	if (!lstate.scanner) {
-		auto &sink = this->sink_state->Cast<OrderGlobalSinkState>();
-		auto &global_sort_state = sink.global_sort_state;
-		lstate.scanner = make_uniq<PayloadScanner>(global_sort_state, lstate.batch_index, true);
-	}
-
-	lstate.scanner->Scan(chunk);
+unique_ptr<GlobalSourceState> PhysicalOrder::GetGlobalSourceState(ClientContext &context) const {
+	return make_uniq<OrderGlobalSourceState>(context, sink_state->Cast<OrderGlobalSinkState>());
+}
 
-	return chunk.size() == 0 ? SourceResultType::FINISHED : SourceResultType::HAVE_MORE_OUTPUT;
+SourceResultType PhysicalOrder::GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const {
+	auto &gstate = input.global_state.Cast<OrderGlobalSourceState>();
+	auto &lstate = input.local_state.Cast<OrderLocalSourceState>();
+	OperatorSourceInput sort_input {*gstate.state, *lstate.state, input.interrupt_state};
+	return gstate.sort.GetData(context, chunk, sort_input);
 }
 
 OperatorPartitionData PhysicalOrder::GetPartitionData(ExecutionContext &context, DataChunk &chunk,
                                                       GlobalSourceState &gstate_p, LocalSourceState &lstate_p,
                                                       const OperatorPartitionInfo &partition_info) const {
+	auto &gstate = gstate_p.Cast<OrderGlobalSourceState>();
+	auto &lstate = lstate_p.Cast<OrderLocalSourceState>();
 	if (partition_info.RequiresPartitionColumns()) {
 		throw InternalException("PhysicalOrder::GetPartitionData: partition columns not supported");
 	}
-	auto &lstate = lstate_p.Cast<PhysicalOrderLocalSourceState>();
-	return OperatorPartitionData(lstate.batch_index);
+	return gstate.sort.GetPartitionData(context, chunk, *gstate.state, *lstate.state, partition_info);
+}
+
+ProgressData PhysicalOrder::GetProgress(ClientContext &context, GlobalSourceState &gstate_p) const {
+	auto &gstate = gstate_p.Cast<OrderGlobalSourceState>();
+	return gstate.sort.GetProgress(context, *gstate.state);
 }
 
 InsertionOrderPreservingMap<string> PhysicalOrder::ParamsToString() const {

package/src/duckdb/src/execution/operator/order/physical_top_n.cpp

@@ -8,10 +8,11 @@
 
 namespace duckdb {
 
-PhysicalTopN::PhysicalTopN(vector<LogicalType> types, vector<BoundOrderByNode> orders, idx_t limit, idx_t offset,
-                           shared_ptr<DynamicFilterData> dynamic_filter_p, idx_t estimated_cardinality)
-    : PhysicalOperator(PhysicalOperatorType::TOP_N, std::move(types), estimated_cardinality), orders(std::move(orders)),
-      limit(limit), offset(offset), dynamic_filter(std::move(dynamic_filter_p)) {
+PhysicalTopN::PhysicalTopN(PhysicalPlan &physical_plan, vector<LogicalType> types, vector<BoundOrderByNode> orders,
+                           idx_t limit, idx_t offset, shared_ptr<DynamicFilterData> dynamic_filter_p,
+                           idx_t estimated_cardinality)
+    : PhysicalOperator(physical_plan, PhysicalOperatorType::TOP_N, std::move(types), estimated_cardinality),
+      orders(std::move(orders)), limit(limit), offset(offset), dynamic_filter(std::move(dynamic_filter_p)) {
 }
 
 PhysicalTopN::~PhysicalTopN() {

package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp

@@ -26,9 +26,11 @@ struct ActiveFlushGuard {
 	atomic<bool> &bool_value;
 };
 
-PhysicalBatchCopyToFile::PhysicalBatchCopyToFile(vector<LogicalType> types, CopyFunction function_p,
-                                                 unique_ptr<FunctionData> bind_data_p, idx_t estimated_cardinality)
-    : PhysicalOperator(PhysicalOperatorType::BATCH_COPY_TO_FILE, std::move(types), estimated_cardinality),
+PhysicalBatchCopyToFile::PhysicalBatchCopyToFile(PhysicalPlan &physical_plan, vector<LogicalType> types,
+                                                 CopyFunction function_p, unique_ptr<FunctionData> bind_data_p,
+                                                 idx_t estimated_cardinality)
+    : PhysicalOperator(physical_plan, PhysicalOperatorType::BATCH_COPY_TO_FILE, std::move(types),
+                       estimated_cardinality),
       function(std::move(function_p)), bind_data(std::move(bind_data_p)) {
 	if (!function.flush_batch || !function.prepare_batch) {
 		throw InternalException("PhysicalFixedBatchCopy created for copy function that does not have "

package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp

@@ -14,16 +14,17 @@
 
 namespace duckdb {
 
-PhysicalBatchInsert::PhysicalBatchInsert(vector<LogicalType> types_p, TableCatalogEntry &table,
+PhysicalBatchInsert::PhysicalBatchInsert(PhysicalPlan &physical_plan, vector<LogicalType> types_p,
+                                         TableCatalogEntry &table,
                                          vector<unique_ptr<BoundConstraint>> bound_constraints_p,
                                          idx_t estimated_cardinality)
-    : PhysicalOperator(PhysicalOperatorType::BATCH_INSERT, std::move(types_p), estimated_cardinality),
+    : PhysicalOperator(physical_plan, PhysicalOperatorType::BATCH_INSERT, std::move(types_p), estimated_cardinality),
       insert_table(&table), insert_types(table.GetTypes()), bound_constraints(std::move(bound_constraints_p)) {
 }
 
-PhysicalBatchInsert::PhysicalBatchInsert(LogicalOperator &op, SchemaCatalogEntry &schema,
+PhysicalBatchInsert::PhysicalBatchInsert(PhysicalPlan &physical_plan, LogicalOperator &op, SchemaCatalogEntry &schema,
                                          unique_ptr<BoundCreateTableInfo> info_p, idx_t estimated_cardinality)
-    : PhysicalOperator(PhysicalOperatorType::BATCH_CREATE_TABLE_AS, op.types, estimated_cardinality),
+    : PhysicalOperator(physical_plan, PhysicalOperatorType::BATCH_CREATE_TABLE_AS, op.types, estimated_cardinality),
       insert_table(nullptr), schema(&schema), info(std::move(info_p)) {
 	PhysicalInsert::GetInsertInfo(*info, insert_types);
 }
@@ -149,6 +150,7 @@ public:
 	    : memory_manager(context, minimum_memory_per_thread), table(table), insert_count(0),
 	      optimistically_written(false), minimum_memory_per_thread(minimum_memory_per_thread) {
 		row_group_size = table.GetStorage().GetRowGroupSize();
+		table.GetStorage().BindIndexes(context);
 	}
 
 	BatchMemoryManager memory_manager;
@@ -224,7 +226,7 @@ public:
 
 		// Merge the collections.
 		if (!l_state.optimistic_writer) {
-			l_state.optimistic_writer = make_uniq<OptimisticDataWriter>(g_state.table.GetStorage());
+			l_state.optimistic_writer = make_uniq<OptimisticDataWriter>(context, g_state.table.GetStorage());
 		}
 		auto result_collection_index = g_state.MergeCollections(context, merge_collections, *l_state.optimistic_writer);
 		merge_collections.clear();
@@ -525,7 +527,7 @@ SinkResultType PhysicalBatchInsert::Sink(ExecutionContext &context, DataChunk &i
 		// no collection yet: create a new one
 		lstate.CreateNewCollection(context.client, table, insert_types);
 		if (!lstate.optimistic_writer) {
-			lstate.optimistic_writer = make_uniq<OptimisticDataWriter>(table.GetStorage());
+			lstate.optimistic_writer = make_uniq<OptimisticDataWriter>(context.client, table.GetStorage());
 		}
 	}
 
@@ -635,7 +637,7 @@ SinkFinalizeType PhysicalBatchInsert::Finalize(Pipeline &pipeline, Event &event,
 		// now that we have created all of the mergers, perform the actual merging
 		vector<PhysicalIndex> final_collections;
 		final_collections.reserve(mergers.size());
-		auto writer = make_uniq<OptimisticDataWriter>(data_table);
+		auto writer = make_uniq<OptimisticDataWriter>(context, data_table);
 		for (auto &merger : mergers) {
 			final_collections.push_back(merger->Flush(*writer));
 		}

package/src/duckdb/src/execution/operator/persistent/physical_copy_database.cpp

@@ -16,9 +16,9 @@
 
 namespace duckdb {
 
-PhysicalCopyDatabase::PhysicalCopyDatabase(vector<LogicalType> types, idx_t estimated_cardinality,
-                                           unique_ptr<CopyDatabaseInfo> info_p)
-    : PhysicalOperator(PhysicalOperatorType::COPY_DATABASE, std::move(types), estimated_cardinality),
+PhysicalCopyDatabase::PhysicalCopyDatabase(PhysicalPlan &physical_plan, vector<LogicalType> types,
+                                           idx_t estimated_cardinality, unique_ptr<CopyDatabaseInfo> info_p)
+    : PhysicalOperator(physical_plan, PhysicalOperatorType::COPY_DATABASE, std::move(types), estimated_cardinality),
       info(std::move(info_p)) {
 }
 
@@ -81,10 +81,12 @@ SourceResultType PhysicalCopyDatabase::GetData(ExecutionContext &context, DataCh
 		storage_info.options.emplace("v1_0_0_storage", false);
 		auto unbound_index = make_uniq<UnboundIndex>(create_index_info.Copy(), storage_info,
 		                                             data_table.GetTableIOManager(), catalog.GetAttached());
-
 		data_table.AddIndex(std::move(unbound_index));
+
+		// We add unbound indexes, so we immediately bind them.
+		// Otherwise, WAL serialization fails due to unbound indexes.
 		auto &data_table_info = *data_table.GetDataTableInfo();
-		data_table_info.GetIndexes().InitializeIndexes(context.client, data_table_info);
+		data_table_info.BindIndexes(context.client);
 	}
 
 	return SourceResultType::FINISHED;

package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp

@@ -8,6 +8,7 @@
 #include "duckdb/common/value_operations/value_operations.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/planner/operator/logical_copy_to_file.hpp"
+#include "duckdb/main/settings.hpp"
 
 #include <algorithm>
 
@@ -50,7 +51,7 @@ public:
 	explicit CopyToFunctionGlobalState(ClientContext &context)
 	    : initialized(false), rows_copied(0), last_file_offset(0),
 	      file_write_lock_if_rotating(make_uniq<StorageLock>()) {
-		max_open_files = ClientConfig::GetConfig(context).partitioned_write_max_open_files;
+		max_open_files = DBConfig::GetSetting<PartitionedWriteMaxOpenFilesSetting>(context);
 	}
 
 	StorageLock lock;
@@ -252,11 +253,14 @@ string PhysicalCopyToFile::GetTrimmedPath(ClientContext &context) const {
 
 class CopyToFunctionLocalState : public LocalSinkState {
 public:
-	explicit CopyToFunctionLocalState(unique_ptr<LocalFunctionData> local_state) : local_state(std::move(local_state)) {
+	explicit CopyToFunctionLocalState(ClientContext &context, unique_ptr<LocalFunctionData> local_state)
+	    : local_state(std::move(local_state)) {
+		partitioned_write_flush_threshold = DBConfig::GetSetting<PartitionedWriteFlushThresholdSetting>(context);
 	}
 	unique_ptr<GlobalFunctionData> global_state;
 	unique_ptr<LocalFunctionData> local_state;
 	idx_t total_rows_copied = 0;
+	idx_t partitioned_write_flush_threshold;
 
 	//! Buffers the tuples in partitions before writing
 	unique_ptr<HivePartitionedColumnData> part_buffer;
@@ -281,7 +285,7 @@ public:
 		}
 		part_buffer->Append(*part_buffer_append_state, chunk);
 		append_count += chunk.size();
-		if (append_count >= ClientConfig::GetConfig(context.client).partitioned_write_flush_threshold) {
+		if (append_count >= partitioned_write_flush_threshold) {
 			// flush all cached partitions
 			FlushPartitions(context, op, g);
 		}
@@ -370,11 +374,12 @@ unique_ptr<LocalSinkState> PhysicalCopyToFile::GetLocalSinkState(ExecutionContex
 	if (partition_output) {
 		auto &g = sink_state->Cast<CopyToFunctionGlobalState>();
 
-		auto state = make_uniq<CopyToFunctionLocalState>(nullptr);
+		auto state = make_uniq<CopyToFunctionLocalState>(context.client, nullptr);
 		state->InitializeAppendState(context.client, *this, g);
 		return std::move(state);
 	}
-	auto res = make_uniq<CopyToFunctionLocalState>(function.copy_to_initialize_local(context, *bind_data));
+	auto res =
+	    make_uniq<CopyToFunctionLocalState>(context.client, function.copy_to_initialize_local(context, *bind_data));
 	return std::move(res);
 }
 
@@ -483,9 +488,9 @@ string PhysicalCopyToFile::GetNonTmpFile(ClientContext &context, const string &t
 	return fs.JoinPath(path, base);
 }
 
-PhysicalCopyToFile::PhysicalCopyToFile(vector<LogicalType> types, CopyFunction function_p,
+PhysicalCopyToFile::PhysicalCopyToFile(PhysicalPlan &physical_plan, vector<LogicalType> types, CopyFunction function_p,
                                        unique_ptr<FunctionData> bind_data, idx_t estimated_cardinality)
-    : PhysicalOperator(PhysicalOperatorType::COPY_TO_FILE, std::move(types), estimated_cardinality),
+    : PhysicalOperator(physical_plan, PhysicalOperatorType::COPY_TO_FILE, std::move(types), estimated_cardinality),
       function(std::move(function_p)), bind_data(std::move(bind_data)), parallel(false) {
 }
 

package/src/duckdb/src/execution/operator/persistent/physical_delete.cpp

@@ -12,10 +12,10 @@
 
 namespace duckdb {
 
-PhysicalDelete::PhysicalDelete(vector<LogicalType> types, TableCatalogEntry &tableref, DataTable &table,
-                               vector<unique_ptr<BoundConstraint>> bound_constraints, idx_t row_id_index,
-                               idx_t estimated_cardinality, bool return_chunk)
-    : PhysicalOperator(PhysicalOperatorType::DELETE_OPERATOR, std::move(types), estimated_cardinality),
+PhysicalDelete::PhysicalDelete(PhysicalPlan &physical_plan, vector<LogicalType> types, TableCatalogEntry &tableref,
+                               DataTable &table, vector<unique_ptr<BoundConstraint>> bound_constraints,
+                               idx_t row_id_index, idx_t estimated_cardinality, bool return_chunk)
+    : PhysicalOperator(physical_plan, PhysicalOperatorType::DELETE_OPERATOR, std::move(types), estimated_cardinality),
       tableref(tableref), table(table), bound_constraints(std::move(bound_constraints)), row_id_index(row_id_index),
       return_chunk(return_chunk) {
 }
@@ -39,6 +39,7 @@ public:
 	mutex delete_lock;
 	idx_t deleted_count;
 	ColumnDataCollection return_collection;
+	unordered_set<row_t> deleted_row_ids;
 	LocalAppendState delete_index_append_state;
 	bool has_unique_indexes;
 };
@@ -147,12 +148,40 @@ SinkResultType PhysicalDelete::Sink(ExecutionContext &context, DataChunk &chunk,
 		});
 	}
 
+	auto deleted_count = table.Delete(*l_state.delete_state, context.client, row_ids, chunk.size());
+	g_state.deleted_count += deleted_count;
+
 	// Append the return_chunk to the return collection.
 	if (return_chunk) {
+		// Rows can be duplicated, so we get the chunk indexes for new row id values.
+		map<row_t, idx_t> new_row_ids_deleted;
+		auto flat_ids = FlatVector::GetData<row_t>(row_ids);
+		for (idx_t i = 0; i < chunk.size(); i++) {
+			// If the row has not been deleted previously
+			// and is not a duplicate within the current chunk,
+			// then we add it to new_row_ids_deleted.
+			auto row_id = flat_ids[i];
+			auto already_deleted = g_state.deleted_row_ids.find(row_id) != g_state.deleted_row_ids.end();
+			auto newly_deleted = new_row_ids_deleted.find(row_id) != new_row_ids_deleted.end();
+			if (!already_deleted && !newly_deleted) {
+				new_row_ids_deleted[row_id] = i;
+				g_state.deleted_row_ids.insert(row_id);
+			}
+		}
+
+		D_ASSERT(new_row_ids_deleted.size() == deleted_count);
+		if (deleted_count < l_state.delete_chunk.size()) {
+			SelectionVector delete_sel(0, deleted_count);
+			idx_t chunk_index = 0;
+			for (auto &row_id_to_chunk_index : new_row_ids_deleted) {
+				delete_sel.set_index(chunk_index, row_id_to_chunk_index.second);
+				chunk_index++;
+			}
+			l_state.delete_chunk.Slice(delete_sel, deleted_count);
+		}
 		g_state.return_collection.Append(l_state.delete_chunk);
 	}
 
-	g_state.deleted_count += table.Delete(*l_state.delete_state, context.client, row_ids, chunk.size());
 	return SinkResultType::NEED_MORE_INPUT;
 }