duckdb 1.3.3-dev4.0 → 1.3.3

package/src/duckdb/src/execution/operator/join/physical_right_delim_join.cpp

@@ -10,13 +10,12 @@
 
 namespace duckdb {
 
-PhysicalRightDelimJoin::PhysicalRightDelimJoin(PhysicalPlan &physical_plan, PhysicalPlanGenerator &planner,
-                                               vector<LogicalType> types, PhysicalOperator &original_join,
-                                               PhysicalOperator &distinct,
+PhysicalRightDelimJoin::PhysicalRightDelimJoin(PhysicalPlanGenerator &planner, vector<LogicalType> types,
+                                               PhysicalOperator &original_join, PhysicalOperator &distinct,
                                                const vector<const_reference<PhysicalOperator>> &delim_scans,
                                                idx_t estimated_cardinality, optional_idx delim_idx)
-    : PhysicalDelimJoin(physical_plan, PhysicalOperatorType::RIGHT_DELIM_JOIN, std::move(types), original_join,
-                        distinct, delim_scans, estimated_cardinality, delim_idx) {
+    : PhysicalDelimJoin(PhysicalOperatorType::RIGHT_DELIM_JOIN, std::move(types), original_join, distinct, delim_scans,
+                        estimated_cardinality, delim_idx) {
 	D_ASSERT(join.children.size() == 2);
 	// now for the original join
 	// we take its right child, this is the side that we will duplicate eliminate

package/src/duckdb/src/execution/operator/order/physical_order.cpp

@@ -1,13 +1,19 @@
 #include "duckdb/execution/operator/order/physical_order.hpp"
 
-#include "duckdb/common/sorting/sort.hpp"
+#include "duckdb/common/sort/sort.hpp"
+#include "duckdb/execution/expression_executor.hpp"
+#include "duckdb/main/client_context.hpp"
+#include "duckdb/parallel/base_pipeline_event.hpp"
+#include "duckdb/parallel/executor_task.hpp"
+#include "duckdb/storage/buffer_manager.hpp"
+#include "duckdb/common/shared_ptr.hpp"
 
 namespace duckdb {
 
-PhysicalOrder::PhysicalOrder(PhysicalPlan &physical_plan, vector<LogicalType> types, vector<BoundOrderByNode> orders,
-                             vector<idx_t> projections, idx_t estimated_cardinality, bool is_index_sort_p)
-    : PhysicalOperator(physical_plan, PhysicalOperatorType::ORDER_BY, std::move(types), estimated_cardinality),
-      orders(std::move(orders)), projections(std::move(projections)), is_index_sort(is_index_sort_p) {
+PhysicalOrder::PhysicalOrder(vector<LogicalType> types, vector<BoundOrderByNode> orders, vector<idx_t> projections,
+                             idx_t estimated_cardinality)
+    : PhysicalOperator(PhysicalOperatorType::ORDER_BY, std::move(types), estimated_cardinality),
+      orders(std::move(orders)), projections(std::move(projections)) {
 }
 
 //===--------------------------------------------------------------------===//
@@ -15,126 +21,261 @@ PhysicalOrder::PhysicalOrder(PhysicalPlan &physical_plan, vector<LogicalType> ty
 //===--------------------------------------------------------------------===//
 class OrderGlobalSinkState : public GlobalSinkState {
 public:
-	OrderGlobalSinkState(const PhysicalOrder &op, ClientContext &context)
-	    : sort(context, op.orders, op.children[0].get().types, op.projections, op.is_index_sort),
-	      state(sort.GetGlobalSinkState(context)) {
+	OrderGlobalSinkState(ClientContext &context, const PhysicalOrder &order, RowLayout &payload_layout)
+	    : order(order), global_sort_state(context, order.orders, payload_layout) {
 	}
 
-public:
-	Sort sort;
-	unique_ptr<GlobalSinkState> state;
+	const PhysicalOrder &order;
+	//! Global sort state
+	GlobalSortState global_sort_state;
+	//! Memory usage per thread
+	idx_t memory_per_thread;
 };
 
 class OrderLocalSinkState : public LocalSinkState {
 public:
-	OrderLocalSinkState() {
+	OrderLocalSinkState(ClientContext &context, const PhysicalOrder &op) : key_executor(context) {
+		// Initialize order clause expression executor and DataChunk
+		vector<LogicalType> key_types;
+		for (auto &order : op.orders) {
+			key_types.push_back(order.expression->return_type);
+			key_executor.AddExpression(*order.expression);
+		}
+		auto &allocator = Allocator::Get(context);
+		keys.Initialize(allocator, key_types);
+		payload.Initialize(allocator, op.types);
 	}
 
 public:
-	unique_ptr<LocalSinkState> state;
+	//! The local sort state
+	LocalSortState local_sort_state;
+	//! Key expression executor, and chunk to hold the vectors
+	ExpressionExecutor key_executor;
+	DataChunk keys;
+	//! Payload chunk to hold the vectors
+	DataChunk payload;
 };
 
-unique_ptr<LocalSinkState> PhysicalOrder::GetLocalSinkState(ExecutionContext &) const {
-	return make_uniq<OrderLocalSinkState>();
+unique_ptr<GlobalSinkState> PhysicalOrder::GetGlobalSinkState(ClientContext &context) const {
+	// Get the payload layout from the return types
+	RowLayout payload_layout;
+	payload_layout.Initialize(types);
+	auto state = make_uniq<OrderGlobalSinkState>(context, *this, payload_layout);
+	// Set external (can be force with the PRAGMA)
+	state->global_sort_state.external = ClientConfig::GetConfig(context).force_external;
+	state->memory_per_thread = GetMaxThreadMemory(context);
+	return std::move(state);
 }
 
-unique_ptr<GlobalSinkState> PhysicalOrder::GetGlobalSinkState(ClientContext &context) const {
-	return make_uniq<OrderGlobalSinkState>(*this, context);
+unique_ptr<LocalSinkState> PhysicalOrder::GetLocalSinkState(ExecutionContext &context) const {
+	return make_uniq<OrderLocalSinkState>(context.client, *this);
 }
 
 SinkResultType PhysicalOrder::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const {
 	auto &gstate = input.global_state.Cast<OrderGlobalSinkState>();
 	auto &lstate = input.local_state.Cast<OrderLocalSinkState>();
-	if (!lstate.state) {
-		lstate.state = gstate.sort.GetLocalSinkState(context);
+
+	auto &global_sort_state = gstate.global_sort_state;
+	auto &local_sort_state = lstate.local_sort_state;
+
+	// Initialize local state (if necessary)
+	if (!local_sort_state.initialized) {
+		local_sort_state.Initialize(global_sort_state, BufferManager::GetBufferManager(context.client));
+	}
+
+	// Obtain sorting columns
+	auto &keys = lstate.keys;
+	keys.Reset();
+	lstate.key_executor.Execute(chunk, keys);
+
+	auto &payload = lstate.payload;
+	payload.ReferenceColumns(chunk, projections);
+
+	// Sink the data into the local sort state
+	keys.Verify();
+	chunk.Verify();
+	local_sort_state.SinkChunk(keys, payload);
+
+	// When sorting data reaches a certain size, we sort it
+	if (local_sort_state.SizeInBytes() >= gstate.memory_per_thread) {
+		local_sort_state.Sort(global_sort_state, true);
 	}
-	OperatorSinkInput sort_input {*gstate.state, *lstate.state, input.interrupt_state};
-	return gstate.sort.Sink(context, chunk, sort_input);
+	return SinkResultType::NEED_MORE_INPUT;
 }
 
 SinkCombineResultType PhysicalOrder::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
 	auto &gstate = input.global_state.Cast<OrderGlobalSinkState>();
 	auto &lstate = input.local_state.Cast<OrderLocalSinkState>();
-	if (!lstate.state) {
-		return SinkCombineResultType::FINISHED;
-	}
-	OperatorSinkCombineInput sort_input {*gstate.state, *lstate.state, input.interrupt_state};
-	return gstate.sort.Combine(context, sort_input);
+	gstate.global_sort_state.AddLocalState(lstate.local_sort_state);
+
+	return SinkCombineResultType::FINISHED;
 }
 
+class PhysicalOrderMergeTask : public ExecutorTask {
+public:
+	PhysicalOrderMergeTask(shared_ptr<Event> event_p, ClientContext &context, OrderGlobalSinkState &state,
+	                       const PhysicalOperator &op_p)
+	    : ExecutorTask(context, std::move(event_p), op_p), context(context), state(state) {
+	}
+
+	TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override {
+		// Initialize merge sorted and iterate until done
+		auto &global_sort_state = state.global_sort_state;
+		MergeSorter merge_sorter(global_sort_state, BufferManager::GetBufferManager(context));
+		merge_sorter.PerformInMergeRound();
+		event->FinishTask();
+		return TaskExecutionResult::TASK_FINISHED;
+	}
+
+	string TaskType() const override {
+		return "PhysicalOrderMergeTask";
+	}
+
+private:
+	ClientContext &context;
+	OrderGlobalSinkState &state;
+};
+
+class OrderMergeEvent : public BasePipelineEvent {
+public:
+	OrderMergeEvent(OrderGlobalSinkState &gstate_p, Pipeline &pipeline_p, const PhysicalOperator &op_p)
+	    : BasePipelineEvent(pipeline_p), gstate(gstate_p), op(op_p) {
+	}
+
+	OrderGlobalSinkState &gstate;
+	const PhysicalOperator &op;
+
+public:
+	void Schedule() override {
+		auto &context = pipeline->GetClientContext();
+
+		// Schedule tasks equal to the number of threads, which will each merge multiple partitions
+		auto &ts = TaskScheduler::GetScheduler(context);
+		auto num_threads = NumericCast<idx_t>(ts.NumberOfThreads());
+
+		vector<shared_ptr<Task>> merge_tasks;
+		for (idx_t tnum = 0; tnum < num_threads; tnum++) {
+			merge_tasks.push_back(make_uniq<PhysicalOrderMergeTask>(shared_from_this(), context, gstate, op));
+		}
+		SetTasks(std::move(merge_tasks));
+	}
+
+	void FinishEvent() override {
+		auto &global_sort_state = gstate.global_sort_state;
+
+		global_sort_state.CompleteMergeRound();
+		if (global_sort_state.sorted_blocks.size() > 1) {
+			// Multiple blocks remaining: Schedule the next round
+			PhysicalOrder::ScheduleMergeTasks(*pipeline, *this, gstate);
+		}
+	}
+};
+
 SinkFinalizeType PhysicalOrder::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
                                          OperatorSinkFinalizeInput &input) const {
-	auto &gstate = input.global_state.Cast<OrderGlobalSinkState>();
-	OperatorSinkFinalizeInput sort_input {*gstate.state, input.interrupt_state};
-	return gstate.sort.Finalize(context, sort_input);
+	auto &state = input.global_state.Cast<OrderGlobalSinkState>();
+	auto &global_sort_state = state.global_sort_state;
+
+	if (global_sort_state.sorted_blocks.empty()) {
+		// Empty input!
+		return SinkFinalizeType::NO_OUTPUT_POSSIBLE;
+	}
+
+	// Prepare for merge sort phase
+	global_sort_state.PrepareMergePhase();
+
+	// Start the merge phase or finish if a merge is not necessary
+	if (global_sort_state.sorted_blocks.size() > 1) {
+		PhysicalOrder::ScheduleMergeTasks(pipeline, event, state);
+	}
+	return SinkFinalizeType::READY;
 }
 
-ProgressData PhysicalOrder::GetSinkProgress(ClientContext &context, GlobalSinkState &gstate_p,
-                                            const ProgressData source_progress) const {
-	auto &gstate = gstate_p.Cast<OrderGlobalSinkState>();
-	return gstate.sort.GetSinkProgress(context, *gstate.state, source_progress);
+void PhysicalOrder::ScheduleMergeTasks(Pipeline &pipeline, Event &event, OrderGlobalSinkState &state) {
+	// Initialize global sort state for a round of merging
+	state.global_sort_state.InitializeMergeRound();
+	auto new_event = make_shared_ptr<OrderMergeEvent>(state, pipeline, state.order);
+	event.InsertEvent(std::move(new_event));
 }
 
 //===--------------------------------------------------------------------===//
 // Source
 //===--------------------------------------------------------------------===//
-class OrderGlobalSourceState : public GlobalSourceState {
+class PhysicalOrderGlobalSourceState : public GlobalSourceState {
 public:
-	explicit OrderGlobalSourceState(ClientContext &context, OrderGlobalSinkState &sink)
-	    : sort(sink.sort), state(sort.GetGlobalSourceState(context, *sink.state)) {
+	explicit PhysicalOrderGlobalSourceState(OrderGlobalSinkState &sink) : next_batch_index(0) {
+		auto &global_sort_state = sink.global_sort_state;
+		if (global_sort_state.sorted_blocks.empty()) {
+			total_batches = 0;
+		} else {
+			D_ASSERT(global_sort_state.sorted_blocks.size() == 1);
+			total_batches = global_sort_state.sorted_blocks[0]->payload_data->data_blocks.size();
+		}
 	}
 
-public:
 	idx_t MaxThreads() override {
-		return state->MaxThreads();
+		return total_batches;
 	}
 
 public:
-	Sort &sort;
-	unique_ptr<GlobalSourceState> state;
+	atomic<idx_t> next_batch_index;
+	idx_t total_batches;
 };
 
-class OrderLocalSourceState : public LocalSourceState {
+unique_ptr<GlobalSourceState> PhysicalOrder::GetGlobalSourceState(ClientContext &context) const {
+	auto &sink = this->sink_state->Cast<OrderGlobalSinkState>();
+	return make_uniq<PhysicalOrderGlobalSourceState>(sink);
+}
+
+class PhysicalOrderLocalSourceState : public LocalSourceState {
 public:
-	explicit OrderLocalSourceState(ExecutionContext &context, OrderGlobalSourceState &gstate)
-	    : state(gstate.sort.GetLocalSourceState(context, *gstate.state)) {
+	explicit PhysicalOrderLocalSourceState(PhysicalOrderGlobalSourceState &gstate)
+	    : batch_index(gstate.next_batch_index++) {
 	}
 
 public:
-	unique_ptr<LocalSourceState> state;
+	idx_t batch_index;
+	unique_ptr<PayloadScanner> scanner;
 };
 
 unique_ptr<LocalSourceState> PhysicalOrder::GetLocalSourceState(ExecutionContext &context,
                                                                 GlobalSourceState &gstate_p) const {
-	auto &gstate = gstate_p.Cast<OrderGlobalSourceState>();
-	return make_uniq<OrderLocalSourceState>(context, gstate);
-}
-
-unique_ptr<GlobalSourceState> PhysicalOrder::GetGlobalSourceState(ClientContext &context) const {
-	return make_uniq<OrderGlobalSourceState>(context, sink_state->Cast<OrderGlobalSinkState>());
+	auto &gstate = gstate_p.Cast<PhysicalOrderGlobalSourceState>();
+	return make_uniq<PhysicalOrderLocalSourceState>(gstate);
 }
 
 SourceResultType PhysicalOrder::GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const {
-	auto &gstate = input.global_state.Cast<OrderGlobalSourceState>();
-	auto &lstate = input.local_state.Cast<OrderLocalSourceState>();
-	OperatorSourceInput sort_input {*gstate.state, *lstate.state, input.interrupt_state};
-	return gstate.sort.GetData(context, chunk, sort_input);
+	auto &gstate = input.global_state.Cast<PhysicalOrderGlobalSourceState>();
+	auto &lstate = input.local_state.Cast<PhysicalOrderLocalSourceState>();
+
+	if (lstate.scanner && lstate.scanner->Remaining() == 0) {
+		lstate.batch_index = gstate.next_batch_index++;
+		lstate.scanner = nullptr;
+	}
+
+	if (lstate.batch_index >= gstate.total_batches) {
+		return SourceResultType::FINISHED;
+	}
+
+	if (!lstate.scanner) {
+		auto &sink = this->sink_state->Cast<OrderGlobalSinkState>();
+		auto &global_sort_state = sink.global_sort_state;
+		lstate.scanner = make_uniq<PayloadScanner>(global_sort_state, lstate.batch_index, true);
+	}
+
+	lstate.scanner->Scan(chunk);
+
+	return chunk.size() == 0 ? SourceResultType::FINISHED : SourceResultType::HAVE_MORE_OUTPUT;
 }
 
 OperatorPartitionData PhysicalOrder::GetPartitionData(ExecutionContext &context, DataChunk &chunk,
                                                       GlobalSourceState &gstate_p, LocalSourceState &lstate_p,
                                                       const OperatorPartitionInfo &partition_info) const {
-	auto &gstate = gstate_p.Cast<OrderGlobalSourceState>();
-	auto &lstate = lstate_p.Cast<OrderLocalSourceState>();
 	if (partition_info.RequiresPartitionColumns()) {
 		throw InternalException("PhysicalOrder::GetPartitionData: partition columns not supported");
 	}
-	return gstate.sort.GetPartitionData(context, chunk, *gstate.state, *lstate.state, partition_info);
-}
-
-ProgressData PhysicalOrder::GetProgress(ClientContext &context, GlobalSourceState &gstate_p) const {
-	auto &gstate = gstate_p.Cast<OrderGlobalSourceState>();
-	return gstate.sort.GetProgress(context, *gstate.state);
+	auto &lstate = lstate_p.Cast<PhysicalOrderLocalSourceState>();
+	return OperatorPartitionData(lstate.batch_index);
 }
 
 InsertionOrderPreservingMap<string> PhysicalOrder::ParamsToString() const {

package/src/duckdb/src/execution/operator/order/physical_top_n.cpp

@@ -8,11 +8,10 @@
 
 namespace duckdb {
 
-PhysicalTopN::PhysicalTopN(PhysicalPlan &physical_plan, vector<LogicalType> types, vector<BoundOrderByNode> orders,
-                           idx_t limit, idx_t offset, shared_ptr<DynamicFilterData> dynamic_filter_p,
-                           idx_t estimated_cardinality)
-    : PhysicalOperator(physical_plan, PhysicalOperatorType::TOP_N, std::move(types), estimated_cardinality),
-      orders(std::move(orders)), limit(limit), offset(offset), dynamic_filter(std::move(dynamic_filter_p)) {
+PhysicalTopN::PhysicalTopN(vector<LogicalType> types, vector<BoundOrderByNode> orders, idx_t limit, idx_t offset,
+                           shared_ptr<DynamicFilterData> dynamic_filter_p, idx_t estimated_cardinality)
+    : PhysicalOperator(PhysicalOperatorType::TOP_N, std::move(types), estimated_cardinality), orders(std::move(orders)),
+      limit(limit), offset(offset), dynamic_filter(std::move(dynamic_filter_p)) {
 }
 
 PhysicalTopN::~PhysicalTopN() {

package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp

@@ -26,11 +26,9 @@ struct ActiveFlushGuard {
 	atomic<bool> &bool_value;
 };
 
-PhysicalBatchCopyToFile::PhysicalBatchCopyToFile(PhysicalPlan &physical_plan, vector<LogicalType> types,
-                                                 CopyFunction function_p, unique_ptr<FunctionData> bind_data_p,
-                                                 idx_t estimated_cardinality)
-    : PhysicalOperator(physical_plan, PhysicalOperatorType::BATCH_COPY_TO_FILE, std::move(types),
-                       estimated_cardinality),
+PhysicalBatchCopyToFile::PhysicalBatchCopyToFile(vector<LogicalType> types, CopyFunction function_p,
+                                                 unique_ptr<FunctionData> bind_data_p, idx_t estimated_cardinality)
+    : PhysicalOperator(PhysicalOperatorType::BATCH_COPY_TO_FILE, std::move(types), estimated_cardinality),
       function(std::move(function_p)), bind_data(std::move(bind_data_p)) {
 	if (!function.flush_batch || !function.prepare_batch) {
 		throw InternalException("PhysicalFixedBatchCopy created for copy function that does not have "

package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp

@@ -14,17 +14,16 @@
 
 namespace duckdb {
 
-PhysicalBatchInsert::PhysicalBatchInsert(PhysicalPlan &physical_plan, vector<LogicalType> types_p,
-                                         TableCatalogEntry &table,
+PhysicalBatchInsert::PhysicalBatchInsert(vector<LogicalType> types_p, TableCatalogEntry &table,
                                          vector<unique_ptr<BoundConstraint>> bound_constraints_p,
                                          idx_t estimated_cardinality)
-    : PhysicalOperator(physical_plan, PhysicalOperatorType::BATCH_INSERT, std::move(types_p), estimated_cardinality),
+    : PhysicalOperator(PhysicalOperatorType::BATCH_INSERT, std::move(types_p), estimated_cardinality),
       insert_table(&table), insert_types(table.GetTypes()), bound_constraints(std::move(bound_constraints_p)) {
 }
 
-PhysicalBatchInsert::PhysicalBatchInsert(PhysicalPlan &physical_plan, LogicalOperator &op, SchemaCatalogEntry &schema,
+PhysicalBatchInsert::PhysicalBatchInsert(LogicalOperator &op, SchemaCatalogEntry &schema,
                                          unique_ptr<BoundCreateTableInfo> info_p, idx_t estimated_cardinality)
-    : PhysicalOperator(physical_plan, PhysicalOperatorType::BATCH_CREATE_TABLE_AS, op.types, estimated_cardinality),
+    : PhysicalOperator(PhysicalOperatorType::BATCH_CREATE_TABLE_AS, op.types, estimated_cardinality),
       insert_table(nullptr), schema(&schema), info(std::move(info_p)) {
 	PhysicalInsert::GetInsertInfo(*info, insert_types);
 }
@@ -150,7 +149,6 @@ public:
 	    : memory_manager(context, minimum_memory_per_thread), table(table), insert_count(0),
 	      optimistically_written(false), minimum_memory_per_thread(minimum_memory_per_thread) {
 		row_group_size = table.GetStorage().GetRowGroupSize();
-		table.GetStorage().BindIndexes(context);
 	}
 
 	BatchMemoryManager memory_manager;
@@ -226,7 +224,7 @@ public:
 
 		// Merge the collections.
 		if (!l_state.optimistic_writer) {
-			l_state.optimistic_writer = make_uniq<OptimisticDataWriter>(context, g_state.table.GetStorage());
+			l_state.optimistic_writer = make_uniq<OptimisticDataWriter>(g_state.table.GetStorage());
 		}
 		auto result_collection_index = g_state.MergeCollections(context, merge_collections, *l_state.optimistic_writer);
 		merge_collections.clear();
@@ -527,7 +525,7 @@ SinkResultType PhysicalBatchInsert::Sink(ExecutionContext &context, DataChunk &i
 		// no collection yet: create a new one
 		lstate.CreateNewCollection(context.client, table, insert_types);
 		if (!lstate.optimistic_writer) {
-			lstate.optimistic_writer = make_uniq<OptimisticDataWriter>(context.client, table.GetStorage());
+			lstate.optimistic_writer = make_uniq<OptimisticDataWriter>(table.GetStorage());
 		}
 	}
 
@@ -637,7 +635,7 @@ SinkFinalizeType PhysicalBatchInsert::Finalize(Pipeline &pipeline, Event &event,
 		// now that we have created all of the mergers, perform the actual merging
 		vector<PhysicalIndex> final_collections;
 		final_collections.reserve(mergers.size());
-		auto writer = make_uniq<OptimisticDataWriter>(context, data_table);
+		auto writer = make_uniq<OptimisticDataWriter>(data_table);
 		for (auto &merger : mergers) {
 			final_collections.push_back(merger->Flush(*writer));
 		}

package/src/duckdb/src/execution/operator/persistent/physical_copy_database.cpp

@@ -16,9 +16,9 @@
 
 namespace duckdb {
 
-PhysicalCopyDatabase::PhysicalCopyDatabase(PhysicalPlan &physical_plan, vector<LogicalType> types,
-                                           idx_t estimated_cardinality, unique_ptr<CopyDatabaseInfo> info_p)
-    : PhysicalOperator(physical_plan, PhysicalOperatorType::COPY_DATABASE, std::move(types), estimated_cardinality),
+PhysicalCopyDatabase::PhysicalCopyDatabase(vector<LogicalType> types, idx_t estimated_cardinality,
+                                           unique_ptr<CopyDatabaseInfo> info_p)
+    : PhysicalOperator(PhysicalOperatorType::COPY_DATABASE, std::move(types), estimated_cardinality),
       info(std::move(info_p)) {
 }
 
@@ -81,12 +81,10 @@ SourceResultType PhysicalCopyDatabase::GetData(ExecutionContext &context, DataCh
 		storage_info.options.emplace("v1_0_0_storage", false);
 		auto unbound_index = make_uniq<UnboundIndex>(create_index_info.Copy(), storage_info,
 		                                             data_table.GetTableIOManager(), catalog.GetAttached());
-		data_table.AddIndex(std::move(unbound_index));
 
-		// We add unbound indexes, so we immediately bind them.
-		// Otherwise, WAL serialization fails due to unbound indexes.
+		data_table.AddIndex(std::move(unbound_index));
 		auto &data_table_info = *data_table.GetDataTableInfo();
-		data_table_info.BindIndexes(context.client);
+		data_table_info.GetIndexes().InitializeIndexes(context.client, data_table_info);
 	}
 
 	return SourceResultType::FINISHED;

package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp

@@ -8,7 +8,6 @@
 #include "duckdb/common/value_operations/value_operations.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/planner/operator/logical_copy_to_file.hpp"
-#include "duckdb/main/settings.hpp"
 
 #include <algorithm>
 
@@ -51,7 +50,7 @@ public:
 	explicit CopyToFunctionGlobalState(ClientContext &context)
 	    : initialized(false), rows_copied(0), last_file_offset(0),
 	      file_write_lock_if_rotating(make_uniq<StorageLock>()) {
-		max_open_files = DBConfig::GetSetting<PartitionedWriteMaxOpenFilesSetting>(context);
+		max_open_files = ClientConfig::GetConfig(context).partitioned_write_max_open_files;
 	}
 
 	StorageLock lock;
@@ -253,14 +252,11 @@ string PhysicalCopyToFile::GetTrimmedPath(ClientContext &context) const {
 
 class CopyToFunctionLocalState : public LocalSinkState {
 public:
-	explicit CopyToFunctionLocalState(ClientContext &context, unique_ptr<LocalFunctionData> local_state)
-	    : local_state(std::move(local_state)) {
-		partitioned_write_flush_threshold = DBConfig::GetSetting<PartitionedWriteFlushThresholdSetting>(context);
+	explicit CopyToFunctionLocalState(unique_ptr<LocalFunctionData> local_state) : local_state(std::move(local_state)) {
 	}
 	unique_ptr<GlobalFunctionData> global_state;
 	unique_ptr<LocalFunctionData> local_state;
 	idx_t total_rows_copied = 0;
-	idx_t partitioned_write_flush_threshold;
 
 	//! Buffers the tuples in partitions before writing
 	unique_ptr<HivePartitionedColumnData> part_buffer;
@@ -285,7 +281,7 @@ public:
 		}
 		part_buffer->Append(*part_buffer_append_state, chunk);
 		append_count += chunk.size();
-		if (append_count >= partitioned_write_flush_threshold) {
+		if (append_count >= ClientConfig::GetConfig(context.client).partitioned_write_flush_threshold) {
 			// flush all cached partitions
 			FlushPartitions(context, op, g);
 		}
@@ -374,12 +370,11 @@ unique_ptr<LocalSinkState> PhysicalCopyToFile::GetLocalSinkState(ExecutionContex
 	if (partition_output) {
 		auto &g = sink_state->Cast<CopyToFunctionGlobalState>();
 
-		auto state = make_uniq<CopyToFunctionLocalState>(context.client, nullptr);
+		auto state = make_uniq<CopyToFunctionLocalState>(nullptr);
 		state->InitializeAppendState(context.client, *this, g);
 		return std::move(state);
 	}
-	auto res =
-	    make_uniq<CopyToFunctionLocalState>(context.client, function.copy_to_initialize_local(context, *bind_data));
+	auto res = make_uniq<CopyToFunctionLocalState>(function.copy_to_initialize_local(context, *bind_data));
 	return std::move(res);
 }
 
@@ -488,9 +483,9 @@ string PhysicalCopyToFile::GetNonTmpFile(ClientContext &context, const string &t
 	return fs.JoinPath(path, base);
 }
 
-PhysicalCopyToFile::PhysicalCopyToFile(PhysicalPlan &physical_plan, vector<LogicalType> types, CopyFunction function_p,
+PhysicalCopyToFile::PhysicalCopyToFile(vector<LogicalType> types, CopyFunction function_p,
                                        unique_ptr<FunctionData> bind_data, idx_t estimated_cardinality)
-    : PhysicalOperator(physical_plan, PhysicalOperatorType::COPY_TO_FILE, std::move(types), estimated_cardinality),
+    : PhysicalOperator(PhysicalOperatorType::COPY_TO_FILE, std::move(types), estimated_cardinality),
       function(std::move(function_p)), bind_data(std::move(bind_data)), parallel(false) {
 }
 

package/src/duckdb/src/execution/operator/persistent/physical_delete.cpp

@@ -12,10 +12,10 @@
 
 namespace duckdb {
 
-PhysicalDelete::PhysicalDelete(PhysicalPlan &physical_plan, vector<LogicalType> types, TableCatalogEntry &tableref,
-                               DataTable &table, vector<unique_ptr<BoundConstraint>> bound_constraints,
-                               idx_t row_id_index, idx_t estimated_cardinality, bool return_chunk)
-    : PhysicalOperator(physical_plan, PhysicalOperatorType::DELETE_OPERATOR, std::move(types), estimated_cardinality),
+PhysicalDelete::PhysicalDelete(vector<LogicalType> types, TableCatalogEntry &tableref, DataTable &table,
+                               vector<unique_ptr<BoundConstraint>> bound_constraints, idx_t row_id_index,
+                               idx_t estimated_cardinality, bool return_chunk)
+    : PhysicalOperator(PhysicalOperatorType::DELETE_OPERATOR, std::move(types), estimated_cardinality),
       tableref(tableref), table(table), bound_constraints(std::move(bound_constraints)), row_id_index(row_id_index),
       return_chunk(return_chunk) {
 }
@@ -39,7 +39,6 @@ public:
 	mutex delete_lock;
 	idx_t deleted_count;
 	ColumnDataCollection return_collection;
-	unordered_set<row_t> deleted_row_ids;
 	LocalAppendState delete_index_append_state;
 	bool has_unique_indexes;
 };
@@ -148,40 +147,12 @@ SinkResultType PhysicalDelete::Sink(ExecutionContext &context, DataChunk &chunk,
 		});
 	}
 
-	auto deleted_count = table.Delete(*l_state.delete_state, context.client, row_ids, chunk.size());
-	g_state.deleted_count += deleted_count;
-
 	// Append the return_chunk to the return collection.
 	if (return_chunk) {
-		// Rows can be duplicated, so we get the chunk indexes for new row id values.
-		map<row_t, idx_t> new_row_ids_deleted;
-		auto flat_ids = FlatVector::GetData<row_t>(row_ids);
-		for (idx_t i = 0; i < chunk.size(); i++) {
-			// If the row has not been deleted previously
-			// and is not a duplicate within the current chunk,
-			// then we add it to new_row_ids_deleted.
-			auto row_id = flat_ids[i];
-			auto already_deleted = g_state.deleted_row_ids.find(row_id) != g_state.deleted_row_ids.end();
-			auto newly_deleted = new_row_ids_deleted.find(row_id) != new_row_ids_deleted.end();
-			if (!already_deleted && !newly_deleted) {
-				new_row_ids_deleted[row_id] = i;
-				g_state.deleted_row_ids.insert(row_id);
-			}
-		}
-
-		D_ASSERT(new_row_ids_deleted.size() == deleted_count);
-		if (deleted_count < l_state.delete_chunk.size()) {
-			SelectionVector delete_sel(0, deleted_count);
-			idx_t chunk_index = 0;
-			for (auto &row_id_to_chunk_index : new_row_ids_deleted) {
-				delete_sel.set_index(chunk_index, row_id_to_chunk_index.second);
-				chunk_index++;
-			}
-			l_state.delete_chunk.Slice(delete_sel, deleted_count);
-		}
 		g_state.return_collection.Append(l_state.delete_chunk);
 	}
 
+	g_state.deleted_count += table.Delete(*l_state.delete_state, context.client, row_ids, chunk.size());
 	return SinkResultType::NEED_MORE_INPUT;
 }