duckdb 0.10.2 → 0.10.3-dev10.0

package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp

@@ -88,7 +88,7 @@ PhysicalHashJoin::PhysicalHashJoin(LogicalOperator &op, unique_ptr<PhysicalOpera
 class HashJoinGlobalSinkState : public GlobalSinkState {
 public:
 	HashJoinGlobalSinkState(const PhysicalHashJoin &op, ClientContext &context_p)
-	    : context(context_p), num_threads(TaskScheduler::GetScheduler(context).NumberOfThreads()),
+	    : context(context_p), num_threads(NumericCast<idx_t>(TaskScheduler::GetScheduler(context).NumberOfThreads())),
 	      temporary_memory_update_count(0),
 	      temporary_memory_state(TemporaryMemoryManager::Get(context).Register(context)), finalized(false),
 	      scanned_data(false) {
@@ -206,7 +206,7 @@ unique_ptr<JoinHashTable> PhysicalHashJoin::InitializeHashTable(ClientContext &c
 			auto count_fun = CountFun::GetFunction();
 			vector<unique_ptr<Expression>> children;
 			// this is a dummy but we need it to make the hash table understand whats going on
-			children.push_back(make_uniq_base<Expression, BoundReferenceExpression>(count_fun.return_type, 0));
+			children.push_back(make_uniq_base<Expression, BoundReferenceExpression>(count_fun.return_type, 0U));
 			aggr = function_binder.BindAggregateFunction(count_fun, std::move(children), nullptr,
 			                                             AggregateType::NON_DISTINCT);
 			correlated_aggregates.push_back(&*aggr);
@@ -321,11 +321,11 @@ public:
 		vector<shared_ptr<Task>> finalize_tasks;
 		auto &ht = *sink.hash_table;
 		const auto chunk_count = ht.GetDataCollection().ChunkCount();
-		const idx_t num_threads = TaskScheduler::GetScheduler(context).NumberOfThreads();
+		const auto num_threads = NumericCast<idx_t>(TaskScheduler::GetScheduler(context).NumberOfThreads());
 		if (num_threads == 1 || (ht.Count() < PARALLEL_CONSTRUCT_THRESHOLD && !context.config.verify_parallelism)) {
 			// Single-threaded finalize
 			finalize_tasks.push_back(
-			    make_uniq<HashJoinFinalizeTask>(shared_from_this(), context, sink, 0, chunk_count, false));
+			    make_uniq<HashJoinFinalizeTask>(shared_from_this(), context, sink, 0U, chunk_count, false));
 		} else {
 			// Parallel finalize
 			auto chunks_per_thread = MaxValue<idx_t>((chunk_count + num_threads - 1) / num_threads, 1);
@@ -359,7 +359,7 @@ void HashJoinGlobalSinkState::ScheduleFinalize(Pipeline &pipeline, Event &event)
 		return;
 	}
 	hash_table->InitializePointerTable();
-	auto new_event = make_shared<HashJoinFinalizeEvent>(pipeline, *this);
+	auto new_event = make_shared_ptr<HashJoinFinalizeEvent>(pipeline, *this);
 	event.InsertEvent(std::move(new_event));
 }
 
@@ -409,7 +409,7 @@ public:
 			total_size += sink_collection.SizeInBytes();
 			total_count += sink_collection.Count();
 		}
-		auto total_blocks = (double(total_size) + Storage::BLOCK_SIZE - 1) / Storage::BLOCK_SIZE;
+		auto total_blocks = NumericCast<idx_t>((double(total_size) + Storage::BLOCK_SIZE - 1) / Storage::BLOCK_SIZE);
 		auto count_per_block = total_count / total_blocks;
 		auto blocks_per_vector = MaxValue<idx_t>(STANDARD_VECTOR_SIZE / count_per_block, 2);
 
@@ -474,7 +474,7 @@ SinkFinalizeType PhysicalHashJoin::Finalize(Pipeline &pipeline, Event &event, Cl
 			// We have to repartition
 			ht.SetRepartitionRadixBits(sink.local_hash_tables, sink.temporary_memory_state->GetReservation(),
 			                           max_partition_size, max_partition_count);
-			auto new_event = make_shared<HashJoinRepartitionEvent>(pipeline, sink, sink.local_hash_tables);
+			auto new_event = make_shared_ptr<HashJoinRepartitionEvent>(pipeline, sink, sink.local_hash_tables);
 			event.InsertEvent(std::move(new_event));
 		} else {
 			// No repartitioning!
@@ -816,7 +816,7 @@ void HashJoinGlobalSourceState::PrepareBuild(HashJoinGlobalSinkState &sink) {
 	build_chunk_count = data_collection.ChunkCount();
 	build_chunk_done = 0;
 
-	auto num_threads = TaskScheduler::GetScheduler(sink.context).NumberOfThreads();
+	auto num_threads = NumericCast<idx_t>(TaskScheduler::GetScheduler(sink.context).NumberOfThreads());
 	build_chunks_per_thread = MaxValue<idx_t>((build_chunk_count + num_threads - 1) / num_threads, 1);
 
 	ht.InitializePointerTable();
@@ -847,7 +847,7 @@ void HashJoinGlobalSourceState::PrepareScanHT(HashJoinGlobalSinkState &sink) {
 	full_outer_chunk_count = data_collection.ChunkCount();
 	full_outer_chunk_done = 0;
 
-	auto num_threads = TaskScheduler::GetScheduler(sink.context).NumberOfThreads();
+	auto num_threads = NumericCast<idx_t>(TaskScheduler::GetScheduler(sink.context).NumberOfThreads());
 	full_outer_chunks_per_thread = MaxValue<idx_t>((full_outer_chunk_count + num_threads - 1) / num_threads, 1);
 
 	global_stage = HashJoinSourceStage::SCAN_HT;

package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp

@@ -325,7 +325,7 @@ idx_t IEJoinUnion::AppendKey(SortedTable &table, ExpressionExecutor &executor, S
 		payload.data[0].Sequence(rid, increment, scan_count);
 		payload.SetCardinality(scan_count);
 		keys.Fuse(payload);
-		rid += increment * scan_count;
+		rid += increment * UnsafeNumericCast<int64_t>(scan_count);
 
 		// Sort on the sort columns (which will no longer be needed)
 		keys.Split(payload, payload_idx);
@@ -385,7 +385,7 @@ IEJoinUnion::IEJoinUnion(ClientContext &context, const PhysicalIEJoin &op, Sorte
 	payload_layout.Initialize(types);
 
 	// Sort on the first expression
-	auto ref = make_uniq<BoundReferenceExpression>(order1.expression->return_type, 0);
+	auto ref = make_uniq<BoundReferenceExpression>(order1.expression->return_type, 0U);
 	vector<BoundOrderByNode> orders;
 	orders.emplace_back(order1.type, order1.null_order, std::move(ref));
 
@@ -426,7 +426,7 @@ IEJoinUnion::IEJoinUnion(ClientContext &context, const PhysicalIEJoin &op, Sorte
 
 	// Sort on the first expression
 	orders.clear();
-	ref = make_uniq<BoundReferenceExpression>(order2.expression->return_type, 0);
+	ref = make_uniq<BoundReferenceExpression>(order2.expression->return_type, 0U);
 	orders.emplace_back(order2.type, order2.null_order, std::move(ref));
 
 	ExpressionExecutor executor(context);
@@ -434,7 +434,7 @@ IEJoinUnion::IEJoinUnion(ClientContext &context, const PhysicalIEJoin &op, Sorte
 
 	l2 = make_uniq<SortedTable>(context, orders, payload_layout);
 	for (idx_t base = 0, block_idx = 0; block_idx < l1->BlockCount(); ++block_idx) {
-		base += AppendKey(*l1, executor, *l2, 1, base, block_idx);
+		base += AppendKey(*l1, executor, *l2, 1, NumericCast<int64_t>(base), block_idx);
 	}
 
 	Sort(*l2);

package/src/duckdb/src/execution/operator/join/physical_left_delim_join.cpp

@@ -23,7 +23,7 @@ PhysicalLeftDelimJoin::PhysicalLeftDelimJoin(vector<LogicalType> types, unique_p
 	// we replace it with a PhysicalColumnDataScan, that scans the ColumnDataCollection that we keep cached
 	// the actual chunk collection to scan will be created in the LeftDelimJoinGlobalState
 	auto cached_chunk_scan = make_uniq<PhysicalColumnDataScan>(
-	    children[0]->GetTypes(), PhysicalOperatorType::COLUMN_DATA_SCAN, estimated_cardinality);
+	    children[0]->GetTypes(), PhysicalOperatorType::COLUMN_DATA_SCAN, estimated_cardinality, nullptr);
 	join->children[0] = std::move(cached_chunk_scan);
 }
 

package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp

@@ -105,7 +105,7 @@ unique_ptr<GlobalSinkState> PhysicalPiecewiseMergeJoin::GetGlobalSinkState(Clien
 
 unique_ptr<LocalSinkState> PhysicalPiecewiseMergeJoin::GetLocalSinkState(ExecutionContext &context) const {
 	// We only sink the RHS
-	return make_uniq<MergeJoinLocalState>(context.client, *this, 1);
+	return make_uniq<MergeJoinLocalState>(context.client, *this, 1U);
 }
 
 SinkResultType PhysicalPiecewiseMergeJoin::Sink(ExecutionContext &context, DataChunk &chunk,
@@ -223,7 +223,7 @@ public:
 	void ResolveJoinKeys(DataChunk &input) {
 		// sort by join key
 		lhs_global_state = make_uniq<GlobalSortState>(buffer_manager, lhs_order, lhs_layout);
-		lhs_local_table = make_uniq<LocalSortedTable>(context, op, 0);
+		lhs_local_table = make_uniq<LocalSortedTable>(context, op, 0U);
 		lhs_local_table->Sink(input, *lhs_global_state);
 
 		// Set external (can be forced with the PRAGMA)

package/src/duckdb/src/execution/operator/join/physical_range_join.cpp

@@ -126,7 +126,7 @@ public:
 
 		// Schedule tasks equal to the number of threads, which will each merge multiple partitions
 		auto &ts = TaskScheduler::GetScheduler(context);
-		idx_t num_threads = ts.NumberOfThreads();
+		auto num_threads = NumericCast<idx_t>(ts.NumberOfThreads());
 
 		vector<shared_ptr<Task>> iejoin_tasks;
 		for (idx_t tnum = 0; tnum < num_threads; tnum++) {
@@ -149,7 +149,7 @@ public:
 void PhysicalRangeJoin::GlobalSortedTable::ScheduleMergeTasks(Pipeline &pipeline, Event &event) {
 	// Initialize global sort state for a round of merging
 	global_sort_state.InitializeMergeRound();
-	auto new_event = make_shared<RangeJoinMergeEvent>(*this, pipeline);
+	auto new_event = make_shared_ptr<RangeJoinMergeEvent>(*this, pipeline);
 	event.InsertEvent(std::move(new_event));
 }
 

package/src/duckdb/src/execution/operator/order/physical_order.cpp

@@ -6,6 +6,7 @@
 #include "duckdb/parallel/base_pipeline_event.hpp"
 #include "duckdb/parallel/executor_task.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
+#include "duckdb/common/shared_ptr.hpp"
 
 namespace duckdb {
 
@@ -143,7 +144,7 @@ public:
 
 		// Schedule tasks equal to the number of threads, which will each merge multiple partitions
 		auto &ts = TaskScheduler::GetScheduler(context);
-		idx_t num_threads = ts.NumberOfThreads();
+		auto num_threads = NumericCast<idx_t>(ts.NumberOfThreads());
 
 		vector<shared_ptr<Task>> merge_tasks;
 		for (idx_t tnum = 0; tnum < num_threads; tnum++) {
@@ -186,7 +187,7 @@ SinkFinalizeType PhysicalOrder::Finalize(Pipeline &pipeline, Event &event, Clien
 void PhysicalOrder::ScheduleMergeTasks(Pipeline &pipeline, Event &event, OrderGlobalSinkState &state) {
 	// Initialize global sort state for a round of merging
 	state.global_sort_state.InitializeMergeRound();
-	auto new_event = make_shared<OrderMergeEvent>(state, pipeline);
+	auto new_event = make_shared_ptr<OrderMergeEvent>(state, pipeline);
 	event.InsertEvent(std::move(new_event));
 }
 

package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp

@@ -308,7 +308,7 @@ SinkFinalizeType PhysicalBatchCopyToFile::Finalize(Pipeline &pipeline, Event &ev
 		FinalFlush(context, input.global_state);
 	} else {
 		// we have multiple tasks remaining - launch an event to execute the tasks in parallel
-		auto new_event = make_shared<ProcessRemainingBatchesEvent>(*this, gstate, pipeline, context);
+		auto new_event = make_shared_ptr<ProcessRemainingBatchesEvent>(*this, gstate, pipeline, context);
 		event.InsertEvent(std::move(new_event));
 	}
 	return SinkFinalizeType::READY;
@@ -434,7 +434,7 @@ void PhysicalBatchCopyToFile::RepartitionBatches(ClientContext &context, GlobalS
 				// create an empty collection
 				auto new_collection =
 				    make_uniq<ColumnDataCollection>(context, children[0]->types, ColumnDataAllocatorType::HYBRID);
-				append_batch = make_uniq<FixedRawBatchData>(0, std::move(new_collection));
+				append_batch = make_uniq<FixedRawBatchData>(0U, std::move(new_collection));
 			}
 			if (append_batch) {
 				append_batch->collection->InitializeAppend(append_state);
@@ -459,7 +459,7 @@ void PhysicalBatchCopyToFile::RepartitionBatches(ClientContext &context, GlobalS
 
 			auto new_collection =
 			    make_uniq<ColumnDataCollection>(context, children[0]->types, ColumnDataAllocatorType::HYBRID);
-			append_batch = make_uniq<FixedRawBatchData>(0, std::move(new_collection));
+			append_batch = make_uniq<FixedRawBatchData>(0U, std::move(new_collection));
 			append_batch->collection->InitializeAppend(append_state);
 		}
 	}
@@ -605,7 +605,7 @@ SourceResultType PhysicalBatchCopyToFile::GetData(ExecutionContext &context, Dat
 	auto &g = sink_state->Cast<FixedBatchCopyGlobalState>();
 
 	chunk.SetCardinality(1);
-	chunk.SetValue(0, 0, Value::BIGINT(g.rows_copied));
+	chunk.SetValue(0, 0, Value::BIGINT(NumericCast<int64_t>(g.rows_copied.load())));
 	return SourceResultType::FINISHED;
 }
 

package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp

@@ -13,12 +13,14 @@
 
 namespace duckdb {
 
-PhysicalBatchInsert::PhysicalBatchInsert(vector<LogicalType> types, TableCatalogEntry &table,
-                                         physical_index_vector_t<idx_t> column_index_map,
-                                         vector<unique_ptr<Expression>> bound_defaults, idx_t estimated_cardinality)
-    : PhysicalOperator(PhysicalOperatorType::BATCH_INSERT, std::move(types), estimated_cardinality),
-      column_index_map(std::move(column_index_map)), insert_table(&table), insert_types(table.GetTypes()),
-      bound_defaults(std::move(bound_defaults)) {
+PhysicalBatchInsert::PhysicalBatchInsert(vector<LogicalType> types_p, TableCatalogEntry &table,
+                                         physical_index_vector_t<idx_t> column_index_map_p,
+                                         vector<unique_ptr<Expression>> bound_defaults_p,
+                                         vector<unique_ptr<BoundConstraint>> bound_constraints_p,
+                                         idx_t estimated_cardinality)
+    : PhysicalOperator(PhysicalOperatorType::BATCH_INSERT, std::move(types_p), estimated_cardinality),
+      column_index_map(std::move(column_index_map_p)), insert_table(&table), insert_types(table.GetTypes()),
+      bound_defaults(std::move(bound_defaults_p)), bound_constraints(std::move(bound_constraints_p)) {
 }
 
 PhysicalBatchInsert::PhysicalBatchInsert(LogicalOperator &op, SchemaCatalogEntry &schema,
@@ -171,11 +173,13 @@ public:
 	TableAppendState current_append_state;
 	unique_ptr<RowGroupCollection> current_collection;
 	optional_ptr<OptimisticDataWriter> writer;
+	unique_ptr<ConstraintState> constraint_state;
 
 	void CreateNewCollection(DuckTableEntry &table, const vector<LogicalType> &insert_types) {
-		auto &table_info = table.GetStorage().info;
+		auto table_info = table.GetStorage().GetDataTableInfo();
 		auto &block_manager = TableIOManager::Get(table.GetStorage()).GetBlockManagerForRowData();
-		current_collection = make_uniq<RowGroupCollection>(table_info, block_manager, insert_types, MAX_ROW_ID);
+		current_collection = make_uniq<RowGroupCollection>(std::move(table_info), block_manager, insert_types,
+		                                                   NumericCast<idx_t>(MAX_ROW_ID));
 		current_collection->InitializeEmpty();
 		current_collection->InitializeAppend(current_append_state);
 	}
@@ -312,8 +316,8 @@ void BatchInsertGlobalState::ScheduleMergeTasks(idx_t min_batch_index) {
 		auto &scheduled_task = to_be_scheduled_tasks[i - 1];
 		if (scheduled_task.start_index + 1 < scheduled_task.end_index) {
 			// erase all entries except the first one
-			collections.erase(collections.begin() + scheduled_task.start_index + 1,
-			                  collections.begin() + scheduled_task.end_index);
+			collections.erase(collections.begin() + NumericCast<int64_t>(scheduled_task.start_index) + 1,
+			                  collections.begin() + NumericCast<int64_t>(scheduled_task.end_index));
 		}
 	}
 }
@@ -493,7 +497,10 @@ SinkResultType PhysicalBatchInsert::Sink(ExecutionContext &context, DataChunk &c
 		throw InternalException("Current batch differs from batch - but NextBatch was not called!?");
 	}
 
-	table.GetStorage().VerifyAppendConstraints(table, context.client, lstate.insert_chunk);
+	if (!lstate.constraint_state) {
+		lstate.constraint_state = table.GetStorage().InitializeConstraintState(table, bound_constraints);
+	}
+	table.GetStorage().VerifyAppendConstraints(*lstate.constraint_state, context.client, lstate.insert_chunk);
 
 	auto new_row_group = lstate.current_collection->Append(lstate.insert_chunk, lstate.current_append_state);
 	if (new_row_group) {
@@ -594,7 +601,7 @@ SinkFinalizeType PhysicalBatchInsert::Finalize(Pipeline &pipeline, Event &event,
 		auto &table = gstate.table;
 		auto &storage = table.GetStorage();
 		LocalAppendState append_state;
-		storage.InitializeLocalAppend(append_state, context);
+		storage.InitializeLocalAppend(append_state, table, context, bound_constraints);
 		auto &transaction = DuckTransaction::Get(context, table.catalog);
 		for (auto &entry : gstate.collections) {
 			if (entry.type != RowGroupBatchType::NOT_FLUSHED) {
@@ -622,7 +629,7 @@ SourceResultType PhysicalBatchInsert::GetData(ExecutionContext &context, DataChu
 	auto &insert_gstate = sink_state->Cast<BatchInsertGlobalState>();
 
 	chunk.SetCardinality(1);
-	chunk.SetValue(0, 0, Value::BIGINT(insert_gstate.insert_count));
+	chunk.SetValue(0, 0, Value::BIGINT(NumericCast<int64_t>(insert_gstate.insert_count)));
 
 	return SourceResultType::FINISHED;
 }

package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp

@@ -52,21 +52,25 @@ public:
 	atomic<idx_t> rows_copied;
 	atomic<idx_t> last_file_offset;
 	unique_ptr<GlobalFunctionData> global_state;
-	idx_t created_directories = 0;
-
+	//! Created directories
+	unordered_set<string> created_directories;
 	//! shared state for HivePartitionedColumnData
 	shared_ptr<GlobalHivePartitionState> partition_state;
 
-	static void CreateDir(const string &dir_path, FileSystem &fs) {
+	void CreateDir(const string &dir_path, FileSystem &fs) {
+		if (created_directories.find(dir_path) != created_directories.end()) {
+			// already attempted to create this directory
+			return;
+		}
 		if (!fs.DirectoryExists(dir_path)) {
 			fs.CreateDirectory(dir_path);
 		}
+		created_directories.insert(dir_path);
 	}
 
-	static void CreateDirectories(const vector<idx_t> &cols, const vector<string> &names, const vector<Value> &values,
-	                              string path, FileSystem &fs) {
+	string GetOrCreateDirectory(const vector<idx_t> &cols, const vector<string> &names, const vector<Value> &values,
+	                            string path, FileSystem &fs) {
 		CreateDir(path, fs);
-
 		for (idx_t i = 0; i < cols.size(); i++) {
 			const auto &partition_col_name = names[cols[i]];
 			const auto &partition_value = values[i];
@@ -74,32 +78,6 @@ public:
 			path = fs.JoinPath(path, p_dir);
 			CreateDir(path, fs);
 		}
-	}
-
-	void CreatePartitionDirectories(ClientContext &context, const PhysicalCopyToFile &op) {
-		auto &fs = FileSystem::GetFileSystem(context);
-
-		auto trimmed_path = op.GetTrimmedPath(context);
-
-		auto l = lock.GetExclusiveLock();
-		lock_guard<mutex> global_lock_on_partition_state(partition_state->lock);
-		const auto &global_partitions = partition_state->partitions;
-		// global_partitions have partitions added only at the back, so it's fine to only traverse the last part
-
-		for (idx_t i = created_directories; i < global_partitions.size(); i++) {
-			CreateDirectories(op.partition_columns, op.names, global_partitions[i]->first.values, trimmed_path, fs);
-		}
-		created_directories = global_partitions.size();
-	}
-
-	static string GetDirectory(const vector<idx_t> &cols, const vector<string> &names, const vector<Value> &values,
-	                           string path, FileSystem &fs) {
-		for (idx_t i = 0; i < cols.size(); i++) {
-			const auto &partition_col_name = names[cols[i]];
-			const auto &partition_value = values[i];
-			string p_dir = partition_col_name + "=" + partition_value.ToString();
-			path = fs.JoinPath(path, p_dir);
-		}
 		return path;
 	}
 
@@ -132,12 +110,8 @@ public:
 		auto &fs = FileSystem::GetFileSystem(context.client);
 		// Create a writer for the current file
 		auto trimmed_path = op.GetTrimmedPath(context.client);
-		string hive_path = GetDirectory(op.partition_columns, op.names, values, trimmed_path, fs);
+		string hive_path = GetOrCreateDirectory(op.partition_columns, op.names, values, trimmed_path, fs);
 		string full_path(op.filename_pattern.CreateFilename(fs, hive_path, op.file_extension, 0));
-		if (fs.FileExists(full_path) && !op.overwrite_or_ignore) {
-			throw IOException("failed to create %s, file exists! Enable OVERWRITE_OR_IGNORE option to force writing",
-			                  full_path);
-		}
 		// initialize writes
 		auto info = make_uniq<PartitionWriteInfo>();
 		info->global_state = op.function.copy_to_initialize_global(context.client, *op.bind_data, full_path);
@@ -161,8 +135,7 @@ string PhysicalCopyToFile::GetTrimmedPath(ClientContext &context) const {
 
 class CopyToFunctionLocalState : public LocalSinkState {
 public:
-	explicit CopyToFunctionLocalState(unique_ptr<LocalFunctionData> local_state)
-	    : local_state(std::move(local_state)), writer_offset(0) {
+	explicit CopyToFunctionLocalState(unique_ptr<LocalFunctionData> local_state) : local_state(std::move(local_state)) {
 	}
 	unique_ptr<GlobalFunctionData> global_state;
 	unique_ptr<LocalFunctionData> local_state;
@@ -171,7 +144,6 @@ public:
 	unique_ptr<HivePartitionedColumnData> part_buffer;
 	unique_ptr<PartitionedColumnDataAppendState> part_buffer_append_state;
 
-	idx_t writer_offset;
 	idx_t append_count = 0;
 
 	void InitializeAppendState(ClientContext &context, const PhysicalCopyToFile &op,
@@ -211,12 +183,13 @@ public:
 		auto &partitions = part_buffer->GetPartitions();
 		auto partition_key_map = part_buffer->GetReverseMap();
 
-		// ensure all partition directories are created before we start writing
-		g.CreatePartitionDirectories(context.client, op);
-
 		for (idx_t i = 0; i < partitions.size(); i++) {
+			auto entry = partition_key_map.find(i);
+			if (entry == partition_key_map.end()) {
+				continue;
+			}
 			// get the partition write info for this buffer
-			auto &info = g.GetPartitionWriteInfo(context, op, partition_key_map[i]->values);
+			auto &info = g.GetPartitionWriteInfo(context, op, entry->second->values);
 
 			auto local_copy_state = op.function.copy_to_initialize_local(context, *op.bind_data);
 			// push the chunks into the write state
@@ -237,9 +210,6 @@ unique_ptr<GlobalFunctionData> PhysicalCopyToFile::CreateFileState(ClientContext
 	idx_t this_file_offset = g.last_file_offset++;
 	auto &fs = FileSystem::GetFileSystem(context);
 	string output_path(filename_pattern.CreateFilename(fs, file_path, file_extension, this_file_offset));
-	if (fs.FileExists(output_path) && !overwrite_or_ignore) {
-		throw IOException("%s exists! Enable OVERWRITE_OR_IGNORE option to force writing", output_path);
-	}
 	return function.copy_to_initialize_global(context, *bind_data, output_path);
 }
 
@@ -248,7 +218,6 @@ unique_ptr<LocalSinkState> PhysicalCopyToFile::GetLocalSinkState(ExecutionContex
 		auto &g = sink_state->Cast<CopyToFunctionGlobalState>();
 
 		auto state = make_uniq<CopyToFunctionLocalState>(nullptr);
-		state->writer_offset = g.last_file_offset++;
 		state->InitializeAppendState(context.client, *this, g);
 		return std::move(state);
 	}
@@ -259,23 +228,64 @@ unique_ptr<LocalSinkState> PhysicalCopyToFile::GetLocalSinkState(ExecutionContex
 	return std::move(res);
 }
 
+void CheckDirectory(FileSystem &fs, const string &file_path, bool overwrite) {
+	if (fs.IsRemoteFile(file_path) && overwrite) {
+		// we only remove files for local file systems
+		// as remote file systems (e.g. S3) do not support RemoveFile
+		return;
+	}
+	vector<string> file_list;
+	vector<string> directory_list;
+	directory_list.push_back(file_path);
+	for (idx_t dir_idx = 0; dir_idx < directory_list.size(); dir_idx++) {
+		auto directory = directory_list[dir_idx];
+		fs.ListFiles(directory, [&](const string &path, bool is_directory) {
+			auto full_path = fs.JoinPath(directory, path);
+			if (is_directory) {
+				directory_list.emplace_back(std::move(full_path));
+			} else {
+				file_list.emplace_back(std::move(full_path));
+			}
+		});
+	}
+	if (file_list.empty()) {
+		return;
+	}
+	if (overwrite) {
+		for (auto &file : file_list) {
+			fs.RemoveFile(file);
+		}
+	} else {
+		throw IOException("Directory \"%s\" is not empty! Enable OVERWRITE_OR_IGNORE option to force writing",
+		                  file_path);
+	}
+}
+
 unique_ptr<GlobalSinkState> PhysicalCopyToFile::GetGlobalSinkState(ClientContext &context) const {
 
 	if (partition_output || per_thread_output || file_size_bytes.IsValid()) {
 		auto &fs = FileSystem::GetFileSystem(context);
-
-		if (fs.FileExists(file_path) && !overwrite_or_ignore) {
-			throw IOException("%s exists! Enable OVERWRITE_OR_IGNORE option to force writing", file_path);
+		if (fs.FileExists(file_path)) {
+			// the target file exists AND is a file (not a directory)
+			if (fs.IsRemoteFile(file_path)) {
+				// for remote files we cannot do anything - as we cannot delete the file
+				throw IOException("Cannot write to \"%s\" - it exists and is a file, not a directory!", file_path);
+			} else {
+				// for local files we can remove the file if OVERWRITE_OR_IGNORE is enabled
+				if (overwrite_or_ignore) {
+					fs.RemoveFile(file_path);
+				} else {
+					throw IOException("Cannot write to \"%s\" - it exists and is a file, not a directory! Enable "
+					                  "OVERWRITE_OR_IGNORE option to force writing",
+					                  file_path);
+				}
+			}
 		}
+		// what if the target exists and is a directory
 		if (!fs.DirectoryExists(file_path)) {
 			fs.CreateDirectory(file_path);
-		} else if (!overwrite_or_ignore) {
-			idx_t n_files = 0;
-			fs.ListFiles(file_path, [&n_files](const string &path, bool) { n_files++; });
-			if (n_files > 0) {
-				throw IOException("Directory %s is not empty! Enable OVERWRITE_OR_IGNORE option to force writing",
-				                  file_path);
-			}
+		} else {
+			CheckDirectory(fs, file_path, overwrite_or_ignore);
 		}
 
 		auto state = make_uniq<CopyToFunctionGlobalState>(nullptr);
@@ -284,7 +294,7 @@ unique_ptr<GlobalSinkState> PhysicalCopyToFile::GetGlobalSinkState(ClientContext
 		}
 
 		if (partition_output) {
-			state->partition_state = make_shared<GlobalHivePartitionState>();
+			state->partition_state = make_shared_ptr<GlobalHivePartitionState>();
 		}
 
 		return std::move(state);
@@ -423,7 +433,7 @@ SourceResultType PhysicalCopyToFile::GetData(ExecutionContext &context, DataChun
 	auto &g = sink_state->Cast<CopyToFunctionGlobalState>();
 
 	chunk.SetCardinality(1);
-	chunk.SetValue(0, 0, Value::BIGINT(g.rows_copied));
+	chunk.SetValue(0, 0, Value::BIGINT(NumericCast<int64_t>(g.rows_copied.load())));
 
 	return SourceResultType::FINISHED;
 }

package/src/duckdb/src/execution/operator/persistent/physical_delete.cpp

@@ -6,9 +6,18 @@
 #include "duckdb/storage/data_table.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/transaction/duck_transaction.hpp"
+#include "duckdb/storage/table/delete_state.hpp"
+#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 
 namespace duckdb {
 
+PhysicalDelete::PhysicalDelete(vector<LogicalType> types, TableCatalogEntry &tableref, DataTable &table,
+                               vector<unique_ptr<BoundConstraint>> bound_constraints, idx_t row_id_index,
+                               idx_t estimated_cardinality, bool return_chunk)
+    : PhysicalOperator(PhysicalOperatorType::DELETE_OPERATOR, std::move(types), estimated_cardinality),
+      tableref(tableref), table(table), bound_constraints(std::move(bound_constraints)), row_id_index(row_id_index),
+      return_chunk(return_chunk) {
+}
 //===--------------------------------------------------------------------===//
 // Sink
 //===--------------------------------------------------------------------===//
@@ -25,10 +34,13 @@ public:
 
 class DeleteLocalState : public LocalSinkState {
 public:
-	DeleteLocalState(Allocator &allocator, const vector<LogicalType> &table_types) {
-		delete_chunk.Initialize(allocator, table_types);
+	DeleteLocalState(ClientContext &context, TableCatalogEntry &table,
+	                 const vector<unique_ptr<BoundConstraint>> &bound_constraints) {
+		delete_chunk.Initialize(Allocator::Get(context), table.GetTypes());
+		delete_state = table.GetStorage().InitializeDelete(table, context, bound_constraints);
 	}
 	DataChunk delete_chunk;
+	unique_ptr<TableDeleteState> delete_state;
 };
 
 SinkResultType PhysicalDelete::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const {
@@ -40,7 +52,7 @@ SinkResultType PhysicalDelete::Sink(ExecutionContext &context, DataChunk &chunk,
 	auto &row_identifiers = chunk.data[row_id_index];
 
 	vector<column_t> column_ids;
-	for (idx_t i = 0; i < table.column_definitions.size(); i++) {
+	for (idx_t i = 0; i < table.ColumnCount(); i++) {
 		column_ids.emplace_back(i);
 	};
 	auto cfs = ColumnFetchState();
@@ -52,8 +64,7 @@ SinkResultType PhysicalDelete::Sink(ExecutionContext &context, DataChunk &chunk,
 		table.Fetch(transaction, ustate.delete_chunk, column_ids, row_identifiers, chunk.size(), cfs);
 		gstate.return_collection.Append(ustate.delete_chunk);
 	}
-	gstate.deleted_count += table.Delete(tableref, context.client, row_identifiers, chunk.size());
-
+	gstate.deleted_count += table.Delete(*ustate.delete_state, context.client, row_identifiers, chunk.size());
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
@@ -62,7 +73,7 @@ unique_ptr<GlobalSinkState> PhysicalDelete::GetGlobalSinkState(ClientContext &co
 }
 
 unique_ptr<LocalSinkState> PhysicalDelete::GetLocalSinkState(ExecutionContext &context) const {
-	return make_uniq<DeleteLocalState>(Allocator::Get(context.client), table.GetTypes());
+	return make_uniq<DeleteLocalState>(context.client, tableref, bound_constraints);
 }
 
 //===--------------------------------------------------------------------===//
@@ -91,7 +102,7 @@ SourceResultType PhysicalDelete::GetData(ExecutionContext &context, DataChunk &c
 	auto &g = sink_state->Cast<DeleteGlobalState>();
 	if (!return_chunk) {
 		chunk.SetCardinality(1);
-		chunk.SetValue(0, 0, Value::BIGINT(g.deleted_count));
+		chunk.SetValue(0, 0, Value::BIGINT(NumericCast<int64_t>(g.deleted_count)));
 		return SourceResultType::FINISHED;
 	}