duckdb 0.10.1-dev6.0 → 0.10.1

package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp

@@ -7,11 +7,42 @@
 #include "duckdb/common/types/uuid.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/storage/storage_lock.hpp"
-
+#include "duckdb/common/value_operations/value_operations.hpp"
 #include <algorithm>
 
 namespace duckdb {
 
+struct PartitionWriteInfo {
+	unique_ptr<GlobalFunctionData> global_state;
+};
+
+struct VectorOfValuesHashFunction {
+	uint64_t operator()(const vector<Value> &values) const {
+		hash_t result = 0;
+		for (auto &val : values) {
+			result ^= val.Hash();
+		}
+		return result;
+	}
+};
+
+struct VectorOfValuesEquality {
+	bool operator()(const vector<Value> &a, const vector<Value> &b) const {
+		if (a.size() != b.size()) {
+			return false;
+		}
+		for (idx_t i = 0; i < a.size(); i++) {
+			if (ValueOperations::DistinctFrom(a[i], b[i])) {
+				return false;
+			}
+		}
+		return true;
+	}
+};
+
+template <class T>
+using vector_of_value_map_t = unordered_map<vector<Value>, T, VectorOfValuesHashFunction, VectorOfValuesEquality>;
+
 class CopyToFunctionGlobalState : public GlobalSinkState {
 public:
 	explicit CopyToFunctionGlobalState(unique_ptr<GlobalFunctionData> global_state)
@@ -25,8 +56,109 @@ public:
 
 	//! shared state for HivePartitionedColumnData
 	shared_ptr<GlobalHivePartitionState> partition_state;
+
+	static void CreateDir(const string &dir_path, FileSystem &fs) {
+		if (!fs.DirectoryExists(dir_path)) {
+			fs.CreateDirectory(dir_path);
+		}
+	}
+
+	static void CreateDirectories(const vector<idx_t> &cols, const vector<string> &names, const vector<Value> &values,
+	                              string path, FileSystem &fs) {
+		CreateDir(path, fs);
+
+		for (idx_t i = 0; i < cols.size(); i++) {
+			const auto &partition_col_name = names[cols[i]];
+			const auto &partition_value = values[i];
+			string p_dir = partition_col_name + "=" + partition_value.ToString();
+			path = fs.JoinPath(path, p_dir);
+			CreateDir(path, fs);
+		}
+	}
+
+	void CreatePartitionDirectories(ClientContext &context, const PhysicalCopyToFile &op) {
+		auto &fs = FileSystem::GetFileSystem(context);
+
+		auto trimmed_path = op.GetTrimmedPath(context);
+
+		auto l = lock.GetExclusiveLock();
+		lock_guard<mutex> global_lock_on_partition_state(partition_state->lock);
+		const auto &global_partitions = partition_state->partitions;
+		// global_partitions have partitions added only at the back, so it's fine to only traverse the last part
+
+		for (idx_t i = created_directories; i < global_partitions.size(); i++) {
+			CreateDirectories(op.partition_columns, op.names, global_partitions[i]->first.values, trimmed_path, fs);
+		}
+		created_directories = global_partitions.size();
+	}
+
+	static string GetDirectory(const vector<idx_t> &cols, const vector<string> &names, const vector<Value> &values,
+	                           string path, FileSystem &fs) {
+		for (idx_t i = 0; i < cols.size(); i++) {
+			const auto &partition_col_name = names[cols[i]];
+			const auto &partition_value = values[i];
+			string p_dir = partition_col_name + "=" + partition_value.ToString();
+			path = fs.JoinPath(path, p_dir);
+		}
+		return path;
+	}
+
+	void FinalizePartition(ClientContext &context, const PhysicalCopyToFile &op, PartitionWriteInfo &info) {
+		if (!info.global_state) {
+			// already finalized
+			return;
+		}
+		// finalize the partition
+		op.function.copy_to_finalize(context, *op.bind_data, *info.global_state);
+		info.global_state.reset();
+	}
+
+	void FinalizePartitions(ClientContext &context, const PhysicalCopyToFile &op) {
+		// finalize any remaining partitions
+		for (auto &entry : active_partitioned_writes) {
+			FinalizePartition(context, op, *entry.second);
+		}
+	}
+
+	PartitionWriteInfo &GetPartitionWriteInfo(ExecutionContext &context, const PhysicalCopyToFile &op,
+	                                          const vector<Value> &values) {
+		auto l = lock.GetExclusiveLock();
+		// check if we have already started writing this partition
+		auto entry = active_partitioned_writes.find(values);
+		if (entry != active_partitioned_writes.end()) {
+			// we have - continue writing in this partition
+			return *entry->second;
+		}
+		auto &fs = FileSystem::GetFileSystem(context.client);
+		// Create a writer for the current file
+		auto trimmed_path = op.GetTrimmedPath(context.client);
+		string hive_path = GetDirectory(op.partition_columns, op.names, values, trimmed_path, fs);
+		string full_path(op.filename_pattern.CreateFilename(fs, hive_path, op.file_extension, 0));
+		if (fs.FileExists(full_path) && !op.overwrite_or_ignore) {
+			throw IOException("failed to create %s, file exists! Enable OVERWRITE_OR_IGNORE option to force writing",
+			                  full_path);
+		}
+		// initialize writes
+		auto info = make_uniq<PartitionWriteInfo>();
+		info->global_state = op.function.copy_to_initialize_global(context.client, *op.bind_data, full_path);
+		auto &result = *info;
+		// store in active write map
+		active_partitioned_writes.insert(make_pair(values, std::move(info)));
+		return result;
+	}
+
+private:
+	//! The active writes per partition (for partitioned write)
+	vector_of_value_map_t<unique_ptr<PartitionWriteInfo>> active_partitioned_writes;
 };
 
+string PhysicalCopyToFile::GetTrimmedPath(ClientContext &context) const {
+	auto &fs = FileSystem::GetFileSystem(context);
+	string trimmed_path = file_path;
+	StringUtil::RTrim(trimmed_path, fs.PathSeparator(trimmed_path));
+	return trimmed_path;
+}
+
 class CopyToFunctionLocalState : public LocalSinkState {
 public:
 	explicit CopyToFunctionLocalState(unique_ptr<LocalFunctionData> local_state)
@@ -40,6 +172,63 @@ public:
 	unique_ptr<PartitionedColumnDataAppendState> part_buffer_append_state;
 
 	idx_t writer_offset;
+	idx_t append_count = 0;
+
+	void InitializeAppendState(ClientContext &context, const PhysicalCopyToFile &op,
+	                           CopyToFunctionGlobalState &gstate) {
+		part_buffer = make_uniq<HivePartitionedColumnData>(context, op.expected_types, op.partition_columns,
+		                                                   gstate.partition_state);
+		part_buffer_append_state = make_uniq<PartitionedColumnDataAppendState>();
+		part_buffer->InitializeAppendState(*part_buffer_append_state);
+		append_count = 0;
+	}
+
+	void AppendToPartition(ExecutionContext &context, const PhysicalCopyToFile &op, CopyToFunctionGlobalState &g,
+	                       DataChunk &chunk) {
+		if (!part_buffer) {
+			// re-initialize the append
+			InitializeAppendState(context.client, op, g);
+		}
+		part_buffer->Append(*part_buffer_append_state, chunk);
+		append_count += chunk.size();
+		if (append_count >= ClientConfig::GetConfig(context.client).partitioned_write_flush_threshold) {
+			// flush all cached partitions
+			FlushPartitions(context, op, g);
+		}
+	}
+
+	void ResetAppendState() {
+		part_buffer_append_state.reset();
+		part_buffer.reset();
+		append_count = 0;
+	}
+
+	void FlushPartitions(ExecutionContext &context, const PhysicalCopyToFile &op, CopyToFunctionGlobalState &g) {
+		if (!part_buffer) {
+			return;
+		}
+		part_buffer->FlushAppendState(*part_buffer_append_state);
+		auto &partitions = part_buffer->GetPartitions();
+		auto partition_key_map = part_buffer->GetReverseMap();
+
+		// ensure all partition directories are created before we start writing
+		g.CreatePartitionDirectories(context.client, op);
+
+		for (idx_t i = 0; i < partitions.size(); i++) {
+			// get the partition write info for this buffer
+			auto &info = g.GetPartitionWriteInfo(context, op, partition_key_map[i]->values);
+
+			auto local_copy_state = op.function.copy_to_initialize_local(context, *op.bind_data);
+			// push the chunks into the write state
+			for (auto &chunk : partitions[i]->Chunks()) {
+				op.function.copy_to_sink(context, *op.bind_data, *info.global_state, *local_copy_state, chunk);
+			}
+			op.function.copy_to_combine(context, *op.bind_data, *info.global_state, *local_copy_state);
+			local_copy_state.reset();
+			partitions[i].reset();
+		}
+		ResetAppendState();
+	}
 };
 
 unique_ptr<GlobalFunctionData> PhysicalCopyToFile::CreateFileState(ClientContext &context,
@@ -60,10 +249,7 @@ unique_ptr<LocalSinkState> PhysicalCopyToFile::GetLocalSinkState(ExecutionContex
 
 		auto state = make_uniq<CopyToFunctionLocalState>(nullptr);
 		state->writer_offset = g.last_file_offset++;
-		state->part_buffer =
-		    make_uniq<HivePartitionedColumnData>(context.client, expected_types, partition_columns, g.partition_state);
-		state->part_buffer_append_state = make_uniq<PartitionedColumnDataAppendState>();
-		state->part_buffer->InitializeAppendState(*state->part_buffer_append_state);
+		state->InitializeAppendState(context.client, *this, g);
 		return std::move(state);
 	}
 	auto res = make_uniq<CopyToFunctionLocalState>(function.copy_to_initialize_local(context, *bind_data));
@@ -139,7 +325,7 @@ SinkResultType PhysicalCopyToFile::Sink(ExecutionContext &context, DataChunk &ch
 	auto &l = input.local_state.Cast<CopyToFunctionLocalState>();
 
 	if (partition_output) {
-		l.part_buffer->Append(*l.part_buffer_append_state, chunk);
+		l.AppendToPartition(context, *this, g, chunk);
 		return SinkResultType::NEED_MORE_INPUT;
 	}
 
@@ -179,79 +365,13 @@ SinkResultType PhysicalCopyToFile::Sink(ExecutionContext &context, DataChunk &ch
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
-static void CreateDir(const string &dir_path, FileSystem &fs) {
-	if (!fs.DirectoryExists(dir_path)) {
-		fs.CreateDirectory(dir_path);
-	}
-}
-
-static void CreateDirectories(const vector<idx_t> &cols, const vector<string> &names, const vector<Value> &values,
-                              string path, FileSystem &fs) {
-	CreateDir(path, fs);
-
-	for (idx_t i = 0; i < cols.size(); i++) {
-		const auto &partition_col_name = names[cols[i]];
-		const auto &partition_value = values[i];
-		string p_dir = partition_col_name + "=" + partition_value.ToString();
-		path = fs.JoinPath(path, p_dir);
-		CreateDir(path, fs);
-	}
-}
-
-static string GetDirectory(const vector<idx_t> &cols, const vector<string> &names, const vector<Value> &values,
-                           string path, FileSystem &fs) {
-	for (idx_t i = 0; i < cols.size(); i++) {
-		const auto &partition_col_name = names[cols[i]];
-		const auto &partition_value = values[i];
-		string p_dir = partition_col_name + "=" + partition_value.ToString();
-		path = fs.JoinPath(path, p_dir);
-	}
-	return path;
-}
-
 SinkCombineResultType PhysicalCopyToFile::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
 	auto &g = input.global_state.Cast<CopyToFunctionGlobalState>();
 	auto &l = input.local_state.Cast<CopyToFunctionLocalState>();
 
 	if (partition_output) {
-		auto &fs = FileSystem::GetFileSystem(context.client);
-		l.part_buffer->FlushAppendState(*l.part_buffer_append_state);
-		auto &partitions = l.part_buffer->GetPartitions();
-		auto partition_key_map = l.part_buffer->GetReverseMap();
-
-		string trimmed_path = file_path;
-		StringUtil::RTrim(trimmed_path, fs.PathSeparator(trimmed_path));
-		{
-			// create directories
-			auto lock = g.lock.GetExclusiveLock();
-			lock_guard<mutex> global_lock_on_partition_state(g.partition_state->lock);
-			const auto &global_partitions = g.partition_state->partitions;
-			// global_partitions have partitions added only at the back, so it's fine to only traverse the last part
-
-			for (idx_t i = g.created_directories; i < global_partitions.size(); i++) {
-				CreateDirectories(partition_columns, names, global_partitions[i]->first.values, trimmed_path, fs);
-			}
-			g.created_directories = global_partitions.size();
-		}
-
-		for (idx_t i = 0; i < partitions.size(); i++) {
-			string hive_path = GetDirectory(partition_columns, names, partition_key_map[i]->values, trimmed_path, fs);
-			string full_path(filename_pattern.CreateFilename(fs, hive_path, file_extension, l.writer_offset));
-			if (fs.FileExists(full_path) && !overwrite_or_ignore) {
-				throw IOException(
-				    "failed to create %s, file exists! Enable OVERWRITE_OR_IGNORE option to force writing", full_path);
-			}
-			// Create a writer for the current file
-			auto fun_data_global = function.copy_to_initialize_global(context.client, *bind_data, full_path);
-			auto fun_data_local = function.copy_to_initialize_local(context, *bind_data);
-
-			for (auto &chunk : partitions[i]->Chunks()) {
-				function.copy_to_sink(context, *bind_data, *fun_data_global, *fun_data_local, chunk);
-			}
-
-			function.copy_to_combine(context, *bind_data, *fun_data_global, *fun_data_local);
-			function.copy_to_finalize(context.client, *bind_data, *fun_data_global);
-		}
+		// flush all remaining partitions
+		l.FlushPartitions(context, *this, g);
 	} else if (function.copy_to_combine) {
 		if (per_thread_output) {
 			// For PER_THREAD_OUTPUT, we can combine/finalize immediately
@@ -272,7 +392,12 @@ SinkCombineResultType PhysicalCopyToFile::Combine(ExecutionContext &context, Ope
 SinkFinalizeType PhysicalCopyToFile::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
                                               OperatorSinkFinalizeInput &input) const {
 	auto &gstate = input.global_state.Cast<CopyToFunctionGlobalState>();
-	if (per_thread_output || partition_output) {
+	if (partition_output) {
+		// finalize any outstanding partitions
+		gstate.FinalizePartitions(context, *this);
+		return SinkFinalizeType::READY;
+	}
+	if (per_thread_output) {
 		// already happened in combine
 		return SinkFinalizeType::READY;
 	}

package/src/duckdb/src/execution/operator/persistent/physical_delete.cpp

@@ -47,6 +47,7 @@ SinkResultType PhysicalDelete::Sink(ExecutionContext &context, DataChunk &chunk,
 
 	lock_guard<mutex> delete_guard(gstate.delete_lock);
 	if (return_chunk) {
+		ustate.delete_chunk.Reset();
 		row_identifiers.Flatten(chunk.size());
 		table.Fetch(transaction, ustate.delete_chunk, column_ids, row_identifiers, chunk.size(), cfs);
 		gstate.return_collection.Append(ustate.delete_chunk);

package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp

@@ -162,6 +162,7 @@ void PhysicalInsert::ResolveDefaults(const TableCatalogEntry &table, DataChunk &
 }
 
 bool AllConflictsMeetCondition(DataChunk &result) {
+	result.Flatten();
 	auto data = FlatVector::GetData<bool>(result.data[0]);
 	for (idx_t i = 0; i < result.size(); i++) {
 		if (!data[i]) {
@@ -236,6 +237,7 @@ static void CreateUpdateChunk(ExecutionContext &context, DataChunk &chunk, Table
 		ExpressionExecutor where_executor(context.client, *do_update_condition);
 		where_executor.Execute(chunk, do_update_filter_result);
 		do_update_filter_result.SetCardinality(chunk.size());
+		do_update_filter_result.Flatten();
 
 		ManagedSelection selection(chunk.size());
 
@@ -431,14 +433,13 @@ SinkResultType PhysicalInsert::Sink(ExecutionContext &context, DataChunk &chunk,
 			gstate.initialized = true;
 		}
 
+		if (return_chunk) {
+			gstate.return_collection.Append(lstate.insert_chunk);
+		}
 		idx_t updated_tuples = OnConflictHandling(table, context, lstate);
 		gstate.insert_count += lstate.insert_chunk.size();
 		gstate.insert_count += updated_tuples;
 		storage.LocalAppend(gstate.append_state, table, context.client, lstate.insert_chunk, true);
-
-		if (return_chunk) {
-			gstate.return_collection.Append(lstate.insert_chunk);
-		}
 	} else {
 		D_ASSERT(!return_chunk);
 		// parallel append
@@ -495,8 +496,8 @@ SinkCombineResultType PhysicalInsert::Combine(ExecutionContext &context, Operato
 		storage.FinalizeLocalAppend(gstate.append_state);
 	} else {
 		// we have written rows to disk optimistically - merge directly into the transaction-local storage
-		gstate.table.GetStorage().FinalizeOptimisticWriter(context.client, *lstate.writer);
 		gstate.table.GetStorage().LocalMerge(context.client, *lstate.local_collection);
+		gstate.table.GetStorage().FinalizeOptimisticWriter(context.client, *lstate.writer);
 	}
 
 	return SinkCombineResultType::FINISHED;

package/src/duckdb/src/execution/operator/persistent/ub_duckdb_operator_persistent.cpp

@@ -0,0 +1,10 @@
+// Unity Build generated by CMake
+#include </Users/carlo/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/persistent/physical_delete.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/persistent/physical_export.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/persistent/physical_insert.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/persistent/physical_update.cpp>

package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp

@@ -33,6 +33,7 @@ PhysicalPivot::PhysicalPivot(vector<LogicalType> types_p, unique_ptr<PhysicalOpe
 OperatorResultType PhysicalPivot::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
                                           GlobalOperatorState &gstate, OperatorState &state) const {
 	// copy the groups as-is
+	input.Flatten();
 	for (idx_t i = 0; i < bound_pivot.group_count; i++) {
 		chunk.data[i].Reference(input.data[i]);
 	}
@@ -68,10 +69,10 @@ OperatorResultType PhysicalPivot::Execute(ExecutionContext &context, DataChunk &
 			for (idx_t aggr = 0; aggr < empty_aggregates.size(); aggr++) {
 				auto pivot_value_lists = FlatVector::GetData<list_entry_t>(input.data[bound_pivot.group_count + aggr]);
 				auto &pivot_value_child = ListVector::GetEntry(input.data[bound_pivot.group_count + aggr]);
-				if (list.offset != pivot_value_lists[r].offset || list.length != pivot_value_lists[r].length) {
+				if (list.length != pivot_value_lists[r].length) {
 					throw InternalException("Pivot - unaligned lists between values and columns!?");
 				}
-				chunk.data[column_idx + aggr].SetValue(r, pivot_value_child.GetValue(list.offset + l));
+				chunk.data[column_idx + aggr].SetValue(r, pivot_value_child.GetValue(pivot_value_lists[r].offset + l));
 			}
 		}
 	}

package/src/duckdb/src/execution/operator/projection/ub_duckdb_operator_projection.cpp

@@ -0,0 +1,5 @@
+// Unity Build generated by CMake
+#include </Users/carlo/duckdb/src/execution/operator/projection/physical_projection.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/projection/physical_tableinout_function.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/projection/physical_pivot.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/projection/physical_unnest.cpp>

package/src/duckdb/src/execution/operator/scan/ub_duckdb_operator_scan.cpp

@@ -0,0 +1,7 @@
+// Unity Build generated by CMake
+#include </Users/carlo/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/scan/physical_dummy_scan.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/scan/physical_empty_result.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/scan/physical_expression_scan.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/scan/physical_positional_scan.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/scan/physical_table_scan.cpp>

package/src/duckdb/src/execution/operator/schema/ub_duckdb_operator_schema.cpp

@@ -0,0 +1,12 @@
+// Unity Build generated by CMake
+#include </Users/carlo/duckdb/src/execution/operator/schema/physical_alter.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/schema/physical_attach.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/schema/physical_create_art_index.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/schema/physical_create_schema.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/schema/physical_create_type.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/schema/physical_create_sequence.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/schema/physical_create_table.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/schema/physical_create_view.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/schema/physical_create_function.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/schema/physical_detach.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/schema/physical_drop.cpp>

package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp

@@ -4,6 +4,7 @@
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/execution/aggregate_hashtable.hpp"
 #include "duckdb/execution/executor.hpp"
+#include "duckdb/execution/operator/scan/physical_column_data_scan.hpp"
 #include "duckdb/parallel/event.hpp"
 #include "duckdb/parallel/meta_pipeline.hpp"
 #include "duckdb/parallel/pipeline.hpp"
@@ -173,6 +174,16 @@ void PhysicalRecursiveCTE::ExecuteRecursivePipelines(ExecutionContext &context)
 //===--------------------------------------------------------------------===//
 // Pipeline Construction
 //===--------------------------------------------------------------------===//
+
+static void GatherColumnDataScans(const PhysicalOperator &op, vector<const_reference<PhysicalOperator>> &delim_scans) {
+	if (op.type == PhysicalOperatorType::DELIM_SCAN || op.type == PhysicalOperatorType::CTE_SCAN) {
+		delim_scans.push_back(op);
+	}
+	for (auto &child : op.children) {
+		GatherColumnDataScans(*child, delim_scans);
+	}
+}
+
 void PhysicalRecursiveCTE::BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) {
 	op_state.reset();
 	sink_state.reset();
@@ -192,6 +203,20 @@ void PhysicalRecursiveCTE::BuildPipelines(Pipeline &current, MetaPipeline &meta_
 	recursive_meta_pipeline = make_shared<MetaPipeline>(executor, state, this);
 	recursive_meta_pipeline->SetRecursiveCTE();
 	recursive_meta_pipeline->Build(*children[1]);
+
+	vector<const_reference<PhysicalOperator>> ops;
+	GatherColumnDataScans(*children[1], ops);
+
+	for (auto op : ops) {
+		auto entry = state.cte_dependencies.find(op);
+		if (entry == state.cte_dependencies.end()) {
+			continue;
+		}
+		// this chunk scan introduces a dependency to the current pipeline
+		// namely a dependency on the CTE pipeline to finish
+		auto cte_dependency = entry->second.get().shared_from_this();
+		current.AddDependency(cte_dependency);
+	}
 }
 
 vector<const_reference<PhysicalOperator>> PhysicalRecursiveCTE::GetSources() const {

package/src/duckdb/src/execution/operator/set/ub_duckdb_operator_set.cpp

@@ -0,0 +1,4 @@
+// Unity Build generated by CMake
+#include </Users/carlo/duckdb/src/execution/operator/set/physical_union.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/set/physical_recursive_cte.cpp>
+#include </Users/carlo/duckdb/src/execution/operator/set/physical_cte.cpp>

package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp

@@ -1,5 +1,5 @@
 #include "duckdb/execution/perfect_aggregate_hashtable.hpp"
-
+#include "duckdb/common/numeric_utils.hpp"
 #include "duckdb/common/row_operations/row_operations.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 
@@ -199,13 +199,13 @@ static void ReconstructGroupVectorTemplated(uint32_t group_values[], Value &min,
 	auto min_data = min.GetValueUnsafe<T>();
 	for (idx_t i = 0; i < entry_count; i++) {
 		// extract the value of this group from the total group index
-		auto group_index = (group_values[i] >> shift) & mask;
+		auto group_index = UnsafeNumericCast<int32_t>((group_values[i] >> shift) & mask);
 		if (group_index == 0) {
 			// if it is 0, the value is NULL
 			validity_mask.SetInvalid(i);
 		} else {
 			// otherwise we add the value (minus 1) to the min value
-			data[i] = min_data + group_index - 1;
+			data[i] = UnsafeNumericCast<T>(min_data + group_index - 1);
 		}
 	}
 }
@@ -254,7 +254,7 @@ void PerfectAggregateHashTable::Scan(idx_t &scan_position, DataChunk &result) {
 		if (group_is_set[scan_position]) {
 			// this group is set: add it to the set of groups to extract
 			data_pointers[entry_count] = data + tuple_size * scan_position;
-			group_values[entry_count] = scan_position;
+			group_values[entry_count] = NumericCast<uint32_t>(scan_position);
 			entry_count++;
 			if (entry_count == STANDARD_VECTOR_SIZE) {
 				scan_position++;

package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp

@@ -20,7 +20,7 @@ static uint32_t RequiredBitsForValue(uint32_t n) {
 		n >>= 1;
 		required_bits++;
 	}
-	return required_bits;
+	return UnsafeNumericCast<uint32_t>(required_bits);
 }
 
 template <class T>
@@ -130,7 +130,7 @@ static bool CanUsePerfectHashAggregate(ClientContext &context, LogicalAggregate
 
 		range += 2;
 		// figure out how many bits we need
-		idx_t required_bits = RequiredBitsForValue(range);
+		idx_t required_bits = RequiredBitsForValue(UnsafeNumericCast<uint32_t>(range));
 		bits_per_group.push_back(required_bits);
 		perfect_hash_bits += required_bits;
 		// check if we have exceeded the bits for the hash

package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp

@@ -82,7 +82,7 @@ void CheckForPerfectJoinOpt(LogicalComparisonJoin &op, PerfectHashJoinStats &joi
 	}
 
 	// and when the build range is smaller than the threshold
-	auto &stats_build = *op.join_stats[0].get(); // lhs stats
+	auto &stats_build = *op.join_stats[1].get(); // rhs stats
 	if (!NumericStats::HasMinMax(stats_build)) {
 		return;
 	}
@@ -97,7 +97,7 @@ void CheckForPerfectJoinOpt(LogicalComparisonJoin &op, PerfectHashJoinStats &joi
 	}
 
 	// Fill join_stats for invisible join
-	auto &stats_probe = *op.join_stats[1].get(); // rhs stats
+	auto &stats_probe = *op.join_stats[0].get(); // lhs stats
 	if (!NumericStats::HasMinMax(stats_probe)) {
 		return;
 	}

package/src/duckdb/src/execution/physical_plan/plan_copy_to_file.cpp

@@ -1,6 +1,5 @@
-#include "duckdb/execution/operator/persistent/physical_batch_copy_to_file.hpp"
 #include "duckdb/execution/operator/persistent/physical_copy_to_file.hpp"
-#include "duckdb/execution/operator/persistent/physical_fixed_batch_copy.hpp"
+#include "duckdb/execution/operator/persistent/physical_batch_copy_to_file.hpp"
 #include "duckdb/execution/physical_plan_generator.hpp"
 #include "duckdb/planner/operator/logical_copy_to_file.hpp"
 
@@ -32,21 +31,12 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCopyToFile
 			throw InternalException("BATCH_COPY_TO_FILE can only be used if batch indexes are supported");
 		}
 		// batched copy to file
-		if (op.function.desired_batch_size) {
-			auto copy = make_uniq<PhysicalFixedBatchCopy>(op.types, op.function, std::move(op.bind_data),
-			                                              op.estimated_cardinality);
-			copy->file_path = op.file_path;
-			copy->use_tmp_file = op.use_tmp_file;
-			copy->children.push_back(std::move(plan));
-			return std::move(copy);
-		} else {
-			auto copy = make_uniq<PhysicalBatchCopyToFile>(op.types, op.function, std::move(op.bind_data),
-			                                               op.estimated_cardinality);
-			copy->file_path = op.file_path;
-			copy->use_tmp_file = op.use_tmp_file;
-			copy->children.push_back(std::move(plan));
-			return std::move(copy);
-		}
+		auto copy = make_uniq<PhysicalBatchCopyToFile>(op.types, op.function, std::move(op.bind_data),
+		                                               op.estimated_cardinality);
+		copy->file_path = op.file_path;
+		copy->use_tmp_file = op.use_tmp_file;
+		copy->children.push_back(std::move(plan));
+		return std::move(copy);
 	}
 	// COPY from select statement to file
 	auto copy = make_uniq<PhysicalCopyToFile>(op.types, op.function, std::move(op.bind_data), op.estimated_cardinality);

package/src/duckdb/src/execution/physical_plan/plan_distinct.cpp

@@ -6,6 +6,7 @@
 #include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/planner/operator/logical_distinct.hpp"
 #include "duckdb/function/function_binder.hpp"
+#include "duckdb/optimizer/rule/ordered_aggregate_optimizer.hpp"
 
 namespace duckdb {
 
@@ -61,6 +62,15 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalDistinct &
 			auto first_aggregate = function_binder.BindAggregateFunction(
 			    FirstFun::GetFunction(logical_type), std::move(first_children), nullptr, AggregateType::NON_DISTINCT);
 			first_aggregate->order_bys = op.order_by ? op.order_by->Copy() : nullptr;
+
+			if (ClientConfig::GetConfig(context).enable_optimizer) {
+				bool changes_made = false;
+				auto new_expr = OrderedAggregateOptimizer::Apply(context, *first_aggregate, groups, changes_made);
+				if (new_expr) {
+					D_ASSERT(new_expr->type == ExpressionType::BOUND_AGGREGATE);
+					first_aggregate = unique_ptr_cast<Expression, BoundAggregateExpression>(std::move(new_expr));
+				}
+			}
 			// add the projection
 			projections.push_back(make_uniq<BoundReferenceExpression>(logical_type, group_count + aggregates.size()));
 			// push it to the list of aggregates

package/src/duckdb/src/execution/physical_plan/plan_insert.cpp

@@ -14,11 +14,18 @@ static OrderPreservationType OrderPreservationRecursive(PhysicalOperator &op) {
 	if (op.IsSource()) {
 		return op.SourceOrder();
 	}
+
+	idx_t child_idx = 0;
 	for (auto &child : op.children) {
+		// Do not take the materialization phase of physical CTEs into account
+		if (op.type == PhysicalOperatorType::CTE && child_idx == 0) {
+			continue;
+		}
 		auto child_preservation = OrderPreservationRecursive(*child);
 		if (child_preservation != OrderPreservationType::INSERTION_ORDER) {
 			return child_preservation;
 		}
+		child_idx++;
 	}
 	return OrderPreservationType::INSERTION_ORDER;
 }