duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp

@@ -1,15 +1,16 @@
 #include "duckdb/execution/operator/persistent/physical_batch_insert.hpp"
+
+#include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
 #include "duckdb/execution/operator/persistent/batch_memory_manager.hpp"
 #include "duckdb/execution/operator/persistent/batch_task_manager.hpp"
 #include "duckdb/parallel/thread_context.hpp"
 #include "duckdb/storage/data_table.hpp"
+#include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/row_group_collection.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/table_io_manager.hpp"
-#include "duckdb/transaction/local_storage.hpp"
-#include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
 #include "duckdb/transaction/duck_transaction.hpp"
-#include "duckdb/storage/table/append_state.hpp"
-#include "duckdb/storage/table/scan_state.hpp"
+#include "duckdb/transaction/local_storage.hpp"
 
 namespace duckdb {
 
@@ -438,7 +439,11 @@ SinkNextBatchType PhysicalBatchInsert::NextBatch(ExecutionContext &context, Oper
 		gstate.AddCollection(context.client, lstate.current_index, lstate.partition_info.min_batch_index.GetIndex(),
 		                     std::move(lstate.current_collection), lstate.writer);
 
-		auto any_unblocked = memory_manager.UnblockTasks();
+		bool any_unblocked;
+		{
+			auto guard = memory_manager.Lock();
+			any_unblocked = memory_manager.UnblockTasks(guard);
+		}
 		if (!any_unblocked) {
 			ExecuteTasks(context.client, gstate, lstate);
 		}
@@ -447,7 +452,8 @@ SinkNextBatchType PhysicalBatchInsert::NextBatch(ExecutionContext &context, Oper
 	lstate.current_index = batch_index;
 
 	// unblock any blocked tasks
-	memory_manager.UnblockTasks();
+	auto guard = memory_manager.Lock();
+	memory_manager.UnblockTasks(guard);
 
 	return SinkNextBatchType::READY;
 }
@@ -475,12 +481,11 @@ SinkResultType PhysicalBatchInsert::Sink(ExecutionContext &context, DataChunk &c
 			// execute tasks while we wait (if any are available)
 			ExecuteTasks(context.client, gstate, lstate);
 
-			lock_guard<mutex> l(memory_manager.GetBlockedTaskLock());
+			auto guard = memory_manager.Lock();
 			if (!memory_manager.IsMinimumBatchIndex(batch_index)) {
 				//  we are not the minimum batch index and we have no memory available to buffer - block the task for
 				//  now
-				memory_manager.BlockTask(input.interrupt_state);
-				return SinkResultType::BLOCKED;
+				return memory_manager.BlockSink(guard, input.interrupt_state);
 			}
 		}
 	}
@@ -518,7 +523,7 @@ SinkCombineResultType PhysicalBatchInsert::Combine(ExecutionContext &context, Op
 	auto &lstate = input.local_state.Cast<BatchInsertLocalState>();
 	auto &memory_manager = gstate.memory_manager;
 	auto &client_profiler = QueryProfiler::Get(context.client);
-	context.thread.profiler.Flush(*this, lstate.default_executor, "default_executor", 1);
+	context.thread.profiler.Flush(*this);
 	client_profiler.Flush(context.thread.profiler);
 
 	memory_manager.UpdateMinBatchIndex(lstate.partition_info.min_batch_index.GetIndex());
@@ -537,7 +542,8 @@ SinkCombineResultType PhysicalBatchInsert::Combine(ExecutionContext &context, Op
 	}
 
 	// unblock any blocked tasks
-	memory_manager.UnblockTasks();
+	auto guard = memory_manager.Lock();
+	memory_manager.UnblockTasks(guard);
 
 	return SinkCombineResultType::FINISHED;
 }

package/src/duckdb/src/execution/operator/persistent/physical_copy_database.cpp

@@ -8,6 +8,7 @@
 #include "duckdb/parser/parsed_data/create_table_info.hpp"
 #include "duckdb/parser/parsed_data/create_type_info.hpp"
 #include "duckdb/parser/parsed_data/create_view_info.hpp"
+#include "duckdb/parser/parsed_data/create_index_info.hpp"
 
 namespace duckdb {
 
@@ -49,7 +50,10 @@ SourceResultType PhysicalCopyDatabase::GetData(ExecutionContext &context, DataCh
 			catalog.CreateTable(context.client, *bound_info);
 			break;
 		}
-		case CatalogType::INDEX_ENTRY:
+		case CatalogType::INDEX_ENTRY: {
+			catalog.CreateIndex(context.client, create_info->Cast<CreateIndexInfo>());
+			break;
+		}
 		default:
 			throw NotImplementedException("Entry type %s not supported in PhysicalCopyDatabase",
 			                              CatalogTypeToString(create_info->type));

package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp

@@ -5,15 +5,17 @@
 #include "duckdb/common/hive_partitioning.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/common/types/uuid.hpp"
-#include "duckdb/common/vector_operations/vector_operations.hpp"
-#include "duckdb/storage/storage_lock.hpp"
 #include "duckdb/common/value_operations/value_operations.hpp"
+#include "duckdb/common/vector_operations/vector_operations.hpp"
+#include "duckdb/planner/operator/logical_copy_to_file.hpp"
+
 #include <algorithm>
 
 namespace duckdb {
 
 struct PartitionWriteInfo {
 	unique_ptr<GlobalFunctionData> global_state;
+	idx_t active_writes = 0;
 };
 
 struct VectorOfValuesHashFunction {
@@ -45,8 +47,9 @@ using vector_of_value_map_t = unordered_map<vector<Value>, T, VectorOfValuesHash
 
 class CopyToFunctionGlobalState : public GlobalSinkState {
 public:
-	explicit CopyToFunctionGlobalState(unique_ptr<GlobalFunctionData> global_state)
+	explicit CopyToFunctionGlobalState(ClientContext &context, unique_ptr<GlobalFunctionData> global_state)
 	    : rows_copied(0), last_file_offset(0), global_state(std::move(global_state)) {
+		max_open_files = ClientConfig::GetConfig(context).partitioned_write_max_open_files;
 	}
 	StorageLock lock;
 	atomic<idx_t> rows_copied;
@@ -56,6 +59,10 @@ public:
 	unordered_set<string> created_directories;
 	//! shared state for HivePartitionedColumnData
 	shared_ptr<GlobalHivePartitionState> partition_state;
+	//! File names
+	vector<Value> file_names;
+	//! Max open files
+	idx_t max_open_files;
 
 	void CreateDir(const string &dir_path, FileSystem &fs) {
 		if (created_directories.find(dir_path) != created_directories.end()) {
@@ -74,13 +81,21 @@ public:
 		for (idx_t i = 0; i < cols.size(); i++) {
 			const auto &partition_col_name = names[cols[i]];
 			const auto &partition_value = values[i];
-			string p_dir = partition_col_name + "=" + partition_value.ToString();
+			string p_dir;
+			p_dir += HivePartitioning::Escape(partition_col_name);
+			p_dir += "=";
+			p_dir += HivePartitioning::Escape(partition_value.ToString());
 			path = fs.JoinPath(path, p_dir);
 			CreateDir(path, fs);
 		}
 		return path;
 	}
 
+	void AddFileName(const StorageLockKey &l, const string &file_name) {
+		D_ASSERT(l.GetType() == StorageLockType::EXCLUSIVE);
+		file_names.emplace_back(file_name);
+	}
+
 	void FinalizePartition(ClientContext &context, const PhysicalCopyToFile &op, PartitionWriteInfo &info) {
 		if (!info.global_state) {
 			// already finalized
@@ -100,30 +115,69 @@ public:
 
 	PartitionWriteInfo &GetPartitionWriteInfo(ExecutionContext &context, const PhysicalCopyToFile &op,
 	                                          const vector<Value> &values) {
-		auto l = lock.GetExclusiveLock();
+		auto global_lock = lock.GetExclusiveLock();
 		// check if we have already started writing this partition
-		auto entry = active_partitioned_writes.find(values);
-		if (entry != active_partitioned_writes.end()) {
+		auto active_write_entry = active_partitioned_writes.find(values);
+		if (active_write_entry != active_partitioned_writes.end()) {
 			// we have - continue writing in this partition
-			return *entry->second;
+			active_write_entry->second->active_writes++;
+			return *active_write_entry->second;
+		}
+		// check if we need to close any writers before we can continue
+		if (active_partitioned_writes.size() >= max_open_files) {
+			// we need to! try to close writers
+			for (auto &entry : active_partitioned_writes) {
+				if (entry.second->active_writes == 0) {
+					// we can evict this entry - evict the partition
+					FinalizePartition(context.client, op, *entry.second);
+					++previous_partitions[entry.first];
+					active_partitioned_writes.erase(entry.first);
+					break;
+				}
+			}
+		}
+		idx_t offset = 0;
+		auto prev_offset = previous_partitions.find(values);
+		if (prev_offset != previous_partitions.end()) {
+			offset = prev_offset->second;
 		}
 		auto &fs = FileSystem::GetFileSystem(context.client);
 		// Create a writer for the current file
 		auto trimmed_path = op.GetTrimmedPath(context.client);
 		string hive_path = GetOrCreateDirectory(op.partition_columns, op.names, values, trimmed_path, fs);
-		string full_path(op.filename_pattern.CreateFilename(fs, hive_path, op.file_extension, 0));
+		string full_path(op.filename_pattern.CreateFilename(fs, hive_path, op.file_extension, offset));
+		if (op.overwrite_mode == CopyOverwriteMode::COPY_APPEND) {
+			// when appending, we first check if the file exists
+			while (fs.FileExists(full_path)) {
+				// file already exists - re-generate name
+				if (!op.filename_pattern.HasUUID()) {
+					throw InternalException("CopyOverwriteMode::COPY_APPEND without {uuid} - and file exists");
+				}
+				full_path = op.filename_pattern.CreateFilename(fs, hive_path, op.file_extension, offset);
+			}
+		}
+		if (op.return_type == CopyFunctionReturnType::CHANGED_ROWS_AND_FILE_LIST) {
+			AddFileName(*global_lock, full_path);
+		}
 		// initialize writes
 		auto info = make_uniq<PartitionWriteInfo>();
 		info->global_state = op.function.copy_to_initialize_global(context.client, *op.bind_data, full_path);
 		auto &result = *info;
+		info->active_writes = 1;
 		// store in active write map
 		active_partitioned_writes.insert(make_pair(values, std::move(info)));
 		return result;
 	}
 
+	void FinishPartitionWrite(PartitionWriteInfo &info) {
+		auto global_lock = lock.GetExclusiveLock();
+		info.active_writes--;
+	}
+
 private:
 	//! The active writes per partition (for partitioned write)
 	vector_of_value_map_t<unique_ptr<PartitionWriteInfo>> active_partitioned_writes;
+	vector_of_value_map_t<idx_t> previous_partitions;
 };
 
 string PhysicalCopyToFile::GetTrimmedPath(ClientContext &context) const {
@@ -175,6 +229,22 @@ public:
 		append_count = 0;
 	}
 
+	void SetDataWithoutPartitions(DataChunk &chunk, const DataChunk &source, const vector<LogicalType> &col_types,
+	                              const vector<idx_t> &part_cols) {
+		D_ASSERT(source.ColumnCount() == col_types.size());
+		auto types = LogicalCopyToFile::GetTypesWithoutPartitions(col_types, part_cols, false);
+		chunk.InitializeEmpty(types);
+		set<idx_t> part_col_set(part_cols.begin(), part_cols.end());
+		idx_t new_col_id = 0;
+		for (idx_t col_idx = 0; col_idx < source.ColumnCount(); col_idx++) {
+			if (part_col_set.find(col_idx) == part_col_set.end()) {
+				chunk.data[new_col_id].Reference(source.data[col_idx]);
+				new_col_id++;
+			}
+		}
+		chunk.SetCardinality(source.size());
+	}
+
 	void FlushPartitions(ExecutionContext &context, const PhysicalCopyToFile &op, CopyToFunctionGlobalState &g) {
 		if (!part_buffer) {
 			return;
@@ -194,22 +264,33 @@ public:
 			auto local_copy_state = op.function.copy_to_initialize_local(context, *op.bind_data);
 			// push the chunks into the write state
 			for (auto &chunk : partitions[i]->Chunks()) {
-				op.function.copy_to_sink(context, *op.bind_data, *info.global_state, *local_copy_state, chunk);
+				if (op.write_partition_columns) {
+					op.function.copy_to_sink(context, *op.bind_data, *info.global_state, *local_copy_state, chunk);
+				} else {
+					DataChunk filtered_chunk;
+					SetDataWithoutPartitions(filtered_chunk, chunk, op.expected_types, op.partition_columns);
+					op.function.copy_to_sink(context, *op.bind_data, *info.global_state, *local_copy_state,
+					                         filtered_chunk);
+				}
 			}
 			op.function.copy_to_combine(context, *op.bind_data, *info.global_state, *local_copy_state);
 			local_copy_state.reset();
 			partitions[i].reset();
+			g.FinishPartitionWrite(info);
 		}
 		ResetAppendState();
 	}
 };
 
-unique_ptr<GlobalFunctionData> PhysicalCopyToFile::CreateFileState(ClientContext &context,
-                                                                   GlobalSinkState &sink) const {
+unique_ptr<GlobalFunctionData> PhysicalCopyToFile::CreateFileState(ClientContext &context, GlobalSinkState &sink,
+                                                                   StorageLockKey &global_lock) const {
 	auto &g = sink.Cast<CopyToFunctionGlobalState>();
 	idx_t this_file_offset = g.last_file_offset++;
 	auto &fs = FileSystem::GetFileSystem(context);
 	string output_path(filename_pattern.CreateFilename(fs, file_path, file_extension, this_file_offset));
+	if (return_type == CopyFunctionReturnType::CHANGED_ROWS_AND_FILE_LIST) {
+		g.AddFileName(global_lock, output_path);
+	}
 	return function.copy_to_initialize_global(context, *bind_data, output_path);
 }
 
@@ -222,14 +303,12 @@ unique_ptr<LocalSinkState> PhysicalCopyToFile::GetLocalSinkState(ExecutionContex
 		return std::move(state);
 	}
 	auto res = make_uniq<CopyToFunctionLocalState>(function.copy_to_initialize_local(context, *bind_data));
-	if (per_thread_output) {
-		res->global_state = CreateFileState(context.client, *sink_state);
-	}
 	return std::move(res);
 }
 
 void CheckDirectory(FileSystem &fs, const string &file_path, CopyOverwriteMode overwrite_mode) {
-	if (overwrite_mode == CopyOverwriteMode::COPY_OVERWRITE_OR_IGNORE) {
+	if (overwrite_mode == CopyOverwriteMode::COPY_OVERWRITE_OR_IGNORE ||
+	    overwrite_mode == CopyOverwriteMode::COPY_APPEND) {
 		// with overwrite or ignore we fully ignore the presence of any files instead of erasing them
 		return;
 	}
@@ -265,8 +344,7 @@ void CheckDirectory(FileSystem &fs, const string &file_path, CopyOverwriteMode o
 }
 
 unique_ptr<GlobalSinkState> PhysicalCopyToFile::GetGlobalSinkState(ClientContext &context) const {
-
-	if (partition_output || per_thread_output || file_size_bytes.IsValid()) {
+	if (partition_output || per_thread_output || rotate) {
 		auto &fs = FileSystem::GetFileSystem(context);
 		if (fs.FileExists(file_path)) {
 			// the target file exists AND is a file (not a directory)
@@ -291,9 +369,10 @@ unique_ptr<GlobalSinkState> PhysicalCopyToFile::GetGlobalSinkState(ClientContext
 			CheckDirectory(fs, file_path, overwrite_mode);
 		}
 
-		auto state = make_uniq<CopyToFunctionGlobalState>(nullptr);
-		if (!per_thread_output && file_size_bytes.IsValid()) {
-			state->global_state = CreateFileState(context, *state);
+		auto state = make_uniq<CopyToFunctionGlobalState>(context, nullptr);
+		if (!per_thread_output && rotate) {
+			auto global_lock = state->lock.GetExclusiveLock();
+			state->global_state = CreateFileState(context, *state, *global_lock);
 		}
 
 		if (partition_output) {
@@ -303,7 +382,15 @@ unique_ptr<GlobalSinkState> PhysicalCopyToFile::GetGlobalSinkState(ClientContext
 		return std::move(state);
 	}
 
-	return make_uniq<CopyToFunctionGlobalState>(function.copy_to_initialize_global(context, *bind_data, file_path));
+	auto state = make_uniq<CopyToFunctionGlobalState>(
+	    context, function.copy_to_initialize_global(context, *bind_data, file_path));
+	if (use_tmp_file) {
+		auto global_lock = state->lock.GetExclusiveLock();
+		state->AddFileName(*global_lock, file_path);
+	} else {
+		state->file_names.emplace_back(file_path);
+	}
+	return std::move(state);
 }
 
 //===--------------------------------------------------------------------===//
@@ -311,6 +398,15 @@ unique_ptr<GlobalSinkState> PhysicalCopyToFile::GetGlobalSinkState(ClientContext
 //===--------------------------------------------------------------------===//
 void PhysicalCopyToFile::MoveTmpFile(ClientContext &context, const string &tmp_file_path) {
 	auto &fs = FileSystem::GetFileSystem(context);
+	auto file_path = GetNonTmpFile(context, tmp_file_path);
+	if (fs.FileExists(file_path)) {
+		fs.RemoveFile(file_path);
+	}
+	fs.MoveFile(tmp_file_path, file_path);
+}
+
+string PhysicalCopyToFile::GetNonTmpFile(ClientContext &context, const string &tmp_file_path) {
+	auto &fs = FileSystem::GetFileSystem(context);
 
 	auto path = StringUtil::GetFilePath(tmp_file_path);
 	auto base = StringUtil::GetFileName(tmp_file_path);
@@ -320,11 +416,7 @@ void PhysicalCopyToFile::MoveTmpFile(ClientContext &context, const string &tmp_f
 		base = base.substr(4);
 	}
 
-	auto file_path = fs.JoinPath(path, base);
-	if (fs.FileExists(file_path)) {
-		fs.RemoveFile(file_path);
-	}
-	fs.MoveFile(tmp_file_path, file_path);
+	return fs.JoinPath(path, base);
 }
 
 PhysicalCopyToFile::PhysicalCopyToFile(vector<LogicalType> types, CopyFunction function_p,
@@ -337,42 +429,46 @@ SinkResultType PhysicalCopyToFile::Sink(ExecutionContext &context, DataChunk &ch
 	auto &g = input.global_state.Cast<CopyToFunctionGlobalState>();
 	auto &l = input.local_state.Cast<CopyToFunctionLocalState>();
 
+	g.rows_copied += chunk.size();
+
 	if (partition_output) {
 		l.AppendToPartition(context, *this, g, chunk);
 		return SinkResultType::NEED_MORE_INPUT;
 	}
 
-	g.rows_copied += chunk.size();
-
 	if (per_thread_output) {
 		auto &gstate = l.global_state;
-		function.copy_to_sink(context, *bind_data, *gstate, *l.local_state, chunk);
-
-		if (file_size_bytes.IsValid() && function.file_size_bytes(*gstate) > file_size_bytes.GetIndex()) {
+		if (!gstate) {
+			// Lazily create file state here to prevent creating empty files
+			auto global_lock = g.lock.GetExclusiveLock();
+			gstate = CreateFileState(context.client, *sink_state, *global_lock);
+		} else if (rotate && function.rotate_next_file(*gstate, *bind_data, file_size_bytes)) {
 			function.copy_to_finalize(context.client, *bind_data, *gstate);
-			gstate = CreateFileState(context.client, *sink_state);
+			auto global_lock = g.lock.GetExclusiveLock();
+			gstate = CreateFileState(context.client, *sink_state, *global_lock);
 		}
+		function.copy_to_sink(context, *bind_data, *gstate, *l.local_state, chunk);
 		return SinkResultType::NEED_MORE_INPUT;
 	}
 
-	if (!file_size_bytes.IsValid()) {
+	if (!file_size_bytes.IsValid() && !rotate) {
 		function.copy_to_sink(context, *bind_data, *g.global_state, *l.local_state, chunk);
 		return SinkResultType::NEED_MORE_INPUT;
 	}
 
-	// FILE_SIZE_BYTES is set, but threads write to the same file, synchronize using lock
+	// FILE_SIZE_BYTES/rotate is set, but threads write to the same file, synchronize using lock
 	auto &gstate = g.global_state;
-	auto lock = g.lock.GetExclusiveLock();
-	if (function.file_size_bytes(*gstate) > file_size_bytes.GetIndex()) {
+	auto global_lock = g.lock.GetExclusiveLock();
+	if (rotate && function.rotate_next_file(*gstate, *bind_data, file_size_bytes)) {
 		auto owned_gstate = std::move(gstate);
-		gstate = CreateFileState(context.client, *sink_state);
-		lock.reset();
+		gstate = CreateFileState(context.client, *sink_state, *global_lock);
+		global_lock.reset();
 		function.copy_to_finalize(context.client, *bind_data, *owned_gstate);
 	} else {
-		lock.reset();
+		global_lock.reset();
 	}
 
-	lock = g.lock.GetSharedLock();
+	global_lock = g.lock.GetSharedLock();
 	function.copy_to_sink(context, *bind_data, *gstate, *l.local_state, chunk);
 
 	return SinkResultType::NEED_MORE_INPUT;
@@ -387,11 +483,13 @@ SinkCombineResultType PhysicalCopyToFile::Combine(ExecutionContext &context, Ope
 		l.FlushPartitions(context, *this, g);
 	} else if (function.copy_to_combine) {
 		if (per_thread_output) {
-			// For PER_THREAD_OUTPUT, we can combine/finalize immediately
-			function.copy_to_combine(context, *bind_data, *l.global_state, *l.local_state);
-			function.copy_to_finalize(context.client, *bind_data, *l.global_state);
-		} else if (file_size_bytes.IsValid()) {
-			// File in global state may change with FILE_SIZE_BYTES, need to grab lock
+			// For PER_THREAD_OUTPUT, we can combine/finalize immediately (if there is a gstate)
+			if (l.global_state) {
+				function.copy_to_combine(context, *bind_data, *l.global_state, *l.local_state);
+				function.copy_to_finalize(context.client, *bind_data, *l.global_state);
+			}
+		} else if (rotate) {
+			// File in global state may change with FILE_SIZE_BYTES/rotate, need to grab lock
 			auto lock = g.lock.GetSharedLock();
 			function.copy_to_combine(context, *bind_data, *g.global_state, *l.local_state);
 		} else {
@@ -421,6 +519,7 @@ SinkFinalizeType PhysicalCopyToFile::Finalize(Pipeline &pipeline, Event &event,
 			D_ASSERT(!per_thread_output);
 			D_ASSERT(!partition_output);
 			D_ASSERT(!file_size_bytes.IsValid());
+			D_ASSERT(!rotate);
 			MoveTmpFile(context, file_path);
 		}
 	}
@@ -436,7 +535,17 @@ SourceResultType PhysicalCopyToFile::GetData(ExecutionContext &context, DataChun
 	auto &g = sink_state->Cast<CopyToFunctionGlobalState>();
 
 	chunk.SetCardinality(1);
-	chunk.SetValue(0, 0, Value::BIGINT(NumericCast<int64_t>(g.rows_copied.load())));
+	switch (return_type) {
+	case CopyFunctionReturnType::CHANGED_ROWS:
+		chunk.SetValue(0, 0, Value::BIGINT(NumericCast<int64_t>(g.rows_copied.load())));
+		break;
+	case CopyFunctionReturnType::CHANGED_ROWS_AND_FILE_LIST:
+		chunk.SetValue(0, 0, Value::BIGINT(NumericCast<int64_t>(g.rows_copied.load())));
+		chunk.SetValue(1, 0, Value::LIST(LogicalType::VARCHAR, g.file_names));
+		break;
+	default:
+		throw NotImplementedException("Unknown CopyFunctionReturnType");
+	}
 
 	return SourceResultType::FINISHED;
 }

package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp

@@ -446,10 +446,18 @@ SinkResultType PhysicalInsert::Sink(ExecutionContext &context, DataChunk &chunk,
 			gstate.initialized = true;
 		}
 
-		if (return_chunk) {
+		if (action_type != OnConflictAction::NOTHING && return_chunk) {
+			// If the action is UPDATE or REPLACE, we will always create either an APPEND or an INSERT
+			// for NOTHING we don't create either an APPEND or an INSERT for the tuple
+			// so it should not be added to the RETURNING chunk
 			gstate.return_collection.Append(lstate.insert_chunk);
 		}
 		idx_t updated_tuples = OnConflictHandling(table, context, lstate);
+		if (action_type == OnConflictAction::NOTHING && return_chunk) {
+			// Because we didn't add to the RETURNING chunk yet
+			// we add the tuples that did not get filtered out now
+			gstate.return_collection.Append(lstate.insert_chunk);
+		}
 		gstate.insert_count += lstate.insert_chunk.size();
 		gstate.insert_count += updated_tuples;
 		storage.LocalAppend(gstate.append_state, table, context.client, lstate.insert_chunk, true);
@@ -488,7 +496,7 @@ SinkCombineResultType PhysicalInsert::Combine(ExecutionContext &context, Operato
 	auto &gstate = input.global_state.Cast<InsertGlobalState>();
 	auto &lstate = input.local_state.Cast<InsertLocalState>();
 	auto &client_profiler = QueryProfiler::Get(context.client);
-	context.thread.profiler.Flush(*this, lstate.default_executor, "default_executor", 1);
+	context.thread.profiler.Flush(*this);
 	client_profiler.Flush(context.thread.profiler);
 
 	if (!parallel || !lstate.local_collection) {

package/src/duckdb/src/execution/operator/persistent/physical_update.cpp

@@ -166,11 +166,9 @@ unique_ptr<LocalSinkState> PhysicalUpdate::GetLocalSinkState(ExecutionContext &c
 }
 
 SinkCombineResultType PhysicalUpdate::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
-	auto &state = input.local_state.Cast<UpdateLocalState>();
 	auto &client_profiler = QueryProfiler::Get(context.client);
-	context.thread.profiler.Flush(*this, state.default_executor, "default_executor", 1);
+	context.thread.profiler.Flush(*this);
 	client_profiler.Flush(context.thread.profiler);
-
 	return SinkCombineResultType::FINISHED;
 }
 

package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp

@@ -20,8 +20,8 @@ PhysicalPivot::PhysicalPivot(vector<LogicalType> types_p, unique_ptr<PhysicalOpe
 	for (auto &aggr_expr : bound_pivot.aggregates) {
 		auto &aggr = aggr_expr->Cast<BoundAggregateExpression>();
 		// for each aggregate, initialize an empty aggregate state and finalize it immediately
-		auto state = make_unsafe_uniq_array<data_t>(aggr.function.state_size());
-		aggr.function.initialize(state.get());
+		auto state = make_unsafe_uniq_array<data_t>(aggr.function.state_size(aggr.function));
+		aggr.function.initialize(aggr.function, state.get());
 		Vector state_vector(Value::POINTER(CastPointerToValue(state.get())));
 		Vector result_vector(aggr_expr->return_type);
 		AggregateInputData aggr_input_data(aggr.bind_info.get(), allocator);

package/src/duckdb/src/execution/operator/projection/physical_projection.cpp

@@ -15,7 +15,7 @@ public:
 
 public:
 	void Finalize(const PhysicalOperator &op, ExecutionContext &context) override {
-		context.thread.profiler.Flush(op, executor, "projection", 0);
+		context.thread.profiler.Flush(op);
 	}
 };
 
@@ -69,12 +69,19 @@ PhysicalProjection::CreateJoinProjection(vector<LogicalType> proj_types, const v
 	return make_uniq<PhysicalProjection>(std::move(proj_types), std::move(proj_selects), estimated_cardinality);
 }
 
-string PhysicalProjection::ParamsToString() const {
-	string extra_info;
-	for (auto &expr : select_list) {
-		extra_info += expr->GetName() + "\n";
+InsertionOrderPreservingMap<string> PhysicalProjection::ParamsToString() const {
+	InsertionOrderPreservingMap<string> result;
+	string projections;
+	for (idx_t i = 0; i < select_list.size(); i++) {
+		if (i > 0) {
+			projections += "\n";
+		}
+		auto &expr = select_list[i];
+		projections += expr->GetName();
 	}
-	return extra_info;
+	result["__projections__"] = projections;
+	SetEstimatedCardinality(result, estimated_cardinality);
+	return result;
 }
 
 } // namespace duckdb

package/src/duckdb/src/execution/operator/projection/physical_tableinout_function.cpp

@@ -38,7 +38,16 @@ unique_ptr<OperatorState> PhysicalTableInOutFunction::GetOperatorState(Execution
 		result->local_state = function.init_local(context, input, gstate.global_state.get());
 	}
 	if (!projected_input.empty()) {
-		result->input_chunk.Initialize(context.client, children[0]->types);
+		vector<LogicalType> input_types;
+		auto &child_types = children[0]->types;
+		idx_t input_length = child_types.size() - projected_input.size();
+		for (idx_t k = 0; k < input_length; k++) {
+			input_types.push_back(child_types[k]);
+		}
+		for (idx_t k = 0; k < projected_input.size(); k++) {
+			D_ASSERT(projected_input[k] >= input_length);
+		}
+		result->input_chunk.Initialize(context.client, input_types);
 	}
 	return std::move(result);
 }
@@ -71,9 +80,8 @@ OperatorResultType PhysicalTableInOutFunction::Execute(ExecutionContext &context
 		}
 		// we are processing a new row: fetch the data for the current row
 		state.input_chunk.Reset();
-		D_ASSERT(input.ColumnCount() == state.input_chunk.ColumnCount());
 		// set up the input data to the table in-out function
-		for (idx_t col_idx = 0; col_idx < input.ColumnCount(); col_idx++) {
+		for (idx_t col_idx = 0; col_idx < state.input_chunk.ColumnCount(); col_idx++) {
 			ConstantVector::Reference(state.input_chunk.data[col_idx], input.data[col_idx], state.row_index, 1);
 		}
 		state.input_chunk.SetCardinality(1);
@@ -100,6 +108,17 @@ OperatorResultType PhysicalTableInOutFunction::Execute(ExecutionContext &context
 	return OperatorResultType::HAVE_MORE_OUTPUT;
 }
 
+InsertionOrderPreservingMap<string> PhysicalTableInOutFunction::ParamsToString() const {
+	InsertionOrderPreservingMap<string> result;
+	if (function.to_string) {
+		result["__text__"] = function.to_string(bind_data.get());
+	} else {
+		result["Name"] = function.name;
+	}
+	SetEstimatedCardinality(result, estimated_cardinality);
+	return result;
+}
+
 OperatorFinalizeResultType PhysicalTableInOutFunction::FinalExecute(ExecutionContext &context, DataChunk &chunk,
                                                                     GlobalOperatorState &gstate_p,
                                                                     OperatorState &state_p) const {

package/src/duckdb/src/execution/operator/projection/physical_unnest.cpp

@@ -88,11 +88,17 @@ static void UnnestNull(idx_t start, idx_t end, Vector &result) {
 	for (idx_t i = start; i < end; i++) {
 		validity.SetInvalid(i);
 	}
-	if (result.GetType().InternalType() == PhysicalType::STRUCT) {
-		auto &struct_children = StructVector::GetEntries(result);
+
+	const auto &logical_type = result.GetType();
+	if (logical_type.InternalType() == PhysicalType::STRUCT) {
+		const auto &struct_children = StructVector::GetEntries(result);
 		for (auto &child : struct_children) {
 			UnnestNull(start, end, *child);
 		}
+	} else if (logical_type.InternalType() == PhysicalType::ARRAY) {
+		auto &array_child = ArrayVector::GetEntry(result);
+		auto array_size = ArrayType::GetSize(logical_type);
+		UnnestNull(start * array_size, end * array_size, array_child);
 	}
 }
 
@@ -205,7 +211,17 @@ static void UnnestVector(UnifiedVectorFormat &child_vector_data, Vector &child_v
 		break;
 	}
 	case PhysicalType::ARRAY: {
-		throw NotImplementedException("ARRAY type not supported for UNNEST.");
+		auto array_size = ArrayType::GetSize(child_vector.GetType());
+		auto &source_array = ArrayVector::GetEntry(child_vector);
+		auto &target_array = ArrayVector::GetEntry(result);
+
+		UnnestValidity(child_vector_data, start, end, result);
+
+		UnifiedVectorFormat child_array_data;
+		source_array.ToUnifiedFormat(list_size * array_size, child_array_data);
+		UnnestVector(child_array_data, source_array, list_size * array_size, start * array_size, end * array_size,
+		             target_array);
+		break;
 	}
 	default:
 		throw InternalException("Unimplemented type for UNNEST.");