duckdb 0.7.2-dev2995.0 → 0.7.2-dev3117.0

package/src/duckdb/src/common/types.cpp

@@ -28,7 +28,7 @@
 #include "duckdb/parser/parser.hpp"
 
 #include "duckdb/common/serializer/format_deserializer.hpp"
-#include "duckdb/common/serializer/enum_serializer.hpp"
+#include "duckdb/common/enum_util.hpp"
 #include "duckdb/common/serializer/format_serializer.hpp"
 
 #include <cmath>
@@ -158,6 +158,11 @@ PhysicalType LogicalType::GetInternalType() {
 	}
 }
 
+// **DEPRECATED**: Use EnumUtil directly instead.
+string LogicalTypeIdToString(LogicalTypeId type) {
+	return EnumUtil::ToString(type);
+}
+
 constexpr const LogicalTypeId LogicalType::INVALID;
 constexpr const LogicalTypeId LogicalType::SQLNULL;
 constexpr const LogicalTypeId LogicalType::BOOLEAN;
@@ -331,10 +336,6 @@ bool TypeIsInteger(PhysicalType type) {
 	return (type >= PhysicalType::UINT8 && type <= PhysicalType::INT64) || type == PhysicalType::INT128;
 }
 
-string LogicalTypeIdToString(LogicalTypeId id) {
-	return EnumSerializer::EnumToString(id);
-}
-
 string LogicalType::ToString() const {
 	auto alias = GetAlias();
 	if (!alias.empty()) {
@@ -406,7 +407,7 @@ string LogicalType::ToString() const {
 		return AggregateStateType::GetTypeName(*this);
 	}
 	default:
-		return LogicalTypeIdToString(id_);
+		return EnumUtil::ToString(id_);
 	}
 }
 // LCOV_EXCL_STOP
@@ -1566,17 +1567,18 @@ void ExtraTypeInfo::Serialize(ExtraTypeInfo *info, FieldWriter &writer) {
 }
 void ExtraTypeInfo::FormatSerialize(FormatSerializer &serializer) const {
 	serializer.WriteProperty("type", type);
+	// BREAKING: we used to write the alias last if there was additional type info, but now we write it second.
 	serializer.WriteProperty("alias", alias);
 }
 
 shared_ptr<ExtraTypeInfo> ExtraTypeInfo::FormatDeserialize(FormatDeserializer &deserializer) {
 	auto type = deserializer.ReadProperty<ExtraTypeInfoType>("type");
+	auto alias = deserializer.ReadProperty<string>("alias");
+	// BREAKING: we used to read the alias last, but now we read it second.
 
 	shared_ptr<ExtraTypeInfo> result;
 	switch (type) {
 	case ExtraTypeInfoType::INVALID_TYPE_INFO: {
-		string alias;
-		deserializer.ReadOptionalProperty("alias", alias);
 		if (!alias.empty()) {
 			return make_shared<ExtraTypeInfo>(type, alias);
 		}
@@ -1620,11 +1622,10 @@ shared_ptr<ExtraTypeInfo> ExtraTypeInfo::FormatDeserialize(FormatDeserializer &d
 	case ExtraTypeInfoType::AGGREGATE_STATE_TYPE_INFO:
 		result = AggregateStateTypeInfo::FormatDeserialize(deserializer);
 		break;
-
 	default:
 		throw InternalException("Unimplemented type info in ExtraTypeInfo::Deserialize");
 	}
-	deserializer.ReadOptionalPropertyOrDefault("alias", result->alias, string());
+	result->alias = alias;
 	return result;
 }
 

package/src/duckdb/src/common/vector_operations/is_distinct_from.cpp

@@ -900,15 +900,15 @@ idx_t VectorOperations::DistinctLessThan(Vector &left, Vector &right, const Sele
 // true := A < B with nulls being minimal
 idx_t VectorOperations::DistinctLessThanNullsFirst(Vector &left, Vector &right, const SelectionVector *sel, idx_t count,
                                                    SelectionVector *true_sel, SelectionVector *false_sel) {
-	return TemplatedDistinctSelectOperation<duckdb::DistinctLessThanNullsFirst, duckdb::DistinctLessThan>(
-	    left, right, sel, count, true_sel, false_sel);
+	return TemplatedDistinctSelectOperation<duckdb::DistinctGreaterThanNullsFirst, duckdb::DistinctGreaterThan>(
+	    right, left, sel, count, true_sel, false_sel);
 }
 
 // true := A <= B with nulls being maximal
 idx_t VectorOperations::DistinctLessThanEquals(Vector &left, Vector &right, const SelectionVector *sel, idx_t count,
                                                SelectionVector *true_sel, SelectionVector *false_sel) {
-	return TemplatedDistinctSelectOperation<duckdb::DistinctGreaterThanEquals>(right, left, sel, count, true_sel,
-	                                                                           false_sel);
+	return count -
+	       TemplatedDistinctSelectOperation<duckdb::DistinctGreaterThan>(left, right, sel, count, false_sel, true_sel);
 }
 
 // true := A != B with nulls being equal, inputs selected

package/src/duckdb/src/core_functions/scalar/date/date_part.cpp

@@ -3,6 +3,7 @@
 #include "duckdb/common/enums/date_part_specifier.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/string_util.hpp"
+#include "duckdb/common/enum_util.hpp"
 #include "duckdb/common/types/date.hpp"
 #include "duckdb/common/types/timestamp.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
@@ -81,7 +82,7 @@ DatePartSpecifier GetDateTypePartSpecifier(const string &specifier, LogicalType
 		break;
 	}
 
-	throw NotImplementedException("\"%s\" units \"%s\" not recognized", LogicalTypeIdToString(type.id()), specifier);
+	throw NotImplementedException("\"%s\" units \"%s\" not recognized", EnumUtil::ToString(type.id()), specifier);
 }
 
 template <int64_t MIN, int64_t MAX>

package/src/duckdb/src/core_functions/scalar/list/list_sort.cpp

@@ -1,11 +1,10 @@
 #include "duckdb/core_functions/scalar/list_functions.hpp"
-#include "duckdb/common/serializer/enum_serializer.hpp"
+#include "duckdb/common/enum_util.hpp"
 #include "duckdb/common/types/chunk_collection.hpp"
 #include "duckdb/planner/expression/bound_function_expression.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/main/config.hpp"
-
 #include "duckdb/common/sort/sort.hpp"
 
 namespace duckdb {
@@ -251,7 +250,7 @@ static T GetOrder(ClientContext &context, Expression &expr) {
 	}
 	Value order_value = ExpressionExecutor::EvaluateScalar(context, expr);
 	auto order_name = StringUtil::Upper(order_value.ToString());
-	return EnumSerializer::StringToEnum<T>(order_name.c_str());
+	return EnumUtil::FromString<T>(order_name.c_str());
 }
 
 static unique_ptr<FunctionData> ListNormalSortBind(ClientContext &context, ScalarFunction &bound_function,

package/src/duckdb/src/execution/aggregate_hashtable.cpp

@@ -236,7 +236,7 @@ void GroupedAggregateHashTable::Resize(idx_t size) {
 
 idx_t GroupedAggregateHashTable::AddChunk(AggregateHTAppendState &state, DataChunk &groups, DataChunk &payload,
                                           AggregateType filter) {
-	vector<idx_t> aggregate_filter;
+	unsafe_vector<idx_t> aggregate_filter;
 
 	auto &aggregates = layout.GetAggregates();
 	for (idx_t i = 0; i < aggregates.size(); i++) {
@@ -249,7 +249,7 @@ idx_t GroupedAggregateHashTable::AddChunk(AggregateHTAppendState &state, DataChu
 }
 
 idx_t GroupedAggregateHashTable::AddChunk(AggregateHTAppendState &state, DataChunk &groups, DataChunk &payload,
-                                          const vector<idx_t> &filter) {
+                                          const unsafe_vector<idx_t> &filter) {
 	Vector hashes(LogicalType::HASH);
 	groups.Hash(hashes);
 
@@ -257,7 +257,7 @@ idx_t GroupedAggregateHashTable::AddChunk(AggregateHTAppendState &state, DataChu
 }
 
 idx_t GroupedAggregateHashTable::AddChunk(AggregateHTAppendState &state, DataChunk &groups, Vector &group_hashes,
-                                          DataChunk &payload, const vector<idx_t> &filter) {
+                                          DataChunk &payload, const unsafe_vector<idx_t> &filter) {
 	D_ASSERT(!is_finalized);
 	if (groups.size() == 0) {
 		return 0;

package/src/duckdb/src/execution/operator/aggregate/distinct_aggregate_data.cpp

@@ -171,7 +171,7 @@ bool DistinctAggregateCollectionInfo::AnyDistinct() const {
 	return !indices.empty();
 }
 
-const vector<idx_t> &DistinctAggregateCollectionInfo::Indices() const {
+const unsafe_vector<idx_t> &DistinctAggregateCollectionInfo::Indices() const {
 	return this->indices;
 }
 

package/src/duckdb/src/execution/operator/aggregate/grouped_aggregate_data.cpp

@@ -12,7 +12,7 @@ const vector<vector<idx_t>> &GroupedAggregateData::GetGroupingFunctions() const
 
 void GroupedAggregateData::InitializeGroupby(vector<unique_ptr<Expression>> groups,
                                              vector<unique_ptr<Expression>> expressions,
-                                             vector<vector<idx_t>> grouping_functions) {
+                                             vector<unsafe_vector<idx_t>> grouping_functions) {
 	InitializeGroupbyGroups(std::move(groups));
 	vector<LogicalType> payload_types_filters;
 
@@ -86,7 +86,7 @@ void GroupedAggregateData::InitializeGroupbyGroups(vector<unique_ptr<Expression>
 	this->groups = std::move(groups);
 }
 
-void GroupedAggregateData::SetGroupingFunctions(vector<vector<idx_t>> &functions) {
+void GroupedAggregateData::SetGroupingFunctions(vector<unsafe_vector<idx_t>> &functions) {
 	grouping_functions.reserve(functions.size());
 	for (idx_t i = 0; i < functions.size(); i++) {
 		grouping_functions.push_back(std::move(functions[i]));

package/src/duckdb/src/execution/operator/aggregate/physical_hash_aggregate.cpp

@@ -117,7 +117,8 @@ PhysicalHashAggregate::PhysicalHashAggregate(ClientContext &context, vector<Logi
                                              vector<unique_ptr<Expression>> expressions,
                                              vector<unique_ptr<Expression>> groups_p,
                                              vector<GroupingSet> grouping_sets_p,
-                                             vector<vector<idx_t>> grouping_functions_p, idx_t estimated_cardinality)
+                                             vector<unsafe_vector<idx_t>> grouping_functions_p,
+                                             idx_t estimated_cardinality)
     : PhysicalOperator(PhysicalOperatorType::HASH_GROUP_BY, std::move(types), estimated_cardinality),
       grouping_sets(std::move(grouping_sets_p)) {
 	// get a list of all aggregates to be computed
@@ -266,7 +267,7 @@ void PhysicalHashAggregate::SinkDistinctGrouping(ExecutionContext &context, Data
 	DataChunk empty_chunk;
 
 	// Create an empty filter for Sink, since we don't need to update any aggregate states here
-	vector<idx_t> empty_filter;
+	unsafe_vector<idx_t> empty_filter;
 
 	for (idx_t &idx : distinct_info.indices) {
 		auto &aggregate = grouped_aggregate_data.aggregates[idx]->Cast<BoundAggregateExpression>();

package/src/duckdb/src/execution/operator/helper/physical_streaming_sample.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/execution/operator/helper/physical_streaming_sample.hpp"
 #include "duckdb/common/random_engine.hpp"
 #include "duckdb/common/to_string.hpp"
+#include "duckdb/common/enum_util.hpp"
 
 namespace duckdb {
 
@@ -68,7 +69,7 @@ OperatorResultType PhysicalStreamingSample::Execute(ExecutionContext &context, D
 }
 
 string PhysicalStreamingSample::ParamsToString() const {
-	return SampleMethodToString(method) + ": " + to_string(100 * percentage) + "%";
+	return EnumUtil::ToString(method) + ": " + to_string(100 * percentage) + "%";
 }
 
 } // namespace duckdb

package/src/duckdb/src/execution/operator/join/physical_blockwise_nl_join.cpp

@@ -6,6 +6,7 @@
 #include "duckdb/execution/operator/join/outer_join_marker.hpp"
 #include "duckdb/execution/operator/join/physical_comparison_join.hpp"
 #include "duckdb/execution/operator/join/physical_cross_product.hpp"
+#include "duckdb/common/enum_util.hpp"
 
 namespace duckdb {
 
@@ -200,7 +201,7 @@ OperatorResultType PhysicalBlockwiseNLJoin::ExecuteInternal(ExecutionContext &co
 }
 
 string PhysicalBlockwiseNLJoin::ParamsToString() const {
-	string extra_info = JoinTypeToString(join_type) + "\n";
+	string extra_info = EnumUtil::ToString(join_type) + "\n";
 	extra_info += condition->GetName();
 	return extra_info;
 }

package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp

@@ -1,5 +1,6 @@
 #include "duckdb/execution/operator/join/physical_comparison_join.hpp"
 #include "duckdb/common/types/chunk_collection.hpp"
+#include "duckdb/common/enum_util.hpp"
 
 namespace duckdb {
 
@@ -24,7 +25,7 @@ PhysicalComparisonJoin::PhysicalComparisonJoin(LogicalOperator &op, PhysicalOper
 }
 
 string PhysicalComparisonJoin::ParamsToString() const {
-	string extra_info = JoinTypeToString(join_type) + "\n";
+	string extra_info = EnumUtil::ToString(join_type) + "\n";
 	for (auto &it : conditions) {
 		string op = ExpressionTypeToOperator(it.comparison);
 		extra_info += it.left->GetName() + " " + op + " " + it.right->GetName() + "\n";

package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp

@@ -0,0 +1,165 @@
+#include "duckdb/execution/operator/persistent/physical_batch_copy_to_file.hpp"
+#include "duckdb/execution/operator/persistent/physical_copy_to_file.hpp"
+#include "duckdb/common/vector_operations/vector_operations.hpp"
+#include "duckdb/common/types/batched_data_collection.hpp"
+#include "duckdb/common/file_system.hpp"
+#include "duckdb/common/file_opener.hpp"
+#include "duckdb/common/allocator.hpp"
+#include <algorithm>
+
+namespace duckdb {
+
+PhysicalBatchCopyToFile::PhysicalBatchCopyToFile(vector<LogicalType> types, CopyFunction function_p,
+                                                 unique_ptr<FunctionData> bind_data, idx_t estimated_cardinality)
+    : PhysicalOperator(PhysicalOperatorType::BATCH_COPY_TO_FILE, std::move(types), estimated_cardinality),
+      function(std::move(function_p)), bind_data(std::move(bind_data)) {
+	if (!function.flush_batch || !function.prepare_batch) {
+		throw InternalException(
+		    "PhysicalBatchCopyToFile created for copy function that does not have prepare_batch/flush_batch defined");
+	}
+}
+
+//===--------------------------------------------------------------------===//
+// Sink
+//===--------------------------------------------------------------------===//
+class BatchCopyToGlobalState : public GlobalSinkState {
+public:
+	explicit BatchCopyToGlobalState(unique_ptr<GlobalFunctionData> global_state)
+	    : rows_copied(0), global_state(std::move(global_state)) {
+	}
+
+	mutex lock;
+	mutex flush_lock;
+	atomic<idx_t> rows_copied;
+	unique_ptr<GlobalFunctionData> global_state;
+	map<idx_t, unique_ptr<PreparedBatchData>> batch_data;
+};
+
+class BatchCopyToLocalState : public LocalSinkState {
+public:
+	explicit BatchCopyToLocalState(unique_ptr<LocalFunctionData> local_state_p)
+	    : local_state(std::move(local_state_p)), rows_copied(0), batch_index(0) {
+	}
+
+	unique_ptr<LocalFunctionData> local_state;
+	unique_ptr<ColumnDataCollection> collection;
+	ColumnDataAppendState append_state;
+	idx_t rows_copied;
+	idx_t batch_index;
+
+	void InitializeCollection(ClientContext &context, const PhysicalOperator &op) {
+		collection = make_uniq<ColumnDataCollection>(Allocator::Get(context), op.children[0]->types);
+		collection->InitializeAppend(append_state);
+	}
+};
+
+SinkResultType PhysicalBatchCopyToFile::Sink(ExecutionContext &context, DataChunk &chunk,
+                                             OperatorSinkInput &input) const {
+	auto &state = input.local_state.Cast<BatchCopyToLocalState>();
+	if (!state.collection) {
+		state.InitializeCollection(context.client, *this);
+	}
+	state.rows_copied += chunk.size();
+	state.collection->Append(state.append_state, chunk);
+	return SinkResultType::NEED_MORE_INPUT;
+}
+
+void PhysicalBatchCopyToFile::Combine(ExecutionContext &context, GlobalSinkState &gstate_p,
+                                      LocalSinkState &lstate) const {
+	auto &state = lstate.Cast<BatchCopyToLocalState>();
+	auto &gstate = gstate_p.Cast<BatchCopyToGlobalState>();
+	gstate.rows_copied += state.rows_copied;
+}
+
+SinkFinalizeType PhysicalBatchCopyToFile::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
+                                                   GlobalSinkState &gstate_p) const {
+	auto &gstate = gstate_p.Cast<BatchCopyToGlobalState>();
+	FlushBatchData(context, gstate_p, NumericLimits<int64_t>::Maximum());
+	if (function.copy_to_finalize) {
+		function.copy_to_finalize(context, *bind_data, *gstate.global_state);
+
+		if (use_tmp_file) {
+			PhysicalCopyToFile::MoveTmpFile(context, file_path);
+		}
+	}
+	return SinkFinalizeType::READY;
+}
+
+void PhysicalBatchCopyToFile::PrepareBatchData(ClientContext &context, GlobalSinkState &gstate_p, idx_t batch_index,
+                                               unique_ptr<ColumnDataCollection> collection) const {
+	auto &gstate = gstate_p.Cast<BatchCopyToGlobalState>();
+
+	// prepare the batch
+	auto batch_data = function.prepare_batch(context, *bind_data, *gstate.global_state, std::move(collection));
+	// move the batch data to the set of prepared batch data
+	lock_guard<mutex> l(gstate.lock);
+	gstate.batch_data[batch_index] = std::move(batch_data);
+}
+
+void PhysicalBatchCopyToFile::FlushBatchData(ClientContext &context, GlobalSinkState &gstate_p, idx_t min_index) const {
+	auto &gstate = gstate_p.Cast<BatchCopyToGlobalState>();
+
+	// flush batch data to disk (if there are any to flush)
+	while (true) {
+		// grab the flush lock - we can only call flush_batch with this lock
+		// otherwise the data might end up in the wrong order
+		lock_guard<mutex> l(gstate.flush_lock);
+		unique_ptr<PreparedBatchData> batch_data;
+		{
+			// fetch the next batch to flush (if any)
+			lock_guard<mutex> l(gstate.lock);
+			if (gstate.batch_data.empty()) {
+				// no batch data left to flush
+				break;
+			}
+			auto entry = gstate.batch_data.begin();
+			if (entry->first >= min_index) {
+				// this data is past the min_index - we cannot write it yet
+				break;
+			}
+			if (!entry->second) {
+				// this batch is in process of being prepared but is not ready yet
+				break;
+			}
+			batch_data = std::move(entry->second);
+			gstate.batch_data.erase(entry);
+		}
+		function.flush_batch(context, *bind_data, *gstate.global_state, *batch_data);
+	}
+}
+
+void PhysicalBatchCopyToFile::NextBatch(ExecutionContext &context, GlobalSinkState &gstate_p,
+                                        LocalSinkState &lstate) const {
+	auto &state = lstate.Cast<BatchCopyToLocalState>();
+	if (state.collection) {
+		// we finished processing this batch
+		// start flushing data
+		PrepareBatchData(context.client, gstate_p, state.batch_index, std::move(state.collection));
+		FlushBatchData(context.client, gstate_p, lstate.partition_info.min_batch_index.GetIndex());
+	}
+	state.batch_index = lstate.partition_info.batch_index.GetIndex();
+
+	state.InitializeCollection(context.client, *this);
+}
+
+unique_ptr<LocalSinkState> PhysicalBatchCopyToFile::GetLocalSinkState(ExecutionContext &context) const {
+	return make_uniq<BatchCopyToLocalState>(function.copy_to_initialize_local(context, *bind_data));
+}
+
+unique_ptr<GlobalSinkState> PhysicalBatchCopyToFile::GetGlobalSinkState(ClientContext &context) const {
+	return make_uniq<BatchCopyToGlobalState>(function.copy_to_initialize_global(context, *bind_data, file_path));
+}
+
+//===--------------------------------------------------------------------===//
+// Source
+//===--------------------------------------------------------------------===//
+SourceResultType PhysicalBatchCopyToFile::GetData(ExecutionContext &context, DataChunk &chunk,
+                                                  OperatorSourceInput &input) const {
+	auto &g = sink_state->Cast<BatchCopyToGlobalState>();
+
+	chunk.SetCardinality(1);
+	chunk.SetValue(0, 0, Value::BIGINT(g.rows_copied));
+	return SourceResultType::FINISHED;
+}
+
+} // namespace duckdb

package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp

@@ -43,7 +43,7 @@ public:
 // Sink
 //===--------------------------------------------------------------------===//
 
-void MoveTmpFile(ClientContext &context, const string &tmp_file_path) {
+void PhysicalCopyToFile::MoveTmpFile(ClientContext &context, const string &tmp_file_path) {
 	auto &fs = FileSystem::GetFileSystem(context);
 	auto file_path = tmp_file_path.substr(0, tmp_file_path.length() - 4);
 	if (fs.FileExists(file_path)) {

package/src/duckdb/src/execution/operator/schema/physical_attach.cpp

@@ -1,11 +1,13 @@
 #include "duckdb/execution/operator/schema/physical_attach.hpp"
-#include "duckdb/parser/parsed_data/attach_info.hpp"
+
 #include "duckdb/catalog/catalog.hpp"
-#include "duckdb/main/database_manager.hpp"
 #include "duckdb/main/attached_database.hpp"
 #include "duckdb/main/database.hpp"
-#include "duckdb/storage/storage_extension.hpp"
+#include "duckdb/main/database_manager.hpp"
+#include "duckdb/main/database_path_and_type.hpp"
 #include "duckdb/main/extension_helper.hpp"
+#include "duckdb/parser/parsed_data/attach_info.hpp"
+#include "duckdb/storage/storage_extension.hpp"
 
 namespace duckdb {
 
@@ -44,11 +46,11 @@ SourceResultType PhysicalAttach::GetData(ExecutionContext &context, DataChunk &c
 	auto &db = DatabaseInstance::GetDatabase(context.client);
 	if (type.empty()) {
 		// try to extract type from path
-		type = db.ExtractDatabaseType(info->path);
-	}
-	if (!type.empty()) {
-		type = ExtensionHelper::ApplyExtensionAlias(type);
+		auto path_and_type = DBPathAndType::Parse(info->path, config);
+		type = path_and_type.type;
+		info->path = path_and_type.path;
 	}
+
 	if (type.empty() && !unrecognized_option.empty()) {
 		throw BinderException("Unrecognized option for attach \"%s\"", unrecognized_option);
 	}

package/src/duckdb/src/execution/partitionable_hashtable.cpp

@@ -48,7 +48,7 @@ HtEntryType PartitionableHashTable::GetHTEntrySize() {
 }
 
 idx_t PartitionableHashTable::ListAddChunk(HashTableList &list, DataChunk &groups, Vector &group_hashes,
-                                           DataChunk &payload, const vector<idx_t> &filter) {
+                                           DataChunk &payload, const unsafe_vector<idx_t> &filter) {
 	// If this is false, a single AddChunk would overflow the max capacity
 	D_ASSERT(list.empty() || groups.size() <= list.back()->MaxCapacity());
 	if (list.empty() || list.back()->Count() + groups.size() >= list.back()->MaxCapacity()) {
@@ -65,7 +65,7 @@ idx_t PartitionableHashTable::ListAddChunk(HashTableList &list, DataChunk &group
 }
 
 idx_t PartitionableHashTable::AddChunk(DataChunk &groups, DataChunk &payload, bool do_partition,
-                                       const vector<idx_t> &filter) {
+                                       const unsafe_vector<idx_t> &filter) {
 	groups.Hash(hashes);
 
 	// we partition when we are asked to or when the unpartitioned ht runs out of space

package/src/duckdb/src/execution/physical_plan/plan_copy_to_file.cpp

@@ -1,17 +1,40 @@
 #include "duckdb/execution/physical_plan_generator.hpp"
 #include "duckdb/execution/operator/persistent/physical_copy_to_file.hpp"
+#include "duckdb/execution/operator/persistent/physical_batch_copy_to_file.hpp"
 #include "duckdb/planner/operator/logical_copy_to_file.hpp"
 
 namespace duckdb {
 
 unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCopyToFile &op) {
 	auto plan = CreatePlan(*op.children[0]);
+	bool preserve_insertion_order = PhysicalPlanGenerator::PreserveInsertionOrder(context, *plan);
+	bool supports_batch_index = PhysicalPlanGenerator::UseBatchIndex(context, *plan);
 	auto &fs = FileSystem::GetFileSystem(context);
 	op.file_path = fs.ExpandPath(op.file_path, FileSystem::GetFileOpener(context));
-
 	if (op.use_tmp_file) {
 		op.file_path += ".tmp";
 	}
+	if (op.per_thread_output || op.partition_output || !op.partition_columns.empty() || op.overwrite_or_ignore) {
+		// hive-partitioning/per-thread output does not care about insertion order, and does not support batch indexes
+		preserve_insertion_order = false;
+		supports_batch_index = false;
+	}
+	auto mode = CopyFunctionExecutionMode::REGULAR_COPY_TO_FILE;
+	if (op.function.execution_mode) {
+		mode = op.function.execution_mode(preserve_insertion_order, supports_batch_index);
+	}
+	if (mode == CopyFunctionExecutionMode::BATCH_COPY_TO_FILE) {
+		if (!supports_batch_index) {
+			throw InternalException("BATCH_COPY_TO_FILE can only be used if batch indexes are supported");
+		}
+		// batched copy to file
+		auto copy = make_uniq<PhysicalBatchCopyToFile>(op.types, op.function, std::move(op.bind_data),
+		                                               op.estimated_cardinality);
+		copy->file_path = op.file_path;
+		copy->use_tmp_file = op.use_tmp_file;
+		copy->children.push_back(std::move(plan));
+		return std::move(copy);
+	}
 	// COPY from select statement to file
 	auto copy = make_uniq<PhysicalCopyToFile>(op.types, op.function, std::move(op.bind_data), op.estimated_cardinality);
 	copy->file_path = op.file_path;
@@ -23,9 +46,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCopyToFile
 	copy->partition_columns = op.partition_columns;
 	copy->names = op.names;
 	copy->expected_types = op.expected_types;
-	if (op.function.parallel) {
-		copy->parallel = op.function.parallel(context, *copy->bind_data);
-	}
+	copy->parallel = mode == CopyFunctionExecutionMode::PARALLEL_COPY_TO_FILE;
 
 	copy->children.push_back(std::move(plan));
 	return std::move(copy);

package/src/duckdb/src/execution/physical_plan/plan_sample.cpp

@@ -2,6 +2,7 @@
 #include "duckdb/execution/operator/helper/physical_streaming_sample.hpp"
 #include "duckdb/execution/physical_plan_generator.hpp"
 #include "duckdb/planner/operator/logical_sample.hpp"
+#include "duckdb/common/enum_util.hpp"
 
 namespace duckdb {
 
@@ -20,7 +21,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalSample &op
 		if (!op.sample_options->is_percentage) {
 			throw ParserException("Sample method %s cannot be used with a discrete sample count, either switch to "
 			                      "reservoir sampling or use a sample_size",
-			                      SampleMethodToString(op.sample_options->method));
+			                      EnumUtil::ToString(op.sample_options->method));
 		}
 		sample = make_uniq<PhysicalStreamingSample>(op.types, op.sample_options->method,
 		                                            op.sample_options->sample_size.GetValue<double>(),

package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp

@@ -131,7 +131,7 @@ void RadixPartitionedHashTable::PopulateGroupChunk(DataChunk &group_chunk, DataC
 }
 
 void RadixPartitionedHashTable::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input,
-                                     DataChunk &payload_input, const vector<idx_t> &filter) const {
+                                     DataChunk &payload_input, const unsafe_vector<idx_t> &filter) const {
 	auto &llstate = input.local_state.Cast<RadixHTLocalState>();
 	auto &gstate = input.global_state.Cast<RadixHTGlobalState>();
 	D_ASSERT(!gstate.is_finalized);

package/src/duckdb/src/function/scalar/operators/arithmetic.cpp

@@ -10,6 +10,7 @@
 #include "duckdb/common/types/time.hpp"
 #include "duckdb/common/types/timestamp.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
+#include "duckdb/common/enum_util.hpp"
 #include "duckdb/function/scalar/operators.hpp"
 #include "duckdb/planner/expression/bound_function_expression.hpp"
 #include "duckdb/function/scalar/nested_functions.hpp"
@@ -365,8 +366,8 @@ ScalarFunction AddFun::GetFunction(const LogicalType &left_type, const LogicalTy
 		break;
 	}
 	// LCOV_EXCL_START
-	throw NotImplementedException("AddFun for types %s, %s", LogicalTypeIdToString(left_type.id()),
-	                              LogicalTypeIdToString(right_type.id()));
+	throw NotImplementedException("AddFun for types %s, %s", EnumUtil::ToString(left_type.id()),
+	                              EnumUtil::ToString(right_type.id()));
 	// LCOV_EXCL_STOP
 }
 
@@ -617,8 +618,8 @@ ScalarFunction SubtractFun::GetFunction(const LogicalType &left_type, const Logi
 		break;
 	}
 	// LCOV_EXCL_START
-	throw NotImplementedException("SubtractFun for types %s, %s", LogicalTypeIdToString(left_type.id()),
-	                              LogicalTypeIdToString(right_type.id()));
+	throw NotImplementedException("SubtractFun for types %s, %s", EnumUtil::ToString(left_type.id()),
+	                              EnumUtil::ToString(right_type.id()));
 	// LCOV_EXCL_STOP
 }