duckdb 1.3.4 → 1.4.0

package/src/duckdb/src/include/duckdb/function/cast/variant/to_variant_fwd.hpp

@@ -0,0 +1,179 @@
+#pragma once
+
+#include "duckdb/common/types/data_chunk.hpp"
+#include "duckdb/function/scalar/variant_utils.hpp"
+#include "duckdb/common/types/vector.hpp"
+#include "duckdb/common/types/variant.hpp"
+#include "duckdb/common/types/selection_vector.hpp"
+#include "duckdb/common/owning_string_map.hpp"
+#include "duckdb/common/optional_ptr.hpp"
+#include "duckdb/common/serializer/varint.hpp"
+#include "duckdb/common/types/decimal.hpp"
+#include "duckdb/common/exception/conversion_exception.hpp"
+
+namespace duckdb {
+namespace variant {
+
+struct OffsetData {
+public:
+	static uint32_t *GetKeys(DataChunk &offsets) {
+		return FlatVector::GetData<uint32_t>(offsets.data[0]);
+	}
+	static uint32_t *GetChildren(DataChunk &offsets) {
+		return FlatVector::GetData<uint32_t>(offsets.data[1]);
+	}
+	static uint32_t *GetValues(DataChunk &offsets) {
+		return FlatVector::GetData<uint32_t>(offsets.data[2]);
+	}
+	static uint32_t *GetBlob(DataChunk &offsets) {
+		return FlatVector::GetData<uint32_t>(offsets.data[3]);
+	}
+};
+
+struct ToVariantGlobalResultData {
+public:
+	ToVariantGlobalResultData(VariantVectorData &variant, DataChunk &offsets,
+	                          OrderedOwningStringMap<uint32_t> &dictionary, SelectionVector &keys_selvec)
+	    : variant(variant), offsets(offsets), dictionary(dictionary), keys_selvec(keys_selvec) {
+	}
+
+public:
+	uint32_t GetOrCreateIndex(const string_t &key) {
+		auto unsorted_idx = dictionary.size();
+		//! This will later be remapped to the sorted idx (see FinalizeVariantKeys in 'to_variant.cpp')
+		return dictionary.emplace(std::make_pair(key, unsorted_idx)).first->second;
+	}
+
+public:
+	VariantVectorData &variant;
+	DataChunk &offsets;
+	//! The dictionary to populate with the (unique and sorted) keys
+	OrderedOwningStringMap<uint32_t> &dictionary;
+	//! The selection vector to populate with mapping from keys index -> dictionary index
+	SelectionVector &keys_selvec;
+};
+
+template <bool WRITE_DATA>
+void WriteContainerData(VariantVectorData &result, idx_t result_index, uint32_t &blob_offset, idx_t length,
+                        idx_t children_offset) {
+	const auto length_varint_size = GetVarintSize(length);
+	const auto child_offset_varint_size =
+	    NumericCast<uint32_t>(length_varint_size ? GetVarintSize(children_offset) : 0);
+
+	if (WRITE_DATA) {
+		auto &blob_value = result.blob_data[result_index];
+		auto blob_value_data = data_ptr_cast(blob_value.GetDataWriteable());
+		VarintEncode(length, blob_value_data + blob_offset);
+		if (length) {
+			VarintEncode(children_offset, blob_value_data + blob_offset + length_varint_size);
+		}
+	}
+
+	blob_offset += length_varint_size + child_offset_varint_size;
+}
+
+struct ContainerSelectionVectors {
+public:
+	explicit ContainerSelectionVectors(idx_t max_size)
+	    : new_selection(0, max_size), non_null_selection(0, max_size), children_selection(0, max_size) {
+	}
+
+public:
+	idx_t count = 0;
+	//! Create a selection vector that maps to the right row in the 'result' for the child vector
+	SelectionVector new_selection;
+	//! Create a selection vector that maps to a subset (the rows where the parent isn't null) of rows of the child
+	//! vector
+	SelectionVector non_null_selection;
+	//! Create a selection vector that maps to the children.values_index entry of the parent
+	SelectionVector children_selection;
+};
+
+template <bool WRITE_DATA>
+void WriteArrayChildren(VariantVectorData &result, uint64_t children_offset, uint32_t &children_offset_data,
+                        const list_entry_t source_entry, idx_t result_index, ContainerSelectionVectors &sel) {
+	for (idx_t child_idx = 0; child_idx < source_entry.length; child_idx++) {
+		//! Set up the selection vector for the child of the list vector
+		sel.new_selection.set_index(child_idx + sel.count, result_index);
+		if (WRITE_DATA) {
+			idx_t children_index = children_offset + children_offset_data;
+			sel.children_selection.set_index(child_idx + sel.count, children_index + child_idx);
+			result.keys_index_validity.SetInvalid(children_index + child_idx);
+		}
+		sel.non_null_selection.set_index(sel.count + child_idx, source_entry.offset + child_idx);
+	}
+	children_offset_data += source_entry.length;
+	sel.count += source_entry.length;
+}
+
+template <bool WRITE_DATA>
+void WriteVariantMetadata(ToVariantGlobalResultData &result, idx_t result_index, uint32_t *values_offsets,
+                          uint32_t blob_offset, optional_ptr<const SelectionVector> value_index_selvec, idx_t i,
+                          VariantLogicalType type_id) {
+
+	auto &values_offset_data = values_offsets[result_index];
+	if (WRITE_DATA) {
+		auto &variant = result.variant;
+		auto &values_list_entry = variant.values_data[result_index];
+		auto values_offset = values_list_entry.offset;
+
+		values_offset = values_list_entry.offset + values_offset_data;
+		variant.type_ids_data[values_offset] = static_cast<uint8_t>(type_id);
+		variant.byte_offset_data[values_offset] = blob_offset;
+		if (value_index_selvec) {
+			variant.values_index_data[value_index_selvec->get_index(i)] = values_offset_data;
+		}
+	}
+	values_offset_data++;
+}
+
+template <bool WRITE_DATA>
+void HandleVariantNull(ToVariantGlobalResultData &result, idx_t result_index, uint32_t *values_offsets,
+                       uint32_t blob_offset, optional_ptr<const SelectionVector> value_index_selvec, idx_t i,
+                       const bool is_root) {
+	if (is_root) {
+		if (WRITE_DATA) {
+			FlatVector::SetNull(result.variant.variant, result_index, true);
+		}
+		return;
+	}
+	WriteVariantMetadata<WRITE_DATA>(result, result_index, values_offsets, blob_offset, value_index_selvec, i,
+	                                 VariantLogicalType::VARIANT_NULL);
+}
+
+struct ToVariantSourceData {
+public:
+	ToVariantSourceData(Vector &source, idx_t source_size) : vec(source), source_size(source_size) {
+		vec.ToUnifiedFormat(source_size, source_format);
+	}
+	ToVariantSourceData(Vector &source, idx_t source_size, const SelectionVector &sel)
+	    : vec(source), source_size(source_size), source_sel(sel) {
+		vec.ToUnifiedFormat(source_size, source_format);
+	}
+	ToVariantSourceData(Vector &source, idx_t source_size, optional_ptr<const SelectionVector> sel)
+	    : vec(source), source_size(source_size), source_sel(sel) {
+		vec.ToUnifiedFormat(source_size, source_format);
+	}
+
+public:
+	uint32_t operator[](idx_t i) const {
+		return static_cast<uint32_t>(source_format.sel->get_index(source_sel ? source_sel->get_index(i) : i));
+	}
+	uint32_t GetMappedIndex(idx_t i) {
+		return static_cast<uint32_t>(source_sel ? source_sel->get_index(i) : i);
+	}
+
+public:
+	Vector &vec;
+	UnifiedVectorFormat source_format;
+	idx_t source_size;
+	optional_ptr<const SelectionVector> source_sel = nullptr;
+};
+
+template <bool WRITE_DATA, bool IGNORE_NULLS = false>
+bool ConvertToVariant(ToVariantSourceData &source, ToVariantGlobalResultData &result, idx_t count,
+                      optional_ptr<const SelectionVector> selvec,
+                      optional_ptr<const SelectionVector> values_index_selvec, const bool is_root);
+
+} // namespace variant
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/function/cast/variant/union_to_variant.hpp

@@ -0,0 +1,59 @@
+#pragma once
+
+#include "duckdb/function/cast/variant/to_variant_fwd.hpp"
+
+namespace duckdb {
+namespace variant {
+
+template <bool WRITE_DATA, bool IGNORE_NULLS>
+bool ConvertUnionToVariant(ToVariantSourceData &source, ToVariantGlobalResultData &result, idx_t count,
+                           optional_ptr<const SelectionVector> selvec,
+                           optional_ptr<const SelectionVector> values_index_selvec, const bool is_root) {
+	auto &children = StructVector::GetEntries(source.vec);
+
+	vector<ToVariantSourceData> member_data;
+	for (idx_t child_idx = 1; child_idx < children.size(); child_idx++) {
+		auto &child = *children[child_idx];
+
+		//! Convert all the children, ignore nulls, only write the non-null values
+		//! UNION will have exactly 1 non-null value for each row
+		member_data.emplace_back(child, source.source_size, source.source_sel);
+		if (!ConvertToVariant<WRITE_DATA, /*ignore_nulls = */ true>(member_data.back(), result, count, selvec,
+		                                                            values_index_selvec, is_root)) {
+			return false;
+		}
+	}
+
+	if (IGNORE_NULLS) {
+		return true;
+	}
+
+	//! For some reason we can have nulls in members, so we need this check
+	//! So we are sure that we handled all nulls
+	auto values_offset_data = OffsetData::GetValues(result.offsets);
+	auto blob_offset_data = OffsetData::GetBlob(result.offsets);
+	for (idx_t i = 0; i < count; i++) {
+		bool is_null = true;
+		for (idx_t child_idx = 1; child_idx < children.size() && is_null; child_idx++) {
+			auto &child = *children[child_idx];
+			if (child.GetType().id() == LogicalTypeId::SQLNULL) {
+				continue;
+			}
+			auto &member_format = member_data[child_idx - 1].source_format;
+			auto &member_validity = member_format.validity;
+			is_null = !member_validity.RowIsValid(member_format.sel->get_index(i));
+		}
+		if (!is_null) {
+			continue;
+		}
+		//! This row is NULL entirely
+		auto result_index = selvec ? selvec->get_index(i) : i;
+		auto &blob_offset = blob_offset_data[result_index];
+		HandleVariantNull<WRITE_DATA>(result, result_index, values_offset_data, blob_offset, values_index_selvec, i,
+		                              is_root);
+	}
+	return true;
+}
+
+} // namespace variant
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/function/cast/variant/variant_to_variant.hpp

@@ -0,0 +1,275 @@
+#pragma once
+
+#include "duckdb/function/cast/variant/to_variant_fwd.hpp"
+
+namespace duckdb {
+namespace variant {
+
+static bool VariantIsTrivialPrimitive(VariantLogicalType type) {
+	switch (type) {
+	case VariantLogicalType::INT8:
+	case VariantLogicalType::INT16:
+	case VariantLogicalType::INT32:
+	case VariantLogicalType::INT64:
+	case VariantLogicalType::INT128:
+	case VariantLogicalType::UINT8:
+	case VariantLogicalType::UINT16:
+	case VariantLogicalType::UINT32:
+	case VariantLogicalType::UINT64:
+	case VariantLogicalType::UINT128:
+	case VariantLogicalType::FLOAT:
+	case VariantLogicalType::DOUBLE:
+	case VariantLogicalType::UUID:
+	case VariantLogicalType::DATE:
+	case VariantLogicalType::TIME_MICROS:
+	case VariantLogicalType::TIME_NANOS:
+	case VariantLogicalType::TIMESTAMP_SEC:
+	case VariantLogicalType::TIMESTAMP_MILIS:
+	case VariantLogicalType::TIMESTAMP_MICROS:
+	case VariantLogicalType::TIMESTAMP_NANOS:
+	case VariantLogicalType::TIME_MICROS_TZ:
+	case VariantLogicalType::TIMESTAMP_MICROS_TZ:
+	case VariantLogicalType::INTERVAL:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static uint32_t VariantTrivialPrimitiveSize(VariantLogicalType type) {
+	switch (type) {
+	case VariantLogicalType::INT8:
+		return sizeof(int8_t);
+	case VariantLogicalType::INT16:
+		return sizeof(int16_t);
+	case VariantLogicalType::INT32:
+		return sizeof(int32_t);
+	case VariantLogicalType::INT64:
+		return sizeof(int64_t);
+	case VariantLogicalType::INT128:
+		return sizeof(hugeint_t);
+	case VariantLogicalType::UINT8:
+		return sizeof(uint8_t);
+	case VariantLogicalType::UINT16:
+		return sizeof(uint16_t);
+	case VariantLogicalType::UINT32:
+		return sizeof(uint32_t);
+	case VariantLogicalType::UINT64:
+		return sizeof(uint64_t);
+	case VariantLogicalType::UINT128:
+		return sizeof(uhugeint_t);
+	case VariantLogicalType::FLOAT:
+		return sizeof(float);
+	case VariantLogicalType::DOUBLE:
+		return sizeof(double);
+	case VariantLogicalType::UUID:
+		return sizeof(hugeint_t);
+	case VariantLogicalType::DATE:
+		return sizeof(int32_t);
+	case VariantLogicalType::TIME_MICROS:
+		return sizeof(dtime_t);
+	case VariantLogicalType::TIME_NANOS:
+		return sizeof(dtime_ns_t);
+	case VariantLogicalType::TIMESTAMP_SEC:
+		return sizeof(timestamp_sec_t);
+	case VariantLogicalType::TIMESTAMP_MILIS:
+		return sizeof(timestamp_ms_t);
+	case VariantLogicalType::TIMESTAMP_MICROS:
+		return sizeof(timestamp_t);
+	case VariantLogicalType::TIMESTAMP_NANOS:
+		return sizeof(timestamp_ns_t);
+	case VariantLogicalType::TIME_MICROS_TZ:
+		return sizeof(dtime_tz_t);
+	case VariantLogicalType::TIMESTAMP_MICROS_TZ:
+		return sizeof(timestamp_tz_t);
+	case VariantLogicalType::INTERVAL:
+		return sizeof(interval_t);
+	default:
+		throw InternalException("VariantLogicalType '%s' is not a trivial primitive", EnumUtil::ToString(type));
+	}
+}
+
+template <bool WRITE_DATA, bool IGNORE_NULLS>
+bool ConvertVariantToVariant(ToVariantSourceData &source_data, ToVariantGlobalResultData &result_data, idx_t count,
+                             optional_ptr<const SelectionVector> selvec,
+                             optional_ptr<const SelectionVector> values_index_selvec, const bool is_root) {
+
+	auto keys_offset_data = OffsetData::GetKeys(result_data.offsets);
+	auto children_offset_data = OffsetData::GetChildren(result_data.offsets);
+	auto values_offset_data = OffsetData::GetValues(result_data.offsets);
+	auto blob_offset_data = OffsetData::GetBlob(result_data.offsets);
+
+	RecursiveUnifiedVectorFormat source_format;
+	Vector::RecursiveToUnifiedFormat(source_data.vec, source_data.source_size, source_format);
+	UnifiedVariantVectorData source(source_format);
+
+	auto &result = result_data.variant;
+	for (idx_t source_index = 0; source_index < count; source_index++) {
+		auto result_index = selvec ? selvec->get_index(source_index) : source_index;
+
+		auto &keys_list_entry = result.keys_data[result_index];
+		auto &children_list_entry = result.children_data[result_index];
+		auto blob_data = data_ptr_cast(result.blob_data[result_index].GetDataWriteable());
+
+		auto &keys_offset = keys_offset_data[result_index];
+		auto &children_offset = children_offset_data[result_index];
+		auto &blob_offset = blob_offset_data[result_index];
+
+		uint32_t keys_count = 0;
+		uint32_t blob_size = 0;
+		if (!source.RowIsValid(source_index)) {
+			if (!IGNORE_NULLS) {
+				HandleVariantNull<WRITE_DATA>(result_data, result_index, values_offset_data, blob_offset,
+				                              values_index_selvec, source_index, is_root);
+			}
+			continue;
+		}
+		if (WRITE_DATA && values_index_selvec) {
+			auto &values_offset = values_offset_data[result_index];
+			//! Write the values_index for the parent of this column
+			result.values_index_data[values_index_selvec->get_index(source_index)] = values_offset;
+		}
+
+		//! FIXME: we might want to add some checks to make sure the NumericLimits<uint32_t>::Maximum isn't exceeded,
+		//! but that's hard to test
+
+		//! First write all children
+		//! NOTE: this has to happen first because we use 'values_offset', which is increased when we write the values
+		auto source_children_list_entry = source.GetChildrenListEntry(source_index);
+		for (idx_t source_children_index = 0; source_children_index < source_children_list_entry.length;
+		     source_children_index++) {
+			//! values_index
+			if (WRITE_DATA) {
+				auto &values_offset = values_offset_data[result_index];
+				auto source_value_index = source.GetValuesIndex(source_index, source_children_index);
+				result.values_index_data[children_list_entry.offset + children_offset + source_children_index] =
+				    values_offset + source_value_index;
+			}
+
+			//! keys_index
+			if (source.KeysIndexIsValid(source_index, source_children_index)) {
+				if (WRITE_DATA) {
+					//! Look up the existing key from 'source'
+					auto source_key_index = source.GetKeysIndex(source_index, source_children_index);
+					auto &source_key_value = source.GetKey(source_index, source_key_index);
+
+					//! Now write this key to the dictionary of the result
+					auto dict_index = result_data.GetOrCreateIndex(source_key_value);
+					result.keys_index_data[children_list_entry.offset + children_offset + source_children_index] =
+					    NumericCast<uint32_t>(keys_offset + keys_count);
+					result_data.keys_selvec.set_index(keys_list_entry.offset + keys_offset + keys_count, dict_index);
+				}
+				keys_count++;
+			} else {
+				if (WRITE_DATA) {
+					result.keys_index_validity.SetInvalid(children_list_entry.offset + children_offset +
+					                                      source_children_index);
+				}
+			}
+		}
+
+		auto source_blob_data = const_data_ptr_cast(source.GetData(source_index).GetData());
+
+		//! Then write all values
+		auto source_values_list_entry = source.GetValuesListEntry(source_index);
+		for (uint32_t source_value_index = 0; source_value_index < source_values_list_entry.length;
+		     source_value_index++) {
+			auto source_type_id = source.GetTypeId(source_index, source_value_index);
+			auto source_byte_offset = source.GetByteOffset(source_index, source_value_index);
+
+			//! NOTE: we have to deserialize these in both passes
+			//! because to figure out the size of the 'data' that is added by the VARIANT, we have to traverse the
+			//! VARIANT solely because the 'child_index' stored in the OBJECT/ARRAY data could require more bits
+			WriteVariantMetadata<WRITE_DATA>(result_data, result_index, values_offset_data, blob_offset + blob_size,
+			                                 nullptr, 0, source_type_id);
+
+			if (source_type_id == VariantLogicalType::ARRAY || source_type_id == VariantLogicalType::OBJECT) {
+				auto source_nested_data = VariantUtils::DecodeNestedData(source, source_index, source_value_index);
+				if (WRITE_DATA) {
+					VarintEncode(source_nested_data.child_count, blob_data + blob_offset + blob_size);
+				}
+				blob_size += GetVarintSize(source_nested_data.child_count);
+				if (source_nested_data.child_count) {
+					auto new_child_index = source_nested_data.children_idx + children_offset;
+					if (WRITE_DATA) {
+						VarintEncode(new_child_index, blob_data + blob_offset + blob_size);
+					}
+					blob_size += GetVarintSize(new_child_index);
+				}
+			} else if (source_type_id == VariantLogicalType::VARIANT_NULL ||
+			           source_type_id == VariantLogicalType::BOOL_FALSE ||
+			           source_type_id == VariantLogicalType::BOOL_TRUE) {
+				// no-op
+			} else if (source_type_id == VariantLogicalType::DECIMAL) {
+				auto decimal_blob_data = source_blob_data + source_byte_offset;
+				auto width = static_cast<uint8_t>(VarintDecode<uint32_t>(decimal_blob_data));
+				auto width_varint_size = GetVarintSize(width);
+				if (WRITE_DATA) {
+					memcpy(blob_data + blob_offset + blob_size, decimal_blob_data - width_varint_size,
+					       width_varint_size);
+				}
+				blob_size += width_varint_size;
+				auto scale = static_cast<uint8_t>(VarintDecode<uint32_t>(decimal_blob_data));
+				auto scale_varint_size = GetVarintSize(scale);
+				if (WRITE_DATA) {
+					memcpy(blob_data + blob_offset + blob_size, decimal_blob_data - scale_varint_size,
+					       scale_varint_size);
+				}
+				blob_size += scale_varint_size;
+
+				if (width > DecimalWidth<int64_t>::max) {
+					if (WRITE_DATA) {
+						memcpy(blob_data + blob_offset + blob_size, decimal_blob_data, sizeof(hugeint_t));
+					}
+					blob_size += sizeof(hugeint_t);
+				} else if (width > DecimalWidth<int32_t>::max) {
+					if (WRITE_DATA) {
+						memcpy(blob_data + blob_offset + blob_size, decimal_blob_data, sizeof(int64_t));
+					}
+					blob_size += sizeof(int64_t);
+				} else if (width > DecimalWidth<int16_t>::max) {
+					if (WRITE_DATA) {
+						memcpy(blob_data + blob_offset + blob_size, decimal_blob_data, sizeof(int32_t));
+					}
+					blob_size += sizeof(int32_t);
+				} else {
+					if (WRITE_DATA) {
+						memcpy(blob_data + blob_offset + blob_size, decimal_blob_data, sizeof(int16_t));
+					}
+					blob_size += sizeof(int16_t);
+				}
+			} else if (source_type_id == VariantLogicalType::BITSTRING ||
+			           source_type_id == VariantLogicalType::BIGNUM || source_type_id == VariantLogicalType::VARCHAR ||
+			           source_type_id == VariantLogicalType::BLOB) {
+				auto str_blob_data = source_blob_data + source_byte_offset;
+				auto str_length = VarintDecode<uint32_t>(str_blob_data);
+				auto str_length_varint_size = GetVarintSize(str_length);
+				if (WRITE_DATA) {
+					memcpy(blob_data + blob_offset + blob_size, str_blob_data - str_length_varint_size,
+					       str_length_varint_size);
+				}
+				blob_size += str_length_varint_size;
+				if (WRITE_DATA) {
+					memcpy(blob_data + blob_offset + blob_size, str_blob_data, str_length);
+				}
+				blob_size += str_length;
+			} else if (VariantIsTrivialPrimitive(source_type_id)) {
+				auto size = VariantTrivialPrimitiveSize(source_type_id);
+				if (WRITE_DATA) {
+					memcpy(blob_data + blob_offset + blob_size, source_blob_data + source_byte_offset, size);
+				}
+				blob_size += size;
+			} else {
+				throw InternalException("Unrecognized VariantLogicalType: %s", EnumUtil::ToString(source_type_id));
+			}
+		}
+
+		keys_offset += keys_count;
+		children_offset += source_children_list_entry.length;
+		blob_offset += blob_size;
+	}
+	return true;
+}
+
+} // namespace variant
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/function/cast/vector_cast_helpers.hpp

@@ -145,11 +145,10 @@ struct VectorCastHelpers {
 		return TemplatedTryCastLoop<SRC, DST, VectorTryCastStringOperator<OP>>(source, result, count, parameters);
 	}
 
-	template <class SRC, class OP>
+	template <class SRC, class OP, class RES = string_t>
 	static bool StringCast(Vector &source, Vector &result, idx_t count, CastParameters &parameters) {
 		D_ASSERT(result.GetType().InternalType() == PhysicalType::VARCHAR);
-		UnaryExecutor::GenericExecute<SRC, string_t, VectorStringCastOperator<OP>>(source, result, count,
-		                                                                           (void *)&result);
+		UnaryExecutor::GenericExecute<SRC, RES, VectorStringCastOperator<OP>>(source, result, count, (void *)&result);
 		return true;
 	}
 

package/src/duckdb/src/include/duckdb/function/copy_function.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/function/table_function.hpp"
 #include "duckdb/parser/parsed_data/copy_info.hpp"
 #include "duckdb/parser/statement/copy_statement.hpp"
+#include "duckdb/common/enums/copy_option_mode.hpp"
 
 namespace duckdb {
 
@@ -76,6 +77,14 @@ struct CopyFunctionBindInput {
 	string file_extension;
 };
 
+struct CopyFromFunctionBindInput {
+	explicit CopyFromFunctionBindInput(const CopyInfo &info_p, TableFunction &tf_p) : info(info_p), tf(tf_p) {
+	}
+
+	const CopyInfo &info;
+	TableFunction &tf;
+};
+
 struct CopyToSelectInput {
 	ClientContext &context;
 	case_insensitive_map_t<vector<Value>> &options;
@@ -83,9 +92,25 @@ struct CopyToSelectInput {
 	CopyToType copy_to_type;
 };
 
+struct CopyOption {
+	CopyOption();
+	explicit CopyOption(LogicalType type_p, CopyOptionMode mode = CopyOptionMode::READ_WRITE);
+
+	LogicalType type;
+	CopyOptionMode mode;
+};
+
+struct CopyOptionsInput {
+	explicit CopyOptionsInput(case_insensitive_map_t<CopyOption> &options) : options(options) {
+	}
+
+	case_insensitive_map_t<CopyOption> &options;
+};
+
 enum class CopyFunctionExecutionMode { REGULAR_COPY_TO_FILE, PARALLEL_COPY_TO_FILE, BATCH_COPY_TO_FILE };
 
 typedef BoundStatement (*copy_to_plan_t)(Binder &binder, CopyStatement &stmt);
+typedef void (*copy_options_t)(ClientContext &context, CopyOptionsInput &input);
 typedef unique_ptr<FunctionData> (*copy_to_bind_t)(ClientContext &context, CopyFunctionBindInput &input,
                                                    const vector<string> &names, const vector<LogicalType> &sql_types);
 typedef unique_ptr<LocalFunctionData> (*copy_to_initialize_local_t)(ExecutionContext &context, FunctionData &bind_data);
@@ -102,7 +127,7 @@ typedef void (*copy_to_serialize_t)(Serializer &serializer, const FunctionData &
 
 typedef unique_ptr<FunctionData> (*copy_to_deserialize_t)(Deserializer &deserializer, CopyFunction &function);
 
-typedef unique_ptr<FunctionData> (*copy_from_bind_t)(ClientContext &context, CopyInfo &info,
+typedef unique_ptr<FunctionData> (*copy_from_bind_t)(ClientContext &context, CopyFromFunctionBindInput &info,
                                                      vector<string> &expected_names,
                                                      vector<LogicalType> &expected_types);
 typedef CopyFunctionExecutionMode (*copy_to_execution_mode_t)(bool preserve_insertion_order, bool supports_batch_index);
@@ -144,20 +169,13 @@ struct CopyFunctionFileStatistics {
 
 class CopyFunction : public Function { // NOLINT: work-around bug in clang-tidy
 public:
-	explicit CopyFunction(const string &name)
-	    : Function(name), plan(nullptr), copy_to_select(nullptr), copy_to_bind(nullptr),
-	      copy_to_initialize_local(nullptr), copy_to_initialize_global(nullptr),
-	      copy_to_get_written_statistics(nullptr), copy_to_sink(nullptr), copy_to_combine(nullptr),
-	      copy_to_finalize(nullptr), execution_mode(nullptr), initialize_operator(nullptr), prepare_batch(nullptr),
-	      flush_batch(nullptr), desired_batch_size(nullptr), rotate_files(nullptr), rotate_next_file(nullptr),
-	      serialize(nullptr), deserialize(nullptr), copy_from_bind(nullptr) {
-	}
+	explicit CopyFunction(const string &name);
 
 	//! Plan rewrite copy function
 	copy_to_plan_t plan;
-
 	copy_to_select_t copy_to_select;
 	copy_to_bind_t copy_to_bind;
+	copy_options_t copy_options;
 	copy_to_initialize_local_t copy_to_initialize_local;
 	copy_to_initialize_global_t copy_to_initialize_global;
 	copy_to_get_written_statistics_t copy_to_get_written_statistics;

package/src/duckdb/src/include/duckdb/function/create_sort_key.hpp

@@ -55,4 +55,17 @@ struct CreateSortKeyHelpers {
 	                                      const idx_t count);
 };
 
+//! We don't add this function to the catalog, for internal use only
+//! Therefore, it's not defined in src/function/scalar/generic/functions.json
+struct DecodeSortKeyFun {
+	static constexpr const char *Name = "decode_sort_key";
+	static constexpr const char *Parameters = "parameters...";
+	static constexpr const char *Description =
+	    "Decodes a sort key created with create_sort_key into a STRUCT based on a set of sort qualifiers";
+	static constexpr const char *Example = "decode_sort_key(sort_key, 'A INTEGER', 'DESC')";
+	static constexpr const char *Categories = "";
+
+	static ScalarFunction GetFunction();
+};
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/function/function_binder.hpp

@@ -76,6 +76,9 @@ public:
 	//! Cast a set of expressions to the arguments of this function
 	void CastToFunctionArguments(SimpleFunction &function, vector<unique_ptr<Expression>> &children);
 
+	void ResolveTemplateTypes(BaseScalarFunction &bound_function, const vector<unique_ptr<Expression>> &children);
+	void CheckTemplateTypesResolved(const BaseScalarFunction &bound_function);
+
 private:
 	optional_idx BindVarArgsFunctionCost(const SimpleFunction &func, const vector<LogicalType> &arguments);
 	optional_idx BindFunctionCost(const SimpleFunction &func, const vector<LogicalType> &arguments);

package/src/duckdb/src/include/duckdb/function/function_list.hpp

@@ -36,7 +36,7 @@ struct CatalogTransaction;
 struct FunctionList {
 	static const StaticFunctionDefinition *GetInternalFunctionList();
 	static void RegisterFunctions(Catalog &catalog, CatalogTransaction transaction);
-	static void RegisterExtensionFunctions(DatabaseInstance &db, const StaticFunctionDefinition *functions);
+	static void RegisterExtensionFunctions(ExtensionLoader &db, const StaticFunctionDefinition *functions);
 };
 
 } // namespace duckdb