duckdb 1.3.4 → 1.4.0

package/src/duckdb/extension/parquet/parquet_writer.cpp

@@ -5,8 +5,6 @@
 #include "parquet_crypto.hpp"
 #include "parquet_timestamp.hpp"
 #include "resizable_buffer.hpp"
-
-#ifndef DUCKDB_AMALGAMATION
 #include "duckdb/common/file_system.hpp"
 #include "duckdb/common/serializer/buffered_file_writer.hpp"
 #include "duckdb/common/serializer/deserializer.hpp"
@@ -19,7 +17,6 @@
 #include "duckdb/parser/parsed_data/create_copy_function_info.hpp"
 #include "duckdb/parser/parsed_data/create_table_function_info.hpp"
 #include "duckdb/common/types/blob.hpp"
-#endif
 
 namespace duckdb {
 
@@ -177,6 +174,13 @@ void ParquetWriter::SetSchemaProperties(const LogicalType &duckdb_type, duckdb_p
 		schema_ele.logicalType.__set_JSON(duckdb_parquet::JsonType());
 		return;
 	}
+	if (duckdb_type.GetAlias() == "WKB_BLOB") {
+		schema_ele.__isset.logicalType = true;
+		schema_ele.logicalType.__isset.GEOMETRY = true;
+		// TODO: Set CRS in the future
+		schema_ele.logicalType.GEOMETRY.__isset.crs = false;
+		return;
+	}
 	switch (duckdb_type.id()) {
 	case LogicalTypeId::TINYINT:
 		schema_ele.converted_type = ConvertedType::INT_8;
@@ -304,7 +308,7 @@ uint32_t ParquetWriter::WriteData(const const_data_ptr_t buffer, const uint32_t
 	}
 }
 
-void VerifyUniqueNames(const vector<string> &names) {
+static void VerifyUniqueNames(const vector<string> &names) {
 #ifdef DEBUG
 	unordered_set<string> name_set;
 	name_set.reserve(names.size());
@@ -332,6 +336,11 @@ struct ColumnStatsUnifier {
 	bool can_have_nan = false;
 	bool has_nan = false;
 
+	unique_ptr<GeometryStats> geo_stats;
+
+	virtual void UnifyGeoStats(const GeometryStats &other) {
+	}
+
 	virtual void UnifyMinMax(const string &new_min, const string &new_max) = 0;
 	virtual string StatsToString(const string &stats) = 0;
 };
@@ -344,10 +353,10 @@ public:
 ParquetWriter::ParquetWriter(ClientContext &context, FileSystem &fs, string file_name_p, vector<LogicalType> types_p,
                              vector<string> names_p, CompressionCodec::type codec, ChildFieldIDs field_ids_p,
                              const vector<pair<string, string>> &kv_metadata,
-                             shared_ptr<ParquetEncryptionConfig> encryption_config_p, idx_t dictionary_size_limit_p,
-                             idx_t string_dictionary_page_size_limit_p, bool enable_bloom_filters_p,
-                             double bloom_filter_false_positive_ratio_p, int64_t compression_level_p,
-                             bool debug_use_openssl_p, ParquetVersion parquet_version)
+                             shared_ptr<ParquetEncryptionConfig> encryption_config_p,
+                             optional_idx dictionary_size_limit_p, idx_t string_dictionary_page_size_limit_p,
+                             bool enable_bloom_filters_p, double bloom_filter_false_positive_ratio_p,
+                             int64_t compression_level_p, bool debug_use_openssl_p, ParquetVersion parquet_version)
     : context(context), file_name(std::move(file_name_p)), sql_types(std::move(types_p)),
       column_names(std::move(names_p)), codec(codec), field_ids(std::move(field_ids_p)),
       encryption_config(std::move(encryption_config_p)), dictionary_size_limit(dictionary_size_limit_p),
@@ -471,7 +480,7 @@ void ParquetWriter::PrepareRowGroup(ColumnDataCollection &buffer, PreparedRowGro
 
 		for (auto &chunk : buffer.Chunks({column_ids})) {
 			for (idx_t i = 0; i < next; i++) {
-				col_writers[i].get().Prepare(*write_states[i], nullptr, chunk.data[i], chunk.size());
+				col_writers[i].get().Prepare(*write_states[i], nullptr, chunk.data[i], chunk.size(), true);
 			}
 		}
 
@@ -543,12 +552,24 @@ void ParquetWriter::FlushRowGroup(PreparedRowGroup &prepared) {
 	// let's make sure all offsets are ay-okay
 	ValidateColumnOffsets(file_name, writer->GetTotalWritten(), row_group);
 
-	// append the row group to the file meta data
+	row_group.total_compressed_size = NumericCast<int64_t>(writer->GetTotalWritten()) - row_group.file_offset;
+	row_group.__isset.total_compressed_size = true;
+
+	if (encryption_config) {
+		auto row_group_ordinal = num_row_groups.load();
+		if (row_group_ordinal > std::numeric_limits<int16_t>::max()) {
+			throw InvalidInputException("RowGroup ordinal exceeds 32767 when encryption enabled");
+		}
+		row_group.ordinal = NumericCast<int16_t>(row_group_ordinal);
+		row_group.__isset.ordinal = true;
+	}
+
+	// append the row group to the file metadata
 	file_meta_data.row_groups.push_back(row_group);
 	file_meta_data.num_rows += row_group.num_rows;
 
 	total_written = writer->GetTotalWritten();
-	num_row_groups++;
+	++num_row_groups;
 }
 
 void ParquetWriter::Flush(ColumnDataCollection &buffer) {
@@ -663,6 +684,50 @@ struct BlobStatsUnifier : public BaseStringStatsUnifier {
 	}
 };
 
+struct GeoStatsUnifier : public ColumnStatsUnifier {
+
+	void UnifyGeoStats(const GeometryStats &other) override {
+		if (geo_stats) {
+			geo_stats->bbox.Combine(other.bbox);
+			geo_stats->types.Combine(other.types);
+		} else {
+			// Make copy
+			geo_stats = make_uniq<GeometryStats>();
+			geo_stats->bbox = other.bbox;
+			geo_stats->types = other.types;
+		}
+	}
+
+	void UnifyMinMax(const string &new_min, const string &new_max) override {
+		// Do nothing
+	}
+
+	string StatsToString(const string &stats) override {
+		if (!geo_stats) {
+			return string();
+		}
+
+		const auto &bbox = geo_stats->bbox;
+		const auto &types = geo_stats->types;
+
+		const auto bbox_value = Value::STRUCT({{"xmin", bbox.xmin},
+		                                       {"xmax", bbox.xmax},
+		                                       {"ymin", bbox.ymin},
+		                                       {"ymax", bbox.ymax},
+		                                       {"zmin", bbox.zmin},
+		                                       {"zmax", bbox.zmax},
+		                                       {"mmin", bbox.mmin},
+		                                       {"mmax", bbox.mmax}});
+
+		vector<Value> type_strings;
+		for (const auto &type : types.ToString(true)) {
+			type_strings.push_back(Value(StringUtil::Lower(type)));
+		}
+
+		return Value::STRUCT({{"bbox", bbox_value}, {"types", Value::LIST(type_strings)}}).ToString();
+	}
+};
+
 struct UUIDStatsUnifier : public BaseStringStatsUnifier {
 	string StatsToString(const string &stats) override {
 		if (stats.size() != 16) {
@@ -692,7 +757,7 @@ struct NullStatsUnifier : public ColumnStatsUnifier {
 	}
 };
 
-unique_ptr<ColumnStatsUnifier> GetBaseStatsUnifier(const LogicalType &type) {
+static unique_ptr<ColumnStatsUnifier> GetBaseStatsUnifier(const LogicalType &type) {
 	switch (type.id()) {
 	case LogicalTypeId::BOOLEAN:
 		return make_uniq<NullStatsUnifier>();
@@ -708,6 +773,8 @@ unique_ptr<ColumnStatsUnifier> GetBaseStatsUnifier(const LogicalType &type) {
 		return make_uniq<NumericStatsUnifier<int64_t>>();
 	case LogicalTypeId::TIME:
 		return make_uniq<NumericStatsUnifier<dtime_t>>();
+	case LogicalTypeId::TIME_NS:
+		return make_uniq<NumericStatsUnifier<dtime_ns_t>>();
 	case LogicalTypeId::TIMESTAMP_SEC:
 	case LogicalTypeId::TIMESTAMP:
 		return make_uniq<NumericStatsUnifier<timestamp_t>>();
@@ -743,7 +810,11 @@ unique_ptr<ColumnStatsUnifier> GetBaseStatsUnifier(const LogicalType &type) {
 		}
 	}
 	case LogicalTypeId::BLOB:
-		return make_uniq<BlobStatsUnifier>();
+		if (type.GetAlias() == "WKB_BLOB") {
+			return make_uniq<GeoStatsUnifier>();
+		} else {
+			return make_uniq<BlobStatsUnifier>();
+		}
 	case LogicalTypeId::VARCHAR:
 		return make_uniq<StringStatsUnifier>();
 	case LogicalTypeId::UUID:
@@ -755,8 +826,8 @@ unique_ptr<ColumnStatsUnifier> GetBaseStatsUnifier(const LogicalType &type) {
 	}
 }
 
-void GetStatsUnifier(const ParquetColumnSchema &schema, vector<unique_ptr<ColumnStatsUnifier>> &unifiers,
-                     string base_name = string()) {
+static void GetStatsUnifier(const ParquetColumnSchema &schema, vector<unique_ptr<ColumnStatsUnifier>> &unifiers,
+                            string base_name = string()) {
 	if (!base_name.empty()) {
 		base_name += ".";
 	}
@@ -800,6 +871,9 @@ void ParquetWriter::FlushColumnStats(idx_t col_idx, duckdb_parquet::ColumnChunk
 		} else {
 			stats_unifier->all_nulls_set = false;
 		}
+		if (writer_stats && writer_stats->HasGeoStats()) {
+			stats_unifier->UnifyGeoStats(*writer_stats->GetGeoStats());
+		}
 		stats_unifier->column_size_bytes += column.meta_data.total_compressed_size;
 	}
 }
@@ -828,6 +902,33 @@ void ParquetWriter::GatherWrittenStatistics() {
 		if (stats_unifier->can_have_nan) {
 			column_stats["has_nan"] = Value::BOOLEAN(stats_unifier->has_nan);
 		}
+		if (stats_unifier->geo_stats) {
+			const auto &bbox = stats_unifier->geo_stats->bbox;
+			const auto &types = stats_unifier->geo_stats->types;
+
+			column_stats["bbox_xmin"] = Value::DOUBLE(bbox.xmin);
+			column_stats["bbox_xmax"] = Value::DOUBLE(bbox.xmax);
+			column_stats["bbox_ymin"] = Value::DOUBLE(bbox.ymin);
+			column_stats["bbox_ymax"] = Value::DOUBLE(bbox.ymax);
+
+			if (bbox.HasZ()) {
+				column_stats["bbox_zmin"] = Value::DOUBLE(bbox.zmin);
+				column_stats["bbox_zmax"] = Value::DOUBLE(bbox.zmax);
+			}
+
+			if (bbox.HasM()) {
+				column_stats["bbox_mmin"] = Value::DOUBLE(bbox.mmin);
+				column_stats["bbox_mmax"] = Value::DOUBLE(bbox.mmax);
+			}
+
+			if (!types.IsEmpty()) {
+				vector<Value> type_strings;
+				for (const auto &type : types.ToString(true)) {
+					type_strings.push_back(Value(StringUtil::Lower(type)));
+				}
+				column_stats["geo_types"] = Value::LIST(type_strings);
+			}
+		}
 		written_stats->column_statistics.insert(make_pair(stats_unifier->column_name, std::move(column_stats)));
 	}
 }
@@ -874,7 +975,7 @@ void ParquetWriter::Finalize() {
 	}
 
 	// Add geoparquet metadata to the file metadata
-	if (geoparquet_data) {
+	if (geoparquet_data && GeoParquetFileMetadata::IsGeoParquetConversionEnabled(context)) {
 		geoparquet_data->Write(file_meta_data);
 	}
 

package/src/duckdb/extension/parquet/reader/decimal_column_reader.cpp

@@ -3,7 +3,7 @@
 namespace duckdb {
 
 template <bool FIXED>
-unique_ptr<ColumnReader> CreateDecimalReaderInternal(ParquetReader &reader, const ParquetColumnSchema &schema) {
+static unique_ptr<ColumnReader> CreateDecimalReaderInternal(ParquetReader &reader, const ParquetColumnSchema &schema) {
 	switch (schema.type.InternalType()) {
 	case PhysicalType::INT16:
 		return make_uniq<DecimalColumnReader<int16_t, FIXED>>(reader, schema);

package/src/duckdb/extension/parquet/reader/variant/variant_binary_decoder.cpp

@@ -0,0 +1,367 @@
+#include "reader/variant/variant_binary_decoder.hpp"
+#include "duckdb/common/printer.hpp"
+#include "utf8proc_wrapper.hpp"
+
+#include "reader/uuid_column_reader.hpp"
+
+#include "duckdb/common/types/timestamp.hpp"
+#include "duckdb/common/types/decimal.hpp"
+#include "duckdb/common/types/uuid.hpp"
+#include "duckdb/common/types/time.hpp"
+#include "duckdb/common/types/date.hpp"
+#include "duckdb/common/types/blob.hpp"
+
+static constexpr uint8_t VERSION_MASK = 0xF;
+static constexpr uint8_t SORTED_STRINGS_MASK = 0x1;
+static constexpr uint8_t SORTED_STRINGS_SHIFT = 4;
+static constexpr uint8_t OFFSET_SIZE_MINUS_ONE_MASK = 0x3;
+static constexpr uint8_t OFFSET_SIZE_MINUS_ONE_SHIFT = 5;
+
+static constexpr uint8_t BASIC_TYPE_MASK = 0x3;
+static constexpr uint8_t VALUE_HEADER_SHIFT = 2;
+
+//! Object and Array header
+static constexpr uint8_t FIELD_OFFSET_SIZE_MINUS_ONE_MASK = 0x3;
+
+//! Object header
+static constexpr uint8_t FIELD_ID_SIZE_MINUS_ONE_MASK = 0x3;
+static constexpr uint8_t FIELD_ID_SIZE_MINUS_ONE_SHIFT = 2;
+
+static constexpr uint8_t OBJECT_IS_LARGE_MASK = 0x1;
+static constexpr uint8_t OBJECT_IS_LARGE_SHIFT = 4;
+
+//! Array header
+static constexpr uint8_t ARRAY_IS_LARGE_MASK = 0x1;
+static constexpr uint8_t ARRAY_IS_LARGE_SHIFT = 2;
+
+using namespace duckdb_yyjson;
+
+namespace duckdb {
+
+namespace {
+
+static idx_t ReadVariableLengthLittleEndian(idx_t length_in_bytes, const_data_ptr_t &ptr) {
+	if (length_in_bytes > sizeof(idx_t)) {
+		throw NotImplementedException("Can't read little-endian value of %d bytes", length_in_bytes);
+	}
+	idx_t result = 0;
+	memcpy(reinterpret_cast<uint8_t *>(&result), ptr, length_in_bytes);
+	ptr += length_in_bytes;
+	return result;
+}
+
+} // namespace
+
+VariantMetadataHeader VariantMetadataHeader::FromHeaderByte(uint8_t byte) {
+	VariantMetadataHeader header;
+	header.version = byte & VERSION_MASK;
+	header.sorted_strings = (byte >> SORTED_STRINGS_SHIFT) & SORTED_STRINGS_MASK;
+	header.offset_size = ((byte >> OFFSET_SIZE_MINUS_ONE_SHIFT) & OFFSET_SIZE_MINUS_ONE_MASK) + 1;
+
+	if (header.version != 1) {
+		throw NotImplementedException("Only version 1 of the Variant encoding scheme is supported, found version: %d",
+		                              header.version);
+	}
+
+	return header;
+}
+
+VariantMetadata::VariantMetadata(const string_t &metadata) : metadata(metadata) {
+	auto metadata_data = metadata.GetData();
+
+	header = VariantMetadataHeader::FromHeaderByte(metadata_data[0]);
+
+	const_data_ptr_t ptr = reinterpret_cast<const_data_ptr_t>(metadata_data + sizeof(uint8_t));
+	idx_t dictionary_size = ReadVariableLengthLittleEndian(header.offset_size, ptr);
+
+	offsets = ptr;
+	bytes = offsets + ((dictionary_size + 1) * header.offset_size);
+	idx_t last_offset = ReadVariableLengthLittleEndian(header.offset_size, ptr);
+	for (idx_t i = 0; i < dictionary_size; i++) {
+		auto next_offset = ReadVariableLengthLittleEndian(header.offset_size, ptr);
+		strings.emplace_back(reinterpret_cast<const char *>(bytes + last_offset), next_offset - last_offset);
+		last_offset = next_offset;
+	}
+}
+
+VariantValueMetadata VariantValueMetadata::FromHeaderByte(uint8_t byte) {
+	VariantValueMetadata result;
+	result.basic_type = VariantBasicTypeFromByte(byte & BASIC_TYPE_MASK);
+	uint8_t value_header = byte >> VALUE_HEADER_SHIFT;
+	switch (result.basic_type) {
+	case VariantBasicType::PRIMITIVE: {
+		result.primitive_type = VariantPrimitiveTypeFromByte(value_header);
+		break;
+	}
+	case VariantBasicType::SHORT_STRING: {
+		result.string_size = value_header;
+		break;
+	}
+	case VariantBasicType::OBJECT: {
+		result.field_offset_size = (value_header & FIELD_OFFSET_SIZE_MINUS_ONE_MASK) + 1;
+		result.field_id_size = ((value_header >> FIELD_ID_SIZE_MINUS_ONE_SHIFT) & FIELD_ID_SIZE_MINUS_ONE_MASK) + 1;
+		result.is_large = (value_header >> OBJECT_IS_LARGE_SHIFT) & OBJECT_IS_LARGE_MASK;
+		break;
+	}
+	case VariantBasicType::ARRAY: {
+		result.field_offset_size = (value_header & FIELD_OFFSET_SIZE_MINUS_ONE_MASK) + 1;
+		result.is_large = (value_header >> ARRAY_IS_LARGE_SHIFT) & ARRAY_IS_LARGE_MASK;
+		break;
+	}
+	default:
+		throw InternalException("VariantBasicType (%d) not handled", static_cast<uint8_t>(result.basic_type));
+	}
+	return result;
+}
+
+template <class T>
+static T DecodeDecimal(const_data_ptr_t data, uint8_t &scale, uint8_t &width) {
+	scale = Load<uint8_t>(data);
+	data++;
+
+	auto result = Load<T>(data);
+	//! FIXME: The spec says:
+	//! The implied precision of a decimal value is `floor(log_10(val)) + 1`
+	width = DecimalWidth<T>::max;
+	return result;
+}
+
+template <>
+hugeint_t DecodeDecimal(const_data_ptr_t data, uint8_t &scale, uint8_t &width) {
+	scale = Load<uint8_t>(data);
+	data++;
+
+	hugeint_t result;
+	result.lower = Load<uint64_t>(data);
+	result.upper = Load<int64_t>(data + sizeof(uint64_t));
+	//! FIXME: The spec says:
+	//! The implied precision of a decimal value is `floor(log_10(val)) + 1`
+	width = DecimalWidth<hugeint_t>::max;
+	return result;
+}
+
+VariantValue VariantBinaryDecoder::PrimitiveTypeDecode(const VariantMetadata &metadata,
+                                                       const VariantValueMetadata &value_metadata,
+                                                       const_data_ptr_t data) {
+	switch (value_metadata.primitive_type) {
+	case VariantPrimitiveType::NULL_TYPE: {
+		return VariantValue(Value());
+	}
+	case VariantPrimitiveType::BOOLEAN_TRUE: {
+		return VariantValue(Value::BOOLEAN(true));
+	}
+	case VariantPrimitiveType::BOOLEAN_FALSE: {
+		return VariantValue(Value::BOOLEAN(false));
+	}
+	case VariantPrimitiveType::INT8: {
+		auto value = Load<int8_t>(data);
+		return VariantValue(Value::TINYINT(value));
+	}
+	case VariantPrimitiveType::INT16: {
+		auto value = Load<int16_t>(data);
+		return VariantValue(Value::SMALLINT(value));
+	}
+	case VariantPrimitiveType::INT32: {
+		auto value = Load<int32_t>(data);
+		return VariantValue(Value::INTEGER(value));
+	}
+	case VariantPrimitiveType::INT64: {
+		auto value = Load<int64_t>(data);
+		return VariantValue(Value::BIGINT(value));
+	}
+	case VariantPrimitiveType::DOUBLE: {
+		double value = Load<double>(data);
+		return VariantValue(Value::DOUBLE(value));
+	}
+	case VariantPrimitiveType::FLOAT: {
+		float value = Load<float>(data);
+		return VariantValue(Value::FLOAT(value));
+	}
+	case VariantPrimitiveType::DECIMAL4: {
+		uint8_t scale;
+		uint8_t width;
+
+		auto value = DecodeDecimal<int32_t>(data, scale, width);
+		auto value_str = Decimal::ToString(value, width, scale);
+		return VariantValue(Value(value_str));
+	}
+	case VariantPrimitiveType::DECIMAL8: {
+		uint8_t scale;
+		uint8_t width;
+
+		auto value = DecodeDecimal<int64_t>(data, scale, width);
+		auto value_str = Decimal::ToString(value, width, scale);
+		return VariantValue(Value(value_str));
+	}
+	case VariantPrimitiveType::DECIMAL16: {
+		uint8_t scale;
+		uint8_t width;
+
+		auto value = DecodeDecimal<hugeint_t>(data, scale, width);
+		auto value_str = Decimal::ToString(value, width, scale);
+		return VariantValue(Value(value_str));
+	}
+	case VariantPrimitiveType::DATE: {
+		date_t value;
+		value.days = Load<int32_t>(data);
+		return VariantValue(Value::DATE(value));
+	}
+	case VariantPrimitiveType::TIMESTAMP_MICROS: {
+		timestamp_tz_t micros_ts_tz;
+		micros_ts_tz.value = Load<int64_t>(data);
+		return VariantValue(Value::TIMESTAMPTZ(micros_ts_tz));
+	}
+	case VariantPrimitiveType::TIMESTAMP_NTZ_MICROS: {
+		timestamp_t micros_ts;
+		micros_ts.value = Load<int64_t>(data);
+
+		auto value = Value::TIMESTAMP(micros_ts);
+		auto value_str = value.ToString();
+		return VariantValue(Value(value_str));
+	}
+	case VariantPrimitiveType::BINARY: {
+		//! Follow the JSON serialization guide by converting BINARY to Base64:
+		//! For example: `"dmFyaWFudAo="`
+		auto size = Load<uint32_t>(data);
+		auto string_data = reinterpret_cast<const char *>(data + sizeof(uint32_t));
+		auto base64_string = Blob::ToBase64(string_t(string_data, size));
+		return VariantValue(Value(base64_string));
+	}
+	case VariantPrimitiveType::STRING: {
+		auto size = Load<uint32_t>(data);
+		auto string_data = reinterpret_cast<const char *>(data + sizeof(uint32_t));
+		if (!Utf8Proc::IsValid(string_data, size)) {
+			throw InternalException("Can't decode Variant short-string, string isn't valid UTF8");
+		}
+		return VariantValue(Value(string(string_data, size)));
+	}
+	case VariantPrimitiveType::TIME_NTZ_MICROS: {
+		dtime_t micros_time;
+		micros_time.micros = Load<int64_t>(data);
+		return VariantValue(Value::TIME(micros_time));
+	}
+	case VariantPrimitiveType::TIMESTAMP_NANOS: {
+		timestamp_ns_t nanos_ts;
+		nanos_ts.value = Load<int64_t>(data);
+
+		//! Convert the nanos timestamp to a micros timestamp (not lossless)
+		auto micros_ts = Timestamp::FromEpochNanoSeconds(nanos_ts.value);
+		return VariantValue(Value::TIMESTAMPTZ(timestamp_tz_t(micros_ts)));
+	}
+	case VariantPrimitiveType::TIMESTAMP_NTZ_NANOS: {
+		timestamp_ns_t nanos_ts;
+		nanos_ts.value = Load<int64_t>(data);
+
+		auto value = Value::TIMESTAMPNS(nanos_ts);
+		auto value_str = value.ToString();
+		return VariantValue(Value(value_str));
+	}
+	case VariantPrimitiveType::UUID: {
+		auto uuid_value = UUIDValueConversion::ReadParquetUUID(data);
+		auto value_str = UUID::ToString(uuid_value);
+		return VariantValue(Value(value_str));
+	}
+	default:
+		throw NotImplementedException("Variant PrimitiveTypeDecode not implemented for type (%d)",
+		                              static_cast<uint8_t>(value_metadata.primitive_type));
+	}
+}
+
+VariantValue VariantBinaryDecoder::ShortStringDecode(const VariantMetadata &metadata,
+                                                     const VariantValueMetadata &value_metadata,
+                                                     const_data_ptr_t data) {
+	D_ASSERT(value_metadata.string_size < 64);
+	auto string_data = reinterpret_cast<const char *>(data);
+	if (!Utf8Proc::IsValid(string_data, value_metadata.string_size)) {
+		throw InternalException("Can't decode Variant short-string, string isn't valid UTF8");
+	}
+	return VariantValue(Value(string(string_data, value_metadata.string_size)));
+}
+
+VariantValue VariantBinaryDecoder::ObjectDecode(const VariantMetadata &metadata,
+                                                const VariantValueMetadata &value_metadata, const_data_ptr_t data) {
+	VariantValue ret(VariantValueType::OBJECT);
+
+	auto field_offset_size = value_metadata.field_offset_size;
+	auto field_id_size = value_metadata.field_id_size;
+	auto is_large = value_metadata.is_large;
+
+	idx_t num_elements;
+	if (is_large) {
+		num_elements = Load<uint32_t>(data);
+		data += sizeof(uint32_t);
+	} else {
+		num_elements = Load<uint8_t>(data);
+		data += sizeof(uint8_t);
+	}
+
+	auto field_ids = data;
+	auto field_offsets = data + (num_elements * field_id_size);
+	auto values = field_offsets + ((num_elements + 1) * field_offset_size);
+
+	idx_t last_offset = ReadVariableLengthLittleEndian(field_offset_size, field_offsets);
+	for (idx_t i = 0; i < num_elements; i++) {
+		auto field_id = ReadVariableLengthLittleEndian(field_id_size, field_ids);
+		auto next_offset = ReadVariableLengthLittleEndian(field_offset_size, field_offsets);
+
+		auto value = Decode(metadata, values + last_offset);
+		auto &key = metadata.strings[field_id];
+
+		ret.AddChild(key, std::move(value));
+		last_offset = next_offset;
+	}
+	return ret;
+}
+
+VariantValue VariantBinaryDecoder::ArrayDecode(const VariantMetadata &metadata,
+                                               const VariantValueMetadata &value_metadata, const_data_ptr_t data) {
+	VariantValue ret(VariantValueType::ARRAY);
+
+	auto field_offset_size = value_metadata.field_offset_size;
+	auto is_large = value_metadata.is_large;
+
+	uint32_t num_elements;
+	if (is_large) {
+		num_elements = Load<uint32_t>(data);
+		data += sizeof(uint32_t);
+	} else {
+		num_elements = Load<uint8_t>(data);
+		data += sizeof(uint8_t);
+	}
+
+	auto field_offsets = data;
+	auto values = field_offsets + ((num_elements + 1) * field_offset_size);
+
+	idx_t last_offset = ReadVariableLengthLittleEndian(field_offset_size, field_offsets);
+	for (idx_t i = 0; i < num_elements; i++) {
+		auto next_offset = ReadVariableLengthLittleEndian(field_offset_size, field_offsets);
+
+		ret.AddItem(Decode(metadata, values + last_offset));
+		last_offset = next_offset;
+	}
+	return ret;
+}
+
+VariantValue VariantBinaryDecoder::Decode(const VariantMetadata &variant_metadata, const_data_ptr_t data) {
+	auto value_metadata = VariantValueMetadata::FromHeaderByte(data[0]);
+
+	data++;
+	switch (value_metadata.basic_type) {
+	case VariantBasicType::PRIMITIVE: {
+		return PrimitiveTypeDecode(variant_metadata, value_metadata, data);
+	}
+	case VariantBasicType::SHORT_STRING: {
+		return ShortStringDecode(variant_metadata, value_metadata, data);
+	}
+	case VariantBasicType::OBJECT: {
+		return ObjectDecode(variant_metadata, value_metadata, data);
+	}
+	case VariantBasicType::ARRAY: {
+		return ArrayDecode(variant_metadata, value_metadata, data);
+	}
+	default:
+		throw InternalException("Unexpected value for VariantBasicType");
+	}
+}
+
+} // namespace duckdb