duckdb 0.8.2-dev37.0 → 0.8.2-dev3989.0

package/src/duckdb/src/common/types/validity_mask.cpp

@@ -96,6 +96,39 @@ void ValidityMask::SliceInPlace(const ValidityMask &other, idx_t target_offset,
 		memcpy(target_validity + target_offset_entries, source_validity + source_offset_entries,
 		       sizeof(validity_t) * EntryCount(count));
 		return;
+	} else if (IsAligned(target_offset)) {
+		//	Simple common case where we are shifting into an aligned mask (e.g., 0 in Slice above)
+		const idx_t entire_units = count / BITS_PER_VALUE;
+		const idx_t ragged = count % BITS_PER_VALUE;
+		const idx_t tail = source_offset % BITS_PER_VALUE;
+		const idx_t head = BITS_PER_VALUE - tail;
+		auto source_validity = other.GetData() + (source_offset / BITS_PER_VALUE);
+		auto target_validity = this->GetData() + (target_offset / BITS_PER_VALUE);
+		auto src_entry = *source_validity++;
+		for (idx_t i = 0; i < entire_units; ++i) {
+			//	Start with head of previous src
+			validity_t tgt_entry = src_entry >> tail;
+			src_entry = *source_validity++;
+			// 	Add in tail of current src
+			tgt_entry |= (src_entry << head);
+			*target_validity++ = tgt_entry;
+		}
+		//	Finish last ragged entry
+		if (ragged) {
+			//	Start with head of previous src
+			validity_t tgt_entry = (src_entry >> tail);
+			//  Add in the tail of the next src, if head was too small
+			if (head < ragged) {
+				src_entry = *source_validity++;
+				tgt_entry |= (src_entry << head);
+			}
+			//  Mask off the bits that go past the ragged end
+			tgt_entry &= (ValidityBuffer::MAX_ENTRY >> (BITS_PER_VALUE - ragged));
+			//	Restore the ragged end of the target
+			tgt_entry |= *target_validity & (ValidityBuffer::MAX_ENTRY << ragged);
+			*target_validity++ = tgt_entry;
+		}
+		return;
 	}
 
 	// FIXME: use bitwise operations here

package/src/duckdb/src/common/types/value.cpp

@@ -231,7 +231,7 @@ Value Value::MinimumValue(const LogicalType &type) {
 	case LogicalTypeId::TIMESTAMP_NS:
 		return Value::TIMESTAMPNS(timestamp_t(NumericLimits<int64_t>::Minimum()));
 	case LogicalTypeId::TIME_TZ:
-		return Value::TIMETZ(dtime_t(0));
+		return Value::TIMETZ(dtime_tz_t(dtime_t(0), dtime_tz_t::MIN_OFFSET));
 	case LogicalTypeId::TIMESTAMP_TZ:
 		return Value::TIMESTAMPTZ(Timestamp::FromDatetime(
 		    Date::FromDate(Timestamp::MIN_YEAR, Timestamp::MIN_MONTH, Timestamp::MIN_DAY), dtime_t(0)));
@@ -300,7 +300,8 @@ Value Value::MaximumValue(const LogicalType &type) {
 	case LogicalTypeId::TIMESTAMP_SEC:
 		return MaximumValue(LogicalType::TIMESTAMP).DefaultCastAs(LogicalType::TIMESTAMP_S);
 	case LogicalTypeId::TIME_TZ:
-		return Value::TIMETZ(dtime_t(Interval::SECS_PER_DAY * Interval::MICROS_PER_SEC - 1));
+		return Value::TIMETZ(
+		    dtime_tz_t(dtime_t(Interval::SECS_PER_DAY * Interval::MICROS_PER_SEC - 1), dtime_tz_t::MAX_OFFSET));
 	case LogicalTypeId::TIMESTAMP_TZ:
 		return MaximumValue(LogicalType::TIMESTAMP);
 	case LogicalTypeId::FLOAT:
@@ -587,9 +588,9 @@ Value Value::TIME(dtime_t value) {
 	return result;
 }
 
-Value Value::TIMETZ(dtime_t value) {
+Value Value::TIMETZ(dtime_tz_t value) {
 	Value result(LogicalType::TIME_TZ);
-	result.value_.time = value;
+	result.value_.timetz = value;
 	result.is_null = false;
 	return result;
 }
@@ -663,6 +664,20 @@ Value Value::MAP(const LogicalType &child_type, vector<Value> values) {
 
 	result.type_ = LogicalType::MAP(child_type);
 	result.is_null = false;
+	for (auto &val : values) {
+		D_ASSERT(val.type().InternalType() == PhysicalType::STRUCT);
+		auto &children = StructValue::GetChildren(val);
+
+		// Ensure that the field containing the keys is called 'key'
+		// and that the field containing the values is called 'value'
+		// this is required to make equality checks work
+		D_ASSERT(children.size() == 2);
+		child_list_t<Value> new_children;
+		new_children.reserve(2);
+		new_children.push_back(std::make_pair("key", children[0]));
+		new_children.push_back(std::make_pair("value", children[1]));
+		val = Value::STRUCT(std::move(new_children));
+	}
 	result.value_info_ = make_shared<NestedValueInfo>(std::move(values));
 	return result;
 }
@@ -678,7 +693,7 @@ Value Value::UNION(child_list_t<LogicalType> members, uint8_t tag, Value value)
 	result.is_null = false;
 	// add the tag to the front of the struct
 	vector<Value> union_values;
-	union_values.emplace_back(Value::TINYINT(tag));
+	union_values.emplace_back(Value::UTINYINT(tag));
 	for (idx_t i = 0; i < members.size(); i++) {
 		if (i != tag) {
 			union_values.emplace_back(members[i].second);
@@ -941,8 +956,9 @@ T Value::GetValueInternal() const {
 	case LogicalTypeId::DATE:
 		return Cast::Operation<date_t, T>(value_.date);
 	case LogicalTypeId::TIME:
-	case LogicalTypeId::TIME_TZ:
 		return Cast::Operation<dtime_t, T>(value_.time);
+	case LogicalTypeId::TIME_TZ:
+		return Cast::Operation<dtime_tz_t, T>(value_.timetz);
 	case LogicalTypeId::TIMESTAMP:
 	case LogicalTypeId::TIMESTAMP_TZ:
 		return Cast::Operation<timestamp_t, T>(value_.timestamp);
@@ -1014,7 +1030,6 @@ int64_t Value::GetValue() const {
 	case LogicalTypeId::TIMESTAMP_NS:
 	case LogicalTypeId::TIMESTAMP_MS:
 	case LogicalTypeId::TIME:
-	case LogicalTypeId::TIME_TZ:
 	case LogicalTypeId::TIMESTAMP_TZ:
 		return value_.bigint;
 	default:
@@ -1132,8 +1147,6 @@ Value Value::Numeric(const LogicalType &type, int64_t value) {
 		return Value::TIMESTAMPMS(timestamp_t(value));
 	case LogicalTypeId::TIMESTAMP_SEC:
 		return Value::TIMESTAMPSEC(timestamp_t(value));
-	case LogicalTypeId::TIME_TZ:
-		return Value::TIMETZ(dtime_t(value));
 	case LogicalTypeId::TIMESTAMP_TZ:
 		return Value::TIMESTAMPTZ(timestamp_t(value));
 	case LogicalTypeId::ENUM:
@@ -1465,20 +1478,25 @@ const vector<Value> &ListValue::GetChildren(const Value &value) {
 }
 
 const Value &UnionValue::GetValue(const Value &value) {
-	D_ASSERT(value.type() == LogicalTypeId::UNION);
+	D_ASSERT(value.type().id() == LogicalTypeId::UNION);
 	auto &children = StructValue::GetChildren(value);
-	auto tag = children[0].GetValueUnsafe<uint8_t>();
+	auto tag = children[0].GetValueUnsafe<union_tag_t>();
 	D_ASSERT(tag < children.size() - 1);
 	return children[tag + 1];
 }
 
-uint8_t UnionValue::GetTag(const Value &value) {
-	D_ASSERT(value.type() == LogicalTypeId::UNION);
+union_tag_t UnionValue::GetTag(const Value &value) {
+	D_ASSERT(value.type().id() == LogicalTypeId::UNION);
 	auto children = StructValue::GetChildren(value);
-	auto tag = children[0].GetValueUnsafe<uint8_t>();
+	auto tag = children[0].GetValueUnsafe<union_tag_t>();
+	D_ASSERT(tag < children.size() - 1);
 	return tag;
 }
 
+const LogicalType &UnionValue::GetType(const Value &value) {
+	return UnionType::GetMemberType(value.type(), UnionValue::GetTag(value));
+}
+
 hugeint_t IntegralValue::Get(const Value &value) {
 	switch (value.type().InternalType()) {
 	case PhysicalType::INT8:
@@ -1758,128 +1776,149 @@ Value Value::Deserialize(Deserializer &main_source) {
 }
 
 void Value::FormatSerialize(FormatSerializer &serializer) const {
-	serializer.WriteProperty("type", type_);
-	serializer.WriteProperty("is_null", is_null);
+	serializer.WriteProperty(100, "type", type_);
+	serializer.WriteProperty(101, "is_null", is_null);
 	if (!IsNull()) {
 		switch (type_.InternalType()) {
+		case PhysicalType::BIT:
+			throw InternalException("BIT type should not be serialized");
 		case PhysicalType::BOOL:
-			serializer.WriteProperty("value", value_.boolean);
+			serializer.WriteProperty(102, "value", value_.boolean);
 			break;
 		case PhysicalType::INT8:
-			serializer.WriteProperty("value", value_.tinyint);
+			serializer.WriteProperty(102, "value", value_.tinyint);
 			break;
 		case PhysicalType::INT16:
-			serializer.WriteProperty("value", value_.smallint);
+			serializer.WriteProperty(102, "value", value_.smallint);
 			break;
 		case PhysicalType::INT32:
-			serializer.WriteProperty("value", value_.integer);
+			serializer.WriteProperty(102, "value", value_.integer);
 			break;
 		case PhysicalType::INT64:
-			serializer.WriteProperty("value", value_.bigint);
+			serializer.WriteProperty(102, "value", value_.bigint);
 			break;
 		case PhysicalType::UINT8:
-			serializer.WriteProperty("value", value_.utinyint);
+			serializer.WriteProperty(102, "value", value_.utinyint);
 			break;
 		case PhysicalType::UINT16:
-			serializer.WriteProperty("value", value_.usmallint);
+			serializer.WriteProperty(102, "value", value_.usmallint);
 			break;
 		case PhysicalType::UINT32:
-			serializer.WriteProperty("value", value_.uinteger);
+			serializer.WriteProperty(102, "value", value_.uinteger);
 			break;
 		case PhysicalType::UINT64:
-			serializer.WriteProperty("value", value_.ubigint);
+			serializer.WriteProperty(102, "value", value_.ubigint);
 			break;
 		case PhysicalType::INT128:
-			serializer.WriteProperty("value", value_.hugeint);
+			serializer.WriteProperty(102, "value", value_.hugeint);
 			break;
 		case PhysicalType::FLOAT:
-			serializer.WriteProperty("value", value_.float_);
+			serializer.WriteProperty(102, "value", value_.float_);
 			break;
 		case PhysicalType::DOUBLE:
-			serializer.WriteProperty("value", value_.double_);
+			serializer.WriteProperty(102, "value", value_.double_);
 			break;
 		case PhysicalType::INTERVAL:
-			serializer.WriteProperty("value", value_.interval);
+			serializer.WriteProperty(102, "value", value_.interval);
 			break;
-		case PhysicalType::VARCHAR:
+		case PhysicalType::VARCHAR: {
 			if (type_.id() == LogicalTypeId::BLOB) {
 				auto blob_str = Blob::ToString(StringValue::Get(*this));
-				serializer.WriteProperty("value", blob_str);
+				serializer.WriteProperty(102, "value", blob_str);
 			} else {
-				serializer.WriteProperty("value", StringValue::Get(*this));
+				serializer.WriteProperty(102, "value", StringValue::Get(*this));
 			}
-			break;
-		default: {
-			Vector v(*this);
-			v.FormatSerialize(serializer, 1);
-			break;
-		}
+		} break;
+		case PhysicalType::LIST: {
+			serializer.WriteObject(102, "value", [&](FormatSerializer &serializer) {
+				auto &children = ListValue::GetChildren(*this);
+				serializer.WriteProperty(100, "children", children);
+			});
+		} break;
+		case PhysicalType::STRUCT: {
+			serializer.WriteObject(102, "value", [&](FormatSerializer &serializer) {
+				auto &children = StructValue::GetChildren(*this);
+				serializer.WriteProperty(100, "children", children);
+			});
+		} break;
+		default:
+			throw NotImplementedException("Unimplemented type for FormatSerialize");
 		}
 	}
 }
 
 Value Value::FormatDeserialize(FormatDeserializer &deserializer) {
-	auto type = deserializer.ReadProperty<LogicalType>("type");
-	auto is_null = deserializer.ReadProperty<bool>("is_null");
+	auto type = deserializer.ReadProperty<LogicalType>(100, "type");
+	auto is_null = deserializer.ReadProperty<bool>(101, "is_null");
 	Value new_value = Value(type);
 	if (is_null) {
 		return new_value;
 	}
 	new_value.is_null = false;
 	switch (type.InternalType()) {
+	case PhysicalType::BIT:
+		throw InternalException("BIT type should not be deserialized");
 	case PhysicalType::BOOL:
-		new_value.value_.boolean = deserializer.ReadProperty<bool>("value");
+		new_value.value_.boolean = deserializer.ReadProperty<bool>(102, "value");
 		break;
 	case PhysicalType::UINT8:
-		new_value.value_.utinyint = deserializer.ReadProperty<uint8_t>("value");
+		new_value.value_.utinyint = deserializer.ReadProperty<uint8_t>(102, "value");
 		break;
 	case PhysicalType::INT8:
-		new_value.value_.tinyint = deserializer.ReadProperty<int8_t>("value");
+		new_value.value_.tinyint = deserializer.ReadProperty<int8_t>(102, "value");
 		break;
 	case PhysicalType::UINT16:
-		new_value.value_.usmallint = deserializer.ReadProperty<uint16_t>("value");
+		new_value.value_.usmallint = deserializer.ReadProperty<uint16_t>(102, "value");
 		break;
 	case PhysicalType::INT16:
-		new_value.value_.smallint = deserializer.ReadProperty<int16_t>("value");
+		new_value.value_.smallint = deserializer.ReadProperty<int16_t>(102, "value");
 		break;
 	case PhysicalType::UINT32:
-		new_value.value_.uinteger = deserializer.ReadProperty<uint32_t>("value");
+		new_value.value_.uinteger = deserializer.ReadProperty<uint32_t>(102, "value");
 		break;
 	case PhysicalType::INT32:
-		new_value.value_.integer = deserializer.ReadProperty<int32_t>("value");
+		new_value.value_.integer = deserializer.ReadProperty<int32_t>(102, "value");
 		break;
 	case PhysicalType::UINT64:
-		new_value.value_.ubigint = deserializer.ReadProperty<uint64_t>("value");
+		new_value.value_.ubigint = deserializer.ReadProperty<uint64_t>(102, "value");
 		break;
 	case PhysicalType::INT64:
-		new_value.value_.bigint = deserializer.ReadProperty<int64_t>("value");
+		new_value.value_.bigint = deserializer.ReadProperty<int64_t>(102, "value");
 		break;
 	case PhysicalType::INT128:
-		new_value.value_.hugeint = deserializer.ReadProperty<hugeint_t>("value");
+		new_value.value_.hugeint = deserializer.ReadProperty<hugeint_t>(102, "value");
 		break;
 	case PhysicalType::FLOAT:
-		new_value.value_.float_ = deserializer.ReadProperty<float>("value");
+		new_value.value_.float_ = deserializer.ReadProperty<float>(102, "value");
 		break;
 	case PhysicalType::DOUBLE:
-		new_value.value_.double_ = deserializer.ReadProperty<double>("value");
+		new_value.value_.double_ = deserializer.ReadProperty<double>(102, "value");
 		break;
 	case PhysicalType::INTERVAL:
-		new_value.value_.interval = deserializer.ReadProperty<interval_t>("value");
+		new_value.value_.interval = deserializer.ReadProperty<interval_t>(102, "value");
 		break;
 	case PhysicalType::VARCHAR: {
-		auto str = deserializer.ReadProperty<string>("value");
+		auto str = deserializer.ReadProperty<string>(102, "value");
 		if (type.id() == LogicalTypeId::BLOB) {
 			new_value.value_info_ = make_shared<StringValueInfo>(Blob::ToBlob(str));
 		} else {
 			new_value.value_info_ = make_shared<StringValueInfo>(str);
 		}
 	} break;
-	default: {
-		Vector v(type);
-		v.FormatDeserialize(deserializer, 1);
-		new_value = v.GetValue(0);
-		break;
-	}
+	case PhysicalType::LIST: {
+		deserializer.ReadObject(102, "value", [&](FormatDeserializer &obj) {
+			auto children = obj.ReadProperty<vector<Value>>(100, "children");
+			new_value.value_info_ = make_shared<NestedValueInfo>(children);
+		});
+	} break;
+	case PhysicalType::STRUCT: {
+		deserializer.ReadObject(102, "value", [&](FormatDeserializer &obj) {
+			auto children = obj.ReadProperty<vector<Value>>(100, "children");
+			new_value.value_info_ = make_shared<NestedValueInfo>(children);
+		});
+	} break;
+	default:
+		throw NotImplementedException("Unimplemented type for FormatDeserialize");
 	}
 	return new_value;
 }

package/src/duckdb/src/common/types/vector.cpp

@@ -99,7 +99,7 @@ void Vector::Reference(const Value &value) {
 	}
 }
 
-void Vector::Reference(Vector &other) {
+void Vector::Reference(const Vector &other) {
 	if (other.GetType().id() != GetType().id()) {
 		throw InternalException("Vector::Reference used on vector of different type");
 	}
@@ -107,12 +107,12 @@ void Vector::Reference(Vector &other) {
 	Reinterpret(other);
 }
 
-void Vector::ReferenceAndSetType(Vector &other) {
+void Vector::ReferenceAndSetType(const Vector &other) {
 	type = other.GetType();
 	Reference(other);
 }
 
-void Vector::Reinterpret(Vector &other) {
+void Vector::Reinterpret(const Vector &other) {
 	vector_type = other.vector_type;
 	AssignSharedPointer(buffer, other.buffer);
 	AssignSharedPointer(auxiliary, other.auxiliary);
@@ -477,7 +477,7 @@ Value Vector::GetValueInternal(const Vector &v_p, idx_t index_p) {
 	case LogicalTypeId::TIME:
 		return Value::TIME(reinterpret_cast<dtime_t *>(data)[index]);
 	case LogicalTypeId::TIME_TZ:
-		return Value::TIMETZ(reinterpret_cast<dtime_t *>(data)[index]);
+		return Value::TIMETZ(reinterpret_cast<dtime_tz_t *>(data)[index]);
 	case LogicalTypeId::BIGINT:
 		return Value::BIGINT(reinterpret_cast<int64_t *>(data)[index]);
 	case LogicalTypeId::UTINYINT:
@@ -892,6 +892,27 @@ void Vector::ToUnifiedFormat(idx_t count, UnifiedVectorFormat &format) {
 	}
 }
 
+void Vector::RecursiveToUnifiedFormat(Vector &input, idx_t count, RecursiveUnifiedVectorFormat &data) {
+
+	input.ToUnifiedFormat(count, data.unified);
+
+	if (input.GetType().InternalType() == PhysicalType::LIST) {
+		auto &child = ListVector::GetEntry(input);
+		auto child_count = ListVector::GetListSize(input);
+		data.children.emplace_back();
+		Vector::RecursiveToUnifiedFormat(child, child_count, data.children.back());
+
+	} else if (input.GetType().InternalType() == PhysicalType::STRUCT) {
+		auto &children = StructVector::GetEntries(input);
+		for (idx_t i = 0; i < children.size(); i++) {
+			data.children.emplace_back();
+		}
+		for (idx_t i = 0; i < children.size(); i++) {
+			Vector::RecursiveToUnifiedFormat(*children[i], count, data.children[i]);
+		}
+	}
+}
+
 void Vector::Sequence(int64_t start, int64_t increment, idx_t count) {
 	this->vector_type = VectorType::SEQUENCE_VECTOR;
 	this->buffer = make_buffer<VectorBuffer>(sizeof(int64_t) * 3);
@@ -977,15 +998,15 @@ void Vector::FormatSerialize(FormatSerializer &serializer, idx_t count) {
 	UnifiedVectorFormat vdata;
 	ToUnifiedFormat(count, vdata);
 
-	const auto all_valid = (count > 0) && !vdata.validity.AllValid();
-	serializer.WriteProperty("all_valid", all_valid);
+	const bool all_valid = (count > 0) && !vdata.validity.AllValid();
+	serializer.WriteProperty(100, "all_valid", all_valid);
 	if (all_valid) {
 		ValidityMask flat_mask(count);
 		for (idx_t i = 0; i < count; ++i) {
 			auto row_idx = vdata.sel->get_index(i);
 			flat_mask.Set(i, vdata.validity.RowIsValid(row_idx));
 		}
-		serializer.WriteProperty("validity", const_data_ptr_cast(flat_mask.GetData()),
+		serializer.WriteProperty(101, "validity", const_data_ptr_cast(flat_mask.GetData()),
 		                         flat_mask.ValidityMaskSize(count));
 	}
 	if (TypeIsConstantSize(logical_type.InternalType())) {
@@ -993,21 +1014,18 @@ void Vector::FormatSerialize(FormatSerializer &serializer, idx_t count) {
 		idx_t write_size = GetTypeIdSize(logical_type.InternalType()) * count;
 		auto ptr = make_unsafe_uniq_array<data_t>(write_size);
 		VectorOperations::WriteToStorage(*this, count, ptr.get());
-		serializer.WriteProperty("data", ptr.get(), write_size);
+		serializer.WriteProperty(102, "data", ptr.get(), write_size);
 	} else {
 		switch (logical_type.InternalType()) {
 		case PhysicalType::VARCHAR: {
 			auto strings = UnifiedVectorFormat::GetData<string_t>(vdata);
 
 			// Serialize data as a list
-			serializer.SetTag("data");
-			serializer.OnListBegin(count);
-			for (idx_t i = 0; i < count; i++) {
+			serializer.WriteList(102, "data", count, [&](FormatSerializer::List &list, idx_t i) {
 				auto idx = vdata.sel->get_index(i);
 				auto str = !vdata.validity.RowIsValid(idx) ? NullValue<string_t>() : strings[idx];
-				serializer.WriteValue(str);
-			}
-			serializer.OnListEnd(count);
+				list.WriteElement(str);
+			});
 			break;
 		}
 		case PhysicalType::STRUCT: {
@@ -1015,14 +1033,9 @@ void Vector::FormatSerialize(FormatSerializer &serializer, idx_t count) {
 			auto &entries = StructVector::GetEntries(*this);
 
 			// Serialize entries as a list
-			serializer.SetTag("children");
-			serializer.OnListBegin(entries.size());
-			for (auto &entry : entries) {
-				serializer.OnObjectBegin();
-				entry->FormatSerialize(serializer, count);
-				serializer.OnObjectEnd();
-			}
-			serializer.OnListEnd(entries.size());
+			serializer.WriteList(103, "children", count, [&](FormatSerializer::List &list, idx_t i) {
+				list.WriteObject([&](FormatSerializer &object) { entries[i]->FormatSerialize(object, count); });
+			});
 			break;
 		}
 		case PhysicalType::LIST: {
@@ -1038,20 +1051,15 @@ void Vector::FormatSerialize(FormatSerializer &serializer, idx_t count) {
 				entries[i].offset = source.offset;
 				entries[i].length = source.length;
 			}
-			serializer.WriteProperty("list_size", list_size);
-			serializer.SetTag("entries");
-			serializer.OnListBegin(count);
-			for (idx_t i = 0; i < count; i++) {
-				serializer.OnObjectBegin();
-				serializer.WriteProperty("offset", entries[i].offset);
-				serializer.WriteProperty("length", entries[i].length);
-				serializer.OnObjectEnd();
-			}
-			serializer.OnListEnd(count);
-			serializer.SetTag("child");
-			serializer.OnObjectBegin();
-			child.FormatSerialize(serializer, list_size);
-			serializer.OnObjectEnd();
+			serializer.WriteProperty(104, "list_size", list_size);
+			serializer.WriteList(105, "entries", count, [&](FormatSerializer::List &list, idx_t i) {
+				list.WriteObject([&](FormatSerializer &object) {
+					object.WriteProperty(100, "offset", entries[i].offset);
+					object.WriteProperty(101, "length", entries[i].length);
+				});
+			});
+			serializer.WriteObject(106, "child",
+			                       [&](FormatSerializer &object) { child.FormatSerialize(object, list_size); });
 			break;
 		}
 		default:
@@ -1065,80 +1073,59 @@ void Vector::FormatDeserialize(FormatDeserializer &deserializer, idx_t count) {
 
 	auto &validity = FlatVector::Validity(*this);
 	validity.Reset();
-	const auto has_validity = deserializer.ReadProperty<bool>("all_valid");
+	const auto has_validity = deserializer.ReadProperty<bool>(100, "all_valid");
 	if (has_validity) {
 		validity.Initialize(count);
-		deserializer.ReadProperty("validity", data_ptr_cast(validity.GetData()), validity.ValidityMaskSize(count));
+		deserializer.ReadProperty(101, "validity", data_ptr_cast(validity.GetData()), validity.ValidityMaskSize(count));
 	}
 
 	if (TypeIsConstantSize(logical_type.InternalType())) {
 		// constant size type: read fixed amount of data
 		auto column_size = GetTypeIdSize(logical_type.InternalType()) * count;
 		auto ptr = make_unsafe_uniq_array<data_t>(column_size);
-		deserializer.ReadProperty("data", ptr.get(), column_size);
+		deserializer.ReadProperty(102, "data", ptr.get(), column_size);
 
 		VectorOperations::ReadFromStorage(ptr.get(), count, *this);
 	} else {
 		switch (logical_type.InternalType()) {
 		case PhysicalType::VARCHAR: {
 			auto strings = FlatVector::GetData<string_t>(*this);
-			deserializer.SetTag("data");
-			auto read_count = deserializer.OnListBegin();
-			D_ASSERT(read_count == count);
-			(void)read_count; // otherwise unused variable error in release mode
-			for (idx_t i = 0; i < count; i++) {
-				// read the strings
-				auto str = deserializer.ReadString();
-				// now add the string to the StringHeap of the vector
-				// and write the pointer into the vector
+			deserializer.ReadList(102, "data", [&](FormatDeserializer::List &list, idx_t i) {
+				auto str = list.ReadElement<string>();
 				if (validity.RowIsValid(i)) {
 					strings[i] = StringVector::AddStringOrBlob(*this, str);
 				}
-			}
-			deserializer.OnListEnd();
+			});
 			break;
 		}
 		case PhysicalType::STRUCT: {
 			auto &entries = StructVector::GetEntries(*this);
 			// Deserialize entries as a list
-			deserializer.SetTag("children");
-			auto read_size = deserializer.OnListBegin();
-			D_ASSERT(read_size == entries.size());
-			(void)read_size;
-			for (auto &entry : entries) {
-				deserializer.OnObjectBegin();
-				entry->FormatDeserialize(deserializer, count);
-				deserializer.OnObjectEnd();
-			}
-			deserializer.OnListEnd();
+			deserializer.ReadList(103, "children", [&](FormatDeserializer::List &list, idx_t i) {
+				list.ReadObject([&](FormatDeserializer &obj) { entries[i]->FormatDeserialize(obj, count); });
+			});
 			break;
 		}
 		case PhysicalType::LIST: {
 			// Read the list size
-			auto list_size = deserializer.ReadProperty<uint64_t>("list_size");
+			auto list_size = deserializer.ReadProperty<uint64_t>(104, "list_size");
 			ListVector::Reserve(*this, list_size);
 			ListVector::SetListSize(*this, list_size);
 
 			// Read the entries
 			auto list_entries = FlatVector::GetData<list_entry_t>(*this);
-			deserializer.SetTag("entries");
-			auto entries_count = deserializer.OnListBegin();
-			D_ASSERT(entries_count == count);
-			(void)entries_count;
-			for (idx_t i = 0; i < count; i++) {
-				deserializer.OnObjectBegin();
-				deserializer.ReadProperty("offset", list_entries[i].offset);
-				deserializer.ReadProperty("length", list_entries[i].length);
-				deserializer.OnObjectEnd();
-			}
-			deserializer.OnListEnd();
+			deserializer.ReadList(105, "entries", [&](FormatDeserializer::List &list, idx_t i) {
+				list.ReadObject([&](FormatDeserializer &obj) {
+					list_entries[i].offset = obj.ReadProperty<uint64_t>(100, "offset");
+					list_entries[i].length = obj.ReadProperty<uint64_t>(101, "length");
+				});
+			});
 
 			// Read the child vector
-			deserializer.SetTag("child");
-			auto &child = ListVector::GetEntry(*this);
-			deserializer.OnObjectBegin();
-			child.FormatDeserialize(deserializer, list_size);
-			deserializer.OnObjectEnd();
+			deserializer.ReadObject(106, "child", [&](FormatDeserializer &obj) {
+				auto &child = ListVector::GetEntry(*this);
+				child.FormatDeserialize(obj, list_size);
+			});
 			break;
 		}
 		default:
@@ -1265,6 +1252,11 @@ void Vector::UTFVerify(idx_t count) {
 void Vector::VerifyMap(Vector &vector_p, const SelectionVector &sel_p, idx_t count) {
 #ifdef DEBUG
 	D_ASSERT(vector_p.GetType().id() == LogicalTypeId::MAP);
+	auto &child = ListType::GetChildType(vector_p.GetType());
+	D_ASSERT(StructType::GetChildCount(child) == 2);
+	D_ASSERT(StructType::GetChildName(child, 0) == "key");
+	D_ASSERT(StructType::GetChildName(child, 1) == "value");
+
 	auto valid_check = MapVector::CheckMapValidity(vector_p, count, sel_p);
 	D_ASSERT(valid_check == MapInvalidReason::VALID);
 #endif // DEBUG
@@ -1389,9 +1381,6 @@ void Vector::Verify(Vector &vector_p, const SelectionVector &sel_p, idx_t count)
 				}
 			}
 		}
-		if (vector->GetType().id() == LogicalTypeId::MAP) {
-			VerifyMap(*vector, *sel, count);
-		}
 
 		if (vector->GetType().id() == LogicalTypeId::UNION) {
 			VerifyUnion(*vector, *sel, count);
@@ -1439,6 +1428,10 @@ void Vector::Verify(Vector &vector_p, const SelectionVector &sel_p, idx_t count)
 			}
 			Vector::Verify(child, child_sel, child_count);
 		}
+
+		if (vector->GetType().id() == LogicalTypeId::MAP) {
+			VerifyMap(*vector, *sel, count);
+		}
 	}
 #endif
 }