duckdb 0.8.2-dev3458.0 → 0.8.2-dev3949.0

package/src/duckdb/src/common/types/row/tuple_data_allocator.cpp

@@ -33,6 +33,10 @@ TupleDataAllocator::TupleDataAllocator(TupleDataAllocator &allocator)
     : buffer_manager(allocator.buffer_manager), layout(allocator.layout.Copy()) {
 }
 
+BufferManager &TupleDataAllocator::GetBufferManager() {
+	return buffer_manager;
+}
+
 Allocator &TupleDataAllocator::GetAllocator() {
 	return buffer_manager.GetBufferAllocator();
 }
@@ -58,7 +62,7 @@ void TupleDataAllocator::Build(TupleDataSegment &segment, TupleDataPinState &pin
 	}
 
 	// Build the chunk parts for the incoming data
-	vector<pair<idx_t, idx_t>> chunk_part_indices;
+	chunk_part_indices.clear();
 	idx_t offset = 0;
 	while (offset != append_count) {
 		if (chunks.empty() || chunks.back().count == STANDARD_VECTOR_SIZE) {
@@ -68,23 +72,26 @@ void TupleDataAllocator::Build(TupleDataSegment &segment, TupleDataPinState &pin
 
 		// Build the next part
 		auto next = MinValue<idx_t>(append_count - offset, STANDARD_VECTOR_SIZE - chunk.count);
-		chunk.AddPart(BuildChunkPart(pin_state, chunk_state, append_offset + offset, next), layout);
-		chunk_part_indices.emplace_back(chunks.size() - 1, chunk.parts.size() - 1);
-
+		chunk.AddPart(BuildChunkPart(pin_state, chunk_state, append_offset + offset, next, chunk), layout);
 		auto &chunk_part = chunk.parts.back();
 		next = chunk_part.count;
+
 		segment.count += next;
+		segment.data_size += chunk_part.count * layout.GetRowWidth();
+		if (!layout.AllConstant()) {
+			segment.data_size += chunk_part.total_heap_size;
+		}
 
 		offset += next;
+		chunk_part_indices.emplace_back(chunks.size() - 1, chunk.parts.size() - 1);
 	}
 
 	// Now initialize the pointers to write the data to
-	vector<TupleDataChunkPart *> parts;
-	parts.reserve(chunk_part_indices.size());
+	chunk_parts.clear();
 	for (auto &indices : chunk_part_indices) {
-		parts.emplace_back(&segment.chunks[indices.first].parts[indices.second]);
+		chunk_parts.emplace_back(segment.chunks[indices.first].parts[indices.second]);
 	}
-	InitializeChunkStateInternal(pin_state, chunk_state, append_offset, false, true, false, parts);
+	InitializeChunkStateInternal(pin_state, chunk_state, append_offset, false, true, false, chunk_parts);
 
 	// To reduce metadata, we try to merge chunk parts where possible
 	// Due to the way chunk parts are constructed, only the last part of the first chunk is eligible for merging
@@ -94,9 +101,10 @@ void TupleDataAllocator::Build(TupleDataSegment &segment, TupleDataPinState &pin
 }
 
 TupleDataChunkPart TupleDataAllocator::BuildChunkPart(TupleDataPinState &pin_state, TupleDataChunkState &chunk_state,
-                                                      const idx_t append_offset, const idx_t append_count) {
+                                                      const idx_t append_offset, const idx_t append_count,
+                                                      TupleDataChunk &chunk) {
 	D_ASSERT(append_count != 0);
-	TupleDataChunkPart result;
+	TupleDataChunkPart result(*chunk.lock);
 
 	// Allocate row block (if needed)
 	if (row_blocks.empty() || row_blocks.back().RemainingCapacity() < layout.GetRowWidth()) {
@@ -176,10 +184,10 @@ void TupleDataAllocator::InitializeChunkState(TupleDataSegment &segment, TupleDa
 	// when chunk 0 needs heap block 0, chunk 1 does not need any heap blocks, and chunk 2 needs heap block 0 again
 	ReleaseOrStoreHandles(pin_state, segment, chunk, !chunk.heap_block_ids.empty());
 
-	vector<TupleDataChunkPart *> parts;
+	unsafe_vector<reference<TupleDataChunkPart>> parts;
 	parts.reserve(chunk.parts.size());
 	for (auto &part : chunk.parts) {
-		parts.emplace_back(&part);
+		parts.emplace_back(part);
 	}
 
 	InitializeChunkStateInternal(pin_state, chunk_state, 0, true, init_heap, init_heap, parts);
@@ -206,17 +214,19 @@ static inline void InitializeHeapSizes(const data_ptr_t row_locations[], idx_t h
 
 void TupleDataAllocator::InitializeChunkStateInternal(TupleDataPinState &pin_state, TupleDataChunkState &chunk_state,
                                                       idx_t offset, bool recompute, bool init_heap_pointers,
-                                                      bool init_heap_sizes, vector<TupleDataChunkPart *> &parts) {
+                                                      bool init_heap_sizes,
+                                                      unsafe_vector<reference<TupleDataChunkPart>> &parts) {
 	auto row_locations = FlatVector::GetData<data_ptr_t>(chunk_state.row_locations);
 	auto heap_sizes = FlatVector::GetData<idx_t>(chunk_state.heap_sizes);
 	auto heap_locations = FlatVector::GetData<data_ptr_t>(chunk_state.heap_locations);
 
-	for (auto &part : parts) {
-		const auto next = part->count;
+	for (auto &part_ref : parts) {
+		auto &part = part_ref.get();
+		const auto next = part.count;
 
 		// Set up row locations for the scan
 		const auto row_width = layout.GetRowWidth();
-		const auto base_row_ptr = GetRowPointer(pin_state, *part);
+		const auto base_row_ptr = GetRowPointer(pin_state, part);
 		for (idx_t i = 0; i < next; i++) {
 			row_locations[offset + i] = base_row_ptr + i * row_width;
 		}
@@ -226,9 +236,9 @@ void TupleDataAllocator::InitializeChunkStateInternal(TupleDataPinState &pin_sta
 			continue;
 		}
 
-		if (part->total_heap_size == 0) {
+		if (part.total_heap_size == 0) {
 			if (init_heap_sizes) { // No heap, but we need the heap sizes
-				InitializeHeapSizes(row_locations, heap_sizes, offset, next, *part, layout.GetHeapSizeOffset());
+				InitializeHeapSizes(row_locations, heap_sizes, offset, next, part, layout.GetHeapSizeOffset());
 			}
 			offset += next;
 			continue;
@@ -236,29 +246,29 @@ void TupleDataAllocator::InitializeChunkStateInternal(TupleDataPinState &pin_sta
 
 		// Check if heap block has changed - re-compute the pointers within each row if so
 		if (recompute && pin_state.properties != TupleDataPinProperties::ALREADY_PINNED) {
-			const auto new_base_heap_ptr = GetBaseHeapPointer(pin_state, *part);
-			if (part->base_heap_ptr != new_base_heap_ptr) {
-				lock_guard<mutex> guard(part->lock);
-				const auto old_base_heap_ptr = part->base_heap_ptr;
+			const auto new_base_heap_ptr = GetBaseHeapPointer(pin_state, part);
+			if (part.base_heap_ptr != new_base_heap_ptr) {
+				lock_guard<mutex> guard(part.lock);
+				const auto old_base_heap_ptr = part.base_heap_ptr;
 				if (old_base_heap_ptr != new_base_heap_ptr) {
 					Vector old_heap_ptrs(
-					    Value::POINTER(CastPointerToValue(old_base_heap_ptr + part->heap_block_offset)));
+					    Value::POINTER(CastPointerToValue(old_base_heap_ptr + part.heap_block_offset)));
 					Vector new_heap_ptrs(
-					    Value::POINTER(CastPointerToValue(new_base_heap_ptr + part->heap_block_offset)));
+					    Value::POINTER(CastPointerToValue(new_base_heap_ptr + part.heap_block_offset)));
 					RecomputeHeapPointers(old_heap_ptrs, *ConstantVector::ZeroSelectionVector(), row_locations,
 					                      new_heap_ptrs, offset, next, layout, 0);
-					part->base_heap_ptr = new_base_heap_ptr;
+					part.base_heap_ptr = new_base_heap_ptr;
 				}
 			}
 		}
 
 		if (init_heap_sizes) {
-			InitializeHeapSizes(row_locations, heap_sizes, offset, next, *part, layout.GetHeapSizeOffset());
+			InitializeHeapSizes(row_locations, heap_sizes, offset, next, part, layout.GetHeapSizeOffset());
 		}
 
 		if (init_heap_pointers) {
 			// Set the pointers where the heap data will be written (if needed)
-			heap_locations[offset] = part->base_heap_ptr + part->heap_block_offset;
+			heap_locations[offset] = part.base_heap_ptr + part.heap_block_offset;
 			for (idx_t i = 1; i < next; i++) {
 				auto idx = offset + i;
 				heap_locations[idx] = heap_locations[idx - 1] + heap_sizes[idx - 1];
@@ -387,11 +397,9 @@ void TupleDataAllocator::ReleaseOrStoreHandles(TupleDataPinState &pin_state, Tup
 	ReleaseOrStoreHandles(pin_state, segment, DUMMY_CHUNK, true);
 }
 
-void TupleDataAllocator::ReleaseOrStoreHandlesInternal(TupleDataSegment &segment, vector<BufferHandle> &pinned_handles,
-                                                       unordered_map<uint32_t, BufferHandle> &handles,
-                                                       const unordered_set<uint32_t> &block_ids,
-                                                       vector<TupleDataBlock> &blocks,
-                                                       TupleDataPinProperties properties) {
+void TupleDataAllocator::ReleaseOrStoreHandlesInternal(
+    TupleDataSegment &segment, unsafe_vector<BufferHandle> &pinned_handles, perfect_map_t<BufferHandle> &handles,
+    const perfect_set_t &block_ids, unsafe_vector<TupleDataBlock> &blocks, TupleDataPinProperties properties) {
 	bool found_handle;
 	do {
 		found_handle = false;

package/src/duckdb/src/common/types/row/tuple_data_collection.cpp

@@ -27,6 +27,7 @@ TupleDataCollection::~TupleDataCollection() {
 void TupleDataCollection::Initialize() {
 	D_ASSERT(!layout.GetTypes().empty());
 	this->count = 0;
+	this->data_size = 0;
 	scatter_functions.reserve(layout.ColumnCount());
 	gather_functions.reserve(layout.ColumnCount());
 	for (idx_t col_idx = 0; col_idx < layout.ColumnCount(); col_idx++) {
@@ -67,23 +68,13 @@ idx_t TupleDataCollection::SizeInBytes() const {
 	return total_size;
 }
 
-void TupleDataCollection::GetBlockPointers(vector<data_ptr_t> &block_pointers) const {
-	D_ASSERT(segments.size() == 1);
-	const auto &segment = segments[0];
-	const auto block_count = segment.allocator->RowBlockCount();
-	D_ASSERT(segment.pinned_row_handles.size() == block_count);
-	block_pointers.resize(block_count);
-	for (idx_t block_idx = 0; block_idx < block_count; block_idx++) {
-		block_pointers[block_idx] = segment.pinned_row_handles[block_idx].Ptr();
-	}
-}
-
 void TupleDataCollection::Unpin() {
 	for (auto &segment : segments) {
 		segment.Unpin();
 	}
 }
 
+// LCOV_EXCL_START
 void VerifyAppendColumns(const TupleDataLayout &layout, const vector<column_t> &column_ids) {
 #ifdef DEBUG
 	for (idx_t col_idx = 0; col_idx < layout.ColumnCount(); col_idx++) {
@@ -105,6 +96,7 @@ void VerifyAppendColumns(const TupleDataLayout &layout, const vector<column_t> &
 	}
 #endif
 }
+// LCOV_EXCL_STOP
 
 void TupleDataCollection::InitializeAppend(TupleDataAppendState &append_state, TupleDataPinProperties properties) {
 	vector<column_t> column_ids;
@@ -260,11 +252,15 @@ void TupleDataCollection::GetVectorData(const TupleDataChunkState &chunk_state,
 
 void TupleDataCollection::Build(TupleDataPinState &pin_state, TupleDataChunkState &chunk_state,
                                 const idx_t append_offset, const idx_t append_count) {
-	segments.back().allocator->Build(segments.back(), pin_state, chunk_state, append_offset, append_count);
+	auto &segment = segments.back();
+	const auto size_before = segment.SizeInBytes();
+	segment.allocator->Build(segment, pin_state, chunk_state, append_offset, append_count);
+	data_size += segment.SizeInBytes() - size_before;
 	count += append_count;
 	Verify();
 }
 
+// LCOV_EXCL_START
 void VerifyHeapSizes(const data_ptr_t source_locations[], const idx_t heap_sizes[], const SelectionVector &append_sel,
                      const idx_t append_count, const idx_t heap_size_offset) {
 #ifdef DEBUG
@@ -275,6 +271,7 @@ void VerifyHeapSizes(const data_ptr_t source_locations[], const idx_t heap_sizes
 	}
 #endif
 }
+// LCOV_EXCL_STOP
 
 void TupleDataCollection::CopyRows(TupleDataChunkState &chunk_state, TupleDataChunkState &input,
                                    const SelectionVector &append_sel, const idx_t append_count) const {
@@ -324,12 +321,17 @@ void TupleDataCollection::Combine(TupleDataCollection &other) {
 	if (this->layout.GetTypes() != other.GetLayout().GetTypes()) {
 		throw InternalException("Attempting to combine TupleDataCollection with mismatching types");
 	}
-	this->count += other.count;
 	this->segments.reserve(this->segments.size() + other.segments.size());
 	for (auto &other_seg : other.segments) {
-		this->segments.emplace_back(std::move(other_seg));
+		AddSegment(std::move(other_seg));
 	}
 	other.Reset();
+}
+
+void TupleDataCollection::AddSegment(TupleDataSegment &&segment) {
+	count += segment.count;
+	data_size += segment.data_size;
+	segments.emplace_back(std::move(segment));
 	Verify();
 }
 
@@ -339,6 +341,7 @@ void TupleDataCollection::Combine(unique_ptr<TupleDataCollection> other) {
 
 void TupleDataCollection::Reset() {
 	count = 0;
+	data_size = 0;
 	segments.clear();
 
 	// Refreshes the TupleDataAllocator to prevent holding on to allocated data unnecessarily
@@ -395,6 +398,10 @@ bool TupleDataCollection::Scan(TupleDataScanState &state, DataChunk &result) {
 	idx_t segment_index;
 	idx_t chunk_index;
 	if (!NextScanIndex(state, segment_index, chunk_index)) {
+		if (!segments.empty()) {
+			FinalizePinState(state.pin_state, segments[segment_index_before]);
+		}
+		result.SetCardinality(0);
 		return false;
 	}
 	if (segment_index_before != DConstants::INVALID_INDEX && segment_index != segment_index_before) {
@@ -408,20 +415,21 @@ bool TupleDataCollection::Scan(TupleDataParallelScanState &gstate, TupleDataLoca
 	lstate.pin_state.properties = gstate.scan_state.pin_state.properties;
 
 	const auto segment_index_before = lstate.segment_index;
-	idx_t segment_index;
-	idx_t chunk_index;
 	{
 		lock_guard<mutex> guard(gstate.lock);
-		if (!NextScanIndex(gstate.scan_state, segment_index, chunk_index)) {
+		if (!NextScanIndex(gstate.scan_state, lstate.segment_index, lstate.chunk_index)) {
+			if (!segments.empty()) {
+				FinalizePinState(lstate.pin_state, segments[segment_index_before]);
+			}
+			result.SetCardinality(0);
 			return false;
 		}
 	}
-	if (segment_index_before != DConstants::INVALID_INDEX && segment_index_before != segment_index) {
+	if (segment_index_before != DConstants::INVALID_INDEX && segment_index_before != lstate.segment_index) {
 		FinalizePinState(lstate.pin_state, segments[lstate.segment_index]);
-		lstate.segment_index = segment_index;
 	}
-	ScanAtIndex(lstate.pin_state, lstate.chunk_state, gstate.scan_state.chunk_state.column_ids, segment_index,
-	            chunk_index, result);
+	ScanAtIndex(lstate.pin_state, lstate.chunk_state, gstate.scan_state.chunk_state.column_ids, lstate.segment_index,
+	            lstate.chunk_index, result);
 	return true;
 }
 
@@ -430,8 +438,8 @@ void TupleDataCollection::FinalizePinState(TupleDataPinState &pin_state, TupleDa
 }
 
 void TupleDataCollection::FinalizePinState(TupleDataPinState &pin_state) {
-	D_ASSERT(segments.size() == 1);
-	allocator->ReleaseOrStoreHandles(pin_state, segments.back());
+	D_ASSERT(!segments.empty());
+	FinalizePinState(pin_state, segments.back());
 }
 
 bool TupleDataCollection::NextScanIndex(TupleDataScanState &state, idx_t &segment_index, idx_t &chunk_index) {
@@ -466,6 +474,7 @@ void TupleDataCollection::ScanAtIndex(TupleDataPinState &pin_state, TupleDataChu
 	result.SetCardinality(chunk.count);
 }
 
+// LCOV_EXCL_START
 string TupleDataCollection::ToString() {
 	DataChunk chunk;
 	InitializeChunk(chunk);
@@ -493,12 +502,15 @@ void TupleDataCollection::Print() {
 
 void TupleDataCollection::Verify() const {
 #ifdef DEBUG
-	idx_t total_segment_count = 0;
+	idx_t total_count = 0;
+	idx_t total_size = 0;
 	for (const auto &segment : segments) {
 		segment.Verify();
-		total_segment_count += segment.count;
+		total_count += segment.count;
+		total_size += segment.data_size;
 	}
-	D_ASSERT(total_segment_count == this->count);
+	D_ASSERT(total_count == this->count);
+	D_ASSERT(total_size == this->data_size);
 #endif
 }
 
@@ -509,5 +521,6 @@ void TupleDataCollection::VerifyEverythingPinned() const {
 	}
 #endif
 }
+// LCOV_EXCL_STOP
 
 } // namespace duckdb

package/src/duckdb/src/common/types/row/tuple_data_layout.cpp

@@ -5,7 +5,8 @@
 namespace duckdb {
 
 TupleDataLayout::TupleDataLayout()
-    : flag_width(0), data_width(0), aggr_width(0), row_width(0), all_constant(true), heap_size_offset(0) {
+    : flag_width(0), data_width(0), aggr_width(0), row_width(0), all_constant(true), heap_size_offset(0),
+      has_destructor(false) {
 }
 
 TupleDataLayout TupleDataLayout::Copy() const {
@@ -25,6 +26,7 @@ TupleDataLayout TupleDataLayout::Copy() const {
 	result.offsets = this->offsets;
 	result.all_constant = this->all_constant;
 	result.heap_size_offset = this->heap_size_offset;
+	result.has_destructor = this->has_destructor;
 	return result;
 }
 
@@ -106,6 +108,14 @@ void TupleDataLayout::Initialize(vector<LogicalType> types_p, Aggregates aggrega
 		row_width = AlignValue(row_width);
 	}
 #endif
+
+	has_destructor = false;
+	for (auto &aggr : GetAggregates()) {
+		if (aggr.function.destructor) {
+			has_destructor = true;
+			break;
+		}
+	}
 }
 
 void TupleDataLayout::Initialize(vector<LogicalType> types_p, bool align, bool heap_offset_p) {

package/src/duckdb/src/common/types/row/tuple_data_segment.cpp

@@ -4,7 +4,7 @@
 
 namespace duckdb {
 
-TupleDataChunkPart::TupleDataChunkPart() {
+TupleDataChunkPart::TupleDataChunkPart(mutex &lock_p) : lock(lock_p) {
 }
 
 void SwapTupleDataChunkPart(TupleDataChunkPart &a, TupleDataChunkPart &b) {
@@ -15,10 +15,10 @@ void SwapTupleDataChunkPart(TupleDataChunkPart &a, TupleDataChunkPart &b) {
 	std::swap(a.base_heap_ptr, b.base_heap_ptr);
 	std::swap(a.total_heap_size, b.total_heap_size);
 	std::swap(a.count, b.count);
-	// Cannot swap the lock, but not needed as move constructor only happens during append, lock only needed for scans
+	std::swap(a.lock, b.lock);
 }
 
-TupleDataChunkPart::TupleDataChunkPart(TupleDataChunkPart &&other) noexcept {
+TupleDataChunkPart::TupleDataChunkPart(TupleDataChunkPart &&other) noexcept : lock((other.lock)) {
 	SwapTupleDataChunkPart(*this, other);
 }
 
@@ -27,7 +27,8 @@ TupleDataChunkPart &TupleDataChunkPart::operator=(TupleDataChunkPart &&other) no
 	return *this;
 }
 
-TupleDataChunk::TupleDataChunk() : count(0) {
+TupleDataChunk::TupleDataChunk() : count(0), lock(make_unsafe_uniq<mutex>()) {
+	parts.reserve(2);
 }
 
 static inline void SwapTupleDataChunk(TupleDataChunk &a, TupleDataChunk &b) noexcept {
@@ -35,6 +36,7 @@ static inline void SwapTupleDataChunk(TupleDataChunk &a, TupleDataChunk &b) noex
 	std::swap(a.row_block_ids, b.row_block_ids);
 	std::swap(a.heap_block_ids, b.heap_block_ids);
 	std::swap(a.count, b.count);
+	std::swap(a.lock, b.lock);
 }
 
 TupleDataChunk::TupleDataChunk(TupleDataChunk &&other) noexcept {
@@ -52,6 +54,7 @@ void TupleDataChunk::AddPart(TupleDataChunkPart &&part, const TupleDataLayout &l
 	if (!layout.AllConstant() && part.total_heap_size > 0) {
 		heap_block_ids.insert(part.heap_block_index);
 	}
+	part.lock = *lock;
 	parts.emplace_back(std::move(part));
 }
 
@@ -98,7 +101,7 @@ void TupleDataChunk::MergeLastChunkPart(const TupleDataLayout &layout) {
 }
 
 TupleDataSegment::TupleDataSegment(shared_ptr<TupleDataAllocator> allocator_p)
-    : allocator(std::move(allocator_p)), count(0) {
+    : allocator(std::move(allocator_p)), count(0), data_size(0) {
 }
 
 TupleDataSegment::~TupleDataSegment() {
@@ -112,6 +115,7 @@ void SwapTupleDataSegment(TupleDataSegment &a, TupleDataSegment &b) {
 	std::swap(a.allocator, b.allocator);
 	std::swap(a.chunks, b.chunks);
 	std::swap(a.count, b.count);
+	std::swap(a.data_size, b.data_size);
 	std::swap(a.pinned_row_handles, b.pinned_row_handles);
 	std::swap(a.pinned_heap_handles, b.pinned_heap_handles);
 }
@@ -130,17 +134,7 @@ idx_t TupleDataSegment::ChunkCount() const {
 }
 
 idx_t TupleDataSegment::SizeInBytes() const {
-	const auto &layout = allocator->GetLayout();
-	idx_t total_size = 0;
-	for (const auto &chunk : chunks) {
-		total_size += chunk.count * layout.GetRowWidth();
-		if (!layout.AllConstant()) {
-			for (const auto &part : chunk.parts) {
-				total_size += part.total_heap_size;
-			}
-		}
-	}
-	return total_size;
+	return data_size;
 }
 
 void TupleDataSegment::Unpin() {
@@ -151,12 +145,23 @@ void TupleDataSegment::Unpin() {
 
 void TupleDataSegment::Verify() const {
 #ifdef DEBUG
+	const auto &layout = allocator->GetLayout();
+
 	idx_t total_count = 0;
+	idx_t total_size = 0;
 	for (const auto &chunk : chunks) {
 		chunk.Verify();
 		total_count += chunk.count;
+
+		total_size += chunk.count * layout.GetRowWidth();
+		if (!layout.AllConstant()) {
+			for (const auto &part : chunk.parts) {
+				total_size += part.total_heap_size;
+			}
+		}
 	}
 	D_ASSERT(total_count == this->count);
+	D_ASSERT(total_size == this->data_size);
 #endif
 }
 

package/src/duckdb/src/common/types/value.cpp

@@ -1780,58 +1780,69 @@ void Value::FormatSerialize(FormatSerializer &serializer) const {
 	serializer.WriteProperty(101, "is_null", is_null);
 	if (!IsNull()) {
 		switch (type_.InternalType()) {
+		case PhysicalType::BIT:
+			throw InternalException("BIT type should not be serialized");
 		case PhysicalType::BOOL:
-			serializer.WriteProperty(100, "value", value_.boolean);
+			serializer.WriteProperty(102, "value", value_.boolean);
 			break;
 		case PhysicalType::INT8:
-			serializer.WriteProperty(100, "value", value_.tinyint);
+			serializer.WriteProperty(102, "value", value_.tinyint);
 			break;
 		case PhysicalType::INT16:
-			serializer.WriteProperty(100, "value", value_.smallint);
+			serializer.WriteProperty(102, "value", value_.smallint);
 			break;
 		case PhysicalType::INT32:
-			serializer.WriteProperty(100, "value", value_.integer);
+			serializer.WriteProperty(102, "value", value_.integer);
 			break;
 		case PhysicalType::INT64:
-			serializer.WriteProperty(100, "value", value_.bigint);
+			serializer.WriteProperty(102, "value", value_.bigint);
 			break;
 		case PhysicalType::UINT8:
-			serializer.WriteProperty(100, "value", value_.utinyint);
+			serializer.WriteProperty(102, "value", value_.utinyint);
 			break;
 		case PhysicalType::UINT16:
-			serializer.WriteProperty(100, "value", value_.usmallint);
+			serializer.WriteProperty(102, "value", value_.usmallint);
 			break;
 		case PhysicalType::UINT32:
-			serializer.WriteProperty(100, "value", value_.uinteger);
+			serializer.WriteProperty(102, "value", value_.uinteger);
 			break;
 		case PhysicalType::UINT64:
-			serializer.WriteProperty(100, "value", value_.ubigint);
+			serializer.WriteProperty(102, "value", value_.ubigint);
 			break;
 		case PhysicalType::INT128:
-			serializer.WriteProperty(100, "value", value_.hugeint);
+			serializer.WriteProperty(102, "value", value_.hugeint);
 			break;
 		case PhysicalType::FLOAT:
-			serializer.WriteProperty(100, "value", value_.float_);
+			serializer.WriteProperty(102, "value", value_.float_);
 			break;
 		case PhysicalType::DOUBLE:
-			serializer.WriteProperty(100, "value", value_.double_);
+			serializer.WriteProperty(102, "value", value_.double_);
 			break;
 		case PhysicalType::INTERVAL:
-			serializer.WriteProperty(100, "value", value_.interval);
+			serializer.WriteProperty(102, "value", value_.interval);
 			break;
-		case PhysicalType::VARCHAR:
+		case PhysicalType::VARCHAR: {
 			if (type_.id() == LogicalTypeId::BLOB) {
 				auto blob_str = Blob::ToString(StringValue::Get(*this));
-				serializer.WriteProperty(100, "value", blob_str);
+				serializer.WriteProperty(102, "value", blob_str);
 			} else {
-				serializer.WriteProperty(100, "value", StringValue::Get(*this));
+				serializer.WriteProperty(102, "value", StringValue::Get(*this));
 			}
-			break;
-		default: {
-			Vector v(*this);
-			v.FormatSerialize(serializer, 1);
-			break;
-		}
+		} break;
+		case PhysicalType::LIST: {
+			serializer.WriteObject(102, "value", [&](FormatSerializer &serializer) {
+				auto &children = ListValue::GetChildren(*this);
+				serializer.WriteProperty(100, "children", children);
+			});
+		} break;
+		case PhysicalType::STRUCT: {
+			serializer.WriteObject(102, "value", [&](FormatSerializer &serializer) {
+				auto &children = StructValue::GetChildren(*this);
+				serializer.WriteProperty(100, "children", children);
+			});
+		} break;
+		default:
+			throw NotImplementedException("Unimplemented type for FormatSerialize");
 		}
 	}
 }
@@ -1845,59 +1856,69 @@ Value Value::FormatDeserialize(FormatDeserializer &deserializer) {
 	}
 	new_value.is_null = false;
 	switch (type.InternalType()) {
+	case PhysicalType::BIT:
+		throw InternalException("BIT type should not be deserialized");
 	case PhysicalType::BOOL:
-		new_value.value_.boolean = deserializer.ReadProperty<bool>(100, "value");
+		new_value.value_.boolean = deserializer.ReadProperty<bool>(102, "value");
 		break;
 	case PhysicalType::UINT8:
-		new_value.value_.utinyint = deserializer.ReadProperty<uint8_t>(100, "value");
+		new_value.value_.utinyint = deserializer.ReadProperty<uint8_t>(102, "value");
 		break;
 	case PhysicalType::INT8:
-		new_value.value_.tinyint = deserializer.ReadProperty<int8_t>(100, "value");
+		new_value.value_.tinyint = deserializer.ReadProperty<int8_t>(102, "value");
 		break;
 	case PhysicalType::UINT16:
-		new_value.value_.usmallint = deserializer.ReadProperty<uint16_t>(100, "value");
+		new_value.value_.usmallint = deserializer.ReadProperty<uint16_t>(102, "value");
 		break;
 	case PhysicalType::INT16:
-		new_value.value_.smallint = deserializer.ReadProperty<int16_t>(100, "value");
+		new_value.value_.smallint = deserializer.ReadProperty<int16_t>(102, "value");
 		break;
 	case PhysicalType::UINT32:
-		new_value.value_.uinteger = deserializer.ReadProperty<uint32_t>(100, "value");
+		new_value.value_.uinteger = deserializer.ReadProperty<uint32_t>(102, "value");
 		break;
 	case PhysicalType::INT32:
-		new_value.value_.integer = deserializer.ReadProperty<int32_t>(100, "value");
+		new_value.value_.integer = deserializer.ReadProperty<int32_t>(102, "value");
 		break;
 	case PhysicalType::UINT64:
-		new_value.value_.ubigint = deserializer.ReadProperty<uint64_t>(100, "value");
+		new_value.value_.ubigint = deserializer.ReadProperty<uint64_t>(102, "value");
 		break;
 	case PhysicalType::INT64:
-		new_value.value_.bigint = deserializer.ReadProperty<int64_t>(100, "value");
+		new_value.value_.bigint = deserializer.ReadProperty<int64_t>(102, "value");
 		break;
 	case PhysicalType::INT128:
-		new_value.value_.hugeint = deserializer.ReadProperty<hugeint_t>(100, "value");
+		new_value.value_.hugeint = deserializer.ReadProperty<hugeint_t>(102, "value");
 		break;
 	case PhysicalType::FLOAT:
-		new_value.value_.float_ = deserializer.ReadProperty<float>(100, "value");
+		new_value.value_.float_ = deserializer.ReadProperty<float>(102, "value");
 		break;
 	case PhysicalType::DOUBLE:
-		new_value.value_.double_ = deserializer.ReadProperty<double>(100, "value");
+		new_value.value_.double_ = deserializer.ReadProperty<double>(102, "value");
 		break;
 	case PhysicalType::INTERVAL:
-		new_value.value_.interval = deserializer.ReadProperty<interval_t>(100, "value");
+		new_value.value_.interval = deserializer.ReadProperty<interval_t>(102, "value");
 		break;
 	case PhysicalType::VARCHAR: {
-		auto str = deserializer.ReadProperty<string>(100, "value");
+		auto str = deserializer.ReadProperty<string>(102, "value");
 		if (type.id() == LogicalTypeId::BLOB) {
 			new_value.value_info_ = make_shared<StringValueInfo>(Blob::ToBlob(str));
 		} else {
 			new_value.value_info_ = make_shared<StringValueInfo>(str);
 		}
 	} break;
-	default: {
-		Vector v(type);
-		v.FormatDeserialize(deserializer, 1);
-		new_value = v.GetValue(0);
-		break;
-	}
+	case PhysicalType::LIST: {
+		deserializer.ReadObject(102, "value", [&](FormatDeserializer &obj) {
+			auto children = obj.ReadProperty<vector<Value>>(100, "children");
+			new_value.value_info_ = make_shared<NestedValueInfo>(children);
+		});
+	} break;
+	case PhysicalType::STRUCT: {
+		deserializer.ReadObject(102, "value", [&](FormatDeserializer &obj) {
+			auto children = obj.ReadProperty<vector<Value>>(100, "children");
+			new_value.value_info_ = make_shared<NestedValueInfo>(children);
+		});
+	} break;
+	default:
+		throw NotImplementedException("Unimplemented type for FormatDeserialize");
 	}
 	return new_value;
 }