duckdb 0.8.2-dev4142.0 → 0.8.2-dev4314.0

package/src/duckdb/src/storage/table/chunk_info.cpp

@@ -1,8 +1,7 @@
 #include "duckdb/storage/table/chunk_info.hpp"
 #include "duckdb/transaction/transaction.hpp"
-#include "duckdb/common/serializer.hpp"
-#include "duckdb/common/serializer/format_serializer.hpp"
-#include "duckdb/common/serializer/format_deserializer.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
 
 namespace duckdb {
 
@@ -30,29 +29,15 @@ static bool UseVersion(TransactionData transaction, transaction_t id) {
 	return TransactionVersionOperator::UseInsertedVersion(transaction.start_time, transaction.transaction_id, id);
 }
 
-unique_ptr<ChunkInfo> ChunkInfo::Deserialize(Deserializer &source) {
-	auto type = source.Read<ChunkInfoType>();
-	switch (type) {
-	case ChunkInfoType::EMPTY_INFO:
-		return nullptr;
-	case ChunkInfoType::CONSTANT_INFO:
-		return ChunkConstantInfo::Deserialize(source);
-	case ChunkInfoType::VECTOR_INFO:
-		return ChunkVectorInfo::Deserialize(source);
-	default:
-		throw SerializationException("Could not deserialize Chunk Info Type: unrecognized type");
-	}
-}
-
-unique_ptr<ChunkInfo> ChunkInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+unique_ptr<ChunkInfo> ChunkInfo::Deserialize(Deserializer &deserializer) {
 	auto type = deserializer.ReadProperty<ChunkInfoType>(100, "type");
 	switch (type) {
 	case ChunkInfoType::EMPTY_INFO:
 		return nullptr;
 	case ChunkInfoType::CONSTANT_INFO:
-		return ChunkConstantInfo::FormatDeserialize(deserializer);
+		return ChunkConstantInfo::Deserialize(deserializer);
 	case ChunkInfoType::VECTOR_INFO:
-		return ChunkVectorInfo::FormatDeserialize(deserializer);
+		return ChunkVectorInfo::Deserialize(deserializer);
 	default:
 		throw SerializationException("Could not deserialize Chunk Info Type: unrecognized type");
 	}
@@ -94,31 +79,11 @@ void ChunkConstantInfo::CommitAppend(transaction_t commit_id, idx_t start, idx_t
 	insert_id = commit_id;
 }
 
-void ChunkConstantInfo::Serialize(Serializer &serializer) {
-	// we only need to write this node if any tuple deletions have been committed
-	bool is_deleted = insert_id >= TRANSACTION_ID_START || delete_id < TRANSACTION_ID_START;
-	if (!is_deleted) {
-		serializer.Write<ChunkInfoType>(ChunkInfoType::EMPTY_INFO);
-		return;
-	}
-	serializer.Write<ChunkInfoType>(type);
-	serializer.Write<idx_t>(start);
-}
-
 idx_t ChunkConstantInfo::GetCommittedDeletedCount(idx_t max_count) {
 	return delete_id < TRANSACTION_ID_START ? max_count : 0;
 }
 
-unique_ptr<ChunkInfo> ChunkConstantInfo::Deserialize(Deserializer &source) {
-	auto start = source.Read<idx_t>();
-
-	auto info = make_uniq<ChunkConstantInfo>(start);
-	info->insert_id = 0;
-	info->delete_id = 0;
-	return std::move(info);
-}
-
-void ChunkConstantInfo::FormatSerialize(FormatSerializer &serializer) const {
+void ChunkConstantInfo::Serialize(Serializer &serializer) const {
 	bool is_deleted = insert_id >= TRANSACTION_ID_START || delete_id < TRANSACTION_ID_START;
 	if (!is_deleted) {
 		serializer.WriteProperty(100, "type", ChunkInfoType::EMPTY_INFO);
@@ -128,7 +93,7 @@ void ChunkConstantInfo::FormatSerialize(FormatSerializer &serializer) const {
 	serializer.WriteProperty(200, "start", start);
 }
 
-unique_ptr<ChunkInfo> ChunkConstantInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+unique_ptr<ChunkInfo> ChunkConstantInfo::Deserialize(Deserializer &deserializer) {
 	auto start = deserializer.ReadProperty<idx_t>(200, "start");
 	auto info = make_uniq<ChunkConstantInfo>(start);
 	info->insert_id = 0;
@@ -253,35 +218,6 @@ void ChunkVectorInfo::CommitAppend(transaction_t commit_id, idx_t start, idx_t e
 	}
 }
 
-void ChunkVectorInfo::Serialize(Serializer &serializer) {
-	SelectionVector sel(STANDARD_VECTOR_SIZE);
-	transaction_t start_time = TRANSACTION_ID_START - 1;
-	transaction_t transaction_id = DConstants::INVALID_INDEX;
-	idx_t count = GetSelVector(start_time, transaction_id, sel, STANDARD_VECTOR_SIZE);
-	if (count == STANDARD_VECTOR_SIZE) {
-		// nothing is deleted: skip writing anything
-		serializer.Write<ChunkInfoType>(ChunkInfoType::EMPTY_INFO);
-		return;
-	}
-	if (count == 0) {
-		// everything is deleted: write a constant vector
-		serializer.Write<ChunkInfoType>(ChunkInfoType::CONSTANT_INFO);
-		serializer.Write<idx_t>(start);
-		return;
-	}
-	// write a boolean vector
-	serializer.Write<ChunkInfoType>(ChunkInfoType::VECTOR_INFO);
-	serializer.Write<idx_t>(start);
-	bool deleted_tuples[STANDARD_VECTOR_SIZE];
-	for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) {
-		deleted_tuples[i] = true;
-	}
-	for (idx_t i = 0; i < count; i++) {
-		deleted_tuples[sel.get_index(i)] = false;
-	}
-	serializer.WriteData(data_ptr_cast(deleted_tuples), sizeof(bool) * STANDARD_VECTOR_SIZE);
-}
-
 idx_t ChunkVectorInfo::GetCommittedDeletedCount(idx_t max_count) {
 	if (!any_deleted) {
 		return 0;
@@ -295,22 +231,7 @@ idx_t ChunkVectorInfo::GetCommittedDeletedCount(idx_t max_count) {
 	return delete_count;
 }
 
-unique_ptr<ChunkInfo> ChunkVectorInfo::Deserialize(Deserializer &source) {
-	auto start = source.Read<idx_t>();
-
-	auto result = make_uniq<ChunkVectorInfo>(start);
-	result->any_deleted = true;
-	bool deleted_tuples[STANDARD_VECTOR_SIZE];
-	source.ReadData(data_ptr_cast(deleted_tuples), sizeof(bool) * STANDARD_VECTOR_SIZE);
-	for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) {
-		if (deleted_tuples[i]) {
-			result->deleted[i] = 0;
-		}
-	}
-	return std::move(result);
-}
-
-void ChunkVectorInfo::FormatSerialize(FormatSerializer &serializer) const {
+void ChunkVectorInfo::Serialize(Serializer &serializer) const {
 	SelectionVector sel(STANDARD_VECTOR_SIZE);
 	transaction_t start_time = TRANSACTION_ID_START - 1;
 	transaction_t transaction_id = DConstants::INVALID_INDEX;
@@ -339,7 +260,7 @@ void ChunkVectorInfo::FormatSerialize(FormatSerializer &serializer) const {
 	serializer.WriteProperty(201, "deleted_tuples", data_ptr_cast(deleted_tuples), sizeof(bool) * STANDARD_VECTOR_SIZE);
 }
 
-unique_ptr<ChunkInfo> ChunkVectorInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+unique_ptr<ChunkInfo> ChunkVectorInfo::Deserialize(Deserializer &deserializer) {
 	auto start = deserializer.ReadProperty<idx_t>(200, "start");
 
 	auto result = make_uniq<ChunkVectorInfo>(start);

package/src/duckdb/src/storage/table/column_checkpoint_state.cpp

@@ -22,9 +22,9 @@ unique_ptr<BaseStatistics> ColumnCheckpointState::GetStatistics() {
 	return std::move(global_stats);
 }
 
-PartialBlockForCheckpoint::PartialBlockForCheckpoint(ColumnData &data, ColumnSegment &segment,
-                                                     BlockManager &block_manager, PartialBlockState state)
-    : PartialBlock(state), block_manager(block_manager), block(segment.block) {
+PartialBlockForCheckpoint::PartialBlockForCheckpoint(ColumnData &data, ColumnSegment &segment, PartialBlockState state,
+                                                     BlockManager &block_manager)
+    : PartialBlock(state, block_manager, segment.block) {
 	AddSegmentToTail(data, segment, 0);
 }
 
@@ -37,24 +37,15 @@ bool PartialBlockForCheckpoint::IsFlushed() {
 	return segments.empty();
 }
 
-void PartialBlockForCheckpoint::AddUninitializedRegion(idx_t start, idx_t end) {
-	uninitialized_regions.push_back({start, end});
-}
+void PartialBlockForCheckpoint::Flush(const idx_t free_space_left) {
 
-void PartialBlockForCheckpoint::Flush(idx_t free_space_left) {
 	if (IsFlushed()) {
 		throw InternalException("Flush called on partial block that was already flushed");
 	}
-	// if we have any free space or uninitialized regions we need to zero-initialize them
-	if (free_space_left > 0 || !uninitialized_regions.empty()) {
-		auto handle = block_manager.buffer_manager.Pin(block);
-		// memset any uninitialized regions
-		for (auto &uninitialized : uninitialized_regions) {
-			memset(handle.Ptr() + uninitialized.start, 0, uninitialized.end - uninitialized.start);
-		}
-		// memset any free space at the end of the block to 0 prior to writing to disk
-		memset(handle.Ptr() + Storage::BLOCK_SIZE - free_space_left, 0, free_space_left);
-	}
+
+	// zero-initialize unused memory
+	FlushInternal(free_space_left);
+
 	// At this point, we've already copied all data from tail_segments
 	// into the page owned by first_segment. We flush all segment data to
 	// disk with the following call.
@@ -63,6 +54,7 @@ void PartialBlockForCheckpoint::Flush(idx_t free_space_left) {
 	if (fetch_new_block) {
 		state.block_id = block_manager.GetFreeBlockId();
 	}
+
 	for (idx_t i = 0; i < segments.size(); i++) {
 		auto &segment = segments[i];
 		segment.data.IncrementVersion();
@@ -71,23 +63,18 @@ void PartialBlockForCheckpoint::Flush(idx_t free_space_left) {
 			D_ASSERT(segment.offset_in_block == 0);
 			segment.segment.ConvertToPersistent(&block_manager, state.block_id);
 			// update the block after it has been converted to a persistent segment
-			block = segment.segment.block;
+			block_handle = segment.segment.block;
 		} else {
 			// subsequent segments are MARKED as persistent - they don't need to be rewritten
-			segment.segment.MarkAsPersistent(block, segment.offset_in_block);
+			segment.segment.MarkAsPersistent(block_handle, segment.offset_in_block);
 			if (fetch_new_block) {
 				// if we fetched a new block we need to increase the reference count to the block
 				block_manager.IncreaseBlockReferenceCount(state.block_id);
 			}
 		}
 	}
-	Clear();
-}
 
-void PartialBlockForCheckpoint::Clear() {
-	uninitialized_regions.clear();
-	block.reset();
-	segments.clear();
+	Clear();
 }
 
 void PartialBlockForCheckpoint::Merge(PartialBlock &other_p, idx_t offset, idx_t other_size) {
@@ -95,13 +82,13 @@ void PartialBlockForCheckpoint::Merge(PartialBlock &other_p, idx_t offset, idx_t
 
 	auto &buffer_manager = block_manager.buffer_manager;
 	// pin the source block
-	auto old_handle = buffer_manager.Pin(other.block);
+	auto old_handle = buffer_manager.Pin(other.block_handle);
 	// pin the target block
-	auto new_handle = buffer_manager.Pin(block);
+	auto new_handle = buffer_manager.Pin(block_handle);
 	// memcpy the contents of the old block to the new block
 	memcpy(new_handle.Ptr() + offset, old_handle.Ptr(), other_size);
 
-	// now copy over all of the segments to the new block
+	// now copy over all segments to the new block
 	// move over the uninitialized regions
 	for (auto &region : other.uninitialized_regions) {
 		region.start += offset;
@@ -113,6 +100,7 @@ void PartialBlockForCheckpoint::Merge(PartialBlock &other_p, idx_t offset, idx_t
 	for (auto &segment : other.segments) {
 		AddSegmentToTail(segment.data, segment.segment, segment.offset_in_block + offset);
 	}
+
 	other.Clear();
 }
 
@@ -120,6 +108,12 @@ void PartialBlockForCheckpoint::AddSegmentToTail(ColumnData &data, ColumnSegment
 	segments.emplace_back(data, segment, offset_in_block);
 }
 
+void PartialBlockForCheckpoint::Clear() {
+	uninitialized_regions.clear();
+	block_handle.reset();
+	segments.clear();
+}
+
 void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_t segment_size) {
 	D_ASSERT(segment_size <= Storage::BLOCK_SIZE);
 	auto tuple_count = segment->count.load();
@@ -140,7 +134,7 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 		// non-constant block
 		PartialBlockAllocation allocation = partial_block_manager.GetBlockAllocation(segment_size);
 		block_id = allocation.state.block_id;
-		offset_in_block = allocation.state.offset_in_block;
+		offset_in_block = allocation.state.offset;
 
 		if (allocation.partial_block) {
 			// Use an existing block.
@@ -149,7 +143,7 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 			// pin the source block
 			auto old_handle = buffer_manager.Pin(segment->block);
 			// pin the target block
-			auto new_handle = buffer_manager.Pin(pstate.block);
+			auto new_handle = buffer_manager.Pin(pstate.block_handle);
 			// memcpy the contents of the old block to the new block
 			memcpy(new_handle.Ptr() + offset_in_block, old_handle.Ptr(), segment_size);
 			pstate.AddSegmentToTail(column_data, *segment, offset_in_block);
@@ -162,8 +156,8 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 				segment->Resize(Storage::BLOCK_SIZE);
 			}
 			D_ASSERT(offset_in_block == 0);
-			allocation.partial_block = make_uniq<PartialBlockForCheckpoint>(
-			    column_data, *segment, *allocation.block_manager, allocation.state);
+			allocation.partial_block = make_uniq<PartialBlockForCheckpoint>(column_data, *segment, allocation.state,
+			                                                                *allocation.block_manager);
 		}
 		// Writer will decide whether to reuse this block.
 		partial_block_manager.RegisterPartialBlock(std::move(allocation));
@@ -193,8 +187,8 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 	data_pointers.push_back(std::move(data_pointer));
 }
 
-void ColumnCheckpointState::WriteDataPointers(RowGroupWriter &writer) {
-	writer.WriteColumnDataPointers(*this);
+void ColumnCheckpointState::WriteDataPointers(RowGroupWriter &writer, Serializer &serializer) {
+	writer.WriteColumnDataPointers(*this, serializer);
 }
 
 } // namespace duckdb

package/src/duckdb/src/storage/table/column_data.cpp

@@ -6,7 +6,6 @@
 #include "duckdb/storage/data_pointer.hpp"
 #include "duckdb/storage/data_table.hpp"
 #include "duckdb/storage/statistics/distinct_statistics.hpp"
-#include "duckdb/storage/storage_manager.hpp"
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/list_column_data.hpp"
 #include "duckdb/storage/table/standard_column_data.hpp"
@@ -16,7 +15,8 @@
 #include "duckdb/storage/table_storage_info.hpp"
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
-#include "duckdb/main/attached_database.hpp"
+#include "duckdb/common/serializer/read_stream.hpp"
+#include "duckdb/common/serializer/binary_deserializer.hpp"
 
 namespace duckdb {
 
@@ -446,45 +446,40 @@ unique_ptr<ColumnCheckpointState> ColumnData::Checkpoint(RowGroup &row_group,
 	return checkpoint_state;
 }
 
-void ColumnData::DeserializeColumn(Deserializer &source) {
+void ColumnData::DeserializeColumn(Deserializer &deserializer) {
 	// load the data pointers for the column
 	this->count = 0;
-	idx_t data_pointer_count = source.Read<idx_t>();
-	for (idx_t data_ptr = 0; data_ptr < data_pointer_count; data_ptr++) {
-		// read the data pointer
-		auto row_start = source.Read<idx_t>();
-		auto tuple_count = source.Read<idx_t>();
-		auto block_pointer_block_id = source.Read<block_id_t>();
-		auto block_pointer_offset = source.Read<uint32_t>();
-		auto compression_type = source.Read<CompressionType>();
-		auto segment_stats = BaseStatistics::Deserialize(source, type);
-		if (stats) {
-			stats->statistics.Merge(segment_stats);
-		}
+	deserializer.Set<LogicalType &>(type);
 
-		DataPointer data_pointer(std::move(segment_stats));
-		data_pointer.row_start = row_start;
-		data_pointer.tuple_count = tuple_count;
-		data_pointer.block_pointer.block_id = block_pointer_block_id;
-		data_pointer.block_pointer.offset = block_pointer_offset;
-		data_pointer.compression_type = compression_type;
+	deserializer.ReadList(100, "data_pointers", [&](Deserializer::List &list, idx_t i) {
+		auto data_pointer = list.ReadElement<DataPointer>();
 
-		this->count += tuple_count;
+		// Update the count and statistics
+		this->count += data_pointer.tuple_count;
+		if (stats) {
+			stats->statistics.Merge(data_pointer.statistics);
+		}
 
 		// create a persistent segment
 		auto segment = ColumnSegment::CreatePersistentSegment(
 		    GetDatabase(), block_manager, data_pointer.block_pointer.block_id, data_pointer.block_pointer.offset, type,
 		    data_pointer.row_start, data_pointer.tuple_count, data_pointer.compression_type,
 		    std::move(data_pointer.statistics));
+
 		data.AppendSegment(std::move(segment));
-	}
+	});
+
+	deserializer.Unset<LogicalType>();
 }
 
 shared_ptr<ColumnData> ColumnData::Deserialize(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
-                                               idx_t start_row, Deserializer &source, const LogicalType &type,
+                                               idx_t start_row, ReadStream &source, const LogicalType &type,
                                                optional_ptr<ColumnData> parent) {
 	auto entry = ColumnData::CreateColumn(block_manager, info, column_index, start_row, type, parent);
-	entry->DeserializeColumn(source);
+	BinaryDeserializer deserializer(source);
+	deserializer.Begin();
+	entry->DeserializeColumn(deserializer);
+	deserializer.End();
 	return entry;
 }
 

package/src/duckdb/src/storage/table/list_column_data.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/storage/table/list_column_data.hpp"
 #include "duckdb/storage/statistics/list_stats.hpp"
-#include "duckdb/transaction/transaction.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
@@ -321,10 +322,12 @@ public:
 		return stats.ToUnique();
 	}
 
-	void WriteDataPointers(RowGroupWriter &writer) override {
-		ColumnCheckpointState::WriteDataPointers(writer);
-		validity_state->WriteDataPointers(writer);
-		child_state->WriteDataPointers(writer);
+	void WriteDataPointers(RowGroupWriter &writer, Serializer &serializer) override {
+		ColumnCheckpointState::WriteDataPointers(writer, serializer);
+		serializer.WriteObject(101, "validity",
+		                       [&](Serializer &serializer) { validity_state->WriteDataPointers(writer, serializer); });
+		serializer.WriteObject(102, "child_column",
+		                       [&](Serializer &serializer) { child_state->WriteDataPointers(writer, serializer); });
 	}
 };
 
@@ -346,10 +349,14 @@ unique_ptr<ColumnCheckpointState> ListColumnData::Checkpoint(RowGroup &row_group
 	return base_state;
 }
 
-void ListColumnData::DeserializeColumn(Deserializer &source) {
-	ColumnData::DeserializeColumn(source);
-	validity.DeserializeColumn(source);
-	child_column->DeserializeColumn(source);
+void ListColumnData::DeserializeColumn(Deserializer &deserializer) {
+	ColumnData::DeserializeColumn(deserializer);
+
+	deserializer.ReadObject(101, "validity",
+	                        [&](Deserializer &deserializer) { validity.DeserializeColumn(deserializer); });
+
+	deserializer.ReadObject(102, "child_column",
+	                        [&](Deserializer &deserializer) { child_column->DeserializeColumn(deserializer); });
 }
 
 void ListColumnData::GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector<duckdb::idx_t> col_path,

package/src/duckdb/src/storage/table/row_group.cpp

@@ -1,7 +1,6 @@
 #include "duckdb/storage/table/row_group.hpp"
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/common/exception.hpp"
-#include "duckdb/common/field_writer.hpp"
 #include "duckdb/storage/table/column_data.hpp"
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
 #include "duckdb/storage/table/update_segment.hpp"
@@ -17,8 +16,9 @@
 #include "duckdb/transaction/duck_transaction.hpp"
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
-#include "duckdb/common/serializer/format_serializer.hpp"
-#include "duckdb/common/serializer/format_deserializer.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
+#include "duckdb/common/serializer/binary_serializer.hpp"
 
 namespace duckdb {
 
@@ -841,94 +841,17 @@ RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, TableStatistics &gl
 		//
 		// Just as above, the state can refer to many other states, so this
 		// can cascade recursively into more pointer writes.
-		state->WriteDataPointers(writer);
+		BinarySerializer serializer(data_writer);
+		serializer.Begin();
+		state->WriteDataPointers(writer, serializer);
+		serializer.End();
 	}
 	row_group_pointer.versions = version_info;
 	Verify();
 	return row_group_pointer;
 }
 
-void RowGroup::CheckpointDeletes(VersionNode *versions, Serializer &serializer) {
-	if (!versions) {
-		// no version information: write nothing
-		serializer.Write<idx_t>(0);
-		return;
-	}
-	// first count how many ChunkInfo's we need to deserialize
-	idx_t chunk_info_count = 0;
-	for (idx_t vector_idx = 0; vector_idx < RowGroup::ROW_GROUP_VECTOR_COUNT; vector_idx++) {
-		auto chunk_info = versions->info[vector_idx].get();
-		if (!chunk_info) {
-			continue;
-		}
-		chunk_info_count++;
-	}
-	// now serialize the actual version information
-	serializer.Write<idx_t>(chunk_info_count);
-	for (idx_t vector_idx = 0; vector_idx < RowGroup::ROW_GROUP_VECTOR_COUNT; vector_idx++) {
-		auto chunk_info = versions->info[vector_idx].get();
-		if (!chunk_info) {
-			continue;
-		}
-		serializer.Write<idx_t>(vector_idx);
-		chunk_info->Serialize(serializer);
-	}
-}
-
-shared_ptr<VersionNode> RowGroup::DeserializeDeletes(Deserializer &source) {
-	auto chunk_count = source.Read<idx_t>();
-	if (chunk_count == 0) {
-		// no deletes
-		return nullptr;
-	}
-	auto version_info = make_shared<VersionNode>();
-	for (idx_t i = 0; i < chunk_count; i++) {
-		idx_t vector_index = source.Read<idx_t>();
-		if (vector_index >= RowGroup::ROW_GROUP_VECTOR_COUNT) {
-			throw Exception("In DeserializeDeletes, vector_index is out of range for the row group. Corrupted file?");
-		}
-		version_info->info[vector_index] = ChunkInfo::Deserialize(source);
-	}
-	return version_info;
-}
-
-void RowGroup::Serialize(RowGroupPointer &pointer, Serializer &main_serializer) {
-	FieldWriter writer(main_serializer);
-	writer.WriteField<uint64_t>(pointer.row_start);
-	writer.WriteField<uint64_t>(pointer.tuple_count);
-	auto &serializer = writer.GetSerializer();
-	for (auto &data_pointer : pointer.data_pointers) {
-		serializer.Write<idx_t>(data_pointer.block_pointer);
-		serializer.Write<uint32_t>(data_pointer.offset);
-	}
-	CheckpointDeletes(pointer.versions.get(), serializer);
-	writer.Finalize();
-}
-
-RowGroupPointer RowGroup::Deserialize(Deserializer &main_source, const vector<LogicalType> &columns) {
-	RowGroupPointer result;
-
-	FieldReader reader(main_source);
-	result.row_start = reader.ReadRequired<uint64_t>();
-	result.tuple_count = reader.ReadRequired<uint64_t>();
-
-	auto physical_columns = columns.size();
-	result.data_pointers.reserve(physical_columns);
-
-	auto &source = reader.GetSource();
-	for (idx_t i = 0; i < physical_columns; i++) {
-		MetaBlockPointer pointer;
-		pointer.block_pointer = source.Read<idx_t>();
-		pointer.offset = source.Read<uint32_t>();
-		result.data_pointers.push_back(pointer);
-	}
-	result.versions = DeserializeDeletes(source);
-
-	reader.Finalize();
-	return result;
-}
-
-void RowGroup::FormatSerialize(RowGroupPointer &pointer, FormatSerializer &serializer) {
+void RowGroup::Serialize(RowGroupPointer &pointer, Serializer &serializer) {
 	serializer.WriteProperty(100, "row_start", pointer.row_start);
 	serializer.WriteProperty(101, "tuple_count", pointer.tuple_count);
 	serializer.WriteProperty(102, "data_pointers", pointer.data_pointers);
@@ -954,17 +877,20 @@ void RowGroup::FormatSerialize(RowGroupPointer &pointer, FormatSerializer &seria
 
 	// now serialize the actual version information
 	serializer.WriteProperty(103, "versions_count", chunk_info_count);
-	serializer.WriteList(104, "versions", chunk_info_count, [&](FormatSerializer::List &list, idx_t i) {
+	if (chunk_info_count == 0) {
+		return;
+	}
+	serializer.WriteList(104, "versions", chunk_info_count, [&](Serializer::List &list, idx_t i) {
 		auto vector_idx = idx_map[i];
 		auto chunk_info = versions->info[vector_idx].get();
-		list.WriteObject([&](FormatSerializer &obj) {
+		list.WriteObject([&](Serializer &obj) {
 			obj.WriteProperty(100, "vector_index", vector_idx);
 			obj.WriteProperty(101, "chunk_info", const_cast<const ChunkInfo *>(chunk_info));
 		});
 	});
 }
 
-RowGroupPointer RowGroup::FormatDeserialize(FormatDeserializer &deserializer) {
+RowGroupPointer RowGroup::Deserialize(Deserializer &deserializer) {
 	RowGroupPointer result;
 	result.row_start = deserializer.ReadProperty<uint64_t>(100, "row_start");
 	result.tuple_count = deserializer.ReadProperty<uint64_t>(101, "tuple_count");
@@ -973,13 +899,11 @@ RowGroupPointer RowGroup::FormatDeserialize(FormatDeserializer &deserializer) {
 	// Deserialize Deletes
 	auto chunk_count = deserializer.ReadProperty<idx_t>(103, "versions_count");
 	if (chunk_count == 0) {
-		// no deletes
 		return result;
 	}
-
 	auto version_info = make_shared<VersionNode>();
-	deserializer.ReadList(104, "versions", [&](FormatDeserializer::List &list, idx_t i) {
-		list.ReadObject([&](FormatDeserializer &obj) {
+	deserializer.ReadList(104, "versions", [&](Deserializer::List &list, idx_t i) {
+		list.ReadObject([&](Deserializer &obj) {
 			auto vector_index = obj.ReadProperty<idx_t>(100, "vector_index");
 			if (vector_index >= RowGroup::ROW_GROUP_VECTOR_COUNT) {
 				throw Exception("In DeserializeDeletes, vector_index is out of range for the row group. Corrupted "

package/src/duckdb/src/storage/table/row_group_collection.cpp

@@ -10,6 +10,7 @@
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/table_storage_info.hpp"
+#include "duckdb/common/serializer/binary_deserializer.hpp"
 
 namespace duckdb {
 
@@ -35,7 +36,10 @@ unique_ptr<RowGroup> RowGroupSegmentTree::LoadSegment() {
 		finished_loading = true;
 		return nullptr;
 	}
-	auto row_group_pointer = RowGroup::Deserialize(*reader, collection.GetTypes());
+	BinaryDeserializer deserializer(*reader);
+	deserializer.Begin();
+	auto row_group_pointer = RowGroup::Deserialize(deserializer);
+	deserializer.End();
 	current_row_group++;
 	return make_uniq<RowGroup>(collection, std::move(row_group_pointer));
 }

package/src/duckdb/src/storage/table/standard_column_data.cpp

@@ -4,8 +4,9 @@
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/data_table.hpp"
 #include "duckdb/planner/table_filter.hpp"
-#include "duckdb/transaction/transaction.hpp"
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
 
 namespace duckdb {
 
@@ -173,9 +174,10 @@ public:
 		return std::move(global_stats);
 	}
 
-	void WriteDataPointers(RowGroupWriter &writer) override {
-		ColumnCheckpointState::WriteDataPointers(writer);
-		validity_state->WriteDataPointers(writer);
+	void WriteDataPointers(RowGroupWriter &writer, Serializer &serializer) override {
+		ColumnCheckpointState::WriteDataPointers(writer, serializer);
+		serializer.WriteObject(101, "validity",
+		                       [&](Serializer &serializer) { validity_state->WriteDataPointers(writer, serializer); });
 	}
 };
 
@@ -202,9 +204,10 @@ void StandardColumnData::CheckpointScan(ColumnSegment &segment, ColumnScanState
 	validity.ScanCommittedRange(row_group_start, offset_in_row_group, count, scan_vector);
 }
 
-void StandardColumnData::DeserializeColumn(Deserializer &source) {
-	ColumnData::DeserializeColumn(source);
-	validity.DeserializeColumn(source);
+void StandardColumnData::DeserializeColumn(Deserializer &deserializer) {
+	ColumnData::DeserializeColumn(deserializer);
+	deserializer.ReadObject(101, "validity",
+	                        [&](Deserializer &deserializer) { validity.DeserializeColumn(deserializer); });
 }
 
 void StandardColumnData::GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector<duckdb::idx_t> col_path,