duckdb 0.8.2-dev4142.0 → 0.8.2-dev4314.0

package/src/duckdb/src/storage/checkpoint_manager.cpp

@@ -8,18 +8,13 @@
 #include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
 #include "duckdb/catalog/catalog_entry/type_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp"
-#include "duckdb/common/field_writer.hpp"
-#include "duckdb/common/serializer.hpp"
 #include "duckdb/execution/index/art/art.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/main/connection.hpp"
 #include "duckdb/main/database.hpp"
-#include "duckdb/parser/column_definition.hpp"
 #include "duckdb/parser/parsed_data/create_schema_info.hpp"
-#include "duckdb/parser/parsed_data/create_table_info.hpp"
 #include "duckdb/parser/parsed_data/create_view_info.hpp"
-#include "duckdb/parser/tableref/basetableref.hpp"
 #include "duckdb/planner/binder.hpp"
 #include "duckdb/planner/bound_tableref.hpp"
 #include "duckdb/planner/expression_binder/index_binder.hpp"
@@ -31,6 +26,8 @@
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
 #include "duckdb/transaction/transaction_manager.hpp"
 #include "duckdb/main/attached_database.hpp"
+#include "duckdb/common/serializer/binary_serializer.hpp"
+#include "duckdb/common/serializer/binary_deserializer.hpp"
 
 namespace duckdb {
 
@@ -54,7 +51,7 @@ MetadataManager &SingleFileCheckpointWriter::GetMetadataManager() {
 }
 
 unique_ptr<TableDataWriter> SingleFileCheckpointWriter::GetTableDataWriter(TableCatalogEntry &table) {
-	return make_uniq<SingleFileTableDataWriter>(*this, table, *table_metadata_writer, GetMetadataWriter());
+	return make_uniq<SingleFileTableDataWriter>(*this, table, *table_metadata_writer);
 }
 
 void SingleFileCheckpointWriter::CreateCheckpoint() {
@@ -81,11 +78,33 @@ void SingleFileCheckpointWriter::CreateCheckpoint() {
 	auto &catalog = Catalog::GetCatalog(db).Cast<DuckCatalog>();
 	catalog.ScanSchemas([&](SchemaCatalogEntry &entry) { schemas.push_back(entry); });
 	// write the actual data into the database
-	// write the amount of schemas
-	metadata_writer->Write<uint32_t>(schemas.size());
-	for (auto &schema : schemas) {
-		WriteSchema(schema.get());
-	}
+
+	// Create a serializer to write the checkpoint data
+	// The serialized format is roughly:
+	/*
+	    {
+	        schemas: [
+	            {
+	                schema: <schema_info>,
+	                custom_types: [ { type: <type_info> }, ... ],
+	                sequences: [ { sequence: <sequence_info> }, ... ],
+	                tables: [ { table: <table_info> }, ... ],
+	                views: [ { view: <view_info> }, ... ],
+	                macros: [ { macro: <macro_info> }, ... ],
+	                table_macros: [ { table_macro: <table_macro_info> }, ... ],
+	                indexes: [ { index: <index_info>, root_offset <block_ptr> }, ... ]
+	            }
+	        ]
+	    }
+	 */
+	BinarySerializer serializer(*metadata_writer);
+	serializer.Begin();
+	serializer.WriteList(100, "schemas", schemas.size(), [&](Serializer::List &list, idx_t i) {
+		auto &schema = schemas[i];
+		list.WriteObject([&](Serializer &obj) { WriteSchema(schema.get(), obj); });
+	});
+	serializer.End();
+
 	partial_block_manager.FlushPartialBlocks();
 	metadata_writer->Flush();
 	table_metadata_writer->Flush();
@@ -121,6 +140,15 @@ void SingleFileCheckpointWriter::CreateCheckpoint() {
 	metadata_manager.MarkBlocksAsModified();
 }
 
+void CheckpointReader::LoadCheckpoint(ClientContext &context, MetadataReader &reader) {
+	BinaryDeserializer deserializer(reader);
+	deserializer.Begin();
+	deserializer.ReadList(100, "schemas", [&](Deserializer::List &list, idx_t i) {
+		return list.ReadObject([&](Deserializer &obj) { ReadSchema(context, obj); });
+	});
+	deserializer.End();
+}
+
 MetadataManager &SingleFileCheckpointReader::GetMetadataManager() {
 	return storage.block_manager->GetMetadataManager();
 }
@@ -143,20 +171,42 @@ void SingleFileCheckpointReader::LoadFromStorage() {
 	con.Commit();
 }
 
-void CheckpointReader::LoadCheckpoint(ClientContext &context, MetadataReader &reader) {
-	uint32_t schema_count = reader.Read<uint32_t>();
-	for (uint32_t i = 0; i < schema_count; i++) {
-		ReadSchema(context, reader);
-	}
-}
-
 //===--------------------------------------------------------------------===//
 // Schema
 //===--------------------------------------------------------------------===//
-void CheckpointWriter::WriteSchema(SchemaCatalogEntry &schema) {
+void CheckpointWriter::WriteSchema(SchemaCatalogEntry &schema, Serializer &serializer) {
 	// write the schema data
-	schema.Serialize(GetMetadataWriter());
-	// then, we fetch the tables/views/sequences information
+	serializer.WriteProperty(100, "schema", &schema);
+
+	// Write the custom types
+	vector<reference<TypeCatalogEntry>> custom_types;
+	schema.Scan(CatalogType::TYPE_ENTRY, [&](CatalogEntry &entry) {
+		if (entry.internal) {
+			return;
+		}
+		custom_types.push_back(entry.Cast<TypeCatalogEntry>());
+	});
+
+	serializer.WriteList(101, "custom_types", custom_types.size(), [&](Serializer::List &list, idx_t i) {
+		auto &entry = custom_types[i];
+		list.WriteObject([&](Serializer &obj) { WriteType(entry, obj); });
+	});
+
+	// Write the sequences
+	vector<reference<SequenceCatalogEntry>> sequences;
+	schema.Scan(CatalogType::SEQUENCE_ENTRY, [&](CatalogEntry &entry) {
+		if (entry.internal) {
+			return;
+		}
+		sequences.push_back(entry.Cast<SequenceCatalogEntry>());
+	});
+
+	serializer.WriteList(102, "sequences", sequences.size(), [&](Serializer::List &list, idx_t i) {
+		auto &entry = sequences[i];
+		list.WriteObject([&](Serializer &obj) { WriteSequence(entry, obj); });
+	});
+
+	// Read the tables and views
 	catalog_entry_vector_t tables;
 	vector<reference<ViewCatalogEntry>> views;
 	schema.Scan(CatalogType::TABLE_ENTRY, [&](CatalogEntry &entry) {
@@ -171,22 +221,22 @@ void CheckpointWriter::WriteSchema(SchemaCatalogEntry &schema) {
 			throw NotImplementedException("Catalog type for entries");
 		}
 	});
-	vector<reference<SequenceCatalogEntry>> sequences;
-	schema.Scan(CatalogType::SEQUENCE_ENTRY, [&](CatalogEntry &entry) {
-		if (entry.internal) {
-			return;
-		}
-		sequences.push_back(entry.Cast<SequenceCatalogEntry>());
+	// Reorder tables because of foreign key constraint
+	ReorderTableEntries(tables);
+	// Tables
+	serializer.WriteList(103, "tables", tables.size(), [&](Serializer::List &list, idx_t i) {
+		auto &entry = tables[i];
+		auto &table = entry.get().Cast<TableCatalogEntry>();
+		list.WriteObject([&](Serializer &obj) { WriteTable(table, obj); });
 	});
 
-	vector<reference<TypeCatalogEntry>> custom_types;
-	schema.Scan(CatalogType::TYPE_ENTRY, [&](CatalogEntry &entry) {
-		if (entry.internal) {
-			return;
-		}
-		custom_types.push_back(entry.Cast<TypeCatalogEntry>());
+	// Views
+	serializer.WriteList(104, "views", views.size(), [&](Serializer::List &list, idx_t i) {
+		auto &entry = views[i];
+		list.WriteObject([&](Serializer &obj) { WriteView(entry.get(), obj); });
 	});
 
+	// Scalar macros
 	vector<reference<ScalarMacroCatalogEntry>> macros;
 	schema.Scan(CatalogType::SCALAR_FUNCTION_ENTRY, [&](CatalogEntry &entry) {
 		if (entry.internal) {
@@ -196,7 +246,12 @@ void CheckpointWriter::WriteSchema(SchemaCatalogEntry &schema) {
 			macros.push_back(entry.Cast<ScalarMacroCatalogEntry>());
 		}
 	});
+	serializer.WriteList(105, "macros", macros.size(), [&](Serializer::List &list, idx_t i) {
+		auto &entry = macros[i];
+		list.WriteObject([&](Serializer &obj) { WriteMacro(entry.get(), obj); });
+	});
 
+	// Table macros
 	vector<reference<TableMacroCatalogEntry>> table_macros;
 	schema.Scan(CatalogType::TABLE_FUNCTION_ENTRY, [&](CatalogEntry &entry) {
 		if (entry.internal) {
@@ -206,159 +261,122 @@ void CheckpointWriter::WriteSchema(SchemaCatalogEntry &schema) {
 			table_macros.push_back(entry.Cast<TableMacroCatalogEntry>());
 		}
 	});
+	serializer.WriteList(106, "table_macros", table_macros.size(), [&](Serializer::List &list, idx_t i) {
+		auto &entry = table_macros[i];
+		list.WriteObject([&](Serializer &obj) { WriteTableMacro(entry.get(), obj); });
+	});
 
+	// Indexes
 	vector<reference<IndexCatalogEntry>> indexes;
 	schema.Scan(CatalogType::INDEX_ENTRY, [&](CatalogEntry &entry) {
 		D_ASSERT(!entry.internal);
 		indexes.push_back(entry.Cast<IndexCatalogEntry>());
 	});
 
-	FieldWriter writer(GetMetadataWriter());
-	writer.WriteField<uint32_t>(custom_types.size());
-	writer.WriteField<uint32_t>(sequences.size());
-	writer.WriteField<uint32_t>(tables.size());
-	writer.WriteField<uint32_t>(views.size());
-	writer.WriteField<uint32_t>(macros.size());
-	writer.WriteField<uint32_t>(table_macros.size());
-	writer.WriteField<uint32_t>(indexes.size());
-	writer.Finalize();
-
-	// write the custom_types
-	for (auto &custom_type : custom_types) {
-		WriteType(custom_type);
-	}
+	serializer.WriteList(107, "indexes", indexes.size(), [&](Serializer::List &list, idx_t i) {
+		auto &entry = indexes[i];
+		list.WriteObject([&](Serializer &obj) { WriteIndex(entry.get(), obj); });
+	});
+}
 
-	// write the sequences
-	for (auto &seq : sequences) {
-		WriteSequence(seq);
-	}
-	// reorder tables because of foreign key constraint
-	ReorderTableEntries(tables);
-	// Write the tables
-	for (auto &entry : tables) {
-		auto &table = entry.get().Cast<TableCatalogEntry>();
-		WriteTable(table);
-	}
-	// Write the views
-	for (auto &view : views) {
-		WriteView(view);
-	}
+void CheckpointReader::ReadSchema(ClientContext &context, Deserializer &deserializer) {
+	// Read the schema and create it in the catalog
+	auto info = deserializer.ReadProperty<unique_ptr<CreateInfo>>(100, "schema");
+	auto &schema_info = info->Cast<CreateSchemaInfo>();
 
-	// Write the macros
-	for (auto &macro : macros) {
-		WriteMacro(macro);
-	}
+	// we set create conflict to IGNORE_ON_CONFLICT, so that we can ignore a failure when recreating the main schema
+	schema_info.on_conflict = OnCreateConflict::IGNORE_ON_CONFLICT;
+	catalog.CreateSchema(context, schema_info);
 
-	// Write the table's macros
-	for (auto &macro : table_macros) {
-		WriteTableMacro(macro);
-	}
-	// Write the indexes
-	for (auto &index : indexes) {
-		WriteIndex(index);
-	}
-}
+	// Read the custom types
+	deserializer.ReadList(101, "custom_types", [&](Deserializer::List &list, idx_t i) {
+		return list.ReadObject([&](Deserializer &obj) { ReadType(context, obj); });
+	});
 
-void CheckpointReader::ReadSchema(ClientContext &context, MetadataReader &reader) {
-	// read the schema and create it in the catalog
-	auto info = CatalogEntry::Deserialize(reader);
+	// Read the sequences
+	deserializer.ReadList(102, "sequences", [&](Deserializer::List &list, idx_t i) {
+		return list.ReadObject([&](Deserializer &obj) { ReadSequence(context, obj); });
+	});
 
-	// we set create conflict to IGNORE_ON_CONFLICT, so that we can ignore a failure when recreating the main schema
-	info->on_conflict = OnCreateConflict::IGNORE_ON_CONFLICT;
-	catalog.CreateSchema(context, info->Cast<CreateSchemaInfo>());
-
-	// first read all the counts
-	FieldReader field_reader(reader);
-	uint32_t enum_count = field_reader.ReadRequired<uint32_t>();
-	uint32_t seq_count = field_reader.ReadRequired<uint32_t>();
-	uint32_t table_count = field_reader.ReadRequired<uint32_t>();
-	uint32_t view_count = field_reader.ReadRequired<uint32_t>();
-	uint32_t macro_count = field_reader.ReadRequired<uint32_t>();
-	uint32_t table_macro_count = field_reader.ReadRequired<uint32_t>();
-	uint32_t table_index_count = field_reader.ReadRequired<uint32_t>();
-	field_reader.Finalize();
-
-	// now read the enums
-	for (uint32_t i = 0; i < enum_count; i++) {
-		ReadType(context, reader);
-	}
+	// Read the tables
+	deserializer.ReadList(103, "tables", [&](Deserializer::List &list, idx_t i) {
+		return list.ReadObject([&](Deserializer &obj) { ReadTable(context, obj); });
+	});
 
-	// read the sequences
-	for (uint32_t i = 0; i < seq_count; i++) {
-		ReadSequence(context, reader);
-	}
-	// read the table count and recreate the tables
-	for (uint32_t i = 0; i < table_count; i++) {
-		ReadTable(context, reader);
-	}
-	// now read the views
-	for (uint32_t i = 0; i < view_count; i++) {
-		ReadView(context, reader);
-	}
+	// Read the views
+	deserializer.ReadList(104, "views", [&](Deserializer::List &list, idx_t i) {
+		return list.ReadObject([&](Deserializer &obj) { ReadView(context, obj); });
+	});
 
-	// finally read the macro's
-	for (uint32_t i = 0; i < macro_count; i++) {
-		ReadMacro(context, reader);
-	}
+	// Read the macros
+	deserializer.ReadList(105, "macros", [&](Deserializer::List &list, idx_t i) {
+		return list.ReadObject([&](Deserializer &obj) { ReadMacro(context, obj); });
+	});
 
-	for (uint32_t i = 0; i < table_macro_count; i++) {
-		ReadTableMacro(context, reader);
-	}
-	for (uint32_t i = 0; i < table_index_count; i++) {
-		ReadIndex(context, reader);
-	}
+	// Read the table macros
+	deserializer.ReadList(106, "table_macros", [&](Deserializer::List &list, idx_t i) {
+		return list.ReadObject([&](Deserializer &obj) { ReadTableMacro(context, obj); });
+	});
+
+	// Read the indexes
+	deserializer.ReadList(107, "indexes", [&](Deserializer::List &list, idx_t i) {
+		return list.ReadObject([&](Deserializer &obj) { ReadIndex(context, obj); });
+	});
 }
 
 //===--------------------------------------------------------------------===//
 // Views
 //===--------------------------------------------------------------------===//
-void CheckpointWriter::WriteView(ViewCatalogEntry &view) {
-	view.Serialize(GetMetadataWriter());
+void CheckpointWriter::WriteView(ViewCatalogEntry &view, Serializer &serializer) {
+	serializer.WriteProperty(100, "view", &view);
 }
 
-void CheckpointReader::ReadView(ClientContext &context, MetadataReader &reader) {
-	auto info = CatalogEntry::Deserialize(reader);
-	catalog.CreateView(context, info->Cast<CreateViewInfo>());
+void CheckpointReader::ReadView(ClientContext &context, Deserializer &deserializer) {
+	auto info = deserializer.ReadProperty<unique_ptr<CreateInfo>>(100, "view");
+	auto &view_info = info->Cast<CreateViewInfo>();
+	catalog.CreateView(context, view_info);
 }
 
 //===--------------------------------------------------------------------===//
 // Sequences
 //===--------------------------------------------------------------------===//
-void CheckpointWriter::WriteSequence(SequenceCatalogEntry &seq) {
-	seq.Serialize(GetMetadataWriter());
+void CheckpointWriter::WriteSequence(SequenceCatalogEntry &seq, Serializer &serializer) {
+	serializer.WriteProperty(100, "sequence", &seq);
 }
 
-void CheckpointReader::ReadSequence(ClientContext &context, MetadataReader &reader) {
-	auto info = SequenceCatalogEntry::Deserialize(reader);
-	catalog.CreateSequence(context, info->Cast<CreateSequenceInfo>());
+void CheckpointReader::ReadSequence(ClientContext &context, Deserializer &deserializer) {
+	auto info = deserializer.ReadProperty<unique_ptr<CreateInfo>>(100, "sequence");
+	auto &sequence_info = info->Cast<CreateSequenceInfo>();
+	catalog.CreateSequence(context, sequence_info);
 }
 
 //===--------------------------------------------------------------------===//
 // Indexes
 //===--------------------------------------------------------------------===//
-void CheckpointWriter::WriteIndex(IndexCatalogEntry &index_catalog) {
-	// we write the index data in WriteTableData
-	// here, we only write the root pointer
-	const auto root_block_pointer = index_catalog.index->GetRootBlockPointer();
-	auto &metadata_writer = GetMetadataWriter();
-	index_catalog.Serialize(metadata_writer);
-	metadata_writer.Write(root_block_pointer);
+void CheckpointWriter::WriteIndex(IndexCatalogEntry &index_catalog, Serializer &serializer) {
+	// The index data is written as part of WriteTableData.
+	// Here, we need only serialize the pointer to that data.
+	auto root_block_pointer = index_catalog.index->GetRootBlockPointer();
+	serializer.WriteProperty(100, "index", &index_catalog);
+	serializer.WriteProperty(101, "root_block_pointer", root_block_pointer);
 }
 
-void CheckpointReader::ReadIndex(ClientContext &context, MetadataReader &reader) {
-	// deserialize the index metadata
-	auto info = IndexCatalogEntry::Deserialize(reader);
+void CheckpointReader::ReadIndex(ClientContext &context, Deserializer &deserializer) {
+
+	// Deserialize the index metadata
+	auto info = deserializer.ReadProperty<unique_ptr<CreateInfo>>(100, "index");
 	auto &index_info = info->Cast<CreateIndexInfo>();
 
-	// create the index in the catalog
+	// Create the index in the catalog
 	auto &schema_catalog = catalog.GetSchema(context, info->schema);
 	auto &table_catalog =
 	    catalog.GetEntry(context, CatalogType::TABLE_ENTRY, info->schema, index_info.table).Cast<DuckTableEntry>();
 	auto &index_catalog = schema_catalog.CreateIndex(context, index_info, table_catalog)->Cast<DuckIndexEntry>();
 	index_catalog.info = table_catalog.GetStorage().info;
 
-	// we deserialize the index lazily, i.e., we only load the root block pointer
-	const auto index_block_pointer = reader.Read<BlockPointer>();
+	// We deserialize the index lazily, i.e., we do not need to load any node information
+	// except the root block pointer
+	auto index_block_pointer = deserializer.ReadProperty<BlockPointer>(101, "root_block_pointer");
 
 	// obtain the expressions of the ART from the index metadata
 	vector<unique_ptr<Expression>> unbound_expressions;
@@ -400,10 +418,10 @@ void CheckpointReader::ReadIndex(ClientContext &context, MetadataReader &reader)
 		auto &storage = table_catalog.GetStorage();
 		auto art = make_uniq<ART>(index_info.column_ids, TableIOManager::Get(storage), std::move(unbound_expressions),
 		                          index_info.constraint_type, storage.db, nullptr, index_block_pointer);
+
 		index_catalog.index = art.get();
 		storage.info->indexes.AddIndex(std::move(art));
-		break;
-	}
+	} break;
 	default:
 		throw InternalException("Unknown index type for ReadIndex");
 	}
@@ -412,79 +430,84 @@ void CheckpointReader::ReadIndex(ClientContext &context, MetadataReader &reader)
 //===--------------------------------------------------------------------===//
 // Custom Types
 //===--------------------------------------------------------------------===//
-void CheckpointWriter::WriteType(TypeCatalogEntry &type) {
-	type.Serialize(GetMetadataWriter());
+void CheckpointWriter::WriteType(TypeCatalogEntry &type, Serializer &serializer) {
+	serializer.WriteProperty(100, "type", &type);
 }
 
-void CheckpointReader::ReadType(ClientContext &context, MetadataReader &reader) {
-	auto info = TypeCatalogEntry::Deserialize(reader);
-	catalog.CreateType(context, info->Cast<CreateTypeInfo>());
+void CheckpointReader::ReadType(ClientContext &context, Deserializer &deserializer) {
+	auto info = deserializer.ReadProperty<unique_ptr<CreateInfo>>(100, "type");
+	auto &type_info = info->Cast<CreateTypeInfo>();
+	catalog.CreateType(context, type_info);
 }
 
 //===--------------------------------------------------------------------===//
 // Macro's
 //===--------------------------------------------------------------------===//
-void CheckpointWriter::WriteMacro(ScalarMacroCatalogEntry &macro) {
-	macro.Serialize(GetMetadataWriter());
+void CheckpointWriter::WriteMacro(ScalarMacroCatalogEntry &macro, Serializer &serializer) {
+	serializer.WriteProperty(100, "macro", &macro);
 }
 
-void CheckpointReader::ReadMacro(ClientContext &context, MetadataReader &reader) {
-	auto info = MacroCatalogEntry::Deserialize(reader);
-	catalog.CreateFunction(context, info->Cast<CreateMacroInfo>());
+void CheckpointReader::ReadMacro(ClientContext &context, Deserializer &deserializer) {
+	auto info = deserializer.ReadProperty<unique_ptr<CreateInfo>>(100, "macro");
+	auto &macro_info = info->Cast<CreateMacroInfo>();
+	catalog.CreateFunction(context, macro_info);
 }
 
-void CheckpointWriter::WriteTableMacro(TableMacroCatalogEntry &macro) {
-	macro.Serialize(GetMetadataWriter());
+void CheckpointWriter::WriteTableMacro(TableMacroCatalogEntry &macro, Serializer &serializer) {
+	serializer.WriteProperty(100, "table_macro", &macro);
 }
 
-void CheckpointReader::ReadTableMacro(ClientContext &context, MetadataReader &reader) {
-	auto info = MacroCatalogEntry::Deserialize(reader);
-	catalog.CreateFunction(context, info->Cast<CreateMacroInfo>());
+void CheckpointReader::ReadTableMacro(ClientContext &context, Deserializer &deserializer) {
+	auto info = deserializer.ReadProperty<unique_ptr<CreateInfo>>(100, "table_macro");
+	auto &macro_info = info->Cast<CreateMacroInfo>();
+	catalog.CreateFunction(context, macro_info);
 }
 
 //===--------------------------------------------------------------------===//
 // Table Metadata
 //===--------------------------------------------------------------------===//
-void CheckpointWriter::WriteTable(TableCatalogEntry &table) {
-	// write the table metadata
-	table.Serialize(GetMetadataWriter());
-	// now we need to write the table data.
+void CheckpointWriter::WriteTable(TableCatalogEntry &table, Serializer &serializer) {
+	// Write the table meta data
+	serializer.WriteProperty(100, "table", &table);
+
+	// Write the table data
 	if (auto writer = GetTableDataWriter(table)) {
-		writer->WriteTableData();
+		writer->WriteTableData(serializer);
 	}
 }
 
-void CheckpointReader::ReadTable(ClientContext &context, MetadataReader &reader) {
+void CheckpointReader::ReadTable(ClientContext &context, Deserializer &deserializer) {
 	// deserialize the table meta data
-	auto info = TableCatalogEntry::Deserialize(reader);
-	// bind the info
+	auto info = deserializer.ReadProperty<unique_ptr<CreateInfo>>(100, "table");
 	auto binder = Binder::CreateBinder(context);
 	auto &schema = catalog.GetSchema(context, info->schema);
 	auto bound_info = binder->BindCreateTableInfo(std::move(info), schema);
 
 	// now read the actual table data and place it into the create table info
-	ReadTableData(context, reader, *bound_info);
+	ReadTableData(context, deserializer, *bound_info);
 
 	// finally create the table in the catalog
 	catalog.CreateTable(context, *bound_info);
 }
 
-void CheckpointReader::ReadTableData(ClientContext &context, MetadataReader &reader, BoundCreateTableInfo &bound_info) {
-	auto block_pointer = reader.Read<idx_t>();
-	auto offset = reader.Read<uint64_t>();
+void CheckpointReader::ReadTableData(ClientContext &context, Deserializer &deserializer,
+                                     BoundCreateTableInfo &bound_info) {
 
-	MetadataReader table_data_reader(reader.GetMetadataManager(), MetaBlockPointer(block_pointer, offset));
-	TableDataReader data_reader(table_data_reader, bound_info);
+	// This is written in "SingleFileTableDataWriter::FinalizeTable"
+	auto table_pointer = deserializer.ReadProperty<MetaBlockPointer>(101, "table_pointer");
+	auto total_rows = deserializer.ReadProperty<idx_t>(102, "total_rows");
+	auto index_pointers = deserializer.ReadProperty<vector<BlockPointer>>(103, "index_pointers");
+
+	// FIXME: icky downcast to get the underlying MetadataReader
+	auto &binary_deserializer = dynamic_cast<BinaryDeserializer &>(deserializer);
+	auto &reader = dynamic_cast<MetadataReader &>(binary_deserializer.GetStream());
 
+	MetadataReader table_data_reader(reader.GetMetadataManager(), table_pointer);
+	TableDataReader data_reader(table_data_reader, bound_info);
 	data_reader.ReadTableData();
-	bound_info.data->total_rows = reader.Read<idx_t>();
 
-	// get the root block pointers of each index
-	idx_t index_count = reader.Read<idx_t>();
-	for (idx_t i = 0; i < index_count; i++) {
-		const auto index_pointer = reader.Read<BlockPointer>();
-		bound_info.indexes.emplace_back(index_pointer);
-	}
+	bound_info.data->total_rows = total_rows;
+	bound_info.indexes = index_pointers;
 }
 
 } // namespace duckdb

package/src/duckdb/src/storage/data_table.cpp

@@ -1282,7 +1282,7 @@ void DataTable::SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> d
 //===--------------------------------------------------------------------===//
 // Checkpoint
 //===--------------------------------------------------------------------===//
-void DataTable::Checkpoint(TableDataWriter &writer) {
+void DataTable::Checkpoint(TableDataWriter &writer, Serializer &metadata_serializer) {
 	// checkpoint each individual row group
 	// FIXME: we might want to combine adjacent row groups in case they have had deletions...
 	TableStatistics global_stats;
@@ -1295,7 +1295,7 @@ void DataTable::Checkpoint(TableDataWriter &writer) {
 	//   row-group pointers
 	//   table pointer
 	//   index data
-	writer.FinalizeTable(std::move(global_stats), info.get());
+	writer.FinalizeTable(std::move(global_stats), info.get(), metadata_serializer);
 }
 
 void DataTable::CommitDropColumn(idx_t index) {

package/src/duckdb/src/storage/metadata/metadata_manager.cpp

@@ -1,6 +1,8 @@
 #include "duckdb/storage/metadata/metadata_manager.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/storage/buffer/block_handle.hpp"
+#include "duckdb/common/serializer/write_stream.hpp"
+#include "duckdb/common/serializer/read_stream.hpp"
 
 namespace duckdb {
 
@@ -170,17 +172,17 @@ void MetadataManager::Flush() {
 	}
 }
 
-void MetadataManager::Serialize(Serializer &serializer) {
-	serializer.Write<uint64_t>(blocks.size());
+void MetadataManager::Write(WriteStream &sink) {
+	sink.Write<uint64_t>(blocks.size());
 	for (auto &kv : blocks) {
-		kv.second.Serialize(serializer);
+		kv.second.Write(sink);
 	}
 }
 
-void MetadataManager::Deserialize(Deserializer &source) {
+void MetadataManager::Read(ReadStream &source) {
 	auto block_count = source.Read<uint64_t>();
 	for (idx_t i = 0; i < block_count; i++) {
-		auto block = MetadataBlock::Deserialize(source);
+		auto block = MetadataBlock::Read(source);
 		auto entry = blocks.find(block.block_id);
 		if (entry == blocks.end()) {
 			// block does not exist yet
@@ -192,12 +194,12 @@ void MetadataManager::Deserialize(Deserializer &source) {
 	}
 }
 
-void MetadataBlock::Serialize(Serializer &serializer) {
-	serializer.Write<block_id_t>(block_id);
-	serializer.Write<idx_t>(FreeBlocksToInteger());
+void MetadataBlock::Write(WriteStream &sink) {
+	sink.Write<block_id_t>(block_id);
+	sink.Write<idx_t>(FreeBlocksToInteger());
 }
 
-MetadataBlock MetadataBlock::Deserialize(Deserializer &source) {
+MetadataBlock MetadataBlock::Read(ReadStream &source) {
 	MetadataBlock result;
 	result.block_id = source.Read<block_id_t>();
 	auto free_list = source.Read<idx_t>();