duckdb 0.8.2-dev2673.0 → 0.8.2-dev2700.0

package/src/duckdb/src/storage/metadata/metadata_reader.cpp

@@ -0,0 +1,80 @@
+#include "duckdb/storage/metadata/metadata_reader.hpp"
+
+namespace duckdb {
+
+MetadataReader::MetadataReader(MetadataManager &manager, MetaBlockPointer pointer, BlockReaderType type)
+    : manager(manager), type(type), next_pointer(FromDiskPointer(pointer)), has_next_block(true), index(0), offset(0),
+      next_offset(pointer.offset), capacity(0) {
+}
+
+MetadataReader::MetadataReader(MetadataManager &manager, BlockPointer pointer)
+    : MetadataReader(manager, MetadataManager::FromBlockPointer(pointer)) {
+}
+
+MetadataPointer MetadataReader::FromDiskPointer(MetaBlockPointer pointer) {
+	if (type == BlockReaderType::EXISTING_BLOCKS) {
+		return manager.FromDiskPointer(pointer);
+	} else {
+		return manager.RegisterDiskPointer(pointer);
+	}
+}
+
+MetadataReader::~MetadataReader() {
+}
+
+void MetadataReader::ReadData(data_ptr_t buffer, idx_t read_size) {
+	while (offset + read_size > capacity) {
+		// cannot read entire entry from block
+		// first read what we can from this block
+		idx_t to_read = capacity - offset;
+		if (to_read > 0) {
+			memcpy(buffer, Ptr(), to_read);
+			read_size -= to_read;
+			buffer += to_read;
+			offset += read_size;
+		}
+		// then move to the next block
+		ReadNextBlock();
+	}
+	// we have enough left in this block to read from the buffer
+	memcpy(buffer, Ptr(), read_size);
+	offset += read_size;
+}
+
+MetaBlockPointer MetadataReader::GetMetaBlockPointer() {
+	return manager.GetDiskPointer(block.pointer, offset);
+}
+
+void MetadataReader::ReadNextBlock() {
+	if (!has_next_block) {
+		throw IOException("No more data remaining in MetadataReader");
+	}
+	block = manager.Pin(next_pointer);
+	index = next_pointer.index;
+
+	idx_t next_block = Load<idx_t>(BasePtr());
+	if (next_block == idx_t(-1)) {
+		has_next_block = false;
+	} else {
+		next_pointer = FromDiskPointer(MetaBlockPointer(next_block, 0));
+	}
+	if (next_offset < sizeof(block_id_t)) {
+		next_offset = sizeof(block_id_t);
+	}
+	if (next_offset > MetadataManager::METADATA_BLOCK_SIZE) {
+		throw InternalException("next_offset cannot be bigger than block size");
+	}
+	offset = next_offset;
+	next_offset = sizeof(block_id_t);
+	capacity = MetadataManager::METADATA_BLOCK_SIZE;
+}
+
+data_ptr_t MetadataReader::BasePtr() {
+	return block.handle.Ptr() + index * MetadataManager::METADATA_BLOCK_SIZE;
+}
+
+data_ptr_t MetadataReader::Ptr() {
+	return BasePtr() + offset;
+}
+
+} // namespace duckdb

package/src/duckdb/src/storage/metadata/metadata_writer.cpp

@@ -0,0 +1,86 @@
+#include "duckdb/storage/metadata/metadata_writer.hpp"
+#include "duckdb/storage/block_manager.hpp"
+
+namespace duckdb {
+
+MetadataWriter::MetadataWriter(MetadataManager &manager) : manager(manager), capacity(0), offset(0) {
+}
+
+MetadataWriter::~MetadataWriter() {
+	// If there's an exception during checkpoint, this can get destroyed without
+	// flushing the data...which is fine, because none of the unwritten data
+	// will be referenced.
+	//
+	// Otherwise, we should have explicitly flushed (and thereby nulled the block).
+	D_ASSERT(!block.handle.IsValid() || Exception::UncaughtException());
+}
+
+BlockPointer MetadataWriter::GetBlockPointer() {
+	return MetadataManager::ToBlockPointer(GetMetaBlockPointer());
+}
+
+MetaBlockPointer MetadataWriter::GetMetaBlockPointer() {
+	if (offset >= capacity) {
+		// at the end of the block - fetch the next block
+		NextBlock();
+		D_ASSERT(capacity > 0);
+	}
+	return manager.GetDiskPointer(block.pointer, offset);
+}
+
+MetadataHandle MetadataWriter::NextHandle() {
+	return manager.AllocateHandle();
+}
+
+void MetadataWriter::NextBlock() {
+	// now we need to get a new block id
+	auto new_handle = NextHandle();
+
+	// write the block id of the new block to the start of the current block
+	if (capacity > 0) {
+		Store<idx_t>(manager.GetDiskPointer(new_handle.pointer).block_pointer, BasePtr());
+	}
+	// now update the block id of the block
+	block = std::move(new_handle);
+	current_pointer = block.pointer;
+	offset = sizeof(idx_t);
+	capacity = MetadataManager::METADATA_BLOCK_SIZE;
+	Store<idx_t>(-1, BasePtr());
+}
+
+void MetadataWriter::WriteData(const_data_ptr_t buffer, idx_t write_size) {
+	while (offset + write_size > capacity) {
+		// we need to make a new block
+		// first copy what we can
+		D_ASSERT(offset <= capacity);
+		idx_t copy_amount = capacity - offset;
+		if (copy_amount > 0) {
+			memcpy(Ptr(), buffer, copy_amount);
+			buffer += copy_amount;
+			offset += copy_amount;
+			write_size -= copy_amount;
+		}
+		// move forward to the next block
+		NextBlock();
+	}
+	memcpy(Ptr(), buffer, write_size);
+	offset += write_size;
+}
+
+void MetadataWriter::Flush() {
+	if (offset < capacity) {
+		// clear remaining bytes of block (if any)
+		memset(Ptr(), 0, capacity - offset);
+	}
+	block.handle.Destroy();
+}
+
+data_ptr_t MetadataWriter::BasePtr() {
+	return block.handle.Ptr() + current_pointer.index * MetadataManager::METADATA_BLOCK_SIZE;
+}
+
+data_ptr_t MetadataWriter::Ptr() {
+	return BasePtr() + offset;
+}
+
+} // namespace duckdb

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -6,8 +6,8 @@
 #include "duckdb/common/serializer/buffered_deserializer.hpp"
 #include "duckdb/common/serializer/buffered_serializer.hpp"
 #include "duckdb/common/field_writer.hpp"
-#include "duckdb/storage/meta_block_reader.hpp"
-#include "duckdb/storage/meta_block_writer.hpp"
+#include "duckdb/storage/metadata/metadata_reader.hpp"
+#include "duckdb/storage/metadata/metadata_writer.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/main/config.hpp"
 
@@ -81,16 +81,16 @@ MainHeader MainHeader::Deserialize(Deserializer &source) {
 
 void DatabaseHeader::Serialize(Serializer &ser) {
 	ser.Write<uint64_t>(iteration);
-	ser.Write<block_id_t>(meta_block);
-	ser.Write<block_id_t>(free_list);
+	ser.Write<idx_t>(meta_block);
+	ser.Write<idx_t>(free_list);
 	ser.Write<uint64_t>(block_count);
 }
 
 DatabaseHeader DatabaseHeader::Deserialize(Deserializer &source) {
 	DatabaseHeader header;
 	header.iteration = source.Read<uint64_t>();
-	header.meta_block = source.Read<block_id_t>();
-	header.free_list = source.Read<block_id_t>();
+	header.meta_block = source.Read<idx_t>();
+	header.free_list = source.Read<idx_t>();
 	header.block_count = source.Read<uint64_t>();
 	return header;
 }
@@ -241,11 +241,12 @@ void SingleFileBlockManager::Initialize(DatabaseHeader &header) {
 }
 
 void SingleFileBlockManager::LoadFreeList() {
-	if (free_list_id == INVALID_BLOCK) {
+	MetaBlockPointer free_pointer(free_list_id, 0);
+	if (!free_pointer.IsValid()) {
 		// no free list
 		return;
 	}
-	MetaBlockReader reader(*this, free_list_id);
+	MetadataReader reader(GetMetadataManager(), free_pointer, BlockReaderType::REGISTER_BLOCKS);
 	auto free_list_count = reader.Read<uint64_t>();
 	free_list.clear();
 	for (idx_t i = 0; i < free_list_count; i++) {
@@ -258,10 +259,12 @@ void SingleFileBlockManager::LoadFreeList() {
 		auto usage_count = reader.Read<uint32_t>();
 		multi_use_blocks[block_id] = usage_count;
 	}
+	GetMetadataManager().Deserialize(reader);
+	GetMetadataManager().MarkBlocksAsModified();
 }
 
-bool SingleFileBlockManager::IsRootBlock(block_id_t root) {
-	return root == meta_block;
+bool SingleFileBlockManager::IsRootBlock(MetaBlockPointer root) {
+	return root.block_pointer == meta_block;
 }
 
 block_id_t SingleFileBlockManager::GetFreeBlockId() {
@@ -327,7 +330,7 @@ void SingleFileBlockManager::IncreaseBlockReferenceCount(block_id_t block_id) {
 	}
 }
 
-block_id_t SingleFileBlockManager::GetMetaBlock() {
+idx_t SingleFileBlockManager::GetMetaBlock() {
 	return meta_block;
 }
 
@@ -391,50 +394,42 @@ void SingleFileBlockManager::Truncate() {
 	handle->Truncate(BLOCK_START + max_block * Storage::BLOCK_ALLOC_SIZE);
 }
 
-vector<block_id_t> SingleFileBlockManager::GetFreeListBlocks() {
-	vector<block_id_t> free_list_blocks;
-
-	if (!free_list.empty() || !multi_use_blocks.empty() || !modified_blocks.empty()) {
-		// there are blocks in the free list or multi_use_blocks
-		// figure out how many blocks we need to write these to the file
-		auto free_list_size = sizeof(uint64_t) + sizeof(block_id_t) * (free_list.size() + modified_blocks.size());
-		auto multi_use_blocks_size =
-		    sizeof(uint64_t) + (sizeof(block_id_t) + sizeof(uint32_t)) * multi_use_blocks.size();
-		auto total_size = free_list_size + multi_use_blocks_size;
-		// because of potential alignment issues and needing to store a next pointer in a block we subtract
-		// a bit from the max block size
-		auto space_in_block = Storage::BLOCK_SIZE - 4 * sizeof(block_id_t);
-		auto total_blocks = (total_size + space_in_block - 1) / space_in_block;
-		D_ASSERT(total_size > 0);
-		D_ASSERT(total_blocks > 0);
-
-		// reserve the blocks that we are going to write
-		// since these blocks are no longer free we cannot just include them in the free list!
-		for (idx_t i = 0; i < total_blocks; i++) {
-			auto block_id = GetFreeBlockId();
-			free_list_blocks.push_back(block_id);
-		}
+vector<MetadataHandle> SingleFileBlockManager::GetFreeListBlocks() {
+	vector<MetadataHandle> free_list_blocks;
+
+	auto free_list_size = sizeof(uint64_t) + sizeof(block_id_t) * (free_list.size() + modified_blocks.size());
+	auto multi_use_blocks_size = sizeof(uint64_t) + (sizeof(block_id_t) + sizeof(uint32_t)) * multi_use_blocks.size();
+	auto metadata_blocks = sizeof(uint64_t) + (sizeof(idx_t) * 2) * GetMetadataManager().BlockCount();
+	auto total_size = free_list_size + multi_use_blocks_size + metadata_blocks;
+
+	// reserve the blocks that we are going to write
+	// since these blocks are no longer free we cannot just include them in the free list!
+	auto block_size = MetadataManager::METADATA_BLOCK_SIZE - sizeof(idx_t);
+	while (total_size > 0) {
+		auto free_list_handle = GetMetadataManager().AllocateHandle();
+		free_list_blocks.push_back(std::move(free_list_handle));
+		total_size -= MinValue<idx_t>(total_size, block_size);
 	}
 
 	return free_list_blocks;
 }
 
-class FreeListBlockWriter : public MetaBlockWriter {
+class FreeListBlockWriter : public MetadataWriter {
 public:
-	FreeListBlockWriter(BlockManager &block_manager, vector<block_id_t> &free_list_blocks_p)
-	    : MetaBlockWriter(block_manager, free_list_blocks_p[0]), free_list_blocks(free_list_blocks_p), index(1) {
+	FreeListBlockWriter(MetadataManager &manager, vector<MetadataHandle> free_list_blocks_p)
+	    : MetadataWriter(manager), free_list_blocks(std::move(free_list_blocks_p)), index(0) {
 	}
 
-	vector<block_id_t> &free_list_blocks;
+	vector<MetadataHandle> free_list_blocks;
 	idx_t index;
 
 protected:
-	block_id_t GetNextBlockId() override {
+	MetadataHandle NextHandle() override {
 		if (index >= free_list_blocks.size()) {
 			throw InternalException(
 			    "Free List Block Writer ran out of blocks, this means not enough blocks were allocated up front");
 		}
-		return free_list_blocks[index++];
+		return std::move(free_list_blocks[index++]);
 	}
 };
 
@@ -442,7 +437,7 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 	// set the iteration count
 	header.iteration = ++iteration_count;
 
-	vector<block_id_t> free_list_blocks = GetFreeListBlocks();
+	auto free_list_blocks = GetFreeListBlocks();
 
 	// now handle the free list
 	// add all modified blocks to the free list: they can now be written to again
@@ -451,20 +446,16 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 	}
 	modified_blocks.clear();
 
+	auto &metadata_manager = GetMetadataManager();
 	if (!free_list_blocks.empty()) {
 		// there are blocks to write, either in the free_list or in the modified_blocks
 		// we write these blocks specifically to the free_list_blocks
-		// a normal MetaBlockWriter will fetch blocks to use from the free_list
+		// a normal MetadataWriter will fetch blocks to use from the free_list
 		// but since we are WRITING the free_list, this behavior is sub-optimal
+		FreeListBlockWriter writer(metadata_manager, std::move(free_list_blocks));
 
-		FreeListBlockWriter writer(*this, free_list_blocks);
-
-		auto ptr = writer.GetBlockPointer();
-		D_ASSERT(ptr.block_id == free_list_blocks[0]);
-		header.free_list = ptr.block_id;
-		for (auto &block_id : free_list_blocks) {
-			modified_blocks.insert(block_id);
-		}
+		auto ptr = writer.GetMetaBlockPointer();
+		header.free_list = ptr.block_pointer;
 
 		writer.Write<uint64_t>(free_list.size());
 		for (auto &block_id : free_list) {
@@ -475,11 +466,13 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 			writer.Write<block_id_t>(entry.first);
 			writer.Write<uint32_t>(entry.second);
 		}
+		GetMetadataManager().Serialize(writer);
 		writer.Flush();
 	} else {
 		// no blocks in the free list
-		header.free_list = INVALID_BLOCK;
+		header.free_list = DConstants::INVALID_INDEX;
 	}
+	metadata_manager.Flush();
 	header.block_count = max_block;
 
 	auto &config = DBConfig::Get(db);
@@ -494,7 +487,9 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 	}
 	// set the header inside the buffer
 	header_buffer.Clear();
-	Store<DatabaseHeader>(header, header_buffer.buffer);
+	BufferedSerializer serializer;
+	header.Serialize(serializer);
+	memcpy(header_buffer.buffer, serializer.blob.data.get(), serializer.blob.size);
 	// now write the header to the file, active_header determines whether we write to h1 or h2
 	// note that if active_header is h1 we write to h2, and vice versa
 	ChecksumAndWrite(header_buffer, active_header == 1 ? Storage::FILE_HEADER_SIZE : Storage::FILE_HEADER_SIZE * 2);

package/src/duckdb/src/storage/storage_info.cpp

@@ -2,7 +2,7 @@
 
 namespace duckdb {
 
-const uint64_t VERSION_NUMBER = 54;
+const uint64_t VERSION_NUMBER = 55;
 
 struct StorageVersionInfo {
 	const char *version_name;

package/src/duckdb/src/storage/storage_manager.cpp

@@ -82,6 +82,9 @@ public:
 	BlockManager &GetBlockManagerForRowData() override {
 		return block_manager;
 	}
+	MetadataManager &GetMetadataManager() override {
+		return block_manager.GetMetadataManager();
+	}
 };
 
 SingleFileStorageManager::SingleFileStorageManager(AttachedDatabase &db, string path, bool read_only)
@@ -140,8 +143,6 @@ void SingleFileStorageManager::LoadDatabase() {
 		//! Load from storage
 		auto checkpointer = SingleFileCheckpointReader(*this);
 		checkpointer.LoadFromStorage();
-		// finish load checkpoint, clear the cached handles of meta blocks
-		block_manager->ClearMetaBlockHandles();
 		// check if the WAL file exists
 		if (fs.FileExists(wal_path)) {
 			// replay the WAL
@@ -223,7 +224,7 @@ unique_ptr<StorageCommitState> SingleFileStorageManager::GenStorageCommitState(T
 	return make_uniq<SingleFileStorageCommitState>(*this, checkpoint);
 }
 
-bool SingleFileStorageManager::IsCheckpointClean(block_id_t checkpoint_id) {
+bool SingleFileStorageManager::IsCheckpointClean(MetaBlockPointer checkpoint_id) {
 	return block_manager->IsRootBlock(checkpoint_id);
 }
 

package/src/duckdb/src/storage/table/column_data_checkpointer.cpp

@@ -242,9 +242,9 @@ void ColumnDataCheckpointer::WritePersistentSegments() {
 	}
 }
 
-void ColumnDataCheckpointer::Checkpoint(vector<SegmentNode<ColumnSegment>> nodes) {
-	D_ASSERT(!nodes.empty());
-	this->nodes = std::move(nodes);
+void ColumnDataCheckpointer::Checkpoint(vector<SegmentNode<ColumnSegment>> nodes_p) {
+	D_ASSERT(!nodes_p.empty());
+	this->nodes = std::move(nodes_p);
 	// first check if any of the segments have changes
 	if (!HasChanges()) {
 		// no changes: only need to write the metadata for this column

package/src/duckdb/src/storage/table/persistent_table_data.cpp

@@ -3,8 +3,7 @@
 
 namespace duckdb {
 
-PersistentTableData::PersistentTableData(idx_t column_count)
-    : total_rows(0), row_group_count(0), block_id(INVALID_BLOCK), offset(0) {
+PersistentTableData::PersistentTableData(idx_t column_count) : total_rows(0), row_group_count(0) {
 }
 
 PersistentTableData::~PersistentTableData() {

package/src/duckdb/src/storage/table/row_group.cpp

@@ -11,7 +11,7 @@
 #include "duckdb/planner/table_filter.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/storage/checkpoint/table_data_writer.hpp"
-#include "duckdb/storage/meta_block_reader.hpp"
+#include "duckdb/storage/metadata/metadata_reader.hpp"
 #include "duckdb/transaction/duck_transaction_manager.hpp"
 #include "duckdb/main/database.hpp"
 #include "duckdb/main/attached_database.hpp"
@@ -116,11 +116,10 @@ ColumnData &RowGroup::GetColumn(storage_t c) {
 	if (column_pointers.size() != columns.size()) {
 		throw InternalException("Lazy loading a column but the pointer was not set");
 	}
-	auto &block_manager = GetCollection().GetBlockManager();
+	auto &metadata_manager = GetCollection().GetMetadataManager();
 	auto &types = GetCollection().GetTypes();
 	auto &block_pointer = column_pointers[c];
-	MetaBlockReader column_data_reader(block_manager, block_pointer.block_id);
-	column_data_reader.offset = block_pointer.offset;
+	MetadataReader column_data_reader(metadata_manager, block_pointer);
 	this->columns[c] =
 	    ColumnData::Deserialize(GetBlockManager(), GetTableInfo(), c, start, column_data_reader, types[c], nullptr);
 	is_loaded[c] = true;
@@ -832,7 +831,7 @@ RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, TableStatistics &gl
 	for (auto &state : result.states) {
 		// get the current position of the table data writer
 		auto &data_writer = writer.GetPayloadWriter();
-		auto pointer = data_writer.GetBlockPointer();
+		auto pointer = data_writer.GetMetaBlockPointer();
 
 		// store the stats and the data pointers in the row group pointers
 		row_group_pointer.data_pointers.push_back(pointer);
@@ -898,8 +897,8 @@ void RowGroup::Serialize(RowGroupPointer &pointer, Serializer &main_serializer)
 	writer.WriteField<uint64_t>(pointer.tuple_count);
 	auto &serializer = writer.GetSerializer();
 	for (auto &data_pointer : pointer.data_pointers) {
-		serializer.Write<block_id_t>(data_pointer.block_id);
-		serializer.Write<uint64_t>(data_pointer.offset);
+		serializer.Write<idx_t>(data_pointer.block_pointer);
+		serializer.Write<uint32_t>(data_pointer.offset);
 	}
 	CheckpointDeletes(pointer.versions.get(), serializer);
 	writer.Finalize();
@@ -917,9 +916,9 @@ RowGroupPointer RowGroup::Deserialize(Deserializer &main_source, const vector<Lo
 
 	auto &source = reader.GetSource();
 	for (idx_t i = 0; i < physical_columns; i++) {
-		BlockPointer pointer;
-		pointer.block_id = source.Read<block_id_t>();
-		pointer.offset = source.Read<uint64_t>();
+		MetaBlockPointer pointer;
+		pointer.block_pointer = source.Read<idx_t>();
+		pointer.offset = source.Read<uint32_t>();
 		result.data_pointers.push_back(pointer);
 	}
 	result.versions = DeserializeDeletes(source);

package/src/duckdb/src/storage/table/row_group_collection.cpp

@@ -6,7 +6,7 @@
 #include "duckdb/planner/constraints/bound_not_null_constraint.hpp"
 #include "duckdb/storage/checkpoint/table_data_writer.hpp"
 #include "duckdb/storage/table/row_group_segment_tree.hpp"
-#include "duckdb/storage/meta_block_reader.hpp"
+#include "duckdb/storage/metadata/metadata_reader.hpp"
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/table_storage_info.hpp"
@@ -27,8 +27,7 @@ void RowGroupSegmentTree::Initialize(PersistentTableData &data) {
 	current_row_group = 0;
 	max_row_group = data.row_group_count;
 	finished_loading = false;
-	reader = make_uniq<MetaBlockReader>(collection.GetBlockManager(), data.block_id);
-	reader->offset = data.offset;
+	reader = make_uniq<MetadataReader>(collection.GetMetadataManager(), data.block_pointer);
 }
 
 unique_ptr<RowGroup> RowGroupSegmentTree::LoadSegment() {
@@ -67,6 +66,10 @@ AttachedDatabase &RowGroupCollection::GetAttached() {
 	return GetTableInfo().db;
 }
 
+MetadataManager &RowGroupCollection::GetMetadataManager() {
+	return GetBlockManager().GetMetadataManager();
+}
+
 //===--------------------------------------------------------------------===//
 // Initialize
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/storage/table_index_list.cpp

@@ -79,7 +79,7 @@ vector<column_t> TableIndexList::GetRequiredColumns() {
 	return result;
 }
 
-vector<BlockPointer> TableIndexList::SerializeIndexes(duckdb::MetaBlockWriter &writer) {
+vector<BlockPointer> TableIndexList::SerializeIndexes(duckdb::MetadataWriter &writer) {
 	vector<BlockPointer> blocks_info;
 	for (auto &index : indexes) {
 		blocks_info.emplace_back(index->Serialize(writer));

package/src/duckdb/src/storage/wal_replay.cpp

@@ -60,7 +60,7 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
 		return false;
 	} // LCOV_EXCL_STOP
 	initial_reader.reset();
-	if (checkpoint_state.checkpoint_id != INVALID_BLOCK) {
+	if (checkpoint_state.checkpoint_id.IsValid()) {
 		// there is a checkpoint flag: check if we need to deserialize the WAL
 		auto &manager = database.GetStorageManager();
 		if (manager.IsCheckpointClean(checkpoint_state.checkpoint_id)) {
@@ -523,7 +523,8 @@ void ReplayState::ReplayUpdate() {
 }
 
 void ReplayState::ReplayCheckpoint() {
-	checkpoint_id = source.Read<block_id_t>();
+	checkpoint_id.block_pointer = source.Read<idx_t>();
+	checkpoint_id.offset = source.Read<uint32_t>();
 }
 
 } // namespace duckdb

package/src/duckdb/src/storage/write_ahead_log.cpp

@@ -48,9 +48,10 @@ void WriteAheadLog::Delete() {
 //===--------------------------------------------------------------------===//
 // Write Entries
 //===--------------------------------------------------------------------===//
-void WriteAheadLog::WriteCheckpoint(block_id_t meta_block) {
+void WriteAheadLog::WriteCheckpoint(MetaBlockPointer meta_block) {
 	writer->Write<WALType>(WALType::CHECKPOINT);
-	writer->Write<block_id_t>(meta_block);
+	writer->Write<idx_t>(meta_block.block_pointer);
+	writer->Write<uint32_t>(meta_block.offset);
 }
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/ub_src_storage.cpp

@@ -14,10 +14,6 @@
 
 #include "src/storage/magic_bytes.cpp"
 
-#include "src/storage/meta_block_reader.cpp"
-
-#include "src/storage/meta_block_writer.cpp"
-
 #include "src/storage/storage_manager.cpp"
 
 #include "src/storage/standard_buffer_manager.cpp"

package/src/duckdb/ub_src_storage_metadata.cpp

@@ -0,0 +1,6 @@
+#include "src/storage/metadata/metadata_manager.cpp"
+
+#include "src/storage/metadata/metadata_reader.cpp"
+
+#include "src/storage/metadata/metadata_writer.cpp"
+

package/src/duckdb/src/include/duckdb/storage/meta_block_reader.hpp

@@ -1,46 +0,0 @@
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/storage/meta_block_reader.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-
-#pragma once
-
-#include "duckdb/common/common.hpp"
-#include "duckdb/common/serializer.hpp"
-#include "duckdb/storage/block.hpp"
-#include "duckdb/storage/buffer/buffer_handle.hpp"
-
-namespace duckdb {
-class BlockHandle;
-class BlockManager;
-class BufferHandle;
-class DatabaseInstance;
-
-//! This struct is responsible for reading meta data from disk
-class MetaBlockReader : public Deserializer {
-public:
-	MetaBlockReader(BlockManager &block_manager, block_id_t block, bool free_blocks_on_read = true);
-	~MetaBlockReader() override;
-
-	BlockManager &block_manager;
-	shared_ptr<BlockHandle> block;
-	BufferHandle handle;
-	idx_t offset;
-	block_id_t next_block;
-	bool free_blocks_on_read;
-
-public:
-	//! Read content of size read_size into the buffer
-	void ReadData(data_ptr_t buffer, idx_t read_size) override;
-
-	ClientContext &GetContext() override;
-	void SetContext(ClientContext &context_p);
-
-private:
-	void ReadNewBlock(block_id_t id);
-	optional_ptr<ClientContext> context;
-};
-} // namespace duckdb