duckdb 0.8.2-dev2669.0 → 0.8.2-dev2700.0

package/src/duckdb/src/parallel/executor.cpp

@@ -589,6 +589,9 @@ bool Executor::GetPipelinesProgress(double &current_progress) { // LCOV_EXCL_STA
 		total_cardinality += child_cardinality;
 	}
 	current_progress = 0;
+	if (total_cardinality == 0) {
+		return true;
+	}
 	for (size_t i = 0; i < progress.size(); i++) {
 		current_progress += progress[i] * double(cardinality[i]) / double(total_cardinality);
 	}

package/src/duckdb/src/storage/buffer/block_manager.cpp

@@ -2,10 +2,15 @@
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/storage/buffer/block_handle.hpp"
 #include "duckdb/storage/buffer/buffer_pool.hpp"
+#include "duckdb/storage/metadata/metadata_manager.hpp"
 
 namespace duckdb {
 
-shared_ptr<BlockHandle> BlockManager::RegisterBlock(block_id_t block_id, bool is_meta_block) {
+BlockManager::BlockManager(BufferManager &buffer_manager)
+    : buffer_manager(buffer_manager), metadata_manager(make_uniq<MetadataManager>(*this, buffer_manager)) {
+}
+
+shared_ptr<BlockHandle> BlockManager::RegisterBlock(block_id_t block_id) {
 	lock_guard<mutex> lock(blocks_lock);
 	// check if the block already exists
 	auto entry = blocks.find(block_id);
@@ -19,19 +24,11 @@ shared_ptr<BlockHandle> BlockManager::RegisterBlock(block_id_t block_id, bool is
 	}
 	// create a new block pointer for this block
 	auto result = make_shared<BlockHandle>(*this, block_id);
-	// for meta block, cache the handle in meta_blocks
-	if (is_meta_block) {
-		meta_blocks[block_id] = result;
-	}
 	// register the block pointer in the set of blocks as a weak pointer
 	blocks[block_id] = weak_ptr<BlockHandle>(result);
 	return result;
 }
 
-void BlockManager::ClearMetaBlockHandles() {
-	meta_blocks.clear();
-}
-
 shared_ptr<BlockHandle> BlockManager::ConvertToPersistent(block_id_t block_id, shared_ptr<BlockHandle> old_block) {
 	// pin the old block to ensure we have it loaded in memory
 	auto old_handle = buffer_manager.Pin(old_block);
@@ -79,6 +76,10 @@ void BlockManager::UnregisterBlock(block_id_t block_id, bool can_destroy) {
 	}
 }
 
+MetadataManager &BlockManager::GetMetadataManager() {
+	return *metadata_manager;
+}
+
 void BlockManager::Truncate() {
 }
 

package/src/duckdb/src/storage/checkpoint/row_group_writer.cpp

@@ -36,7 +36,7 @@ void SingleFileRowGroupWriter::WriteColumnDataPointers(ColumnCheckpointState &co
 	}
 }
 
-MetaBlockWriter &SingleFileRowGroupWriter::GetPayloadWriter() {
+MetadataWriter &SingleFileRowGroupWriter::GetPayloadWriter() {
 	return table_data_writer;
 }
 

package/src/duckdb/src/storage/checkpoint/table_data_reader.cpp

@@ -1,5 +1,5 @@
 #include "duckdb/storage/checkpoint/table_data_reader.hpp"
-#include "duckdb/storage/meta_block_reader.hpp"
+#include "duckdb/storage/metadata/metadata_reader.hpp"
 #include "duckdb/common/types/null_value.hpp"
 
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
@@ -10,7 +10,7 @@
 
 namespace duckdb {
 
-TableDataReader::TableDataReader(MetaBlockReader &reader, BoundCreateTableInfo &info) : reader(reader), info(info) {
+TableDataReader::TableDataReader(MetadataReader &reader, BoundCreateTableInfo &info) : reader(reader), info(info) {
 	info.data = make_uniq<PersistentTableData>(info.Base().columns.LogicalColumnCount());
 }
 
@@ -23,8 +23,7 @@ void TableDataReader::ReadTableData() {
 
 	// deserialize each of the individual row groups
 	info.data->row_group_count = reader.Read<uint64_t>();
-	info.data->block_id = reader.block->BlockId();
-	info.data->offset = reader.offset;
+	info.data->block_pointer = reader.GetMetaBlockPointer();
 }
 
 } // namespace duckdb

package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp

@@ -31,8 +31,8 @@ void TableDataWriter::AddRowGroup(RowGroupPointer &&row_group_pointer, unique_pt
 }
 
 SingleFileTableDataWriter::SingleFileTableDataWriter(SingleFileCheckpointWriter &checkpoint_manager,
-                                                     TableCatalogEntry &table, MetaBlockWriter &table_data_writer,
-                                                     MetaBlockWriter &meta_data_writer)
+                                                     TableCatalogEntry &table, MetadataWriter &table_data_writer,
+                                                     MetadataWriter &meta_data_writer)
     : TableDataWriter(table), checkpoint_manager(checkpoint_manager), table_data_writer(table_data_writer),
       meta_data_writer(meta_data_writer) {
 }
@@ -44,7 +44,7 @@ unique_ptr<RowGroupWriter> SingleFileTableDataWriter::GetRowGroupWriter(RowGroup
 void SingleFileTableDataWriter::FinalizeTable(TableStatistics &&global_stats, DataTableInfo *info) {
 	// store the current position in the metadata writer
 	// this is where the row groups for this table start
-	auto pointer = table_data_writer.GetBlockPointer();
+	auto pointer = table_data_writer.GetMetaBlockPointer();
 
 	global_stats.Serialize(table_data_writer);
 
@@ -60,18 +60,18 @@ void SingleFileTableDataWriter::FinalizeTable(TableStatistics &&global_stats, Da
 	}
 
 	// Pointer to the table itself goes to the metadata stream.
-	meta_data_writer.Write<block_id_t>(pointer.block_id);
+	meta_data_writer.Write<idx_t>(pointer.block_pointer);
 	meta_data_writer.Write<uint64_t>(pointer.offset);
 	meta_data_writer.Write<idx_t>(total_rows);
 
 	// Now we serialize indexes in the table_metadata_writer
-	vector<BlockPointer> index_pointers = info->indexes.SerializeIndexes(table_data_writer);
+	auto index_pointers = info->indexes.SerializeIndexes(table_data_writer);
 
 	// Write-off to metadata block ids and offsets of indexes
 	meta_data_writer.Write<idx_t>(index_pointers.size());
 	for (auto &block_info : index_pointers) {
-		meta_data_writer.Write<idx_t>(block_info.block_id);
-		meta_data_writer.Write<idx_t>(block_info.offset);
+		meta_data_writer.Write<block_id_t>(block_info.block_id);
+		meta_data_writer.Write<uint32_t>(block_info.offset);
 	}
 }
 

package/src/duckdb/src/storage/checkpoint_manager.cpp

@@ -27,7 +27,7 @@
 #include "duckdb/storage/block_manager.hpp"
 #include "duckdb/storage/checkpoint/table_data_reader.hpp"
 #include "duckdb/storage/checkpoint/table_data_writer.hpp"
-#include "duckdb/storage/meta_block_reader.hpp"
+#include "duckdb/storage/metadata/metadata_reader.hpp"
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
 #include "duckdb/transaction/transaction_manager.hpp"
 #include "duckdb/main/attached_database.hpp"
@@ -45,12 +45,16 @@ BlockManager &SingleFileCheckpointWriter::GetBlockManager() {
 	return *storage_manager.block_manager;
 }
 
-MetaBlockWriter &SingleFileCheckpointWriter::GetMetaBlockWriter() {
+MetadataWriter &SingleFileCheckpointWriter::GetMetadataWriter() {
 	return *metadata_writer;
 }
 
+MetadataManager &SingleFileCheckpointWriter::GetMetadataManager() {
+	return GetBlockManager().GetMetadataManager();
+}
+
 unique_ptr<TableDataWriter> SingleFileCheckpointWriter::GetTableDataWriter(TableCatalogEntry &table) {
-	return make_uniq<SingleFileTableDataWriter>(*this, table, *table_metadata_writer, GetMetaBlockWriter());
+	return make_uniq<SingleFileTableDataWriter>(*this, table, *table_metadata_writer, GetMetadataWriter());
 }
 
 void SingleFileCheckpointWriter::CreateCheckpoint() {
@@ -63,13 +67,14 @@ void SingleFileCheckpointWriter::CreateCheckpoint() {
 	D_ASSERT(!metadata_writer);
 
 	auto &block_manager = GetBlockManager();
+	auto &metadata_manager = GetMetadataManager();
 
 	//! Set up the writers for the checkpoints
-	metadata_writer = make_uniq<MetaBlockWriter>(block_manager);
-	table_metadata_writer = make_uniq<MetaBlockWriter>(block_manager);
+	metadata_writer = make_uniq<MetadataWriter>(metadata_manager);
+	table_metadata_writer = make_uniq<MetadataWriter>(metadata_manager);
 
 	// get the id of the first meta block
-	block_id_t meta_block = metadata_writer->GetBlockPointer().block_id;
+	auto meta_block = metadata_writer->GetMetaBlockPointer();
 
 	vector<reference<SchemaCatalogEntry>> schemas;
 	// we scan the set of committed schemas
@@ -82,7 +87,6 @@ void SingleFileCheckpointWriter::CreateCheckpoint() {
 		WriteSchema(schema.get());
 	}
 	partial_block_manager.FlushPartialBlocks();
-	// flush the meta data to disk
 	metadata_writer->Flush();
 	table_metadata_writer->Flush();
 
@@ -101,7 +105,7 @@ void SingleFileCheckpointWriter::CreateCheckpoint() {
 
 	// finally write the updated header
 	DatabaseHeader header;
-	header.meta_block = meta_block;
+	header.meta_block = meta_block.block_pointer;
 	block_manager.WriteHeader(header);
 
 	if (config.options.checkpoint_abort == CheckpointAbort::DEBUG_ABORT_BEFORE_TRUNCATE) {
@@ -114,29 +118,32 @@ void SingleFileCheckpointWriter::CreateCheckpoint() {
 	// truncate the file
 	block_manager.Truncate();
 
-	// mark all blocks written as part of the metadata as modified
-	metadata_writer->MarkWrittenBlocks();
-	table_metadata_writer->MarkWrittenBlocks();
+	metadata_manager.MarkBlocksAsModified();
+}
+
+MetadataManager &SingleFileCheckpointReader::GetMetadataManager() {
+	return storage.block_manager->GetMetadataManager();
 }
 
 void SingleFileCheckpointReader::LoadFromStorage() {
 	auto &block_manager = *storage.block_manager;
-	block_id_t meta_block = block_manager.GetMetaBlock();
-	if (meta_block < 0) {
+	auto &metadata_manager = GetMetadataManager();
+	MetaBlockPointer meta_block(block_manager.GetMetaBlock(), 0);
+	if (!meta_block.IsValid()) {
 		// storage is empty
 		return;
 	}
 
 	Connection con(storage.GetDatabase());
 	con.BeginTransaction();
-	// create the MetaBlockReader to read from the storage
-	MetaBlockReader reader(block_manager, meta_block);
-	reader.SetContext(*con.context);
+	// create the MetadataReader to read from the storage
+	MetadataReader reader(metadata_manager, meta_block);
+	//	reader.SetContext(*con.context);
 	LoadCheckpoint(*con.context, reader);
 	con.Commit();
 }
 
-void CheckpointReader::LoadCheckpoint(ClientContext &context, MetaBlockReader &reader) {
+void CheckpointReader::LoadCheckpoint(ClientContext &context, MetadataReader &reader) {
 	uint32_t schema_count = reader.Read<uint32_t>();
 	for (uint32_t i = 0; i < schema_count; i++) {
 		ReadSchema(context, reader);
@@ -148,7 +155,7 @@ void CheckpointReader::LoadCheckpoint(ClientContext &context, MetaBlockReader &r
 //===--------------------------------------------------------------------===//
 void CheckpointWriter::WriteSchema(SchemaCatalogEntry &schema) {
 	// write the schema data
-	schema.Serialize(GetMetaBlockWriter());
+	schema.Serialize(GetMetadataWriter());
 	// then, we fetch the tables/views/sequences information
 	vector<reference<TableCatalogEntry>> tables;
 	vector<reference<ViewCatalogEntry>> views;
@@ -206,7 +213,7 @@ void CheckpointWriter::WriteSchema(SchemaCatalogEntry &schema) {
 		indexes.push_back(entry.Cast<IndexCatalogEntry>());
 	});
 
-	FieldWriter writer(GetMetaBlockWriter());
+	FieldWriter writer(GetMetadataWriter());
 	writer.WriteField<uint32_t>(custom_types.size());
 	writer.WriteField<uint32_t>(sequences.size());
 	writer.WriteField<uint32_t>(tables.size());
@@ -251,7 +258,7 @@ void CheckpointWriter::WriteSchema(SchemaCatalogEntry &schema) {
 	}
 }
 
-void CheckpointReader::ReadSchema(ClientContext &context, MetaBlockReader &reader) {
+void CheckpointReader::ReadSchema(ClientContext &context, MetadataReader &reader) {
 	// read the schema and create it in the catalog
 	auto info = CatalogEntry::Deserialize(reader);
 	// we set create conflict to ignore to ignore the failure of recreating the main schema
@@ -304,10 +311,10 @@ void CheckpointReader::ReadSchema(ClientContext &context, MetaBlockReader &reade
 // Views
 //===--------------------------------------------------------------------===//
 void CheckpointWriter::WriteView(ViewCatalogEntry &view) {
-	view.Serialize(GetMetaBlockWriter());
+	view.Serialize(GetMetadataWriter());
 }
 
-void CheckpointReader::ReadView(ClientContext &context, MetaBlockReader &reader) {
+void CheckpointReader::ReadView(ClientContext &context, MetadataReader &reader) {
 	auto info = CatalogEntry::Deserialize(reader);
 	catalog.CreateView(context, info->Cast<CreateViewInfo>());
 }
@@ -316,10 +323,10 @@ void CheckpointReader::ReadView(ClientContext &context, MetaBlockReader &reader)
 // Sequences
 //===--------------------------------------------------------------------===//
 void CheckpointWriter::WriteSequence(SequenceCatalogEntry &seq) {
-	seq.Serialize(GetMetaBlockWriter());
+	seq.Serialize(GetMetadataWriter());
 }
 
-void CheckpointReader::ReadSequence(ClientContext &context, MetaBlockReader &reader) {
+void CheckpointReader::ReadSequence(ClientContext &context, MetadataReader &reader) {
 	auto info = SequenceCatalogEntry::Deserialize(reader);
 	catalog.CreateSequence(context, info->Cast<CreateSequenceInfo>());
 }
@@ -331,14 +338,14 @@ void CheckpointWriter::WriteIndex(IndexCatalogEntry &index_catalog) {
 	// The index data should already have been written as part of WriteTableData.
 	// Here, we need only serialize the pointer to that data.
 	auto root_offset = index_catalog.index->GetSerializedDataPointer();
-	auto &metadata_writer = GetMetaBlockWriter();
+	auto &metadata_writer = GetMetadataWriter();
 	index_catalog.Serialize(metadata_writer);
 	// Serialize the Block id and offset of root node
 	metadata_writer.Write(root_offset.block_id);
 	metadata_writer.Write(root_offset.offset);
 }
 
-void CheckpointReader::ReadIndex(ClientContext &context, MetaBlockReader &reader) {
+void CheckpointReader::ReadIndex(ClientContext &context, MetadataReader &reader) {
 	// deserialize the index metadata
 	auto info = IndexCatalogEntry::Deserialize(reader);
 	auto &index_info = info->Cast<CreateIndexInfo>();
@@ -393,8 +400,9 @@ void CheckpointReader::ReadIndex(ClientContext &context, MetaBlockReader &reader
 	switch (index_info.index_type) {
 	case IndexType::ART: {
 		auto &storage = table_catalog.GetStorage();
-		auto art = make_uniq<ART>(index_info.column_ids, TableIOManager::Get(storage), std::move(unbound_expressions),
-		                          index_info.constraint_type, storage.db, nullptr, root_block_id, root_offset);
+		auto art =
+		    make_uniq<ART>(index_info.column_ids, TableIOManager::Get(storage), std::move(unbound_expressions),
+		                   index_info.constraint_type, storage.db, nullptr, BlockPointer(root_block_id, root_offset));
 		index_catalog.index = art.get();
 		storage.info->indexes.AddIndex(std::move(art));
 		break;
@@ -408,10 +416,10 @@ void CheckpointReader::ReadIndex(ClientContext &context, MetaBlockReader &reader
 // Custom Types
 //===--------------------------------------------------------------------===//
 void CheckpointWriter::WriteType(TypeCatalogEntry &type) {
-	type.Serialize(GetMetaBlockWriter());
+	type.Serialize(GetMetadataWriter());
 }
 
-void CheckpointReader::ReadType(ClientContext &context, MetaBlockReader &reader) {
+void CheckpointReader::ReadType(ClientContext &context, MetadataReader &reader) {
 	auto info = TypeCatalogEntry::Deserialize(reader);
 	catalog.CreateType(context, info->Cast<CreateTypeInfo>());
 }
@@ -420,19 +428,19 @@ void CheckpointReader::ReadType(ClientContext &context, MetaBlockReader &reader)
 // Macro's
 //===--------------------------------------------------------------------===//
 void CheckpointWriter::WriteMacro(ScalarMacroCatalogEntry &macro) {
-	macro.Serialize(GetMetaBlockWriter());
+	macro.Serialize(GetMetadataWriter());
 }
 
-void CheckpointReader::ReadMacro(ClientContext &context, MetaBlockReader &reader) {
+void CheckpointReader::ReadMacro(ClientContext &context, MetadataReader &reader) {
 	auto info = MacroCatalogEntry::Deserialize(reader);
 	catalog.CreateFunction(context, info->Cast<CreateMacroInfo>());
 }
 
 void CheckpointWriter::WriteTableMacro(TableMacroCatalogEntry &macro) {
-	macro.Serialize(GetMetaBlockWriter());
+	macro.Serialize(GetMetadataWriter());
 }
 
-void CheckpointReader::ReadTableMacro(ClientContext &context, MetaBlockReader &reader) {
+void CheckpointReader::ReadTableMacro(ClientContext &context, MetadataReader &reader) {
 	auto info = MacroCatalogEntry::Deserialize(reader);
 	catalog.CreateFunction(context, info->Cast<CreateMacroInfo>());
 }
@@ -442,14 +450,14 @@ void CheckpointReader::ReadTableMacro(ClientContext &context, MetaBlockReader &r
 //===--------------------------------------------------------------------===//
 void CheckpointWriter::WriteTable(TableCatalogEntry &table) {
 	// write the table meta data
-	table.Serialize(GetMetaBlockWriter());
+	table.Serialize(GetMetadataWriter());
 	// now we need to write the table data.
 	if (auto writer = GetTableDataWriter(table)) {
 		writer->WriteTableData();
 	}
 }
 
-void CheckpointReader::ReadTable(ClientContext &context, MetaBlockReader &reader) {
+void CheckpointReader::ReadTable(ClientContext &context, MetadataReader &reader) {
 	// deserialize the table meta data
 	auto info = TableCatalogEntry::Deserialize(reader);
 	// bind the info
@@ -464,13 +472,11 @@ void CheckpointReader::ReadTable(ClientContext &context, MetaBlockReader &reader
 	catalog.CreateTable(context, *bound_info);
 }
 
-void CheckpointReader::ReadTableData(ClientContext &context, MetaBlockReader &reader,
-                                     BoundCreateTableInfo &bound_info) {
-	auto block_id = reader.Read<block_id_t>();
+void CheckpointReader::ReadTableData(ClientContext &context, MetadataReader &reader, BoundCreateTableInfo &bound_info) {
+	auto block_pointer = reader.Read<idx_t>();
 	auto offset = reader.Read<uint64_t>();
 
-	MetaBlockReader table_data_reader(reader.block_manager, block_id);
-	table_data_reader.offset = offset;
+	MetadataReader table_data_reader(reader.GetMetadataManager(), MetaBlockPointer(block_pointer, offset));
 	TableDataReader data_reader(table_data_reader, bound_info);
 
 	data_reader.ReadTableData();
@@ -479,8 +485,8 @@ void CheckpointReader::ReadTableData(ClientContext &context, MetaBlockReader &re
 	// Get any indexes block info
 	idx_t num_indexes = reader.Read<idx_t>();
 	for (idx_t i = 0; i < num_indexes; i++) {
-		auto idx_block_id = reader.Read<idx_t>();
-		auto idx_offset = reader.Read<idx_t>();
+		auto idx_block_id = reader.Read<block_id_t>();
+		auto idx_offset = reader.Read<uint32_t>();
 		bound_info.indexes.emplace_back(idx_block_id, idx_offset);
 	}
 }

package/src/duckdb/src/storage/index.cpp

@@ -108,7 +108,7 @@ bool Index::IndexIsUpdated(const vector<PhysicalIndex> &column_ids) const {
 	return false;
 }
 
-BlockPointer Index::Serialize(MetaBlockWriter &writer) {
+BlockPointer Index::Serialize(MetadataWriter &writer) {
 	throw NotImplementedException("The implementation of this index serialization does not exist.");
 }
 

package/src/duckdb/src/storage/metadata/metadata_manager.cpp

@@ -0,0 +1,267 @@
+#include "duckdb/storage/metadata/metadata_manager.hpp"
+#include "duckdb/storage/buffer_manager.hpp"
+#include "duckdb/storage/buffer/block_handle.hpp"
+
+namespace duckdb {
+
+MetadataManager::MetadataManager(BlockManager &block_manager, BufferManager &buffer_manager)
+    : block_manager(block_manager), buffer_manager(buffer_manager) {
+}
+
+MetadataManager::~MetadataManager() {
+}
+
+MetadataHandle MetadataManager::AllocateHandle() {
+	// check if there is any free space left in an existing block
+	// if not allocate a new bloc
+	block_id_t free_block = INVALID_BLOCK;
+	for (auto &kv : blocks) {
+		auto &block = kv.second;
+		D_ASSERT(kv.first == block.block_id);
+		if (!block.free_blocks.empty()) {
+			free_block = kv.first;
+			break;
+		}
+	}
+	if (free_block == INVALID_BLOCK) {
+		free_block = AllocateNewBlock();
+	}
+	D_ASSERT(free_block != INVALID_BLOCK);
+
+	// select the first free metadata block we can find
+	MetadataPointer pointer;
+	pointer.block_index = free_block;
+	auto &block = blocks[free_block];
+	D_ASSERT(!block.free_blocks.empty());
+	pointer.index = block.free_blocks.back();
+	// mark the block as used
+	block.free_blocks.pop_back();
+	D_ASSERT(pointer.index < METADATA_BLOCK_COUNT);
+	// pin the block
+	return Pin(pointer);
+}
+
+MetadataHandle MetadataManager::Pin(MetadataPointer pointer) {
+	D_ASSERT(pointer.index < METADATA_BLOCK_COUNT);
+	auto &block = blocks[pointer.block_index];
+
+	MetadataHandle handle;
+	handle.pointer.block_index = pointer.block_index;
+	handle.pointer.index = pointer.index;
+	handle.handle = buffer_manager.Pin(block.block);
+	return handle;
+}
+
+block_id_t MetadataManager::AllocateNewBlock() {
+	auto new_block_id = GetNextBlockId();
+
+	MetadataBlock new_block;
+	buffer_manager.Allocate(Storage::BLOCK_SIZE, false, &new_block.block);
+	new_block.block_id = new_block_id;
+	for (idx_t i = 0; i < METADATA_BLOCK_COUNT; i++) {
+		new_block.free_blocks.push_back(METADATA_BLOCK_COUNT - i - 1);
+	}
+	AddBlock(std::move(new_block));
+	return new_block_id;
+}
+
+void MetadataManager::AddBlock(MetadataBlock new_block, bool if_exists) {
+	if (blocks.find(new_block.block_id) != blocks.end()) {
+		if (if_exists) {
+			return;
+		}
+		throw InternalException("Block id with id %llu already exists", new_block.block_id);
+	}
+	blocks[new_block.block_id] = std::move(new_block);
+}
+
+void MetadataManager::AddAndRegisterBlock(MetadataBlock block) {
+	if (block.block) {
+		throw InternalException("Calling AddAndRegisterBlock on block that already exists");
+	}
+	block.block = block_manager.RegisterBlock(block.block_id);
+	AddBlock(std::move(block), true);
+}
+
+MetaBlockPointer MetadataManager::GetDiskPointer(MetadataPointer pointer, uint32_t offset) {
+	idx_t block_pointer = idx_t(pointer.block_index);
+	block_pointer |= idx_t(pointer.index) << 56ULL;
+	return MetaBlockPointer(block_pointer, offset);
+}
+
+block_id_t MetaBlockPointer::GetBlockId() {
+	return block_id_t(block_pointer & ~(idx_t(0xFF) << 56ULL));
+}
+
+uint32_t MetaBlockPointer::GetBlockIndex() {
+	return block_pointer >> 56ULL;
+}
+
+MetadataPointer MetadataManager::FromDiskPointer(MetaBlockPointer pointer) {
+	auto block_id = pointer.GetBlockId();
+	auto index = pointer.GetBlockIndex();
+	auto entry = blocks.find(block_id);
+	if (entry == blocks.end()) { // LCOV_EXCL_START
+		throw InternalException("Failed to load metadata pointer (id %llu, idx %llu, ptr %llu)\n", block_id, index,
+		                        pointer.block_pointer);
+	} // LCOV_EXCL_STOP
+	MetadataPointer result;
+	result.block_index = block_id;
+	result.index = index;
+	return result;
+}
+
+MetadataPointer MetadataManager::RegisterDiskPointer(MetaBlockPointer pointer) {
+	auto block_id = pointer.GetBlockId();
+	MetadataBlock block;
+	block.block_id = block_id;
+	AddAndRegisterBlock(block);
+	return FromDiskPointer(pointer);
+}
+
+BlockPointer MetadataManager::ToBlockPointer(MetaBlockPointer meta_pointer) {
+	BlockPointer result;
+	result.block_id = meta_pointer.GetBlockId();
+	result.offset = meta_pointer.GetBlockIndex() * MetadataManager::METADATA_BLOCK_SIZE + meta_pointer.offset;
+	D_ASSERT(result.offset < MetadataManager::METADATA_BLOCK_SIZE * MetadataManager::METADATA_BLOCK_COUNT);
+	return result;
+}
+
+MetaBlockPointer MetadataManager::FromBlockPointer(BlockPointer block_pointer) {
+	if (!block_pointer.IsValid()) {
+		return MetaBlockPointer();
+	}
+	idx_t index = block_pointer.offset / MetadataManager::METADATA_BLOCK_SIZE;
+	auto offset = block_pointer.offset % MetadataManager::METADATA_BLOCK_SIZE;
+	D_ASSERT(index < MetadataManager::METADATA_BLOCK_COUNT);
+	D_ASSERT(offset < MetadataManager::METADATA_BLOCK_SIZE);
+	MetaBlockPointer result;
+	result.block_pointer = idx_t(block_pointer.block_id) | index << 56ULL;
+	result.offset = offset;
+	return result;
+}
+
+idx_t MetadataManager::BlockCount() {
+	return blocks.size();
+}
+
+void MetadataManager::Flush() {
+	const idx_t total_metadata_size = MetadataManager::METADATA_BLOCK_SIZE * MetadataManager::METADATA_BLOCK_COUNT;
+	// write the blocks of the metadata manager to disk
+	for (auto &kv : blocks) {
+		auto &block = kv.second;
+		auto handle = buffer_manager.Pin(block.block);
+		// zero-initialize any free blocks
+		for (auto free_block : block.free_blocks) {
+			memset(handle.Ptr() + free_block * MetadataManager::METADATA_BLOCK_SIZE, 0,
+			       MetadataManager::METADATA_BLOCK_SIZE);
+		}
+		// there are a few bytes left-over at the end of the block, zero-initialize them
+		memset(handle.Ptr() + total_metadata_size, 0, Storage::BLOCK_SIZE - total_metadata_size);
+		D_ASSERT(kv.first == block.block_id);
+		if (block.block->BlockId() >= MAXIMUM_BLOCK) {
+			// temporary block - convert to persistent
+			block.block = block_manager.ConvertToPersistent(kv.first, std::move(block.block));
+		} else {
+			// already a persistent block - only need to write it
+			D_ASSERT(block.block->BlockId() == block.block_id);
+			block_manager.Write(handle.GetFileBuffer(), block.block_id);
+		}
+	}
+}
+
+void MetadataManager::Serialize(Serializer &serializer) {
+	serializer.Write<uint64_t>(blocks.size());
+	for (auto &kv : blocks) {
+		kv.second.Serialize(serializer);
+	}
+}
+
+void MetadataManager::Deserialize(Deserializer &source) {
+	auto block_count = source.Read<uint64_t>();
+	for (idx_t i = 0; i < block_count; i++) {
+		auto block = MetadataBlock::Deserialize(source);
+		auto entry = blocks.find(block.block_id);
+		if (entry == blocks.end()) {
+			// block does not exist yet
+			AddAndRegisterBlock(std::move(block));
+		} else {
+			// block was already created - only copy over the free list
+			entry->second.free_blocks = std::move(block.free_blocks);
+		}
+	}
+}
+
+void MetadataBlock::Serialize(Serializer &serializer) {
+	serializer.Write<block_id_t>(block_id);
+	serializer.Write<idx_t>(FreeBlocksToInteger());
+}
+
+MetadataBlock MetadataBlock::Deserialize(Deserializer &source) {
+	MetadataBlock result;
+	result.block_id = source.Read<block_id_t>();
+	auto free_list = source.Read<idx_t>();
+	result.FreeBlocksFromInteger(free_list);
+	return result;
+}
+
+idx_t MetadataBlock::FreeBlocksToInteger() {
+	idx_t result = 0;
+	for (idx_t i = 0; i < free_blocks.size(); i++) {
+		D_ASSERT(free_blocks[i] < idx_t(64));
+		idx_t mask = idx_t(1) << idx_t(free_blocks[i]);
+		result |= mask;
+	}
+	return result;
+}
+
+void MetadataBlock::FreeBlocksFromInteger(idx_t free_list) {
+	free_blocks.clear();
+	if (free_list == 0) {
+		return;
+	}
+	for (idx_t i = 64; i > 0; i--) {
+		auto index = i - 1;
+		idx_t mask = idx_t(1) << index;
+		if (free_list & mask) {
+			free_blocks.push_back(index);
+		}
+	}
+}
+
+void MetadataManager::MarkBlocksAsModified() {
+	if (!modified_blocks.empty()) {
+		// for any blocks that were modified in the last checkpoint - set them to free blocks currently
+		for (auto &kv : modified_blocks) {
+			auto block_id = kv.first;
+			idx_t modified_list = kv.second;
+			auto entry = blocks.find(block_id);
+			D_ASSERT(entry != blocks.end());
+			auto &block = entry->second;
+			idx_t current_free_blocks = block.FreeBlocksToInteger();
+			// merge the current set of free blocks with the modified blocks
+			idx_t new_free_blocks = current_free_blocks | modified_list;
+			//			if (new_free_blocks == NumericLimits<idx_t>::Maximum()) {
+			//				// if new free_blocks is all blocks - mark entire block as modified
+			//				blocks.erase(entry);
+			//				block_manager.MarkBlockAsModified(block_id);
+			//			} else {
+			// set the new set of free blocks
+			block.FreeBlocksFromInteger(new_free_blocks);
+			//			}
+		}
+	}
+	modified_blocks.clear();
+	for (auto &kv : blocks) {
+		auto &block = kv.second;
+		idx_t free_list = block.FreeBlocksToInteger();
+		idx_t occupied_list = ~free_list;
+		modified_blocks[block.block_id] = occupied_list;
+	}
+}
+
+block_id_t MetadataManager::GetNextBlockId() {
+	return block_manager.GetFreeBlockId();
+}
+
+} // namespace duckdb