duckdb 0.8.2-dev33.0 → 0.8.2-dev3334.0

package/src/duckdb/src/include/duckdb/storage/checkpoint_manager.hpp

@@ -16,7 +16,7 @@ namespace duckdb {
 class DatabaseInstance;
 class ClientContext;
 class ColumnSegment;
-class MetaBlockReader;
+class MetadataReader;
 class SchemaCatalogEntry;
 class SequenceCatalogEntry;
 class TableCatalogEntry;
@@ -33,7 +33,8 @@ public:
 	//! The database
 	AttachedDatabase &db;
 
-	virtual MetaBlockWriter &GetMetaBlockWriter() = 0;
+	virtual MetadataManager &GetMetadataManager() = 0;
+	virtual MetadataWriter &GetMetadataWriter() = 0;
 	virtual unique_ptr<TableDataWriter> GetTableDataWriter(TableCatalogEntry &table) = 0;
 
 protected:
@@ -58,17 +59,17 @@ protected:
 	Catalog &catalog;
 
 protected:
-	virtual void LoadCheckpoint(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadSchema(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadTable(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadView(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadSequence(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadMacro(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadTableMacro(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadIndex(ClientContext &context, MetaBlockReader &reader);
-	virtual void ReadType(ClientContext &context, MetaBlockReader &reader);
-
-	virtual void ReadTableData(ClientContext &context, MetaBlockReader &reader, BoundCreateTableInfo &bound_info);
+	virtual void LoadCheckpoint(ClientContext &context, MetadataReader &reader);
+	virtual void ReadSchema(ClientContext &context, MetadataReader &reader);
+	virtual void ReadTable(ClientContext &context, MetadataReader &reader);
+	virtual void ReadView(ClientContext &context, MetadataReader &reader);
+	virtual void ReadSequence(ClientContext &context, MetadataReader &reader);
+	virtual void ReadMacro(ClientContext &context, MetadataReader &reader);
+	virtual void ReadTableMacro(ClientContext &context, MetadataReader &reader);
+	virtual void ReadIndex(ClientContext &context, MetadataReader &reader);
+	virtual void ReadType(ClientContext &context, MetadataReader &reader);
+
+	virtual void ReadTableData(ClientContext &context, MetadataReader &reader, BoundCreateTableInfo &bound_info);
 };
 
 class SingleFileCheckpointReader final : public CheckpointReader {
@@ -78,6 +79,7 @@ public:
 	}
 
 	void LoadFromStorage();
+	MetadataManager &GetMetadataManager();
 
 	//! The database
 	SingleFileStorageManager &storage;
@@ -98,16 +100,17 @@ public:
 	//! connection is available because right now the checkpointing cannot be done online. (TODO)
 	void CreateCheckpoint();
 
-	virtual MetaBlockWriter &GetMetaBlockWriter() override;
+	virtual MetadataWriter &GetMetadataWriter() override;
+	virtual MetadataManager &GetMetadataManager() override;
 	virtual unique_ptr<TableDataWriter> GetTableDataWriter(TableCatalogEntry &table) override;
 
 	BlockManager &GetBlockManager();
 
 private:
 	//! The metadata writer is responsible for writing schema information
-	unique_ptr<MetaBlockWriter> metadata_writer;
+	unique_ptr<MetadataWriter> metadata_writer;
 	//! The table data writer is responsible for writing the DataPointers used by the table chunks
-	unique_ptr<MetaBlockWriter> table_metadata_writer;
+	unique_ptr<MetadataWriter> table_metadata_writer;
 	//! Because this is single-file storage, we can share partial blocks across
 	//! an entire checkpoint.
 	PartialBlockManager partial_block_manager;

package/src/duckdb/src/include/duckdb/storage/data_pointer.hpp

@@ -33,7 +33,7 @@ struct RowGroupPointer {
 	uint64_t row_start;
 	uint64_t tuple_count;
 	//! The data pointers of the column segments stored in the row group
-	vector<BlockPointer> data_pointers;
+	vector<MetaBlockPointer> data_pointers;
 	//! The versions information of the row group (if any)
 	shared_ptr<VersionNode> versions;
 };

package/src/duckdb/src/include/duckdb/storage/data_table.hpp

@@ -51,7 +51,7 @@ public:
 	          const string &table, vector<ColumnDefinition> column_definitions_p,
 	          unique_ptr<PersistentTableData> data = nullptr);
 	//! Constructs a DataTable as a delta on an existing data table with a newly added column
-	DataTable(ClientContext &context, DataTable &parent, ColumnDefinition &new_column, Expression *default_value);
+	DataTable(ClientContext &context, DataTable &parent, ColumnDefinition &new_column, Expression &default_value);
 	//! Constructs a DataTable as a delta on an existing data table but with one column removed
 	DataTable(ClientContext &context, DataTable &parent, idx_t removed_column);
 	//! Constructs a DataTable as a delta on an existing data table but with one column changed type
@@ -125,7 +125,7 @@ public:
 	void UpdateColumn(TableCatalogEntry &table, ClientContext &context, Vector &row_ids,
 	                  const vector<column_t> &column_path, DataChunk &updates);
 
-	//! Add an index to the DataTable. NOTE: for CREATE (UNIQUE) INDEX statements, we use the PhysicalCreateIndex
+	//! Add an index to the DataTable. NOTE: for CREATE (UNIQUE) INDEX statements, we use the PhysicalCreateARTIndex
 	//! operator. This function is only used during the WAL replay, and is a much less performant index creation
 	//! approach.
 	void WALAddIndex(ClientContext &context, unique_ptr<Index> index,

package/src/duckdb/src/include/duckdb/storage/in_memory_block_manager.hpp

@@ -29,7 +29,7 @@ public:
 	block_id_t GetFreeBlockId() override {
 		throw InternalException("Cannot perform IO in in-memory database - GetFreeBlockId!");
 	}
-	bool IsRootBlock(block_id_t root) override {
+	bool IsRootBlock(MetaBlockPointer root) override {
 		throw InternalException("Cannot perform IO in in-memory database - IsRootBlock!");
 	}
 	void MarkBlockAsFree(block_id_t block_id) override {
@@ -41,7 +41,7 @@ public:
 	void IncreaseBlockReferenceCount(block_id_t block_id) override {
 		throw InternalException("Cannot perform IO in in-memory database - IncreaseBlockReferenceCount!");
 	}
-	block_id_t GetMetaBlock() override {
+	idx_t GetMetaBlock() override {
 		throw InternalException("Cannot perform IO in in-memory database - GetMetaBlock!");
 	}
 	void Read(Block &block) override {

package/src/duckdb/src/include/duckdb/storage/index.hpp

@@ -14,7 +14,7 @@
 #include "duckdb/common/sort/sort.hpp"
 #include "duckdb/parser/parsed_expression.hpp"
 #include "duckdb/planner/expression.hpp"
-#include "duckdb/storage/meta_block_writer.hpp"
+#include "duckdb/storage/metadata/metadata_writer.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/common/types/constraint_conflict_info.hpp"
 
@@ -126,7 +126,7 @@ public:
 	}
 
 	//! Serializes the index and returns the pair of block_id offset positions
-	virtual BlockPointer Serialize(MetaBlockWriter &writer);
+	virtual BlockPointer Serialize(MetadataWriter &writer);
 	//! Returns the serialized data pointer to the block and offset of the serialized index
 	BlockPointer GetSerializedDataPointer() const {
 		return serialized_data_pointer;

package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp

@@ -0,0 +1,88 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/metadata/metadata_manager.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/serializer.hpp"
+#include "duckdb/storage/block.hpp"
+#include "duckdb/storage/block_manager.hpp"
+#include "duckdb/common/set.hpp"
+#include "duckdb/storage/buffer/buffer_handle.hpp"
+
+namespace duckdb {
+class DatabaseInstance;
+
+struct MetadataBlock {
+	shared_ptr<BlockHandle> block;
+	block_id_t block_id;
+	vector<uint8_t> free_blocks;
+
+	void Serialize(Serializer &serializer);
+	static MetadataBlock Deserialize(Deserializer &source);
+
+	idx_t FreeBlocksToInteger();
+	void FreeBlocksFromInteger(idx_t blocks);
+};
+
+struct MetadataPointer {
+	idx_t block_index : 56;
+	uint8_t index : 8;
+};
+
+struct MetadataHandle {
+	MetadataPointer pointer;
+	BufferHandle handle;
+};
+
+class MetadataManager {
+public:
+	//! The size of metadata blocks
+	static constexpr const idx_t METADATA_BLOCK_SIZE = 4088;
+	//! The amount of metadata blocks per storage block
+	static constexpr const idx_t METADATA_BLOCK_COUNT = 64;
+
+public:
+	MetadataManager(BlockManager &block_manager, BufferManager &buffer_manager);
+	~MetadataManager();
+
+	MetadataHandle AllocateHandle();
+	MetadataHandle Pin(MetadataPointer pointer);
+
+	MetaBlockPointer GetDiskPointer(MetadataPointer pointer, uint32_t offset = 0);
+	MetadataPointer FromDiskPointer(MetaBlockPointer pointer);
+	MetadataPointer RegisterDiskPointer(MetaBlockPointer pointer);
+
+	static BlockPointer ToBlockPointer(MetaBlockPointer meta_pointer);
+	static MetaBlockPointer FromBlockPointer(BlockPointer block_pointer);
+
+	//! Flush all blocks to disk
+	void Flush();
+
+	void MarkBlocksAsModified();
+
+	idx_t BlockCount();
+
+	void Serialize(Serializer &serializer);
+	void Deserialize(Deserializer &source);
+
+protected:
+	BlockManager &block_manager;
+	BufferManager &buffer_manager;
+	unordered_map<block_id_t, MetadataBlock> blocks;
+	unordered_map<block_id_t, idx_t> modified_blocks;
+
+protected:
+	block_id_t AllocateNewBlock();
+	block_id_t GetNextBlockId();
+
+	void AddBlock(MetadataBlock new_block, bool if_exists = false);
+	void AddAndRegisterBlock(MetadataBlock block);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/metadata/metadata_reader.hpp

@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/metadata/metadata_reader.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/metadata/metadata_manager.hpp"
+
+namespace duckdb {
+
+enum class BlockReaderType { EXISTING_BLOCKS, REGISTER_BLOCKS };
+
+class MetadataReader : public Deserializer {
+public:
+	MetadataReader(MetadataManager &manager, MetaBlockPointer pointer,
+	               BlockReaderType type = BlockReaderType::EXISTING_BLOCKS);
+	MetadataReader(MetadataManager &manager, BlockPointer pointer);
+	~MetadataReader() override;
+
+public:
+	//! Read content of size read_size into the buffer
+	void ReadData(data_ptr_t buffer, idx_t read_size) override;
+
+	MetaBlockPointer GetMetaBlockPointer();
+
+	MetadataManager &GetMetadataManager() {
+		return manager;
+	}
+
+private:
+	data_ptr_t BasePtr();
+	data_ptr_t Ptr();
+
+	void ReadNextBlock();
+
+	MetadataPointer FromDiskPointer(MetaBlockPointer pointer);
+
+private:
+	MetadataManager &manager;
+	BlockReaderType type;
+	MetadataHandle block;
+	MetadataPointer next_pointer;
+	bool has_next_block;
+	idx_t index;
+	idx_t offset;
+	idx_t next_offset;
+	idx_t capacity;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/metadata/metadata_writer.hpp

@@ -0,0 +1,45 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/metadata/metadata_writer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/metadata/metadata_manager.hpp"
+
+namespace duckdb {
+
+class MetadataWriter : public Serializer {
+public:
+	explicit MetadataWriter(MetadataManager &manager);
+	~MetadataWriter() override;
+
+public:
+	void WriteData(const_data_ptr_t buffer, idx_t write_size) override;
+
+	void Flush();
+
+	BlockPointer GetBlockPointer();
+	MetaBlockPointer GetMetaBlockPointer();
+
+protected:
+	virtual MetadataHandle NextHandle();
+
+private:
+	data_ptr_t BasePtr();
+	data_ptr_t Ptr();
+
+	void NextBlock();
+
+private:
+	MetadataManager &manager;
+	MetadataHandle block;
+	MetadataPointer current_pointer;
+	idx_t capacity;
+	idx_t offset;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/object_cache.hpp

@@ -47,11 +47,33 @@ public:
 		return std::static_pointer_cast<T, ObjectCacheEntry>(object);
 	}
 
+	template <class T, class... Args>
+	shared_ptr<T> GetOrCreate(const string &key, Args &&...args) {
+		lock_guard<mutex> glock(lock);
+
+		auto entry = cache.find(key);
+		if (entry == cache.end()) {
+			auto value = make_shared<T>(args...);
+			cache[key] = value;
+			return value;
+		}
+		auto object = entry->second;
+		if (!object || object->GetObjectType() != T::ObjectType()) {
+			return nullptr;
+		}
+		return std::static_pointer_cast<T, ObjectCacheEntry>(object);
+	}
+
 	void Put(string key, shared_ptr<ObjectCacheEntry> value) {
 		lock_guard<mutex> glock(lock);
 		cache[key] = std::move(value);
 	}
 
+	void Delete(const string &key) {
+		lock_guard<mutex> glock(lock);
+		cache.erase(key);
+	}
+
 	DUCKDB_API static ObjectCache &GetObjectCache(ClientContext &context);
 	DUCKDB_API static bool ObjectCacheEnabled(ClientContext &context);
 

package/src/duckdb/src/include/duckdb/storage/partial_block_manager.hpp

@@ -11,14 +11,14 @@
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/map.hpp"
 #include "duckdb/storage/storage_manager.hpp"
-#include "duckdb/storage/meta_block_writer.hpp"
+#include "duckdb/storage/metadata/metadata_writer.hpp"
 #include "duckdb/storage/data_pointer.hpp"
 
 namespace duckdb {
 class DatabaseInstance;
 class ClientContext;
 class ColumnSegment;
-class MetaBlockReader;
+class MetadataReader;
 class SchemaCatalogEntry;
 class SequenceCatalogEntry;
 class TableCatalogEntry;

package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp

@@ -20,6 +20,7 @@
 namespace duckdb {
 
 class DatabaseInstance;
+struct MetadataHandle;
 
 struct StorageManagerOptions {
 	bool read_only = false;
@@ -45,7 +46,7 @@ public:
 	//! Return the next free block id
 	block_id_t GetFreeBlockId() override;
 	//! Returns whether or not a specified block is the root block
-	bool IsRootBlock(block_id_t root) override;
+	bool IsRootBlock(MetaBlockPointer root) override;
 	//! Mark a block as free (immediately re-writeable)
 	void MarkBlockAsFree(block_id_t block_id) override;
 	//! Mark a block as modified (re-writeable after a checkpoint)
@@ -53,13 +54,15 @@ public:
 	//! Increase the reference count of a block. The block should hold at least one reference
 	void IncreaseBlockReferenceCount(block_id_t block_id) override;
 	//! Return the meta block id
-	block_id_t GetMetaBlock() override;
+	idx_t GetMetaBlock() override;
 	//! Read the content of the block from disk
 	void Read(Block &block) override;
 	//! Write the given block to disk
 	void Write(FileBuffer &block, block_id_t block_id) override;
 	//! Write the header to disk, this is the final step of the checkpointing process
 	void WriteHeader(DatabaseHeader header) override;
+	//! Truncate the underlying database file after a checkpoint
+	void Truncate() override;
 
 	//! Returns the number of total blocks
 	idx_t TotalBlocks() override;
@@ -76,7 +79,7 @@ private:
 	void ChecksumAndWrite(FileBuffer &handle, uint64_t location) const;
 
 	//! Return the blocks to which we will write the free list and modified blocks
-	vector<block_id_t> GetFreeListBlocks();
+	vector<MetadataHandle> GetFreeListBlocks();
 
 private:
 	AttachedDatabase &db;
@@ -97,11 +100,11 @@ private:
 	//! The list of blocks that will be added to the free list
 	unordered_set<block_id_t> modified_blocks;
 	//! The current meta block id
-	block_id_t meta_block;
+	idx_t meta_block;
 	//! The current maximum block id, this id will be given away first after the free_list runs out
 	block_id_t max_block;
 	//! The block id where the free list can be found
-	block_id_t free_list_id;
+	idx_t free_list_id;
 	//! The current header iteration count
 	uint64_t iteration_count;
 	//! The storage manager options

package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp

@@ -48,6 +48,10 @@ struct StringStats {
 	DUCKDB_API static uint32_t MaxStringLength(const BaseStatistics &stats);
 	//! Whether or not the strings can contain unicode
 	DUCKDB_API static bool CanContainUnicode(const BaseStatistics &stats);
+	//! Returns the min value (up to a length of StringStatsData::MAX_STRING_MINMAX_SIZE)
+	DUCKDB_API static string Min(const BaseStatistics &stats);
+	//! Returns the max value (up to a length of StringStatsData::MAX_STRING_MINMAX_SIZE)
+	DUCKDB_API static string Max(const BaseStatistics &stats);
 
 	//! Resets the max string length so HasMaxStringLength() is false
 	DUCKDB_API static void ResetMaxStringLength(BaseStatistics &stats);

package/src/duckdb/src/include/duckdb/storage/storage_info.hpp

@@ -65,9 +65,9 @@ struct DatabaseHeader {
 	//! The iteration count, increases by 1 every time the storage is checkpointed.
 	uint64_t iteration;
 	//! A pointer to the initial meta block
-	block_id_t meta_block;
+	idx_t meta_block;
 	//! A pointer to the block containing the free list
-	block_id_t free_list;
+	idx_t free_list;
 	//! The number of blocks that is in the file as of this database header. If the file is larger than BLOCK_SIZE *
 	//! block_count any blocks appearing AFTER block_count are implicitly part of the free_list.
 	uint64_t block_count;

package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp

@@ -65,7 +65,7 @@ public:
 
 	virtual bool AutomaticCheckpoint(idx_t estimated_wal_bytes) = 0;
 	virtual unique_ptr<StorageCommitState> GenStorageCommitState(Transaction &transaction, bool checkpoint) = 0;
-	virtual bool IsCheckpointClean(block_id_t checkpoint_id) = 0;
+	virtual bool IsCheckpointClean(MetaBlockPointer checkpoint_id) = 0;
 	virtual void CreateCheckpoint(bool delete_wal = false, bool force_checkpoint = false) = 0;
 	virtual DatabaseSize GetDatabaseSize() = 0;
 	virtual shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) = 0;
@@ -109,7 +109,7 @@ public:
 public:
 	bool AutomaticCheckpoint(idx_t estimated_wal_bytes) override;
 	unique_ptr<StorageCommitState> GenStorageCommitState(Transaction &transaction, bool checkpoint) override;
-	bool IsCheckpointClean(block_id_t checkpoint_id) override;
+	bool IsCheckpointClean(MetaBlockPointer checkpoint_id) override;
 	void CreateCheckpoint(bool delete_wal, bool force_checkpoint) override;
 	DatabaseSize GetDatabaseSize() override;
 	shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) override;

package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp

@@ -41,6 +41,7 @@ public:
 	//! Returns whether or not a single row in the ChunkInfo should be used or not for the given transaction
 	virtual bool Fetch(TransactionData transaction, row_t row) = 0;
 	virtual void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) = 0;
+	virtual idx_t GetCommittedDeletedCount(idx_t max_count) = 0;
 
 	virtual void Serialize(Serializer &serialize) = 0;
 	static unique_ptr<ChunkInfo> Deserialize(Deserializer &source);
@@ -79,6 +80,7 @@ public:
 	                            SelectionVector &sel_vector, idx_t max_count) override;
 	bool Fetch(TransactionData transaction, row_t row) override;
 	void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override;
+	idx_t GetCommittedDeletedCount(idx_t max_count) override;
 
 	void Serialize(Serializer &serialize) override;
 	static unique_ptr<ChunkInfo> Deserialize(Deserializer &source);
@@ -113,6 +115,7 @@ public:
 	                            SelectionVector &sel_vector, idx_t max_count) override;
 	bool Fetch(TransactionData transaction, row_t row) override;
 	void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override;
+	idx_t GetCommittedDeletedCount(idx_t max_count) override;
 
 	void Append(idx_t start, idx_t end, transaction_t commit_id);
 

package/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/common/vector.hpp"
 #include "duckdb/storage/data_pointer.hpp"
 #include "duckdb/storage/table/table_statistics.hpp"
+#include "duckdb/storage/metadata/metadata_manager.hpp"
 
 namespace duckdb {
 class BaseStatistics;
@@ -24,8 +25,7 @@ public:
 	TableStatistics table_stats;
 	idx_t total_rows;
 	idx_t row_group_count;
-	block_id_t block_id;
-	idx_t offset;
+	MetaBlockPointer block_pointer;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -74,7 +74,6 @@ public:
 	RowGroupCollection &GetCollection() {
 		return collection.get();
 	}
-	DatabaseInstance &GetDatabase();
 	BlockManager &GetBlockManager();
 	DataTableInfo &GetTableInfo();
 
@@ -82,7 +81,7 @@ public:
 	                               ExpressionExecutor &executor, CollectionScanState &scan_state,
 	                               DataChunk &scan_chunk);
 	unique_ptr<RowGroup> AddColumn(RowGroupCollection &collection, ColumnDefinition &new_column,
-	                               ExpressionExecutor &executor, Expression *default_value, Vector &intermediate);
+	                               ExpressionExecutor &executor, Expression &default_value, Vector &intermediate);
 	unique_ptr<RowGroup> RemoveColumn(RowGroupCollection &collection, idx_t removed_column);
 
 	void CommitDrop();
@@ -123,6 +122,7 @@ public:
 	idx_t Delete(TransactionData transaction, DataTable &table, row_t *row_ids, idx_t count);
 
 	RowGroupWriteData WriteToDisk(PartialBlockManager &manager, const vector<CompressionType> &compression_types);
+	bool AllDeleted();
 	RowGroupPointer Checkpoint(RowGroupWriter &writer, TableStatistics &global_stats);
 	static void Serialize(RowGroupPointer &pointer, Serializer &serializer);
 	static RowGroupPointer Deserialize(Deserializer &source, const vector<LogicalType> &columns);
@@ -162,7 +162,7 @@ private:
 private:
 	mutex row_group_lock;
 	mutex stats_lock;
-	vector<BlockPointer> column_pointers;
+	vector<MetaBlockPointer> column_pointers;
 	unique_ptr<atomic<bool>[]> is_loaded;
 };
 
@@ -170,6 +170,7 @@ struct VersionNode {
 	unique_ptr<ChunkInfo> info[RowGroup::ROW_GROUP_VECTOR_COUNT];
 
 	void SetStart(idx_t start);
+	idx_t GetCommittedDeletedCount(idx_t count);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp

@@ -27,6 +27,7 @@ class DuckTransaction;
 class BoundConstraint;
 class RowGroupSegmentTree;
 struct ColumnSegmentInfo;
+class MetadataManager;
 
 class RowGroupCollection {
 public:
@@ -45,7 +46,6 @@ public:
 	void AppendRowGroup(SegmentLock &l, idx_t start_row);
 	//! Get the nth row-group, negative numbers start from the back (so -1 is the last row group, etc)
 	RowGroup *GetRowGroup(int64_t index);
-	idx_t RowGroupCount();
 	void Verify();
 
 	void InitializeScan(CollectionScanState &state, const vector<column_t> &column_ids, TableFilterSet *table_filters);
@@ -93,7 +93,7 @@ public:
 	const vector<LogicalType> &GetTypes() const;
 
 	shared_ptr<RowGroupCollection> AddColumn(ClientContext &context, ColumnDefinition &new_column,
-	                                         Expression *default_value);
+	                                         Expression &default_value);
 	shared_ptr<RowGroupCollection> RemoveColumn(idx_t col_idx);
 	shared_ptr<RowGroupCollection> AlterType(ClientContext &context, idx_t changed_idx, const LogicalType &target_type,
 	                                         vector<column_t> bound_columns, Expression &cast_expr);
@@ -104,10 +104,10 @@ public:
 	void SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats);
 
 	AttachedDatabase &GetAttached();
-	DatabaseInstance &GetDatabase();
 	BlockManager &GetBlockManager() {
 		return block_manager;
 	}
+	MetadataManager &GetMetadataManager();
 	DataTableInfo &GetTableInfo() {
 		return *info;
 	}

package/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp

@@ -14,7 +14,7 @@
 namespace duckdb {
 struct DataTableInfo;
 class PersistentTableData;
-class MetaBlockReader;
+class MetadataReader;
 
 class RowGroupSegmentTree : public SegmentTree<RowGroup, true> {
 public:
@@ -29,7 +29,7 @@ protected:
 	RowGroupCollection &collection;
 	idx_t current_row_group;
 	idx_t max_row_group;
-	unique_ptr<MetaBlockReader> reader;
+	unique_ptr<MetadataReader> reader;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/table_index_list.hpp

@@ -47,7 +47,7 @@ public:
 	void VerifyForeignKey(const vector<PhysicalIndex> &fk_keys, DataChunk &chunk, ConflictManager &conflict_manager);
 
 	//! Serialize all indexes owned by this table, returns a vector of block info of all indexes
-	vector<BlockPointer> SerializeIndexes(duckdb::MetaBlockWriter &writer);
+	vector<BlockPointer> SerializeIndexes(duckdb::MetadataWriter &writer);
 
 	vector<column_t> GetRequiredColumns();
 

package/src/duckdb/src/include/duckdb/storage/table_io_manager.hpp

@@ -13,6 +13,7 @@
 namespace duckdb {
 class BlockManager;
 class DataTable;
+class MetadataManager;
 
 class TableIOManager {
 public:
@@ -27,6 +28,8 @@ public:
 
 	//! The block manager used for storing row group data
 	virtual BlockManager &GetBlockManagerForRowData() = 0;
+
+	virtual MetadataManager &GetMetadataManager() = 0;
 };
 
 } // namespace duckdb