duckdb 0.8.2-dev37.0 → 0.8.2-dev3989.0

package/src/duckdb/src/include/duckdb/storage/index.hpp

@@ -14,7 +14,7 @@
 #include "duckdb/common/sort/sort.hpp"
 #include "duckdb/parser/parsed_expression.hpp"
 #include "duckdb/planner/expression.hpp"
-#include "duckdb/storage/meta_block_writer.hpp"
+#include "duckdb/storage/metadata/metadata_writer.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/common/types/constraint_conflict_info.hpp"
 
@@ -126,7 +126,7 @@ public:
 	}
 
 	//! Serializes the index and returns the pair of block_id offset positions
-	virtual BlockPointer Serialize(MetaBlockWriter &writer);
+	virtual BlockPointer Serialize(MetadataWriter &writer);
 	//! Returns the serialized data pointer to the block and offset of the serialized index
 	BlockPointer GetSerializedDataPointer() const {
 		return serialized_data_pointer;

package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp

@@ -0,0 +1,88 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/metadata/metadata_manager.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/serializer.hpp"
+#include "duckdb/storage/block.hpp"
+#include "duckdb/storage/block_manager.hpp"
+#include "duckdb/common/set.hpp"
+#include "duckdb/storage/buffer/buffer_handle.hpp"
+
+namespace duckdb {
+class DatabaseInstance;
+
+struct MetadataBlock {
+	shared_ptr<BlockHandle> block;
+	block_id_t block_id;
+	vector<uint8_t> free_blocks;
+
+	void Serialize(Serializer &serializer);
+	static MetadataBlock Deserialize(Deserializer &source);
+
+	idx_t FreeBlocksToInteger();
+	void FreeBlocksFromInteger(idx_t blocks);
+};
+
+struct MetadataPointer {
+	idx_t block_index : 56;
+	uint8_t index : 8;
+};
+
+struct MetadataHandle {
+	MetadataPointer pointer;
+	BufferHandle handle;
+};
+
+class MetadataManager {
+public:
+	//! The size of metadata blocks
+	static constexpr const idx_t METADATA_BLOCK_SIZE = 4088;
+	//! The amount of metadata blocks per storage block
+	static constexpr const idx_t METADATA_BLOCK_COUNT = 64;
+
+public:
+	MetadataManager(BlockManager &block_manager, BufferManager &buffer_manager);
+	~MetadataManager();
+
+	MetadataHandle AllocateHandle();
+	MetadataHandle Pin(MetadataPointer pointer);
+
+	MetaBlockPointer GetDiskPointer(MetadataPointer pointer, uint32_t offset = 0);
+	MetadataPointer FromDiskPointer(MetaBlockPointer pointer);
+	MetadataPointer RegisterDiskPointer(MetaBlockPointer pointer);
+
+	static BlockPointer ToBlockPointer(MetaBlockPointer meta_pointer);
+	static MetaBlockPointer FromBlockPointer(BlockPointer block_pointer);
+
+	//! Flush all blocks to disk
+	void Flush();
+
+	void MarkBlocksAsModified();
+
+	idx_t BlockCount();
+
+	void Serialize(Serializer &serializer);
+	void Deserialize(Deserializer &source);
+
+protected:
+	BlockManager &block_manager;
+	BufferManager &buffer_manager;
+	unordered_map<block_id_t, MetadataBlock> blocks;
+	unordered_map<block_id_t, idx_t> modified_blocks;
+
+protected:
+	block_id_t AllocateNewBlock();
+	block_id_t GetNextBlockId();
+
+	void AddBlock(MetadataBlock new_block, bool if_exists = false);
+	void AddAndRegisterBlock(MetadataBlock block);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/metadata/metadata_reader.hpp

@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/metadata/metadata_reader.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/metadata/metadata_manager.hpp"
+
+namespace duckdb {
+
+enum class BlockReaderType { EXISTING_BLOCKS, REGISTER_BLOCKS };
+
+class MetadataReader : public Deserializer {
+public:
+	MetadataReader(MetadataManager &manager, MetaBlockPointer pointer,
+	               BlockReaderType type = BlockReaderType::EXISTING_BLOCKS);
+	MetadataReader(MetadataManager &manager, BlockPointer pointer);
+	~MetadataReader() override;
+
+public:
+	//! Read content of size read_size into the buffer
+	void ReadData(data_ptr_t buffer, idx_t read_size) override;
+
+	MetaBlockPointer GetMetaBlockPointer();
+
+	MetadataManager &GetMetadataManager() {
+		return manager;
+	}
+
+private:
+	data_ptr_t BasePtr();
+	data_ptr_t Ptr();
+
+	void ReadNextBlock();
+
+	MetadataPointer FromDiskPointer(MetaBlockPointer pointer);
+
+private:
+	MetadataManager &manager;
+	BlockReaderType type;
+	MetadataHandle block;
+	MetadataPointer next_pointer;
+	bool has_next_block;
+	idx_t index;
+	idx_t offset;
+	idx_t next_offset;
+	idx_t capacity;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/metadata/metadata_writer.hpp

@@ -0,0 +1,45 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/metadata/metadata_writer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/metadata/metadata_manager.hpp"
+
+namespace duckdb {
+
+class MetadataWriter : public Serializer {
+public:
+	explicit MetadataWriter(MetadataManager &manager);
+	~MetadataWriter() override;
+
+public:
+	void WriteData(const_data_ptr_t buffer, idx_t write_size) override;
+
+	void Flush();
+
+	BlockPointer GetBlockPointer();
+	MetaBlockPointer GetMetaBlockPointer();
+
+protected:
+	virtual MetadataHandle NextHandle();
+
+private:
+	data_ptr_t BasePtr();
+	data_ptr_t Ptr();
+
+	void NextBlock();
+
+private:
+	MetadataManager &manager;
+	MetadataHandle block;
+	MetadataPointer current_pointer;
+	idx_t capacity;
+	idx_t offset;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/object_cache.hpp

@@ -47,11 +47,33 @@ public:
 		return std::static_pointer_cast<T, ObjectCacheEntry>(object);
 	}
 
+	template <class T, class... Args>
+	shared_ptr<T> GetOrCreate(const string &key, Args &&...args) {
+		lock_guard<mutex> glock(lock);
+
+		auto entry = cache.find(key);
+		if (entry == cache.end()) {
+			auto value = make_shared<T>(args...);
+			cache[key] = value;
+			return value;
+		}
+		auto object = entry->second;
+		if (!object || object->GetObjectType() != T::ObjectType()) {
+			return nullptr;
+		}
+		return std::static_pointer_cast<T, ObjectCacheEntry>(object);
+	}
+
 	void Put(string key, shared_ptr<ObjectCacheEntry> value) {
 		lock_guard<mutex> glock(lock);
 		cache[key] = std::move(value);
 	}
 
+	void Delete(const string &key) {
+		lock_guard<mutex> glock(lock);
+		cache.erase(key);
+	}
+
 	DUCKDB_API static ObjectCache &GetObjectCache(ClientContext &context);
 	DUCKDB_API static bool ObjectCacheEnabled(ClientContext &context);
 

package/src/duckdb/src/include/duckdb/storage/partial_block_manager.hpp

@@ -11,14 +11,14 @@
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/map.hpp"
 #include "duckdb/storage/storage_manager.hpp"
-#include "duckdb/storage/meta_block_writer.hpp"
+#include "duckdb/storage/metadata/metadata_writer.hpp"
 #include "duckdb/storage/data_pointer.hpp"
 
 namespace duckdb {
 class DatabaseInstance;
 class ClientContext;
 class ColumnSegment;
-class MetaBlockReader;
+class MetadataReader;
 class SchemaCatalogEntry;
 class SequenceCatalogEntry;
 class TableCatalogEntry;

package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp

@@ -20,6 +20,7 @@
 namespace duckdb {
 
 class DatabaseInstance;
+struct MetadataHandle;
 
 struct StorageManagerOptions {
 	bool read_only = false;
@@ -45,7 +46,7 @@ public:
 	//! Return the next free block id
 	block_id_t GetFreeBlockId() override;
 	//! Returns whether or not a specified block is the root block
-	bool IsRootBlock(block_id_t root) override;
+	bool IsRootBlock(MetaBlockPointer root) override;
 	//! Mark a block as free (immediately re-writeable)
 	void MarkBlockAsFree(block_id_t block_id) override;
 	//! Mark a block as modified (re-writeable after a checkpoint)
@@ -53,13 +54,15 @@ public:
 	//! Increase the reference count of a block. The block should hold at least one reference
 	void IncreaseBlockReferenceCount(block_id_t block_id) override;
 	//! Return the meta block id
-	block_id_t GetMetaBlock() override;
+	idx_t GetMetaBlock() override;
 	//! Read the content of the block from disk
 	void Read(Block &block) override;
 	//! Write the given block to disk
 	void Write(FileBuffer &block, block_id_t block_id) override;
 	//! Write the header to disk, this is the final step of the checkpointing process
 	void WriteHeader(DatabaseHeader header) override;
+	//! Truncate the underlying database file after a checkpoint
+	void Truncate() override;
 
 	//! Returns the number of total blocks
 	idx_t TotalBlocks() override;
@@ -76,7 +79,7 @@ private:
 	void ChecksumAndWrite(FileBuffer &handle, uint64_t location) const;
 
 	//! Return the blocks to which we will write the free list and modified blocks
-	vector<block_id_t> GetFreeListBlocks();
+	vector<MetadataHandle> GetFreeListBlocks();
 
 private:
 	AttachedDatabase &db;
@@ -97,11 +100,11 @@ private:
 	//! The list of blocks that will be added to the free list
 	unordered_set<block_id_t> modified_blocks;
 	//! The current meta block id
-	block_id_t meta_block;
+	idx_t meta_block;
 	//! The current maximum block id, this id will be given away first after the free_list runs out
 	block_id_t max_block;
 	//! The block id where the free list can be found
-	block_id_t free_list_id;
+	idx_t free_list_id;
 	//! The current header iteration count
 	uint64_t iteration_count;
 	//! The storage manager options

package/src/duckdb/src/include/duckdb/storage/statistics/base_statistics.hpp

@@ -19,6 +19,9 @@
 namespace duckdb {
 struct SelectionVector;
 
+class FormatSerializer;
+class FormatDeserializer;
+
 class Serializer;
 class Deserializer;
 class FieldWriter;
@@ -94,17 +97,18 @@ public:
 
 	void Serialize(Serializer &serializer) const;
 	void Serialize(FieldWriter &writer) const;
-
-	idx_t GetDistinctCount();
-
 	static BaseStatistics Deserialize(Deserializer &source, LogicalType type);
 
+	void FormatSerialize(FormatSerializer &serializer) const;
+	static BaseStatistics FormatDeserialize(FormatDeserializer &deserializer);
+
 	//! Verify that a vector does not violate the statistics
 	void Verify(Vector &vector, const SelectionVector &sel, idx_t count) const;
 	void Verify(Vector &vector, idx_t count) const;
 
 	string ToString() const;
 
+	idx_t GetDistinctCount();
 	static BaseStatistics FromConstant(const Value &input);
 
 private:

package/src/duckdb/src/include/duckdb/storage/statistics/column_statistics.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/storage/statistics/distinct_statistics.hpp"
 
 namespace duckdb {
+class FormatSerializer;
 
 class ColumnStatistics {
 public:
@@ -35,6 +36,9 @@ public:
 	void Serialize(Serializer &serializer) const;
 	static shared_ptr<ColumnStatistics> Deserialize(Deserializer &source, const LogicalType &type);
 
+	void FormatSerialize(FormatSerializer &serializer) const;
+	static shared_ptr<ColumnStatistics> FormatDeserialize(FormatDeserializer &source);
+
 private:
 	BaseStatistics stats;
 	//! The approximate count distinct stats of the column

package/src/duckdb/src/include/duckdb/storage/statistics/distinct_statistics.hpp

@@ -16,6 +16,8 @@ namespace duckdb {
 class Serializer;
 class Deserializer;
 class Vector;
+class FormatSerializer;
+class FormatDeserializer;
 
 class DistinctStatistics {
 public:
@@ -48,6 +50,9 @@ public:
 
 	static bool TypeIsSupported(const LogicalType &type);
 
+	void FormatSerialize(FormatSerializer &serializer) const;
+	static unique_ptr<DistinctStatistics> FormatDeserialize(FormatDeserializer &deserializer);
+
 private:
 	//! For distinct statistics we sample the input to speed up insertions
 	static constexpr const double SAMPLE_RATE = 0.1;

package/src/duckdb/src/include/duckdb/storage/statistics/list_stats.hpp

@@ -31,6 +31,9 @@ struct ListStats {
 	DUCKDB_API static void Serialize(const BaseStatistics &stats, FieldWriter &writer);
 	DUCKDB_API static BaseStatistics Deserialize(FieldReader &reader, LogicalType type);
 
+	DUCKDB_API static void FormatSerialize(const BaseStatistics &stats, FormatSerializer &serializer);
+	DUCKDB_API static BaseStatistics FormatDeserialize(FormatDeserializer &deserializer, LogicalType type);
+
 	DUCKDB_API static string ToString(const BaseStatistics &stats);
 
 	DUCKDB_API static void Merge(BaseStatistics &stats, const BaseStatistics &other);

package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats.hpp

@@ -64,6 +64,9 @@ struct NumericStats {
 	DUCKDB_API static void Serialize(const BaseStatistics &stats, FieldWriter &writer);
 	DUCKDB_API static BaseStatistics Deserialize(FieldReader &reader, LogicalType type);
 
+	DUCKDB_API static void FormatSerialize(const BaseStatistics &stats, FormatSerializer &serializer);
+	DUCKDB_API static BaseStatistics FormatDeserialize(FormatDeserializer &deserializer, LogicalType type);
+
 	DUCKDB_API static string ToString(const BaseStatistics &stats);
 
 	template <class T>

package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp

@@ -48,6 +48,10 @@ struct StringStats {
 	DUCKDB_API static uint32_t MaxStringLength(const BaseStatistics &stats);
 	//! Whether or not the strings can contain unicode
 	DUCKDB_API static bool CanContainUnicode(const BaseStatistics &stats);
+	//! Returns the min value (up to a length of StringStatsData::MAX_STRING_MINMAX_SIZE)
+	DUCKDB_API static string Min(const BaseStatistics &stats);
+	//! Returns the max value (up to a length of StringStatsData::MAX_STRING_MINMAX_SIZE)
+	DUCKDB_API static string Max(const BaseStatistics &stats);
 
 	//! Resets the max string length so HasMaxStringLength() is false
 	DUCKDB_API static void ResetMaxStringLength(BaseStatistics &stats);
@@ -57,6 +61,9 @@ struct StringStats {
 	DUCKDB_API static void Serialize(const BaseStatistics &stats, FieldWriter &writer);
 	DUCKDB_API static BaseStatistics Deserialize(FieldReader &reader, LogicalType type);
 
+	DUCKDB_API static void FormatSerialize(const BaseStatistics &stats, FormatSerializer &serializer);
+	DUCKDB_API static BaseStatistics FormatDeserialize(FormatDeserializer &deserializer, LogicalType type);
+
 	DUCKDB_API static string ToString(const BaseStatistics &stats);
 
 	DUCKDB_API static FilterPropagateResult CheckZonemap(const BaseStatistics &stats, ExpressionType comparison_type,

package/src/duckdb/src/include/duckdb/storage/statistics/struct_stats.hpp

@@ -32,6 +32,9 @@ struct StructStats {
 	DUCKDB_API static void Serialize(const BaseStatistics &stats, FieldWriter &writer);
 	DUCKDB_API static BaseStatistics Deserialize(FieldReader &reader, LogicalType type);
 
+	DUCKDB_API static void FormatSerialize(const BaseStatistics &stats, FormatSerializer &serializer);
+	DUCKDB_API static BaseStatistics FormatDeserialize(FormatDeserializer &deserializer, LogicalType type);
+
 	DUCKDB_API static string ToString(const BaseStatistics &stats);
 
 	DUCKDB_API static void Merge(BaseStatistics &stats, const BaseStatistics &other);

package/src/duckdb/src/include/duckdb/storage/storage_info.hpp

@@ -65,9 +65,9 @@ struct DatabaseHeader {
 	//! The iteration count, increases by 1 every time the storage is checkpointed.
 	uint64_t iteration;
 	//! A pointer to the initial meta block
-	block_id_t meta_block;
+	idx_t meta_block;
 	//! A pointer to the block containing the free list
-	block_id_t free_list;
+	idx_t free_list;
 	//! The number of blocks that is in the file as of this database header. If the file is larger than BLOCK_SIZE *
 	//! block_count any blocks appearing AFTER block_count are implicitly part of the free_list.
 	uint64_t block_count;

package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp

@@ -65,7 +65,7 @@ public:
 
 	virtual bool AutomaticCheckpoint(idx_t estimated_wal_bytes) = 0;
 	virtual unique_ptr<StorageCommitState> GenStorageCommitState(Transaction &transaction, bool checkpoint) = 0;
-	virtual bool IsCheckpointClean(block_id_t checkpoint_id) = 0;
+	virtual bool IsCheckpointClean(MetaBlockPointer checkpoint_id) = 0;
 	virtual void CreateCheckpoint(bool delete_wal = false, bool force_checkpoint = false) = 0;
 	virtual DatabaseSize GetDatabaseSize() = 0;
 	virtual shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) = 0;
@@ -109,7 +109,7 @@ public:
 public:
 	bool AutomaticCheckpoint(idx_t estimated_wal_bytes) override;
 	unique_ptr<StorageCommitState> GenStorageCommitState(Transaction &transaction, bool checkpoint) override;
-	bool IsCheckpointClean(block_id_t checkpoint_id) override;
+	bool IsCheckpointClean(MetaBlockPointer checkpoint_id) override;
 	void CreateCheckpoint(bool delete_wal, bool force_checkpoint) override;
 	DatabaseSize GetDatabaseSize() override;
 	shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) override;

package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp

@@ -18,6 +18,9 @@ struct SelectionVector;
 class Transaction;
 struct TransactionData;
 
+class FormatSerializer;
+class FormatDeserializer;
+
 enum class ChunkInfoType : uint8_t { CONSTANT_INFO, VECTOR_INFO, EMPTY_INFO };
 
 class ChunkInfo {
@@ -41,10 +44,14 @@ public:
 	//! Returns whether or not a single row in the ChunkInfo should be used or not for the given transaction
 	virtual bool Fetch(TransactionData transaction, row_t row) = 0;
 	virtual void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) = 0;
+	virtual idx_t GetCommittedDeletedCount(idx_t max_count) = 0;
 
 	virtual void Serialize(Serializer &serialize) = 0;
 	static unique_ptr<ChunkInfo> Deserialize(Deserializer &source);
 
+	virtual void FormatSerialize(FormatSerializer &serializer) const = 0;
+	static unique_ptr<ChunkInfo> FormatDeserialize(FormatDeserializer &deserializer);
+
 public:
 	template <class TARGET>
 	TARGET &Cast() {
@@ -79,14 +86,18 @@ public:
 	                            SelectionVector &sel_vector, idx_t max_count) override;
 	bool Fetch(TransactionData transaction, row_t row) override;
 	void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override;
+	idx_t GetCommittedDeletedCount(idx_t max_count) override;
 
 	void Serialize(Serializer &serialize) override;
 	static unique_ptr<ChunkInfo> Deserialize(Deserializer &source);
 
+	void FormatSerialize(FormatSerializer &serializer) const override;
+	static unique_ptr<ChunkInfo> FormatDeserialize(FormatDeserializer &deserializer);
+
 private:
 	template <class OP>
 	idx_t TemplatedGetSelVector(transaction_t start_time, transaction_t transaction_id, SelectionVector &sel_vector,
-	                            idx_t max_count);
+	                            idx_t max_count) const;
 };
 
 class ChunkVectorInfo : public ChunkInfo {
@@ -107,12 +118,13 @@ public:
 
 public:
 	idx_t GetSelVector(transaction_t start_time, transaction_t transaction_id, SelectionVector &sel_vector,
-	                   idx_t max_count);
+	                   idx_t max_count) const;
 	idx_t GetSelVector(TransactionData transaction, SelectionVector &sel_vector, idx_t max_count) override;
 	idx_t GetCommittedSelVector(transaction_t min_start_id, transaction_t min_transaction_id,
 	                            SelectionVector &sel_vector, idx_t max_count) override;
 	bool Fetch(TransactionData transaction, row_t row) override;
 	void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override;
+	idx_t GetCommittedDeletedCount(idx_t max_count) override;
 
 	void Append(idx_t start, idx_t end, transaction_t commit_id);
 
@@ -127,10 +139,13 @@ public:
 	void Serialize(Serializer &serialize) override;
 	static unique_ptr<ChunkInfo> Deserialize(Deserializer &source);
 
+	void FormatSerialize(FormatSerializer &serializer) const override;
+	static unique_ptr<ChunkInfo> FormatDeserialize(FormatDeserializer &deserializer);
+
 private:
 	template <class OP>
 	idx_t TemplatedGetSelVector(transaction_t start_time, transaction_t transaction_id, SelectionVector &sel_vector,
-	                            idx_t max_count);
+	                            idx_t max_count) const;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/common/vector.hpp"
 #include "duckdb/storage/data_pointer.hpp"
 #include "duckdb/storage/table/table_statistics.hpp"
+#include "duckdb/storage/metadata/metadata_manager.hpp"
 
 namespace duckdb {
 class BaseStatistics;
@@ -24,8 +25,7 @@ public:
 	TableStatistics table_stats;
 	idx_t total_rows;
 	idx_t row_group_count;
-	block_id_t block_id;
-	idx_t offset;
+	MetaBlockPointer block_pointer;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -74,7 +74,6 @@ public:
 	RowGroupCollection &GetCollection() {
 		return collection.get();
 	}
-	DatabaseInstance &GetDatabase();
 	BlockManager &GetBlockManager();
 	DataTableInfo &GetTableInfo();
 
@@ -82,7 +81,7 @@ public:
 	                               ExpressionExecutor &executor, CollectionScanState &scan_state,
 	                               DataChunk &scan_chunk);
 	unique_ptr<RowGroup> AddColumn(RowGroupCollection &collection, ColumnDefinition &new_column,
-	                               ExpressionExecutor &executor, Expression *default_value, Vector &intermediate);
+	                               ExpressionExecutor &executor, Expression &default_value, Vector &intermediate);
 	unique_ptr<RowGroup> RemoveColumn(RowGroupCollection &collection, idx_t removed_column);
 
 	void CommitDrop();
@@ -123,6 +122,7 @@ public:
 	idx_t Delete(TransactionData transaction, DataTable &table, row_t *row_ids, idx_t count);
 
 	RowGroupWriteData WriteToDisk(PartialBlockManager &manager, const vector<CompressionType> &compression_types);
+	bool AllDeleted();
 	RowGroupPointer Checkpoint(RowGroupWriter &writer, TableStatistics &global_stats);
 	static void Serialize(RowGroupPointer &pointer, Serializer &serializer);
 	static RowGroupPointer Deserialize(Deserializer &source, const vector<LogicalType> &columns);
@@ -147,6 +147,10 @@ public:
 
 	void NextVector(CollectionScanState &state);
 
+	// Serialization
+	static void FormatSerialize(RowGroupPointer &pointer, FormatSerializer &serializer);
+	static RowGroupPointer FormatDeserialize(FormatDeserializer &deserializer);
+
 private:
 	ChunkInfo *GetChunkInfo(idx_t vector_idx);
 	ColumnData &GetColumn(storage_t c);
@@ -162,7 +166,7 @@ private:
 private:
 	mutex row_group_lock;
 	mutex stats_lock;
-	vector<BlockPointer> column_pointers;
+	vector<MetaBlockPointer> column_pointers;
 	unique_ptr<atomic<bool>[]> is_loaded;
 };
 
@@ -170,6 +174,7 @@ struct VersionNode {
 	unique_ptr<ChunkInfo> info[RowGroup::ROW_GROUP_VECTOR_COUNT];
 
 	void SetStart(idx_t start);
+	idx_t GetCommittedDeletedCount(idx_t count);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp

@@ -27,6 +27,7 @@ class DuckTransaction;
 class BoundConstraint;
 class RowGroupSegmentTree;
 struct ColumnSegmentInfo;
+class MetadataManager;
 
 class RowGroupCollection {
 public:
@@ -45,7 +46,6 @@ public:
 	void AppendRowGroup(SegmentLock &l, idx_t start_row);
 	//! Get the nth row-group, negative numbers start from the back (so -1 is the last row group, etc)
 	RowGroup *GetRowGroup(int64_t index);
-	idx_t RowGroupCount();
 	void Verify();
 
 	void InitializeScan(CollectionScanState &state, const vector<column_t> &column_ids, TableFilterSet *table_filters);
@@ -93,7 +93,7 @@ public:
 	const vector<LogicalType> &GetTypes() const;
 
 	shared_ptr<RowGroupCollection> AddColumn(ClientContext &context, ColumnDefinition &new_column,
-	                                         Expression *default_value);
+	                                         Expression &default_value);
 	shared_ptr<RowGroupCollection> RemoveColumn(idx_t col_idx);
 	shared_ptr<RowGroupCollection> AlterType(ClientContext &context, idx_t changed_idx, const LogicalType &target_type,
 	                                         vector<column_t> bound_columns, Expression &cast_expr);
@@ -104,10 +104,10 @@ public:
 	void SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats);
 
 	AttachedDatabase &GetAttached();
-	DatabaseInstance &GetDatabase();
 	BlockManager &GetBlockManager() {
 		return block_manager;
 	}
+	MetadataManager &GetMetadataManager();
 	DataTableInfo &GetTableInfo() {
 		return *info;
 	}

package/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp

@@ -14,7 +14,7 @@
 namespace duckdb {
 struct DataTableInfo;
 class PersistentTableData;
-class MetaBlockReader;
+class MetadataReader;
 
 class RowGroupSegmentTree : public SegmentTree<RowGroup, true> {
 public:
@@ -29,7 +29,7 @@ protected:
 	RowGroupCollection &collection;
 	idx_t current_row_group;
 	idx_t max_row_group;
-	unique_ptr<MetaBlockReader> reader;
+	unique_ptr<MetadataReader> reader;
 };
 
 } // namespace duckdb