duckdb 0.5.2-dev547.0 → 0.5.2-dev561.0

package/src/duckdb.cpp

@@ -1711,20 +1711,22 @@ unique_ptr<Key> Key::CreateKey(const char *value);
 
 namespace duckdb {
 class BlockHandle;
+class BlockManager;
 class BufferHandle;
 class DatabaseInstance;
 
 //! This struct is responsible for reading meta data from disk
 class MetaBlockReader : public Deserializer {
 public:
-	MetaBlockReader(DatabaseInstance &db, block_id_t block);
+	MetaBlockReader(BlockManager &block_manager, block_id_t block, bool free_blocks_on_read = true);
 	~MetaBlockReader() override;
 
-	DatabaseInstance &db;
+	BlockManager &block_manager;
 	shared_ptr<BlockHandle> block;
 	BufferHandle handle;
 	idx_t offset;
 	block_id_t next_block;
+	bool free_blocks_on_read;
 
 public:
 	//! Read content of size read_size into the buffer
@@ -2241,8 +2243,9 @@ enum VerifyExistenceType : uint8_t {
 
 class ART : public Index {
 public:
-	ART(const vector<column_t> &column_ids, const vector<unique_ptr<Expression>> &unbound_expressions,
-	    IndexConstraintType constraint_type, DatabaseInstance &db, idx_t block_id = DConstants::INVALID_INDEX,
+	ART(const vector<column_t> &column_ids, TableIOManager &table_io_manager,
+	    const vector<unique_ptr<Expression>> &unbound_expressions, IndexConstraintType constraint_type,
+	    DatabaseInstance &db, idx_t block_id = DConstants::INVALID_INDEX,
 	    idx_t block_offset = DConstants::INVALID_INDEX);
 	~ART() override;
 
@@ -2251,6 +2254,7 @@ public:
 
 	DatabaseInstance &db;
 
+public:
 	//! Initialize a scan on the index with the given expression and column ids
 	//! to fetch from the base table for a single predicate
 	unique_ptr<IndexScanState> InitializeScanSinglePredicate(Transaction &transaction, Value value,
@@ -3487,6 +3491,39 @@ protected:
 
 
 
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table_io_manager.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+namespace duckdb {
+class BlockManager;
+class DataTable;
+
+class TableIOManager {
+public:
+	virtual ~TableIOManager() {
+	}
+
+	//! Obtains a reference to the TableIOManager of a specific table
+	static TableIOManager &Get(DataTable &table);
+
+	//! The block manager used for managing index data
+	virtual BlockManager &GetIndexBlockManager() = 0;
+
+	//! The block manager used for storing row group data
+	virtual BlockManager &GetBlockManagerForRowData() = 0;
+};
+
+} // namespace duckdb
+
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -3586,10 +3623,10 @@ class TransactionManager;
 //! server crashes or is shut down.
 class WriteAheadLog {
 public:
-	explicit WriteAheadLog(DatabaseInstance &database);
+	//! Initialize the WAL in the specified directory
+	explicit WriteAheadLog(DatabaseInstance &database, const string &path);
+	virtual ~WriteAheadLog();
 
-	//! Whether or not the WAL has been initialized
-	bool initialized;
 	//! Skip writing to the WAL
 	bool skip_writing;
 
@@ -3597,14 +3634,12 @@ public:
 	//! Replay the WAL
 	static bool Replay(DatabaseInstance &database, string &path);
 
-	//! Initialize the WAL in the specified directory
-	void Initialize(string &path);
 	//! Returns the current size of the WAL in bytes
 	int64_t GetWALSize();
 	//! Gets the total bytes written to the WAL since startup
 	idx_t GetTotalWritten();
 
-	void WriteCreateTable(TableCatalogEntry *entry);
+	virtual void WriteCreateTable(TableCatalogEntry *entry);
 	void WriteDropTable(TableCatalogEntry *entry);
 
 	void WriteCreateSchema(SchemaCatalogEntry *entry);
@@ -3650,7 +3685,7 @@ public:
 
 	void WriteCheckpoint(block_id_t meta_block);
 
-private:
+protected:
 	DatabaseInstance &database;
 	unique_ptr<BufferedFileWriter> writer;
 	string wal_path;
@@ -3662,19 +3697,38 @@ private:
 namespace duckdb {
 class BlockManager;
 class Catalog;
+class CheckpointWriter;
 class DatabaseInstance;
 class TransactionManager;
 class TableCatalogEntry;
 
+struct DatabaseSize {
+	idx_t total_blocks = 0;
+	idx_t block_size = 0;
+	idx_t free_blocks = 0;
+	idx_t used_blocks = 0;
+	idx_t bytes = 0;
+	idx_t wal_size = 0;
+};
+
+class StorageCommitState {
+public:
+	// Destruction of this object, without prior call to FlushCommit,
+	// will roll back the committed changes.
+	virtual ~StorageCommitState() {
+	}
+
+	// Make the commit persistent
+	virtual void FlushCommit() = 0;
+};
+
 //! StorageManager is responsible for managing the physical storage of the
 //! database on disk
 class StorageManager {
 public:
 	StorageManager(DatabaseInstance &db, string path, bool read_only);
-	~StorageManager();
+	virtual ~StorageManager();
 
-	//! The BlockManager to read/store meta information and data in blocks
-	unique_ptr<BlockManager> block_manager;
 	//! The BufferManager of the database
 	unique_ptr<BufferManager> buffer_manager;
 	//! The database this storagemanager belongs to
@@ -3686,35 +3740,62 @@ public:
 
 	//! Initialize a database or load an existing database from the given path
 	void Initialize();
-	//! Get the WAL of the StorageManager, returns nullptr if in-memory
-	WriteAheadLog *GetWriteAheadLog() {
-		return wal.initialized ? &wal : nullptr;
-	}
 
 	DatabaseInstance &GetDatabase() {
 		return db;
 	}
 
-	void CreateCheckpoint(bool delete_wal = false, bool force_checkpoint = false);
+	//! Get the WAL of the StorageManager, returns nullptr if in-memory
+	WriteAheadLog *GetWriteAheadLog() {
+		return wal.get();
+	}
 
 	string GetDBPath() {
 		return path;
 	}
 	bool InMemory();
 
-private:
-	//! Load the database from a directory
-	void LoadDatabase();
+	virtual bool AutomaticCheckpoint(idx_t estimated_wal_bytes) = 0;
+	virtual unique_ptr<StorageCommitState> GenStorageCommitState(Transaction &transaction, bool checkpoint) = 0;
+	virtual bool IsCheckpointClean(block_id_t checkpoint_id) = 0;
+	virtual void CreateCheckpoint(bool delete_wal = false, bool force_checkpoint = false) = 0;
+	virtual DatabaseSize GetDatabaseSize() = 0;
+	virtual shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) = 0;
+
+protected:
+	virtual void LoadDatabase() = 0;
+	virtual void CreateBufferManager();
 
 	//! The path of the database
 	string path;
 	//! The WriteAheadLog of the storage manager
-	WriteAheadLog wal;
+	unique_ptr<WriteAheadLog> wal;
 
 	//! Whether or not the database is opened in read-only mode
 	bool read_only;
 };
 
+//! Stores database in a single file.
+class SingleFileStorageManager : public StorageManager {
+public:
+	SingleFileStorageManager(DatabaseInstance &db, string path, bool read_only);
+
+	//! The BlockManager to read/store meta information and data in blocks
+	unique_ptr<BlockManager> block_manager;
+	//! TableIoManager
+	unique_ptr<TableIOManager> table_io_manager;
+
+public:
+	bool AutomaticCheckpoint(idx_t estimated_wal_bytes) override;
+	unique_ptr<StorageCommitState> GenStorageCommitState(Transaction &transaction, bool checkpoint) override;
+	bool IsCheckpointClean(block_id_t checkpoint_id) override;
+	void CreateCheckpoint(bool delete_wal, bool force_checkpoint) override;
+	DatabaseSize GetDatabaseSize() override;
+	shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) override;
+
+protected:
+	void LoadDatabase() override;
+};
 } // namespace duckdb
 
 
@@ -3766,13 +3847,14 @@ void AddDataTableIndex(DataTable *storage, vector<ColumnDefinition> &columns, ve
 		bound_expressions.push_back(make_unique<BoundReferenceExpression>(columns[key].Type(), key_nr++));
 		column_ids.push_back(column.StorageOid());
 	}
-	// create an adaptive radix tree around the expressions
 	unique_ptr<ART> art;
+	// create an adaptive radix tree around the expressions
 	if (index_block) {
-		art = make_unique<ART>(column_ids, move(unbound_expressions), constraint_type, storage->db,
-		                       index_block->block_id, index_block->offset);
+		art = make_unique<ART>(column_ids, TableIOManager::Get(*storage), move(unbound_expressions), constraint_type,
+		                       storage->db, index_block->block_id, index_block->offset);
 	} else {
-		art = make_unique<ART>(column_ids, move(unbound_expressions), constraint_type, storage->db);
+		art = make_unique<ART>(column_ids, TableIOManager::Get(*storage), move(unbound_expressions), constraint_type,
+		                       storage->db);
 		if (!storage->IsRoot()) {
 			throw TransactionException("Transaction conflict: cannot add an index to a table that has been altered!");
 		}
@@ -3810,7 +3892,10 @@ TableCatalogEntry::TableCatalogEntry(Catalog *catalog, SchemaCatalogEntry *schem
 			}
 			storage_columns.push_back(col_def.Copy());
 		}
-		storage = make_shared<DataTable>(catalog->db, schema->name, name, move(storage_columns), move(info->data));
+		storage =
+		    make_shared<DataTable>(catalog->db, StorageManager::GetStorageManager(catalog->db).GetTableIOManager(info),
+		                           schema->name, name, move(storage_columns), move(info->data));
+
 		// create the unique indexes for the UNIQUE and PRIMARY KEY and FOREIGN KEY constraints
 		idx_t indexes_idx = 0;
 		for (idx_t i = 0; i < bound_constraints.size(); i++) {
@@ -58079,15 +58164,17 @@ ExpressionExecutorState::ExpressionExecutorState(const string &name) : profiler(
 
 
 
+
 #include <algorithm>
 #include <cstring>
 #include <ctgmath>
 
 namespace duckdb {
 
-ART::ART(const vector<column_t> &column_ids, const vector<unique_ptr<Expression>> &unbound_expressions,
-         IndexConstraintType constraint_type, DatabaseInstance &db, idx_t block_id, idx_t block_offset)
-    : Index(IndexType::ART, column_ids, unbound_expressions, constraint_type), db(db) {
+ART::ART(const vector<column_t> &column_ids, TableIOManager &table_io_manager,
+         const vector<unique_ptr<Expression>> &unbound_expressions, IndexConstraintType constraint_type,
+         DatabaseInstance &db, idx_t block_id, idx_t block_offset)
+    : Index(IndexType::ART, table_io_manager, column_ids, unbound_expressions, constraint_type), db(db) {
 	if (block_id != DConstants::INVALID_INDEX) {
 		tree = Node::Deserialize(*this, block_id, block_offset);
 	} else {
@@ -58377,7 +58464,8 @@ void ART::ConstructAndMerge(IndexLock &lock, PayloadScanner &scanner, Allocator
 	payload_types.emplace_back(LogicalType::ROW_TYPE);
 
 	auto skipped_all_nulls = false;
-	auto temp_art = make_unique<ART>(this->column_ids, this->unbound_expressions, this->constraint_type, this->db);
+	auto temp_art = make_unique<ART>(this->column_ids, this->table_io_manager, this->unbound_expressions,
+	                                 this->constraint_type, this->db);
 	for (;;) {
 		DataChunk ordered_chunk;
 		ordered_chunk.Initialize(allocator, payload_types);
@@ -58420,7 +58508,8 @@ void ART::ConstructAndMerge(IndexLock &lock, PayloadScanner &scanner, Allocator
 		auto row_ids = FlatVector::GetData<row_t>(row_identifiers);
 
 		// construct the ART of this chunk
-		auto art = make_unique<ART>(this->column_ids, this->unbound_expressions, this->constraint_type, this->db);
+		auto art = make_unique<ART>(this->column_ids, this->table_io_manager, this->unbound_expressions,
+		                            this->constraint_type, this->db);
 		auto key_section = KeySection(start_idx, ordered_chunk.size() - 1, 0, 0);
 		auto has_constraint = IsPrimary() || IsUnique();
 		Construct(keys, row_ids, art->tree, key_section, has_constraint);
@@ -59404,8 +59493,7 @@ void Leaf::Remove(row_t row_id) {
 }
 
 BlockPointer Leaf::Serialize(duckdb::MetaBlockWriter &writer) {
-	auto block_id = writer.block->id;
-	uint32_t offset = writer.offset;
+	auto ptr = writer.GetBlockPointer();
 	// Write Node Type
 	writer.Write(type);
 	// Write compression Info
@@ -59417,7 +59505,7 @@ BlockPointer Leaf::Serialize(duckdb::MetaBlockWriter &writer) {
 	for (idx_t i = 0; i < count; i++) {
 		writer.Write(row_ids[i]);
 	}
-	return {block_id, offset};
+	return ptr;
 }
 
 Leaf *Leaf::Deserialize(MetaBlockReader &reader) {
@@ -59450,8 +59538,6 @@ void Leaf::Merge(bool &has_constraint, Node *&l_node, Node *&r_node) {
 
 
 
-
-
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -59469,6 +59555,9 @@ using std::swap;
 }
 
 
+
+
+
 namespace duckdb {
 
 InternalType::InternalType(Node *n) {
@@ -59602,8 +59691,7 @@ BlockPointer Node::SerializeInternal(ART &art, duckdb::MetaBlockWriter &writer,
 	for (idx_t i = 0; i < internal_type.children_size; i++) {
 		child_offsets.emplace_back(internal_type.children[i].Serialize(art, writer));
 	}
-	auto block_id = writer.block->id;
-	uint32_t offset = writer.offset;
+	auto ptr = writer.GetBlockPointer();
 	// Write Node Type
 	writer.Write(type);
 	// Write count
@@ -59619,7 +59707,7 @@ BlockPointer Node::SerializeInternal(ART &art, duckdb::MetaBlockWriter &writer,
 		writer.Write(offsets.block_id);
 		writer.Write(offsets.offset);
 	}
-	return {block_id, offset};
+	return ptr;
 }
 
 BlockPointer Node::Serialize(ART &art, duckdb::MetaBlockWriter &writer) {
@@ -59655,7 +59743,7 @@ void Node::DeserializeInternal(duckdb::MetaBlockReader &reader) {
 }
 
 Node *Node::Deserialize(ART &art, idx_t block_id, idx_t offset) {
-	MetaBlockReader reader(art.db, block_id);
+	MetaBlockReader reader(art.table_io_manager.GetIndexBlockManager(), block_id);
 	reader.offset = offset;
 	auto n = reader.Read<uint8_t>();
 	NodeType node_type(static_cast<NodeType>(n));
@@ -79673,6 +79761,7 @@ public:
 
 
 
+
 namespace duckdb {
 
 //===--------------------------------------------------------------------===//
@@ -79710,7 +79799,8 @@ unique_ptr<GlobalSinkState> PhysicalCreateIndex::GetGlobalSinkState(ClientContex
 	// create the global index
 	switch (info->index_type) {
 	case IndexType::ART: {
-		state->global_index = make_unique<ART>(storage_ids, unbound_expressions, info->constraint_type, *context.db);
+		state->global_index = make_unique<ART>(storage_ids, TableIOManager::Get(*table.storage), unbound_expressions,
+		                                       info->constraint_type, *context.db);
 		break;
 	}
 	default:
@@ -79729,8 +79819,8 @@ unique_ptr<LocalSinkState> PhysicalCreateIndex::GetLocalSinkState(ExecutionConte
 	// create the local index
 	switch (info->index_type) {
 	case IndexType::ART: {
-		state->local_index =
-		    make_unique<ART>(storage_ids, unbound_expressions, info->constraint_type, *context.client.db);
+		state->local_index = make_unique<ART>(storage_ids, TableIOManager::Get(*table.storage), unbound_expressions,
+		                                      info->constraint_type, *context.client.db);
 		break;
 	}
 	default:
@@ -122487,32 +122577,18 @@ void PragmaDatabaseSizeFunction(ClientContext &context, TableFunctionInput &data
 		return;
 	}
 	auto &storage = StorageManager::GetStorageManager(context);
-	auto &block_manager = BlockManager::GetBlockManager(context);
 	auto &buffer_manager = BufferManager::GetBufferManager(context);
 
+	auto ds = storage.GetDatabaseSize();
+
 	output.SetCardinality(1);
-	if (!storage.InMemory()) {
-		auto total_blocks = block_manager.TotalBlocks();
-		auto block_size = Storage::BLOCK_ALLOC_SIZE;
-		auto free_blocks = block_manager.FreeBlocks();
-		auto used_blocks = total_blocks - free_blocks;
-		auto bytes = (total_blocks * block_size);
-		auto wal = storage.GetWriteAheadLog();
-		auto wal_size = wal ? wal->GetWALSize() : 0;
-		output.data[0].SetValue(0, Value(StringUtil::BytesToHumanReadableString(bytes)));
-		output.data[1].SetValue(0, Value::BIGINT(block_size));
-		output.data[2].SetValue(0, Value::BIGINT(total_blocks));
-		output.data[3].SetValue(0, Value::BIGINT(used_blocks));
-		output.data[4].SetValue(0, Value::BIGINT(free_blocks));
-		output.data[5].SetValue(0, Value(StringUtil::BytesToHumanReadableString(wal_size)));
-	} else {
-		output.data[0].SetValue(0, Value());
-		output.data[1].SetValue(0, Value());
-		output.data[2].SetValue(0, Value());
-		output.data[3].SetValue(0, Value());
-		output.data[4].SetValue(0, Value());
-		output.data[5].SetValue(0, Value());
-	}
+	output.data[0].SetValue(0, Value(StringUtil::BytesToHumanReadableString(ds.bytes)));
+	output.data[1].SetValue(0, Value::BIGINT(ds.block_size));
+	output.data[2].SetValue(0, Value::BIGINT(ds.total_blocks));
+	output.data[3].SetValue(0, Value::BIGINT(ds.used_blocks));
+	output.data[4].SetValue(0, Value::BIGINT(ds.free_blocks));
+	output.data[5].SetValue(0, Value(StringUtil::BytesToHumanReadableString(ds.wal_size)));
+
 	output.data[6].SetValue(0, Value(StringUtil::BytesToHumanReadableString(buffer_manager.GetUsedMemory())));
 	auto max_memory = buffer_manager.GetMaxMemory();
 	output.data[7].SetValue(0, max_memory == (idx_t)-1 ? Value("Unlimited")
@@ -130510,14 +130586,6 @@ BufferManager &BufferManager::GetBufferManager(DatabaseInstance &db) {
 	return *db.GetStorageManager().buffer_manager;
 }
 
-BlockManager &BlockManager::GetBlockManager(DatabaseInstance &db) {
-	return *db.GetStorageManager().block_manager;
-}
-
-BlockManager &BlockManager::GetBlockManager(ClientContext &context) {
-	return BlockManager::GetBlockManager(DatabaseInstance::GetDatabase(context));
-}
-
 DatabaseInstance &DatabaseInstance::GetDatabase(ClientContext &context) {
 	return *context.db;
 }
@@ -130604,10 +130672,11 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
 		config.options.temporary_directory = string();
 	}
 
-	//	config.create_storage_manager(config.options.database_path,
-	//	                              config.options.access_mode == AccessMode::READ_ONLY);
-	storage = make_unique<StorageManager>(*this, config.options.database_path,
-	                                      config.options.access_mode == AccessMode::READ_ONLY);
+	// TODO: Support an extension here, to generate different storage managers
+	// depending on the DB path structure/prefix.
+	const string dbPath = config.options.database_path;
+	storage = make_unique<SingleFileStorageManager>(*this, dbPath, config.options.access_mode == AccessMode::READ_ONLY);
+
 	catalog = make_unique<Catalog>(*this);
 	transaction_manager = make_unique<TransactionManager>(*this);
 	scheduler = make_unique<TaskScheduler>(*this);
@@ -192836,10 +192905,7 @@ public:
 	using BlockManager::BlockManager;
 
 	// LCOV_EXCL_START
-	void StartCheckpoint() override {
-		throw InternalException("Cannot perform IO in in-memory database!");
-	}
-	unique_ptr<Block> CreateBlock(block_id_t block_id) override {
+	unique_ptr<Block> CreateBlock(block_id_t block_id, FileBuffer *source_buffer) override {
 		throw InternalException("Cannot perform IO in in-memory database!");
 	}
 	block_id_t GetFreeBlockId() override {
@@ -193756,7 +193822,17 @@ Allocator &BufferManager::GetBufferAllocator() {
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/storage/checkpoint/table_data_reader.hpp
+// duckdb/storage/checkpoint/table_data_writer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/checkpoint/row_group_writer.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -193773,6 +193849,14 @@ Allocator &BufferManager::GetBufferAllocator() {
 
 
 
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/partial_block_manager.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
 
 
 
@@ -193792,83 +193876,390 @@ class TableCatalogEntry;
 class ViewCatalogEntry;
 class TypeCatalogEntry;
 
-struct PartialColumnSegment {
-	ColumnSegment *segment;
+struct PartialBlockState {
+	block_id_t block_id;
+	//! How big is the block we're writing to. (Total bytes to assign).
+	uint32_t block_size;
+	//! How many bytes of the allocation are used. (offset_in_block of next allocation)
 	uint32_t offset_in_block;
+	//! How many times has the block been used?
+	uint32_t block_use_count;
 };
 
 struct PartialBlock {
-	block_id_t block_id;
-	//! The block handle that stores this block
-	shared_ptr<BlockHandle> block;
-	//! The segments that are involved in the partial block
-	vector<PartialColumnSegment> segments;
+	explicit PartialBlock(PartialBlockState state) : state(move(state)) {
+	}
+	virtual ~PartialBlock() {
+	}
+
+	PartialBlockState state;
 
-	void FlushToDisk(DatabaseInstance &db);
+public:
+	virtual void Flush() = 0;
+};
+
+struct PartialBlockAllocation {
+	// BlockManager owning the block_id
+	BlockManager *block_manager {nullptr};
+	//! How many bytes assigned to the caller?
+	uint32_t allocation_size;
+	//! State of assigned block.
+	PartialBlockState state;
+	//! Arbitrary state related to partial block storage.
+	unique_ptr<PartialBlock> partial_block;
 };
 
-//! CheckpointManager is responsible for checkpointing the database
-class CheckpointManager {
+//! Enables sharing blocks across some scope. Scope is whatever we want to share
+//! blocks across. It may be an entire checkpoint or just a single row group.
+//! In any case, they must share a block manager.
+class PartialBlockManager {
 public:
-	static constexpr const idx_t PARTIAL_BLOCK_THRESHOLD = Storage::BLOCK_SIZE / 5 * 4;
+	// 20% free / 80% utilization
+	static constexpr const idx_t DEFAULT_MAX_PARTIAL_BLOCK_SIZE = Storage::BLOCK_SIZE / 5 * 4;
+	// Max number of shared references to a block. No effective limit by default.
+	static constexpr const idx_t DEFAULT_MAX_USE_COUNT = 1 << 20;
+	// No point letting map size grow unbounded. We'll drop blocks with the
+	// least free space first.
+	static constexpr const idx_t MAX_BLOCK_MAP_SIZE = 1 << 31;
 
 public:
-	explicit CheckpointManager(DatabaseInstance &db);
+	PartialBlockManager(BlockManager &block_manager, uint32_t max_partial_block_size = DEFAULT_MAX_PARTIAL_BLOCK_SIZE,
+	                    uint32_t max_use_count = DEFAULT_MAX_USE_COUNT)
+	    : block_manager(block_manager), max_partial_block_size(max_partial_block_size), max_use_count(max_use_count) {
+	}
+
+public:
+	//! Flush any remaining partial blocks to disk
+	void FlushPartialBlocks();
+
+	PartialBlockAllocation GetBlockAllocation(uint32_t segment_size);
+
+	virtual void AllocateBlock(PartialBlockState &state, uint32_t segment_size);
+
+	//! Register a partially filled block that is filled with "segment_size" entries
+	void RegisterPartialBlock(PartialBlockAllocation &&allocation);
+
+protected:
+	BlockManager &block_manager;
+	//! A map of (available space -> PartialBlock) for partially filled blocks
+	//! This is a multimap because there might be outstanding partial blocks with
+	//! the same amount of left-over space
+	multimap<idx_t, unique_ptr<PartialBlock>> partially_filled_blocks;
+
+	//! The maximum size (in bytes) at which a partial block will be considered a partial block
+	uint32_t max_partial_block_size;
+	uint32_t max_use_count;
+
+protected:
+	//! Try to obtain a partially filled block that can fit "segment_size" bytes
+	//! If successful, returns true and returns the block_id and offset_in_block to write to
+	//! Otherwise, returns false
+	bool GetPartialBlock(idx_t segment_size, unique_ptr<PartialBlock> &state);
+};
+
+} // namespace duckdb
+
+
+
+namespace duckdb {
+class DatabaseInstance;
+class ClientContext;
+class ColumnSegment;
+class MetaBlockReader;
+class SchemaCatalogEntry;
+class SequenceCatalogEntry;
+class TableCatalogEntry;
+class ViewCatalogEntry;
+class TypeCatalogEntry;
+
+class CheckpointWriter {
+public:
+	explicit CheckpointWriter(DatabaseInstance &db) : db(db) {
+	}
+	virtual ~CheckpointWriter() {
+	}
 
 	//! The database
 	DatabaseInstance &db;
+
+	virtual MetaBlockWriter &GetMetaBlockWriter() = 0;
+	virtual unique_ptr<TableDataWriter> GetTableDataWriter(TableCatalogEntry &table) = 0;
+	virtual BlockPointer WriteIndexData(IndexCatalogEntry &index_catalog) = 0;
+
+protected:
+	virtual void WriteSchema(SchemaCatalogEntry &schema);
+	virtual void WriteTable(TableCatalogEntry &table);
+	virtual void WriteView(ViewCatalogEntry &table);
+	virtual void WriteSequence(SequenceCatalogEntry &table);
+	virtual void WriteMacro(ScalarMacroCatalogEntry &table);
+	virtual void WriteTableMacro(TableMacroCatalogEntry &table);
+	virtual void WriteIndex(IndexCatalogEntry &index_catalog);
+	virtual void WriteType(TypeCatalogEntry &table);
+};
+
+class CheckpointReader {
+public:
+	virtual ~CheckpointReader() {
+	}
+
+protected:
+	virtual void LoadCheckpoint(ClientContext &context, MetaBlockReader &reader);
+	virtual void ReadSchema(ClientContext &context, MetaBlockReader &reader);
+	virtual void ReadTable(ClientContext &context, MetaBlockReader &reader);
+	virtual void ReadView(ClientContext &context, MetaBlockReader &reader);
+	virtual void ReadSequence(ClientContext &context, MetaBlockReader &reader);
+	virtual void ReadMacro(ClientContext &context, MetaBlockReader &reader);
+	virtual void ReadTableMacro(ClientContext &context, MetaBlockReader &reader);
+	virtual void ReadIndex(ClientContext &context, MetaBlockReader &reader);
+	virtual void ReadType(ClientContext &context, MetaBlockReader &reader);
+
+	virtual void ReadTableData(ClientContext &context, MetaBlockReader &reader, BoundCreateTableInfo &bound_info);
+};
+
+class SingleFileCheckpointReader final : public CheckpointReader {
+public:
+	explicit SingleFileCheckpointReader(SingleFileStorageManager &storage) : storage(storage) {
+	}
+
+	void LoadFromStorage();
+
+	//! The database
+	SingleFileStorageManager &storage;
+};
+
+//! CheckpointWriter is responsible for checkpointing the database
+class SingleFileRowGroupWriter;
+class SingleFileTableDataWriter;
+
+class SingleFileCheckpointWriter final : public CheckpointWriter {
+	friend class SingleFileRowGroupWriter;
+	friend class SingleFileTableDataWriter;
+
+public:
+	explicit SingleFileCheckpointWriter(DatabaseInstance &db, BlockManager &block_manager)
+	    : CheckpointWriter(db), partial_block_manager(block_manager) {
+	}
+
+	//! Checkpoint the current state of the WAL and flush it to the main storage. This should be called BEFORE any
+	//! connection is available because right now the checkpointing cannot be done online. (TODO)
+	void CreateCheckpoint();
+
+	virtual MetaBlockWriter &GetMetaBlockWriter() override;
+	virtual unique_ptr<TableDataWriter> GetTableDataWriter(TableCatalogEntry &table) override;
+	virtual BlockPointer WriteIndexData(IndexCatalogEntry &index_catalog) override;
+
+	BlockManager &GetBlockManager();
+
+private:
 	//! The metadata writer is responsible for writing schema information
 	unique_ptr<MetaBlockWriter> metadata_writer;
 	//! The table data writer is responsible for writing the DataPointers used by the table chunks
-	unique_ptr<MetaBlockWriter> tabledata_writer;
+	unique_ptr<MetaBlockWriter> table_metadata_writer;
+	//! Because this is single-file storage, we can share partial blocks across
+	//! an entire checkpoint.
+	PartialBlockManager partial_block_manager;
+};
 
+} // namespace duckdb
+
+
+namespace duckdb {
+struct ColumnCheckpointState;
+class CheckpointWriter;
+class ColumnData;
+class ColumnSegment;
+class RowGroup;
+class BaseStatistics;
+class SegmentStatistics;
+
+// Writes data for an entire row group.
+class RowGroupWriter {
 public:
-	//! Checkpoint the current state of the WAL and flush it to the main storage. This should be called BEFORE any
-	//! connction is available because right now the checkpointing cannot be done online. (TODO)
-	void CreateCheckpoint();
-	//! Load from a stored checkpoint
-	void LoadFromStorage();
+	RowGroupWriter(TableCatalogEntry &table, PartialBlockManager &partial_block_manager)
+	    : table(table), partial_block_manager(partial_block_manager) {
+	}
+	virtual ~RowGroupWriter() {
+	}
 
-	//! Try to obtain a partially filled block that can fit "segment_size" bytes
-	//! If successful, returns true and returns the block_id and offset_in_block to write to
-	//! Otherwise, returns false
-	bool GetPartialBlock(ColumnSegment *segment, idx_t segment_size, block_id_t &block_id, uint32_t &offset_in_block,
-	                     PartialBlock *&partial_block_ptr, unique_ptr<PartialBlock> &owned_partial_block);
+	CompressionType GetColumnCompressionType(idx_t i);
 
-	//! Register a partially filled block that is filled with "segment_size" entries
-	void RegisterPartialBlock(ColumnSegment *segment, idx_t segment_size, block_id_t block_id);
+	virtual void WriteColumnDataPointers(ColumnCheckpointState &column_checkpoint_state) = 0;
 
-	//! Flush any remaining partial segments to disk
-	void FlushPartialSegments();
+	virtual MetaBlockWriter &GetPayloadWriter() = 0;
 
-private:
-	void WriteSchema(SchemaCatalogEntry &schema);
-	void WriteTable(TableCatalogEntry &table);
-	void WriteView(ViewCatalogEntry &table);
-	void WriteSequence(SequenceCatalogEntry &table);
-	void WriteMacro(ScalarMacroCatalogEntry &table);
-	void WriteTableMacro(TableMacroCatalogEntry &table);
-	void WriteIndex(IndexCatalogEntry &index_catalog);
-	void WriteType(TypeCatalogEntry &table);
-
-	void ReadSchema(ClientContext &context, MetaBlockReader &reader);
-	void ReadTable(ClientContext &context, MetaBlockReader &reader);
-	void ReadView(ClientContext &context, MetaBlockReader &reader);
-	void ReadSequence(ClientContext &context, MetaBlockReader &reader);
-	void ReadMacro(ClientContext &context, MetaBlockReader &reader);
-	void ReadTableMacro(ClientContext &context, MetaBlockReader &reader);
-	void ReadIndex(ClientContext &context, MetaBlockReader &reader);
-	void ReadType(ClientContext &context, MetaBlockReader &reader);
+	void RegisterPartialBlock(PartialBlockAllocation &&allocation);
+	PartialBlockAllocation GetBlockAllocation(uint32_t segment_size);
+
+protected:
+	TableCatalogEntry &table;
+	PartialBlockManager &partial_block_manager;
+};
+
+// Writes data for an entire row group.
+class SingleFileRowGroupWriter : public RowGroupWriter {
+public:
+	SingleFileRowGroupWriter(TableCatalogEntry &table, PartialBlockManager &partial_block_manager,
+	                         MetaBlockWriter &table_data_writer)
+	    : RowGroupWriter(table, partial_block_manager), table_data_writer(table_data_writer) {
+	}
+
+	//! MetaBlockWriter is a cursor on a given BlockManager. This returns the
+	//! cursor against which we should write payload data for the specified RowGroup.
+	MetaBlockWriter &table_data_writer;
+
+public:
+	virtual void WriteColumnDataPointers(ColumnCheckpointState &column_checkpoint_state) override;
+
+	virtual MetaBlockWriter &GetPayloadWriter() override;
+};
+
+} // namespace duckdb
+
+
+namespace duckdb {
+
+//! The table data writer is responsible for writing the data of a table to
+//! storage.
+//
+//! This is meant to encapsulate and abstract:
+//!  - Storage/encoding of table metadata (block pointers)
+//!  - Mapping management of data block locations
+//! Abstraction will support, for example: tiering, versioning, or splitting into multiple block managers.
+class TableDataWriter {
+public:
+	explicit TableDataWriter(TableCatalogEntry &table);
+	virtual ~TableDataWriter();
+
+public:
+	void WriteTableData();
+
+	CompressionType GetColumnCompressionType(idx_t i);
+
+	virtual void FinalizeTable(vector<unique_ptr<BaseStatistics>> &&global_stats, DataTableInfo *info) = 0;
+	virtual unique_ptr<RowGroupWriter> GetRowGroupWriter(RowGroup &row_group) = 0;
+
+	virtual void AddRowGroup(RowGroupPointer &&row_group_pointer, unique_ptr<RowGroupWriter> &&writer);
+
+protected:
+	TableCatalogEntry &table;
+	// Pointers to the start of each row group.
+	vector<RowGroupPointer> row_group_pointers;
+};
+
+class SingleFileTableDataWriter : public TableDataWriter {
+public:
+	SingleFileTableDataWriter(SingleFileCheckpointWriter &checkpoint_manager, TableCatalogEntry &table,
+	                          MetaBlockWriter &table_data_writer, MetaBlockWriter &meta_data_writer);
+
+public:
+	virtual void FinalizeTable(vector<unique_ptr<BaseStatistics>> &&global_stats, DataTableInfo *info) override;
+	virtual unique_ptr<RowGroupWriter> GetRowGroupWriter(RowGroup &row_group) override;
 
 private:
-	//! A map of (available space -> PartialBlock) for partially filled blocks
-	//! This is a multimap because there might be outstanding partial blocks with the same amount of left-over space
-	multimap<idx_t, unique_ptr<PartialBlock>> partially_filled_blocks;
+	SingleFileCheckpointWriter &checkpoint_manager;
+	// Writes the actual table data
+	MetaBlockWriter &table_data_writer;
+	// Writes the metadata of the table
+	MetaBlockWriter &meta_data_writer;
+};
+
+} // namespace duckdb
+
+
+
+
+
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/column_checkpoint_state.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+
+
+
+
+namespace duckdb {
+class ColumnData;
+class DatabaseInstance;
+class RowGroup;
+class TableDataWriter;
+
+struct ColumnCheckpointState {
+	ColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, RowGroupWriter &writer);
+	virtual ~ColumnCheckpointState();
+
+	RowGroup &row_group;
+	ColumnData &column_data;
+	RowGroupWriter &writer;
+	SegmentTree new_tree;
+	vector<DataPointer> data_pointers;
+	unique_ptr<BaseStatistics> global_stats;
+
+public:
+	virtual unique_ptr<BaseStatistics> GetStatistics();
+
+	virtual void FlushSegment(unique_ptr<ColumnSegment> segment, idx_t segment_size);
+	virtual void WriteDataPointers();
 };
 
 } // namespace duckdb
 
 
+namespace duckdb {
+
+CompressionType RowGroupWriter::GetColumnCompressionType(idx_t i) {
+	return table.columns[i].CompressionType();
+}
+
+void RowGroupWriter::RegisterPartialBlock(PartialBlockAllocation &&allocation) {
+	partial_block_manager.RegisterPartialBlock(move(allocation));
+}
+
+PartialBlockAllocation RowGroupWriter::GetBlockAllocation(uint32_t segment_size) {
+	return partial_block_manager.GetBlockAllocation(segment_size);
+}
+
+void SingleFileRowGroupWriter::WriteColumnDataPointers(ColumnCheckpointState &column_checkpoint_state) {
+	auto &meta_writer = table_data_writer;
+	const auto &data_pointers = column_checkpoint_state.data_pointers;
+
+	meta_writer.Write<idx_t>(data_pointers.size());
+	// then write the data pointers themselves
+	for (idx_t k = 0; k < data_pointers.size(); k++) {
+		auto &data_pointer = data_pointers[k];
+		meta_writer.Write<idx_t>(data_pointer.row_start);
+		meta_writer.Write<idx_t>(data_pointer.tuple_count);
+		meta_writer.Write<block_id_t>(data_pointer.block_pointer.block_id);
+		meta_writer.Write<uint32_t>(data_pointer.block_pointer.offset);
+		meta_writer.Write<CompressionType>(data_pointer.compression_type);
+		data_pointer.statistics->Serialize(meta_writer);
+	}
+}
+
+MetaBlockWriter &SingleFileRowGroupWriter::GetPayloadWriter() {
+	return table_data_writer;
+}
+
+} // namespace duckdb
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/checkpoint/table_data_reader.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
 namespace duckdb {
 struct BoundCreateTableInfo;
 
@@ -193931,61 +194322,6 @@ void TableDataReader::ReadTableData() {
 }
 
 } // namespace duckdb
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/storage/checkpoint/table_data_writer.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-
-
-
-
-
-namespace duckdb {
-class CheckpointManager;
-class ColumnData;
-class ColumnSegment;
-class RowGroup;
-class BaseStatistics;
-class SegmentStatistics;
-
-//! The table data writer is responsible for writing the data of a table to the block manager
-class TableDataWriter {
-	friend class ColumnData;
-
-public:
-	TableDataWriter(DatabaseInstance &db, CheckpointManager &checkpoint_manager, TableCatalogEntry &table,
-	                MetaBlockWriter &table_data_writer, MetaBlockWriter &meta_data_writer);
-	~TableDataWriter();
-
-	void WriteTableData();
-
-	MetaBlockWriter &GetTableWriter() {
-		return table_data_writer;
-	}
-	MetaBlockWriter &GetMetaWriter() {
-		return meta_data_writer;
-	}
-
-	CheckpointManager &GetCheckpointManager() {
-		return checkpoint_manager;
-	}
-
-	CompressionType GetColumnCompressionType(idx_t i);
-
-private:
-	CheckpointManager &checkpoint_manager;
-	TableCatalogEntry &table;
-	// Writes the actual table data
-	MetaBlockWriter &table_data_writer;
-	// Writes the metadata of the table
-	MetaBlockWriter &meta_data_writer;
-};
-
-} // namespace duckdb
-
 
 
 
@@ -193995,10 +194331,7 @@ private:
 
 namespace duckdb {
 
-TableDataWriter::TableDataWriter(DatabaseInstance &, CheckpointManager &checkpoint_manager, TableCatalogEntry &table,
-                                 MetaBlockWriter &table_data_writer, MetaBlockWriter &meta_data_writer)
-    : checkpoint_manager(checkpoint_manager), table(table), table_data_writer(table_data_writer),
-      meta_data_writer(meta_data_writer) {
+TableDataWriter::TableDataWriter(TableCatalogEntry &table) : table(table) {
 }
 
 TableDataWriter::~TableDataWriter() {
@@ -194013,6 +194346,51 @@ CompressionType TableDataWriter::GetColumnCompressionType(idx_t i) {
 	return table.columns[i].CompressionType();
 }
 
+void TableDataWriter::AddRowGroup(RowGroupPointer &&row_group_pointer, unique_ptr<RowGroupWriter> &&writer) {
+	row_group_pointers.push_back(move(row_group_pointer));
+	writer.reset();
+}
+
+SingleFileTableDataWriter::SingleFileTableDataWriter(SingleFileCheckpointWriter &checkpoint_manager,
+                                                     TableCatalogEntry &table, MetaBlockWriter &table_data_writer,
+                                                     MetaBlockWriter &meta_data_writer)
+    : TableDataWriter(table), checkpoint_manager(checkpoint_manager), table_data_writer(table_data_writer),
+      meta_data_writer(meta_data_writer) {
+}
+
+unique_ptr<RowGroupWriter> SingleFileTableDataWriter::GetRowGroupWriter(RowGroup &row_group) {
+	return make_unique<SingleFileRowGroupWriter>(table, checkpoint_manager.partial_block_manager, table_data_writer);
+}
+
+void SingleFileTableDataWriter::FinalizeTable(vector<unique_ptr<BaseStatistics>> &&global_stats, DataTableInfo *info) {
+	// store the current position in the metadata writer
+	// this is where the row groups for this table start
+	auto pointer = table_data_writer.GetBlockPointer();
+
+	for (auto &stats : global_stats) {
+		stats->Serialize(table_data_writer);
+	}
+	// now start writing the row group pointers to disk
+	table_data_writer.Write<uint64_t>(row_group_pointers.size());
+	for (auto &row_group_pointer : row_group_pointers) {
+		RowGroup::Serialize(row_group_pointer, table_data_writer);
+	}
+
+	// Pointer to the table itself goes to the metadata stream.
+	meta_data_writer.Write<block_id_t>(pointer.block_id);
+	meta_data_writer.Write<uint64_t>(pointer.offset);
+
+	// Now we serialize indexes in the table_metadata_writer
+	std::vector<BlockPointer> index_pointers = info->indexes.SerializeIndexes(table_data_writer);
+
+	// Write-off to metadata block ids and offsets of indexes
+	meta_data_writer.Write<idx_t>(index_pointers.size());
+	for (auto &block_info : index_pointers) {
+		meta_data_writer.Write<idx_t>(block_info.block_id);
+		meta_data_writer.Write<idx_t>(block_info.offset);
+	}
+}
+
 } // namespace duckdb
 //===----------------------------------------------------------------------===//
 //                         DuckDB
@@ -194030,11 +194408,11 @@ namespace duckdb {
 
 class WriteOverflowStringsToDisk : public OverflowStringWriter {
 public:
-	explicit WriteOverflowStringsToDisk(DatabaseInstance &db);
+	explicit WriteOverflowStringsToDisk(BlockManager &block_manager);
 	~WriteOverflowStringsToDisk() override;
 
-	//! The checkpoint manager
-	DatabaseInstance &db;
+	//! The block manager
+	BlockManager &block_manager;
 
 	//! Temporary buffer
 	BufferHandle handle;
@@ -194060,20 +194438,18 @@ private:
 
 namespace duckdb {
 
-WriteOverflowStringsToDisk::WriteOverflowStringsToDisk(DatabaseInstance &db)
-    : db(db), block_id(INVALID_BLOCK), offset(0) {
+WriteOverflowStringsToDisk::WriteOverflowStringsToDisk(BlockManager &block_manager)
+    : block_manager(block_manager), block_id(INVALID_BLOCK), offset(0) {
 }
 
 WriteOverflowStringsToDisk::~WriteOverflowStringsToDisk() {
-	auto &block_manager = BlockManager::GetBlockManager(db);
 	if (offset > 0) {
 		block_manager.Write(handle.GetFileBuffer(), block_id);
 	}
 }
 
 void WriteOverflowStringsToDisk::WriteString(string_t string, block_id_t &result_block, int32_t &result_offset) {
-	auto &buffer_manager = BufferManager::GetBufferManager(db);
-	auto &block_manager = BlockManager::GetBlockManager(db);
+	auto &buffer_manager = block_manager.buffer_manager;
 	if (!handle.IsValid()) {
 		handle = buffer_manager.Allocate(Storage::BLOCK_SIZE);
 	}
@@ -194123,7 +194499,6 @@ void WriteOverflowStringsToDisk::WriteString(string_t string, block_id_t &result
 }
 
 void WriteOverflowStringsToDisk::AllocateNewBlock(block_id_t new_block_id) {
-	auto &block_manager = BlockManager::GetBlockManager(db);
 	if (block_id != INVALID_BLOCK) {
 		// there is an old block, write it first
 		block_manager.Write(handle.GetFileBuffer(), block_id);
@@ -194163,33 +194538,47 @@ void WriteOverflowStringsToDisk::AllocateNewBlock(block_id_t new_block_id) {
 
 
 
+
 
 
 namespace duckdb {
 
 void ReorderTableEntries(vector<TableCatalogEntry *> &tables);
 
-CheckpointManager::CheckpointManager(DatabaseInstance &db) : db(db) {
+BlockManager &SingleFileCheckpointWriter::GetBlockManager() {
+	auto &storage_manager = (SingleFileStorageManager &)db.GetStorageManager();
+	return *storage_manager.block_manager;
+}
+
+BlockPointer SingleFileCheckpointWriter::WriteIndexData(IndexCatalogEntry &index_catalog) {
+	return index_catalog.index->Serialize(*table_metadata_writer);
 }
 
-void CheckpointManager::CreateCheckpoint() {
+MetaBlockWriter &SingleFileCheckpointWriter::GetMetaBlockWriter() {
+	return *metadata_writer;
+}
+
+unique_ptr<TableDataWriter> SingleFileCheckpointWriter::GetTableDataWriter(TableCatalogEntry &table) {
+	return make_unique<SingleFileTableDataWriter>(*this, table, *table_metadata_writer, GetMetaBlockWriter());
+}
+
+void SingleFileCheckpointWriter::CreateCheckpoint() {
 	auto &config = DBConfig::GetConfig(db);
-	auto &storage_manager = StorageManager::GetStorageManager(db);
+	auto &storage_manager = (SingleFileStorageManager &)db.GetStorageManager();
 	if (storage_manager.InMemory()) {
 		return;
 	}
 	// assert that the checkpoint manager hasn't been used before
 	D_ASSERT(!metadata_writer);
 
-	auto &block_manager = BlockManager::GetBlockManager(db);
-	block_manager.StartCheckpoint();
+	auto &block_manager = GetBlockManager();
 
 	//! Set up the writers for the checkpoints
-	metadata_writer = make_unique<MetaBlockWriter>(db);
-	tabledata_writer = make_unique<MetaBlockWriter>(db);
+	metadata_writer = make_unique<MetaBlockWriter>(block_manager);
+	table_metadata_writer = make_unique<MetaBlockWriter>(block_manager);
 
 	// get the id of the first meta block
-	block_id_t meta_block = metadata_writer->block->id;
+	block_id_t meta_block = metadata_writer->GetBlockPointer().block_id;
 
 	vector<SchemaCatalogEntry *> schemas;
 	// we scan the set of committed schemas
@@ -194201,10 +194590,10 @@ void CheckpointManager::CreateCheckpoint() {
 	for (auto &schema : schemas) {
 		WriteSchema(*schema);
 	}
-	FlushPartialSegments();
+	partial_block_manager.FlushPartialBlocks();
 	// flush the meta data to disk
 	metadata_writer->Flush();
-	tabledata_writer->Flush();
+	table_metadata_writer->Flush();
 
 	// write a checkpoint flag to the WAL
 	// this protects against the rare event that the database crashes AFTER writing the file, but BEFORE truncating the
@@ -194232,39 +194621,39 @@ void CheckpointManager::CreateCheckpoint() {
 	wal->Truncate(0);
 
 	// mark all blocks written as part of the metadata as modified
-	for (auto &block_id : metadata_writer->written_blocks) {
-		block_manager.MarkBlockAsModified(block_id);
-	}
-	for (auto &block_id : tabledata_writer->written_blocks) {
-		block_manager.MarkBlockAsModified(block_id);
-	}
+	metadata_writer->MarkWrittenBlocks();
+	table_metadata_writer->MarkWrittenBlocks();
 }
 
-void CheckpointManager::LoadFromStorage() {
-	auto &block_manager = BlockManager::GetBlockManager(db);
+void SingleFileCheckpointReader::LoadFromStorage() {
+	auto &block_manager = *storage.block_manager;
 	block_id_t meta_block = block_manager.GetMetaBlock();
 	if (meta_block < 0) {
 		// storage is empty
 		return;
 	}
 
-	Connection con(db);
+	Connection con(storage.db);
 	con.BeginTransaction();
 	// create the MetaBlockReader to read from the storage
-	MetaBlockReader reader(db, meta_block);
+	MetaBlockReader reader(block_manager, meta_block);
+	LoadCheckpoint(*con.context, reader);
+	con.Commit();
+}
+
+void CheckpointReader::LoadCheckpoint(ClientContext &context, MetaBlockReader &reader) {
 	uint32_t schema_count = reader.Read<uint32_t>();
 	for (uint32_t i = 0; i < schema_count; i++) {
-		ReadSchema(*con.context, reader);
+		ReadSchema(context, reader);
 	}
-	con.Commit();
 }
 
 //===--------------------------------------------------------------------===//
 // Schema
 //===--------------------------------------------------------------------===//
-void CheckpointManager::WriteSchema(SchemaCatalogEntry &schema) {
+void CheckpointWriter::WriteSchema(SchemaCatalogEntry &schema) {
 	// write the schema data
-	schema.Serialize(*metadata_writer);
+	schema.Serialize(GetMetaBlockWriter());
 	// then, we fetch the tables/views/sequences information
 	vector<TableCatalogEntry *> tables;
 	vector<ViewCatalogEntry *> views;
@@ -194322,7 +194711,7 @@ void CheckpointManager::WriteSchema(SchemaCatalogEntry &schema) {
 		indexes.push_back((IndexCatalogEntry *)entry);
 	});
 
-	FieldWriter writer(*metadata_writer);
+	FieldWriter writer(GetMetaBlockWriter());
 	writer.WriteField<uint32_t>(custom_types.size());
 	writer.WriteField<uint32_t>(sequences.size());
 	writer.WriteField<uint32_t>(tables.size());
@@ -194367,8 +194756,8 @@ void CheckpointManager::WriteSchema(SchemaCatalogEntry &schema) {
 	}
 }
 
-void CheckpointManager::ReadSchema(ClientContext &context, MetaBlockReader &reader) {
-	auto &catalog = Catalog::GetCatalog(db);
+void CheckpointReader::ReadSchema(ClientContext &context, MetaBlockReader &reader) {
+	auto &catalog = Catalog::GetCatalog(context);
 
 	// read the schema and create it in the catalog
 	auto info = SchemaCatalogEntry::Deserialize(reader);
@@ -194421,51 +194810,52 @@ void CheckpointManager::ReadSchema(ClientContext &context, MetaBlockReader &read
 //===--------------------------------------------------------------------===//
 // Views
 //===--------------------------------------------------------------------===//
-void CheckpointManager::WriteView(ViewCatalogEntry &view) {
-	view.Serialize(*metadata_writer);
+void CheckpointWriter::WriteView(ViewCatalogEntry &view) {
+	view.Serialize(GetMetaBlockWriter());
 }
 
-void CheckpointManager::ReadView(ClientContext &context, MetaBlockReader &reader) {
+void CheckpointReader::ReadView(ClientContext &context, MetaBlockReader &reader) {
 	auto info = ViewCatalogEntry::Deserialize(reader, context);
 
-	auto &catalog = Catalog::GetCatalog(db);
+	auto &catalog = Catalog::GetCatalog(context);
 	catalog.CreateView(context, info.get());
 }
 
 //===--------------------------------------------------------------------===//
 // Sequences
 //===--------------------------------------------------------------------===//
-void CheckpointManager::WriteSequence(SequenceCatalogEntry &seq) {
-	seq.Serialize(*metadata_writer);
+void CheckpointWriter::WriteSequence(SequenceCatalogEntry &seq) {
+	seq.Serialize(GetMetaBlockWriter());
 }
 
-void CheckpointManager::ReadSequence(ClientContext &context, MetaBlockReader &reader) {
+void CheckpointReader::ReadSequence(ClientContext &context, MetaBlockReader &reader) {
 	auto info = SequenceCatalogEntry::Deserialize(reader);
 
-	auto &catalog = Catalog::GetCatalog(db);
+	auto &catalog = Catalog::GetCatalog(context);
 	catalog.CreateSequence(context, info.get());
 }
 
 //===--------------------------------------------------------------------===//
 // Indexes
 //===--------------------------------------------------------------------===//
-void CheckpointManager::WriteIndex(IndexCatalogEntry &index_catalog) {
+void CheckpointWriter::WriteIndex(IndexCatalogEntry &index_catalog) {
 	// Write the index data and metadata
 	// Serialize the necessary meta data for index catalog construction.
-	auto root_offset = index_catalog.index->Serialize(*tabledata_writer);
-	index_catalog.Serialize(*metadata_writer);
+	auto root_offset = WriteIndexData(index_catalog);
+	auto &metadata_writer = GetMetaBlockWriter();
+	index_catalog.Serialize(metadata_writer);
 	// Serialize the Block id and offset of root node
-	metadata_writer->Write(root_offset.block_id);
-	metadata_writer->Write(root_offset.offset);
+	metadata_writer.Write(root_offset.block_id);
+	metadata_writer.Write(root_offset.offset);
 }
 
-void CheckpointManager::ReadIndex(ClientContext &context, MetaBlockReader &reader) {
+void CheckpointReader::ReadIndex(ClientContext &context, MetaBlockReader &reader) {
 
 	// Deserialize the index meta data
 	auto info = IndexCatalogEntry::Deserialize(reader, context);
 
 	// Create index in the catalog
-	auto &catalog = Catalog::GetCatalog(db);
+	auto &catalog = Catalog::GetCatalog(context);
 	auto schema_catalog = catalog.GetSchema(context, info->schema);
 	auto table_catalog =
 	    (TableCatalogEntry *)catalog.GetEntry(context, CatalogType::TABLE_ENTRY, info->schema, info->table->table_name);
@@ -194506,8 +194896,9 @@ void CheckpointManager::ReadIndex(ClientContext &context, MetaBlockReader &reade
 
 	switch (info->index_type) {
 	case IndexType::ART: {
-		auto art = make_unique<ART>(info->column_ids, move(unbound_expressions), info->constraint_type, db,
-		                            root_block_id, root_offset);
+		auto art =
+		    make_unique<ART>(info->column_ids, TableIOManager::Get(*table_catalog->storage), move(unbound_expressions),
+		                     info->constraint_type, *context.db, root_block_id, root_offset);
 		index_catalog->index = art.get();
 		table_catalog->storage->info->indexes.AddIndex(move(art));
 		break;
@@ -194520,52 +194911,53 @@ void CheckpointManager::ReadIndex(ClientContext &context, MetaBlockReader &reade
 //===--------------------------------------------------------------------===//
 // Custom Types
 //===--------------------------------------------------------------------===//
-void CheckpointManager::WriteType(TypeCatalogEntry &table) {
-	table.Serialize(*metadata_writer);
+void CheckpointWriter::WriteType(TypeCatalogEntry &table) {
+	table.Serialize(GetMetaBlockWriter());
 }
 
-void CheckpointManager::ReadType(ClientContext &context, MetaBlockReader &reader) {
+void CheckpointReader::ReadType(ClientContext &context, MetaBlockReader &reader) {
 	auto info = TypeCatalogEntry::Deserialize(reader);
 
-	auto &catalog = Catalog::GetCatalog(db);
+	auto &catalog = Catalog::GetCatalog(context);
 	catalog.CreateType(context, info.get());
 }
 
 //===--------------------------------------------------------------------===//
 // Macro's
 //===--------------------------------------------------------------------===//
-void CheckpointManager::WriteMacro(ScalarMacroCatalogEntry &macro) {
-	macro.Serialize(*metadata_writer);
+void CheckpointWriter::WriteMacro(ScalarMacroCatalogEntry &macro) {
+	macro.Serialize(GetMetaBlockWriter());
 }
 
-void CheckpointManager::ReadMacro(ClientContext &context, MetaBlockReader &reader) {
+void CheckpointReader::ReadMacro(ClientContext &context, MetaBlockReader &reader) {
 	auto info = ScalarMacroCatalogEntry::Deserialize(reader, context);
-	auto &catalog = Catalog::GetCatalog(db);
+	auto &catalog = Catalog::GetCatalog(context);
 	catalog.CreateFunction(context, info.get());
 }
 
-void CheckpointManager::WriteTableMacro(TableMacroCatalogEntry &macro) {
-	macro.Serialize(*metadata_writer);
+void CheckpointWriter::WriteTableMacro(TableMacroCatalogEntry &macro) {
+	macro.Serialize(GetMetaBlockWriter());
 }
 
-void CheckpointManager::ReadTableMacro(ClientContext &context, MetaBlockReader &reader) {
+void CheckpointReader::ReadTableMacro(ClientContext &context, MetaBlockReader &reader) {
 	auto info = TableMacroCatalogEntry::Deserialize(reader, context);
-	auto &catalog = Catalog::GetCatalog(db);
+	auto &catalog = Catalog::GetCatalog(context);
 	catalog.CreateFunction(context, info.get());
 }
 
 //===--------------------------------------------------------------------===//
 // Table Metadata
 //===--------------------------------------------------------------------===//
-void CheckpointManager::WriteTable(TableCatalogEntry &table) {
+void CheckpointWriter::WriteTable(TableCatalogEntry &table) {
 	// write the table meta data
-	table.Serialize(*metadata_writer);
-	// now we need to write the table data
-	TableDataWriter writer(db, *this, table, *tabledata_writer, *metadata_writer);
-	writer.WriteTableData();
+	table.Serialize(GetMetaBlockWriter());
+	// now we need to write the table data.
+	if (auto writer = GetTableDataWriter(table)) {
+		writer->WriteTableData();
+	}
 }
 
-void CheckpointManager::ReadTable(ClientContext &context, MetaBlockReader &reader) {
+void CheckpointReader::ReadTable(ClientContext &context, MetaBlockReader &reader) {
 	// deserialize the table meta data
 	auto info = TableCatalogEntry::Deserialize(reader, context);
 	// bind the info
@@ -194573,11 +194965,22 @@ void CheckpointManager::ReadTable(ClientContext &context, MetaBlockReader &reade
 	auto bound_info = binder->BindCreateTableInfo(move(info));
 
 	// now read the actual table data and place it into the create table info
+	ReadTableData(context, reader, *bound_info);
+
+	// finally create the table in the catalog
+	auto &catalog = Catalog::GetCatalog(context);
+	catalog.CreateTable(context, bound_info.get());
+}
+
+void CheckpointReader::ReadTableData(ClientContext &context, MetaBlockReader &reader,
+                                     BoundCreateTableInfo &bound_info) {
 	auto block_id = reader.Read<block_id_t>();
 	auto offset = reader.Read<uint64_t>();
-	MetaBlockReader table_data_reader(db, block_id);
+
+	MetaBlockReader table_data_reader(reader.block_manager, block_id);
 	table_data_reader.offset = offset;
-	TableDataReader data_reader(table_data_reader, *bound_info);
+	TableDataReader data_reader(table_data_reader, bound_info);
+
 	data_reader.ReadTableData();
 
 	// Get any indexes block info
@@ -194585,82 +194988,7 @@ void CheckpointManager::ReadTable(ClientContext &context, MetaBlockReader &reade
 	for (idx_t i = 0; i < num_indexes; i++) {
 		auto idx_block_id = reader.Read<idx_t>();
 		auto idx_offset = reader.Read<idx_t>();
-		bound_info->indexes.emplace_back(idx_block_id, idx_offset);
-	}
-
-	// finally create the table in the catalog
-	auto &catalog = Catalog::GetCatalog(db);
-	catalog.CreateTable(context, bound_info.get());
-}
-
-//===--------------------------------------------------------------------===//
-// Partial Blocks
-//===--------------------------------------------------------------------===//
-bool CheckpointManager::GetPartialBlock(ColumnSegment *segment, idx_t segment_size, block_id_t &block_id,
-                                        uint32_t &offset_in_block, PartialBlock *&partial_block_ptr,
-                                        unique_ptr<PartialBlock> &owned_partial_block) {
-	auto entry = partially_filled_blocks.lower_bound(segment_size);
-	if (entry == partially_filled_blocks.end()) {
-		return false;
-	}
-	// found a partially filled block! fill in the info
-	auto partial_block = move(entry->second);
-	partial_block_ptr = partial_block.get();
-	block_id = partial_block->block_id;
-	offset_in_block = Storage::BLOCK_SIZE - entry->first;
-	partially_filled_blocks.erase(entry);
-	PartialColumnSegment partial_segment;
-	partial_segment.segment = segment;
-	partial_segment.offset_in_block = offset_in_block;
-	partial_block->segments.push_back(partial_segment);
-
-	D_ASSERT(offset_in_block > 0);
-	D_ASSERT(ValueIsAligned(offset_in_block));
-
-	// check if the block is STILL partially filled after adding the segment_size
-	auto new_size = AlignValue(offset_in_block + segment_size);
-	if (new_size <= CheckpointManager::PARTIAL_BLOCK_THRESHOLD) {
-		// the block is still partially filled: add it to the partially_filled_blocks list
-		auto new_space_left = Storage::BLOCK_SIZE - new_size;
-		partially_filled_blocks.insert(make_pair(new_space_left, move(partial_block)));
-		// should not write the block yet: perhaps more columns will be added
-	} else {
-		// we are done with this block after the current write: write it to disk
-		owned_partial_block = move(partial_block);
-	}
-	return true;
-}
-
-void CheckpointManager::RegisterPartialBlock(ColumnSegment *segment, idx_t segment_size, block_id_t block_id) {
-	D_ASSERT(segment_size <= CheckpointManager::PARTIAL_BLOCK_THRESHOLD);
-	auto partial_block = make_unique<PartialBlock>();
-	partial_block->block_id = block_id;
-	partial_block->block = segment->block;
-
-	PartialColumnSegment partial_segment;
-	partial_segment.segment = segment;
-	partial_segment.offset_in_block = 0;
-	partial_block->segments.push_back(partial_segment);
-	auto space_left = Storage::BLOCK_SIZE - AlignValue(segment_size);
-	partially_filled_blocks.insert(make_pair(space_left, move(partial_block)));
-}
-
-void CheckpointManager::FlushPartialSegments() {
-	for (auto &entry : partially_filled_blocks) {
-		entry.second->FlushToDisk(db);
-	}
-}
-
-void PartialBlock::FlushToDisk(DatabaseInstance &db) {
-	auto &block_manager = BlockManager::GetBlockManager(db);
-
-	// the data for the block might already exists in-memory of our block
-	// instead of copying the data we alter some metadata so the buffer points to an on-disk block
-	block = block_manager.ConvertToPersistent(block_id, move(block));
-
-	// now set this block as the block for all segments
-	for (auto &seg : segments) {
-		seg.segment->ConvertToPersistent(block, block_id, seg.offset_in_block);
+		bound_info.indexes.emplace_back(idx_block_id, idx_offset);
 	}
 }
 
@@ -196111,47 +196439,6 @@ private:
 
 
 
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/storage/table/column_checkpoint_state.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-
-
-
-
-
-
-
-
-
-namespace duckdb {
-class ColumnData;
-class DatabaseInstance;
-class RowGroup;
-class TableDataWriter;
-
-struct ColumnCheckpointState {
-	ColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, TableDataWriter &writer);
-	virtual ~ColumnCheckpointState();
-
-	RowGroup &row_group;
-	ColumnData &column_data;
-	TableDataWriter &writer;
-	SegmentTree new_tree;
-	vector<DataPointer> data_pointers;
-	unique_ptr<BaseStatistics> global_stats;
-
-public:
-	virtual unique_ptr<BaseStatistics> GetStatistics();
-
-	virtual void FlushSegment(unique_ptr<ColumnSegment> segment, idx_t segment_size);
-	virtual void FlushToDisk();
-};
-
-} // namespace duckdb
 
 
 
@@ -196160,6 +196447,7 @@ class ColumnData;
 class ColumnSegment;
 class DatabaseInstance;
 class RowGroup;
+class RowGroupWriter;
 class TableDataWriter;
 struct TransactionData;
 
@@ -196174,11 +196462,13 @@ class ColumnData {
 	friend class ColumnDataCheckpointer;
 
 public:
-	ColumnData(DataTableInfo &info, idx_t column_index, idx_t start_row, LogicalType type, ColumnData *parent);
+	ColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row, LogicalType type,
+	           ColumnData *parent);
 	ColumnData(ColumnData &other, idx_t start, ColumnData *parent);
-
 	virtual ~ColumnData();
 
+	//! The block manager
+	BlockManager &block_manager;
 	//! Table info for the column
 	DataTableInfo &info;
 	//! The column index of the column, either within the parent table or within the parent
@@ -196242,25 +196532,27 @@ public:
 
 	virtual void CommitDropColumn();
 
-	virtual unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group, TableDataWriter &writer);
-	virtual unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, TableDataWriter &writer,
+	virtual unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group, RowGroupWriter &writer);
+	virtual unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, RowGroupWriter &writer,
 	                                                     ColumnCheckpointInfo &checkpoint_info);
 
 	virtual void CheckpointScan(ColumnSegment *segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
 	                            Vector &scan_vector);
 
 	virtual void DeserializeColumn(Deserializer &source);
-	static shared_ptr<ColumnData> Deserialize(DataTableInfo &info, idx_t column_index, idx_t start_row,
-	                                          Deserializer &source, const LogicalType &type, ColumnData *parent);
+	static shared_ptr<ColumnData> Deserialize(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
+	                                          idx_t start_row, Deserializer &source, const LogicalType &type,
+	                                          ColumnData *parent);
 
 	virtual void GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, vector<vector<Value>> &result);
 	virtual void Verify(RowGroup &parent);
 
-	static shared_ptr<ColumnData> CreateColumn(DataTableInfo &info, idx_t column_index, idx_t start_row,
-	                                           const LogicalType &type, ColumnData *parent = nullptr);
+	static shared_ptr<ColumnData> CreateColumn(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
+	                                           idx_t start_row, const LogicalType &type, ColumnData *parent = nullptr);
 	static shared_ptr<ColumnData> CreateColumn(ColumnData &other, idx_t start_row, ColumnData *parent = nullptr);
-	static unique_ptr<ColumnData> CreateColumnUnique(DataTableInfo &info, idx_t column_index, idx_t start_row,
-	                                                 const LogicalType &type, ColumnData *parent = nullptr);
+	static unique_ptr<ColumnData> CreateColumnUnique(BlockManager &block_manager, DataTableInfo &info,
+	                                                 idx_t column_index, idx_t start_row, const LogicalType &type,
+	                                                 ColumnData *parent = nullptr);
 	static unique_ptr<ColumnData> CreateColumnUnique(ColumnData &other, idx_t start_row, ColumnData *parent = nullptr);
 
 protected:
@@ -197032,7 +197324,7 @@ public:
 		next_width = 0;
 
 		// Reset the pointers into the current segment
-		auto &buffer_manager = BufferManager::GetBufferManager(current_segment->db);
+		auto &buffer_manager = BufferManager::GetBufferManager(checkpointer.GetDatabase());
 		current_handle = buffer_manager.Pin(current_segment->block);
 		current_dictionary = DictionaryCompressionStorage::GetDictionary(*current_segment, current_handle);
 		current_end_ptr = current_handle.Ptr() + current_dictionary.end;
@@ -197116,7 +197408,7 @@ public:
 	}
 
 	idx_t Finalize() {
-		auto &buffer_manager = BufferManager::GetBufferManager(current_segment->db);
+		auto &buffer_manager = BufferManager::GetBufferManager(checkpointer.GetDatabase());
 		auto handle = buffer_manager.Pin(current_segment->block);
 		D_ASSERT(current_dictionary.end == Storage::BLOCK_SIZE);
 
@@ -197559,7 +197851,7 @@ void UncompressedCompressState::CreateEmptySegment(idx_t row_start) {
 	auto compressed_segment = ColumnSegment::CreateTransientSegment(db, type, row_start);
 	if (type.InternalType() == PhysicalType::VARCHAR) {
 		auto &state = (UncompressedStringSegmentState &)*compressed_segment->GetSegmentState();
-		state.overflow_writer = make_unique<WriteOverflowStringsToDisk>(db);
+		state.overflow_writer = make_unique<WriteOverflowStringsToDisk>(checkpointer.GetColumnData().block_manager);
 	}
 	current_segment = move(compressed_segment);
 }
@@ -199357,8 +199649,8 @@ string_t UncompressedStringStorage::ReadOverflowString(ColumnSegment &segment, V
 	D_ASSERT(block != INVALID_BLOCK);
 	D_ASSERT(offset < Storage::BLOCK_SIZE);
 
-	auto &block_manager = BlockManager::GetBlockManager(segment.db);
-	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
+	auto &block_manager = segment.GetBlockManager();
+	auto &buffer_manager = block_manager.buffer_manager;
 	auto &state = (UncompressedStringSegmentState &)*segment.GetSegmentState();
 	if (block < MAXIMUM_BLOCK) {
 		// read the overflow string from disk
@@ -200042,7 +200334,8 @@ namespace duckdb {
 //! Validity column data represents the validity data (i.e. which values are null)
 class ValidityColumnData : public ColumnData {
 public:
-	ValidityColumnData(DataTableInfo &info, idx_t column_index, idx_t start_row, ColumnData *parent);
+	ValidityColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
+	                   ColumnData *parent);
 	ValidityColumnData(ColumnData &original, idx_t start_row, ColumnData *parent = nullptr);
 
 public:
@@ -200057,8 +200350,8 @@ namespace duckdb {
 //! Standard column data represents a regular flat column (e.g. a column of type INTEGER or STRING)
 class StandardColumnData : public ColumnData {
 public:
-	StandardColumnData(DataTableInfo &info, idx_t column_index, idx_t start_row, LogicalType type,
-	                   ColumnData *parent = nullptr);
+	StandardColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
+	                   LogicalType type, ColumnData *parent = nullptr);
 	StandardColumnData(ColumnData &original, idx_t start_row, ColumnData *parent = nullptr);
 
 	//! The validity column data
@@ -200088,8 +200381,8 @@ public:
 
 	void CommitDropColumn() override;
 
-	unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group, TableDataWriter &writer) override;
-	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, TableDataWriter &writer,
+	unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group, RowGroupWriter &writer) override;
+	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, RowGroupWriter &writer,
 	                                             ColumnCheckpointInfo &checkpoint_info) override;
 	void CheckpointScan(ColumnSegment *segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
 	                    Vector &scan_vector) override;
@@ -200112,13 +200405,15 @@ private:
 
 namespace duckdb {
 
-DataTable::DataTable(DatabaseInstance &db, const string &schema, const string &table,
-                     vector<ColumnDefinition> column_definitions_p, unique_ptr<PersistentTableData> data)
-    : info(make_shared<DataTableInfo>(db, schema, table)), column_definitions(move(column_definitions_p)), db(db),
-      is_root(true) {
+DataTable::DataTable(DatabaseInstance &db, shared_ptr<TableIOManager> table_io_manager_p, const string &schema,
+                     const string &table, vector<ColumnDefinition> column_definitions_p,
+                     unique_ptr<PersistentTableData> data)
+    : info(make_shared<DataTableInfo>(db, move(table_io_manager_p), schema, table)),
+      column_definitions(move(column_definitions_p)), db(db), is_root(true) {
 	// initialize the table with the existing data from disk, if any
 	auto types = GetTypes();
-	this->row_groups = make_shared<RowGroupCollection>(info, types, 0);
+	this->row_groups =
+	    make_shared<RowGroupCollection>(info, TableIOManager::Get(*this).GetBlockManagerForRowData(), types, 0);
 	if (data && !data->row_groups.empty()) {
 		this->row_groups->Initialize(*data);
 		stats.Initialize(types, *data);
@@ -200268,6 +200563,10 @@ vector<LogicalType> DataTable::GetTypes() {
 	return types;
 }
 
+TableIOManager &TableIOManager::Get(DataTable &table) {
+	return *table.info->table_io_manager;
+}
+
 //===--------------------------------------------------------------------===//
 // Scan
 //===--------------------------------------------------------------------===//
@@ -201006,38 +201305,14 @@ void DataTable::Checkpoint(TableDataWriter &writer) {
 		global_stats.push_back(stats.CopyStats(i));
 	}
 
-	vector<RowGroupPointer> row_group_pointers;
-	row_groups->Checkpoint(writer, row_group_pointers, global_stats);
-
-	// store the current position in the metadata writer
-	// this is where the row groups for this table start
-	auto &data_writer = writer.GetTableWriter();
-	auto pointer = data_writer.GetBlockPointer();
-
-	for (auto &stats : global_stats) {
-		stats->Serialize(data_writer);
-	}
-	// now start writing the row group pointers to disk
-	data_writer.Write<uint64_t>(row_group_pointers.size());
-	for (auto &row_group_pointer : row_group_pointers) {
-		RowGroup::Serialize(row_group_pointer, data_writer);
-	}
-	// Now we serialize indexes in the tabledata_writer
-	auto blocks_info = info->indexes.SerializeIndexes(data_writer);
-
-	// metadata writing time
-	auto &metadata_writer = writer.GetMetaWriter();
+	row_groups->Checkpoint(writer, global_stats);
 
-	// write the block pointer for the table info
-	metadata_writer.Write<block_id_t>(pointer.block_id);
-	metadata_writer.Write<uint64_t>(pointer.offset);
-
-	// Write-off block ids and offsets of indexes
-	metadata_writer.Write<idx_t>(blocks_info.size());
-	for (auto &block_info : blocks_info) {
-		metadata_writer.Write<idx_t>(block_info.block_id);
-		metadata_writer.Write<idx_t>(block_info.offset);
-	}
+	// The rowgroup payload data has been written. Now write:
+	//   column stats
+	//   row-group pointers
+	//   table pointer
+	//   index data
+	writer.FinalizeTable(move(global_stats), info.get());
 }
 
 void DataTable::CommitDropColumn(idx_t index) {
@@ -201071,9 +201346,9 @@ vector<vector<Value>> DataTable::GetStorageInfo() {
 
 namespace duckdb {
 
-Index::Index(IndexType type, const vector<column_t> &column_ids_p,
+Index::Index(IndexType type, TableIOManager &table_io_manager, const vector<column_t> &column_ids_p,
              const vector<unique_ptr<Expression>> &unbound_expressions, IndexConstraintType constraint_type_p)
-    : type(type), column_ids(column_ids_p), constraint_type(constraint_type_p),
+    : type(type), table_io_manager(table_io_manager), column_ids(column_ids_p), constraint_type(constraint_type_p),
       executor(Allocator::DefaultAllocator()) {
 	for (auto &expr : unbound_expressions) {
 		types.push_back(expr->return_type.InternalType());
@@ -201160,12 +201435,15 @@ BlockPointer Index::Serialize(duckdb::MetaBlockWriter &writer) {
 
 
 
+
 namespace duckdb {
 
 LocalTableStorage::LocalTableStorage(DataTable &table)
     : table(table), allocator(Allocator::Get(table.db)), deleted_rows(0) {
 	auto types = table.GetTypes();
-	row_groups = make_shared<RowGroupCollection>(table.info, types, MAX_ROW_ID, 0);
+	row_groups = make_shared<RowGroupCollection>(table.info, TableIOManager::Get(table).GetBlockManagerForRowData(),
+	                                             types, MAX_ROW_ID, 0);
+
 	stats.InitializeEmpty(types);
 	table.info->indexes.Scan([&](Index &index) {
 		D_ASSERT(index.type == IndexType::ART);
@@ -201176,7 +201454,8 @@ LocalTableStorage::LocalTableStorage(DataTable &table)
 			for (auto &expr : art.unbound_expressions) {
 				unbound_expressions.push_back(expr->Copy());
 			}
-			indexes.AddIndex(make_unique<ART>(art.column_ids, move(unbound_expressions), art.constraint_type, art.db));
+			indexes.AddIndex(make_unique<ART>(art.column_ids, art.table_io_manager, move(unbound_expressions),
+			                                  art.constraint_type, art.db));
 		}
 		return false;
 	});
@@ -201518,7 +201797,8 @@ void LocalStorage::VerifyNewConstraint(DataTable &parent, const BoundConstraint
 
 namespace duckdb {
 
-MetaBlockReader::MetaBlockReader(DatabaseInstance &db, block_id_t block_id) : db(db), offset(0), next_block(-1) {
+MetaBlockReader::MetaBlockReader(BlockManager &block_manager, block_id_t block_id, bool free_blocks_on_read)
+    : block_manager(block_manager), offset(0), next_block(-1), free_blocks_on_read(free_blocks_on_read) {
 	ReadNewBlock(block_id);
 }
 
@@ -201536,6 +201816,9 @@ void MetaBlockReader::ReadData(data_ptr_t buffer, idx_t read_size) {
 			buffer += to_read;
 		}
 		// then move to the next block
+		if (next_block == INVALID_BLOCK) {
+			throw IOException("Cannot read from INVALID_BLOCK.");
+		}
 		ReadNewBlock(next_block);
 	}
 	// we have enough left in this block to read from the buffer
@@ -201544,10 +201827,15 @@ void MetaBlockReader::ReadData(data_ptr_t buffer, idx_t read_size) {
 }
 
 void MetaBlockReader::ReadNewBlock(block_id_t id) {
-	auto &block_manager = BlockManager::GetBlockManager(db);
-	auto &buffer_manager = BufferManager::GetBufferManager(db);
+	auto &buffer_manager = block_manager.buffer_manager;
 
-	block_manager.MarkBlockAsModified(id);
+	// Marking these blocks as modified will cause them to be moved to the free
+	// list upon the next successful checkpoint. Marking them modified here
+	// assumes MetaBlockReader is exclusively used for reading checkpoint data,
+	// and thus any blocks we're reading will be obviated by the next checkpoint.
+	if (free_blocks_on_read) {
+		block_manager.MarkBlockAsModified(id);
+	}
 	block = block_manager.RegisterBlock(id);
 	handle = buffer_manager.Pin(block);
 
@@ -201563,28 +201851,26 @@ void MetaBlockReader::ReadNewBlock(block_id_t id) {
 
 namespace duckdb {
 
-MetaBlockWriter::MetaBlockWriter(DatabaseInstance &db, block_id_t initial_block_id) : db(db) {
+MetaBlockWriter::MetaBlockWriter(BlockManager &block_manager, block_id_t initial_block_id)
+    : block_manager(block_manager) {
 	if (initial_block_id == INVALID_BLOCK) {
 		initial_block_id = GetNextBlockId();
 	}
-	auto &block_manager = BlockManager::GetBlockManager(db);
-	block = block_manager.CreateBlock(initial_block_id);
+	block = block_manager.CreateBlock(initial_block_id, nullptr);
 	Store<block_id_t>(-1, block->buffer);
 	offset = sizeof(block_id_t);
 }
 
 MetaBlockWriter::~MetaBlockWriter() {
-	if (Exception::UncaughtException()) {
-		return;
-	}
-	try {
-		Flush();
-	} catch (...) {
-	}
+	// If there's an exception during checkpoint, this can get destroyed without
+	// flushing the data...which is fine, because none of the unwritten data
+	// will be referenced.
+	//
+	// Otherwise, we should have explicitly flushed (and thereby nulled the block).
+	D_ASSERT(!block || Exception::UncaughtException());
 }
 
 block_id_t MetaBlockWriter::GetNextBlockId() {
-	auto &block_manager = BlockManager::GetBlockManager(db);
 	return block_manager.GetFreeBlockId();
 }
 
@@ -201596,9 +201882,13 @@ BlockPointer MetaBlockWriter::GetBlockPointer() {
 }
 
 void MetaBlockWriter::Flush() {
+	AdvanceBlock();
+	block = nullptr;
+}
+
+void MetaBlockWriter::AdvanceBlock() {
 	written_blocks.insert(block->id);
 	if (offset > sizeof(block_id_t)) {
-		auto &block_manager = BlockManager::GetBlockManager(db);
 		block_manager.Write(*block);
 		offset = sizeof(block_id_t);
 	}
@@ -201621,8 +201911,8 @@ void MetaBlockWriter::WriteData(const_data_ptr_t buffer, idx_t write_size) {
 		// write the block id of the new block to the start of the current block
 		Store<block_id_t>(new_block_id, block->buffer);
 		// first flush the old block
-		Flush();
-		// now update the block id of the lbock
+		AdvanceBlock();
+		// now update the block id of the block
 		block->id = new_block_id;
 		Store<block_id_t>(-1, block->buffer);
 	}
@@ -201630,6 +201920,87 @@ void MetaBlockWriter::WriteData(const_data_ptr_t buffer, idx_t write_size) {
 	offset += write_size;
 }
 
+} // namespace duckdb
+
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Partial Blocks
+//===--------------------------------------------------------------------===//
+PartialBlockAllocation PartialBlockManager::GetBlockAllocation(uint32_t segment_size) {
+	PartialBlockAllocation allocation;
+	allocation.block_manager = &block_manager;
+	allocation.allocation_size = segment_size;
+
+	// if the block is less than 80% full, we consider it a "partial block"
+	// which means we will try to fit it with other blocks
+	// check if there is a partial block available we can write to
+	if (segment_size <= max_partial_block_size && GetPartialBlock(segment_size, allocation.partial_block)) {
+		//! there is! increase the reference count of this block
+		allocation.partial_block->state.block_use_count += 1;
+		allocation.state = allocation.partial_block->state;
+		block_manager.IncreaseBlockReferenceCount(allocation.state.block_id);
+	} else {
+		// full block: get a free block to write to
+		AllocateBlock(allocation.state, segment_size);
+	}
+	return allocation;
+}
+
+void PartialBlockManager::AllocateBlock(PartialBlockState &state, uint32_t segment_size) {
+	D_ASSERT(segment_size <= Storage::BLOCK_SIZE);
+	state.block_id = block_manager.GetFreeBlockId();
+	state.block_size = Storage::BLOCK_SIZE;
+	state.offset_in_block = 0;
+	state.block_use_count = 1;
+}
+
+bool PartialBlockManager::GetPartialBlock(idx_t segment_size, unique_ptr<PartialBlock> &partial_block) {
+	auto entry = partially_filled_blocks.lower_bound(segment_size);
+	if (entry == partially_filled_blocks.end()) {
+		return false;
+	}
+	// found a partially filled block! fill in the info
+	partial_block = move(entry->second);
+	partially_filled_blocks.erase(entry);
+
+	D_ASSERT(partial_block->state.offset_in_block > 0);
+	D_ASSERT(ValueIsAligned(partial_block->state.offset_in_block));
+	return true;
+}
+
+void PartialBlockManager::RegisterPartialBlock(PartialBlockAllocation &&allocation) {
+	auto &state(allocation.partial_block->state);
+	if (state.block_use_count < max_use_count) {
+		auto new_size = AlignValue(allocation.allocation_size + state.offset_in_block);
+		state.offset_in_block = new_size;
+		auto new_space_left = state.block_size - new_size;
+		// check if the block is STILL partially filled after adding the segment_size
+		if (new_space_left >= Storage::BLOCK_SIZE - max_partial_block_size) {
+			// the block is still partially filled: add it to the partially_filled_blocks list
+			partially_filled_blocks.insert(make_pair(new_space_left, move(allocation.partial_block)));
+		}
+	}
+	auto block_to_free = move(allocation.partial_block);
+	if (!block_to_free && partially_filled_blocks.size() > MAX_BLOCK_MAP_SIZE) {
+		// Free the page with the least space free.
+		auto itr = partially_filled_blocks.begin();
+		block_to_free = move(itr->second);
+		partially_filled_blocks.erase(itr);
+	}
+	// Flush any block that we're not going to reuse.
+	if (block_to_free) {
+		block_to_free->Flush();
+	}
+}
+
+void PartialBlockManager::FlushPartialBlocks() {
+	for (auto &e : partially_filled_blocks) {
+		e.second->Flush();
+	}
+}
+
 } // namespace duckdb
 //===----------------------------------------------------------------------===//
 //                         DuckDB
@@ -201660,9 +202031,8 @@ class SingleFileBlockManager : public BlockManager {
 public:
 	SingleFileBlockManager(DatabaseInstance &db, string path, bool read_only, bool create_new, bool use_direct_io);
 
-	void StartCheckpoint() override;
 	//! Creates a new Block using the specified block_id and returns a pointer
-	unique_ptr<Block> CreateBlock(block_id_t block_id) override;
+	unique_ptr<Block> CreateBlock(block_id_t block_id, FileBuffer *source_buffer) override;
 	//! Return the next free block id
 	block_id_t GetFreeBlockId() override;
 	//! Returns whether or not a specified block is the root block
@@ -201688,10 +202058,11 @@ public:
 	idx_t FreeBlocks() override {
 		return free_list.size();
 	}
+
+private:
 	//! Load the free list from the file
 	void LoadFreeList();
 
-private:
 	void Initialize(DatabaseHeader &header);
 
 	//! Return the blocks to which we will write the free list and modified blocks
@@ -201911,6 +202282,7 @@ SingleFileBlockManager::SingleFileBlockManager(DatabaseInstance &db, string path
 			active_header = 1;
 			Initialize(h2);
 		}
+		LoadFreeList();
 	}
 }
 
@@ -201930,7 +202302,7 @@ void SingleFileBlockManager::LoadFreeList() {
 		// no free list
 		return;
 	}
-	MetaBlockReader reader(db, free_list_id);
+	MetaBlockReader reader(*this, free_list_id);
 	auto free_list_count = reader.Read<uint64_t>();
 	free_list.clear();
 	for (idx_t i = 0; i < free_list_count; i++) {
@@ -201945,9 +202317,6 @@ void SingleFileBlockManager::LoadFreeList() {
 	}
 }
 
-void SingleFileBlockManager::StartCheckpoint() {
-}
-
 bool SingleFileBlockManager::IsRootBlock(block_id_t root) {
 	return root == meta_block;
 }
@@ -201981,6 +202350,10 @@ void SingleFileBlockManager::MarkBlockAsModified(block_id_t block_id) {
 		}
 		return;
 	}
+	// Check for multi-free
+	// TODO: Fix the bug that causes this assert to fire, then uncomment it.
+	// D_ASSERT(modified_blocks.find(block_id) == modified_blocks.end());
+	D_ASSERT(free_list.find(block_id) == free_list.end());
 	modified_blocks.insert(block_id);
 }
 
@@ -201998,8 +202371,13 @@ block_id_t SingleFileBlockManager::GetMetaBlock() {
 	return meta_block;
 }
 
-unique_ptr<Block> SingleFileBlockManager::CreateBlock(block_id_t block_id) {
-	return make_unique<Block>(Allocator::Get(db), block_id);
+unique_ptr<Block> SingleFileBlockManager::CreateBlock(block_id_t block_id, FileBuffer *source_buffer) {
+	if (source_buffer) {
+		D_ASSERT(source_buffer->AllocSize() == Storage::BLOCK_ALLOC_SIZE);
+		return make_unique<Block>(*source_buffer, block_id);
+	} else {
+		return make_unique<Block>(Allocator::Get(db), block_id);
+	}
 }
 
 void SingleFileBlockManager::Read(Block &block) {
@@ -202047,8 +202425,8 @@ vector<block_id_t> SingleFileBlockManager::GetFreeListBlocks() {
 
 class FreeListBlockWriter : public MetaBlockWriter {
 public:
-	FreeListBlockWriter(DatabaseInstance &db_p, vector<block_id_t> &free_list_blocks_p)
-	    : MetaBlockWriter(db_p, free_list_blocks_p[0]), free_list_blocks(free_list_blocks_p), index(1) {
+	FreeListBlockWriter(BlockManager &block_manager, vector<block_id_t> &free_list_blocks_p)
+	    : MetaBlockWriter(block_manager, free_list_blocks_p[0]), free_list_blocks(free_list_blocks_p), index(1) {
 	}
 
 	vector<block_id_t> &free_list_blocks;
@@ -202083,10 +202461,11 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 		// a normal MetaBlockWriter will fetch blocks to use from the free_list
 		// but since we are WRITING the free_list, this behavior is sub-optimal
 
-		FreeListBlockWriter writer(db, free_list_blocks);
+		FreeListBlockWriter writer(*this, free_list_blocks);
 
-		D_ASSERT(writer.block->id == free_list_blocks[0]);
-		header.free_list = writer.block->id;
+		auto ptr = writer.GetBlockPointer();
+		D_ASSERT(ptr.block_id == free_list_blocks[0]);
+		header.free_list = ptr.block_id;
 		for (auto &block_id : free_list_blocks) {
 			modified_blocks.insert(block_id);
 		}
@@ -203268,7 +203647,7 @@ void StorageLock::ReleaseSharedLock() {
 namespace duckdb {
 
 StorageManager::StorageManager(DatabaseInstance &db, string path, bool read_only)
-    : db(db), path(move(path)), wal(db), read_only(read_only) {
+    : db(db), path(move(path)), read_only(read_only) {
 }
 
 StorageManager::~StorageManager() {
@@ -203294,14 +203673,20 @@ bool StorageManager::InMemory() {
 	return path.empty() || path == ":memory:";
 }
 
+void StorageManager::CreateBufferManager() {
+	auto &config = DBConfig::GetConfig(db);
+	buffer_manager = make_unique<BufferManager>(db, config.options.temporary_directory, config.options.maximum_memory);
+}
+
 void StorageManager::Initialize() {
 	bool in_memory = InMemory();
 	if (in_memory && read_only) {
 		throw CatalogException("Cannot launch in-memory database in read-only mode!");
 	}
+	CreateBufferManager();
+
 	auto &config = DBConfig::GetConfig(db);
 	auto &catalog = Catalog::GetCatalog(db);
-	buffer_manager = make_unique<BufferManager>(db, config.options.temporary_directory, config.options.maximum_memory);
 
 	// first initialize the base system catalogs
 	// these are never written to the WAL
@@ -203323,15 +203708,38 @@ void StorageManager::Initialize() {
 	// commit transactions
 	con.Commit();
 
-	if (!in_memory) {
-		// create or load the database from disk, if not in-memory mode
-		LoadDatabase();
-	} else {
-		block_manager = make_unique<InMemoryBlockManager>(*buffer_manager);
+	// create or load the database from disk, if not in-memory mode
+	LoadDatabase();
+}
+
+///////////////////////////////////////////////////////////////////////////
+class SingleFileTableIOManager : public TableIOManager {
+public:
+	explicit SingleFileTableIOManager(BlockManager &block_manager) : block_manager(block_manager) {
 	}
+
+	BlockManager &block_manager;
+
+public:
+	BlockManager &GetIndexBlockManager() override {
+		return block_manager;
+	}
+	BlockManager &GetBlockManagerForRowData() override {
+		return block_manager;
+	}
+};
+
+SingleFileStorageManager::SingleFileStorageManager(DatabaseInstance &db, string path, bool read_only)
+    : StorageManager(db, move(path), read_only) {
 }
 
-void StorageManager::LoadDatabase() {
+void SingleFileStorageManager::LoadDatabase() {
+	if (InMemory()) {
+		block_manager = make_unique<InMemoryBlockManager>(*buffer_manager);
+		table_io_manager = make_unique<SingleFileTableIOManager>(*block_manager);
+		return;
+	}
+
 	string wal_path = path + ".wal";
 	auto &fs = db.GetFileSystem();
 	auto &config = db.config;
@@ -203349,15 +203757,14 @@ void StorageManager::LoadDatabase() {
 		}
 		// initialize the block manager while creating a new db file
 		block_manager = make_unique<SingleFileBlockManager>(db, path, read_only, true, config.options.use_direct_io);
+		table_io_manager = make_unique<SingleFileTableIOManager>(*block_manager);
 	} else {
 		// initialize the block manager while loading the current db file
-		auto sf_bm = make_unique<SingleFileBlockManager>(db, path, read_only, false, config.options.use_direct_io);
-		auto sf = sf_bm.get();
-		block_manager = move(sf_bm);
-		sf->LoadFreeList();
+		block_manager = make_unique<SingleFileBlockManager>(db, path, read_only, false, config.options.use_direct_io);
+		table_io_manager = make_unique<SingleFileTableIOManager>(*block_manager);
 
 		//! Load from storage
-		CheckpointManager checkpointer(db);
+		auto checkpointer = SingleFileCheckpointReader(*this);
 		checkpointer.LoadFromStorage();
 		// check if the WAL file exists
 		if (fs.FileExists(wal_path)) {
@@ -203367,27 +203774,131 @@ void StorageManager::LoadDatabase() {
 	}
 	// initialize the WAL file
 	if (!read_only) {
-		wal.Initialize(wal_path);
+		wal = make_unique<WriteAheadLog>(db, wal_path);
 		if (truncate_wal) {
-			wal.Truncate(0);
+			wal->Truncate(0);
 		}
 	}
 }
 
-void StorageManager::CreateCheckpoint(bool delete_wal, bool force_checkpoint) {
-	if (InMemory() || read_only || !wal.initialized) {
+///////////////////////////////////////////////////////////////////////////////
+
+class SingleFileStorageCommitState : public StorageCommitState {
+	idx_t initial_wal_size = 0;
+	idx_t initial_written = 0;
+	WriteAheadLog *log;
+	bool checkpoint;
+
+public:
+	SingleFileStorageCommitState(StorageManager &storage_manager, bool checkpoint);
+	~SingleFileStorageCommitState() override;
+
+	// Make the commit persistent
+	void FlushCommit() override;
+};
+
+SingleFileStorageCommitState::SingleFileStorageCommitState(StorageManager &storage_manager, bool checkpoint)
+    : checkpoint(checkpoint) {
+	log = storage_manager.GetWriteAheadLog();
+	if (log) {
+		auto initial_size = log->GetWALSize();
+		initial_written = log->GetTotalWritten();
+		initial_wal_size = initial_size < 0 ? 0 : idx_t(initial_size);
+
+		if (checkpoint) {
+			// check if we are checkpointing after this commit
+			// if we are checkpointing, we don't need to write anything to the WAL
+			// this saves us a lot of unnecessary writes to disk in the case of large commits
+			log->skip_writing = true;
+		}
+	} else {
+		D_ASSERT(!checkpoint);
+	}
+}
+
+// Make the commit persistent
+void SingleFileStorageCommitState::FlushCommit() {
+	if (log) {
+		// flush the WAL if any changes were made
+		if (log->GetTotalWritten() > initial_written) {
+			D_ASSERT(!checkpoint);
+			D_ASSERT(!log->skip_writing);
+			log->Flush();
+		}
+		log->skip_writing = false;
+	}
+	// Null so that the destructor will not truncate the log.
+	log = nullptr;
+}
+
+SingleFileStorageCommitState::~SingleFileStorageCommitState() {
+	// If log is non-null, then commit threw an exception before flushing.
+	if (log) {
+		log->skip_writing = false;
+		if (log->GetTotalWritten() > initial_written) {
+			// remove any entries written into the WAL by truncating it
+			log->Truncate(initial_wal_size);
+		}
+	}
+}
+
+unique_ptr<StorageCommitState> SingleFileStorageManager::GenStorageCommitState(Transaction &transaction,
+                                                                               bool checkpoint) {
+	return make_unique<SingleFileStorageCommitState>(*this, checkpoint);
+}
+
+bool SingleFileStorageManager::IsCheckpointClean(block_id_t checkpoint_id) {
+	return block_manager->IsRootBlock(checkpoint_id);
+}
+
+void SingleFileStorageManager::CreateCheckpoint(bool delete_wal, bool force_checkpoint) {
+	if (InMemory() || read_only || !wal) {
 		return;
 	}
-	if (wal.GetWALSize() > 0 || db.config.options.force_checkpoint || force_checkpoint) {
+	if (wal->GetWALSize() > 0 || db.config.options.force_checkpoint || force_checkpoint) {
 		// we only need to checkpoint if there is anything in the WAL
-		CheckpointManager checkpointer(db);
+		SingleFileCheckpointWriter checkpointer(db, *block_manager);
 		checkpointer.CreateCheckpoint();
 	}
 	if (delete_wal) {
-		wal.Delete();
+		wal->Delete();
+		wal.reset();
 	}
 }
 
+DatabaseSize SingleFileStorageManager::GetDatabaseSize() {
+	// All members default to zero
+	DatabaseSize ds;
+	if (!InMemory()) {
+		ds.total_blocks = block_manager->TotalBlocks();
+		ds.block_size = Storage::BLOCK_ALLOC_SIZE;
+		ds.free_blocks = block_manager->FreeBlocks();
+		ds.used_blocks = ds.total_blocks - ds.free_blocks;
+		ds.bytes = (ds.total_blocks * ds.block_size);
+		if (auto wal = GetWriteAheadLog()) {
+			ds.wal_size = wal->GetWALSize();
+		}
+	}
+	return ds;
+}
+
+bool SingleFileStorageManager::AutomaticCheckpoint(idx_t estimated_wal_bytes) {
+	auto log = GetWriteAheadLog();
+	if (!log) {
+		return false;
+	}
+
+	auto initial_size = log->GetWALSize();
+	idx_t expected_wal_size = initial_size + estimated_wal_bytes;
+	return expected_wal_size > db.config.options.checkpoint_wal_size;
+}
+
+shared_ptr<TableIOManager> SingleFileStorageManager::GetTableIOManager(BoundCreateTableInfo *info /*info*/) {
+	// This is an unmanaged reference. No ref/deref overhead. Lifetime of the
+	// TableIoManager follows lifetime of the StorageManager (this).
+	return shared_ptr<TableIOManager>(shared_ptr<char>(nullptr), table_io_manager.get());
+}
+
 } // namespace duckdb
 
 
@@ -203679,8 +204190,8 @@ namespace duckdb {
 //! List column data represents a list
 class ListColumnData : public ColumnData {
 public:
-	ListColumnData(DataTableInfo &info, idx_t column_index, idx_t start_row, LogicalType type,
-	               ColumnData *parent = nullptr);
+	ListColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
+	               LogicalType type, ColumnData *parent = nullptr);
 	ListColumnData(ColumnData &original, idx_t start_row, ColumnData *parent = nullptr);
 
 	//! The child-column of the list
@@ -203714,8 +204225,8 @@ public:
 
 	void CommitDropColumn() override;
 
-	unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group, TableDataWriter &writer) override;
-	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, TableDataWriter &writer,
+	unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group, RowGroupWriter &writer) override;
+	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, RowGroupWriter &writer,
 	                                             ColumnCheckpointInfo &checkpoint_info) override;
 
 	void DeserializeColumn(Deserializer &source) override;
@@ -203736,7 +204247,7 @@ private:
 
 namespace duckdb {
 
-ColumnCheckpointState::ColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, TableDataWriter &writer)
+ColumnCheckpointState::ColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, RowGroupWriter &writer)
     : row_group(row_group), column_data(column_data), writer(writer) {
 }
 
@@ -203748,6 +204259,56 @@ unique_ptr<BaseStatistics> ColumnCheckpointState::GetStatistics() {
 	return move(global_stats);
 }
 
+struct PartialBlockForCheckpoint : PartialBlock {
+	struct PartialColumnSegment {
+		ColumnSegment *segment;
+		uint32_t offset_in_block;
+	};
+
+public:
+	PartialBlockForCheckpoint(ColumnSegment *first_segment, BlockManager &block_manager, PartialBlockState state)
+	    : PartialBlock(state), first_segment(first_segment), block_manager(block_manager) {
+	}
+
+	~PartialBlockForCheckpoint() override {
+		D_ASSERT(IsFlushed() || Exception::UncaughtException());
+	}
+
+	// We will copy all subsequent segment data into the memory corresponding
+	// to the first segment. Once the block is full (or checkpoint is complete)
+	// we'll invoke Flush(), which will cause
+	// the block to get written to storage (via BlockManger::ConvertToPersistent),
+	// and all segments to have their references updated
+	// (via ColumnSegment::ConvertToPersistent)
+	ColumnSegment *first_segment;
+	BlockManager &block_manager;
+	vector<PartialColumnSegment> tail_segments;
+
+public:
+	bool IsFlushed() {
+		// first_segment is zeroed on Flush
+		return !first_segment;
+	}
+
+	void Flush() override {
+		// At this point, we've already copied all data from tail_segments
+		// into the page owned by first_segment. We flush all segment data to
+		// disk with the following call.
+		first_segment->ConvertToPersistent(&block_manager, state.block_id);
+		// Now that the page is persistent, update tail_segments to point to the
+		// newly persistent block.
+		for (auto e : tail_segments) {
+			e.segment->MarkAsPersistent(first_segment->block, e.offset_in_block);
+		}
+		first_segment = nullptr;
+		tail_segments.clear();
+	}
+
+	void AddSegmentToTail(ColumnSegment *segment, uint32_t offset_in_block) {
+		tail_segments.push_back({segment, offset_in_block});
+	}
+};
+
 void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_t segment_size) {
 	D_ASSERT(segment_size <= Storage::BLOCK_SIZE);
 	auto tuple_count = segment->count.load();
@@ -203761,46 +204322,41 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 	// get the buffer of the segment and pin it
 	auto &db = column_data.GetDatabase();
 	auto &buffer_manager = BufferManager::GetBufferManager(db);
-	auto &block_manager = BlockManager::GetBlockManager(db);
-	auto &checkpoint_manager = writer.GetCheckpointManager();
-
-	bool block_is_constant = segment->stats.statistics->IsConstant();
-
 	block_id_t block_id = INVALID_BLOCK;
 	uint32_t offset_in_block = 0;
-	bool need_to_write = true;
-	PartialBlock *partial_block = nullptr;
-	unique_ptr<PartialBlock> owned_partial_block;
-	if (!block_is_constant) {
+
+	if (!segment->stats.statistics->IsConstant()) {
 		// non-constant block
-		// if the block is less than 80% full, we consider it a "partial block"
-		// which means we will try to fit it with other blocks
-		if (segment_size <= CheckpointManager::PARTIAL_BLOCK_THRESHOLD) {
-			// the block is a partial block
-			// check if there is a partial block available we can write to
-			if (checkpoint_manager.GetPartialBlock(segment.get(), segment_size, block_id, offset_in_block,
-			                                       partial_block, owned_partial_block)) {
-				//! there is! increase the reference count of this block
-				block_manager.IncreaseBlockReferenceCount(block_id);
-			} else {
-				// there isn't: generate a new block for this segment
-				block_id = block_manager.GetFreeBlockId();
-				offset_in_block = 0;
-				need_to_write = false;
-				// now register this block as a partial block
-				checkpoint_manager.RegisterPartialBlock(segment.get(), segment_size, block_id);
-			}
+		PartialBlockAllocation allocation = writer.GetBlockAllocation(segment_size);
+		block_id = allocation.state.block_id;
+		offset_in_block = allocation.state.offset_in_block;
+
+		if (allocation.partial_block) {
+			// Use an existing block.
+			D_ASSERT(offset_in_block > 0);
+			auto pstate = (PartialBlockForCheckpoint *)allocation.partial_block.get();
+			// pin the source block
+			auto old_handle = buffer_manager.Pin(segment->block);
+			// pin the target block
+			auto new_handle = buffer_manager.Pin(pstate->first_segment->block);
+			// memcpy the contents of the old block to the new block
+			memcpy(new_handle.Ptr() + offset_in_block, old_handle.Ptr(), segment_size);
+			pstate->AddSegmentToTail(segment.get(), offset_in_block);
 		} else {
-			// full block: get a free block to write to
-			block_id = block_manager.GetFreeBlockId();
-			offset_in_block = 0;
+			// Create a new block for future reuse.
+			D_ASSERT(offset_in_block == 0);
+			allocation.partial_block =
+			    make_unique<PartialBlockForCheckpoint>(segment.get(), *allocation.block_manager, allocation.state);
 		}
+		// Writer will decide whether to reuse this block.
+		writer.RegisterPartialBlock(move(allocation));
 	} else {
 		// constant block: no need to write anything to disk besides the stats
 		// set up the compression function to constant
 		auto &config = DBConfig::GetConfig(db);
 		segment->function =
 		    config.GetCompressionFunction(CompressionType::COMPRESSION_CONSTANT, segment->type.InternalType());
+		segment->ConvertToPersistent(nullptr, INVALID_BLOCK);
 	}
 
 	// construct the data pointer
@@ -203816,43 +204372,13 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 	data_pointer.compression_type = segment->function->type;
 	data_pointer.statistics = segment->stats.statistics->Copy();
 
-	if (need_to_write) {
-		if (partial_block) {
-			// pin the current block
-			auto old_handle = buffer_manager.Pin(segment->block);
-			// pin the new block
-			auto new_handle = buffer_manager.Pin(partial_block->block);
-			// memcpy the contents of the old block to the new block
-			memcpy(new_handle.Ptr() + offset_in_block, old_handle.Ptr(), segment_size);
-		} else {
-			// convert the segment into a persistent segment that points to this block
-			segment->ConvertToPersistent(block_id);
-		}
-	}
-	if (owned_partial_block) {
-		// the partial block has become full: write it to disk
-		owned_partial_block->FlushToDisk(db);
-	}
-
 	// append the segment to the new segment tree
 	new_tree.AppendSegment(move(segment));
 	data_pointers.push_back(move(data_pointer));
 }
 
-void ColumnCheckpointState::FlushToDisk() {
-	auto &meta_writer = writer.GetTableWriter();
-
-	meta_writer.Write<idx_t>(data_pointers.size());
-	// then write the data pointers themselves
-	for (idx_t k = 0; k < data_pointers.size(); k++) {
-		auto &data_pointer = data_pointers[k];
-		meta_writer.Write<idx_t>(data_pointer.row_start);
-		meta_writer.Write<idx_t>(data_pointer.tuple_count);
-		meta_writer.Write<block_id_t>(data_pointer.block_pointer.block_id);
-		meta_writer.Write<uint32_t>(data_pointer.block_pointer.offset);
-		meta_writer.Write<CompressionType>(data_pointer.compression_type);
-		data_pointer.statistics->Serialize(meta_writer);
-	}
+void ColumnCheckpointState::WriteDataPointers() {
+	writer.WriteColumnDataPointers(*this);
 }
 
 } // namespace duckdb
@@ -203888,8 +204414,8 @@ namespace duckdb {
 //! Struct column data represents a struct
 class StructColumnData : public ColumnData {
 public:
-	StructColumnData(DataTableInfo &info, idx_t column_index, idx_t start_row, LogicalType type,
-	                 ColumnData *parent = nullptr);
+	StructColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
+	                 LogicalType type, ColumnData *parent = nullptr);
 	StructColumnData(ColumnData &original, idx_t start_row, ColumnData *parent = nullptr);
 
 	//! The sub-columns of the struct
@@ -203924,8 +204450,8 @@ public:
 
 	void CommitDropColumn() override;
 
-	unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group, TableDataWriter &writer) override;
-	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, TableDataWriter &writer,
+	unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group, RowGroupWriter &writer) override;
+	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, RowGroupWriter &writer,
 	                                             ColumnCheckpointInfo &checkpoint_info) override;
 
 	void DeserializeColumn(Deserializer &source) override;
@@ -204049,13 +204575,15 @@ struct UpdateNode {
 
 namespace duckdb {
 
-ColumnData::ColumnData(DataTableInfo &info, idx_t column_index, idx_t start_row, LogicalType type, ColumnData *parent)
-    : info(info), column_index(column_index), start(start_row), type(move(type)), parent(parent) {
+ColumnData::ColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
+                       LogicalType type, ColumnData *parent)
+    : block_manager(block_manager), info(info), column_index(column_index), start(start_row), type(move(type)),
+      parent(parent) {
 }
 
 ColumnData::ColumnData(ColumnData &other, idx_t start, ColumnData *parent)
-    : info(other.info), column_index(other.column_index), start(start), type(move(other.type)), parent(parent),
-      updates(move(other.updates)) {
+    : block_manager(other.block_manager), info(other.info), column_index(other.column_index), start(start),
+      type(move(other.type)), parent(parent), updates(move(other.updates)) {
 	idx_t offset = 0;
 	for (auto segment = other.data.GetRootSegment(); segment; segment = segment->next.get()) {
 		auto &other = (ColumnSegment &)*segment;
@@ -204251,6 +204779,7 @@ void ColumnData::InitializeAppend(ColumnAppendState &state) {
 
 	D_ASSERT(state.current->segment_type == ColumnSegmentType::TRANSIENT);
 	state.current->InitializeAppend(state);
+	D_ASSERT(state.current->function->append);
 }
 
 void ColumnData::AppendData(BaseStatistics &stats, ColumnAppendState &state, UnifiedVectorFormat &vdata, idx_t count) {
@@ -204353,7 +204882,6 @@ void ColumnData::AppendTransientSegment(idx_t start_row) {
 }
 
 void ColumnData::CommitDropColumn() {
-	auto &block_manager = BlockManager::GetBlockManager(GetDatabase());
 	auto segment = (ColumnSegment *)data.GetRootSegment();
 	while (segment) {
 		if (segment->segment_type == ColumnSegmentType::PERSISTENT) {
@@ -204366,7 +204894,7 @@ void ColumnData::CommitDropColumn() {
 	}
 }
 
-unique_ptr<ColumnCheckpointState> ColumnData::CreateCheckpointState(RowGroup &row_group, TableDataWriter &writer) {
+unique_ptr<ColumnCheckpointState> ColumnData::CreateCheckpointState(RowGroup &row_group, RowGroupWriter &writer) {
 	return make_unique<ColumnCheckpointState>(row_group, *this, writer);
 }
 
@@ -204379,7 +204907,7 @@ void ColumnData::CheckpointScan(ColumnSegment *segment, ColumnScanState &state,
 	}
 }
 
-unique_ptr<ColumnCheckpointState> ColumnData::Checkpoint(RowGroup &row_group, TableDataWriter &writer,
+unique_ptr<ColumnCheckpointState> ColumnData::Checkpoint(RowGroup &row_group, RowGroupWriter &writer,
                                                          ColumnCheckpointInfo &checkpoint_info) {
 	// scan the segments of the column data
 	// set up the checkpoint state
@@ -204416,16 +204944,17 @@ void ColumnData::DeserializeColumn(Deserializer &source) {
 
 		// create a persistent segment
 		auto segment = ColumnSegment::CreatePersistentSegment(
-		    GetDatabase(), data_pointer.block_pointer.block_id, data_pointer.block_pointer.offset, type,
+		    GetDatabase(), block_manager, data_pointer.block_pointer.block_id, data_pointer.block_pointer.offset, type,
 		    data_pointer.row_start, data_pointer.tuple_count, data_pointer.compression_type,
 		    move(data_pointer.statistics));
 		data.AppendSegment(move(segment));
 	}
 }
 
-shared_ptr<ColumnData> ColumnData::Deserialize(DataTableInfo &info, idx_t column_index, idx_t start_row,
-                                               Deserializer &source, const LogicalType &type, ColumnData *parent) {
-	auto entry = ColumnData::CreateColumn(info, column_index, start_row, type, parent);
+shared_ptr<ColumnData> ColumnData::Deserialize(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
+                                               idx_t start_row, Deserializer &source, const LogicalType &type,
+                                               ColumnData *parent) {
+	auto entry = ColumnData::CreateColumn(block_manager, info, column_index, start_row, type, parent);
 	entry->DeserializeColumn(source);
 	return entry;
 }
@@ -204510,16 +205039,16 @@ void ColumnData::Verify(RowGroup &parent) {
 }
 
 template <class RET, class OP>
-static RET CreateColumnInternal(DataTableInfo &info, idx_t column_index, idx_t start_row, const LogicalType &type,
-                                ColumnData *parent) {
+static RET CreateColumnInternal(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
+                                const LogicalType &type, ColumnData *parent) {
 	if (type.InternalType() == PhysicalType::STRUCT) {
-		return OP::template Create<StructColumnData>(info, column_index, start_row, type, parent);
+		return OP::template Create<StructColumnData>(block_manager, info, column_index, start_row, type, parent);
 	} else if (type.InternalType() == PhysicalType::LIST) {
-		return OP::template Create<ListColumnData>(info, column_index, start_row, type, parent);
+		return OP::template Create<ListColumnData>(block_manager, info, column_index, start_row, type, parent);
 	} else if (type.id() == LogicalTypeId::VALIDITY) {
-		return OP::template Create<ValidityColumnData>(info, column_index, start_row, parent);
+		return OP::template Create<ValidityColumnData>(block_manager, info, column_index, start_row, parent);
 	}
-	return OP::template Create<StandardColumnData>(info, column_index, start_row, type, parent);
+	return OP::template Create<StandardColumnData>(block_manager, info, column_index, start_row, type, parent);
 }
 
 template <class RET, class OP>
@@ -204534,18 +205063,21 @@ static RET CreateColumnInternal(ColumnData &other, idx_t start_row, ColumnData *
 	return OP::template Create<StandardColumnData>(other, start_row, parent);
 }
 
-shared_ptr<ColumnData> ColumnData::CreateColumn(DataTableInfo &info, idx_t column_index, idx_t start_row,
-                                                const LogicalType &type, ColumnData *parent) {
-	return CreateColumnInternal<shared_ptr<ColumnData>, SharedConstructor>(info, column_index, start_row, type, parent);
+shared_ptr<ColumnData> ColumnData::CreateColumn(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
+                                                idx_t start_row, const LogicalType &type, ColumnData *parent) {
+	return CreateColumnInternal<shared_ptr<ColumnData>, SharedConstructor>(block_manager, info, column_index, start_row,
+	                                                                       type, parent);
 }
 
 shared_ptr<ColumnData> ColumnData::CreateColumn(ColumnData &other, idx_t start_row, ColumnData *parent) {
 	return CreateColumnInternal<shared_ptr<ColumnData>, SharedConstructor>(other, start_row, parent);
 }
 
-unique_ptr<ColumnData> ColumnData::CreateColumnUnique(DataTableInfo &info, idx_t column_index, idx_t start_row,
-                                                      const LogicalType &type, ColumnData *parent) {
-	return CreateColumnInternal<unique_ptr<ColumnData>, UniqueConstructor>(info, column_index, start_row, type, parent);
+unique_ptr<ColumnData> ColumnData::CreateColumnUnique(BlockManager &block_manager, DataTableInfo &info,
+                                                      idx_t column_index, idx_t start_row, const LogicalType &type,
+                                                      ColumnData *parent) {
+	return CreateColumnInternal<unique_ptr<ColumnData>, UniqueConstructor>(block_manager, info, column_index, start_row,
+	                                                                       type, parent);
 }
 
 unique_ptr<ColumnData> ColumnData::CreateColumnUnique(ColumnData &other, idx_t start_row, ColumnData *parent) {
@@ -204715,7 +205247,7 @@ void ColumnDataCheckpointer::WriteToDisk() {
 	// first we check the current segments
 	// if there are any persistent segments, we will mark their old block ids as modified
 	// since the segments will be rewritten their old on disk data is no longer required
-	auto &block_manager = BlockManager::GetBlockManager(GetDatabase());
+	auto &block_manager = col_data.block_manager;
 	for (auto segment = (ColumnSegment *)owned_segment.get(); segment; segment = (ColumnSegment *)segment->next.get()) {
 		if (segment->segment_type == ColumnSegmentType::PERSISTENT) {
 			// persistent segment has updates: mark it as modified and rewrite the block with the merged updates
@@ -204825,26 +205357,32 @@ void ColumnDataCheckpointer::Checkpoint(unique_ptr<SegmentBase> segment) {
 
 namespace duckdb {
 
-unique_ptr<ColumnSegment> ColumnSegment::CreatePersistentSegment(DatabaseInstance &db, block_id_t block_id,
-                                                                 idx_t offset, const LogicalType &type, idx_t start,
-                                                                 idx_t count, CompressionType compression_type,
+unique_ptr<ColumnSegment> ColumnSegment::CreatePersistentSegment(DatabaseInstance &db, BlockManager &block_manager,
+                                                                 block_id_t block_id, idx_t offset,
+                                                                 const LogicalType &type, idx_t start, idx_t count,
+                                                                 CompressionType compression_type,
                                                                  unique_ptr<BaseStatistics> statistics) {
 	auto &config = DBConfig::GetConfig(db);
 	CompressionFunction *function;
+	shared_ptr<BlockHandle> block;
 	if (block_id == INVALID_BLOCK) {
+		// constant segment, no need to allocate an actual block
 		function = config.GetCompressionFunction(CompressionType::COMPRESSION_CONSTANT, type.InternalType());
 	} else {
 		function = config.GetCompressionFunction(compression_type, type.InternalType());
+		block = block_manager.RegisterBlock(block_id);
 	}
-	return make_unique<ColumnSegment>(db, type, ColumnSegmentType::PERSISTENT, start, count, function, move(statistics),
-	                                  block_id, offset);
+	return make_unique<ColumnSegment>(db, block, type, ColumnSegmentType::PERSISTENT, start, count, function,
+	                                  move(statistics), block_id, offset);
 }
 
 unique_ptr<ColumnSegment> ColumnSegment::CreateTransientSegment(DatabaseInstance &db, const LogicalType &type,
                                                                 idx_t start) {
 	auto &config = DBConfig::GetConfig(db);
 	auto function = config.GetCompressionFunction(CompressionType::COMPRESSION_UNCOMPRESSED, type.InternalType());
-	return make_unique<ColumnSegment>(db, type, ColumnSegmentType::TRANSIENT, start, 0, function, nullptr,
+	// transient: allocate a buffer for the uncompressed segment
+	auto block = BufferManager::GetBufferManager(db).RegisterMemory(Storage::BLOCK_SIZE, false);
+	return make_unique<ColumnSegment>(db, block, type, ColumnSegmentType::TRANSIENT, start, 0, function, nullptr,
 	                                  INVALID_BLOCK, 0);
 }
 
@@ -204852,27 +205390,13 @@ unique_ptr<ColumnSegment> ColumnSegment::CreateSegment(ColumnSegment &other, idx
 	return make_unique<ColumnSegment>(other, start);
 }
 
-ColumnSegment::ColumnSegment(DatabaseInstance &db, LogicalType type_p, ColumnSegmentType segment_type, idx_t start,
-                             idx_t count, CompressionFunction *function_p, unique_ptr<BaseStatistics> statistics,
-                             block_id_t block_id_p, idx_t offset_p)
+ColumnSegment::ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block, LogicalType type_p,
+                             ColumnSegmentType segment_type, idx_t start, idx_t count, CompressionFunction *function_p,
+                             unique_ptr<BaseStatistics> statistics, block_id_t block_id_p, idx_t offset_p)
     : SegmentBase(start, count), db(db), type(move(type_p)), type_size(GetTypeIdSize(type.InternalType())),
-      segment_type(segment_type), function(function_p), stats(type, move(statistics)), block_id(block_id_p),
-      offset(offset_p) {
+      segment_type(segment_type), function(function_p), stats(type, move(statistics)), block(move(block)),
+      block_id(block_id_p), offset(offset_p) {
 	D_ASSERT(function);
-	auto &block_manager = BlockManager::GetBlockManager(db);
-	auto &buffer_manager = BufferManager::GetBufferManager(db);
-	if (block_id == INVALID_BLOCK) {
-		// no block id specified
-		// there are two cases here:
-		// transient: allocate a buffer for the uncompressed segment
-		// persistent: constant segment, no need to allocate anything
-		if (segment_type == ColumnSegmentType::TRANSIENT) {
-			this->block = buffer_manager.RegisterMemory(Storage::BLOCK_SIZE, false);
-		}
-	} else {
-		D_ASSERT(segment_type == ColumnSegmentType::PERSISTENT);
-		this->block = block_manager.RegisterBlock(block_id);
-	}
 	if (function->init_segment) {
 		segment_state = function->init_segment(*this, block_id);
 	}
@@ -204960,22 +205484,23 @@ void ColumnSegment::RevertAppend(idx_t start_row) {
 //===--------------------------------------------------------------------===//
 // Convert To Persistent
 //===--------------------------------------------------------------------===//
-void ColumnSegment::ConvertToPersistent(block_id_t block_id_p) {
+void ColumnSegment::ConvertToPersistent(BlockManager *block_manager, block_id_t block_id_p) {
 	D_ASSERT(segment_type == ColumnSegmentType::TRANSIENT);
 	segment_type = ColumnSegmentType::PERSISTENT;
+
 	block_id = block_id_p;
 	offset = 0;
 
 	if (block_id == INVALID_BLOCK) {
 		// constant block: reset the block buffer
+		D_ASSERT(stats.statistics->IsConstant());
 		block.reset();
 	} else {
+		D_ASSERT(!stats.statistics->IsConstant());
 		// non-constant block: write the block to disk
-		auto &block_manager = BlockManager::GetBlockManager(db);
-
 		// the data for the block already exists in-memory of our block
 		// instead of copying the data we alter some metadata so the buffer points to an on-disk block
-		block = block_manager.ConvertToPersistent(block_id, move(block));
+		block = block_manager->ConvertToPersistent(block_id, move(block));
 	}
 
 	segment_state.reset();
@@ -204984,10 +205509,11 @@ void ColumnSegment::ConvertToPersistent(block_id_t block_id_p) {
 	}
 }
 
-void ColumnSegment::ConvertToPersistent(shared_ptr<BlockHandle> block_p, block_id_t block_id_p, uint32_t offset_p) {
+void ColumnSegment::MarkAsPersistent(shared_ptr<BlockHandle> block_p, uint32_t offset_p) {
 	D_ASSERT(segment_type == ColumnSegmentType::TRANSIENT);
 	segment_type = ColumnSegmentType::PERSISTENT;
-	block_id = block_id_p;
+
+	block_id = block_p->BlockId();
 	offset = offset_p;
 	block = move(block_p);
 
@@ -205278,13 +205804,14 @@ idx_t ColumnSegment::FilterSelection(SelectionVector &sel, Vector &result, const
 
 namespace duckdb {
 
-ListColumnData::ListColumnData(DataTableInfo &info, idx_t column_index, idx_t start_row, LogicalType type_p,
-                               ColumnData *parent)
-    : ColumnData(info, column_index, start_row, move(type_p), parent), validity(info, 0, start_row, this) {
+ListColumnData::ListColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
+                               LogicalType type_p, ColumnData *parent)
+    : ColumnData(block_manager, info, column_index, start_row, move(type_p), parent),
+      validity(block_manager, info, 0, start_row, this) {
 	D_ASSERT(type.InternalType() == PhysicalType::LIST);
 	auto &child_type = ListType::GetChildType(type);
 	// the child column, with column index 1 (0 is the validity mask)
-	child_column = ColumnData::CreateColumnUnique(info, 1, start_row, child_type, this);
+	child_column = ColumnData::CreateColumnUnique(block_manager, info, 1, start_row, child_type, this);
 }
 
 ListColumnData::ListColumnData(ColumnData &original, idx_t start_row, ColumnData *parent)
@@ -205595,7 +206122,7 @@ void ListColumnData::CommitDropColumn() {
 }
 
 struct ListColumnCheckpointState : public ColumnCheckpointState {
-	ListColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, TableDataWriter &writer)
+	ListColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, RowGroupWriter &writer)
 	    : ColumnCheckpointState(row_group, column_data, writer) {
 		global_stats = make_unique<ListStatistics>(column_data.type);
 	}
@@ -205612,18 +206139,18 @@ public:
 		return stats;
 	}
 
-	void FlushToDisk() override {
-		ColumnCheckpointState::FlushToDisk();
-		validity_state->FlushToDisk();
-		child_state->FlushToDisk();
+	void WriteDataPointers() override {
+		ColumnCheckpointState::WriteDataPointers();
+		validity_state->WriteDataPointers();
+		child_state->WriteDataPointers();
 	}
 };
 
-unique_ptr<ColumnCheckpointState> ListColumnData::CreateCheckpointState(RowGroup &row_group, TableDataWriter &writer) {
+unique_ptr<ColumnCheckpointState> ListColumnData::CreateCheckpointState(RowGroup &row_group, RowGroupWriter &writer) {
 	return make_unique<ListColumnCheckpointState>(row_group, *this, writer);
 }
 
-unique_ptr<ColumnCheckpointState> ListColumnData::Checkpoint(RowGroup &row_group, TableDataWriter &writer,
+unique_ptr<ColumnCheckpointState> ListColumnData::Checkpoint(RowGroup &row_group, RowGroupWriter &writer,
                                                              ColumnCheckpointInfo &checkpoint_info) {
 	auto validity_state = validity.Checkpoint(row_group, writer, checkpoint_info);
 	auto base_state = ColumnData::Checkpoint(row_group, writer, checkpoint_info);
@@ -205682,24 +206209,27 @@ namespace duckdb {
 constexpr const idx_t RowGroup::ROW_GROUP_VECTOR_COUNT;
 constexpr const idx_t RowGroup::ROW_GROUP_SIZE;
 
-RowGroup::RowGroup(DatabaseInstance &db, DataTableInfo &table_info, idx_t start, idx_t count)
-    : SegmentBase(start, count), db(db), table_info(table_info) {
+RowGroup::RowGroup(DatabaseInstance &db, BlockManager &block_manager, DataTableInfo &table_info, idx_t start,
+                   idx_t count)
+    : SegmentBase(start, count), db(db), block_manager(block_manager), table_info(table_info) {
 
 	Verify();
 }
 
-RowGroup::RowGroup(DatabaseInstance &db, DataTableInfo &table_info, const vector<LogicalType> &types,
-                   RowGroupPointer &pointer)
-    : SegmentBase(pointer.row_start, pointer.tuple_count), db(db), table_info(table_info) {
+RowGroup::RowGroup(DatabaseInstance &db, BlockManager &block_manager, DataTableInfo &table_info,
+                   const vector<LogicalType> &types, RowGroupPointer &&pointer)
+    : SegmentBase(pointer.row_start, pointer.tuple_count), db(db), block_manager(block_manager),
+      table_info(table_info) {
 	// deserialize the columns
 	if (pointer.data_pointers.size() != types.size()) {
 		throw IOException("Row group column count is unaligned with table column count. Corrupt file?");
 	}
 	for (idx_t i = 0; i < pointer.data_pointers.size(); i++) {
 		auto &block_pointer = pointer.data_pointers[i];
-		MetaBlockReader column_data_reader(db, block_pointer.block_id);
+		MetaBlockReader column_data_reader(block_manager, block_pointer.block_id);
 		column_data_reader.offset = block_pointer.offset;
-		this->columns.push_back(ColumnData::Deserialize(table_info, i, start, column_data_reader, types[i], nullptr));
+		this->columns.push_back(
+		    ColumnData::Deserialize(block_manager, table_info, i, start, column_data_reader, types[i], nullptr));
 	}
 
 	// set up the statistics
@@ -205713,8 +206243,8 @@ RowGroup::RowGroup(DatabaseInstance &db, DataTableInfo &table_info, const vector
 }
 
 RowGroup::RowGroup(RowGroup &row_group, idx_t start)
-    : SegmentBase(start, row_group.count), db(row_group.db), table_info(row_group.table_info),
-      version_info(move(row_group.version_info)), stats(move(row_group.stats)) {
+    : SegmentBase(start, row_group.count), db(row_group.db), block_manager(row_group.block_manager),
+      table_info(row_group.table_info), version_info(move(row_group.version_info)), stats(move(row_group.stats)) {
 	for (auto &column : row_group.columns) {
 		this->columns.push_back(ColumnData::CreateColumn(*column, start));
 	}
@@ -205740,7 +206270,7 @@ RowGroup::~RowGroup() {
 void RowGroup::InitializeEmpty(const vector<LogicalType> &types) {
 	// set up the segment trees for the column segments
 	for (idx_t i = 0; i < types.size(); i++) {
-		auto column_data = ColumnData::CreateColumn(GetTableInfo(), i, start, types[i]);
+		auto column_data = ColumnData::CreateColumn(block_manager, GetTableInfo(), i, start, types[i]);
 		stats.push_back(make_shared<SegmentStatistics>(types[i]));
 		columns.push_back(move(column_data));
 	}
@@ -205802,7 +206332,7 @@ unique_ptr<RowGroup> RowGroup::AlterType(const LogicalType &target_type, idx_t c
 	Verify();
 
 	// construct a new column data for this type
-	auto column_data = ColumnData::CreateColumn(GetTableInfo(), changed_idx, start, target_type);
+	auto column_data = ColumnData::CreateColumn(block_manager, GetTableInfo(), changed_idx, start, target_type);
 
 	ColumnAppendState append_state;
 	column_data->InitializeAppend(append_state);
@@ -205825,7 +206355,7 @@ unique_ptr<RowGroup> RowGroup::AlterType(const LogicalType &target_type, idx_t c
 	}
 
 	// set up the row_group based on this row_group
-	auto row_group = make_unique<RowGroup>(db, table_info, this->start, this->count);
+	auto row_group = make_unique<RowGroup>(db, block_manager, table_info, this->start, this->count);
 	row_group->version_info = version_info;
 	for (idx_t i = 0; i < columns.size(); i++) {
 		if (i == changed_idx) {
@@ -205847,7 +206377,8 @@ unique_ptr<RowGroup> RowGroup::AddColumn(ColumnDefinition &new_column, Expressio
 	Verify();
 
 	// construct a new column data for the new column
-	auto added_column = ColumnData::CreateColumn(GetTableInfo(), columns.size(), start, new_column.Type());
+	auto added_column =
+	    ColumnData::CreateColumn(block_manager, GetTableInfo(), columns.size(), start, new_column.Type());
 	auto added_col_stats = make_shared<SegmentStatistics>(
 	    new_column.Type(), BaseStatistics::CreateEmpty(new_column.Type(), StatisticsType::LOCAL_STATS));
 
@@ -205868,7 +206399,7 @@ unique_ptr<RowGroup> RowGroup::AddColumn(ColumnDefinition &new_column, Expressio
 	}
 
 	// set up the row_group based on this row_group
-	auto row_group = make_unique<RowGroup>(db, table_info, this->start, this->count);
+	auto row_group = make_unique<RowGroup>(db, block_manager, table_info, this->start, this->count);
 	row_group->version_info = version_info;
 	row_group->columns = columns;
 	row_group->stats = stats;
@@ -205885,7 +206416,7 @@ unique_ptr<RowGroup> RowGroup::RemoveColumn(idx_t removed_column) {
 
 	D_ASSERT(removed_column < columns.size());
 
-	auto row_group = make_unique<RowGroup>(db, table_info, this->start, this->count);
+	auto row_group = make_unique<RowGroup>(db, block_manager, table_info, this->start, this->count);
 	row_group->version_info = version_info;
 	row_group->columns = columns;
 	row_group->stats = stats;
@@ -206342,12 +206873,19 @@ void RowGroup::MergeIntoStatistics(idx_t column_idx, BaseStatistics &other) {
 	other.Merge(*stats[column_idx]->statistics);
 }
 
-RowGroupPointer RowGroup::Checkpoint(TableDataWriter &writer, vector<unique_ptr<BaseStatistics>> &global_stats) {
+RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, vector<unique_ptr<BaseStatistics>> &global_stats) {
 	RowGroupPointer row_group_pointer;
 	vector<unique_ptr<ColumnCheckpointState>> states;
 	states.reserve(columns.size());
 
-	// checkpoint the individual columns of the row group
+	// Checkpoint the individual columns of the row group
+	// Here we're iterating over columns. Each column can have multiple segments.
+	// (Some columns will be wider than others, and require different numbers
+	// of blocks to encode.) Segments cannot span blocks.
+	//
+	// Some of these columns are composite (list, struct). The data is written
+	// first sequentially, and the pointers are written later, so that the
+	// pointers all end up densely packed, and thus more cache-friendly.
 	for (idx_t column_idx = 0; column_idx < columns.size(); column_idx++) {
 		auto &column = columns[column_idx];
 		ColumnCheckpointInfo checkpoint_info {writer.GetColumnCompressionType(column_idx)};
@@ -206367,15 +206905,18 @@ RowGroupPointer RowGroup::Checkpoint(TableDataWriter &writer, vector<unique_ptr<
 	row_group_pointer.row_start = start;
 	row_group_pointer.tuple_count = count;
 	for (auto &state : states) {
-		// get the current position of the meta data writer
-		auto &meta_writer = writer.GetTableWriter();
-		auto pointer = meta_writer.GetBlockPointer();
+		// get the current position of the table data writer
+		auto &data_writer = writer.GetPayloadWriter();
+		auto pointer = data_writer.GetBlockPointer();
 
 		// store the stats and the data pointers in the row group pointers
 		row_group_pointer.data_pointers.push_back(pointer);
 
-		// now flush the actual column data to disk
-		state->FlushToDisk();
+		// Write pointers to the column segments.
+		//
+		// Just as above, the state can refer to many other states, so this
+		// can cascade recursively into more pointer writes.
+		state->WriteDataPointers();
 	}
 	row_group_pointer.versions = version_info;
 	Verify();
@@ -206592,11 +207133,13 @@ void VersionDeleteState::Flush() {
 
 
 
+
 namespace duckdb {
 
-RowGroupCollection::RowGroupCollection(shared_ptr<DataTableInfo> info_p, vector<LogicalType> types_p, idx_t row_start_p,
-                                       idx_t total_rows_p)
-    : total_rows(total_rows_p), info(move(info_p)), types(move(types_p)), row_start(row_start_p) {
+RowGroupCollection::RowGroupCollection(shared_ptr<DataTableInfo> info_p, BlockManager &block_manager,
+                                       vector<LogicalType> types_p, idx_t row_start_p, idx_t total_rows_p)
+    : block_manager(block_manager), total_rows(total_rows_p), info(move(info_p)), types(move(types_p)),
+      row_start(row_start_p) {
 	row_groups = make_shared<SegmentTree>();
 }
 
@@ -206618,7 +207161,7 @@ Allocator &RowGroupCollection::GetAllocator() const {
 void RowGroupCollection::Initialize(PersistentTableData &data) {
 	D_ASSERT(this->row_start == 0);
 	for (auto &row_group_pointer : data.row_groups) {
-		auto new_row_group = make_unique<RowGroup>(info->db, *info, types, row_group_pointer);
+		auto new_row_group = make_unique<RowGroup>(info->db, block_manager, *info, types, move(row_group_pointer));
 		auto row_group_count = new_row_group->start + new_row_group->count;
 		if (row_group_count > this->total_rows) {
 			this->total_rows = row_group_count;
@@ -206629,7 +207172,7 @@ void RowGroupCollection::Initialize(PersistentTableData &data) {
 
 void RowGroupCollection::AppendRowGroup(idx_t start_row) {
 	D_ASSERT(start_row >= row_start);
-	auto new_row_group = make_unique<RowGroup>(info->db, *info, start_row, 0);
+	auto new_row_group = make_unique<RowGroup>(info->db, block_manager, *info, start_row, 0);
 	new_row_group->InitializeEmpty(types);
 	row_groups->AppendSegment(move(new_row_group));
 }
@@ -206997,13 +207540,12 @@ void RowGroupCollection::UpdateColumn(TransactionData transaction, Vector &row_i
 //===--------------------------------------------------------------------===//
 // Checkpoint
 //===--------------------------------------------------------------------===//
-void RowGroupCollection::Checkpoint(TableDataWriter &writer, vector<RowGroupPointer> &row_group_pointers,
-                                    vector<unique_ptr<BaseStatistics>> &global_stats) {
-	auto row_group = (RowGroup *)row_groups->GetRootSegment();
-	while (row_group) {
-		auto pointer = row_group->Checkpoint(writer, global_stats);
-		row_group_pointers.push_back(move(pointer));
-		row_group = (RowGroup *)row_group->next.get();
+void RowGroupCollection::Checkpoint(TableDataWriter &writer, vector<unique_ptr<BaseStatistics>> &global_stats) {
+	for (auto row_group = (RowGroup *)row_groups->GetRootSegment(); row_group;
+	     row_group = (RowGroup *)row_group->next.get()) {
+		auto rowg_writer = writer.GetRowGroupWriter(*row_group);
+		auto pointer = row_group->Checkpoint(*rowg_writer, global_stats);
+		writer.AddRowGroup(move(pointer), move(rowg_writer));
 	}
 }
 
@@ -207052,7 +207594,7 @@ shared_ptr<RowGroupCollection> RowGroupCollection::AddColumn(ColumnDefinition &n
 	idx_t new_column_idx = types.size();
 	auto new_types = types;
 	new_types.push_back(new_column.GetType());
-	auto result = make_shared<RowGroupCollection>(info, move(new_types), row_start, total_rows.load());
+	auto result = make_shared<RowGroupCollection>(info, block_manager, move(new_types), row_start, total_rows.load());
 
 	ExpressionExecutor executor(GetAllocator());
 	DataChunk dummy_chunk;
@@ -207082,7 +207624,7 @@ shared_ptr<RowGroupCollection> RowGroupCollection::RemoveColumn(idx_t col_idx) {
 	auto new_types = types;
 	new_types.erase(new_types.begin() + col_idx);
 
-	auto result = make_shared<RowGroupCollection>(info, move(new_types), row_start, total_rows.load());
+	auto result = make_shared<RowGroupCollection>(info, block_manager, move(new_types), row_start, total_rows.load());
 
 	auto current_row_group = (RowGroup *)row_groups->GetRootSegment();
 	while (current_row_group) {
@@ -207100,7 +207642,7 @@ shared_ptr<RowGroupCollection> RowGroupCollection::AlterType(idx_t changed_idx,
 	auto new_types = types;
 	new_types[changed_idx] = target_type;
 
-	auto result = make_shared<RowGroupCollection>(info, move(new_types), row_start, total_rows.load());
+	auto result = make_shared<RowGroupCollection>(info, block_manager, move(new_types), row_start, total_rows.load());
 
 	vector<LogicalType> scan_types;
 	for (idx_t i = 0; i < bound_columns.size(); i++) {
@@ -207366,9 +207908,10 @@ void SegmentTree::Replace(SegmentTree &other) {
 
 namespace duckdb {
 
-StandardColumnData::StandardColumnData(DataTableInfo &info, idx_t column_index, idx_t start_row, LogicalType type,
-                                       ColumnData *parent)
-    : ColumnData(info, column_index, start_row, move(type), parent), validity(info, 0, start_row, this) {
+StandardColumnData::StandardColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
+                                       idx_t start_row, LogicalType type, ColumnData *parent)
+    : ColumnData(block_manager, info, column_index, start_row, move(type), parent),
+      validity(block_manager, info, 0, start_row, this) {
 }
 
 StandardColumnData::StandardColumnData(ColumnData &original, idx_t start_row, ColumnData *parent)
@@ -207516,7 +208059,7 @@ void StandardColumnData::CommitDropColumn() {
 }
 
 struct StandardColumnCheckpointState : public ColumnCheckpointState {
-	StandardColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, TableDataWriter &writer)
+	StandardColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, RowGroupWriter &writer)
 	    : ColumnCheckpointState(row_group, column_data, writer) {
 	}
 
@@ -207529,18 +208072,18 @@ public:
 		return move(global_stats);
 	}
 
-	void FlushToDisk() override {
-		ColumnCheckpointState::FlushToDisk();
-		validity_state->FlushToDisk();
+	void WriteDataPointers() override {
+		ColumnCheckpointState::WriteDataPointers();
+		validity_state->WriteDataPointers();
 	}
 };
 
 unique_ptr<ColumnCheckpointState> StandardColumnData::CreateCheckpointState(RowGroup &row_group,
-                                                                            TableDataWriter &writer) {
+                                                                            RowGroupWriter &writer) {
 	return make_unique<StandardColumnCheckpointState>(row_group, *this, writer);
 }
 
-unique_ptr<ColumnCheckpointState> StandardColumnData::Checkpoint(RowGroup &row_group, TableDataWriter &writer,
+unique_ptr<ColumnCheckpointState> StandardColumnData::Checkpoint(RowGroup &row_group, RowGroupWriter &writer,
                                                                  ColumnCheckpointInfo &checkpoint_info) {
 	auto validity_state = validity.Checkpoint(row_group, writer, checkpoint_info);
 	auto base_state = ColumnData::Checkpoint(row_group, writer, checkpoint_info);
@@ -207582,9 +208125,10 @@ void StandardColumnData::Verify(RowGroup &parent) {
 
 namespace duckdb {
 
-StructColumnData::StructColumnData(DataTableInfo &info, idx_t column_index, idx_t start_row, LogicalType type_p,
-                                   ColumnData *parent)
-    : ColumnData(info, column_index, start_row, move(type_p), parent), validity(info, 0, start_row, this) {
+StructColumnData::StructColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
+                                   idx_t start_row, LogicalType type_p, ColumnData *parent)
+    : ColumnData(block_manager, info, column_index, start_row, move(type_p), parent),
+      validity(block_manager, info, 0, start_row, this) {
 	D_ASSERT(type.InternalType() == PhysicalType::STRUCT);
 	auto &child_types = StructType::GetChildTypes(type);
 	D_ASSERT(child_types.size() > 0);
@@ -207592,7 +208136,7 @@ StructColumnData::StructColumnData(DataTableInfo &info, idx_t column_index, idx_
 	idx_t sub_column_index = 1;
 	for (auto &child_type : child_types) {
 		sub_columns.push_back(
-		    ColumnData::CreateColumnUnique(info, sub_column_index, start_row, child_type.second, this));
+		    ColumnData::CreateColumnUnique(block_manager, info, sub_column_index, start_row, child_type.second, this));
 		sub_column_index++;
 	}
 }
@@ -207814,7 +208358,7 @@ void StructColumnData::CommitDropColumn() {
 }
 
 struct StructColumnCheckpointState : public ColumnCheckpointState {
-	StructColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, TableDataWriter &writer)
+	StructColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, RowGroupWriter &writer)
 	    : ColumnCheckpointState(row_group, column_data, writer) {
 		global_stats = make_unique<StructStatistics>(column_data.type);
 	}
@@ -207834,20 +208378,19 @@ public:
 		return move(stats);
 	}
 
-	void FlushToDisk() override {
-		validity_state->FlushToDisk();
+	void WriteDataPointers() override {
+		validity_state->WriteDataPointers();
 		for (auto &state : child_states) {
-			state->FlushToDisk();
+			state->WriteDataPointers();
 		}
 	}
 };
 
-unique_ptr<ColumnCheckpointState> StructColumnData::CreateCheckpointState(RowGroup &row_group,
-                                                                          TableDataWriter &writer) {
+unique_ptr<ColumnCheckpointState> StructColumnData::CreateCheckpointState(RowGroup &row_group, RowGroupWriter &writer) {
 	return make_unique<StructColumnCheckpointState>(row_group, *this, writer);
 }
 
-unique_ptr<ColumnCheckpointState> StructColumnData::Checkpoint(RowGroup &row_group, TableDataWriter &writer,
+unique_ptr<ColumnCheckpointState> StructColumnData::Checkpoint(RowGroup &row_group, RowGroupWriter &writer,
                                                                ColumnCheckpointInfo &checkpoint_info) {
 	auto checkpoint_state = make_unique<StructColumnCheckpointState>(row_group, *this, writer);
 	checkpoint_state->validity_state = validity.Checkpoint(row_group, writer, checkpoint_info);
@@ -209279,8 +209822,9 @@ bool UpdateSegment::HasUpdates(idx_t start_row_index, idx_t end_row_index) {
 
 namespace duckdb {
 
-ValidityColumnData::ValidityColumnData(DataTableInfo &info, idx_t column_index, idx_t start_row, ColumnData *parent)
-    : ColumnData(info, column_index, start_row, LogicalType(LogicalTypeId::VALIDITY), parent) {
+ValidityColumnData::ValidityColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
+                                       idx_t start_row, ColumnData *parent)
+    : ColumnData(block_manager, info, column_index, start_row, LogicalType(LogicalTypeId::VALIDITY), parent) {
 }
 
 ValidityColumnData::ValidityColumnData(ColumnData &original, idx_t start_row, ColumnData *parent)
@@ -209384,6 +209928,7 @@ vector<BlockPointer> TableIndexList::SerializeIndexes(duckdb::MetaBlockWriter &w
 
 
 
+
 namespace duckdb {
 
 class ReplayState {
@@ -209472,8 +210017,8 @@ bool WriteAheadLog::Replay(DatabaseInstance &database, string &path) {
 	initial_reader.reset();
 	if (checkpoint_state.checkpoint_id != INVALID_BLOCK) {
 		// there is a checkpoint flag: check if we need to deserialize the WAL
-		auto &manager = BlockManager::GetBlockManager(database);
-		if (manager.IsRootBlock(checkpoint_state.checkpoint_id)) {
+		auto &manager = StorageManager::GetStorageManager(database);
+		if (manager.IsCheckpointClean(checkpoint_state.checkpoint_id)) {
 			// the contents of the WAL have already been checkpointed
 			// we can safely truncate the WAL and ignore its contents
 			return true;
@@ -209904,15 +210449,14 @@ void ReplayState::ReplayCheckpoint() {
 
 namespace duckdb {
 
-WriteAheadLog::WriteAheadLog(DatabaseInstance &database) : initialized(false), skip_writing(false), database(database) {
-}
-
-void WriteAheadLog::Initialize(string &path) {
+WriteAheadLog::WriteAheadLog(DatabaseInstance &database, const string &path) : skip_writing(false), database(database) {
 	wal_path = path;
 	writer = make_unique<BufferedFileWriter>(database.GetFileSystem(), path.c_str(),
 	                                         FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE |
 	                                             FileFlags::FILE_FLAGS_APPEND);
-	initialized = true;
+}
+
+WriteAheadLog::~WriteAheadLog() {
 }
 
 int64_t WriteAheadLog::GetWALSize() {
@@ -209930,10 +210474,9 @@ void WriteAheadLog::Truncate(int64_t size) {
 }
 
 void WriteAheadLog::Delete() {
-	if (!initialized) {
+	if (!writer) {
 		return;
 	}
-	initialized = false;
 	writer.reset();
 
 	auto &fs = FileSystem::GetFileSystem(database);
@@ -210869,41 +211412,23 @@ bool Transaction::ChangesMade() {
 }
 
 bool Transaction::AutomaticCheckpoint(DatabaseInstance &db) {
-	auto &config = DBConfig::GetConfig(db);
 	auto &storage_manager = StorageManager::GetStorageManager(db);
-	auto log = storage_manager.GetWriteAheadLog();
-	if (!log) {
-		return false;
-	}
-
-	auto initial_size = log->GetWALSize();
-	idx_t expected_wal_size = initial_size + storage.EstimatedSize() + undo_buffer.EstimatedSize();
-	return expected_wal_size > config.options.checkpoint_wal_size;
+	return storage_manager.AutomaticCheckpoint(storage.EstimatedSize() + undo_buffer.EstimatedSize());
 }
 
 string Transaction::Commit(DatabaseInstance &db, transaction_t commit_id, bool checkpoint) noexcept {
+	// "checkpoint" parameter indicates if the caller will checkpoint. If checkpoint ==
+	//    true: Then this function will NOT write to the WAL or flush/persist.
+	//          This method only makes commit in memory, expecting caller to checkpoint/flush.
+	//    false: Then this function WILL write to the WAL and Flush/Persist it.
 	this->commit_id = commit_id;
 	auto &storage_manager = StorageManager::GetStorageManager(db);
 	auto log = storage_manager.GetWriteAheadLog();
 
 	UndoBuffer::IteratorState iterator_state;
 	LocalStorage::CommitState commit_state;
-	idx_t initial_wal_size = 0;
-	idx_t initial_written = 0;
-	if (log) {
-		auto initial_size = log->GetWALSize();
-		initial_written = log->GetTotalWritten();
-		initial_wal_size = initial_size < 0 ? 0 : idx_t(initial_size);
-	} else {
-		D_ASSERT(!checkpoint);
-	}
+	auto storage_commit_state = storage_manager.GenStorageCommitState(*this, checkpoint);
 	try {
-		if (checkpoint) {
-			// check if we are checkpointing after this commit
-			// if we are checkpointing, we don't need to write anything to the WAL
-			// this saves us a lot of unnecessary writes to disk in the case of large commits
-			log->skip_writing = true;
-		}
 		storage.Commit(commit_state, *this, log, commit_id);
 		undo_buffer.Commit(iterator_state, log, commit_id);
 		if (log) {
@@ -210911,25 +211436,11 @@ string Transaction::Commit(DatabaseInstance &db, transaction_t commit_id, bool c
 			for (auto &entry : sequence_usage) {
 				log->WriteSequenceValue(entry.first, entry.second);
 			}
-			// flush the WAL if any changes were made
-			if (log->GetTotalWritten() > initial_written) {
-				D_ASSERT(!checkpoint);
-				D_ASSERT(!log->skip_writing);
-				log->Flush();
-			}
-			log->skip_writing = false;
 		}
+		storage_commit_state->FlushCommit();
 		return string();
 	} catch (std::exception &ex) {
 		undo_buffer.RevertCommit(iterator_state, transaction_id);
-		if (log) {
-			log->skip_writing = false;
-			if (log->GetTotalWritten() > initial_written) {
-				// remove any entries written into the WAL by truncating it
-				log->Truncate(initial_wal_size);
-			}
-		}
-		D_ASSERT(!log || !log->skip_writing);
 		return ex.what();
 	}
 }