duckdb 0.5.2-dev764.0 → 0.5.2-dev802.0

package/src/duckdb.cpp

@@ -63958,8 +63958,9 @@ OperatorResultType PhysicalStreamingWindow::Execute(ExecutionContext &context, D
 			if (wexpr.children.empty()) {
 				D_ASSERT(GetTypeIdSize(result.GetType().InternalType()) == sizeof(int64_t));
 				auto data = FlatVector::GetData<int64_t>(result);
+				int64_t start_row = gstate.row_number;
 				for (idx_t i = 0; i < input.size(); ++i) {
-					data[i] = gstate.row_number + i;
+					data[i] = start_row + i;
 				}
 				break;
 			}
@@ -64006,9 +64007,10 @@ OperatorResultType PhysicalStreamingWindow::Execute(ExecutionContext &context, D
 		}
 		case ExpressionType::WINDOW_ROW_NUMBER: {
 			// Set row numbers
+			int64_t start_row = gstate.row_number;
 			auto rdata = FlatVector::GetData<int64_t>(chunk.data[col_idx]);
 			for (idx_t i = 0; i < count; i++) {
-				rdata[i] = gstate.row_number + i;
+				rdata[i] = start_row + i;
 			}
 			break;
 		}
@@ -64114,6 +64116,7 @@ private:
 
 
 
+
 #include <functional>
 
 
@@ -64580,6 +64583,10 @@ public:
 	void Schedule() override {
 		auto &distinct_data = *gstate.distinct_data;
 
+		//! Now that all tables are combined, it's time to do the distinct aggregations
+		auto new_event = make_shared<DistinctAggregateFinalizeEvent>(op, gstate, *pipeline, client);
+		this->InsertEvent(move(new_event));
+
 		vector<unique_ptr<Task>> tasks;
 		for (idx_t table_idx = 0; table_idx < distinct_data.radix_tables.size(); table_idx++) {
 			distinct_data.radix_tables[table_idx]->ScheduleTasks(pipeline->executor, shared_from_this(),
@@ -64587,10 +64594,6 @@ public:
 		}
 		D_ASSERT(!tasks.empty());
 		SetTasks(move(tasks));
-
-		//! Now that all tables are combined, it's time to do the distinct aggregations
-		auto new_event = make_shared<DistinctAggregateFinalizeEvent>(op, gstate, *pipeline, client);
-		this->InsertEvent(move(new_event));
 	}
 };
 
@@ -71671,6 +71674,7 @@ private:
 
 
 
+
 #include <thread>
 
 namespace duckdb {
@@ -82285,7 +82289,8 @@ static bool CanPlanIndexJoin(Transaction &transaction, TableScanBindData *bind_d
 		return false;
 	}
 	auto table = bind_data->table;
-	if (transaction.storage.Find(table->storage.get())) {
+	auto &local_storage = LocalStorage::Get(transaction);
+	if (local_storage.Find(table->storage.get())) {
 		// transaction local appends: skip index join
 		return false;
 	}
@@ -105300,6 +105305,7 @@ void ListFlattenFun::RegisterFunction(BuiltinFunctions &set) {
 
 
 
+
 namespace duckdb {
 
 // FIXME: use a local state for each thread to increase performance?
@@ -116691,6 +116697,8 @@ void StructPackFun::RegisterFunction(BuiltinFunctions &set) {
 
 
 
+
+
 namespace duckdb {
 
 // aggregate state export
@@ -124159,8 +124167,8 @@ unique_ptr<GlobalTableFunctionState> TableScanInitGlobal(ClientContext &context,
 static unique_ptr<BaseStatistics> TableScanStatistics(ClientContext &context, const FunctionData *bind_data_p,
                                                       column_t column_id) {
 	auto &bind_data = (const TableScanBindData &)*bind_data_p;
-	auto &transaction = Transaction::GetTransaction(context);
-	if (transaction.storage.Find(bind_data.table->storage.get())) {
+	auto &local_storage = LocalStorage::Get(context);
+	if (local_storage.Find(bind_data.table->storage.get())) {
 		// we don't emit any statistics for tables that have outstanding transaction-local data
 		return nullptr;
 	}
@@ -124238,9 +124246,9 @@ void TableScanDependency(unordered_set<CatalogEntry *> &entries, const FunctionD
 
 unique_ptr<NodeStatistics> TableScanCardinality(ClientContext &context, const FunctionData *bind_data_p) {
 	auto &bind_data = (const TableScanBindData &)*bind_data_p;
-	auto &transaction = Transaction::GetTransaction(context);
+	auto &local_storage = LocalStorage::Get(context);
 	idx_t estimated_cardinality =
-	    bind_data.table->storage->info->cardinality + transaction.storage.AddedRows(bind_data.table->storage.get());
+	    bind_data.table->storage->info->cardinality + local_storage.AddedRows(bind_data.table->storage.get());
 	return make_unique<NodeStatistics>(bind_data.table->storage->info->cardinality, estimated_cardinality);
 }
 
@@ -124265,11 +124273,11 @@ static unique_ptr<GlobalTableFunctionState> IndexScanInitGlobal(ClientContext &c
 		row_id_data = (data_ptr_t)&bind_data.result_ids[0];
 	}
 	auto result = make_unique<IndexScanGlobalState>(row_id_data);
-	auto &transaction = Transaction::GetTransaction(context);
+	auto &local_storage = LocalStorage::Get(context);
 	result->column_ids = input.column_ids;
 	result->local_storage_state.Initialize(input.column_ids, input.filters);
-	transaction.storage.InitializeScan(bind_data.table->storage.get(), result->local_storage_state.local_state,
-	                                   input.filters);
+	local_storage.InitializeScan(bind_data.table->storage.get(), result->local_storage_state.local_state,
+	                             input.filters);
 
 	result->finished = false;
 	return move(result);
@@ -124279,13 +124287,15 @@ static void IndexScanFunction(ClientContext &context, TableFunctionInput &data_p
 	auto &bind_data = (const TableScanBindData &)*data_p.bind_data;
 	auto &state = (IndexScanGlobalState &)*data_p.global_state;
 	auto &transaction = Transaction::GetTransaction(context);
+	auto &local_storage = LocalStorage::Get(transaction);
+
 	if (!state.finished) {
 		bind_data.table->storage->Fetch(transaction, output, state.column_ids, state.row_ids,
 		                                bind_data.result_ids.size(), state.fetch_state);
 		state.finished = true;
 	}
 	if (output.size() == 0) {
-		transaction.storage.Scan(state.local_storage_state.local_state, state.column_ids, output);
+		local_storage.Scan(state.local_storage_state.local_state, state.column_ids, output);
 	}
 }
 
@@ -178018,6 +178028,7 @@ void BindContext::AddContext(BindContext other) {
 
 
 
+
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -178215,6 +178226,7 @@ public:
 
 
 
+
 namespace duckdb {
 
 static void InvertPercentileFractions(unique_ptr<ParsedExpression> &fractions) {
@@ -179094,6 +179106,7 @@ BindResult ExpressionBinder::BindExpression(ConstantExpression &expr, idx_t dept
 
 
 
+
 namespace duckdb {
 
 BindResult ExpressionBinder::BindExpression(FunctionExpression &function, idx_t depth,
@@ -186562,6 +186575,7 @@ BoundLimitPercentModifier::BoundLimitPercentModifier()
 
 
 
+
 namespace duckdb {
 
 class FunctionSerializer {
@@ -194519,6 +194533,8 @@ struct PartialBlock {
 
 public:
 	virtual void Flush() = 0;
+	virtual void Clear() {
+	}
 };
 
 struct PartialBlockAllocation {
@@ -194547,9 +194563,8 @@ public:
 
 public:
 	PartialBlockManager(BlockManager &block_manager, uint32_t max_partial_block_size = DEFAULT_MAX_PARTIAL_BLOCK_SIZE,
-	                    uint32_t max_use_count = DEFAULT_MAX_USE_COUNT)
-	    : block_manager(block_manager), max_partial_block_size(max_partial_block_size), max_use_count(max_use_count) {
-	}
+	                    uint32_t max_use_count = DEFAULT_MAX_USE_COUNT);
+	virtual ~PartialBlockManager();
 
 public:
 	//! Flush any remaining partial blocks to disk
@@ -194562,6 +194577,9 @@ public:
 	//! Register a partially filled block that is filled with "segment_size" entries
 	void RegisterPartialBlock(PartialBlockAllocation &&allocation);
 
+	//! Clears all blocks
+	void Clear();
+
 protected:
 	BlockManager &block_manager;
 	//! A map of (available space -> PartialBlock) for partially filled blocks
@@ -194711,6 +194729,10 @@ public:
 	void RegisterPartialBlock(PartialBlockAllocation &&allocation);
 	PartialBlockAllocation GetBlockAllocation(uint32_t segment_size);
 
+	PartialBlockManager &GetPartialBlockManager() {
+		return partial_block_manager;
+	}
+
 protected:
 	TableCatalogEntry &table;
 	PartialBlockManager &partial_block_manager;
@@ -194806,28 +194828,33 @@ private:
 
 
 
+
 namespace duckdb {
 class ColumnData;
 class DatabaseInstance;
 class RowGroup;
+class PartialBlockManager;
 class TableDataWriter;
 
 struct ColumnCheckpointState {
-	ColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, RowGroupWriter &writer);
+	ColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, PartialBlockManager &partial_block_manager);
 	virtual ~ColumnCheckpointState();
 
 	RowGroup &row_group;
 	ColumnData &column_data;
-	RowGroupWriter &writer;
 	SegmentTree new_tree;
 	vector<DataPointer> data_pointers;
 	unique_ptr<BaseStatistics> global_stats;
 
+protected:
+	PartialBlockManager &partial_block_manager;
+
 public:
 	virtual unique_ptr<BaseStatistics> GetStatistics();
 
 	virtual void FlushSegment(unique_ptr<ColumnSegment> segment, idx_t segment_size);
-	virtual void WriteDataPointers();
+	virtual void WriteDataPointers(RowGroupWriter &writer);
+	virtual void GetBlockIds(unordered_set<block_id_t> &result);
 };
 
 } // namespace duckdb
@@ -197104,6 +197131,8 @@ public:
 	DataTableInfo &GetTableInfo() const;
 	virtual idx_t GetMaxEntry();
 
+	void IncrementVersion();
+
 	//! The root type of the column
 	const LogicalType &RootType() const;
 
@@ -197149,9 +197178,10 @@ public:
 
 	virtual void CommitDropColumn();
 
-	virtual unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group, RowGroupWriter &writer);
-	virtual unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, RowGroupWriter &writer,
-	                                                     ColumnCheckpointInfo &checkpoint_info);
+	virtual unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group,
+	                                                                PartialBlockManager &partial_block_manager);
+	virtual unique_ptr<ColumnCheckpointState>
+	Checkpoint(RowGroup &row_group, PartialBlockManager &partial_block_manager, ColumnCheckpointInfo &checkpoint_info);
 
 	virtual void CheckpointScan(ColumnSegment *segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
 	                            Vector &scan_vector);
@@ -197190,6 +197220,8 @@ protected:
 	mutex update_lock;
 	//! The updates for this column segment
 	unique_ptr<UpdateSegment> updates;
+	//! The internal version of the column data
+	idx_t version;
 };
 
 } // namespace duckdb
@@ -200657,12 +200689,14 @@ idx_t ValidityFinalAnalyze(AnalyzeState &state_p) {
 //===--------------------------------------------------------------------===//
 struct ValidityScanState : public SegmentScanState {
 	BufferHandle handle;
+	block_id_t block_id;
 };
 
 unique_ptr<SegmentScanState> ValidityInitScan(ColumnSegment &segment) {
 	auto result = make_unique<ValidityScanState>();
 	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
 	result->handle = buffer_manager.Pin(segment.block);
+	result->block_id = segment.block->BlockId();
 	return move(result);
 }
 
@@ -200678,6 +200712,7 @@ void ValidityScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t s
 
 	auto &result_mask = FlatVector::Validity(result);
 	auto buffer_ptr = scan_state.handle.Ptr() + segment.GetBlockOffset();
+	D_ASSERT(scan_state.block_id == segment.block->BlockId());
 	auto input_data = (validity_t *)buffer_ptr;
 
 #ifdef DEBUG
@@ -200803,6 +200838,7 @@ void ValidityScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_cou
 		// it is not required for correctness
 		auto &result_mask = FlatVector::Validity(result);
 		auto buffer_ptr = scan_state.handle.Ptr() + segment.GetBlockOffset();
+		D_ASSERT(scan_state.block_id == segment.block->BlockId());
 		auto input_data = (validity_t *)buffer_ptr;
 		auto result_data = (validity_t *)result_mask.GetData();
 		idx_t start_offset = start / ValidityMask::BITS_PER_VALUE;
@@ -201019,8 +201055,9 @@ public:
 
 	void CommitDropColumn() override;
 
-	unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group, RowGroupWriter &writer) override;
-	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, RowGroupWriter &writer,
+	unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group,
+	                                                        PartialBlockManager &partial_block_manager) override;
+	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, PartialBlockManager &partial_block_manager,
 	                                             ColumnCheckpointInfo &checkpoint_info) override;
 	void CheckpointScan(ColumnSegment *segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
 	                    Vector &scan_vector) override;
@@ -201080,11 +201117,11 @@ DataTable::DataTable(ClientContext &context, DataTable &parent, ColumnDefinition
 	// add the column definitions from this DataTable
 	column_definitions.emplace_back(new_column.Copy());
 
-	auto &transaction = Transaction::GetTransaction(context);
 	this->row_groups = parent.row_groups->AddColumn(new_column, default_value, stats.GetStats(new_column_idx));
 
 	// also add this column to client local storage
-	transaction.storage.AddColumn(&parent, this, new_column, default_value);
+	auto &local_storage = LocalStorage::Get(context);
+	local_storage.AddColumn(&parent, this, new_column, default_value);
 
 	// this table replaces the previous table, hence the parent is no longer the root DataTable
 	parent.is_root = false;
@@ -201131,8 +201168,8 @@ DataTable::DataTable(ClientContext &context, DataTable &parent, idx_t removed_co
 	this->row_groups = parent.row_groups->RemoveColumn(removed_column);
 
 	// scan the original table, and fill the new column with the transformed value
-	auto &transaction = Transaction::GetTransaction(context);
-	transaction.storage.DropColumn(&parent, this, removed_column);
+	auto &local_storage = LocalStorage::Get(context);
+	local_storage.DropColumn(&parent, this, removed_column);
 
 	// this table replaces the previous table, hence the parent is no longer the root DataTable
 	parent.is_root = false;
@@ -201152,8 +201189,8 @@ DataTable::DataTable(ClientContext &context, DataTable &parent, unique_ptr<Bound
 	VerifyNewConstraint(context, parent, constraint.get());
 
 	// Get the local data ownership from old dt
-	auto &transaction = Transaction::GetTransaction(context);
-	transaction.storage.MoveStorage(&parent, this);
+	auto &local_storage = LocalStorage::Get(context);
+	local_storage.MoveStorage(&parent, this);
 	// this table replaces the previous table, hence the parent is no longer the root DataTable
 	parent.is_root = false;
 }
@@ -201186,8 +201223,8 @@ DataTable::DataTable(ClientContext &context, DataTable &parent, idx_t changed_id
 	    parent.row_groups->AlterType(changed_idx, target_type, bound_columns, cast_expr, stats.GetStats(changed_idx));
 
 	// scan the original table, and fill the new column with the transformed value
-	auto &transaction = Transaction::GetTransaction(context);
-	transaction.storage.ChangeType(&parent, this, changed_idx, target_type, bound_columns, cast_expr);
+	auto &local_storage = LocalStorage::Get(context);
+	local_storage.ChangeType(&parent, this, changed_idx, target_type, bound_columns, cast_expr);
 
 	// this table replaces the previous table, hence the parent is no longer the root DataTable
 	parent.is_root = false;
@@ -201217,7 +201254,8 @@ void DataTable::InitializeScan(TableScanState &state, const vector<column_t> &co
 void DataTable::InitializeScan(Transaction &transaction, TableScanState &state, const vector<column_t> &column_ids,
                                TableFilterSet *table_filters) {
 	InitializeScan(state, column_ids, table_filters);
-	transaction.storage.InitializeScan(this, state.local_state, table_filters);
+	auto &local_storage = LocalStorage::Get(transaction);
+	local_storage.InitializeScan(this, state.local_state, table_filters);
 }
 
 void DataTable::InitializeScanWithOffset(TableScanState &state, const vector<column_t> &column_ids, idx_t start_row,
@@ -201237,16 +201275,17 @@ idx_t DataTable::MaxThreads(ClientContext &context) {
 
 void DataTable::InitializeParallelScan(ClientContext &context, ParallelTableScanState &state) {
 	row_groups->InitializeParallelScan(state.scan_state);
-	auto &transaction = Transaction::GetTransaction(context);
-	transaction.storage.InitializeParallelScan(this, state.local_state);
+
+	auto &local_storage = LocalStorage::Get(context);
+	local_storage.InitializeParallelScan(this, state.local_state);
 }
 
 bool DataTable::NextParallelScan(ClientContext &context, ParallelTableScanState &state, TableScanState &scan_state) {
 	if (row_groups->NextParallelScan(context, state.scan_state, scan_state.table_state)) {
 		return true;
 	}
-	auto &transaction = Transaction::GetTransaction(context);
-	if (transaction.storage.NextParallelScan(context, this, state.local_state, scan_state.local_state)) {
+	auto &local_storage = LocalStorage::Get(context);
+	if (local_storage.NextParallelScan(context, this, state.local_state, scan_state.local_state)) {
 		return true;
 	} else {
 		// finished all scans: no more scans remaining
@@ -201262,7 +201301,8 @@ void DataTable::Scan(Transaction &transaction, DataChunk &result, TableScanState
 	}
 
 	// scan the transaction-local segments
-	transaction.storage.Scan(state.local_state, state.GetColumnIds(), result);
+	auto &local_storage = LocalStorage::Get(transaction);
+	local_storage.Scan(state.local_state, state.GetColumnIds(), result);
 }
 
 bool DataTable::CreateIndexScan(TableScanState &state, DataChunk &result, TableScanType type) {
@@ -201384,10 +201424,10 @@ static void VerifyForeignKeyConstraint(const BoundForeignKeyConstraint &bfk, Cli
 
 	data_table->info->indexes.VerifyForeignKey(*dst_keys_ptr, is_append, dst_chunk, err_msgs);
 	// check whether or not the chunk can be inserted or deleted into the referenced table' transaction local storage
-	auto &transaction = Transaction::GetTransaction(context);
-	bool transaction_check = transaction.storage.Find(data_table);
+	auto &local_storage = LocalStorage::Get(context);
+	bool transaction_check = local_storage.Find(data_table);
 	if (transaction_check) {
-		auto &transact_index = transaction.storage.GetIndexes(data_table);
+		auto &transact_index = local_storage.GetIndexes(data_table);
 		transact_index.VerifyForeignKey(*dst_keys_ptr, is_append, dst_chunk, tran_err_msgs);
 	}
 
@@ -201439,8 +201479,8 @@ void DataTable::VerifyNewConstraint(ClientContext &context, DataTable &parent, c
 	}
 
 	parent.row_groups->VerifyNewConstraint(parent, *constraint);
-	auto &transaction = Transaction::GetTransaction(context);
-	transaction.storage.VerifyNewConstraint(parent, *constraint);
+	auto &local_storage = LocalStorage::Get(context);
+	local_storage.VerifyNewConstraint(parent, *constraint);
 }
 
 void DataTable::VerifyAppendConstraints(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk) {
@@ -201500,8 +201540,8 @@ void DataTable::InitializeLocalAppend(LocalAppendState &state, ClientContext &co
 	if (!is_root) {
 		throw TransactionException("Transaction conflict: adding entries to a table that has been altered!");
 	}
-	auto &transaction = Transaction::GetTransaction(context);
-	transaction.storage.InitializeAppend(state, this);
+	auto &local_storage = LocalStorage::Get(context);
+	local_storage.InitializeAppend(state, this);
 }
 
 void DataTable::LocalAppend(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context,
@@ -201761,6 +201801,7 @@ idx_t DataTable::Delete(TableCatalogEntry &table, ClientContext &context, Vector
 	}
 
 	auto &transaction = Transaction::GetTransaction(context);
+	auto &local_storage = LocalStorage::Get(context);
 
 	row_identifiers.Flatten(count);
 	auto ids = FlatVector::GetData<row_t>(row_identifiers);
@@ -201771,7 +201812,7 @@ idx_t DataTable::Delete(TableCatalogEntry &table, ClientContext &context, Vector
 	// and we only need to fetch columns that are part of this constraint
 	DataChunk verify_chunk;
 	if (first_id >= MAX_ROW_ID) {
-		transaction.storage.FetchChunk(this, row_identifiers, count, verify_chunk);
+		local_storage.FetchChunk(this, row_identifiers, count, verify_chunk);
 	} else {
 		ColumnFetchState fetch_state;
 		vector<column_t> col_ids;
@@ -201787,7 +201828,7 @@ idx_t DataTable::Delete(TableCatalogEntry &table, ClientContext &context, Vector
 
 	if (first_id >= MAX_ROW_ID) {
 		// deletion is in transaction-local storage: push delete into local chunk collection
-		return transaction.storage.Delete(this, row_identifiers, count);
+		return local_storage.Delete(this, row_identifiers, count);
 	} else {
 		return row_groups->Delete(transaction, this, ids, count);
 	}
@@ -201902,7 +201943,8 @@ void DataTable::Update(TableCatalogEntry &table, ClientContext &context, Vector
 	auto first_id = FlatVector::GetValue<row_t>(row_ids, 0);
 	if (first_id >= MAX_ROW_ID) {
 		// update is in transaction-local storage: push update into local storage
-		transaction.storage.Update(this, row_ids, column_ids, updates);
+		auto &local_storage = LocalStorage::Get(context);
+		local_storage.Update(this, row_ids, column_ids, updates);
 		return;
 	}
 
@@ -202099,8 +202141,13 @@ BlockPointer Index::Serialize(duckdb::MetaBlockWriter &writer) {
 
 
 
+
+
 namespace duckdb {
 
+//===--------------------------------------------------------------------===//
+// Local Table Storage
+//===--------------------------------------------------------------------===//
 LocalTableStorage::LocalTableStorage(DataTable &table)
     : table(table), allocator(Allocator::Get(table.db)), deleted_rows(0) {
 	auto types = table.GetTypes();
@@ -202127,7 +202174,11 @@ LocalTableStorage::LocalTableStorage(DataTable &table)
 LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &parent, idx_t changed_idx,
                                      const LogicalType &target_type, const vector<column_t> &bound_columns,
                                      Expression &cast_expr)
-    : table(new_dt), allocator(Allocator::Get(table.db)), deleted_rows(parent.deleted_rows) {
+    : table(new_dt), allocator(Allocator::Get(table.db)), deleted_rows(parent.deleted_rows),
+      partial_manager(move(parent.partial_manager)), written_blocks(move(parent.written_blocks)) {
+	if (partial_manager) {
+		partial_manager->FlushPartialBlocks();
+	}
 	stats.InitializeAlterType(parent.stats, changed_idx, target_type);
 	row_groups =
 	    parent.row_groups->AlterType(changed_idx, target_type, bound_columns, cast_expr, stats.GetStats(changed_idx));
@@ -202136,7 +202187,11 @@ LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &paren
 }
 
 LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &parent, idx_t drop_idx)
-    : table(new_dt), allocator(Allocator::Get(table.db)), deleted_rows(parent.deleted_rows) {
+    : table(new_dt), allocator(Allocator::Get(table.db)), deleted_rows(parent.deleted_rows),
+      partial_manager(move(parent.partial_manager)), written_blocks(move(parent.written_blocks)) {
+	if (partial_manager) {
+		partial_manager->FlushPartialBlocks();
+	}
 	stats.InitializeRemoveColumn(parent.stats, drop_idx);
 	row_groups = parent.row_groups->RemoveColumn(drop_idx);
 	parent.row_groups.reset();
@@ -202145,7 +202200,8 @@ LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &paren
 
 LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &parent, ColumnDefinition &new_column,
                                      Expression *default_value)
-    : table(new_dt), allocator(Allocator::Get(table.db)), deleted_rows(parent.deleted_rows) {
+    : table(new_dt), allocator(Allocator::Get(table.db)), deleted_rows(parent.deleted_rows),
+      partial_manager(move(parent.partial_manager)), written_blocks(move(parent.written_blocks)) {
 	idx_t new_column_idx = parent.table.column_definitions.size();
 	stats.InitializeAddColumn(parent.stats, new_column.GetType());
 	row_groups = parent.row_groups->AddColumn(new_column, default_value, stats.GetStats(new_column_idx));
@@ -202177,6 +202233,20 @@ idx_t LocalTableStorage::EstimatedSize() {
 	return appended_rows * row_size;
 }
 
+//===--------------------------------------------------------------------===//
+// LocalStorage
+//===--------------------------------------------------------------------===//
+LocalStorage::LocalStorage(Transaction &transaction) : transaction(transaction) {
+}
+
+LocalStorage &LocalStorage::Get(Transaction &transaction) {
+	return transaction.GetLocalStorage();
+}
+
+LocalStorage &LocalStorage::Get(ClientContext &context) {
+	return Transaction::GetTransaction(context).GetLocalStorage();
+}
+
 void LocalStorage::InitializeScan(DataTable *table, CollectionScanState &state, TableFilterSet *table_filters) {
 	auto entry = table_storage.find(table);
 	if (entry == table_storage.end()) {
@@ -202230,8 +202300,68 @@ void LocalStorage::Append(LocalAppendState &state, DataChunk &chunk) {
 		throw ConstraintException("PRIMARY KEY or UNIQUE constraint violated: duplicated key");
 	}
 
-	//! Append to the chunk
-	storage->row_groups->Append(chunk, state.append_state, storage->stats);
+	//! Append the chunk to the local storage
+	auto new_row_group = storage->row_groups->Append(chunk, state.append_state, storage->stats);
+
+	//! Check if we should pre-emptively flush blocks to disk
+	if (new_row_group) {
+		storage->CheckFlushToDisk();
+	}
+}
+
+void LocalTableStorage::CheckFlushToDisk() {
+	// we finished writing a complete row group
+	// check if we should pre-emptively write it to disk
+	if (table.info->IsTemporary() || StorageManager::GetStorageManager(table.db).InMemory()) {
+		return;
+	}
+	if (!table.info->indexes.Empty()) {
+		// we have indexes - we cannot merge
+		return;
+	}
+	if (deleted_rows != 0) {
+		// we have deletes - we cannot merge
+		return;
+	}
+	// we should! write the second-to-last row group to disk
+	// allocate the partial block-manager if none is allocated yet
+	if (!partial_manager) {
+		auto &block_manager = table.info->table_io_manager->GetBlockManagerForRowData();
+		partial_manager = make_unique<PartialBlockManager>(block_manager);
+	}
+	// flush second-to-last row group
+	auto row_group = row_groups->GetRowGroup(-2);
+	FlushToDisk(row_group);
+}
+
+void LocalTableStorage::FlushToDisk(RowGroup *row_group) {
+	// flush the specified row group
+	D_ASSERT(row_group);
+	D_ASSERT(deleted_rows == 0);
+	D_ASSERT(table.info->indexes.Empty());
+	D_ASSERT(partial_manager);
+	//! The set of column compression types (if any)
+	vector<CompressionType> compression_types;
+	D_ASSERT(compression_types.empty());
+	for (auto &column : table.column_definitions) {
+		compression_types.push_back(column.CompressionType());
+	}
+	auto row_group_pointer = row_group->WriteToDisk(*partial_manager, compression_types);
+	for (idx_t col_idx = 0; col_idx < row_group_pointer.statistics.size(); col_idx++) {
+		row_group_pointer.states[col_idx]->GetBlockIds(written_blocks);
+		stats.MergeStats(col_idx, *row_group_pointer.statistics[col_idx]);
+	}
+}
+void LocalTableStorage::FlushToDisk() {
+	// no partial manager - nothing to flush
+	if (!partial_manager) {
+		return;
+	}
+	// flush the last row group
+	FlushToDisk(row_groups->GetRowGroup(-1));
+	// then flush the partial manager
+	partial_manager->FlushPartialBlocks();
+	partial_manager.reset();
 }
 
 void LocalStorage::FinalizeAppend(LocalAppendState &state) {
@@ -202252,6 +202382,10 @@ idx_t LocalStorage::EstimatedSize() {
 	return estimated_size;
 }
 
+bool LocalTableStorage::HasWrittenBlocks() {
+	return partial_manager || !written_blocks.empty();
+}
+
 idx_t LocalStorage::Delete(DataTable *table, Vector &row_ids, idx_t count) {
 	auto storage = GetStorage(table);
 	D_ASSERT(storage);
@@ -202319,10 +202453,16 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) {
 	table.AppendLock(append_state);
 	if ((append_state.row_start == 0 || storage.row_groups->GetTotalRows() >= MERGE_THRESHOLD) &&
 	    storage.table.info->indexes.Empty() && storage.deleted_rows == 0) {
-		// table is currently empty OR we are bulk appending to a table with existing storage: move over the storage
-		// directly
+		// table is currently empty OR we are bulk appending: move over the storage directly
+		// first flush any out-standing storage nodes
+		storage.FlushToDisk();
 		table.MergeStorage(*storage.row_groups, storage.indexes, storage.stats);
 	} else {
+		if (storage.partial_manager || !storage.written_blocks.empty()) {
+			// we have written data but cannot merge to disk after all
+			// revert the data we have already written
+			storage.Rollback();
+		}
 		bool constraint_violated = false;
 		table.InitializeAppend(transaction, append_state, append_count);
 		ScanTableStorage(table, storage, [&](DataChunk &chunk) -> bool {
@@ -202368,6 +202508,27 @@ void LocalStorage::Commit(LocalStorage::CommitState &commit_state, Transaction &
 	table_storage.clear();
 }
 
+void LocalStorage::Rollback() {
+	for (auto &entry : table_storage) {
+		auto storage = entry.second.get();
+		if (!storage) {
+			continue;
+		}
+		storage->Rollback();
+	}
+}
+
+void LocalTableStorage::Rollback() {
+	if (partial_manager) {
+		partial_manager->Clear();
+		partial_manager.reset();
+	}
+	auto &block_manager = table.info->table_io_manager->GetBlockManagerForRowData();
+	for (auto block_id : written_blocks) {
+		block_manager.MarkBlockAsModified(block_id);
+	}
+}
+
 idx_t LocalStorage::AddedRows(DataTable *table) {
 	auto entry = table_storage.find(table);
 	if (entry == table_storage.end()) {
@@ -202593,6 +202754,12 @@ void MetaBlockWriter::WriteData(const_data_ptr_t buffer, idx_t write_size) {
 
 namespace duckdb {
 
+PartialBlockManager::PartialBlockManager(BlockManager &block_manager, uint32_t max_partial_block_size,
+                                         uint32_t max_use_count)
+    : block_manager(block_manager), max_partial_block_size(max_partial_block_size), max_use_count(max_use_count) {
+}
+PartialBlockManager::~PartialBlockManager() {
+}
 //===--------------------------------------------------------------------===//
 // Partial Blocks
 //===--------------------------------------------------------------------===//
@@ -202667,6 +202834,14 @@ void PartialBlockManager::FlushPartialBlocks() {
 	for (auto &e : partially_filled_blocks) {
 		e.second->Flush();
 	}
+	partially_filled_blocks.clear();
+}
+
+void PartialBlockManager::Clear() {
+	for (auto &e : partially_filled_blocks) {
+		e.second->Clear();
+	}
+	partially_filled_blocks.clear();
 }
 
 } // namespace duckdb
@@ -202719,13 +202894,9 @@ public:
 	void WriteHeader(DatabaseHeader header) override;
 
 	//! Returns the number of total blocks
-	idx_t TotalBlocks() override {
-		return max_block;
-	}
+	idx_t TotalBlocks() override;
 	//! Returns the number of free blocks
-	idx_t FreeBlocks() override {
-		return free_list.size();
-	}
+	idx_t FreeBlocks() override;
 
 private:
 	//! Load the free list from the file
@@ -202766,6 +202937,8 @@ private:
 	bool read_only;
 	//! Whether or not to use Direct IO to read the blocks
 	bool use_direct_io;
+	//! Lock for performing various operations in the single file block manager
+	mutex block_lock;
 };
 } // namespace duckdb
 
@@ -202990,6 +203163,7 @@ bool SingleFileBlockManager::IsRootBlock(block_id_t root) {
 }
 
 block_id_t SingleFileBlockManager::GetFreeBlockId() {
+	lock_guard<mutex> lock(block_lock);
 	block_id_t block;
 	if (!free_list.empty()) {
 		// free list is non empty
@@ -203004,6 +203178,7 @@ block_id_t SingleFileBlockManager::GetFreeBlockId() {
 }
 
 void SingleFileBlockManager::MarkBlockAsModified(block_id_t block_id) {
+	lock_guard<mutex> lock(block_lock);
 	D_ASSERT(block_id >= 0);
 
 	// check if the block is a multi-use block
@@ -203026,6 +203201,7 @@ void SingleFileBlockManager::MarkBlockAsModified(block_id_t block_id) {
 }
 
 void SingleFileBlockManager::IncreaseBlockReferenceCount(block_id_t block_id) {
+	lock_guard<mutex> lock(block_lock);
 	D_ASSERT(free_list.find(block_id) == free_list.end());
 	auto entry = multi_use_blocks.find(block_id);
 	if (entry != multi_use_blocks.end()) {
@@ -203039,6 +203215,16 @@ block_id_t SingleFileBlockManager::GetMetaBlock() {
 	return meta_block;
 }
 
+idx_t SingleFileBlockManager::TotalBlocks() {
+	lock_guard<mutex> lock(block_lock);
+	return max_block;
+}
+
+idx_t SingleFileBlockManager::FreeBlocks() {
+	lock_guard<mutex> lock(block_lock);
+	return free_list.size();
+}
+
 unique_ptr<Block> SingleFileBlockManager::CreateBlock(block_id_t block_id, FileBuffer *source_buffer) {
 	if (source_buffer) {
 		D_ASSERT(source_buffer->AllocSize() == Storage::BLOCK_ALLOC_SIZE);
@@ -204894,8 +205080,9 @@ public:
 
 	void CommitDropColumn() override;
 
-	unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group, RowGroupWriter &writer) override;
-	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, RowGroupWriter &writer,
+	unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group,
+	                                                        PartialBlockManager &partial_block_manager) override;
+	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, PartialBlockManager &partial_block_manager,
 	                                             ColumnCheckpointInfo &checkpoint_info) override;
 
 	void DeserializeColumn(Deserializer &source) override;
@@ -204916,8 +205103,9 @@ private:
 
 namespace duckdb {
 
-ColumnCheckpointState::ColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, RowGroupWriter &writer)
-    : row_group(row_group), column_data(column_data), writer(writer) {
+ColumnCheckpointState::ColumnCheckpointState(RowGroup &row_group, ColumnData &column_data,
+                                             PartialBlockManager &partial_block_manager)
+    : row_group(row_group), column_data(column_data), partial_block_manager(partial_block_manager) {
 }
 
 ColumnCheckpointState::~ColumnCheckpointState() {
@@ -204930,13 +205118,15 @@ unique_ptr<BaseStatistics> ColumnCheckpointState::GetStatistics() {
 
 struct PartialBlockForCheckpoint : PartialBlock {
 	struct PartialColumnSegment {
+		ColumnData *data;
 		ColumnSegment *segment;
 		uint32_t offset_in_block;
 	};
 
 public:
-	PartialBlockForCheckpoint(ColumnSegment *first_segment, BlockManager &block_manager, PartialBlockState state)
-	    : PartialBlock(state), first_segment(first_segment), block_manager(block_manager) {
+	PartialBlockForCheckpoint(ColumnData *first_data, ColumnSegment *first_segment, BlockManager &block_manager,
+	                          PartialBlockState state)
+	    : PartialBlock(state), first_data(first_data), first_segment(first_segment), block_manager(block_manager) {
 	}
 
 	~PartialBlockForCheckpoint() override {
@@ -204949,6 +205139,7 @@ public:
 	// the block to get written to storage (via BlockManger::ConvertToPersistent),
 	// and all segments to have their references updated
 	// (via ColumnSegment::ConvertToPersistent)
+	ColumnData *first_data;
 	ColumnSegment *first_segment;
 	BlockManager &block_manager;
 	vector<PartialColumnSegment> tail_segments;
@@ -204963,18 +205154,26 @@ public:
 		// At this point, we've already copied all data from tail_segments
 		// into the page owned by first_segment. We flush all segment data to
 		// disk with the following call.
+		first_data->IncrementVersion();
 		first_segment->ConvertToPersistent(&block_manager, state.block_id);
 		// Now that the page is persistent, update tail_segments to point to the
 		// newly persistent block.
 		for (auto e : tail_segments) {
+			e.data->IncrementVersion();
 			e.segment->MarkAsPersistent(first_segment->block, e.offset_in_block);
 		}
 		first_segment = nullptr;
 		tail_segments.clear();
 	}
 
-	void AddSegmentToTail(ColumnSegment *segment, uint32_t offset_in_block) {
-		tail_segments.push_back({segment, offset_in_block});
+	void Clear() override {
+		first_data = nullptr;
+		first_segment = nullptr;
+		tail_segments.clear();
+	}
+
+	void AddSegmentToTail(ColumnData *data, ColumnSegment *segment, uint32_t offset_in_block) {
+		tail_segments.push_back({data, segment, offset_in_block});
 	}
 };
 
@@ -204996,7 +205195,7 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 
 	if (!segment->stats.statistics->IsConstant()) {
 		// non-constant block
-		PartialBlockAllocation allocation = writer.GetBlockAllocation(segment_size);
+		PartialBlockAllocation allocation = partial_block_manager.GetBlockAllocation(segment_size);
 		block_id = allocation.state.block_id;
 		offset_in_block = allocation.state.offset_in_block;
 
@@ -205010,15 +205209,15 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 			auto new_handle = buffer_manager.Pin(pstate->first_segment->block);
 			// memcpy the contents of the old block to the new block
 			memcpy(new_handle.Ptr() + offset_in_block, old_handle.Ptr(), segment_size);
-			pstate->AddSegmentToTail(segment.get(), offset_in_block);
+			pstate->AddSegmentToTail(&column_data, segment.get(), offset_in_block);
 		} else {
 			// Create a new block for future reuse.
 			D_ASSERT(offset_in_block == 0);
-			allocation.partial_block =
-			    make_unique<PartialBlockForCheckpoint>(segment.get(), *allocation.block_manager, allocation.state);
+			allocation.partial_block = make_unique<PartialBlockForCheckpoint>(
+			    &column_data, segment.get(), *allocation.block_manager, allocation.state);
 		}
 		// Writer will decide whether to reuse this block.
-		writer.RegisterPartialBlock(move(allocation));
+		partial_block_manager.RegisterPartialBlock(move(allocation));
 	} else {
 		// constant block: no need to write anything to disk besides the stats
 		// set up the compression function to constant
@@ -205046,10 +205245,19 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 	data_pointers.push_back(move(data_pointer));
 }
 
-void ColumnCheckpointState::WriteDataPointers() {
+void ColumnCheckpointState::WriteDataPointers(RowGroupWriter &writer) {
 	writer.WriteColumnDataPointers(*this);
 }
 
+void ColumnCheckpointState::GetBlockIds(unordered_set<block_id_t> &result) {
+	for (auto &pointer : data_pointers) {
+		if (pointer.block_pointer.block_id == INVALID_BLOCK) {
+			continue;
+		}
+		result.insert(pointer.block_pointer.block_id);
+	}
+}
+
 } // namespace duckdb
 
 
@@ -205119,8 +205327,9 @@ public:
 
 	void CommitDropColumn() override;
 
-	unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group, RowGroupWriter &writer) override;
-	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, RowGroupWriter &writer,
+	unique_ptr<ColumnCheckpointState> CreateCheckpointState(RowGroup &row_group,
+	                                                        PartialBlockManager &partial_block_manager) override;
+	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, PartialBlockManager &partial_block_manager,
 	                                             ColumnCheckpointInfo &checkpoint_info) override;
 
 	void DeserializeColumn(Deserializer &source) override;
@@ -205247,12 +205456,12 @@ namespace duckdb {
 ColumnData::ColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
                        LogicalType type, ColumnData *parent)
     : block_manager(block_manager), info(info), column_index(column_index), start(start_row), type(move(type)),
-      parent(parent) {
+      parent(parent), version(0) {
 }
 
 ColumnData::ColumnData(ColumnData &other, idx_t start, ColumnData *parent)
     : block_manager(other.block_manager), info(other.info), column_index(other.column_index), start(start),
-      type(move(other.type)), parent(parent), updates(move(other.updates)) {
+      type(move(other.type)), parent(parent), updates(move(other.updates)), version(parent ? parent->version + 1 : 0) {
 	idx_t offset = 0;
 	for (auto segment = other.data.GetRootSegment(); segment; segment = segment->next.get()) {
 		auto &other = (ColumnSegment &)*segment;
@@ -205279,6 +205488,10 @@ const LogicalType &ColumnData::RootType() const {
 	return type;
 }
 
+void ColumnData::IncrementVersion() {
+	version++;
+}
+
 idx_t ColumnData::GetMaxEntry() {
 	auto first_segment = data.GetRootSegment();
 	auto last_segment = data.GetLastSegment();
@@ -205296,6 +205509,8 @@ void ColumnData::InitializeScan(ColumnScanState &state) {
 	state.row_index = state.current ? state.current->start : 0;
 	state.internal_index = state.row_index;
 	state.initialized = false;
+	state.version = version;
+	state.scan_state.reset();
 }
 
 void ColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) {
@@ -205303,15 +205518,23 @@ void ColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx)
 	state.row_index = row_idx;
 	state.internal_index = state.current->start;
 	state.initialized = false;
+	state.version = version;
+	state.scan_state.reset();
 }
 
 idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remaining) {
-	if (!state.initialized) {
+	if (state.version != version) {
+		InitializeScanWithOffset(state, state.row_index);
+		state.current->InitializeScan(state);
+		state.initialized = true;
+	} else if (!state.initialized) {
 		D_ASSERT(state.current);
 		state.current->InitializeScan(state);
 		state.internal_index = state.current->start;
 		state.initialized = true;
 	}
+	D_ASSERT(data.HasSegment(state.current));
+	D_ASSERT(state.version == version);
 	D_ASSERT(state.internal_index <= state.row_index);
 	if (state.internal_index < state.row_index) {
 		state.current->Skip(state);
@@ -205563,8 +205786,9 @@ void ColumnData::CommitDropColumn() {
 	}
 }
 
-unique_ptr<ColumnCheckpointState> ColumnData::CreateCheckpointState(RowGroup &row_group, RowGroupWriter &writer) {
-	return make_unique<ColumnCheckpointState>(row_group, *this, writer);
+unique_ptr<ColumnCheckpointState> ColumnData::CreateCheckpointState(RowGroup &row_group,
+                                                                    PartialBlockManager &partial_block_manager) {
+	return make_unique<ColumnCheckpointState>(row_group, *this, partial_block_manager);
 }
 
 void ColumnData::CheckpointScan(ColumnSegment *segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
@@ -205576,11 +205800,12 @@ void ColumnData::CheckpointScan(ColumnSegment *segment, ColumnScanState &state,
 	}
 }
 
-unique_ptr<ColumnCheckpointState> ColumnData::Checkpoint(RowGroup &row_group, RowGroupWriter &writer,
+unique_ptr<ColumnCheckpointState> ColumnData::Checkpoint(RowGroup &row_group,
+                                                         PartialBlockManager &partial_block_manager,
                                                          ColumnCheckpointInfo &checkpoint_info) {
 	// scan the segments of the column data
 	// set up the checkpoint state
-	auto checkpoint_state = CreateCheckpointState(row_group, writer);
+	auto checkpoint_state = CreateCheckpointState(row_group, partial_block_manager);
 	checkpoint_state->global_stats = BaseStatistics::CreateEmpty(type, StatisticsType::LOCAL_STATS);
 
 	if (!data.root_node) {
@@ -205594,6 +205819,7 @@ unique_ptr<ColumnCheckpointState> ColumnData::Checkpoint(RowGroup &row_group, Ro
 
 	// replace the old tree with the new one
 	data.Replace(checkpoint_state->new_tree);
+	version++;
 
 	return checkpoint_state;
 }
@@ -206166,6 +206392,7 @@ void ColumnSegment::ConvertToPersistent(BlockManager *block_manager, block_id_t
 	block_id = block_id_p;
 	offset = 0;
 
+	D_ASSERT(stats.statistics);
 	if (block_id == INVALID_BLOCK) {
 		// constant block: reset the block buffer
 		D_ASSERT(stats.statistics->IsConstant());
@@ -206797,8 +207024,8 @@ void ListColumnData::CommitDropColumn() {
 }
 
 struct ListColumnCheckpointState : public ColumnCheckpointState {
-	ListColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, RowGroupWriter &writer)
-	    : ColumnCheckpointState(row_group, column_data, writer) {
+	ListColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, PartialBlockManager &partial_block_manager)
+	    : ColumnCheckpointState(row_group, column_data, partial_block_manager) {
 		global_stats = make_unique<ListStatistics>(column_data.type);
 	}
 
@@ -206814,22 +207041,29 @@ public:
 		return stats;
 	}
 
-	void WriteDataPointers() override {
-		ColumnCheckpointState::WriteDataPointers();
-		validity_state->WriteDataPointers();
-		child_state->WriteDataPointers();
+	void WriteDataPointers(RowGroupWriter &writer) override {
+		ColumnCheckpointState::WriteDataPointers(writer);
+		validity_state->WriteDataPointers(writer);
+		child_state->WriteDataPointers(writer);
+	}
+	void GetBlockIds(unordered_set<block_id_t> &result) override {
+		ColumnCheckpointState::GetBlockIds(result);
+		validity_state->GetBlockIds(result);
+		child_state->GetBlockIds(result);
 	}
 };
 
-unique_ptr<ColumnCheckpointState> ListColumnData::CreateCheckpointState(RowGroup &row_group, RowGroupWriter &writer) {
-	return make_unique<ListColumnCheckpointState>(row_group, *this, writer);
+unique_ptr<ColumnCheckpointState> ListColumnData::CreateCheckpointState(RowGroup &row_group,
+                                                                        PartialBlockManager &partial_block_manager) {
+	return make_unique<ListColumnCheckpointState>(row_group, *this, partial_block_manager);
 }
 
-unique_ptr<ColumnCheckpointState> ListColumnData::Checkpoint(RowGroup &row_group, RowGroupWriter &writer,
+unique_ptr<ColumnCheckpointState> ListColumnData::Checkpoint(RowGroup &row_group,
+                                                             PartialBlockManager &partial_block_manager,
                                                              ColumnCheckpointInfo &checkpoint_info) {
-	auto validity_state = validity.Checkpoint(row_group, writer, checkpoint_info);
-	auto base_state = ColumnData::Checkpoint(row_group, writer, checkpoint_info);
-	auto child_state = child_column->Checkpoint(row_group, writer, checkpoint_info);
+	auto validity_state = validity.Checkpoint(row_group, partial_block_manager, checkpoint_info);
+	auto base_state = ColumnData::Checkpoint(row_group, partial_block_manager, checkpoint_info);
+	auto child_state = child_column->Checkpoint(row_group, partial_block_manager, checkpoint_info);
 
 	auto &checkpoint_state = (ListColumnCheckpointState &)*base_state;
 	checkpoint_state.validity_state = move(validity_state);
@@ -207545,10 +207779,11 @@ void RowGroup::MergeIntoStatistics(idx_t column_idx, BaseStatistics &other) {
 	other.Merge(*stats[column_idx]->statistics);
 }
 
-RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, vector<unique_ptr<BaseStatistics>> &global_stats) {
-	RowGroupPointer row_group_pointer;
-	vector<unique_ptr<ColumnCheckpointState>> states;
-	states.reserve(columns.size());
+RowGroupWriteData RowGroup::WriteToDisk(PartialBlockManager &manager,
+                                        const vector<CompressionType> &compression_types) {
+	RowGroupWriteData result;
+	result.states.reserve(columns.size());
+	result.statistics.reserve(columns.size());
 
 	// Checkpoint the individual columns of the row group
 	// Here we're iterating over columns. Each column can have multiple segments.
@@ -207560,23 +207795,39 @@ RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, vector<unique_ptr<B
 	// pointers all end up densely packed, and thus more cache-friendly.
 	for (idx_t column_idx = 0; column_idx < columns.size(); column_idx++) {
 		auto &column = columns[column_idx];
-		ColumnCheckpointInfo checkpoint_info {writer.GetColumnCompressionType(column_idx)};
-		auto checkpoint_state = column->Checkpoint(*this, writer, checkpoint_info);
+		ColumnCheckpointInfo checkpoint_info {compression_types[column_idx]};
+		auto checkpoint_state = column->Checkpoint(*this, manager, checkpoint_info);
 		D_ASSERT(checkpoint_state);
 
 		auto stats = checkpoint_state->GetStatistics();
 		D_ASSERT(stats);
 
-		global_stats[column_idx]->Merge(*stats);
-		row_group_pointer.statistics.push_back(move(stats));
-		states.push_back(move(checkpoint_state));
+		result.statistics.push_back(move(stats));
+		result.states.push_back(move(checkpoint_state));
 	}
+	D_ASSERT(result.states.size() == result.statistics.size());
+	return result;
+}
+
+RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, vector<unique_ptr<BaseStatistics>> &global_stats) {
+	RowGroupPointer row_group_pointer;
+
+	vector<CompressionType> compression_types;
+	compression_types.reserve(columns.size());
+	for (idx_t column_idx = 0; column_idx < columns.size(); column_idx++) {
+		compression_types.push_back(writer.GetColumnCompressionType(column_idx));
+	}
+	auto result = WriteToDisk(writer.GetPartialBlockManager(), compression_types);
+	for (idx_t column_idx = 0; column_idx < columns.size(); column_idx++) {
+		global_stats[column_idx]->Merge(*result.statistics[column_idx]);
+	}
+	row_group_pointer.statistics = move(result.statistics);
 
 	// construct the row group pointer and write the column meta data to disk
-	D_ASSERT(states.size() == columns.size());
+	D_ASSERT(result.states.size() == columns.size());
 	row_group_pointer.row_start = start;
 	row_group_pointer.tuple_count = count;
-	for (auto &state : states) {
+	for (auto &state : result.states) {
 		// get the current position of the table data writer
 		auto &data_writer = writer.GetPayloadWriter();
 		auto pointer = data_writer.GetBlockPointer();
@@ -207588,7 +207839,7 @@ RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, vector<unique_ptr<B
 		//
 		// Just as above, the state can refer to many other states, so this
 		// can cascade recursively into more pointer writes.
-		state->WriteDataPointers();
+		state->WriteDataPointers(writer);
 	}
 	row_group_pointer.versions = version_info;
 	Verify();
@@ -207849,6 +208100,10 @@ void RowGroupCollection::AppendRowGroup(idx_t start_row) {
 	row_groups->AppendSegment(move(new_row_group));
 }
 
+RowGroup *RowGroupCollection::GetRowGroup(int64_t index) {
+	return (RowGroup *)row_groups->GetSegmentByIndex(index);
+}
+
 void RowGroupCollection::Verify() {
 #ifdef DEBUG
 	idx_t current_total_rows = 0;
@@ -207999,10 +208254,11 @@ void RowGroupCollection::InitializeAppend(TableAppendState &state) {
 	InitializeAppend(tdata, state, 0);
 }
 
-void RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state, TableStatistics &stats) {
+bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state, TableStatistics &stats) {
 	D_ASSERT(chunk.ColumnCount() == types.size());
 	chunk.Verify();
 
+	bool new_row_group = false;
 	idx_t append_count = chunk.size();
 	idx_t remaining = chunk.size();
 	state.total_append_count += append_count;
@@ -208036,6 +208292,7 @@ void RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state, Table
 				chunk.Slice(sel, remaining);
 			}
 			// append a new row_group
+			new_row_group = true;
 			auto next_start = current_row_group->start + state.row_group_append_state.offset_in_row_group;
 			AppendRowGroup(next_start);
 			// set up the append state for this row_group
@@ -208059,6 +208316,7 @@ void RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state, Table
 		}
 		stats.GetStats(col_idx).stats->UpdateDistinctStatistics(chunk.data[col_idx], chunk.size());
 	}
+	return new_row_group;
 }
 
 void RowGroupCollection::FinalizeAppend(TransactionData transaction, TableAppendState &state) {
@@ -208099,12 +208357,7 @@ void RowGroupCollection::CommitAppend(transaction_t commit_id, idx_t row_start,
 
 void RowGroupCollection::RevertAppendInternal(idx_t start_row, idx_t count) {
 	if (total_rows != start_row + count) {
-		// interleaved append: don't do anything
-		// in this case the rows will stay as "inserted by transaction X", but will never be committed
-		// they will never be used by any other transaction and will essentially leave a gap
-		// this situation is rare, and as such we don't care about optimizing it (yet?)
-		// it only happens if C1 appends a lot of data -> C2 appends a lot of data -> C1 rolls back
-		return;
+		throw InternalException("Interleaved appends: this should no longer happen");
 	}
 	total_rows = start_row;
 
@@ -208559,6 +208812,21 @@ SegmentBase *SegmentTree::GetRootSegment() {
 	return root_node.get();
 }
 
+SegmentBase *SegmentTree::GetSegmentByIndex(int64_t index) {
+	if (index < 0) {
+		index = nodes.size() + index;
+		if (index < 0) {
+			return nullptr;
+		}
+		return nodes[index].node;
+	} else {
+		if (idx_t(index) >= nodes.size()) {
+			return nullptr;
+		}
+		return nodes[index].node;
+	}
+}
+
 SegmentBase *SegmentTree::GetLastSegment() {
 	return nodes.empty() ? nullptr : nodes.back().node;
 }
@@ -208591,6 +208859,16 @@ idx_t SegmentTree::GetSegmentIndex(idx_t row_number) {
 	throw InternalException("Could not find node in column segment tree!");
 }
 
+bool SegmentTree::HasSegment(SegmentBase *segment) {
+	lock_guard<mutex> tree_lock(node_lock);
+	for (auto &node : nodes) {
+		if (node.node == segment) {
+			return true;
+		}
+	}
+	return false;
+}
+
 void SegmentTree::AppendSegment(unique_ptr<SegmentBase> segment) {
 	D_ASSERT(segment);
 	// add the node to the list of nodes
@@ -208775,8 +209053,9 @@ void StandardColumnData::CommitDropColumn() {
 }
 
 struct StandardColumnCheckpointState : public ColumnCheckpointState {
-	StandardColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, RowGroupWriter &writer)
-	    : ColumnCheckpointState(row_group, column_data, writer) {
+	StandardColumnCheckpointState(RowGroup &row_group, ColumnData &column_data,
+	                              PartialBlockManager &partial_block_manager)
+	    : ColumnCheckpointState(row_group, column_data, partial_block_manager) {
 	}
 
 	unique_ptr<ColumnCheckpointState> validity_state;
@@ -208788,21 +209067,27 @@ public:
 		return move(global_stats);
 	}
 
-	void WriteDataPointers() override {
-		ColumnCheckpointState::WriteDataPointers();
-		validity_state->WriteDataPointers();
+	void WriteDataPointers(RowGroupWriter &writer) override {
+		ColumnCheckpointState::WriteDataPointers(writer);
+		validity_state->WriteDataPointers(writer);
+	}
+
+	void GetBlockIds(unordered_set<block_id_t> &result) override {
+		ColumnCheckpointState::GetBlockIds(result);
+		validity_state->GetBlockIds(result);
 	}
 };
 
-unique_ptr<ColumnCheckpointState> StandardColumnData::CreateCheckpointState(RowGroup &row_group,
-                                                                            RowGroupWriter &writer) {
-	return make_unique<StandardColumnCheckpointState>(row_group, *this, writer);
+unique_ptr<ColumnCheckpointState>
+StandardColumnData::CreateCheckpointState(RowGroup &row_group, PartialBlockManager &partial_block_manager) {
+	return make_unique<StandardColumnCheckpointState>(row_group, *this, partial_block_manager);
 }
 
-unique_ptr<ColumnCheckpointState> StandardColumnData::Checkpoint(RowGroup &row_group, RowGroupWriter &writer,
+unique_ptr<ColumnCheckpointState> StandardColumnData::Checkpoint(RowGroup &row_group,
+                                                                 PartialBlockManager &partial_block_manager,
                                                                  ColumnCheckpointInfo &checkpoint_info) {
-	auto validity_state = validity.Checkpoint(row_group, writer, checkpoint_info);
-	auto base_state = ColumnData::Checkpoint(row_group, writer, checkpoint_info);
+	auto validity_state = validity.Checkpoint(row_group, partial_block_manager, checkpoint_info);
+	auto base_state = ColumnData::Checkpoint(row_group, partial_block_manager, checkpoint_info);
 	auto &checkpoint_state = (StandardColumnCheckpointState &)*base_state;
 	checkpoint_state.validity_state = move(validity_state);
 	return base_state;
@@ -209074,8 +209359,9 @@ void StructColumnData::CommitDropColumn() {
 }
 
 struct StructColumnCheckpointState : public ColumnCheckpointState {
-	StructColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, RowGroupWriter &writer)
-	    : ColumnCheckpointState(row_group, column_data, writer) {
+	StructColumnCheckpointState(RowGroup &row_group, ColumnData &column_data,
+	                            PartialBlockManager &partial_block_manager)
+	    : ColumnCheckpointState(row_group, column_data, partial_block_manager) {
 		global_stats = make_unique<StructStatistics>(column_data.type);
 	}
 
@@ -209094,24 +209380,33 @@ public:
 		return move(stats);
 	}
 
-	void WriteDataPointers() override {
-		validity_state->WriteDataPointers();
+	void WriteDataPointers(RowGroupWriter &writer) override {
+		validity_state->WriteDataPointers(writer);
+		for (auto &state : child_states) {
+			state->WriteDataPointers(writer);
+		}
+	}
+	void GetBlockIds(unordered_set<block_id_t> &result) override {
+		validity_state->GetBlockIds(result);
 		for (auto &state : child_states) {
-			state->WriteDataPointers();
+			state->GetBlockIds(result);
 		}
 	}
 };
 
-unique_ptr<ColumnCheckpointState> StructColumnData::CreateCheckpointState(RowGroup &row_group, RowGroupWriter &writer) {
-	return make_unique<StructColumnCheckpointState>(row_group, *this, writer);
+unique_ptr<ColumnCheckpointState> StructColumnData::CreateCheckpointState(RowGroup &row_group,
+                                                                          PartialBlockManager &partial_block_manager) {
+	return make_unique<StructColumnCheckpointState>(row_group, *this, partial_block_manager);
 }
 
-unique_ptr<ColumnCheckpointState> StructColumnData::Checkpoint(RowGroup &row_group, RowGroupWriter &writer,
+unique_ptr<ColumnCheckpointState> StructColumnData::Checkpoint(RowGroup &row_group,
+                                                               PartialBlockManager &partial_block_manager,
                                                                ColumnCheckpointInfo &checkpoint_info) {
-	auto checkpoint_state = make_unique<StructColumnCheckpointState>(row_group, *this, writer);
-	checkpoint_state->validity_state = validity.Checkpoint(row_group, writer, checkpoint_info);
+	auto checkpoint_state = make_unique<StructColumnCheckpointState>(row_group, *this, partial_block_manager);
+	checkpoint_state->validity_state = validity.Checkpoint(row_group, partial_block_manager, checkpoint_info);
 	for (auto &sub_column : sub_columns) {
-		checkpoint_state->child_states.push_back(sub_column->Checkpoint(row_group, writer, checkpoint_info));
+		checkpoint_state->child_states.push_back(
+		    sub_column->Checkpoint(row_group, partial_block_manager, checkpoint_info));
 	}
 	return move(checkpoint_state);
 }
@@ -212062,6 +212357,7 @@ void RollbackState::RollbackEntry(UndoFlags type, data_ptr_t data) {
 
 
 
+
 #include <cstring>
 
 namespace duckdb {
@@ -212077,13 +212373,21 @@ Transaction::Transaction(weak_ptr<ClientContext> context_p, transaction_t start_
                          timestamp_t start_timestamp, idx_t catalog_version)
     : context(move(context_p)), start_time(start_time), transaction_id(transaction_id), commit_id(0),
       highest_active_query(0), active_query(MAXIMUM_QUERY_ID), start_timestamp(start_timestamp),
-      catalog_version(catalog_version), storage(*this), is_invalidated(false), undo_buffer(context.lock()) {
+      catalog_version(catalog_version), is_invalidated(false), undo_buffer(context.lock()),
+      storage(make_unique<LocalStorage>(*this)) {
+}
+
+Transaction::~Transaction() {
 }
 
 Transaction &Transaction::GetTransaction(ClientContext &context) {
 	return context.ActiveTransaction();
 }
 
+LocalStorage &Transaction::GetLocalStorage() {
+	return *storage;
+}
+
 void Transaction::PushCatalogEntry(CatalogEntry *entry, data_ptr_t extra_data, idx_t extra_data_size) {
 	idx_t alloc_size = sizeof(CatalogEntry *);
 	if (extra_data_size > 0) {
@@ -212131,12 +212435,12 @@ UpdateInfo *Transaction::CreateUpdateInfo(idx_t type_size, idx_t entries) {
 }
 
 bool Transaction::ChangesMade() {
-	return undo_buffer.ChangesMade() || storage.ChangesMade();
+	return undo_buffer.ChangesMade() || storage->ChangesMade();
 }
 
 bool Transaction::AutomaticCheckpoint(DatabaseInstance &db) {
 	auto &storage_manager = StorageManager::GetStorageManager(db);
-	return storage_manager.AutomaticCheckpoint(storage.EstimatedSize() + undo_buffer.EstimatedSize());
+	return storage_manager.AutomaticCheckpoint(storage->EstimatedSize() + undo_buffer.EstimatedSize());
 }
 
 string Transaction::Commit(DatabaseInstance &db, transaction_t commit_id, bool checkpoint) noexcept {
@@ -212152,7 +212456,7 @@ string Transaction::Commit(DatabaseInstance &db, transaction_t commit_id, bool c
 	LocalStorage::CommitState commit_state;
 	auto storage_commit_state = storage_manager.GenStorageCommitState(*this, checkpoint);
 	try {
-		storage.Commit(commit_state, *this);
+		storage->Commit(commit_state, *this);
 		undo_buffer.Commit(iterator_state, log, commit_id);
 		if (log) {
 			// commit any sequences that were used to the WAL
@@ -212168,6 +212472,22 @@ string Transaction::Commit(DatabaseInstance &db, transaction_t commit_id, bool c
 	}
 }
 
+void Transaction::Rollback() noexcept {
+	storage->Rollback();
+	undo_buffer.Rollback();
+}
+
+void Transaction::Cleanup() {
+	undo_buffer.Cleanup();
+}
+
+void Transaction::Invalidate() {
+	is_invalidated = true;
+}
+bool Transaction::IsInvalidated() {
+	return is_invalidated;
+}
+
 } // namespace duckdb