duckdb 0.7.2-dev2867.0 → 0.7.2-dev2995.0

package/src/duckdb/src/planner/operator/logical_aggregate.cpp

@@ -1,6 +1,8 @@
 #include "duckdb/planner/operator/logical_aggregate.hpp"
-#include "duckdb/common/string_util.hpp"
+
 #include "duckdb/common/field_writer.hpp"
+#include "duckdb/common/string_util.hpp"
+#include "duckdb/main/config.hpp"
 
 namespace duckdb {
 
@@ -117,4 +119,14 @@ vector<idx_t> LogicalAggregate::GetTableIndex() const {
 	return result;
 }
 
+string LogicalAggregate::GetName() const {
+#ifdef DEBUG
+	if (DBConfigOptions::debug_print_bindings) {
+		return LogicalOperator::GetName() +
+		       StringUtil::Format(" #%llu, #%llu, #%llu", group_index, aggregate_index, groupings_index);
+	}
+#endif
+	return LogicalOperator::GetName();
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/operator/logical_column_data_get.cpp

@@ -1,6 +1,8 @@
 #include "duckdb/planner/operator/logical_column_data_get.hpp"
+
 #include "duckdb/common/field_writer.hpp"
 #include "duckdb/common/types/data_chunk.hpp"
+#include "duckdb/main/config.hpp"
 
 namespace duckdb {
 
@@ -42,4 +44,13 @@ vector<idx_t> LogicalColumnDataGet::GetTableIndex() const {
 	return vector<idx_t> {table_index};
 }
 
+string LogicalColumnDataGet::GetName() const {
+#ifdef DEBUG
+	if (DBConfigOptions::debug_print_bindings) {
+		return LogicalOperator::GetName() + StringUtil::Format(" #%llu", table_index);
+	}
+#endif
+	return LogicalOperator::GetName();
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/operator/logical_cteref.cpp

@@ -1,5 +1,7 @@
 #include "duckdb/planner/operator/logical_cteref.hpp"
+
 #include "duckdb/common/field_writer.hpp"
+#include "duckdb/main/config.hpp"
 
 namespace duckdb {
 
@@ -22,4 +24,13 @@ vector<idx_t> LogicalCTERef::GetTableIndex() const {
 	return vector<idx_t> {table_index};
 }
 
+string LogicalCTERef::GetName() const {
+#ifdef DEBUG
+	if (DBConfigOptions::debug_print_bindings) {
+		return LogicalOperator::GetName() + StringUtil::Format(" #%llu", table_index);
+	}
+#endif
+	return LogicalOperator::GetName();
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/operator/logical_delete.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/planner/operator/logical_delete.hpp"
 
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
+#include "duckdb/main/config.hpp"
 #include "duckdb/parser/parsed_data/create_table_info.hpp"
 
 namespace duckdb {
@@ -51,4 +52,13 @@ void LogicalDelete::ResolveTypes() {
 	}
 }
 
+string LogicalDelete::GetName() const {
+#ifdef DEBUG
+	if (DBConfigOptions::debug_print_bindings) {
+		return LogicalOperator::GetName() + StringUtil::Format(" #%llu", table_index);
+	}
+#endif
+	return LogicalOperator::GetName();
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/operator/logical_delim_get.cpp

@@ -1,6 +1,8 @@
-#include "duckdb/common/field_writer.hpp"
 #include "duckdb/planner/operator/logical_delim_get.hpp"
 
+#include "duckdb/common/field_writer.hpp"
+#include "duckdb/main/config.hpp"
+
 namespace duckdb {
 
 void LogicalDelimGet::Serialize(FieldWriter &writer) const {
@@ -18,4 +20,13 @@ vector<idx_t> LogicalDelimGet::GetTableIndex() const {
 	return vector<idx_t> {table_index};
 }
 
+string LogicalDelimGet::GetName() const {
+#ifdef DEBUG
+	if (DBConfigOptions::debug_print_bindings) {
+		return LogicalOperator::GetName() + StringUtil::Format(" #%llu", table_index);
+	}
+#endif
+	return LogicalOperator::GetName();
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/operator/logical_dummy_scan.cpp

@@ -1,6 +1,8 @@
-#include "duckdb/common/field_writer.hpp"
 #include "duckdb/planner/operator/logical_dummy_scan.hpp"
 
+#include "duckdb/common/field_writer.hpp"
+#include "duckdb/main/config.hpp"
+
 namespace duckdb {
 
 void LogicalDummyScan::Serialize(FieldWriter &writer) const {
@@ -16,4 +18,13 @@ vector<idx_t> LogicalDummyScan::GetTableIndex() const {
 	return vector<idx_t> {table_index};
 }
 
+string LogicalDummyScan::GetName() const {
+#ifdef DEBUG
+	if (DBConfigOptions::debug_print_bindings) {
+		return LogicalOperator::GetName() + StringUtil::Format(" #%llu", table_index);
+	}
+#endif
+	return LogicalOperator::GetName();
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/operator/logical_expression_get.cpp

@@ -1,6 +1,8 @@
-#include "duckdb/common/field_writer.hpp"
 #include "duckdb/planner/operator/logical_expression_get.hpp"
 
+#include "duckdb/common/field_writer.hpp"
+#include "duckdb/main/config.hpp"
+
 namespace duckdb {
 
 void LogicalExpressionGet::Serialize(FieldWriter &writer) const {
@@ -30,4 +32,13 @@ vector<idx_t> LogicalExpressionGet::GetTableIndex() const {
 	return vector<idx_t> {table_index};
 }
 
+string LogicalExpressionGet::GetName() const {
+#ifdef DEBUG
+	if (DBConfigOptions::debug_print_bindings) {
+		return LogicalOperator::GetName() + StringUtil::Format(" #%llu", table_index);
+	}
+#endif
+	return LogicalOperator::GetName();
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/operator/logical_get.cpp

@@ -6,6 +6,7 @@
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/function/function_serialization.hpp"
 #include "duckdb/function/table/table_scan.hpp"
+#include "duckdb/main/config.hpp"
 #include "duckdb/storage/data_table.hpp"
 
 namespace duckdb {
@@ -16,10 +17,6 @@ LogicalGet::LogicalGet(idx_t table_index, TableFunction function, unique_ptr<Fun
       bind_data(std::move(bind_data)), returned_types(std::move(returned_types)), names(std::move(returned_names)) {
 }
 
-string LogicalGet::GetName() const {
-	return StringUtil::Upper(function.name);
-}
-
 optional_ptr<TableCatalogEntry> LogicalGet::GetTable() const {
 	return TableScanFunction::GetTableEntry(function, bind_data.get());
 }
@@ -201,4 +198,13 @@ vector<idx_t> LogicalGet::GetTableIndex() const {
 	return vector<idx_t> {table_index};
 }
 
+string LogicalGet::GetName() const {
+#ifdef DEBUG
+	if (DBConfigOptions::debug_print_bindings) {
+		return StringUtil::Upper(function.name) + StringUtil::Format(" #%llu", table_index);
+	}
+#endif
+	return StringUtil::Upper(function.name);
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/operator/logical_insert.cpp

@@ -1,6 +1,8 @@
+#include "duckdb/planner/operator/logical_insert.hpp"
+
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/common/field_writer.hpp"
-#include "duckdb/planner/operator/logical_insert.hpp"
+#include "duckdb/main/config.hpp"
 #include "duckdb/parser/parsed_data/create_table_info.hpp"
 
 namespace duckdb {
@@ -78,4 +80,13 @@ void LogicalInsert::ResolveTypes() {
 	}
 }
 
+string LogicalInsert::GetName() const {
+#ifdef DEBUG
+	if (DBConfigOptions::debug_print_bindings) {
+		return LogicalOperator::GetName() + StringUtil::Format(" #%llu", table_index);
+	}
+#endif
+	return LogicalOperator::GetName();
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/operator/logical_pivot.cpp

@@ -1,5 +1,7 @@
 #include "duckdb/planner/operator/logical_pivot.hpp"
 
+#include "duckdb/main/config.hpp"
+
 namespace duckdb {
 
 LogicalPivot::LogicalPivot(idx_t pivot_idx, unique_ptr<LogicalOperator> plan, BoundPivotInfo info_p)
@@ -32,4 +34,13 @@ void LogicalPivot::ResolveTypes() {
 	this->types = bound_pivot.types;
 }
 
+string LogicalPivot::GetName() const {
+#ifdef DEBUG
+	if (DBConfigOptions::debug_print_bindings) {
+		return LogicalOperator::GetName() + StringUtil::Format(" #%llu", pivot_index);
+	}
+#endif
+	return LogicalOperator::GetName();
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/operator/logical_projection.cpp

@@ -1,5 +1,7 @@
 #include "duckdb/planner/operator/logical_projection.hpp"
+
 #include "duckdb/common/field_writer.hpp"
+#include "duckdb/main/config.hpp"
 
 namespace duckdb {
 
@@ -32,4 +34,13 @@ vector<idx_t> LogicalProjection::GetTableIndex() const {
 	return vector<idx_t> {table_index};
 }
 
+string LogicalProjection::GetName() const {
+#ifdef DEBUG
+	if (DBConfigOptions::debug_print_bindings) {
+		return LogicalOperator::GetName() + StringUtil::Format(" #%llu", table_index);
+	}
+#endif
+	return LogicalOperator::GetName();
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/operator/logical_recursive_cte.cpp

@@ -1,5 +1,7 @@
 #include "duckdb/planner/operator/logical_recursive_cte.hpp"
+
 #include "duckdb/common/field_writer.hpp"
+#include "duckdb/main/config.hpp"
 
 namespace duckdb {
 
@@ -21,4 +23,13 @@ vector<idx_t> LogicalRecursiveCTE::GetTableIndex() const {
 	return vector<idx_t> {table_index};
 }
 
+string LogicalRecursiveCTE::GetName() const {
+#ifdef DEBUG
+	if (DBConfigOptions::debug_print_bindings) {
+		return LogicalOperator::GetName() + StringUtil::Format(" #%llu", table_index);
+	}
+#endif
+	return LogicalOperator::GetName();
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/operator/logical_set_operation.cpp

@@ -1,5 +1,7 @@
 #include "duckdb/planner/operator/logical_set_operation.hpp"
+
 #include "duckdb/common/field_writer.hpp"
+#include "duckdb/main/config.hpp"
 
 namespace duckdb {
 
@@ -19,4 +21,13 @@ vector<idx_t> LogicalSetOperation::GetTableIndex() const {
 	return vector<idx_t> {table_index};
 }
 
+string LogicalSetOperation::GetName() const {
+#ifdef DEBUG
+	if (DBConfigOptions::debug_print_bindings) {
+		return LogicalOperator::GetName() + StringUtil::Format(" #%llu", table_index);
+	}
+#endif
+	return LogicalOperator::GetName();
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/operator/logical_unnest.cpp

@@ -1,6 +1,8 @@
-#include "duckdb/common/field_writer.hpp"
 #include "duckdb/planner/operator/logical_unnest.hpp"
 
+#include "duckdb/common/field_writer.hpp"
+#include "duckdb/main/config.hpp"
+
 namespace duckdb {
 
 vector<ColumnBinding> LogicalUnnest::GetColumnBindings() {
@@ -35,4 +37,13 @@ vector<idx_t> LogicalUnnest::GetTableIndex() const {
 	return vector<idx_t> {unnest_index};
 }
 
+string LogicalUnnest::GetName() const {
+#ifdef DEBUG
+	if (DBConfigOptions::debug_print_bindings) {
+		return LogicalOperator::GetName() + StringUtil::Format(" #%llu", unnest_index);
+	}
+#endif
+	return LogicalOperator::GetName();
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/operator/logical_update.cpp

@@ -2,6 +2,7 @@
 #include "duckdb/common/field_writer.hpp"
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/parser/parsed_data/create_table_info.hpp"
+#include "duckdb/main/config.hpp"
 
 namespace duckdb {
 
@@ -52,4 +53,13 @@ void LogicalUpdate::ResolveTypes() {
 	}
 }
 
+string LogicalUpdate::GetName() const {
+#ifdef DEBUG
+	if (DBConfigOptions::debug_print_bindings) {
+		return LogicalOperator::GetName() + StringUtil::Format(" #%llu", table_index);
+	}
+#endif
+	return LogicalOperator::GetName();
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/operator/logical_window.cpp

@@ -1,5 +1,7 @@
 #include "duckdb/planner/operator/logical_window.hpp"
+
 #include "duckdb/common/field_writer.hpp"
+#include "duckdb/main/config.hpp"
 
 namespace duckdb {
 
@@ -34,4 +36,13 @@ vector<idx_t> LogicalWindow::GetTableIndex() const {
 	return vector<idx_t> {window_index};
 }
 
+string LogicalWindow::GetName() const {
+#ifdef DEBUG
+	if (DBConfigOptions::debug_print_bindings) {
+		return LogicalOperator::GetName() + StringUtil::Format(" #%llu", window_index);
+	}
+#endif
+	return LogicalOperator::GetName();
+}
+
 } // namespace duckdb

package/src/duckdb/src/storage/checkpoint_manager.cpp

@@ -37,7 +37,7 @@ namespace duckdb {
 void ReorderTableEntries(vector<reference<TableCatalogEntry>> &tables);
 
 SingleFileCheckpointWriter::SingleFileCheckpointWriter(AttachedDatabase &db, BlockManager &block_manager)
-    : CheckpointWriter(db), partial_block_manager(block_manager) {
+    : CheckpointWriter(db), partial_block_manager(block_manager, CheckpointType::FULL_CHECKPOINT) {
 }
 
 BlockManager &SingleFileCheckpointWriter::GetBlockManager() {

package/src/duckdb/src/storage/data_table.cpp

@@ -665,6 +665,11 @@ OptimisticDataWriter &DataTable::CreateOptimisticWriter(ClientContext &context)
 	return local_storage.CreateOptimisticWriter(*this);
 }
 
+void DataTable::FinalizeOptimisticWriter(ClientContext &context, OptimisticDataWriter &writer) {
+	auto &local_storage = LocalStorage::Get(context, db);
+	local_storage.FinalizeOptimisticWriter(*this, writer);
+}
+
 void DataTable::LocalMerge(ClientContext &context, RowGroupCollection &collection) {
 	auto &local_storage = LocalStorage::Get(context, db);
 	local_storage.LocalMerge(*this, collection);

package/src/duckdb/src/storage/local_storage.cpp

@@ -9,111 +9,14 @@
 #include "duckdb/storage/partial_block_manager.hpp"
 
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
-#include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/storage/table_io_manager.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 
-//===--------------------------------------------------------------------===//
-// OptimisticDataWriter
-//===--------------------------------------------------------------------===//
-OptimisticDataWriter::OptimisticDataWriter(DataTable &table) : table(table) {
-}
-
-OptimisticDataWriter::OptimisticDataWriter(DataTable &table, OptimisticDataWriter &parent)
-    : table(table), partial_manager(std::move(parent.partial_manager)),
-      written_blocks(std::move(parent.written_blocks)) {
-	if (partial_manager) {
-		partial_manager->FlushPartialBlocks();
-	}
-}
-
-OptimisticDataWriter::~OptimisticDataWriter() {
-}
-
-bool OptimisticDataWriter::PrepareWrite() {
-	// check if we should pre-emptively write the table to disk
-	if (table.info->IsTemporary() || StorageManager::Get(table.info->db).InMemory()) {
-		return false;
-	}
-	// we should! write the second-to-last row group to disk
-	// allocate the partial block-manager if none is allocated yet
-	if (!partial_manager) {
-		auto &block_manager = table.info->table_io_manager->GetBlockManagerForRowData();
-		partial_manager = make_uniq<PartialBlockManager>(block_manager);
-	}
-	return true;
-}
-
-void OptimisticDataWriter::CheckFlushToDisk(RowGroupCollection &row_groups) {
-	// we finished writing a complete row group
-	if (!PrepareWrite()) {
-		return;
-	}
-	// flush second-to-last row group
-	auto row_group = row_groups.GetRowGroup(-2);
-	FlushToDisk(row_group);
-}
-
-void OptimisticDataWriter::FlushToDisk(RowGroup *row_group) {
-	// flush the specified row group
-	D_ASSERT(row_group);
-	//! The set of column compression types (if any)
-	vector<CompressionType> compression_types;
-	D_ASSERT(compression_types.empty());
-	for (auto &column : table.column_definitions) {
-		compression_types.push_back(column.CompressionType());
-	}
-	auto row_group_pointer = row_group->WriteToDisk(*partial_manager, compression_types);
-
-	// update the set of written blocks
-	for (idx_t col_idx = 0; col_idx < row_group_pointer.statistics.size(); col_idx++) {
-		row_group_pointer.states[col_idx]->GetBlockIds(written_blocks);
-	}
-}
-
-void OptimisticDataWriter::FlushToDisk(RowGroupCollection &row_groups, bool force) {
-	if (!partial_manager) {
-		if (!force) {
-			// no partial manager - nothing to flush
-			return;
-		}
-		if (!PrepareWrite()) {
-			return;
-		}
-	}
-	// flush the last row group
-	FlushToDisk(row_groups.GetRowGroup(-1));
-}
-
-void OptimisticDataWriter::FinalFlush() {
-	if (!partial_manager) {
-		return;
-	}
-	// then flush the partial manager
-	partial_manager->FlushPartialBlocks();
-	partial_manager.reset();
-}
-
-void OptimisticDataWriter::Rollback() {
-	if (partial_manager) {
-		partial_manager->Clear();
-		partial_manager.reset();
-	}
-	if (!written_blocks.empty()) {
-		auto &block_manager = table.info->table_io_manager->GetBlockManagerForRowData();
-		for (auto block_id : written_blocks) {
-			block_manager.MarkBlockAsFree(block_id);
-		}
-	}
-}
-
-//===--------------------------------------------------------------------===//
-// Local Table Storage
-//===--------------------------------------------------------------------===//
 LocalTableStorage::LocalTableStorage(DataTable &table)
-    : table_ref(table), allocator(Allocator::Get(table.db)), deleted_rows(0), optimistic_writer(table) {
+    : table_ref(table), allocator(Allocator::Get(table.db)), deleted_rows(0), optimistic_writer(table),
+      merged_storage(false) {
 	auto types = table.GetTypes();
 	row_groups = make_shared<RowGroupCollection>(table.info, TableIOManager::Get(table).GetBlockManagerForRowData(),
 	                                             types, MAX_ROW_ID, 0);
@@ -140,7 +43,8 @@ LocalTableStorage::LocalTableStorage(ClientContext &context, DataTable &new_dt,
                                      idx_t changed_idx, const LogicalType &target_type,
                                      const vector<column_t> &bound_columns, Expression &cast_expr)
     : table_ref(new_dt), allocator(Allocator::Get(new_dt.db)), deleted_rows(parent.deleted_rows),
-      optimistic_writer(new_dt, parent.optimistic_writer), optimistic_writers(std::move(parent.optimistic_writers)) {
+      optimistic_writer(new_dt, parent.optimistic_writer), optimistic_writers(std::move(parent.optimistic_writers)),
+      merged_storage(parent.merged_storage) {
 	row_groups = parent.row_groups->AlterType(context, changed_idx, target_type, bound_columns, cast_expr);
 	parent.row_groups.reset();
 	indexes.Move(parent.indexes);
@@ -148,7 +52,8 @@ LocalTableStorage::LocalTableStorage(ClientContext &context, DataTable &new_dt,
 
 LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &parent, idx_t drop_idx)
     : table_ref(new_dt), allocator(Allocator::Get(new_dt.db)), deleted_rows(parent.deleted_rows),
-      optimistic_writer(new_dt, parent.optimistic_writer), optimistic_writers(std::move(parent.optimistic_writers)) {
+      optimistic_writer(new_dt, parent.optimistic_writer), optimistic_writers(std::move(parent.optimistic_writers)),
+      merged_storage(parent.merged_storage) {
 	row_groups = parent.row_groups->RemoveColumn(drop_idx);
 	parent.row_groups.reset();
 	indexes.Move(parent.indexes);
@@ -157,7 +62,8 @@ LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &paren
 LocalTableStorage::LocalTableStorage(ClientContext &context, DataTable &new_dt, LocalTableStorage &parent,
                                      ColumnDefinition &new_column, optional_ptr<Expression> default_value)
     : table_ref(new_dt), allocator(Allocator::Get(new_dt.db)), deleted_rows(parent.deleted_rows),
-      optimistic_writer(new_dt, parent.optimistic_writer), optimistic_writers(std::move(parent.optimistic_writers)) {
+      optimistic_writer(new_dt, parent.optimistic_writer), optimistic_writers(std::move(parent.optimistic_writers)),
+      merged_storage(parent.merged_storage) {
 	row_groups = parent.row_groups->AddColumn(context, new_column, default_value.get());
 	parent.row_groups.reset();
 	indexes.Move(parent.indexes);
@@ -187,16 +93,18 @@ idx_t LocalTableStorage::EstimatedSize() {
 	return appended_rows * row_size;
 }
 
-void LocalTableStorage::CheckFlushToDisk() {
+void LocalTableStorage::WriteNewRowGroup() {
 	if (deleted_rows != 0) {
 		// we have deletes - we cannot merge row groups
 		return;
 	}
-	optimistic_writer.CheckFlushToDisk(*row_groups);
+	optimistic_writer.WriteNewRowGroup(*row_groups);
 }
 
-void LocalTableStorage::FlushToDisk() {
-	optimistic_writer.FlushToDisk(*row_groups);
+void LocalTableStorage::FlushBlocks() {
+	if (!merged_storage && row_groups->GetTotalRows() > RowGroup::ROW_GROUP_SIZE) {
+		optimistic_writer.WriteLastRowGroup(*row_groups);
+	}
 	optimistic_writer.FinalFlush();
 }
 
@@ -286,12 +194,28 @@ OptimisticDataWriter &LocalTableStorage::CreateOptimisticWriter() {
 	return *optimistic_writers.back();
 }
 
+void LocalTableStorage::FinalizeOptimisticWriter(OptimisticDataWriter &writer) {
+	// remove the writer from the set of optimistic writers
+	unique_ptr<OptimisticDataWriter> owned_writer;
+	for (idx_t i = 0; i < optimistic_writers.size(); i++) {
+		if (optimistic_writers[i].get() == &writer) {
+			owned_writer = std::move(optimistic_writers[i]);
+			optimistic_writers.erase(optimistic_writers.begin() + i);
+			break;
+		}
+	}
+	if (!owned_writer) {
+		throw InternalException("Error in FinalizeOptimisticWriter - could not find writer");
+	}
+	optimistic_writer.Merge(*owned_writer);
+}
+
 void LocalTableStorage::Rollback() {
-	optimistic_writer.Rollback();
 	for (auto &writer : optimistic_writers) {
 		writer->Rollback();
 	}
 	optimistic_writers.clear();
+	optimistic_writer.Rollback();
 }
 
 //===--------------------------------------------------------------------===//
@@ -429,7 +353,7 @@ void LocalStorage::Append(LocalAppendState &state, DataChunk &chunk) {
 
 	//! Check if we should pre-emptively flush blocks to disk
 	if (new_row_group) {
-		storage->CheckFlushToDisk();
+		storage->WriteNewRowGroup();
 	}
 }
 
@@ -448,6 +372,7 @@ void LocalStorage::LocalMerge(DataTable &table, RowGroupCollection &collection)
 		}
 	}
 	storage.row_groups->MergeStorage(collection);
+	storage.merged_storage = true;
 }
 
 OptimisticDataWriter &LocalStorage::CreateOptimisticWriter(DataTable &table) {
@@ -455,6 +380,11 @@ OptimisticDataWriter &LocalStorage::CreateOptimisticWriter(DataTable &table) {
 	return storage.CreateOptimisticWriter();
 }
 
+void LocalStorage::FinalizeOptimisticWriter(DataTable &table, OptimisticDataWriter &writer) {
+	auto &storage = table_manager.GetOrCreateStorage(table);
+	storage.FinalizeOptimisticWriter(writer);
+}
+
 bool LocalStorage::ChangesMade() noexcept {
 	return !table_manager.IsEmpty();
 }
@@ -502,8 +432,8 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) {
 	if ((append_state.row_start == 0 || storage.row_groups->GetTotalRows() >= MERGE_THRESHOLD) &&
 	    storage.deleted_rows == 0) {
 		// table is currently empty OR we are bulk appending: move over the storage directly
-		// first flush any out-standing storage nodes
-		storage.FlushToDisk();
+		// first flush any outstanding blocks
+		storage.FlushBlocks();
 		// now append to the indexes (if there are any)
 		// FIXME: we should be able to merge the transaction-local index directly into the main table index
 		// as long we just rewrite some row-ids