duckdb 0.7.2-dev2867.0 → 0.7.2-dev2995.0

package/src/duckdb/src/include/duckdb/planner/operator/logical_delete.hpp

@@ -29,6 +29,7 @@ public:
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	idx_t EstimateCardinality(ClientContext &context) override;
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	vector<ColumnBinding> GetColumnBindings() override;

package/src/duckdb/src/include/duckdb/planner/operator/logical_delim_get.hpp

@@ -36,6 +36,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override {

package/src/duckdb/src/include/duckdb/planner/operator/logical_dummy_scan.hpp

@@ -35,6 +35,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override {

package/src/duckdb/src/include/duckdb/planner/operator/logical_expression_get.hpp

@@ -41,6 +41,7 @@ public:
 		return expressions.size();
 	}
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override {

package/src/duckdb/src/include/duckdb/planner/operator/logical_insert.hpp

@@ -69,5 +69,6 @@ protected:
 
 	idx_t EstimateCardinality(ClientContext &context) override;
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/operator/logical_pivot.hpp

@@ -29,6 +29,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override;

package/src/duckdb/src/include/duckdb/planner/operator/logical_projection.hpp

@@ -27,6 +27,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override;

package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp

@@ -41,6 +41,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override {

package/src/duckdb/src/include/duckdb/planner/operator/logical_set_operation.hpp

@@ -41,6 +41,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override {

package/src/duckdb/src/include/duckdb/planner/operator/logical_unnest.hpp

@@ -29,6 +29,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override;

package/src/duckdb/src/include/duckdb/planner/operator/logical_update.hpp

@@ -34,6 +34,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	idx_t EstimateCardinality(ClientContext &context) override;
+	string GetName() const override;
 
 protected:
 	vector<ColumnBinding> GetColumnBindings() override;

package/src/duckdb/src/include/duckdb/planner/operator/logical_window.hpp

@@ -30,6 +30,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override;

package/src/duckdb/src/include/duckdb/storage/data_table.hpp

@@ -106,6 +106,7 @@ public:
 	void LocalMerge(ClientContext &context, RowGroupCollection &collection);
 	//! Creates an optimistic writer for this table - used for optimistically writing parallel appends
 	OptimisticDataWriter &CreateOptimisticWriter(ClientContext &context);
+	void FinalizeOptimisticWriter(ClientContext &context, OptimisticDataWriter &writer);
 
 	//! Delete the entries with the specified row identifier from the table
 	idx_t Delete(TableCatalogEntry &table, ClientContext &context, Vector &row_ids, idx_t count);

package/src/duckdb/src/include/duckdb/storage/optimistic_data_writer.hpp

@@ -0,0 +1,46 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/optimistic_data_writer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/table/row_group_collection.hpp"
+
+namespace duckdb {
+class PartialBlockManager;
+
+class OptimisticDataWriter {
+public:
+	OptimisticDataWriter(DataTable &table);
+	OptimisticDataWriter(DataTable &table, OptimisticDataWriter &parent);
+	~OptimisticDataWriter();
+
+	//! Write a new row group to disk (if possible)
+	void WriteNewRowGroup(RowGroupCollection &row_groups);
+	//! Write the last row group of a collection to disk
+	void WriteLastRowGroup(RowGroupCollection &row_groups);
+	//! Final flush of the optimistic writer - fully flushes the partial block manager
+	void FinalFlush();
+	//! Flushes a specific row group to disk
+	void FlushToDisk(RowGroup *row_group);
+	//! Merge the partially written blocks from one optimistic writer into another
+	void Merge(OptimisticDataWriter &other);
+	//! Rollback
+	void Rollback();
+
+private:
+	//! Prepare a write to disk
+	bool PrepareWrite();
+
+private:
+	//! The table
+	DataTable &table;
+	//! The partial block manager (if we created one yet)
+	unique_ptr<PartialBlockManager> partial_manager;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/partial_block_manager.hpp

@@ -48,6 +48,14 @@ public:
 	virtual void Flush(idx_t free_space_left) = 0;
 	virtual void Clear() {
 	}
+	virtual void Merge(PartialBlock &other, idx_t offset, idx_t other_size);
+
+public:
+	template <class TARGET>
+	TARGET &Cast() {
+		D_ASSERT(dynamic_cast<TARGET *>(this));
+		return (TARGET &)*this;
+	}
 };
 
 struct PartialBlockAllocation {
@@ -61,6 +69,8 @@ struct PartialBlockAllocation {
 	unique_ptr<PartialBlock> partial_block;
 };
 
+enum class CheckpointType { FULL_CHECKPOINT, APPEND_TO_TABLE };
+
 //! Enables sharing blocks across some scope. Scope is whatever we want to share
 //! blocks across. It may be an entire checkpoint or just a single row group.
 //! In any case, they must share a block manager.
@@ -75,7 +85,8 @@ public:
 	static constexpr const idx_t MAX_BLOCK_MAP_SIZE = 1u << 31;
 
 public:
-	PartialBlockManager(BlockManager &block_manager, uint32_t max_partial_block_size = DEFAULT_MAX_PARTIAL_BLOCK_SIZE,
+	PartialBlockManager(BlockManager &block_manager, CheckpointType checkpoint_type,
+	                    uint32_t max_partial_block_size = DEFAULT_MAX_PARTIAL_BLOCK_SIZE,
 	                    uint32_t max_use_count = DEFAULT_MAX_USE_COUNT);
 	virtual ~PartialBlockManager();
 
@@ -87,18 +98,25 @@ public:
 
 	virtual void AllocateBlock(PartialBlockState &state, uint32_t segment_size);
 
+	void Merge(PartialBlockManager &other);
 	//! Register a partially filled block that is filled with "segment_size" entries
 	void RegisterPartialBlock(PartialBlockAllocation &&allocation);
 
-	//! Clears all blocks
-	void Clear();
+	//! Clear remaining blocks without writing them to disk
+	void ClearBlocks();
+
+	//! Rollback all data written by this partial block manager
+	void Rollback();
 
 protected:
 	BlockManager &block_manager;
+	CheckpointType checkpoint_type;
 	//! A map of (available space -> PartialBlock) for partially filled blocks
 	//! This is a multimap because there might be outstanding partial blocks with
 	//! the same amount of left-over space
 	multimap<idx_t, unique_ptr<PartialBlock>> partially_filled_blocks;
+	//! The set of written blocks
+	unordered_set<block_id_t> written_blocks;
 
 	//! The maximum size (in bytes) at which a partial block will be considered a partial block
 	uint32_t max_partial_block_size;
@@ -109,6 +127,9 @@ protected:
 	//! If successful, returns true and returns the block_id and offset_in_block to write to
 	//! Otherwise, returns false
 	bool GetPartialBlock(idx_t segment_size, unique_ptr<PartialBlock> &state);
+
+	bool HasBlockAllocation(uint32_t segment_size);
+	void AddWrittenBlock(block_id_t block);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/column_checkpoint_state.hpp

@@ -15,6 +15,7 @@
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/storage/table/column_data.hpp"
 #include "duckdb/common/unordered_set.hpp"
+#include "duckdb/storage/partial_block_manager.hpp"
 
 namespace duckdb {
 class ColumnData;
@@ -41,7 +42,51 @@ public:
 
 	virtual void FlushSegment(unique_ptr<ColumnSegment> segment, idx_t segment_size);
 	virtual void WriteDataPointers(RowGroupWriter &writer);
-	virtual void GetBlockIds(unordered_set<block_id_t> &result);
+};
+
+struct PartialBlockForCheckpoint : public PartialBlock {
+	struct PartialColumnSegment {
+		PartialColumnSegment(ColumnData &data, ColumnSegment &segment, uint32_t offset_in_block)
+		    : data(data), segment(segment), offset_in_block(offset_in_block) {
+		}
+
+		ColumnData &data;
+		ColumnSegment &segment;
+		uint32_t offset_in_block;
+	};
+
+public:
+	PartialBlockForCheckpoint(ColumnData &data, ColumnSegment &segment, BlockManager &block_manager,
+	                          PartialBlockState state);
+	~PartialBlockForCheckpoint() override;
+
+	// We will copy all segment data into the memory of the shared block.
+	// Once the block is full (or checkpoint is complete) we'll invoke Flush().
+	// This will cause the block to get written to storage (via BlockManger::ConvertToPersistent),
+	// and all segments to have their references updated (via ColumnSegment::ConvertToPersistent)
+	BlockManager &block_manager;
+	shared_ptr<BlockHandle> block;
+	vector<PartialColumnSegment> segments;
+
+private:
+	struct UninitializedRegion {
+		idx_t start;
+		idx_t end;
+	};
+	vector<UninitializedRegion> uninitialized_regions;
+
+public:
+	bool IsFlushed();
+
+	void AddUninitializedRegion(idx_t start, idx_t end) override;
+
+	void Flush(idx_t free_space_left) override;
+
+	void Clear() override;
+
+	void Merge(PartialBlock &other, idx_t offset, idx_t other_size) override;
+
+	void AddSegmentToTail(ColumnData &data, ColumnSegment &segment, uint32_t offset_in_block);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp

@@ -39,12 +39,11 @@ class ColumnData {
 
 public:
 	ColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row, LogicalType type,
-	           ColumnData *parent);
-	ColumnData(ColumnData &other, idx_t start, ColumnData *parent);
+	           optional_ptr<ColumnData> parent);
 	virtual ~ColumnData();
 
 	//! The start row
-	const idx_t start;
+	idx_t start;
 	//! The count of the column data
 	idx_t count;
 	//! The block manager
@@ -56,7 +55,7 @@ public:
 	//! The type of the column
 	LogicalType type;
 	//! The parent column (if any)
-	ColumnData *parent;
+	optional_ptr<ColumnData> parent;
 
 public:
 	virtual bool CheckZonemap(ColumnScanState &state, TableFilter &filter) = 0;
@@ -70,6 +69,7 @@ public:
 
 	void IncrementVersion();
 
+	virtual void SetStart(idx_t new_start);
 	//! The root type of the column
 	const LogicalType &RootType() const;
 
@@ -122,13 +122,13 @@ public:
 	virtual unique_ptr<ColumnCheckpointState>
 	Checkpoint(RowGroup &row_group, PartialBlockManager &partial_block_manager, ColumnCheckpointInfo &checkpoint_info);
 
-	virtual void CheckpointScan(ColumnSegment *segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
+	virtual void CheckpointScan(ColumnSegment &segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
 	                            Vector &scan_vector);
 
 	virtual void DeserializeColumn(Deserializer &source);
 	static shared_ptr<ColumnData> Deserialize(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
 	                                          idx_t start_row, Deserializer &source, const LogicalType &type,
-	                                          ColumnData *parent);
+	                                          optional_ptr<ColumnData> parent);
 
 	virtual void GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, TableStorageInfo &result);
 	virtual void Verify(RowGroup &parent);
@@ -136,12 +136,11 @@ public:
 	bool CheckZonemap(TableFilter &filter);
 
 	static shared_ptr<ColumnData> CreateColumn(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
-	                                           idx_t start_row, const LogicalType &type, ColumnData *parent = nullptr);
-	static shared_ptr<ColumnData> CreateColumn(ColumnData &other, idx_t start_row, ColumnData *parent = nullptr);
+	                                           idx_t start_row, const LogicalType &type,
+	                                           optional_ptr<ColumnData> parent = nullptr);
 	static unique_ptr<ColumnData> CreateColumnUnique(BlockManager &block_manager, DataTableInfo &info,
 	                                                 idx_t column_index, idx_t start_row, const LogicalType &type,
-	                                                 ColumnData *parent = nullptr);
-	static unique_ptr<ColumnData> CreateColumnUnique(ColumnData &other, idx_t start_row, ColumnData *parent = nullptr);
+	                                                 optional_ptr<ColumnData> parent = nullptr);
 
 	void MergeStatistics(const BaseStatistics &other);
 	void MergeIntoStatistics(BaseStatistics &other);

package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp

@@ -93,7 +93,7 @@ public:
 
 	//! Convert a transient in-memory segment into a persistent segment blocked by an on-disk block.
 	//! Only used during checkpointing.
-	void ConvertToPersistent(BlockManager *block_manager, block_id_t block_id);
+	void ConvertToPersistent(optional_ptr<BlockManager> block_manager, block_id_t block_id);
 	//! Updates pointers to refer to the given block and offset. This is only used
 	//! when sharing a block among segments. This is invoked only AFTER the block is written.
 	void MarkAsPersistent(shared_ptr<BlockHandle> block, uint32_t offset_in_block);

package/src/duckdb/src/include/duckdb/storage/table/list_column_data.hpp

@@ -17,8 +17,7 @@ namespace duckdb {
 class ListColumnData : public ColumnData {
 public:
 	ListColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
-	               LogicalType type, ColumnData *parent = nullptr);
-	ListColumnData(ColumnData &original, idx_t start_row, ColumnData *parent = nullptr);
+	               LogicalType type, optional_ptr<ColumnData> parent = nullptr);
 
 	//! The child-column of the list
 	unique_ptr<ColumnData> child_column;
@@ -26,6 +25,7 @@ public:
 	ValidityColumnData validity;
 
 public:
+	void SetStart(idx_t new_start) override;
 	bool CheckZonemap(ColumnScanState &state, TableFilter &filter) override;
 
 	void InitializeScan(ColumnScanState &state) override;

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -59,20 +59,20 @@ public:
 public:
 	RowGroup(RowGroupCollection &collection, idx_t start, idx_t count);
 	RowGroup(RowGroupCollection &collection, RowGroupPointer &&pointer);
-	RowGroup(RowGroup &row_group, RowGroupCollection &collection, idx_t start);
 	~RowGroup();
 
 private:
 	//! The RowGroupCollection this row-group is a part of
-	RowGroupCollection &collection;
+	reference<RowGroupCollection> collection;
 	//! The version info of the row_group (inserted and deleted tuple info)
 	shared_ptr<VersionNode> version_info;
 	//! The column data of the row_group
 	vector<shared_ptr<ColumnData>> columns;
 
 public:
+	void MoveToCollection(RowGroupCollection &collection, idx_t new_start);
 	RowGroupCollection &GetCollection() {
-		return collection;
+		return collection.get();
 	}
 	DatabaseInstance &GetDatabase();
 	BlockManager &GetBlockManager();

package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp

@@ -44,6 +44,7 @@ public:
 	void AppendRowGroup(SegmentLock &l, idx_t start_row);
 	//! Get the nth row-group, negative numbers start from the back (so -1 is the last row group, etc)
 	RowGroup *GetRowGroup(int64_t index);
+	idx_t RowGroupCount();
 	void Verify();
 
 	void InitializeScan(CollectionScanState &state, const vector<column_t> &column_ids, TableFilterSet *table_filters);

package/src/duckdb/src/include/duckdb/storage/table/segment_base.hpp

@@ -27,7 +27,7 @@ public:
 	}
 
 	//! The start row id of this chunk
-	const idx_t start;
+	idx_t start;
 	//! The amount of entries in this storage chunk
 	atomic<idx_t> count;
 	//! The next segment after this one

package/src/duckdb/src/include/duckdb/storage/table/segment_tree.hpp

@@ -63,6 +63,14 @@ public:
 		LoadAllSegments(l);
 		return std::move(nodes);
 	}
+	vector<SegmentNode<T>> MoveSegments() {
+		auto l = Lock();
+		return MoveSegments(l);
+	}
+	idx_t GetSegmentCount() {
+		auto l = Lock();
+		return nodes.size();
+	}
 	//! Gets a pointer to the nth segment. Negative numbers start from the back.
 	T *GetSegmentByIndex(int64_t index) {
 		auto l = Lock();
@@ -243,6 +251,20 @@ public:
 		return SegmentIterationHelper(*this);
 	}
 
+	void Reinitialize() {
+		if (nodes.empty()) {
+			return;
+		}
+		idx_t offset = nodes[0].node->start;
+		for (auto &entry : nodes) {
+			if (entry.node->start != offset) {
+				throw InternalException("In SegmentTree::Reinitialize - gap found between nodes!");
+			}
+			entry.row_start = offset;
+			offset += entry.node->count;
+		}
+	}
+
 protected:
 	atomic<bool> finished_loading;
 

package/src/duckdb/src/include/duckdb/storage/table/standard_column_data.hpp

@@ -17,13 +17,13 @@ namespace duckdb {
 class StandardColumnData : public ColumnData {
 public:
 	StandardColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
-	                   LogicalType type, ColumnData *parent = nullptr);
-	StandardColumnData(ColumnData &original, idx_t start_row, ColumnData *parent = nullptr);
+	                   LogicalType type, optional_ptr<ColumnData> parent = nullptr);
 
 	//! The validity column data
 	ValidityColumnData validity;
 
 public:
+	void SetStart(idx_t new_start) override;
 	bool CheckZonemap(ColumnScanState &state, TableFilter &filter) override;
 
 	void InitializeScan(ColumnScanState &state) override;
@@ -51,7 +51,7 @@ public:
 	                                                        PartialBlockManager &partial_block_manager) override;
 	unique_ptr<ColumnCheckpointState> Checkpoint(RowGroup &row_group, PartialBlockManager &partial_block_manager,
 	                                             ColumnCheckpointInfo &checkpoint_info) override;
-	void CheckpointScan(ColumnSegment *segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
+	void CheckpointScan(ColumnSegment &segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
 	                    Vector &scan_vector) override;
 
 	void DeserializeColumn(Deserializer &source) override;

package/src/duckdb/src/include/duckdb/storage/table/struct_column_data.hpp

@@ -17,8 +17,7 @@ namespace duckdb {
 class StructColumnData : public ColumnData {
 public:
 	StructColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
-	                 LogicalType type, ColumnData *parent = nullptr);
-	StructColumnData(ColumnData &original, idx_t start_row, ColumnData *parent = nullptr);
+	                 LogicalType type, optional_ptr<ColumnData> parent = nullptr);
 
 	//! The sub-columns of the struct
 	vector<unique_ptr<ColumnData>> sub_columns;
@@ -26,6 +25,7 @@ public:
 	ValidityColumnData validity;
 
 public:
+	void SetStart(idx_t new_start) override;
 	bool CheckZonemap(ColumnScanState &state, TableFilter &filter) override;
 	idx_t GetMaxEntry() override;
 

package/src/duckdb/src/include/duckdb/storage/table/update_segment.hpp

@@ -23,8 +23,6 @@ struct UpdateNode;
 class UpdateSegment {
 public:
 	UpdateSegment(ColumnData &column_data);
-	// Construct a duplicate of 'other' with 'new_owner' as it's column data
-	UpdateSegment(UpdateSegment &other, ColumnData &new_owner);
 	~UpdateSegment();
 
 	ColumnData &column_data;

package/src/duckdb/src/include/duckdb/storage/table/validity_column_data.hpp

@@ -16,8 +16,7 @@ namespace duckdb {
 class ValidityColumnData : public ColumnData {
 public:
 	ValidityColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
-	                   ColumnData *parent);
-	ValidityColumnData(ColumnData &original, idx_t start_row, ColumnData *parent = nullptr);
+	                   ColumnData &parent);
 
 public:
 	bool CheckZonemap(ColumnScanState &state, TableFilter &filter) override;

package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp

@@ -11,6 +11,7 @@
 #include "duckdb/storage/table/row_group_collection.hpp"
 #include "duckdb/storage/table/table_index_list.hpp"
 #include "duckdb/storage/table/table_statistics.hpp"
+#include "duckdb/storage/optimistic_data_writer.hpp"
 
 namespace duckdb {
 class AttachedDatabase;
@@ -20,35 +21,6 @@ class WriteAheadLog;
 struct LocalAppendState;
 struct TableAppendState;
 
-class OptimisticDataWriter {
-public:
-	OptimisticDataWriter(DataTable &table);
-	OptimisticDataWriter(DataTable &table, OptimisticDataWriter &parent);
-	~OptimisticDataWriter();
-
-	void CheckFlushToDisk(RowGroupCollection &row_groups);
-	//! Flushes a specific row group to disk
-	void FlushToDisk(RowGroup *row_group);
-	//! Flushes the final row group to disk (if any)
-	void FlushToDisk(RowGroupCollection &row_groups, bool force = false);
-	//! Final flush: flush the partial block manager to disk
-	void FinalFlush();
-
-	void Rollback();
-
-private:
-	//! Prepare a write to disk
-	bool PrepareWrite();
-
-private:
-	//! The table
-	DataTable &table;
-	//! The partial block manager (if we created one yet)
-	unique_ptr<PartialBlockManager> partial_manager;
-	//! The set of blocks that have been pre-emptively written to disk
-	unordered_set<block_id_t> written_blocks;
-};
-
 class LocalTableStorage : public std::enable_shared_from_this<LocalTableStorage> {
 public:
 	// Create a new LocalTableStorage
@@ -76,13 +48,14 @@ public:
 	OptimisticDataWriter optimistic_writer;
 	//! The set of all optimistic data writers associated with this table
 	vector<unique_ptr<OptimisticDataWriter>> optimistic_writers;
+	//! Whether or not storage was merged
+	bool merged_storage = false;
 
 public:
 	void InitializeScan(CollectionScanState &state, optional_ptr<TableFilterSet> table_filters = nullptr);
-	//! Check if we should flush the previously written row-group to disk
-	void CheckFlushToDisk();
-	//! Flushes the final row group to disk (if any)
-	void FlushToDisk();
+	//! Write a new row group to disk (if possible)
+	void WriteNewRowGroup();
+	void FlushBlocks();
 	void Rollback();
 	idx_t EstimatedSize();
 
@@ -93,6 +66,7 @@ public:
 
 	//! Creates an optimistic writer for this table
 	OptimisticDataWriter &CreateOptimisticWriter();
+	void FinalizeOptimisticWriter(OptimisticDataWriter &writer);
 };
 
 class LocalTableManager {
@@ -114,7 +88,7 @@ private:
 class LocalStorage {
 public:
 	// Threshold to merge row groups instead of appending
-	static constexpr const idx_t MERGE_THRESHOLD = RowGroup::ROW_GROUP_SIZE / 2;
+	static constexpr const idx_t MERGE_THRESHOLD = RowGroup::ROW_GROUP_SIZE;
 
 public:
 	struct CommitState {
@@ -150,6 +124,7 @@ public:
 	void LocalMerge(DataTable &table, RowGroupCollection &collection);
 	//! Create an optimistic writer for the specified table
 	OptimisticDataWriter &CreateOptimisticWriter(DataTable &table);
+	void FinalizeOptimisticWriter(DataTable &table, OptimisticDataWriter &writer);
 
 	//! Delete a set of rows from the local storage
 	idx_t Delete(DataTable &table, Vector &row_ids, idx_t count);

package/src/duckdb/src/include/duckdb/verification/no_operator_caching_verifier.hpp

@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/verification/unoptimized_statement_verifier.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/verification/statement_verifier.hpp"
+
+namespace duckdb {
+
+class NoOperatorCachingVerifier : public StatementVerifier {
+public:
+	explicit NoOperatorCachingVerifier(unique_ptr<SQLStatement> statement_p);
+	static unique_ptr<StatementVerifier> Create(const SQLStatement &statement_p);
+
+	bool DisableOperatorCaching() const override {
+		return true;
+	}
+};
+
+} // namespace duckdb