duckdb 0.7.2-dev2820.0 → 0.7.2-dev2995.0

package/src/duckdb/src/include/duckdb/parallel/pipeline_executor.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/common/types/data_chunk.hpp"
+#include "duckdb/parallel/interrupt.hpp"
 #include "duckdb/parallel/pipeline.hpp"
 #include "duckdb/execution/physical_operator.hpp"
 #include "duckdb/parallel/thread_context.hpp"
@@ -20,16 +21,27 @@
 namespace duckdb {
 class Executor;
 
+//! The result of executing a PipelineExecutor
+enum class PipelineExecuteResult {
+	//! PipelineExecutor is fully executed: the source is completely exhausted
+	FINISHED,
+	//! PipelineExecutor is not yet fully executed and can be called again immediately
+	NOT_FINISHED,
+	//! The PipelineExecutor was interrupted and should not be called again until the interrupt is handled as specified
+	//! in the InterruptMode
+	INTERRUPTED
+};
+
 //! The Pipeline class represents an execution pipeline
 class PipelineExecutor {
 public:
 	PipelineExecutor(ClientContext &context, Pipeline &pipeline);
 
 	//! Fully execute a pipeline with a source and a sink until the source is completely exhausted
-	void Execute();
-	//! Execute a pipeline with a source and a sink until finished, or until max_chunks have been processed
+	PipelineExecuteResult Execute();
+	//! Execute a pipeline with a source and a sink until finished, or until max_chunks were processed from the source
 	//! Returns true if execution is finished, false if Execute should be called again
-	bool Execute(idx_t max_chunks);
+	PipelineExecuteResult Execute(idx_t max_chunks);
 
 	//! Push a single input DataChunk into the pipeline.
 	//! Returns either OperatorResultType::NEED_MORE_INPUT or OperatorResultType::FINISHED
@@ -48,6 +60,9 @@ public:
 	//! This flushes profiler states
 	void PullFinalize();
 
+	//! Registers the task in the interrupt_state to allow Source/Sink operators to block the task
+	void SetTaskForInterrupts(weak_ptr<Task> current_task);
+
 private:
 	//! The pipeline to process
 	Pipeline &pipeline;
@@ -65,6 +80,8 @@ private:
 	unique_ptr<LocalSourceState> local_source_state;
 	//! The local sink state (if any)
 	unique_ptr<LocalSinkState> local_sink_state;
+	//! The interrupt state, holding required information for sink/source operators to block
+	InterruptState interrupt_state;
 
 	//! The final chunk used for moving data into the sink
 	DataChunk final_chunk;
@@ -79,28 +96,55 @@ private:
 	//! Whether or not this pipeline requires keeping track of the batch index of the source
 	bool requires_batch_index = false;
 
+	//! Source has indicated it is exhausted
+	bool exhausted_source = false;
+	//! Flushing of intermediate operators has started
+	bool started_flushing = false;
+	//! Flushing of caching operators is done
+	bool done_flushing = false;
+
+	//! This flag is set when the pipeline gets interrupted by the Sink -> the final_chunk should be re-sink-ed.
+	bool remaining_sink_chunk = false;
+
+	//! Current operator being flushed
+	idx_t flushing_idx;
+	//! Whether the current flushing_idx should be flushed: this needs to be stored to make flushing code re-entrant
+	bool should_flush_current_idx = true;
+
 private:
 	void StartOperator(PhysicalOperator &op);
 	void EndOperator(PhysicalOperator &op, optional_ptr<DataChunk> chunk);
 
 	//! Reset the operator index to the first operator
 	void GoToSource(idx_t &current_idx, idx_t initial_idx);
-	void FetchFromSource(DataChunk &result);
+	SourceResultType FetchFromSource(DataChunk &result);
 
 	void FinishProcessing(int32_t operator_idx = -1);
 	bool IsFinished();
 
+	//! Wrappers for sink/source calls to respective operators
+	SourceResultType GetData(DataChunk &chunk, OperatorSourceInput &input);
+	SinkResultType Sink(DataChunk &chunk, OperatorSinkInput &input);
+
 	OperatorResultType ExecutePushInternal(DataChunk &input, idx_t initial_idx = 0);
 	//! Pushes a chunk through the pipeline and returns a single result chunk
 	//! Returns whether or not a new input chunk is needed, or whether or not we are finished
 	OperatorResultType Execute(DataChunk &input, DataChunk &result, idx_t initial_index = 0);
 
-	//! FlushCachedOperators methods push/pull any remaining cached results through the pipeline
-	void FlushCachingOperatorsPull(DataChunk &result);
-	void FlushCachingOperatorsPush();
+	//! Tries to flush all state from intermediate operators. Will return true if all state is flushed, false in the
+	//! case of a blocked sink.
+	bool TryFlushCachingOperators();
 
 	static bool CanCacheType(const LogicalType &type);
 	void CacheChunk(DataChunk &input, idx_t operator_idx);
+
+#ifdef DUCKDB_DEBUG_ASYNC_SINK_SOURCE
+	//! Debugging state: number of times blocked
+	int debug_blocked_sink_count = 0;
+	int debug_blocked_source_count = 0;
+	//! Number of times the Sink/Source will block before actually returning data
+	int debug_blocked_target_count = 1;
+#endif
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/parallel/task.hpp

@@ -13,13 +13,16 @@
 namespace duckdb {
 class ClientContext;
 class Executor;
+class Task;
+class DatabaseInstance;
+struct ProducerToken;
 
 enum class TaskExecutionMode : uint8_t { PROCESS_ALL, PROCESS_PARTIAL };
 
-enum class TaskExecutionResult : uint8_t { TASK_FINISHED, TASK_NOT_FINISHED, TASK_ERROR };
+enum class TaskExecutionResult : uint8_t { TASK_FINISHED, TASK_NOT_FINISHED, TASK_ERROR, TASK_BLOCKED };
 
 //! Generic parallel task
-class Task {
+class Task : public std::enable_shared_from_this<Task> {
 public:
 	virtual ~Task() {
 	}
@@ -28,7 +31,20 @@ public:
 	//! If mode is PROCESS_ALL, Execute should always finish processing and return TASK_FINISHED
 	//! If mode is PROCESS_PARTIAL, Execute can return TASK_NOT_FINISHED, in which case Execute will be called again
 	//! In case of an error, TASK_ERROR is returned
+	//! In case the task has interrupted, BLOCKED is returned.
 	virtual TaskExecutionResult Execute(TaskExecutionMode mode) = 0;
+
+	//! Descheduling a task ensures the task is not executed, but remains available for rescheduling as long as
+	//! required, generally until some code in an operator calls the InterruptState::Callback() method of a state of the
+	//! InterruptMode::TASK mode.
+	virtual void Deschedule() {
+		throw InternalException("Cannot deschedule task of base Task class");
+	};
+
+	//! Ensures a task is rescheduled to the correct queue
+	virtual void Reschedule() {
+		throw InternalException("Cannot reschedule task of base Task class");
+	}
 };
 
 //! Execute a task within an executor, including exception handling
@@ -39,6 +55,9 @@ public:
 	ExecutorTask(ClientContext &context);
 	virtual ~ExecutorTask();
 
+	void Deschedule() override;
+	void Reschedule() override;
+
 	Executor &executor;
 
 public:

package/src/duckdb/src/include/duckdb/parallel/task_counter.hpp

@@ -18,7 +18,7 @@ public:
 	    : scheduler(scheduler_p), token(scheduler_p.CreateProducer()), task_count(0), tasks_completed(0) {
 	}
 
-	virtual void AddTask(unique_ptr<Task> task) {
+	virtual void AddTask(shared_ptr<Task> task) {
 		++task_count;
 		scheduler.ScheduleTask(*token, std::move(task));
 	}
@@ -29,7 +29,7 @@ public:
 
 	virtual void Finish() {
 		while (tasks_completed < task_count) {
-			unique_ptr<Task> task;
+			shared_ptr<Task> task;
 			if (scheduler.GetTaskFromProducer(*token, task)) {
 				task->Execute();
 				task.reset();

package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp

@@ -47,9 +47,9 @@ public:
 
 	unique_ptr<ProducerToken> CreateProducer();
 	//! Schedule a task to be executed by the task scheduler
-	void ScheduleTask(ProducerToken &producer, unique_ptr<Task> task);
+	void ScheduleTask(ProducerToken &producer, shared_ptr<Task> task);
 	//! Fetches a task from a specific producer, returns true if successful or false if no tasks were available
-	bool GetTaskFromProducer(ProducerToken &token, unique_ptr<Task> &task);
+	bool GetTaskFromProducer(ProducerToken &token, shared_ptr<Task> &task);
 	//! Run tasks forever until "marker" is set to false, "marker" must remain valid until the thread is joined
 	void ExecuteForever(atomic<bool> *marker);
 	//! Run tasks until `marker` is set to false, `max_tasks` have been completed, or until there are no more tasks

package/src/duckdb/src/include/duckdb/parser/tableref/pivotref.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/parser/tableref.hpp"
+#include "duckdb/parser/query_node/select_node.hpp"
 
 namespace duckdb {
 
@@ -43,6 +44,8 @@ struct PivotColumn {
 	vector<PivotColumnEntry> entries;
 	//! The enum to read pivot values from (if any)
 	string pivot_enum;
+	//! Subquery (if any) - used during transform only
+	unique_ptr<QueryNode> subquery;
 
 	string ToString() const;
 	bool Equals(const PivotColumn &other) const;

package/src/duckdb/src/include/duckdb/parser/transformer.hpp

@@ -45,6 +45,7 @@ class Transformer {
 		string enum_name;
 		unique_ptr<SelectNode> base;
 		unique_ptr<ParsedExpression> column;
+		unique_ptr<QueryNode> subquery;
 	};
 
 public:
@@ -85,10 +86,13 @@ private:
 	bool GetNamedParam(const string &name, int32_t &index);
 	bool HasNamedParameters() const;
 
-	void AddPivotEntry(string enum_name, unique_ptr<SelectNode> source, unique_ptr<ParsedExpression> column);
+	void AddPivotEntry(string enum_name, unique_ptr<SelectNode> source, unique_ptr<ParsedExpression> column,
+	                   unique_ptr<QueryNode> subquery);
 	unique_ptr<SQLStatement> GenerateCreateEnumStmt(unique_ptr<CreatePivotEntry> entry);
 	bool HasPivotEntries();
 	idx_t PivotEntryCount();
+	vector<unique_ptr<CreatePivotEntry>> &GetPivotEntries();
+	void PivotEntryCheck(const string &type);
 	void ExtractCTEsRecursive(CommonTableExpressionMap &cte_map);
 
 private:

package/src/duckdb/src/include/duckdb/planner/column_binding.hpp

@@ -9,6 +9,8 @@
 #pragma once
 
 #include "duckdb/common/common.hpp"
+#include "duckdb/common/to_string.hpp"
+
 #include <functional>
 
 namespace duckdb {
@@ -23,6 +25,10 @@ struct ColumnBinding {
 	ColumnBinding(idx_t table, idx_t column) : table_index(table), column_index(column) {
 	}
 
+	string ToString() const {
+		return "#[" + to_string(table_index) + "." + to_string(column_index) + "]";
+	}
+
 	bool operator==(const ColumnBinding &rhs) const {
 		return table_index == rhs.table_index && column_index == rhs.column_index;
 	}

package/src/duckdb/src/include/duckdb/planner/expression/bound_columnref_expression.hpp

@@ -42,6 +42,7 @@ public:
 	}
 
 	string ToString() const override;
+	string GetName() const override;
 
 	bool Equals(const BaseExpression *other) const override;
 	hash_t Hash() const override;

package/src/duckdb/src/include/duckdb/planner/operator/logical_aggregate.hpp

@@ -47,6 +47,7 @@ public:
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	idx_t EstimateCardinality(ClientContext &context) override;
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override;

package/src/duckdb/src/include/duckdb/planner/operator/logical_column_data_get.hpp

@@ -34,6 +34,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override {

package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp

@@ -41,6 +41,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override {

package/src/duckdb/src/include/duckdb/planner/operator/logical_delete.hpp

@@ -29,6 +29,7 @@ public:
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	idx_t EstimateCardinality(ClientContext &context) override;
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	vector<ColumnBinding> GetColumnBindings() override;

package/src/duckdb/src/include/duckdb/planner/operator/logical_delim_get.hpp

@@ -36,6 +36,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override {

package/src/duckdb/src/include/duckdb/planner/operator/logical_dummy_scan.hpp

@@ -35,6 +35,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override {

package/src/duckdb/src/include/duckdb/planner/operator/logical_expression_get.hpp

@@ -41,6 +41,7 @@ public:
 		return expressions.size();
 	}
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override {

package/src/duckdb/src/include/duckdb/planner/operator/logical_insert.hpp

@@ -69,5 +69,6 @@ protected:
 
 	idx_t EstimateCardinality(ClientContext &context) override;
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/operator/logical_pivot.hpp

@@ -29,6 +29,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override;

package/src/duckdb/src/include/duckdb/planner/operator/logical_projection.hpp

@@ -27,6 +27,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override;

package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp

@@ -41,6 +41,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override {

package/src/duckdb/src/include/duckdb/planner/operator/logical_set_operation.hpp

@@ -41,6 +41,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override {

package/src/duckdb/src/include/duckdb/planner/operator/logical_unnest.hpp

@@ -29,6 +29,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override;

package/src/duckdb/src/include/duckdb/planner/operator/logical_update.hpp

@@ -34,6 +34,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	idx_t EstimateCardinality(ClientContext &context) override;
+	string GetName() const override;
 
 protected:
 	vector<ColumnBinding> GetColumnBindings() override;

package/src/duckdb/src/include/duckdb/planner/operator/logical_window.hpp

@@ -30,6 +30,7 @@ public:
 	void Serialize(FieldWriter &writer) const override;
 	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
 	vector<idx_t> GetTableIndex() const override;
+	string GetName() const override;
 
 protected:
 	void ResolveTypes() override;

package/src/duckdb/src/include/duckdb/storage/data_table.hpp

@@ -106,6 +106,7 @@ public:
 	void LocalMerge(ClientContext &context, RowGroupCollection &collection);
 	//! Creates an optimistic writer for this table - used for optimistically writing parallel appends
 	OptimisticDataWriter &CreateOptimisticWriter(ClientContext &context);
+	void FinalizeOptimisticWriter(ClientContext &context, OptimisticDataWriter &writer);
 
 	//! Delete the entries with the specified row identifier from the table
 	idx_t Delete(TableCatalogEntry &table, ClientContext &context, Vector &row_ids, idx_t count);

package/src/duckdb/src/include/duckdb/storage/optimistic_data_writer.hpp

@@ -0,0 +1,46 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/optimistic_data_writer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/table/row_group_collection.hpp"
+
+namespace duckdb {
+class PartialBlockManager;
+
+class OptimisticDataWriter {
+public:
+	OptimisticDataWriter(DataTable &table);
+	OptimisticDataWriter(DataTable &table, OptimisticDataWriter &parent);
+	~OptimisticDataWriter();
+
+	//! Write a new row group to disk (if possible)
+	void WriteNewRowGroup(RowGroupCollection &row_groups);
+	//! Write the last row group of a collection to disk
+	void WriteLastRowGroup(RowGroupCollection &row_groups);
+	//! Final flush of the optimistic writer - fully flushes the partial block manager
+	void FinalFlush();
+	//! Flushes a specific row group to disk
+	void FlushToDisk(RowGroup *row_group);
+	//! Merge the partially written blocks from one optimistic writer into another
+	void Merge(OptimisticDataWriter &other);
+	//! Rollback
+	void Rollback();
+
+private:
+	//! Prepare a write to disk
+	bool PrepareWrite();
+
+private:
+	//! The table
+	DataTable &table;
+	//! The partial block manager (if we created one yet)
+	unique_ptr<PartialBlockManager> partial_manager;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/partial_block_manager.hpp

@@ -48,6 +48,14 @@ public:
 	virtual void Flush(idx_t free_space_left) = 0;
 	virtual void Clear() {
 	}
+	virtual void Merge(PartialBlock &other, idx_t offset, idx_t other_size);
+
+public:
+	template <class TARGET>
+	TARGET &Cast() {
+		D_ASSERT(dynamic_cast<TARGET *>(this));
+		return (TARGET &)*this;
+	}
 };
 
 struct PartialBlockAllocation {
@@ -61,6 +69,8 @@ struct PartialBlockAllocation {
 	unique_ptr<PartialBlock> partial_block;
 };
 
+enum class CheckpointType { FULL_CHECKPOINT, APPEND_TO_TABLE };
+
 //! Enables sharing blocks across some scope. Scope is whatever we want to share
 //! blocks across. It may be an entire checkpoint or just a single row group.
 //! In any case, they must share a block manager.
@@ -75,7 +85,8 @@ public:
 	static constexpr const idx_t MAX_BLOCK_MAP_SIZE = 1u << 31;
 
 public:
-	PartialBlockManager(BlockManager &block_manager, uint32_t max_partial_block_size = DEFAULT_MAX_PARTIAL_BLOCK_SIZE,
+	PartialBlockManager(BlockManager &block_manager, CheckpointType checkpoint_type,
+	                    uint32_t max_partial_block_size = DEFAULT_MAX_PARTIAL_BLOCK_SIZE,
 	                    uint32_t max_use_count = DEFAULT_MAX_USE_COUNT);
 	virtual ~PartialBlockManager();
 
@@ -87,18 +98,25 @@ public:
 
 	virtual void AllocateBlock(PartialBlockState &state, uint32_t segment_size);
 
+	void Merge(PartialBlockManager &other);
 	//! Register a partially filled block that is filled with "segment_size" entries
 	void RegisterPartialBlock(PartialBlockAllocation &&allocation);
 
-	//! Clears all blocks
-	void Clear();
+	//! Clear remaining blocks without writing them to disk
+	void ClearBlocks();
+
+	//! Rollback all data written by this partial block manager
+	void Rollback();
 
 protected:
 	BlockManager &block_manager;
+	CheckpointType checkpoint_type;
 	//! A map of (available space -> PartialBlock) for partially filled blocks
 	//! This is a multimap because there might be outstanding partial blocks with
 	//! the same amount of left-over space
 	multimap<idx_t, unique_ptr<PartialBlock>> partially_filled_blocks;
+	//! The set of written blocks
+	unordered_set<block_id_t> written_blocks;
 
 	//! The maximum size (in bytes) at which a partial block will be considered a partial block
 	uint32_t max_partial_block_size;
@@ -109,6 +127,9 @@ protected:
 	//! If successful, returns true and returns the block_id and offset_in_block to write to
 	//! Otherwise, returns false
 	bool GetPartialBlock(idx_t segment_size, unique_ptr<PartialBlock> &state);
+
+	bool HasBlockAllocation(uint32_t segment_size);
+	void AddWrittenBlock(block_id_t block);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/column_checkpoint_state.hpp

@@ -15,6 +15,7 @@
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/storage/table/column_data.hpp"
 #include "duckdb/common/unordered_set.hpp"
+#include "duckdb/storage/partial_block_manager.hpp"
 
 namespace duckdb {
 class ColumnData;
@@ -41,7 +42,51 @@ public:
 
 	virtual void FlushSegment(unique_ptr<ColumnSegment> segment, idx_t segment_size);
 	virtual void WriteDataPointers(RowGroupWriter &writer);
-	virtual void GetBlockIds(unordered_set<block_id_t> &result);
+};
+
+struct PartialBlockForCheckpoint : public PartialBlock {
+	struct PartialColumnSegment {
+		PartialColumnSegment(ColumnData &data, ColumnSegment &segment, uint32_t offset_in_block)
+		    : data(data), segment(segment), offset_in_block(offset_in_block) {
+		}
+
+		ColumnData &data;
+		ColumnSegment &segment;
+		uint32_t offset_in_block;
+	};
+
+public:
+	PartialBlockForCheckpoint(ColumnData &data, ColumnSegment &segment, BlockManager &block_manager,
+	                          PartialBlockState state);
+	~PartialBlockForCheckpoint() override;
+
+	// We will copy all segment data into the memory of the shared block.
+	// Once the block is full (or checkpoint is complete) we'll invoke Flush().
+	// This will cause the block to get written to storage (via BlockManger::ConvertToPersistent),
+	// and all segments to have their references updated (via ColumnSegment::ConvertToPersistent)
+	BlockManager &block_manager;
+	shared_ptr<BlockHandle> block;
+	vector<PartialColumnSegment> segments;
+
+private:
+	struct UninitializedRegion {
+		idx_t start;
+		idx_t end;
+	};
+	vector<UninitializedRegion> uninitialized_regions;
+
+public:
+	bool IsFlushed();
+
+	void AddUninitializedRegion(idx_t start, idx_t end) override;
+
+	void Flush(idx_t free_space_left) override;
+
+	void Clear() override;
+
+	void Merge(PartialBlock &other, idx_t offset, idx_t other_size) override;
+
+	void AddSegmentToTail(ColumnData &data, ColumnSegment &segment, uint32_t offset_in_block);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp

@@ -39,12 +39,11 @@ class ColumnData {
 
 public:
 	ColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row, LogicalType type,
-	           ColumnData *parent);
-	ColumnData(ColumnData &other, idx_t start, ColumnData *parent);
+	           optional_ptr<ColumnData> parent);
 	virtual ~ColumnData();
 
 	//! The start row
-	const idx_t start;
+	idx_t start;
 	//! The count of the column data
 	idx_t count;
 	//! The block manager
@@ -56,7 +55,7 @@ public:
 	//! The type of the column
 	LogicalType type;
 	//! The parent column (if any)
-	ColumnData *parent;
+	optional_ptr<ColumnData> parent;
 
 public:
 	virtual bool CheckZonemap(ColumnScanState &state, TableFilter &filter) = 0;
@@ -70,6 +69,7 @@ public:
 
 	void IncrementVersion();
 
+	virtual void SetStart(idx_t new_start);
 	//! The root type of the column
 	const LogicalType &RootType() const;
 
@@ -122,13 +122,13 @@ public:
 	virtual unique_ptr<ColumnCheckpointState>
 	Checkpoint(RowGroup &row_group, PartialBlockManager &partial_block_manager, ColumnCheckpointInfo &checkpoint_info);
 
-	virtual void CheckpointScan(ColumnSegment *segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
+	virtual void CheckpointScan(ColumnSegment &segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
 	                            Vector &scan_vector);
 
 	virtual void DeserializeColumn(Deserializer &source);
 	static shared_ptr<ColumnData> Deserialize(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
 	                                          idx_t start_row, Deserializer &source, const LogicalType &type,
-	                                          ColumnData *parent);
+	                                          optional_ptr<ColumnData> parent);
 
 	virtual void GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, TableStorageInfo &result);
 	virtual void Verify(RowGroup &parent);
@@ -136,12 +136,11 @@ public:
 	bool CheckZonemap(TableFilter &filter);
 
 	static shared_ptr<ColumnData> CreateColumn(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
-	                                           idx_t start_row, const LogicalType &type, ColumnData *parent = nullptr);
-	static shared_ptr<ColumnData> CreateColumn(ColumnData &other, idx_t start_row, ColumnData *parent = nullptr);
+	                                           idx_t start_row, const LogicalType &type,
+	                                           optional_ptr<ColumnData> parent = nullptr);
 	static unique_ptr<ColumnData> CreateColumnUnique(BlockManager &block_manager, DataTableInfo &info,
 	                                                 idx_t column_index, idx_t start_row, const LogicalType &type,
-	                                                 ColumnData *parent = nullptr);
-	static unique_ptr<ColumnData> CreateColumnUnique(ColumnData &other, idx_t start_row, ColumnData *parent = nullptr);
+	                                                 optional_ptr<ColumnData> parent = nullptr);
 
 	void MergeStatistics(const BaseStatistics &other);
 	void MergeIntoStatistics(BaseStatistics &other);