duckdb 0.7.2-dev2867.0 → 0.7.2-dev3117.0

package/src/duckdb/src/storage/table/column_checkpoint_state.cpp

@@ -22,86 +22,103 @@ unique_ptr<BaseStatistics> ColumnCheckpointState::GetStatistics() {
 	return std::move(global_stats);
 }
 
-struct PartialBlockForCheckpoint : PartialBlock {
-	struct PartialColumnSegment {
-		ColumnData *data;
-		ColumnSegment *segment;
-		uint32_t offset_in_block;
-	};
-
-public:
-	PartialBlockForCheckpoint(ColumnData *first_data, ColumnSegment *first_segment, BlockManager &block_manager,
-	                          PartialBlockState state)
-	    : PartialBlock(state), first_data(first_data), first_segment(first_segment), block_manager(block_manager) {
-	}
+PartialBlockForCheckpoint::PartialBlockForCheckpoint(ColumnData &data, ColumnSegment &segment,
+                                                     BlockManager &block_manager, PartialBlockState state)
+    : PartialBlock(state), block_manager(block_manager), block(segment.block) {
+	AddSegmentToTail(data, segment, 0);
+}
 
-	~PartialBlockForCheckpoint() override {
-		D_ASSERT(IsFlushed() || Exception::UncaughtException());
-	}
+PartialBlockForCheckpoint::~PartialBlockForCheckpoint() {
+	D_ASSERT(IsFlushed() || Exception::UncaughtException());
+}
 
-	// We will copy all subsequent segment data into the memory corresponding
-	// to the first segment. Once the block is full (or checkpoint is complete)
-	// we'll invoke Flush(), which will cause
-	// the block to get written to storage (via BlockManger::ConvertToPersistent),
-	// and all segments to have their references updated
-	// (via ColumnSegment::ConvertToPersistent)
-	ColumnData *first_data;
-	ColumnSegment *first_segment;
-	BlockManager &block_manager;
-	vector<PartialColumnSegment> tail_segments;
-
-private:
-	struct UninitializedRegion {
-		idx_t start;
-		idx_t end;
-	};
-	vector<UninitializedRegion> uninitialized_regions;
-
-public:
-	bool IsFlushed() {
-		// first_segment is zeroed on Flush
-		return !first_segment;
-	}
+bool PartialBlockForCheckpoint::IsFlushed() {
+	// segments are cleared on Flush
+	return segments.empty();
+}
 
-	void AddUninitializedRegion(idx_t start, idx_t end) override {
-		uninitialized_regions.push_back({start, end});
-	}
+void PartialBlockForCheckpoint::AddUninitializedRegion(idx_t start, idx_t end) {
+	uninitialized_regions.push_back({start, end});
+}
 
-	void Flush(idx_t free_space_left) override {
-		// At this point, we've already copied all data from tail_segments
-		// into the page owned by first_segment. We flush all segment data to
-		// disk with the following call.
-		if (free_space_left > 0 || !uninitialized_regions.empty()) {
-			auto handle = block_manager.buffer_manager.Pin(first_segment->block);
-			// memset any uninitialized regions
-			for (auto &uninitialized : uninitialized_regions) {
-				memset(handle.Ptr() + uninitialized.start, 0, uninitialized.end - uninitialized.start);
-			}
-			// memset any free space at the end of the block to 0 prior to writing to disk
-			memset(handle.Ptr() + Storage::BLOCK_SIZE - free_space_left, 0, free_space_left);
+void PartialBlockForCheckpoint::Flush(idx_t free_space_left) {
+	if (IsFlushed()) {
+		throw InternalException("Flush called on partial block that was already flushed");
+	}
+	// if we have any free space or uninitialized regions we need to zero-initialize them
+	if (free_space_left > 0 || !uninitialized_regions.empty()) {
+		auto handle = block_manager.buffer_manager.Pin(block);
+		// memset any uninitialized regions
+		for (auto &uninitialized : uninitialized_regions) {
+			memset(handle.Ptr() + uninitialized.start, 0, uninitialized.end - uninitialized.start);
 		}
-		first_data->IncrementVersion();
-		first_segment->ConvertToPersistent(&block_manager, state.block_id);
-		// Now that the page is persistent, update tail_segments to point to the
-		// newly persistent block.
-		for (auto e : tail_segments) {
-			e.data->IncrementVersion();
-			e.segment->MarkAsPersistent(first_segment->block, e.offset_in_block);
+		// memset any free space at the end of the block to 0 prior to writing to disk
+		memset(handle.Ptr() + Storage::BLOCK_SIZE - free_space_left, 0, free_space_left);
+	}
+	// At this point, we've already copied all data from tail_segments
+	// into the page owned by first_segment. We flush all segment data to
+	// disk with the following call.
+	// persist the first segment to disk and point the remaining segments to the same block
+	bool fetch_new_block = state.block_id == INVALID_BLOCK;
+	if (fetch_new_block) {
+		state.block_id = block_manager.GetFreeBlockId();
+	}
+	for (idx_t i = 0; i < segments.size(); i++) {
+		auto &segment = segments[i];
+		segment.data.IncrementVersion();
+		if (i == 0) {
+			// the first segment is converted to persistent - this writes the data for ALL segments to disk
+			D_ASSERT(segment.offset_in_block == 0);
+			segment.segment.ConvertToPersistent(&block_manager, state.block_id);
+			// update the block after it has been converted to a persistent segment
+			block = segment.segment.block;
+		} else {
+			// subsequent segments are MARKED as persistent - they don't need to be rewritten
+			segment.segment.MarkAsPersistent(block, segment.offset_in_block);
+			if (fetch_new_block) {
+				// if we fetched a new block we need to increase the reference count to the block
+				block_manager.IncreaseBlockReferenceCount(state.block_id);
+			}
 		}
-		first_segment = nullptr;
-		tail_segments.clear();
 	}
+	Clear();
+}
 
-	void Clear() override {
-		first_data = nullptr;
-		first_segment = nullptr;
-		tail_segments.clear();
+void PartialBlockForCheckpoint::Clear() {
+	uninitialized_regions.clear();
+	block.reset();
+	segments.clear();
+}
+
+void PartialBlockForCheckpoint::Merge(PartialBlock &other_p, idx_t offset, idx_t other_size) {
+	auto &other = other_p.Cast<PartialBlockForCheckpoint>();
+
+	auto &buffer_manager = block_manager.buffer_manager;
+	// pin the source block
+	auto old_handle = buffer_manager.Pin(other.block);
+	// pin the target block
+	auto new_handle = buffer_manager.Pin(block);
+	// memcpy the contents of the old block to the new block
+	memcpy(new_handle.Ptr() + offset, old_handle.Ptr(), other_size);
+
+	// now copy over all of the segments to the new block
+	// move over the uninitialized regions
+	for (auto &region : other.uninitialized_regions) {
+		region.start += offset;
+		region.end += offset;
+		uninitialized_regions.push_back(region);
 	}
 
-	void AddSegmentToTail(ColumnData *data, ColumnSegment *segment, uint32_t offset_in_block) {
-		tail_segments.push_back({data, segment, offset_in_block});
+	// move over the segments
+	for (auto &segment : other.segments) {
+		AddSegmentToTail(segment.data, segment.segment, segment.offset_in_block + offset);
 	}
-};
+	other.Clear();
+}
+
+void PartialBlockForCheckpoint::AddSegmentToTail(ColumnData &data, ColumnSegment &segment, uint32_t offset_in_block) {
+	segments.emplace_back(data, segment, offset_in_block);
+}
 
 void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_t segment_size) {
 	D_ASSERT(segment_size <= Storage::BLOCK_SIZE);
@@ -128,14 +145,14 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 		if (allocation.partial_block) {
 			// Use an existing block.
 			D_ASSERT(offset_in_block > 0);
-			auto pstate = (PartialBlockForCheckpoint *)allocation.partial_block.get();
+			auto &pstate = allocation.partial_block->Cast<PartialBlockForCheckpoint>();
 			// pin the source block
 			auto old_handle = buffer_manager.Pin(segment->block);
 			// pin the target block
-			auto new_handle = buffer_manager.Pin(pstate->first_segment->block);
+			auto new_handle = buffer_manager.Pin(pstate.block);
 			// memcpy the contents of the old block to the new block
 			memcpy(new_handle.Ptr() + offset_in_block, old_handle.Ptr(), segment_size);
-			pstate->AddSegmentToTail(&column_data, segment.get(), offset_in_block);
+			pstate.AddSegmentToTail(column_data, *segment, offset_in_block);
 		} else {
 			// Create a new block for future reuse.
 			if (segment->SegmentSize() != Storage::BLOCK_SIZE) {
@@ -146,7 +163,7 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 			}
 			D_ASSERT(offset_in_block == 0);
 			allocation.partial_block = make_uniq<PartialBlockForCheckpoint>(
-			    &column_data, segment.get(), *allocation.block_manager, allocation.state);
+			    column_data, *segment, *allocation.block_manager, allocation.state);
 		}
 		// Writer will decide whether to reuse this block.
 		partial_block_manager.RegisterPartialBlock(std::move(allocation));
@@ -180,13 +197,4 @@ void ColumnCheckpointState::WriteDataPointers(RowGroupWriter &writer) {
 	writer.WriteColumnDataPointers(*this);
 }
 
-void ColumnCheckpointState::GetBlockIds(unordered_set<block_id_t> &result) {
-	for (auto &pointer : data_pointers) {
-		if (pointer.block_pointer.block_id == INVALID_BLOCK) {
-			continue;
-		}
-		result.insert(pointer.block_pointer.block_id);
-	}
-}
-
 } // namespace duckdb

package/src/duckdb/src/storage/table/column_data.cpp

@@ -21,7 +21,7 @@
 namespace duckdb {
 
 ColumnData::ColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
-                       LogicalType type_p, ColumnData *parent)
+                       LogicalType type_p, optional_ptr<ColumnData> parent)
     : start(start_row), count(0), block_manager(block_manager), info(info), column_index(column_index),
       type(std::move(type_p)), parent(parent), version(0) {
 	if (!parent) {
@@ -29,24 +29,17 @@ ColumnData::ColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t c
 	}
 }
 
-ColumnData::ColumnData(ColumnData &other, idx_t start, ColumnData *parent)
-    : start(start), count(other.count), block_manager(other.block_manager), info(other.info),
-      column_index(other.column_index), type(std::move(other.type)), parent(parent),
-      version(parent ? parent->version + 1 : 0) {
-	if (other.updates) {
-		updates = make_uniq<UpdateSegment>(*other.updates, *this);
-	}
-	if (other.stats) {
-		stats = make_uniq<SegmentStatistics>(other.stats->statistics.Copy());
-	}
+ColumnData::~ColumnData() {
+}
+
+void ColumnData::SetStart(idx_t new_start) {
+	this->start = new_start;
 	idx_t offset = 0;
-	for (auto &segment : other.data.Segments()) {
-		this->data.AppendSegment(ColumnSegment::CreateSegment(segment, start + offset));
+	for (auto &segment : data.Segments()) {
+		segment.start = start + offset;
 		offset += segment.count;
 	}
-}
-
-ColumnData::~ColumnData() {
+	data.Reinitialize();
 }
 
 DatabaseInstance &ColumnData::GetDatabase() const {
@@ -278,8 +271,8 @@ void ColumnData::InitializeAppend(ColumnAppendState &state) {
 		AppendTransientSegment(l, start);
 	}
 	auto segment = data.GetLastSegment(l);
-	if (segment->segment_type == ColumnSegmentType::PERSISTENT) {
-		// no transient segments yet
+	if (segment->segment_type == ColumnSegmentType::PERSISTENT || !segment->function.get().init_append) {
+		// we cannot append to this segment - append a new segment
 		auto total_rows = segment->start + segment->count;
 		AppendTransientSegment(l, total_rows);
 		state.current = data.GetLastSegment(l);
@@ -418,9 +411,9 @@ unique_ptr<ColumnCheckpointState> ColumnData::CreateCheckpointState(RowGroup &ro
 	return make_uniq<ColumnCheckpointState>(row_group, *this, partial_block_manager);
 }
 
-void ColumnData::CheckpointScan(ColumnSegment *segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
+void ColumnData::CheckpointScan(ColumnSegment &segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
                                 Vector &scan_vector) {
-	segment->Scan(state, count, scan_vector, 0, true);
+	segment.Scan(state, count, scan_vector, 0, true);
 	if (updates) {
 		scan_vector.Flatten(count);
 		updates->FetchCommittedRange(state.row_index - row_group_start, count, scan_vector);
@@ -489,7 +482,7 @@ void ColumnData::DeserializeColumn(Deserializer &source) {
 
 shared_ptr<ColumnData> ColumnData::Deserialize(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
                                                idx_t start_row, Deserializer &source, const LogicalType &type,
-                                               ColumnData *parent) {
+                                               optional_ptr<ColumnData> parent) {
 	auto entry = ColumnData::CreateColumn(block_manager, info, column_index, start_row, type, parent);
 	entry->DeserializeColumn(source);
 	return entry;
@@ -565,48 +558,29 @@ void ColumnData::Verify(RowGroup &parent) {
 
 template <class RET, class OP>
 static RET CreateColumnInternal(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
-                                const LogicalType &type, ColumnData *parent) {
+                                const LogicalType &type, optional_ptr<ColumnData> parent) {
 	if (type.InternalType() == PhysicalType::STRUCT) {
 		return OP::template Create<StructColumnData>(block_manager, info, column_index, start_row, type, parent);
 	} else if (type.InternalType() == PhysicalType::LIST) {
 		return OP::template Create<ListColumnData>(block_manager, info, column_index, start_row, type, parent);
 	} else if (type.id() == LogicalTypeId::VALIDITY) {
-		return OP::template Create<ValidityColumnData>(block_manager, info, column_index, start_row, parent);
+		return OP::template Create<ValidityColumnData>(block_manager, info, column_index, start_row, *parent);
 	}
 	return OP::template Create<StandardColumnData>(block_manager, info, column_index, start_row, type, parent);
 }
 
-template <class RET, class OP>
-static RET CreateColumnInternal(ColumnData &other, idx_t start_row, ColumnData *parent) {
-	if (other.type.InternalType() == PhysicalType::STRUCT) {
-		return OP::template Create<StructColumnData>(other, start_row, parent);
-	} else if (other.type.InternalType() == PhysicalType::LIST) {
-		return OP::template Create<ListColumnData>(other, start_row, parent);
-	} else if (other.type.id() == LogicalTypeId::VALIDITY) {
-		return OP::template Create<ValidityColumnData>(other, start_row, parent);
-	}
-	return OP::template Create<StandardColumnData>(other, start_row, parent);
-}
-
 shared_ptr<ColumnData> ColumnData::CreateColumn(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
-                                                idx_t start_row, const LogicalType &type, ColumnData *parent) {
+                                                idx_t start_row, const LogicalType &type,
+                                                optional_ptr<ColumnData> parent) {
 	return CreateColumnInternal<shared_ptr<ColumnData>, SharedConstructor>(block_manager, info, column_index, start_row,
 	                                                                       type, parent);
 }
 
-shared_ptr<ColumnData> ColumnData::CreateColumn(ColumnData &other, idx_t start_row, ColumnData *parent) {
-	return CreateColumnInternal<shared_ptr<ColumnData>, SharedConstructor>(other, start_row, parent);
-}
-
 unique_ptr<ColumnData> ColumnData::CreateColumnUnique(BlockManager &block_manager, DataTableInfo &info,
                                                       idx_t column_index, idx_t start_row, const LogicalType &type,
-                                                      ColumnData *parent) {
+                                                      optional_ptr<ColumnData> parent) {
 	return CreateColumnInternal<unique_ptr<ColumnData>, UniqueConstructor>(block_manager, info, column_index, start_row,
 	                                                                       type, parent);
 }
 
-unique_ptr<ColumnData> ColumnData::CreateColumnUnique(ColumnData &other, idx_t start_row, ColumnData *parent) {
-	return CreateColumnInternal<unique_ptr<ColumnData>, UniqueConstructor>(other, start_row, parent);
-}
-
 } // namespace duckdb

package/src/duckdb/src/storage/table/column_data_checkpointer.cpp

@@ -43,16 +43,16 @@ ColumnCheckpointState &ColumnDataCheckpointer::GetCheckpointState() {
 void ColumnDataCheckpointer::ScanSegments(const std::function<void(Vector &, idx_t)> &callback) {
 	Vector scan_vector(intermediate.GetType(), nullptr);
 	for (idx_t segment_idx = 0; segment_idx < nodes.size(); segment_idx++) {
-		auto segment = nodes[segment_idx].node.get();
+		auto &segment = *nodes[segment_idx].node;
 		ColumnScanState scan_state;
-		scan_state.current = segment;
-		segment->InitializeScan(scan_state);
+		scan_state.current = &segment;
+		segment.InitializeScan(scan_state);
 
-		for (idx_t base_row_index = 0; base_row_index < segment->count; base_row_index += STANDARD_VECTOR_SIZE) {
+		for (idx_t base_row_index = 0; base_row_index < segment.count; base_row_index += STANDARD_VECTOR_SIZE) {
 			scan_vector.Reference(intermediate);
 
-			idx_t count = MinValue<idx_t>(segment->count - base_row_index, STANDARD_VECTOR_SIZE);
-			scan_state.row_index = segment->start + base_row_index;
+			idx_t count = MinValue<idx_t>(segment.count - base_row_index, STANDARD_VECTOR_SIZE);
+			scan_state.row_index = segment.start + base_row_index;
 
 			col_data.CheckpointScan(segment, scan_state, row_group.start, count, scan_vector);
 
@@ -250,7 +250,7 @@ void ColumnDataCheckpointer::Checkpoint(vector<SegmentNode<ColumnSegment>> nodes
 		// no changes: only need to write the metadata for this column
 		WritePersistentSegments();
 	} else {
-		// there are changes: rewrite the set of columns
+		// there are changes: rewrite the set of columns);
 		WriteToDisk();
 	}
 }

package/src/duckdb/src/storage/table/column_segment.cpp

@@ -172,7 +172,7 @@ void ColumnSegment::RevertAppend(idx_t start_row) {
 //===--------------------------------------------------------------------===//
 // Convert To Persistent
 //===--------------------------------------------------------------------===//
-void ColumnSegment::ConvertToPersistent(BlockManager *block_manager, block_id_t block_id_p) {
+void ColumnSegment::ConvertToPersistent(optional_ptr<BlockManager> block_manager, block_id_t block_id_p) {
 	D_ASSERT(segment_type == ColumnSegmentType::TRANSIENT);
 	segment_type = ColumnSegmentType::PERSISTENT;
 

package/src/duckdb/src/storage/table/list_column_data.cpp

@@ -8,19 +8,19 @@
 namespace duckdb {
 
 ListColumnData::ListColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
-                               LogicalType type_p, ColumnData *parent)
+                               LogicalType type_p, optional_ptr<ColumnData> parent)
     : ColumnData(block_manager, info, column_index, start_row, std::move(type_p), parent),
-      validity(block_manager, info, 0, start_row, this) {
+      validity(block_manager, info, 0, start_row, *this) {
 	D_ASSERT(type.InternalType() == PhysicalType::LIST);
 	auto &child_type = ListType::GetChildType(type);
 	// the child column, with column index 1 (0 is the validity mask)
 	child_column = ColumnData::CreateColumnUnique(block_manager, info, 1, start_row, child_type, this);
 }
 
-ListColumnData::ListColumnData(ColumnData &original, idx_t start_row, ColumnData *parent)
-    : ColumnData(original, start_row, parent), validity(((ListColumnData &)original).validity, start_row, this) {
-	auto &list_data = (ListColumnData &)original;
-	child_column = ColumnData::CreateColumnUnique(*list_data.child_column, start_row, this);
+void ListColumnData::SetStart(idx_t new_start) {
+	ColumnData::SetStart(new_start);
+	child_column->SetStart(new_start);
+	validity.SetStart(new_start);
 }
 
 bool ListColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {
@@ -322,11 +322,6 @@ public:
 		validity_state->WriteDataPointers(writer);
 		child_state->WriteDataPointers(writer);
 	}
-	void GetBlockIds(unordered_set<block_id_t> &result) override {
-		ColumnCheckpointState::GetBlockIds(result);
-		validity_state->GetBlockIds(result);
-		child_state->GetBlockIds(result);
-	}
 };
 
 unique_ptr<ColumnCheckpointState> ListColumnData::CreateCheckpointState(RowGroup &row_group,

package/src/duckdb/src/storage/table/row_group.cpp

@@ -45,16 +45,15 @@ RowGroup::RowGroup(RowGroupCollection &collection, RowGroupPointer &&pointer)
 	Verify();
 }
 
-RowGroup::RowGroup(RowGroup &row_group, RowGroupCollection &collection, idx_t start)
-    : SegmentBase<RowGroup>(start, row_group.count.load()), collection(collection),
-      version_info(std::move(row_group.version_info)) {
-	for (auto &column : row_group.GetColumns()) {
-		this->columns.push_back(ColumnData::CreateColumn(*column, start));
+void RowGroup::MoveToCollection(RowGroupCollection &collection, idx_t new_start) {
+	this->collection = collection;
+	this->start = new_start;
+	for (auto &column : GetColumns()) {
+		column->SetStart(new_start);
 	}
 	if (version_info) {
-		version_info->SetStart(start);
+		version_info->SetStart(new_start);
 	}
-	Verify();
 }
 
 void VersionNode::SetStart(idx_t start) {
@@ -101,8 +100,8 @@ ColumnData &RowGroup::GetColumn(idx_t c) {
 	if (column_pointers.size() != columns.size()) {
 		throw InternalException("Lazy loading a column but the pointer was not set");
 	}
-	auto &block_manager = collection.GetBlockManager();
-	auto &types = collection.GetTypes();
+	auto &block_manager = GetCollection().GetBlockManager();
+	auto &types = GetCollection().GetTypes();
 	auto &block_pointer = column_pointers[c];
 	MetaBlockReader column_data_reader(block_manager, block_pointer.block_id);
 	column_data_reader.offset = block_pointer.offset;
@@ -113,14 +112,14 @@ ColumnData &RowGroup::GetColumn(idx_t c) {
 }
 
 DatabaseInstance &RowGroup::GetDatabase() {
-	return collection.GetDatabase();
+	return GetCollection().GetDatabase();
 }
 
 BlockManager &RowGroup::GetBlockManager() {
-	return collection.GetBlockManager();
+	return GetCollection().GetBlockManager();
 }
 DataTableInfo &RowGroup::GetTableInfo() {
-	return collection.GetTableInfo();
+	return GetCollection().GetTableInfo();
 }
 
 void RowGroup::InitializeEmpty(const vector<LogicalType> &types) {
@@ -231,7 +230,7 @@ unique_ptr<RowGroup> RowGroup::AlterType(RowGroupCollection &new_collection, con
 	column_data->InitializeAppend(append_state);
 
 	// scan the original table, and fill the new column with the transformed value
-	scan_state.Initialize(collection.GetTypes());
+	scan_state.Initialize(GetCollection().GetTypes());
 	InitializeScan(scan_state);
 
 	DataChunk append_chunk;
@@ -537,7 +536,7 @@ void RowGroup::Scan(TransactionData transaction, CollectionScanState &state, Dat
 }
 
 void RowGroup::ScanCommitted(CollectionScanState &state, DataChunk &result, TableScanType type) {
-	auto &transaction_manager = DuckTransactionManager::Get(collection.GetAttached());
+	auto &transaction_manager = DuckTransactionManager::Get(GetCollection().GetAttached());
 
 	auto lowest_active_start = transaction_manager.LowestActiveStart();
 	auto lowest_active_id = transaction_manager.LowestActiveId();

package/src/duckdb/src/storage/table/row_group_collection.cpp

@@ -96,6 +96,10 @@ RowGroup *RowGroupCollection::GetRowGroup(int64_t index) {
 	return (RowGroup *)row_groups->GetSegmentByIndex(index);
 }
 
+idx_t RowGroupCollection::RowGroupCount() {
+	return row_groups->GetSegmentCount();
+}
+
 void RowGroupCollection::Verify() {
 #ifdef DEBUG
 	idx_t current_total_rows = 0;
@@ -444,10 +448,12 @@ void RowGroupCollection::RevertAppendInternal(idx_t start_row, idx_t count) {
 void RowGroupCollection::MergeStorage(RowGroupCollection &data) {
 	D_ASSERT(data.types == types);
 	auto index = row_start + total_rows.load();
-	for (auto &row_group : data.row_groups->Segments()) {
-		auto new_group = make_uniq<RowGroup>(row_group, *this, index);
-		index += new_group->count;
-		row_groups->AppendSegment(std::move(new_group));
+	auto segments = data.row_groups->MoveSegments();
+	for (auto &entry : segments) {
+		auto &row_group = entry.node;
+		row_group->MoveToCollection(*this, index);
+		index += row_group->count;
+		row_groups->AppendSegment(std::move(row_group));
 	}
 	stats.MergeStats(data.stats);
 	total_rows += data.total_rows.load();

package/src/duckdb/src/storage/table/standard_column_data.cpp

@@ -10,13 +10,14 @@
 namespace duckdb {
 
 StandardColumnData::StandardColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
-                                       idx_t start_row, LogicalType type, ColumnData *parent)
+                                       idx_t start_row, LogicalType type, optional_ptr<ColumnData> parent)
     : ColumnData(block_manager, info, column_index, start_row, std::move(type), parent),
-      validity(block_manager, info, 0, start_row, this) {
+      validity(block_manager, info, 0, start_row, *this) {
 }
 
-StandardColumnData::StandardColumnData(ColumnData &original, idx_t start_row, ColumnData *parent)
-    : ColumnData(original, start_row, parent), validity(((StandardColumnData &)original).validity, start_row, this) {
+void StandardColumnData::SetStart(idx_t new_start) {
+	ColumnData::SetStart(new_start);
+	validity.SetStart(new_start);
 }
 
 bool StandardColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {
@@ -176,11 +177,6 @@ public:
 		ColumnCheckpointState::WriteDataPointers(writer);
 		validity_state->WriteDataPointers(writer);
 	}
-
-	void GetBlockIds(unordered_set<block_id_t> &result) override {
-		ColumnCheckpointState::GetBlockIds(result);
-		validity_state->GetBlockIds(result);
-	}
 };
 
 unique_ptr<ColumnCheckpointState>
@@ -198,7 +194,7 @@ unique_ptr<ColumnCheckpointState> StandardColumnData::Checkpoint(RowGroup &row_g
 	return base_state;
 }
 
-void StandardColumnData::CheckpointScan(ColumnSegment *segment, ColumnScanState &state, idx_t row_group_start,
+void StandardColumnData::CheckpointScan(ColumnSegment &segment, ColumnScanState &state, idx_t row_group_start,
                                         idx_t count, Vector &scan_vector) {
 	ColumnData::CheckpointScan(segment, state, row_group_start, count, scan_vector);
 

package/src/duckdb/src/storage/table/struct_column_data.cpp

@@ -8,9 +8,9 @@
 namespace duckdb {
 
 StructColumnData::StructColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
-                                   idx_t start_row, LogicalType type_p, ColumnData *parent)
+                                   idx_t start_row, LogicalType type_p, optional_ptr<ColumnData> parent)
     : ColumnData(block_manager, info, column_index, start_row, std::move(type_p), parent),
-      validity(block_manager, info, 0, start_row, this) {
+      validity(block_manager, info, 0, start_row, *this) {
 	D_ASSERT(type.InternalType() == PhysicalType::STRUCT);
 	auto &child_types = StructType::GetChildTypes(type);
 	D_ASSERT(child_types.size() > 0);
@@ -23,12 +23,12 @@ StructColumnData::StructColumnData(BlockManager &block_manager, DataTableInfo &i
 	}
 }
 
-StructColumnData::StructColumnData(ColumnData &original, idx_t start_row, ColumnData *parent)
-    : ColumnData(original, start_row, parent), validity(((StructColumnData &)original).validity, start_row, this) {
-	auto &struct_data = (StructColumnData &)original;
-	for (auto &child_col : struct_data.sub_columns) {
-		sub_columns.push_back(ColumnData::CreateColumnUnique(*child_col, start_row, this));
+void StructColumnData::SetStart(idx_t new_start) {
+	this->start = new_start;
+	for (auto &sub_column : sub_columns) {
+		sub_column->SetStart(new_start);
 	}
+	validity.SetStart(new_start);
 }
 
 bool StructColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {
@@ -246,12 +246,6 @@ public:
 			state->WriteDataPointers(writer);
 		}
 	}
-	void GetBlockIds(unordered_set<block_id_t> &result) override {
-		validity_state->GetBlockIds(result);
-		for (auto &state : child_states) {
-			state->GetBlockIds(result);
-		}
-	}
 };
 
 unique_ptr<ColumnCheckpointState> StructColumnData::CreateCheckpointState(RowGroup &row_group,

package/src/duckdb/src/storage/table/update_segment.cpp

@@ -37,31 +37,6 @@ UpdateSegment::UpdateSegment(ColumnData &column_data)
 	this->statistics_update_function = GetStatisticsUpdateFunction(physical_type);
 }
 
-UpdateSegment::UpdateSegment(UpdateSegment &other, ColumnData &owner)
-    : column_data(owner), root(std::move(other.root)), stats(std::move(other.stats)), type_size(other.type_size) {
-
-	this->heap.Move(other.heap);
-	// update the segment links
-	if (root) {
-		for (idx_t i = 0; i < RowGroup::ROW_GROUP_VECTOR_COUNT; i++) {
-			if (!root->info[i]) {
-				continue;
-			}
-			for (auto info = root->info[i]->info.get(); info; info = info->next) {
-				info->segment = this;
-			}
-		}
-	}
-	initialize_update_function = other.initialize_update_function;
-	merge_update_function = other.merge_update_function;
-	fetch_update_function = other.fetch_update_function;
-	fetch_committed_function = other.fetch_committed_function;
-	fetch_committed_range = other.fetch_committed_range;
-	fetch_row_function = other.fetch_row_function;
-	rollback_update_function = other.rollback_update_function;
-	statistics_update_function = other.statistics_update_function;
-}
-
 UpdateSegment::~UpdateSegment() {
 }
 

package/src/duckdb/src/storage/table/validity_column_data.cpp

@@ -5,12 +5,8 @@
 namespace duckdb {
 
 ValidityColumnData::ValidityColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
-                                       idx_t start_row, ColumnData *parent)
-    : ColumnData(block_manager, info, column_index, start_row, LogicalType(LogicalTypeId::VALIDITY), parent) {
-}
-
-ValidityColumnData::ValidityColumnData(ColumnData &original, idx_t start_row, ColumnData *parent)
-    : ColumnData(original, start_row, parent) {
+                                       idx_t start_row, ColumnData &parent)
+    : ColumnData(block_manager, info, column_index, start_row, LogicalType(LogicalTypeId::VALIDITY), &parent) {
 }
 
 bool ValidityColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {