duckdb 0.7.2-dev2867.0 → 0.7.2-dev2995.0

package/src/duckdb/src/storage/optimistic_data_writer.cpp

@@ -0,0 +1,96 @@
+#include "duckdb/storage/optimistic_data_writer.hpp"
+#include "duckdb/storage/table/column_segment.hpp"
+#include "duckdb/storage/partial_block_manager.hpp"
+#include "duckdb/storage/table/column_checkpoint_state.hpp"
+
+namespace duckdb {
+
+OptimisticDataWriter::OptimisticDataWriter(DataTable &table) : table(table) {
+}
+
+OptimisticDataWriter::OptimisticDataWriter(DataTable &table, OptimisticDataWriter &parent) : table(table) {
+	if (parent.partial_manager) {
+		parent.partial_manager->ClearBlocks();
+	}
+}
+
+OptimisticDataWriter::~OptimisticDataWriter() {
+}
+
+bool OptimisticDataWriter::PrepareWrite() {
+	// check if we should pre-emptively write the table to disk
+	if (table.info->IsTemporary() || StorageManager::Get(table.info->db).InMemory()) {
+		return false;
+	}
+	// we should! write the second-to-last row group to disk
+	// allocate the partial block-manager if none is allocated yet
+	if (!partial_manager) {
+		auto &block_manager = table.info->table_io_manager->GetBlockManagerForRowData();
+		partial_manager = make_uniq<PartialBlockManager>(block_manager, CheckpointType::APPEND_TO_TABLE);
+	}
+	return true;
+}
+
+void OptimisticDataWriter::WriteNewRowGroup(RowGroupCollection &row_groups) {
+	// we finished writing a complete row group
+	if (!PrepareWrite()) {
+		return;
+	}
+	// flush second-to-last row group
+	auto row_group = row_groups.GetRowGroup(-2);
+	FlushToDisk(row_group);
+}
+
+void OptimisticDataWriter::WriteLastRowGroup(RowGroupCollection &row_groups) {
+	// we finished writing a complete row group
+	if (!PrepareWrite()) {
+		return;
+	}
+	// flush second-to-last row group
+	auto row_group = row_groups.GetRowGroup(-1);
+	if (!row_group) {
+		return;
+	}
+	FlushToDisk(row_group);
+}
+
+void OptimisticDataWriter::FlushToDisk(RowGroup *row_group) {
+	if (!row_group) {
+		throw InternalException("FlushToDisk called without a RowGroup");
+	}
+	//! The set of column compression types (if any)
+	vector<CompressionType> compression_types;
+	D_ASSERT(compression_types.empty());
+	for (auto &column : table.column_definitions) {
+		compression_types.push_back(column.CompressionType());
+	}
+	row_group->WriteToDisk(*partial_manager, compression_types);
+}
+
+void OptimisticDataWriter::Merge(OptimisticDataWriter &other) {
+	if (!other.partial_manager) {
+		return;
+	}
+	if (!partial_manager) {
+		partial_manager = std::move(other.partial_manager);
+		return;
+	}
+	partial_manager->Merge(*other.partial_manager);
+	other.partial_manager.reset();
+}
+
+void OptimisticDataWriter::FinalFlush() {
+	if (partial_manager) {
+		partial_manager->FlushPartialBlocks();
+		partial_manager.reset();
+	}
+}
+
+void OptimisticDataWriter::Rollback() {
+	if (partial_manager) {
+		partial_manager->Rollback();
+		partial_manager.reset();
+	}
+}
+
+} // namespace duckdb

package/src/duckdb/src/storage/partial_block_manager.cpp

@@ -2,9 +2,10 @@
 
 namespace duckdb {
 
-PartialBlockManager::PartialBlockManager(BlockManager &block_manager, uint32_t max_partial_block_size,
-                                         uint32_t max_use_count)
-    : block_manager(block_manager), max_partial_block_size(max_partial_block_size), max_use_count(max_use_count) {
+PartialBlockManager::PartialBlockManager(BlockManager &block_manager, CheckpointType checkpoint_type,
+                                         uint32_t max_partial_block_size, uint32_t max_use_count)
+    : block_manager(block_manager), checkpoint_type(checkpoint_type), max_partial_block_size(max_partial_block_size),
+      max_use_count(max_use_count) {
 }
 PartialBlockManager::~PartialBlockManager() {
 }
@@ -23,7 +24,9 @@ PartialBlockAllocation PartialBlockManager::GetBlockAllocation(uint32_t segment_
 		//! there is! increase the reference count of this block
 		allocation.partial_block->state.block_use_count += 1;
 		allocation.state = allocation.partial_block->state;
-		block_manager.IncreaseBlockReferenceCount(allocation.state.block_id);
+		if (checkpoint_type == CheckpointType::FULL_CHECKPOINT) {
+			block_manager.IncreaseBlockReferenceCount(allocation.state.block_id);
+		}
 	} else {
 		// full block: get a free block to write to
 		AllocateBlock(allocation.state, segment_size);
@@ -31,9 +34,18 @@ PartialBlockAllocation PartialBlockManager::GetBlockAllocation(uint32_t segment_
 	return allocation;
 }
 
+bool PartialBlockManager::HasBlockAllocation(uint32_t segment_size) {
+	return segment_size <= max_partial_block_size &&
+	       partially_filled_blocks.lower_bound(segment_size) != partially_filled_blocks.end();
+}
+
 void PartialBlockManager::AllocateBlock(PartialBlockState &state, uint32_t segment_size) {
 	D_ASSERT(segment_size <= Storage::BLOCK_SIZE);
-	state.block_id = block_manager.GetFreeBlockId();
+	if (checkpoint_type == CheckpointType::FULL_CHECKPOINT) {
+		state.block_id = block_manager.GetFreeBlockId();
+	} else {
+		state.block_id = INVALID_BLOCK;
+	}
 	state.block_size = Storage::BLOCK_SIZE;
 	state.offset_in_block = 0;
 	state.block_use_count = 1;
@@ -82,21 +94,73 @@ void PartialBlockManager::RegisterPartialBlock(PartialBlockAllocation &&allocati
 	// Flush any block that we're not going to reuse.
 	if (block_to_free) {
 		block_to_free->Flush(free_space);
+		AddWrittenBlock(block_to_free->state.block_id);
 	}
 }
 
-void PartialBlockManager::FlushPartialBlocks() {
+void PartialBlock::Merge(PartialBlock &other, idx_t offset, idx_t other_size) {
+	throw InternalException("PartialBlock::Merge not implemented for this block type");
+}
+
+void PartialBlockManager::Merge(PartialBlockManager &other) {
+	if (&other == this) {
+		throw InternalException("Cannot merge into itself");
+	}
+	// for each partially filled block in the other manager, check if we can merge it into an existing block in this
+	// manager
+	for (auto &e : other.partially_filled_blocks) {
+		if (!e.second) {
+			throw InternalException("Empty partially filled block found");
+		}
+		auto used_space = Storage::BLOCK_SIZE - e.first;
+		if (HasBlockAllocation(used_space)) {
+			// we can merge this block into an existing block - merge them
+			// merge blocks
+			auto allocation = GetBlockAllocation(used_space);
+			allocation.partial_block->Merge(*e.second, allocation.state.offset_in_block, used_space);
+
+			// re-register the partial block
+			allocation.state.offset_in_block += used_space;
+			RegisterPartialBlock(std::move(allocation));
+		} else {
+			// we cannot merge this block - append it directly to the current block manager
+			partially_filled_blocks.insert(make_pair(e.first, std::move(e.second)));
+		}
+	}
+	// copy over the written blocks
+	for (auto &block_id : other.written_blocks) {
+		AddWrittenBlock(block_id);
+	}
+	other.written_blocks.clear();
+	other.partially_filled_blocks.clear();
+}
+
+void PartialBlockManager::AddWrittenBlock(block_id_t block) {
+	auto entry = written_blocks.insert(block);
+	if (!entry.second) {
+		throw InternalException("Written block already exists");
+	}
+}
+
+void PartialBlockManager::ClearBlocks() {
 	for (auto &e : partially_filled_blocks) {
-		e.second->Flush(e.first);
+		e.second->Clear();
 	}
 	partially_filled_blocks.clear();
 }
 
-void PartialBlockManager::Clear() {
+void PartialBlockManager::FlushPartialBlocks() {
 	for (auto &e : partially_filled_blocks) {
-		e.second->Clear();
+		e.second->Flush(e.first);
 	}
 	partially_filled_blocks.clear();
 }
 
+void PartialBlockManager::Rollback() {
+	ClearBlocks();
+	for (auto &block_id : written_blocks) {
+		block_manager.MarkBlockAsFree(block_id);
+	}
+}
+
 } // namespace duckdb

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -287,7 +287,9 @@ void SingleFileBlockManager::MarkBlockAsFree(block_id_t block_id) {
 	lock_guard<mutex> lock(block_lock);
 	D_ASSERT(block_id >= 0);
 	D_ASSERT(block_id < max_block);
-	D_ASSERT(free_list.find(block_id) == free_list.end());
+	if (free_list.find(block_id) != free_list.end()) {
+		throw InternalException("MarkBlockAsFree called but block %llu was already freed!", block_id);
+	}
 	multi_use_blocks.erase(block_id);
 	free_list.insert(block_id);
 }

package/src/duckdb/src/storage/standard_buffer_manager.cpp

@@ -90,15 +90,18 @@ TempBufferPoolReservation StandardBufferManager::EvictBlocksOrThrow(idx_t memory
                                                                     ARGS... args) {
 	auto r = buffer_pool.EvictBlocks(memory_delta, buffer_pool.maximum_memory, buffer);
 	if (!r.success) {
-		throw OutOfMemoryException(args..., InMemoryWarning());
+		string extra_text = StringUtil::Format(" (%s/%s used)", StringUtil::BytesToHumanReadableString(GetUsedMemory()),
+		                                       StringUtil::BytesToHumanReadableString(GetMaxMemory()));
+		extra_text += InMemoryWarning();
+		throw OutOfMemoryException(args..., extra_text);
 	}
 	return std::move(r.reservation);
 }
 
 shared_ptr<BlockHandle> StandardBufferManager::RegisterSmallMemory(idx_t block_size) {
 	D_ASSERT(block_size < Storage::BLOCK_SIZE);
-	auto res = EvictBlocksOrThrow(block_size, nullptr, "could not allocate block of %lld bytes (%lld/%lld used) %s",
-	                              block_size, GetUsedMemory(), GetMaxMemory());
+	auto res = EvictBlocksOrThrow(block_size, nullptr, "could not allocate block of size %s%s",
+	                              StringUtil::BytesToHumanReadableString(block_size));
 
 	auto buffer = ConstructManagedBuffer(block_size, nullptr, FileBufferType::TINY_BUFFER);
 
@@ -112,9 +115,8 @@ shared_ptr<BlockHandle> StandardBufferManager::RegisterMemory(idx_t block_size,
 	auto alloc_size = GetAllocSize(block_size);
 	// first evict blocks until we have enough memory to store this buffer
 	unique_ptr<FileBuffer> reusable_buffer;
-	auto res =
-	    EvictBlocksOrThrow(alloc_size, &reusable_buffer, "could not allocate block of %lld bytes (%lld/%lld used) %s",
-	                       alloc_size, GetUsedMemory(), GetMaxMemory());
+	auto res = EvictBlocksOrThrow(alloc_size, &reusable_buffer, "could not allocate block of size %s%s",
+	                              StringUtil::BytesToHumanReadableString(alloc_size));
 
 	auto buffer = ConstructManagedBuffer(block_size, std::move(reusable_buffer));
 
@@ -144,8 +146,9 @@ void StandardBufferManager::ReAllocate(shared_ptr<BlockHandle> &handle, idx_t bl
 		return;
 	} else if (memory_delta > 0) {
 		// evict blocks until we have space to resize this block
-		auto reservation = EvictBlocksOrThrow(memory_delta, nullptr, "failed to resize block from %lld to %lld%s",
-		                                      handle->memory_usage, req.alloc_size);
+		auto reservation = EvictBlocksOrThrow(memory_delta, nullptr, "failed to resize block from %s to %s%s",
+		                                      StringUtil::BytesToHumanReadableString(handle->memory_usage),
+		                                      StringUtil::BytesToHumanReadableString(req.alloc_size));
 		// EvictBlocks decrements 'current_memory' for us.
 		handle->memory_charge.Merge(std::move(reservation));
 	} else {
@@ -171,8 +174,8 @@ BufferHandle StandardBufferManager::Pin(shared_ptr<BlockHandle> &handle) {
 	}
 	// evict blocks until we have space for the current block
 	unique_ptr<FileBuffer> reusable_buffer;
-	auto reservation =
-	    EvictBlocksOrThrow(required_memory, &reusable_buffer, "failed to pin block of size %lld%s", required_memory);
+	auto reservation = EvictBlocksOrThrow(required_memory, &reusable_buffer, "failed to pin block of size %s%s",
+	                                      StringUtil::BytesToHumanReadableString(required_memory));
 	// lock the handle again and repeat the check (in case anybody loaded in the mean time)
 	lock_guard<mutex> lock(handle->lock);
 	// check if the block is already loaded
@@ -732,7 +735,8 @@ void StandardBufferManager::ReserveMemory(idx_t size) {
 	if (size == 0) {
 		return;
 	}
-	auto reservation = EvictBlocksOrThrow(size, nullptr, "failed to reserve memory data of size %lld%s", size);
+	auto reservation = EvictBlocksOrThrow(size, nullptr, "failed to reserve memory data of size %s%s",
+	                                      StringUtil::BytesToHumanReadableString(size));
 	reservation.size = 0;
 }
 
@@ -748,7 +752,8 @@ void StandardBufferManager::FreeReservedMemory(idx_t size) {
 //===--------------------------------------------------------------------===//
 data_ptr_t StandardBufferManager::BufferAllocatorAllocate(PrivateAllocatorData *private_data, idx_t size) {
 	auto &data = (BufferAllocatorData &)*private_data;
-	auto reservation = data.manager.EvictBlocksOrThrow(size, nullptr, "failed to allocate data of size %lld%s", size);
+	auto reservation = data.manager.EvictBlocksOrThrow(size, nullptr, "failed to allocate data of size %s%s",
+	                                                   StringUtil::BytesToHumanReadableString(size));
 	// We rely on manual tracking of this one. :(
 	reservation.size = 0;
 	return Allocator::Get(data.manager.db).AllocateData(size);

package/src/duckdb/src/storage/statistics/base_statistics.cpp

@@ -271,6 +271,7 @@ void BaseStatistics::Serialize(Serializer &serializer) const {
 	FieldWriter writer(serializer);
 	writer.WriteField<bool>(has_null);
 	writer.WriteField<bool>(has_no_null);
+	writer.WriteField<idx_t>(distinct_count);
 	Serialize(writer);
 	writer.Finalize();
 }
@@ -316,9 +317,11 @@ BaseStatistics BaseStatistics::Deserialize(Deserializer &source, LogicalType typ
 	FieldReader reader(source);
 	bool has_null = reader.ReadRequired<bool>();
 	bool has_no_null = reader.ReadRequired<bool>();
+	idx_t distinct_count = reader.ReadRequired<idx_t>();
 	auto result = DeserializeType(reader, std::move(type));
 	result.has_null = has_null;
 	result.has_no_null = has_no_null;
+	result.distinct_count = distinct_count;
 	reader.Finalize();
 	return result;
 }

package/src/duckdb/src/storage/table/column_checkpoint_state.cpp

@@ -22,86 +22,103 @@ unique_ptr<BaseStatistics> ColumnCheckpointState::GetStatistics() {
 	return std::move(global_stats);
 }
 
-struct PartialBlockForCheckpoint : PartialBlock {
-	struct PartialColumnSegment {
-		ColumnData *data;
-		ColumnSegment *segment;
-		uint32_t offset_in_block;
-	};
-
-public:
-	PartialBlockForCheckpoint(ColumnData *first_data, ColumnSegment *first_segment, BlockManager &block_manager,
-	                          PartialBlockState state)
-	    : PartialBlock(state), first_data(first_data), first_segment(first_segment), block_manager(block_manager) {
-	}
+PartialBlockForCheckpoint::PartialBlockForCheckpoint(ColumnData &data, ColumnSegment &segment,
+                                                     BlockManager &block_manager, PartialBlockState state)
+    : PartialBlock(state), block_manager(block_manager), block(segment.block) {
+	AddSegmentToTail(data, segment, 0);
+}
 
-	~PartialBlockForCheckpoint() override {
-		D_ASSERT(IsFlushed() || Exception::UncaughtException());
-	}
+PartialBlockForCheckpoint::~PartialBlockForCheckpoint() {
+	D_ASSERT(IsFlushed() || Exception::UncaughtException());
+}
 
-	// We will copy all subsequent segment data into the memory corresponding
-	// to the first segment. Once the block is full (or checkpoint is complete)
-	// we'll invoke Flush(), which will cause
-	// the block to get written to storage (via BlockManger::ConvertToPersistent),
-	// and all segments to have their references updated
-	// (via ColumnSegment::ConvertToPersistent)
-	ColumnData *first_data;
-	ColumnSegment *first_segment;
-	BlockManager &block_manager;
-	vector<PartialColumnSegment> tail_segments;
-
-private:
-	struct UninitializedRegion {
-		idx_t start;
-		idx_t end;
-	};
-	vector<UninitializedRegion> uninitialized_regions;
-
-public:
-	bool IsFlushed() {
-		// first_segment is zeroed on Flush
-		return !first_segment;
-	}
+bool PartialBlockForCheckpoint::IsFlushed() {
+	// segments are cleared on Flush
+	return segments.empty();
+}
 
-	void AddUninitializedRegion(idx_t start, idx_t end) override {
-		uninitialized_regions.push_back({start, end});
-	}
+void PartialBlockForCheckpoint::AddUninitializedRegion(idx_t start, idx_t end) {
+	uninitialized_regions.push_back({start, end});
+}
 
-	void Flush(idx_t free_space_left) override {
-		// At this point, we've already copied all data from tail_segments
-		// into the page owned by first_segment. We flush all segment data to
-		// disk with the following call.
-		if (free_space_left > 0 || !uninitialized_regions.empty()) {
-			auto handle = block_manager.buffer_manager.Pin(first_segment->block);
-			// memset any uninitialized regions
-			for (auto &uninitialized : uninitialized_regions) {
-				memset(handle.Ptr() + uninitialized.start, 0, uninitialized.end - uninitialized.start);
-			}
-			// memset any free space at the end of the block to 0 prior to writing to disk
-			memset(handle.Ptr() + Storage::BLOCK_SIZE - free_space_left, 0, free_space_left);
+void PartialBlockForCheckpoint::Flush(idx_t free_space_left) {
+	if (IsFlushed()) {
+		throw InternalException("Flush called on partial block that was already flushed");
+	}
+	// if we have any free space or uninitialized regions we need to zero-initialize them
+	if (free_space_left > 0 || !uninitialized_regions.empty()) {
+		auto handle = block_manager.buffer_manager.Pin(block);
+		// memset any uninitialized regions
+		for (auto &uninitialized : uninitialized_regions) {
+			memset(handle.Ptr() + uninitialized.start, 0, uninitialized.end - uninitialized.start);
 		}
-		first_data->IncrementVersion();
-		first_segment->ConvertToPersistent(&block_manager, state.block_id);
-		// Now that the page is persistent, update tail_segments to point to the
-		// newly persistent block.
-		for (auto e : tail_segments) {
-			e.data->IncrementVersion();
-			e.segment->MarkAsPersistent(first_segment->block, e.offset_in_block);
+		// memset any free space at the end of the block to 0 prior to writing to disk
+		memset(handle.Ptr() + Storage::BLOCK_SIZE - free_space_left, 0, free_space_left);
+	}
+	// At this point, we've already copied all data from tail_segments
+	// into the page owned by first_segment. We flush all segment data to
+	// disk with the following call.
+	// persist the first segment to disk and point the remaining segments to the same block
+	bool fetch_new_block = state.block_id == INVALID_BLOCK;
+	if (fetch_new_block) {
+		state.block_id = block_manager.GetFreeBlockId();
+	}
+	for (idx_t i = 0; i < segments.size(); i++) {
+		auto &segment = segments[i];
+		segment.data.IncrementVersion();
+		if (i == 0) {
+			// the first segment is converted to persistent - this writes the data for ALL segments to disk
+			D_ASSERT(segment.offset_in_block == 0);
+			segment.segment.ConvertToPersistent(&block_manager, state.block_id);
+			// update the block after it has been converted to a persistent segment
+			block = segment.segment.block;
+		} else {
+			// subsequent segments are MARKED as persistent - they don't need to be rewritten
+			segment.segment.MarkAsPersistent(block, segment.offset_in_block);
+			if (fetch_new_block) {
+				// if we fetched a new block we need to increase the reference count to the block
+				block_manager.IncreaseBlockReferenceCount(state.block_id);
+			}
 		}
-		first_segment = nullptr;
-		tail_segments.clear();
 	}
+	Clear();
+}
 
-	void Clear() override {
-		first_data = nullptr;
-		first_segment = nullptr;
-		tail_segments.clear();
+void PartialBlockForCheckpoint::Clear() {
+	uninitialized_regions.clear();
+	block.reset();
+	segments.clear();
+}
+
+void PartialBlockForCheckpoint::Merge(PartialBlock &other_p, idx_t offset, idx_t other_size) {
+	auto &other = other_p.Cast<PartialBlockForCheckpoint>();
+
+	auto &buffer_manager = block_manager.buffer_manager;
+	// pin the source block
+	auto old_handle = buffer_manager.Pin(other.block);
+	// pin the target block
+	auto new_handle = buffer_manager.Pin(block);
+	// memcpy the contents of the old block to the new block
+	memcpy(new_handle.Ptr() + offset, old_handle.Ptr(), other_size);
+
+	// now copy over all of the segments to the new block
+	// move over the uninitialized regions
+	for (auto &region : other.uninitialized_regions) {
+		region.start += offset;
+		region.end += offset;
+		uninitialized_regions.push_back(region);
 	}
 
-	void AddSegmentToTail(ColumnData *data, ColumnSegment *segment, uint32_t offset_in_block) {
-		tail_segments.push_back({data, segment, offset_in_block});
+	// move over the segments
+	for (auto &segment : other.segments) {
+		AddSegmentToTail(segment.data, segment.segment, segment.offset_in_block + offset);
 	}
-};
+	other.Clear();
+}
+
+void PartialBlockForCheckpoint::AddSegmentToTail(ColumnData &data, ColumnSegment &segment, uint32_t offset_in_block) {
+	segments.emplace_back(data, segment, offset_in_block);
+}
 
 void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_t segment_size) {
 	D_ASSERT(segment_size <= Storage::BLOCK_SIZE);
@@ -128,14 +145,14 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 		if (allocation.partial_block) {
 			// Use an existing block.
 			D_ASSERT(offset_in_block > 0);
-			auto pstate = (PartialBlockForCheckpoint *)allocation.partial_block.get();
+			auto &pstate = allocation.partial_block->Cast<PartialBlockForCheckpoint>();
 			// pin the source block
 			auto old_handle = buffer_manager.Pin(segment->block);
 			// pin the target block
-			auto new_handle = buffer_manager.Pin(pstate->first_segment->block);
+			auto new_handle = buffer_manager.Pin(pstate.block);
 			// memcpy the contents of the old block to the new block
 			memcpy(new_handle.Ptr() + offset_in_block, old_handle.Ptr(), segment_size);
-			pstate->AddSegmentToTail(&column_data, segment.get(), offset_in_block);
+			pstate.AddSegmentToTail(column_data, *segment, offset_in_block);
 		} else {
 			// Create a new block for future reuse.
 			if (segment->SegmentSize() != Storage::BLOCK_SIZE) {
@@ -146,7 +163,7 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 			}
 			D_ASSERT(offset_in_block == 0);
 			allocation.partial_block = make_uniq<PartialBlockForCheckpoint>(
-			    &column_data, segment.get(), *allocation.block_manager, allocation.state);
+			    column_data, *segment, *allocation.block_manager, allocation.state);
 		}
 		// Writer will decide whether to reuse this block.
 		partial_block_manager.RegisterPartialBlock(std::move(allocation));
@@ -180,13 +197,4 @@ void ColumnCheckpointState::WriteDataPointers(RowGroupWriter &writer) {
 	writer.WriteColumnDataPointers(*this);
 }
 
-void ColumnCheckpointState::GetBlockIds(unordered_set<block_id_t> &result) {
-	for (auto &pointer : data_pointers) {
-		if (pointer.block_pointer.block_id == INVALID_BLOCK) {
-			continue;
-		}
-		result.insert(pointer.block_pointer.block_id);
-	}
-}
-
 } // namespace duckdb

package/src/duckdb/src/storage/table/column_data.cpp

@@ -21,7 +21,7 @@
 namespace duckdb {
 
 ColumnData::ColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
-                       LogicalType type_p, ColumnData *parent)
+                       LogicalType type_p, optional_ptr<ColumnData> parent)
     : start(start_row), count(0), block_manager(block_manager), info(info), column_index(column_index),
       type(std::move(type_p)), parent(parent), version(0) {
 	if (!parent) {
@@ -29,24 +29,17 @@ ColumnData::ColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t c
 	}
 }
 
-ColumnData::ColumnData(ColumnData &other, idx_t start, ColumnData *parent)
-    : start(start), count(other.count), block_manager(other.block_manager), info(other.info),
-      column_index(other.column_index), type(std::move(other.type)), parent(parent),
-      version(parent ? parent->version + 1 : 0) {
-	if (other.updates) {
-		updates = make_uniq<UpdateSegment>(*other.updates, *this);
-	}
-	if (other.stats) {
-		stats = make_uniq<SegmentStatistics>(other.stats->statistics.Copy());
-	}
+ColumnData::~ColumnData() {
+}
+
+void ColumnData::SetStart(idx_t new_start) {
+	this->start = new_start;
 	idx_t offset = 0;
-	for (auto &segment : other.data.Segments()) {
-		this->data.AppendSegment(ColumnSegment::CreateSegment(segment, start + offset));
+	for (auto &segment : data.Segments()) {
+		segment.start = start + offset;
 		offset += segment.count;
 	}
-}
-
-ColumnData::~ColumnData() {
+	data.Reinitialize();
 }
 
 DatabaseInstance &ColumnData::GetDatabase() const {
@@ -278,8 +271,8 @@ void ColumnData::InitializeAppend(ColumnAppendState &state) {
 		AppendTransientSegment(l, start);
 	}
 	auto segment = data.GetLastSegment(l);
-	if (segment->segment_type == ColumnSegmentType::PERSISTENT) {
-		// no transient segments yet
+	if (segment->segment_type == ColumnSegmentType::PERSISTENT || !segment->function.get().init_append) {
+		// we cannot append to this segment - append a new segment
 		auto total_rows = segment->start + segment->count;
 		AppendTransientSegment(l, total_rows);
 		state.current = data.GetLastSegment(l);
@@ -418,9 +411,9 @@ unique_ptr<ColumnCheckpointState> ColumnData::CreateCheckpointState(RowGroup &ro
 	return make_uniq<ColumnCheckpointState>(row_group, *this, partial_block_manager);
 }
 
-void ColumnData::CheckpointScan(ColumnSegment *segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
+void ColumnData::CheckpointScan(ColumnSegment &segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
                                 Vector &scan_vector) {
-	segment->Scan(state, count, scan_vector, 0, true);
+	segment.Scan(state, count, scan_vector, 0, true);
 	if (updates) {
 		scan_vector.Flatten(count);
 		updates->FetchCommittedRange(state.row_index - row_group_start, count, scan_vector);
@@ -489,7 +482,7 @@ void ColumnData::DeserializeColumn(Deserializer &source) {
 
 shared_ptr<ColumnData> ColumnData::Deserialize(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
                                                idx_t start_row, Deserializer &source, const LogicalType &type,
-                                               ColumnData *parent) {
+                                               optional_ptr<ColumnData> parent) {
 	auto entry = ColumnData::CreateColumn(block_manager, info, column_index, start_row, type, parent);
 	entry->DeserializeColumn(source);
 	return entry;
@@ -565,48 +558,29 @@ void ColumnData::Verify(RowGroup &parent) {
 
 template <class RET, class OP>
 static RET CreateColumnInternal(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
-                                const LogicalType &type, ColumnData *parent) {
+                                const LogicalType &type, optional_ptr<ColumnData> parent) {
 	if (type.InternalType() == PhysicalType::STRUCT) {
 		return OP::template Create<StructColumnData>(block_manager, info, column_index, start_row, type, parent);
 	} else if (type.InternalType() == PhysicalType::LIST) {
 		return OP::template Create<ListColumnData>(block_manager, info, column_index, start_row, type, parent);
 	} else if (type.id() == LogicalTypeId::VALIDITY) {
-		return OP::template Create<ValidityColumnData>(block_manager, info, column_index, start_row, parent);
+		return OP::template Create<ValidityColumnData>(block_manager, info, column_index, start_row, *parent);
 	}
 	return OP::template Create<StandardColumnData>(block_manager, info, column_index, start_row, type, parent);
 }
 
-template <class RET, class OP>
-static RET CreateColumnInternal(ColumnData &other, idx_t start_row, ColumnData *parent) {
-	if (other.type.InternalType() == PhysicalType::STRUCT) {
-		return OP::template Create<StructColumnData>(other, start_row, parent);
-	} else if (other.type.InternalType() == PhysicalType::LIST) {
-		return OP::template Create<ListColumnData>(other, start_row, parent);
-	} else if (other.type.id() == LogicalTypeId::VALIDITY) {
-		return OP::template Create<ValidityColumnData>(other, start_row, parent);
-	}
-	return OP::template Create<StandardColumnData>(other, start_row, parent);
-}
-
 shared_ptr<ColumnData> ColumnData::CreateColumn(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
-                                                idx_t start_row, const LogicalType &type, ColumnData *parent) {
+                                                idx_t start_row, const LogicalType &type,
+                                                optional_ptr<ColumnData> parent) {
 	return CreateColumnInternal<shared_ptr<ColumnData>, SharedConstructor>(block_manager, info, column_index, start_row,
 	                                                                       type, parent);
 }
 
-shared_ptr<ColumnData> ColumnData::CreateColumn(ColumnData &other, idx_t start_row, ColumnData *parent) {
-	return CreateColumnInternal<shared_ptr<ColumnData>, SharedConstructor>(other, start_row, parent);
-}
-
 unique_ptr<ColumnData> ColumnData::CreateColumnUnique(BlockManager &block_manager, DataTableInfo &info,
                                                       idx_t column_index, idx_t start_row, const LogicalType &type,
-                                                      ColumnData *parent) {
+                                                      optional_ptr<ColumnData> parent) {
 	return CreateColumnInternal<unique_ptr<ColumnData>, UniqueConstructor>(block_manager, info, column_index, start_row,
 	                                                                       type, parent);
 }
 
-unique_ptr<ColumnData> ColumnData::CreateColumnUnique(ColumnData &other, idx_t start_row, ColumnData *parent) {
-	return CreateColumnInternal<unique_ptr<ColumnData>, UniqueConstructor>(other, start_row, parent);
-}
-
 } // namespace duckdb