duckdb 0.8.2-dev77.0 → 0.9.0

package/src/duckdb/src/storage/table/column_checkpoint_state.cpp

@@ -22,9 +22,9 @@ unique_ptr<BaseStatistics> ColumnCheckpointState::GetStatistics() {
 	return std::move(global_stats);
 }
 
-PartialBlockForCheckpoint::PartialBlockForCheckpoint(ColumnData &data, ColumnSegment &segment,
-                                                     BlockManager &block_manager, PartialBlockState state)
-    : PartialBlock(state), block_manager(block_manager), block(segment.block) {
+PartialBlockForCheckpoint::PartialBlockForCheckpoint(ColumnData &data, ColumnSegment &segment, PartialBlockState state,
+                                                     BlockManager &block_manager)
+    : PartialBlock(state, block_manager, segment.block) {
 	AddSegmentToTail(data, segment, 0);
 }
 
@@ -37,24 +37,15 @@ bool PartialBlockForCheckpoint::IsFlushed() {
 	return segments.empty();
 }
 
-void PartialBlockForCheckpoint::AddUninitializedRegion(idx_t start, idx_t end) {
-	uninitialized_regions.push_back({start, end});
-}
+void PartialBlockForCheckpoint::Flush(const idx_t free_space_left) {
 
-void PartialBlockForCheckpoint::Flush(idx_t free_space_left) {
 	if (IsFlushed()) {
 		throw InternalException("Flush called on partial block that was already flushed");
 	}
-	// if we have any free space or uninitialized regions we need to zero-initialize them
-	if (free_space_left > 0 || !uninitialized_regions.empty()) {
-		auto handle = block_manager.buffer_manager.Pin(block);
-		// memset any uninitialized regions
-		for (auto &uninitialized : uninitialized_regions) {
-			memset(handle.Ptr() + uninitialized.start, 0, uninitialized.end - uninitialized.start);
-		}
-		// memset any free space at the end of the block to 0 prior to writing to disk
-		memset(handle.Ptr() + Storage::BLOCK_SIZE - free_space_left, 0, free_space_left);
-	}
+
+	// zero-initialize unused memory
+	FlushInternal(free_space_left);
+
 	// At this point, we've already copied all data from tail_segments
 	// into the page owned by first_segment. We flush all segment data to
 	// disk with the following call.
@@ -63,6 +54,7 @@ void PartialBlockForCheckpoint::Flush(idx_t free_space_left) {
 	if (fetch_new_block) {
 		state.block_id = block_manager.GetFreeBlockId();
 	}
+
 	for (idx_t i = 0; i < segments.size(); i++) {
 		auto &segment = segments[i];
 		segment.data.IncrementVersion();
@@ -71,23 +63,18 @@ void PartialBlockForCheckpoint::Flush(idx_t free_space_left) {
 			D_ASSERT(segment.offset_in_block == 0);
 			segment.segment.ConvertToPersistent(&block_manager, state.block_id);
 			// update the block after it has been converted to a persistent segment
-			block = segment.segment.block;
+			block_handle = segment.segment.block;
 		} else {
 			// subsequent segments are MARKED as persistent - they don't need to be rewritten
-			segment.segment.MarkAsPersistent(block, segment.offset_in_block);
+			segment.segment.MarkAsPersistent(block_handle, segment.offset_in_block);
 			if (fetch_new_block) {
 				// if we fetched a new block we need to increase the reference count to the block
 				block_manager.IncreaseBlockReferenceCount(state.block_id);
 			}
 		}
 	}
-	Clear();
-}
 
-void PartialBlockForCheckpoint::Clear() {
-	uninitialized_regions.clear();
-	block.reset();
-	segments.clear();
+	Clear();
 }
 
 void PartialBlockForCheckpoint::Merge(PartialBlock &other_p, idx_t offset, idx_t other_size) {
@@ -95,13 +82,13 @@ void PartialBlockForCheckpoint::Merge(PartialBlock &other_p, idx_t offset, idx_t
 
 	auto &buffer_manager = block_manager.buffer_manager;
 	// pin the source block
-	auto old_handle = buffer_manager.Pin(other.block);
+	auto old_handle = buffer_manager.Pin(other.block_handle);
 	// pin the target block
-	auto new_handle = buffer_manager.Pin(block);
+	auto new_handle = buffer_manager.Pin(block_handle);
 	// memcpy the contents of the old block to the new block
 	memcpy(new_handle.Ptr() + offset, old_handle.Ptr(), other_size);
 
-	// now copy over all of the segments to the new block
+	// now copy over all segments to the new block
 	// move over the uninitialized regions
 	for (auto &region : other.uninitialized_regions) {
 		region.start += offset;
@@ -113,6 +100,7 @@ void PartialBlockForCheckpoint::Merge(PartialBlock &other_p, idx_t offset, idx_t
 	for (auto &segment : other.segments) {
 		AddSegmentToTail(segment.data, segment.segment, segment.offset_in_block + offset);
 	}
+
 	other.Clear();
 }
 
@@ -120,6 +108,12 @@ void PartialBlockForCheckpoint::AddSegmentToTail(ColumnData &data, ColumnSegment
 	segments.emplace_back(data, segment, offset_in_block);
 }
 
+void PartialBlockForCheckpoint::Clear() {
+	uninitialized_regions.clear();
+	block_handle.reset();
+	segments.clear();
+}
+
 void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_t segment_size) {
 	D_ASSERT(segment_size <= Storage::BLOCK_SIZE);
 	auto tuple_count = segment->count.load();
@@ -140,7 +134,7 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 		// non-constant block
 		PartialBlockAllocation allocation = partial_block_manager.GetBlockAllocation(segment_size);
 		block_id = allocation.state.block_id;
-		offset_in_block = allocation.state.offset_in_block;
+		offset_in_block = allocation.state.offset;
 
 		if (allocation.partial_block) {
 			// Use an existing block.
@@ -149,7 +143,7 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 			// pin the source block
 			auto old_handle = buffer_manager.Pin(segment->block);
 			// pin the target block
-			auto new_handle = buffer_manager.Pin(pstate.block);
+			auto new_handle = buffer_manager.Pin(pstate.block_handle);
 			// memcpy the contents of the old block to the new block
 			memcpy(new_handle.Ptr() + offset_in_block, old_handle.Ptr(), segment_size);
 			pstate.AddSegmentToTail(column_data, *segment, offset_in_block);
@@ -162,8 +156,8 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 				segment->Resize(Storage::BLOCK_SIZE);
 			}
 			D_ASSERT(offset_in_block == 0);
-			allocation.partial_block = make_uniq<PartialBlockForCheckpoint>(
-			    column_data, *segment, *allocation.block_manager, allocation.state);
+			allocation.partial_block = make_uniq<PartialBlockForCheckpoint>(column_data, *segment, allocation.state,
+			                                                                *allocation.block_manager);
 		}
 		// Writer will decide whether to reuse this block.
 		partial_block_manager.RegisterPartialBlock(std::move(allocation));
@@ -187,14 +181,17 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 	}
 	data_pointer.tuple_count = tuple_count;
 	data_pointer.compression_type = segment->function.get().type;
+	if (segment->function.get().serialize_state) {
+		data_pointer.segment_state = segment->function.get().serialize_state(*segment);
+	}
 
 	// append the segment to the new segment tree
 	new_tree.AppendSegment(std::move(segment));
 	data_pointers.push_back(std::move(data_pointer));
 }
 
-void ColumnCheckpointState::WriteDataPointers(RowGroupWriter &writer) {
-	writer.WriteColumnDataPointers(*this);
+void ColumnCheckpointState::WriteDataPointers(RowGroupWriter &writer, Serializer &serializer) {
+	writer.WriteColumnDataPointers(*this, serializer);
 }
 
 } // namespace duckdb

package/src/duckdb/src/storage/table/column_data.cpp

@@ -6,7 +6,6 @@
 #include "duckdb/storage/data_pointer.hpp"
 #include "duckdb/storage/data_table.hpp"
 #include "duckdb/storage/statistics/distinct_statistics.hpp"
-#include "duckdb/storage/storage_manager.hpp"
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/list_column_data.hpp"
 #include "duckdb/storage/table/standard_column_data.hpp"
@@ -16,7 +15,8 @@
 #include "duckdb/storage/table_storage_info.hpp"
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
-#include "duckdb/main/attached_database.hpp"
+#include "duckdb/common/serializer/read_stream.hpp"
+#include "duckdb/common/serializer/binary_deserializer.hpp"
 
 namespace duckdb {
 
@@ -87,7 +87,7 @@ void ColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx)
 	state.last_offset = 0;
 }
 
-idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remaining) {
+idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remaining, bool has_updates) {
 	state.previous_states.clear();
 	if (state.version != version) {
 		InitializeScanWithOffset(state, state.row_index);
@@ -113,7 +113,8 @@ idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remai
 		idx_t scan_count = MinValue<idx_t>(remaining, state.current->start + state.current->count - state.row_index);
 		idx_t result_offset = initial_remaining - remaining;
 		if (scan_count > 0) {
-			state.current->Scan(state, scan_count, result, result_offset, scan_count == initial_remaining);
+			state.current->Scan(state, scan_count, result, result_offset,
+			                    !has_updates && scan_count == initial_remaining);
 
 			state.row_index += scan_count;
 			remaining -= scan_count;
@@ -138,10 +139,14 @@ idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remai
 
 template <bool SCAN_COMMITTED, bool ALLOW_UPDATES>
 idx_t ColumnData::ScanVector(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result) {
-	auto scan_count = ScanVector(state, result, STANDARD_VECTOR_SIZE);
-
-	lock_guard<mutex> update_guard(update_lock);
-	if (updates) {
+	bool has_updates;
+	{
+		lock_guard<mutex> update_guard(update_lock);
+		has_updates = updates ? true : false;
+	}
+	auto scan_count = ScanVector(state, result, STANDARD_VECTOR_SIZE, has_updates);
+	if (has_updates) {
+		lock_guard<mutex> update_guard(update_lock);
 		if (!ALLOW_UPDATES && updates->HasUncommittedUpdates(vector_index)) {
 			throw TransactionException("Cannot create index with outstanding updates");
 		}
@@ -179,7 +184,7 @@ idx_t ColumnData::ScanCommitted(idx_t vector_index, ColumnScanState &state, Vect
 void ColumnData::ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t count, Vector &result) {
 	ColumnScanState child_state;
 	InitializeScanWithOffset(child_state, row_group_start + offset_in_row_group);
-	auto scan_count = ScanVector(child_state, result, count);
+	auto scan_count = ScanVector(child_state, result, count, updates ? true : false);
 	if (updates) {
 		result.Flatten(scan_count);
 		updates->FetchCommittedRange(offset_in_row_group, count, result);
@@ -192,7 +197,7 @@ idx_t ColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t count)
 	}
 	// ScanCount can only be used if there are no updates
 	D_ASSERT(!updates);
-	return ScanVector(state, result, count);
+	return ScanVector(state, result, count, false);
 }
 
 void ColumnData::Select(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
@@ -339,7 +344,7 @@ idx_t ColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &result) {
 	state.row_index = start + ((row_id - start) / STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE);
 	state.current = data.GetSegment(state.row_index);
 	state.internal_index = state.current->start;
-	return ScanVector(state, result, STANDARD_VECTOR_SIZE);
+	return ScanVector(state, result, STANDARD_VECTOR_SIZE, false);
 }
 
 void ColumnData::FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result,
@@ -397,12 +402,7 @@ void ColumnData::AppendTransientSegment(SegmentLock &l, idx_t start_row) {
 void ColumnData::CommitDropColumn() {
 	for (auto &segment_p : data.Segments()) {
 		auto &segment = segment_p;
-		if (segment.segment_type == ColumnSegmentType::PERSISTENT) {
-			auto block_id = segment.GetBlockId();
-			if (block_id != INVALID_BLOCK) {
-				block_manager.MarkBlockAsModified(block_id);
-			}
-		}
+		segment.CommitDropSegment();
 	}
 }
 
@@ -446,45 +446,44 @@ unique_ptr<ColumnCheckpointState> ColumnData::Checkpoint(RowGroup &row_group,
 	return checkpoint_state;
 }
 
-void ColumnData::DeserializeColumn(Deserializer &source) {
+void ColumnData::DeserializeColumn(Deserializer &deserializer) {
 	// load the data pointers for the column
+	deserializer.Set<DatabaseInstance &>(info.db.GetDatabase());
+	deserializer.Set<LogicalType &>(type);
+
+	vector<DataPointer> data_pointers;
+	deserializer.ReadProperty(100, "data_pointers", data_pointers);
+
+	deserializer.Unset<DatabaseInstance>();
+	deserializer.Unset<LogicalType>();
+
+	// construct the segments based on the data pointers
 	this->count = 0;
-	idx_t data_pointer_count = source.Read<idx_t>();
-	for (idx_t data_ptr = 0; data_ptr < data_pointer_count; data_ptr++) {
-		// read the data pointer
-		auto row_start = source.Read<idx_t>();
-		auto tuple_count = source.Read<idx_t>();
-		auto block_pointer_block_id = source.Read<block_id_t>();
-		auto block_pointer_offset = source.Read<uint32_t>();
-		auto compression_type = source.Read<CompressionType>();
-		auto segment_stats = BaseStatistics::Deserialize(source, type);
+	for (auto &data_pointer : data_pointers) {
+		// Update the count and statistics
+		this->count += data_pointer.tuple_count;
 		if (stats) {
-			stats->statistics.Merge(segment_stats);
+			stats->statistics.Merge(data_pointer.statistics);
 		}
 
-		DataPointer data_pointer(std::move(segment_stats));
-		data_pointer.row_start = row_start;
-		data_pointer.tuple_count = tuple_count;
-		data_pointer.block_pointer.block_id = block_pointer_block_id;
-		data_pointer.block_pointer.offset = block_pointer_offset;
-		data_pointer.compression_type = compression_type;
-
-		this->count += tuple_count;
-
 		// create a persistent segment
 		auto segment = ColumnSegment::CreatePersistentSegment(
 		    GetDatabase(), block_manager, data_pointer.block_pointer.block_id, data_pointer.block_pointer.offset, type,
 		    data_pointer.row_start, data_pointer.tuple_count, data_pointer.compression_type,
-		    std::move(data_pointer.statistics));
+		    std::move(data_pointer.statistics), std::move(data_pointer.segment_state));
+
 		data.AppendSegment(std::move(segment));
 	}
 }
 
 shared_ptr<ColumnData> ColumnData::Deserialize(BlockManager &block_manager, DataTableInfo &info, idx_t column_index,
-                                               idx_t start_row, Deserializer &source, const LogicalType &type,
+                                               idx_t start_row, ReadStream &source, const LogicalType &type,
                                                optional_ptr<ColumnData> parent) {
 	auto entry = ColumnData::CreateColumn(block_manager, info, column_index, start_row, type, parent);
-	entry->DeserializeColumn(source);
+	BinaryDeserializer deserializer(source);
+	deserializer.Begin();
+	entry->DeserializeColumn(deserializer);
+	deserializer.End();
 	return entry;
 }
 
@@ -530,10 +529,14 @@ void ColumnData::GetColumnSegmentInfo(idx_t row_group_index, vector<idx_t> col_p
 		} else {
 			column_info.persistent = false;
 		}
+		auto segment_state = segment->GetSegmentState();
+		if (segment_state) {
+			column_info.segment_info = segment_state->GetSegmentInfo();
+		}
 		result.emplace_back(column_info);
 
 		segment_idx++;
-		segment = (ColumnSegment *)data.GetNextSegment(segment);
+		segment = data.GetNextSegment(segment);
 	}
 }
 

package/src/duckdb/src/storage/table/column_data_checkpointer.cpp

@@ -168,16 +168,9 @@ void ColumnDataCheckpointer::WriteToDisk() {
 	// first we check the current segments
 	// if there are any persistent segments, we will mark their old block ids as modified
 	// since the segments will be rewritten their old on disk data is no longer required
-	auto &block_manager = col_data.GetBlockManager();
 	for (idx_t segment_idx = 0; segment_idx < nodes.size(); segment_idx++) {
 		auto segment = nodes[segment_idx].node.get();
-		if (segment->segment_type == ColumnSegmentType::PERSISTENT) {
-			// persistent segment has updates: mark it as modified and rewrite the block with the merged updates
-			auto block_id = segment->GetBlockId();
-			if (block_id != INVALID_BLOCK) {
-				block_manager.MarkBlockAsModified(block_id);
-			}
-		}
+		segment->CommitDropSegment();
 	}
 
 	// now we need to write our segment
@@ -231,6 +224,9 @@ void ColumnDataCheckpointer::WritePersistentSegments() {
 		pointer.row_start = segment->start;
 		pointer.tuple_count = segment->count;
 		pointer.compression_type = segment->function.get().type;
+		if (segment->function.get().serialize_state) {
+			pointer.segment_state = segment->function.get().serialize_state(*segment);
+		}
 
 		// merge the persistent stats into the global column stats
 		state.global_stats->Merge(segment->stats.statistics);
@@ -242,9 +238,9 @@ void ColumnDataCheckpointer::WritePersistentSegments() {
 	}
 }
 
-void ColumnDataCheckpointer::Checkpoint(vector<SegmentNode<ColumnSegment>> nodes) {
-	D_ASSERT(!nodes.empty());
-	this->nodes = std::move(nodes);
+void ColumnDataCheckpointer::Checkpoint(vector<SegmentNode<ColumnSegment>> nodes_p) {
+	D_ASSERT(!nodes_p.empty());
+	this->nodes = std::move(nodes_p);
 	// first check if any of the segments have changes
 	if (!HasChanges()) {
 		// no changes: only need to write the metadata for this column

package/src/duckdb/src/storage/table/column_segment.cpp

@@ -9,6 +9,7 @@
 #include "duckdb/planner/filter/constant_filter.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
+#include "duckdb/storage/data_pointer.hpp"
 
 #include <cstring>
 
@@ -18,7 +19,8 @@ unique_ptr<ColumnSegment> ColumnSegment::CreatePersistentSegment(DatabaseInstanc
                                                                  block_id_t block_id, idx_t offset,
                                                                  const LogicalType &type, idx_t start, idx_t count,
                                                                  CompressionType compression_type,
-                                                                 BaseStatistics statistics) {
+                                                                 BaseStatistics statistics,
+                                                                 unique_ptr<ColumnSegmentState> segment_state) {
 	auto &config = DBConfig::GetConfig(db);
 	optional_ptr<CompressionFunction> function;
 	shared_ptr<BlockHandle> block;
@@ -31,7 +33,7 @@ unique_ptr<ColumnSegment> ColumnSegment::CreatePersistentSegment(DatabaseInstanc
 	}
 	auto segment_size = Storage::BLOCK_SIZE;
 	return make_uniq<ColumnSegment>(db, std::move(block), type, ColumnSegmentType::PERSISTENT, start, count, *function,
-	                                std::move(statistics), block_id, offset, segment_size);
+	                                std::move(statistics), block_id, offset, segment_size, std::move(segment_state));
 }
 
 unique_ptr<ColumnSegment> ColumnSegment::CreateTransientSegment(DatabaseInstance &db, const LogicalType &type,
@@ -56,13 +58,14 @@ unique_ptr<ColumnSegment> ColumnSegment::CreateSegment(ColumnSegment &other, idx
 
 ColumnSegment::ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block, LogicalType type_p,
                              ColumnSegmentType segment_type, idx_t start, idx_t count, CompressionFunction &function_p,
-                             BaseStatistics statistics, block_id_t block_id_p, idx_t offset_p, idx_t segment_size_p)
+                             BaseStatistics statistics, block_id_t block_id_p, idx_t offset_p, idx_t segment_size_p,
+                             unique_ptr<ColumnSegmentState> segment_state)
     : SegmentBase<ColumnSegment>(start, count), db(db), type(std::move(type_p)),
       type_size(GetTypeIdSize(type.InternalType())), segment_type(segment_type), function(function_p),
       stats(std::move(statistics)), block(std::move(block)), block_id(block_id_p), offset(offset_p),
       segment_size(segment_size_p) {
 	if (function.get().init_segment) {
-		segment_state = function.get().init_segment(*this, block_id);
+		this->segment_state = function.get().init_segment(*this, block_id, segment_state.get());
 	}
 }
 
@@ -190,11 +193,6 @@ void ColumnSegment::ConvertToPersistent(optional_ptr<BlockManager> block_manager
 		// instead of copying the data we alter some metadata so the buffer points to an on-disk block
 		block = block_manager->ConvertToPersistent(block_id, std::move(block));
 	}
-
-	segment_state.reset();
-	if (function.get().init_segment) {
-		segment_state = function.get().init_segment(*this, block_id);
-	}
 }
 
 void ColumnSegment::MarkAsPersistent(shared_ptr<BlockHandle> block_p, uint32_t offset_p) {
@@ -204,10 +202,21 @@ void ColumnSegment::MarkAsPersistent(shared_ptr<BlockHandle> block_p, uint32_t o
 	block_id = block_p->BlockId();
 	offset = offset_p;
 	block = std::move(block_p);
+}
 
-	segment_state.reset();
-	if (function.get().init_segment) {
-		segment_state = function.get().init_segment(*this, block_id);
+//===--------------------------------------------------------------------===//
+// Drop Segment
+//===--------------------------------------------------------------------===//
+void ColumnSegment::CommitDropSegment() {
+	if (segment_type != ColumnSegmentType::PERSISTENT) {
+		// not persistent
+		return;
+	}
+	if (block_id != INVALID_BLOCK) {
+		GetBlockManager().MarkBlockAsModified(block_id);
+	}
+	if (function.get().cleanup_state) {
+		function.get().cleanup_state(*this);
 	}
 }
 

package/src/duckdb/src/storage/table/list_column_data.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/storage/table/list_column_data.hpp"
 #include "duckdb/storage/statistics/list_stats.hpp"
-#include "duckdb/transaction/transaction.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
@@ -85,20 +86,23 @@ idx_t ListColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t co
 	D_ASSERT(!updates);
 
 	Vector offset_vector(LogicalType::UBIGINT, count);
-	idx_t scan_count = ScanVector(state, offset_vector, count);
+	idx_t scan_count = ScanVector(state, offset_vector, count, false);
 	D_ASSERT(scan_count > 0);
 	validity.ScanCount(state.child_states[0], result, count);
 
-	auto data = FlatVector::GetData<uint64_t>(offset_vector);
-	auto last_entry = data[scan_count - 1];
+	UnifiedVectorFormat offsets;
+	offset_vector.ToUnifiedFormat(scan_count, offsets);
+	auto data = UnifiedVectorFormat::GetData<uint64_t>(offsets);
+	auto last_entry = data[offsets.sel->get_index(scan_count - 1)];
 
 	// shift all offsets so they are 0 at the first entry
 	auto result_data = FlatVector::GetData<list_entry_t>(result);
 	auto base_offset = state.last_offset;
 	idx_t current_offset = 0;
 	for (idx_t i = 0; i < scan_count; i++) {
+		auto offset_index = offsets.sel->get_index(i);
 		result_data[i].offset = current_offset;
-		result_data[i].length = data[i] - current_offset - base_offset;
+		result_data[i].length = data[offset_index] - current_offset - base_offset;
 		current_offset += result_data[i].length;
 	}
 
@@ -128,7 +132,7 @@ void ListColumnData::Skip(ColumnScanState &state, idx_t count) {
 	// note that we only need to read the first and last entry
 	// however, let's just read all "count" entries for now
 	Vector result(LogicalType::UBIGINT, count);
-	idx_t scan_count = ScanVector(state, result, count);
+	idx_t scan_count = ScanVector(state, result, count, false);
 	if (scan_count == 0) {
 		return;
 	}
@@ -318,10 +322,12 @@ public:
 		return stats.ToUnique();
 	}
 
-	void WriteDataPointers(RowGroupWriter &writer) override {
-		ColumnCheckpointState::WriteDataPointers(writer);
-		validity_state->WriteDataPointers(writer);
-		child_state->WriteDataPointers(writer);
+	void WriteDataPointers(RowGroupWriter &writer, Serializer &serializer) override {
+		ColumnCheckpointState::WriteDataPointers(writer, serializer);
+		serializer.WriteObject(101, "validity",
+		                       [&](Serializer &serializer) { validity_state->WriteDataPointers(writer, serializer); });
+		serializer.WriteObject(102, "child_column",
+		                       [&](Serializer &serializer) { child_state->WriteDataPointers(writer, serializer); });
 	}
 };
 
@@ -343,10 +349,14 @@ unique_ptr<ColumnCheckpointState> ListColumnData::Checkpoint(RowGroup &row_group
 	return base_state;
 }
 
-void ListColumnData::DeserializeColumn(Deserializer &source) {
-	ColumnData::DeserializeColumn(source);
-	validity.DeserializeColumn(source);
-	child_column->DeserializeColumn(source);
+void ListColumnData::DeserializeColumn(Deserializer &deserializer) {
+	ColumnData::DeserializeColumn(deserializer);
+
+	deserializer.ReadObject(101, "validity",
+	                        [&](Deserializer &deserializer) { validity.DeserializeColumn(deserializer); });
+
+	deserializer.ReadObject(102, "child_column",
+	                        [&](Deserializer &deserializer) { child_column->DeserializeColumn(deserializer); });
 }
 
 void ListColumnData::GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector<duckdb::idx_t> col_path,

package/src/duckdb/src/storage/table/persistent_table_data.cpp

@@ -3,8 +3,7 @@
 
 namespace duckdb {
 
-PersistentTableData::PersistentTableData(idx_t column_count)
-    : total_rows(0), row_group_count(0), block_id(INVALID_BLOCK), offset(0) {
+PersistentTableData::PersistentTableData(idx_t column_count) : total_rows(0), row_group_count(0) {
 }
 
 PersistentTableData::~PersistentTableData() {