duckdb 0.7.2-dev0.0 → 0.7.2-dev1034.0

package/src/duckdb/src/storage/table/column_data.cpp

@@ -32,10 +32,9 @@ ColumnData::ColumnData(ColumnData &other, idx_t start, ColumnData *parent)
 		updates = make_unique<UpdateSegment>(*other.updates, *this);
 	}
 	idx_t offset = 0;
-	for (auto segment = other.data.GetRootSegment(); segment; segment = segment->Next()) {
-		auto &other = (ColumnSegment &)*segment;
-		this->data.AppendSegment(ColumnSegment::CreateSegment(other, start + offset));
-		offset += segment->count;
+	for (auto &segment : other.data.Segments()) {
+		this->data.AppendSegment(ColumnSegment::CreateSegment(segment, start + offset));
+		offset += segment.count;
 	}
 }
 
@@ -75,7 +74,8 @@ idx_t ColumnData::GetMaxEntry() {
 }
 
 void ColumnData::InitializeScan(ColumnScanState &state) {
-	state.current = (ColumnSegment *)data.GetRootSegment();
+	state.current = data.GetRootSegment();
+	state.segment_tree = &data;
 	state.row_index = state.current ? state.current->start : 0;
 	state.internal_index = state.row_index;
 	state.initialized = false;
@@ -84,7 +84,8 @@ void ColumnData::InitializeScan(ColumnScanState &state) {
 }
 
 void ColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) {
-	state.current = (ColumnSegment *)data.GetSegment(row_idx);
+	state.current = data.GetSegment(row_idx);
+	state.segment_tree = &data;
 	state.row_index = row_idx;
 	state.internal_index = state.current->start;
 	state.initialized = false;
@@ -125,11 +126,12 @@ idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remai
 		}
 
 		if (remaining > 0) {
-			if (!state.current->next) {
+			auto next = data.GetNextSegment(state.current);
+			if (!next) {
 				break;
 			}
 			state.previous_states.emplace_back(std::move(state.scan_state));
-			state.current = (ColumnSegment *)state.current->Next();
+			state.current = next;
 			state.current->InitializeScan(state);
 			state.segment_checked = false;
 			D_ASSERT(state.row_index >= state.current->start &&
@@ -234,14 +236,14 @@ void ColumnData::InitializeAppend(ColumnAppendState &state) {
 		// no segments yet, append an empty segment
 		AppendTransientSegment(l, start);
 	}
-	auto segment = (ColumnSegment *)data.GetLastSegment(l);
+	auto segment = data.GetLastSegment(l);
 	if (segment->segment_type == ColumnSegmentType::PERSISTENT) {
 		// no transient segments yet
 		auto total_rows = segment->start + segment->count;
 		AppendTransientSegment(l, total_rows);
-		state.current = (ColumnSegment *)data.GetLastSegment(l);
+		state.current = data.GetLastSegment(l);
 	} else {
-		state.current = (ColumnSegment *)segment;
+		state.current = segment;
 	}
 
 	D_ASSERT(state.current->segment_type == ColumnSegmentType::TRANSIENT);
@@ -254,7 +256,7 @@ void ColumnData::AppendData(BaseStatistics &stats, ColumnAppendState &state, Uni
 	while (true) {
 		// append the data from the vector
 		idx_t copied_elements = state.current->Append(state, vdata, offset, count);
-		stats.Merge(*state.current->stats.statistics);
+		stats.Merge(state.current->stats.statistics);
 		if (copied_elements == count) {
 			// finished copying everything
 			break;
@@ -264,7 +266,7 @@ void ColumnData::AppendData(BaseStatistics &stats, ColumnAppendState &state, Uni
 		{
 			auto l = data.Lock();
 			AppendTransientSegment(l, state.current->start + state.current->count);
-			state.current = (ColumnSegment *)data.GetLastSegment(l);
+			state.current = data.GetLastSegment(l);
 			state.current->InitializeAppend(state);
 		}
 		offset += copied_elements;
@@ -284,7 +286,7 @@ void ColumnData::RevertAppend(row_t start_row) {
 	// find the segment index that the current row belongs to
 	idx_t segment_index = data.GetSegmentIndex(l, start_row);
 	auto segment = data.GetSegmentByIndex(l, segment_index);
-	auto &transient = (ColumnSegment &)*segment;
+	auto &transient = *segment;
 	D_ASSERT(transient.segment_type == ColumnSegmentType::TRANSIENT);
 
 	// remove any segments AFTER this segment: they should be deleted entirely
@@ -299,14 +301,14 @@ idx_t ColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &result) {
 	D_ASSERT(idx_t(row_id) >= start);
 	// perform the fetch within the segment
 	state.row_index = start + ((row_id - start) / STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE);
-	state.current = (ColumnSegment *)data.GetSegment(state.row_index);
+	state.current = data.GetSegment(state.row_index);
 	state.internal_index = state.current->start;
 	return ScanVector(state, result, STANDARD_VECTOR_SIZE);
 }
 
 void ColumnData::FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result,
                           idx_t result_idx) {
-	auto segment = (ColumnSegment *)data.GetSegment(row_id);
+	auto segment = data.GetSegment(row_id);
 
 	// now perform the fetch within the segment
 	segment->FetchRow(state, row_id, result, result_idx);
@@ -357,15 +359,14 @@ void ColumnData::AppendTransientSegment(SegmentLock &l, idx_t start_row) {
 }
 
 void ColumnData::CommitDropColumn() {
-	auto segment = (ColumnSegment *)data.GetRootSegment();
-	while (segment) {
-		if (segment->segment_type == ColumnSegmentType::PERSISTENT) {
-			auto block_id = segment->GetBlockId();
+	for (auto &segment_p : data.Segments()) {
+		auto &segment = segment_p;
+		if (segment.segment_type == ColumnSegmentType::PERSISTENT) {
+			auto block_id = segment.GetBlockId();
 			if (block_id != INVALID_BLOCK) {
 				block_manager.MarkBlockAsModified(block_id);
 			}
 		}
-		segment = (ColumnSegment *)segment->Next();
 	}
 }
 
@@ -389,7 +390,7 @@ unique_ptr<ColumnCheckpointState> ColumnData::Checkpoint(RowGroup &row_group,
 	// scan the segments of the column data
 	// set up the checkpoint state
 	auto checkpoint_state = CreateCheckpointState(row_group, partial_block_manager);
-	checkpoint_state->global_stats = BaseStatistics::CreateEmpty(type, StatisticsType::LOCAL_STATS);
+	checkpoint_state->global_stats = BaseStatistics::CreateEmpty(type).ToUnique();
 
 	auto l = data.Lock();
 	auto nodes = data.MoveSegments(l);
@@ -414,13 +415,19 @@ void ColumnData::DeserializeColumn(Deserializer &source) {
 	idx_t data_pointer_count = source.Read<idx_t>();
 	for (idx_t data_ptr = 0; data_ptr < data_pointer_count; data_ptr++) {
 		// read the data pointer
-		DataPointer data_pointer;
-		data_pointer.row_start = source.Read<idx_t>();
-		data_pointer.tuple_count = source.Read<idx_t>();
-		data_pointer.block_pointer.block_id = source.Read<block_id_t>();
-		data_pointer.block_pointer.offset = source.Read<uint32_t>();
-		data_pointer.compression_type = source.Read<CompressionType>();
-		data_pointer.statistics = BaseStatistics::Deserialize(source, type);
+		auto row_start = source.Read<idx_t>();
+		auto tuple_count = source.Read<idx_t>();
+		auto block_pointer_block_id = source.Read<block_id_t>();
+		auto block_pointer_offset = source.Read<uint32_t>();
+		auto compression_type = source.Read<CompressionType>();
+		auto stats = BaseStatistics::Deserialize(source, type);
+
+		DataPointer data_pointer(std::move(stats));
+		data_pointer.row_start = row_start;
+		data_pointer.tuple_count = tuple_count;
+		data_pointer.block_pointer.block_id = block_pointer_block_id;
+		data_pointer.block_pointer.offset = block_pointer_offset;
+		data_pointer.compression_type = compression_type;
 
 		// create a persistent segment
 		auto segment = ColumnSegment::CreatePersistentSegment(
@@ -458,7 +465,6 @@ void ColumnData::GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, T
 	while (segment) {
 		ColumnSegmentInfo column_info;
 		column_info.row_group_index = row_group_index;
-		;
 		column_info.column_id = col_path[0];
 		column_info.column_path = col_path_str;
 		column_info.segment_idx = segment_idx;
@@ -466,8 +472,7 @@ void ColumnData::GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, T
 		column_info.segment_start = segment->start;
 		column_info.segment_count = segment->count;
 		column_info.compression_type = CompressionTypeToString(segment->function->type);
-		column_info.segment_stats =
-		    segment->stats.statistics ? segment->stats.statistics->ToString() : string("No Stats");
+		column_info.segment_stats = segment->stats.statistics.ToString();
 		column_info.has_updates = updates ? true : false;
 		// persistent
 		// block_id
@@ -482,7 +487,7 @@ void ColumnData::GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, T
 		result.column_segments.push_back(std::move(column_info));
 
 		segment_idx++;
-		segment = (ColumnSegment *)segment->Next();
+		segment = (ColumnSegment *)data.GetNextSegment(segment);
 	}
 }
 
@@ -490,19 +495,13 @@ void ColumnData::Verify(RowGroup &parent) {
 #ifdef DEBUG
 	D_ASSERT(this->start == parent.start);
 	data.Verify();
-	auto root = data.GetRootSegment();
-	if (root) {
-		D_ASSERT(root != nullptr);
-		D_ASSERT(root->start == this->start);
-		idx_t prev_end = root->start;
-		while (root) {
-			D_ASSERT(prev_end == root->start);
-			prev_end = root->start + root->count;
-			if (!root->next) {
-				D_ASSERT(prev_end == parent.start + parent.count);
-			}
-			root = root->Next();
-		}
+	idx_t current_index = 0;
+	idx_t current_start = this->start;
+	for (auto &segment : data.Segments()) {
+		D_ASSERT(segment.index == current_index);
+		D_ASSERT(segment.start == current_start);
+		current_start += segment.count;
+		current_index++;
 	}
 #endif
 }

package/src/duckdb/src/storage/table/column_data_checkpointer.cpp

@@ -38,7 +38,7 @@ ColumnCheckpointState &ColumnDataCheckpointer::GetCheckpointState() {
 void ColumnDataCheckpointer::ScanSegments(const std::function<void(Vector &, idx_t)> &callback) {
 	Vector scan_vector(intermediate.GetType(), nullptr);
 	for (idx_t segment_idx = 0; segment_idx < nodes.size(); segment_idx++) {
-		auto segment = (ColumnSegment *)nodes[segment_idx].node.get();
+		auto segment = nodes[segment_idx].node.get();
 		ColumnScanState scan_state;
 		scan_state.current = segment;
 		segment->InitializeScan(scan_state);
@@ -163,7 +163,7 @@ void ColumnDataCheckpointer::WriteToDisk() {
 	// since the segments will be rewritten their old on disk data is no longer required
 	auto &block_manager = col_data.block_manager;
 	for (idx_t segment_idx = 0; segment_idx < nodes.size(); segment_idx++) {
-		auto segment = (ColumnSegment *)nodes[segment_idx].node.get();
+		auto segment = nodes[segment_idx].node.get();
 		if (segment->segment_type == ColumnSegmentType::PERSISTENT) {
 			// persistent segment has updates: mark it as modified and rewrite the block with the merged updates
 			auto block_id = segment->GetBlockId();
@@ -194,7 +194,7 @@ void ColumnDataCheckpointer::WriteToDisk() {
 
 bool ColumnDataCheckpointer::HasChanges() {
 	for (idx_t segment_idx = 0; segment_idx < nodes.size(); segment_idx++) {
-		auto segment = (ColumnSegment *)nodes[segment_idx].node.get();
+		auto segment = nodes[segment_idx].node.get();
 		if (segment->segment_type == ColumnSegmentType::TRANSIENT) {
 			// transient segment: always need to write to disk
 			return true;
@@ -214,20 +214,19 @@ void ColumnDataCheckpointer::WritePersistentSegments() {
 	// all segments are persistent and there are no updates
 	// we only need to write the metadata
 	for (idx_t segment_idx = 0; segment_idx < nodes.size(); segment_idx++) {
-		auto segment = (ColumnSegment *)nodes[segment_idx].node.get();
+		auto segment = nodes[segment_idx].node.get();
 		D_ASSERT(segment->segment_type == ColumnSegmentType::PERSISTENT);
 
 		// set up the data pointer directly using the data from the persistent segment
-		DataPointer pointer;
+		DataPointer pointer(segment->stats.statistics.Copy());
 		pointer.block_pointer.block_id = segment->GetBlockId();
 		pointer.block_pointer.offset = segment->GetBlockOffset();
 		pointer.row_start = segment->start;
 		pointer.tuple_count = segment->count;
 		pointer.compression_type = segment->function->type;
-		pointer.statistics = segment->stats.statistics->Copy();
 
 		// merge the persistent stats into the global column stats
-		state.global_stats->Merge(*segment->stats.statistics);
+		state.global_stats->Merge(segment->stats.statistics);
 
 		// directly append the current segment to the new tree
 		state.new_tree.AppendSegment(std::move(nodes[segment_idx].node));
@@ -236,7 +235,7 @@ void ColumnDataCheckpointer::WritePersistentSegments() {
 	}
 }
 
-void ColumnDataCheckpointer::Checkpoint(vector<SegmentNode> nodes) {
+void ColumnDataCheckpointer::Checkpoint(vector<SegmentNode<ColumnSegment>> nodes) {
 	D_ASSERT(!nodes.empty());
 	this->nodes = std::move(nodes);
 	// first check if any of the segments have changes

package/src/duckdb/src/storage/table/column_segment.cpp

@@ -19,7 +19,7 @@ unique_ptr<ColumnSegment> ColumnSegment::CreatePersistentSegment(DatabaseInstanc
                                                                  block_id_t block_id, idx_t offset,
                                                                  const LogicalType &type, idx_t start, idx_t count,
                                                                  CompressionType compression_type,
-                                                                 unique_ptr<BaseStatistics> statistics) {
+                                                                 BaseStatistics statistics) {
 	auto &config = DBConfig::GetConfig(db);
 	CompressionFunction *function;
 	shared_ptr<BlockHandle> block;
@@ -48,7 +48,7 @@ unique_ptr<ColumnSegment> ColumnSegment::CreateTransientSegment(DatabaseInstance
 		buffer_manager.Allocate(segment_size, false, &block);
 	}
 	return make_unique<ColumnSegment>(db, std::move(block), type, ColumnSegmentType::TRANSIENT, start, 0, function,
-	                                  nullptr, INVALID_BLOCK, 0, segment_size);
+	                                  BaseStatistics::CreateEmpty(type), INVALID_BLOCK, 0, segment_size);
 }
 
 unique_ptr<ColumnSegment> ColumnSegment::CreateSegment(ColumnSegment &other, idx_t start) {
@@ -57,11 +57,11 @@ unique_ptr<ColumnSegment> ColumnSegment::CreateSegment(ColumnSegment &other, idx
 
 ColumnSegment::ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block, LogicalType type_p,
                              ColumnSegmentType segment_type, idx_t start, idx_t count, CompressionFunction *function_p,
-                             unique_ptr<BaseStatistics> statistics, block_id_t block_id_p, idx_t offset_p,
-                             idx_t segment_size_p)
-    : SegmentBase(start, count), db(db), type(std::move(type_p)), type_size(GetTypeIdSize(type.InternalType())),
-      segment_type(segment_type), function(function_p), stats(type, std::move(statistics)), block(std::move(block)),
-      block_id(block_id_p), offset(offset_p), segment_size(segment_size_p) {
+                             BaseStatistics statistics, block_id_t block_id_p, idx_t offset_p, idx_t segment_size_p)
+    : SegmentBase<ColumnSegment>(start, count), db(db), type(std::move(type_p)),
+      type_size(GetTypeIdSize(type.InternalType())), segment_type(segment_type), function(function_p),
+      stats(std::move(statistics)), block(std::move(block)), block_id(block_id_p), offset(offset_p),
+      segment_size(segment_size_p) {
 	D_ASSERT(function);
 	if (function->init_segment) {
 		segment_state = function->init_segment(*this, block_id);
@@ -69,10 +69,10 @@ ColumnSegment::ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block
 }
 
 ColumnSegment::ColumnSegment(ColumnSegment &other, idx_t start)
-    : SegmentBase(start, other.count), db(other.db), type(std::move(other.type)), type_size(other.type_size),
-      segment_type(other.segment_type), function(other.function), stats(std::move(other.stats)),
-      block(std::move(other.block)), block_id(other.block_id), offset(other.offset), segment_size(other.segment_size),
-      segment_state(std::move(other.segment_state)) {
+    : SegmentBase<ColumnSegment>(start, other.count.load()), db(other.db), type(std::move(other.type)),
+      type_size(other.type_size), segment_type(other.segment_type), function(other.function),
+      stats(std::move(other.stats)), block(std::move(other.block)), block_id(other.block_id), offset(other.offset),
+      segment_size(other.segment_size), segment_state(std::move(other.segment_state)) {
 }
 
 ColumnSegment::~ColumnSegment() {
@@ -181,13 +181,12 @@ void ColumnSegment::ConvertToPersistent(BlockManager *block_manager, block_id_t
 	block_id = block_id_p;
 	offset = 0;
 
-	D_ASSERT(stats.statistics);
 	if (block_id == INVALID_BLOCK) {
 		// constant block: reset the block buffer
-		D_ASSERT(stats.statistics->IsConstant());
+		D_ASSERT(stats.statistics.IsConstant());
 		block.reset();
 	} else {
-		D_ASSERT(!stats.statistics->IsConstant());
+		D_ASSERT(!stats.statistics.IsConstant());
 		// non-constant block: write the block to disk
 		// the data for the block already exists in-memory of our block
 		// instead of copying the data we alter some metadata so the buffer points to an on-disk block

package/src/duckdb/src/storage/table/list_column_data.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/storage/table/list_column_data.hpp"
-#include "duckdb/storage/statistics/list_statistics.hpp"
+#include "duckdb/storage/statistics/list_stats.hpp"
 #include "duckdb/transaction/transaction.hpp"
+#include "duckdb/storage/table/column_checkpoint_state.hpp"
 
 namespace duckdb {
 
@@ -39,15 +40,14 @@ void ListColumnData::InitializeScan(ColumnScanState &state) {
 	state.child_states.push_back(std::move(child_state));
 }
 
-list_entry_t ListColumnData::FetchListEntry(idx_t row_idx) {
-	auto segment = (ColumnSegment *)data.GetSegment(row_idx);
+uint64_t ListColumnData::FetchListOffset(idx_t row_idx) {
+	auto segment = data.GetSegment(row_idx);
 	ColumnFetchState fetch_state;
 	Vector result(type, 1);
 	segment->FetchRow(fetch_state, row_idx, result, 0);
 
 	// initialize the child scan with the required offset
-	auto list_data = FlatVector::GetData<list_entry_t>(result);
-	return list_data[0];
+	return FlatVector::GetData<uint64_t>(result)[0];
 }
 
 void ListColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) {
@@ -63,8 +63,7 @@ void ListColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_
 	state.child_states.push_back(std::move(validity_state));
 
 	// we need to read the list at position row_idx to get the correct row offset of the child
-	auto list_entry = FetchListEntry(row_idx);
-	auto child_offset = list_entry.offset;
+	auto child_offset = row_idx == start ? 0 : FetchListOffset(row_idx - 1);
 
 	D_ASSERT(child_offset <= child_column->GetMaxEntry());
 	ColumnScanState child_state;
@@ -89,26 +88,26 @@ idx_t ListColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t co
 	// updates not supported for lists
 	D_ASSERT(!updates);
 
-	idx_t scan_count = ScanVector(state, result, count);
+	Vector offset_vector(LogicalType::UBIGINT, count);
+	idx_t scan_count = ScanVector(state, offset_vector, count);
 	D_ASSERT(scan_count > 0);
 	validity.ScanCount(state.child_states[0], result, count);
 
-	auto data = FlatVector::GetData<list_entry_t>(result);
-	auto first_entry = data[0];
+	auto data = FlatVector::GetData<uint64_t>(offset_vector);
 	auto last_entry = data[scan_count - 1];
 
-#ifdef DEBUG
-	for (idx_t i = 1; i < scan_count; i++) {
-		D_ASSERT(data[i].offset == data[i - 1].offset + data[i - 1].length);
-	}
-#endif
 	// shift all offsets so they are 0 at the first entry
+	auto result_data = FlatVector::GetData<list_entry_t>(result);
+	auto base_offset = state.last_offset;
+	idx_t current_offset = 0;
 	for (idx_t i = 0; i < scan_count; i++) {
-		data[i].offset -= first_entry.offset;
+		result_data[i].offset = current_offset;
+		result_data[i].length = data[i] - current_offset - base_offset;
+		current_offset += result_data[i].length;
 	}
 
-	D_ASSERT(last_entry.offset >= first_entry.offset);
-	idx_t child_scan_count = last_entry.offset + last_entry.length - first_entry.offset;
+	D_ASSERT(last_entry >= base_offset);
+	idx_t child_scan_count = last_entry - base_offset;
 	ListVector::Reserve(result, child_scan_count);
 
 	if (child_scan_count > 0) {
@@ -118,6 +117,7 @@ idx_t ListColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t co
 		             child_column->start + child_column->GetMaxEntry());
 		child_column->ScanCount(state.child_states[1], child_entry, child_scan_count);
 	}
+	state.last_offset = last_entry;
 
 	ListVector::SetListSize(result, child_scan_count);
 	return scan_count;
@@ -130,19 +130,19 @@ void ListColumnData::Skip(ColumnScanState &state, idx_t count) {
 	// we need to read the list entries/offsets to figure out how much to skip
 	// note that we only need to read the first and last entry
 	// however, let's just read all "count" entries for now
-	auto data = unique_ptr<list_entry_t[]>(new list_entry_t[count]);
-	Vector result(type, (data_ptr_t)data.get());
+	Vector result(LogicalType::UBIGINT, count);
 	idx_t scan_count = ScanVector(state, result, count);
 	if (scan_count == 0) {
 		return;
 	}
 
-	auto &first_entry = data[0];
-	auto &last_entry = data[scan_count - 1];
-	idx_t child_scan_count = last_entry.offset + last_entry.length - first_entry.offset;
+	auto data = FlatVector::GetData<uint64_t>(result);
+	auto last_entry = data[scan_count - 1];
+	idx_t child_scan_count = last_entry - state.last_offset;
 	if (child_scan_count == 0) {
 		return;
 	}
+	state.last_offset = last_entry;
 
 	// skip the child state forward by the child_scan_count
 	child_column->Skip(state.child_states[1], child_scan_count);
@@ -163,10 +163,8 @@ void ListColumnData::InitializeAppend(ColumnAppendState &state) {
 	state.child_appends.push_back(std::move(child_append_state));
 }
 
-void ListColumnData::Append(BaseStatistics &stats_p, ColumnAppendState &state, Vector &vector, idx_t count) {
+void ListColumnData::Append(BaseStatistics &stats, ColumnAppendState &state, Vector &vector, idx_t count) {
 	D_ASSERT(count > 0);
-	auto &stats = (ListStatistics &)stats_p;
-
 	UnifiedVectorFormat list_data;
 	vector.ToUnifiedFormat(count, list_data);
 	auto &list_validity = list_data.validity;
@@ -177,8 +175,8 @@ void ListColumnData::Append(BaseStatistics &stats_p, ColumnAppendState &state, V
 	idx_t child_count = 0;
 
 	ValidityMask append_mask(count);
-	auto append_offsets = unique_ptr<list_entry_t[]>(new list_entry_t[count]);
-	bool child_contiguous = false;
+	auto append_offsets = unique_ptr<uint64_t[]>(new uint64_t[count]);
+	bool child_contiguous = true;
 	for (idx_t i = 0; i < count; i++) {
 		auto input_idx = list_data.sel->get_index(i);
 		if (list_validity.RowIsValid(input_idx)) {
@@ -186,17 +184,11 @@ void ListColumnData::Append(BaseStatistics &stats_p, ColumnAppendState &state, V
 			if (input_list.offset != child_count) {
 				child_contiguous = false;
 			}
-			append_offsets[i].offset = start_offset + child_count;
-			append_offsets[i].length = input_list.length;
+			append_offsets[i] = start_offset + child_count + input_list.length;
 			child_count += input_list.length;
 		} else {
 			append_mask.SetInvalid(i);
-			if (i > 0) {
-				append_offsets[i].offset = append_offsets[i - 1].offset + append_offsets[i - 1].length;
-			} else {
-				append_offsets[i].offset = start_offset;
-			}
-			append_offsets[i].length = 0;
+			append_offsets[i] = start_offset + child_count;
 		}
 	}
 	auto &list_child = ListVector::GetEntry(vector);
@@ -218,27 +210,19 @@ void ListColumnData::Append(BaseStatistics &stats_p, ColumnAppendState &state, V
 		D_ASSERT(current_count == child_count);
 		child_vector.Slice(list_child, child_sel, child_count);
 	}
-#ifdef DEBUG
-	D_ASSERT(append_offsets[0].offset == start_offset);
-	for (idx_t i = 1; i < count; i++) {
-		D_ASSERT(append_offsets[i].offset == append_offsets[i - 1].offset + append_offsets[i - 1].length);
-	}
-	D_ASSERT(append_offsets[count - 1].offset + append_offsets[count - 1].length - append_offsets[0].offset ==
-	         child_count);
-#endif
 
 	UnifiedVectorFormat vdata;
-	vdata.validity = append_mask;
 	vdata.sel = FlatVector::IncrementalSelectionVector();
 	vdata.data = (data_ptr_t)append_offsets.get();
 
 	// append the list offsets
 	ColumnData::AppendData(stats, state, vdata, count);
 	// append the validity data
-	validity.AppendData(*stats.validity_stats, state.child_appends[0], vdata, count);
+	vdata.validity = append_mask;
+	validity.AppendData(stats, state.child_appends[0], vdata, count);
 	// append the child vector
 	if (child_count > 0) {
-		child_column->Append(*stats.child_stats, state.child_appends[1], child_vector, child_count);
+		child_column->Append(ListStats::GetChildStats(stats), state.child_appends[1], child_vector, child_count);
 	}
 }
 
@@ -248,8 +232,8 @@ void ListColumnData::RevertAppend(row_t start_row) {
 	auto column_count = GetMaxEntry();
 	if (column_count > start) {
 		// revert append in the child column
-		auto list_entry = FetchListEntry(column_count - 1);
-		child_column->RevertAppend(list_entry.offset + list_entry.length);
+		auto list_offset = FetchListOffset(column_count - 1);
+		child_column->RevertAppend(list_offset);
 	}
 }
 
@@ -281,19 +265,18 @@ void ListColumnData::FetchRow(TransactionData transaction, ColumnFetchState &sta
 		auto child_state = make_unique<ColumnFetchState>();
 		state.child_states.push_back(std::move(child_state));
 	}
-	// fetch the list_entry_t and the validity mask for that list
-	auto segment = (ColumnSegment *)data.GetSegment(row_id);
 
 	// now perform the fetch within the segment
-	segment->FetchRow(state, row_id, result, result_idx);
+	auto start_offset = idx_t(row_id) == this->start ? 0 : FetchListOffset(row_id - 1);
+	auto end_offset = FetchListOffset(row_id);
 	validity.FetchRow(transaction, *state.child_states[0], row_id, result, result_idx);
 
 	auto &validity = FlatVector::Validity(result);
 	auto list_data = FlatVector::GetData<list_entry_t>(result);
 	auto &list_entry = list_data[result_idx];
-	auto original_offset = list_entry.offset;
 	// set the list entry offset to the size of the current list
 	list_entry.offset = ListVector::GetListSize(result);
+	list_entry.length = end_offset - start_offset;
 	if (!validity.RowIsValid(result_idx)) {
 		// the list is NULL! no need to fetch the child
 		D_ASSERT(list_entry.length == 0);
@@ -307,7 +290,7 @@ void ListColumnData::FetchRow(TransactionData transaction, ColumnFetchState &sta
 		auto &child_type = ListType::GetChildType(result.GetType());
 		Vector child_scan(child_type, child_scan_count);
 		// seek the scan towards the specified position and read [length] entries
-		child_column->InitializeScanWithOffset(*child_state, start + original_offset);
+		child_column->InitializeScanWithOffset(*child_state, start + start_offset);
 		D_ASSERT(child_type.InternalType() == PhysicalType::STRUCT ||
 		         child_state->row_index + child_scan_count - this->start <= child_column->GetMaxEntry());
 		child_column->ScanCount(*child_state, child_scan, child_scan_count);
@@ -324,7 +307,7 @@ void ListColumnData::CommitDropColumn() {
 struct ListColumnCheckpointState : public ColumnCheckpointState {
 	ListColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, PartialBlockManager &partial_block_manager)
 	    : ColumnCheckpointState(row_group, column_data, partial_block_manager) {
-		global_stats = make_unique<ListStatistics>(column_data.type);
+		global_stats = ListStats::CreateEmpty(column_data.type).ToUnique();
 	}
 
 	unique_ptr<ColumnCheckpointState> validity_state;
@@ -333,10 +316,8 @@ struct ListColumnCheckpointState : public ColumnCheckpointState {
 public:
 	unique_ptr<BaseStatistics> GetStatistics() override {
 		auto stats = global_stats->Copy();
-		auto &list_stats = (ListStatistics &)*stats;
-		stats->validity_stats = validity_state->GetStatistics();
-		list_stats.child_stats = child_state->GetStatistics();
-		return stats;
+		ListStats::SetChildStats(stats, child_state->GetStatistics());
+		return stats.ToUnique();
 	}
 
 	void WriteDataPointers(RowGroupWriter &writer) override {
@@ -376,6 +357,7 @@ void ListColumnData::DeserializeColumn(Deserializer &source) {
 }
 
 void ListColumnData::GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, TableStorageInfo &result) {
+	ColumnData::GetStorageInfo(row_group_index, col_path, result);
 	col_path.push_back(0);
 	validity.GetStorageInfo(row_group_index, col_path, result);
 	col_path.back() = 1;

package/src/duckdb/src/storage/table/persistent_table_data.cpp

@@ -3,7 +3,8 @@
 
 namespace duckdb {
 
-PersistentTableData::PersistentTableData(idx_t column_count) {
+PersistentTableData::PersistentTableData(idx_t column_count)
+    : total_rows(0), row_group_count(0), block_id(INVALID_BLOCK), offset(0) {
 }
 
 PersistentTableData::~PersistentTableData() {