duckdb 0.7.1 → 0.7.2-dev1034.0

package/src/duckdb/src/storage/table/row_group.cpp

@@ -4,7 +4,7 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/field_writer.hpp"
 #include "duckdb/storage/table/column_data.hpp"
-#include "duckdb/storage/table/standard_column_data.hpp"
+#include "duckdb/storage/table/column_checkpoint_state.hpp"
 #include "duckdb/storage/table/update_segment.hpp"
 #include "duckdb/common/chrono.hpp"
 #include "duckdb/planner/table_filter.hpp"
@@ -23,14 +23,14 @@ constexpr const idx_t RowGroup::ROW_GROUP_SIZE;
 
 RowGroup::RowGroup(AttachedDatabase &db, BlockManager &block_manager, DataTableInfo &table_info, idx_t start,
                    idx_t count)
-    : SegmentBase(start, count), db(db), block_manager(block_manager), table_info(table_info) {
+    : SegmentBase<RowGroup>(start, count), db(db), block_manager(block_manager), table_info(table_info) {
 
 	Verify();
 }
 
 RowGroup::RowGroup(AttachedDatabase &db, BlockManager &block_manager, DataTableInfo &table_info,
                    const vector<LogicalType> &types, RowGroupPointer &&pointer)
-    : SegmentBase(pointer.row_start, pointer.tuple_count), db(db), block_manager(block_manager),
+    : SegmentBase<RowGroup>(pointer.row_start, pointer.tuple_count), db(db), block_manager(block_manager),
       table_info(table_info) {
 	// deserialize the columns
 	if (pointer.data_pointers.size() != types.size()) {
@@ -46,8 +46,7 @@ RowGroup::RowGroup(AttachedDatabase &db, BlockManager &block_manager, DataTableI
 
 	// set up the statistics
 	for (auto &stats : pointer.statistics) {
-		auto stats_type = stats->type;
-		this->stats.push_back(make_shared<SegmentStatistics>(stats_type, std::move(stats)));
+		this->stats.emplace_back(std::move(stats));
 	}
 	this->version_info = std::move(pointer.versions);
 
@@ -55,7 +54,7 @@ RowGroup::RowGroup(AttachedDatabase &db, BlockManager &block_manager, DataTableI
 }
 
 RowGroup::RowGroup(RowGroup &row_group, idx_t start)
-    : SegmentBase(start, row_group.count), db(row_group.db), block_manager(row_group.block_manager),
+    : SegmentBase<RowGroup>(start, row_group.count.load()), db(row_group.db), block_manager(row_group.block_manager),
       table_info(row_group.table_info), version_info(std::move(row_group.version_info)),
       stats(std::move(row_group.stats)) {
 	for (auto &column : row_group.columns) {
@@ -88,7 +87,7 @@ void RowGroup::InitializeEmpty(const vector<LogicalType> &types) {
 	// set up the segment trees for the column segments
 	for (idx_t i = 0; i < types.size(); i++) {
 		auto column_data = ColumnData::CreateColumn(block_manager, GetTableInfo(), i, start, types[i]);
-		stats.push_back(make_shared<SegmentStatistics>(types[i]));
+		stats.emplace_back(types[i]);
 		columns.push_back(std::move(column_data));
 	}
 }
@@ -157,8 +156,12 @@ unique_ptr<RowGroup> RowGroup::AlterType(const LogicalType &target_type, idx_t c
 	// scan the original table, and fill the new column with the transformed value
 	InitializeScan(scan_state);
 
-	Vector append_vector(target_type);
-	auto altered_col_stats = make_shared<SegmentStatistics>(target_type);
+	DataChunk append_chunk;
+	vector<LogicalType> append_types;
+	append_types.push_back(target_type);
+	append_chunk.Initialize(Allocator::DefaultAllocator(), append_types);
+	auto &append_vector = append_chunk.data[0];
+	SegmentStatistics altered_col_stats(target_type);
 	while (true) {
 		// scan the table
 		scan_chunk.Reset();
@@ -167,8 +170,9 @@ unique_ptr<RowGroup> RowGroup::AlterType(const LogicalType &target_type, idx_t c
 			break;
 		}
 		// execute the expression
+		append_chunk.Reset();
 		executor.ExecuteExpression(scan_chunk, append_vector);
-		column_data->Append(*altered_col_stats->statistics, append_state, append_vector, scan_chunk.size());
+		column_data->Append(altered_col_stats.statistics, append_state, append_vector, scan_chunk.size());
 	}
 
 	// set up the row_group based on this row_group
@@ -178,11 +182,11 @@ unique_ptr<RowGroup> RowGroup::AlterType(const LogicalType &target_type, idx_t c
 		if (i == changed_idx) {
 			// this is the altered column: use the new column
 			row_group->columns.push_back(std::move(column_data));
-			row_group->stats.push_back(std::move(altered_col_stats));
+			row_group->stats.push_back(std::move(altered_col_stats)); // NOLINT: false positive
 		} else {
 			// this column was not altered: use the data directly
 			row_group->columns.push_back(columns[i]);
-			row_group->stats.push_back(stats[i]);
+			row_group->stats.emplace_back(stats[i].statistics.Copy());
 		}
 	}
 	row_group->Verify();
@@ -196,8 +200,7 @@ unique_ptr<RowGroup> RowGroup::AddColumn(ColumnDefinition &new_column, Expressio
 	// construct a new column data for the new column
 	auto added_column =
 	    ColumnData::CreateColumn(block_manager, GetTableInfo(), columns.size(), start, new_column.Type());
-	auto added_col_stats = make_shared<SegmentStatistics>(
-	    new_column.Type(), BaseStatistics::CreateEmpty(new_column.Type(), StatisticsType::LOCAL_STATS));
+	SegmentStatistics added_col_stats(new_column.Type());
 
 	idx_t rows_to_write = this->count;
 	if (rows_to_write > 0) {
@@ -211,7 +214,7 @@ unique_ptr<RowGroup> RowGroup::AddColumn(ColumnDefinition &new_column, Expressio
 				dummy_chunk.SetCardinality(rows_in_this_vector);
 				executor.ExecuteExpression(dummy_chunk, result);
 			}
-			added_column->Append(*added_col_stats->statistics, state, result, rows_in_this_vector);
+			added_column->Append(added_col_stats.statistics, state, result, rows_in_this_vector);
 		}
 	}
 
@@ -219,7 +222,9 @@ unique_ptr<RowGroup> RowGroup::AddColumn(ColumnDefinition &new_column, Expressio
 	auto row_group = make_unique<RowGroup>(db, block_manager, table_info, this->start, this->count);
 	row_group->version_info = version_info;
 	row_group->columns = columns;
-	row_group->stats = stats;
+	for (auto &stat : stats) {
+		row_group->stats.emplace_back(stat.statistics.Copy());
+	}
 	// now add the new column
 	row_group->columns.push_back(std::move(added_column));
 	row_group->stats.push_back(std::move(added_col_stats));
@@ -236,7 +241,9 @@ unique_ptr<RowGroup> RowGroup::RemoveColumn(idx_t removed_column) {
 	auto row_group = make_unique<RowGroup>(db, block_manager, table_info, this->start, this->count);
 	row_group->version_info = version_info;
 	row_group->columns = columns;
-	row_group->stats = stats;
+	for (auto &stat : stats) {
+		row_group->stats.emplace_back(stat.statistics.Copy());
+	}
 	// now remove the column
 	row_group->columns.erase(row_group->columns.begin() + removed_column);
 	row_group->stats.erase(row_group->stats.begin() + removed_column);
@@ -275,7 +282,7 @@ bool RowGroup::CheckZonemap(TableFilterSet &filters, const vector<column_t> &col
 		auto &filter = entry.second;
 		auto base_column_index = column_ids[column_index];
 
-		auto propagate_result = filter->CheckStatistics(*stats[base_column_index]->statistics);
+		auto propagate_result = filter->CheckStatistics(stats[base_column_index].statistics);
 		if (propagate_result == FilterPropagateResult::FILTER_ALWAYS_FALSE ||
 		    propagate_result == FilterPropagateResult::FILTER_FALSE_OR_NULL) {
 			return false;
@@ -628,7 +635,7 @@ void RowGroup::InitializeAppend(RowGroupAppendState &append_state) {
 void RowGroup::Append(RowGroupAppendState &state, DataChunk &chunk, idx_t append_count) {
 	// append to the current row_group
 	for (idx_t i = 0; i < columns.size(); i++) {
-		columns[i]->Append(*stats[i]->statistics, state.states[i], chunk.data[i], append_count);
+		columns[i]->Append(stats[i].statistics, state.states[i], chunk.data[i], append_count);
 	}
 	state.offset_in_row_group += append_count;
 }
@@ -671,21 +678,21 @@ unique_ptr<BaseStatistics> RowGroup::GetStatistics(idx_t column_idx) {
 	D_ASSERT(column_idx < stats.size());
 
 	lock_guard<mutex> slock(stats_lock);
-	return stats[column_idx]->statistics->Copy();
+	return stats[column_idx].statistics.ToUnique();
 }
 
 void RowGroup::MergeStatistics(idx_t column_idx, const BaseStatistics &other) {
 	D_ASSERT(column_idx < stats.size());
 
 	lock_guard<mutex> slock(stats_lock);
-	stats[column_idx]->statistics->Merge(other);
+	stats[column_idx].statistics.Merge(other);
 }
 
 void RowGroup::MergeIntoStatistics(idx_t column_idx, BaseStatistics &other) {
 	D_ASSERT(column_idx < stats.size());
 
 	lock_guard<mutex> slock(stats_lock);
-	other.Merge(*stats[column_idx]->statistics);
+	other.Merge(stats[column_idx].statistics);
 }
 
 RowGroupWriteData RowGroup::WriteToDisk(PartialBlockManager &manager,
@@ -711,14 +718,14 @@ RowGroupWriteData RowGroup::WriteToDisk(PartialBlockManager &manager,
 		auto stats = checkpoint_state->GetStatistics();
 		D_ASSERT(stats);
 
-		result.statistics.push_back(std::move(stats));
+		result.statistics.push_back(stats->Copy());
 		result.states.push_back(std::move(checkpoint_state));
 	}
 	D_ASSERT(result.states.size() == result.statistics.size());
 	return result;
 }
 
-RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, vector<unique_ptr<BaseStatistics>> &global_stats) {
+RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, TableStatistics &global_stats) {
 	RowGroupPointer row_group_pointer;
 
 	vector<CompressionType> compression_types;
@@ -728,7 +735,7 @@ RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, vector<unique_ptr<B
 	}
 	auto result = WriteToDisk(writer.GetPartialBlockManager(), compression_types);
 	for (idx_t column_idx = 0; column_idx < columns.size(); column_idx++) {
-		global_stats[column_idx]->Merge(*result.statistics[column_idx]);
+		global_stats.GetStats(column_idx).Statistics().Merge(result.statistics[column_idx]);
 	}
 	row_group_pointer.statistics = std::move(result.statistics);
 
@@ -805,7 +812,7 @@ void RowGroup::Serialize(RowGroupPointer &pointer, Serializer &main_serializer)
 	writer.WriteField<uint64_t>(pointer.tuple_count);
 	auto &serializer = writer.GetSerializer();
 	for (auto &stats : pointer.statistics) {
-		stats->Serialize(serializer);
+		stats.Serialize(serializer);
 	}
 	for (auto &data_pointer : pointer.data_pointers) {
 		serializer.Write<block_id_t>(data_pointer.block_id);
@@ -815,23 +822,22 @@ void RowGroup::Serialize(RowGroupPointer &pointer, Serializer &main_serializer)
 	writer.Finalize();
 }
 
-RowGroupPointer RowGroup::Deserialize(Deserializer &main_source, const ColumnList &columns) {
+RowGroupPointer RowGroup::Deserialize(Deserializer &main_source, const vector<LogicalType> &columns) {
 	RowGroupPointer result;
 
 	FieldReader reader(main_source);
 	result.row_start = reader.ReadRequired<uint64_t>();
 	result.tuple_count = reader.ReadRequired<uint64_t>();
 
-	auto physical_columns = columns.PhysicalColumnCount();
+	auto physical_columns = columns.size();
 	result.data_pointers.reserve(physical_columns);
 	result.statistics.reserve(physical_columns);
 
 	auto &source = reader.GetSource();
-	for (auto &col : columns.Physical()) {
-		auto stats = BaseStatistics::Deserialize(source, col.Type());
-		result.statistics.push_back(std::move(stats));
+	for (auto &col_type : columns) {
+		result.statistics.push_back(BaseStatistics::Deserialize(source, col_type));
 	}
-	for (idx_t i = 0; i < columns.PhysicalColumnCount(); i++) {
+	for (idx_t i = 0; i < columns.size(); i++) {
 		BlockPointer pointer;
 		pointer.block_id = source.Read<block_id_t>();
 		pointer.offset = source.Read<uint64_t>();

package/src/duckdb/src/storage/table/row_group_collection.cpp

@@ -6,14 +6,48 @@
 #include "duckdb/transaction/transaction.hpp"
 #include "duckdb/planner/constraints/bound_not_null_constraint.hpp"
 #include "duckdb/storage/checkpoint/table_data_writer.hpp"
+#include "duckdb/storage/table/row_group_segment_tree.hpp"
+#include "duckdb/storage/meta_block_reader.hpp"
 
 namespace duckdb {
 
+//===--------------------------------------------------------------------===//
+// Row Group Segment Tree
+//===--------------------------------------------------------------------===//
+RowGroupSegmentTree::RowGroupSegmentTree(DataTableInfo &table_info_p, BlockManager &block_manager_p,
+                                         vector<LogicalType> column_types_p)
+    : SegmentTree<RowGroup, true>(), info(table_info_p), block_manager(block_manager_p),
+      column_types(std::move(column_types_p)), current_row_group(0), max_row_group(0) {
+}
+RowGroupSegmentTree::~RowGroupSegmentTree() {
+}
+
+void RowGroupSegmentTree::Initialize(PersistentTableData &data) {
+	D_ASSERT(data.row_group_count > 0);
+	current_row_group = 0;
+	max_row_group = data.row_group_count;
+	finished_loading = false;
+	reader = make_unique<MetaBlockReader>(block_manager, data.block_id);
+	reader->offset = data.offset;
+}
+
+unique_ptr<RowGroup> RowGroupSegmentTree::LoadSegment() {
+	if (current_row_group >= max_row_group) {
+		finished_loading = true;
+		return nullptr;
+	}
+	auto row_group_pointer = RowGroup::Deserialize(*reader, column_types);
+	current_row_group++;
+	return make_unique<RowGroup>(info.db, block_manager, info, column_types, std::move(row_group_pointer));
+}
+//===--------------------------------------------------------------------===//
+// Row Group Collection
+//===--------------------------------------------------------------------===//
 RowGroupCollection::RowGroupCollection(shared_ptr<DataTableInfo> info_p, BlockManager &block_manager,
                                        vector<LogicalType> types_p, idx_t row_start_p, idx_t total_rows_p)
     : block_manager(block_manager), total_rows(total_rows_p), info(std::move(info_p)), types(std::move(types_p)),
       row_start(row_start_p) {
-	row_groups = make_shared<SegmentTree>();
+	row_groups = make_shared<RowGroupSegmentTree>(*info, block_manager, types);
 }
 
 idx_t RowGroupCollection::GetTotalRows() const {
@@ -34,14 +68,8 @@ Allocator &RowGroupCollection::GetAllocator() const {
 void RowGroupCollection::Initialize(PersistentTableData &data) {
 	D_ASSERT(this->row_start == 0);
 	auto l = row_groups->Lock();
-	for (auto &row_group_pointer : data.row_groups) {
-		auto new_row_group = make_unique<RowGroup>(info->db, block_manager, *info, types, std::move(row_group_pointer));
-		auto row_group_count = new_row_group->start + new_row_group->count;
-		if (row_group_count > this->total_rows) {
-			this->total_rows = row_group_count;
-		}
-		row_groups->AppendSegment(l, std::move(new_row_group));
-	}
+	this->total_rows = data.total_rows;
+	row_groups->Initialize(data);
 	stats.Initialize(types, data);
 }
 
@@ -64,8 +92,7 @@ void RowGroupCollection::Verify() {
 #ifdef DEBUG
 	idx_t current_total_rows = 0;
 	row_groups->Verify();
-	for (auto segment = row_groups->GetRootSegment(); segment; segment = segment->Next()) {
-		auto &row_group = (RowGroup &)*segment;
+	for (auto &row_group : row_groups->Segments()) {
 		row_group.Verify();
 		D_ASSERT(row_group.start == this->row_start + current_total_rows);
 		current_total_rows += row_group.count;
@@ -79,11 +106,12 @@ void RowGroupCollection::Verify() {
 //===--------------------------------------------------------------------===//
 void RowGroupCollection::InitializeScan(CollectionScanState &state, const vector<column_t> &column_ids,
                                         TableFilterSet *table_filters) {
-	auto row_group = (RowGroup *)row_groups->GetRootSegment();
+	auto row_group = row_groups->GetRootSegment();
 	D_ASSERT(row_group);
+	state.row_groups = row_groups.get();
 	state.max_row = row_start + total_rows;
 	while (row_group && !row_group->InitializeScan(state.row_group_state)) {
-		row_group = (RowGroup *)row_group->Next();
+		row_group = row_groups->GetNextSegment(row_group);
 	}
 }
 
@@ -93,8 +121,9 @@ void RowGroupCollection::InitializeCreateIndexScan(CreateIndexScanState &state)
 
 void RowGroupCollection::InitializeScanWithOffset(CollectionScanState &state, const vector<column_t> &column_ids,
                                                   idx_t start_row, idx_t end_row) {
-	auto row_group = (RowGroup *)row_groups->GetSegment(start_row);
+	auto row_group = row_groups->GetSegment(start_row);
 	D_ASSERT(row_group);
+	state.row_groups = row_groups.get();
 	state.max_row = end_row;
 	idx_t start_vector = (start_row - row_group->start) / STANDARD_VECTOR_SIZE;
 	if (!row_group->InitializeScanWithOffset(state.row_group_state, start_vector)) {
@@ -102,14 +131,17 @@ void RowGroupCollection::InitializeScanWithOffset(CollectionScanState &state, co
 	}
 }
 
-bool RowGroupCollection::InitializeScanInRowGroup(CollectionScanState &state, RowGroup *row_group, idx_t vector_index,
+bool RowGroupCollection::InitializeScanInRowGroup(CollectionScanState &state,
+                                                  ParallelCollectionScanState &parallel_state, idx_t vector_index,
                                                   idx_t max_row) {
 	state.max_row = max_row;
-	return row_group->InitializeScanWithOffset(state.row_group_state, vector_index);
+	state.row_groups = parallel_state.collection->row_groups.get();
+	return parallel_state.current_row_group->InitializeScanWithOffset(state.row_group_state, vector_index);
 }
 
 void RowGroupCollection::InitializeParallelScan(ParallelCollectionScanState &state) {
-	state.current_row_group = (RowGroup *)row_groups->GetRootSegment();
+	state.collection = this;
+	state.current_row_group = row_groups->GetRootSegment();
 	state.vector_index = 0;
 	state.max_row = row_start + total_rows;
 	state.batch_index = 0;
@@ -131,15 +163,15 @@ bool RowGroupCollection::NextParallelScan(ClientContext &context, ParallelCollec
 			max_row = state.current_row_group->start + state.current_row_group->count;
 		}
 		max_row = MinValue<idx_t>(max_row, state.max_row);
-		bool need_to_scan = InitializeScanInRowGroup(scan_state, state.current_row_group, vector_index, max_row);
+		bool need_to_scan = InitializeScanInRowGroup(scan_state, state, vector_index, max_row);
 		if (ClientConfig::GetConfig(context).verify_parallelism) {
 			state.vector_index++;
 			if (state.vector_index * STANDARD_VECTOR_SIZE >= state.current_row_group->count) {
-				state.current_row_group = (RowGroup *)state.current_row_group->Next();
+				state.current_row_group = row_groups->GetNextSegment(state.current_row_group);
 				state.vector_index = 0;
 			}
 		} else {
-			state.current_row_group = (RowGroup *)state.current_row_group->Next();
+			state.current_row_group = row_groups->GetNextSegment(state.current_row_group);
 		}
 		scan_state.batch_index = ++state.batch_index;
 		if (!need_to_scan) {
@@ -204,7 +236,7 @@ void RowGroupCollection::Fetch(TransactionData transaction, DataChunk &result, c
 				// in parallel append scenarios it is possible for the row_id
 				continue;
 			}
-			row_group = (RowGroup *)row_groups->GetSegmentByIndex(l, segment_index);
+			row_group = row_groups->GetSegmentByIndex(l, segment_index);
 		}
 		if (!row_group->Fetch(transaction, row_id - row_group->start)) {
 			continue;
@@ -246,7 +278,7 @@ void RowGroupCollection::InitializeAppend(TransactionData transaction, TableAppe
 		// empty row group collection: empty first row group
 		AppendRowGroup(l, row_start);
 	}
-	state.start_row_group = (RowGroup *)row_groups->GetLastSegment(l);
+	state.start_row_group = row_groups->GetLastSegment(l);
 	D_ASSERT(this->row_start + total_rows == state.start_row_group->start + state.start_row_group->count);
 	state.start_row_group->InitializeAppend(state.row_group_append_state);
 	state.remaining = append_count;
@@ -280,7 +312,7 @@ bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state) {
 			// merge the stats
 			auto stats_lock = stats.GetLock();
 			for (idx_t i = 0; i < types.size(); i++) {
-				current_row_group->MergeIntoStatistics(i, *stats.GetStats(i).stats);
+				current_row_group->MergeIntoStatistics(i, stats.GetStats(i).Statistics());
 			}
 		}
 		remaining -= append_count;
@@ -306,7 +338,7 @@ bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state) {
 			auto l = row_groups->Lock();
 			AppendRowGroup(l, next_start);
 			// set up the append state for this row_group
-			auto last_row_group = (RowGroup *)row_groups->GetLastSegment(l);
+			auto last_row_group = row_groups->GetLastSegment(l);
 			last_row_group->InitializeAppend(state.row_group_append_state);
 			if (state.remaining > 0) {
 				last_row_group->AppendVersionInfo(state.transaction, state.remaining);
@@ -319,11 +351,7 @@ bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state) {
 	state.current_row += append_count;
 	auto stats_lock = stats.GetLock();
 	for (idx_t col_idx = 0; col_idx < types.size(); col_idx++) {
-		auto type = types[col_idx].InternalType();
-		if (type == PhysicalType::LIST || type == PhysicalType::STRUCT) {
-			continue;
-		}
-		stats.GetStats(col_idx).stats->UpdateDistinctStatistics(chunk.data[col_idx], chunk.size());
+		stats.GetStats(col_idx).UpdateDistinctStatistics(chunk.data[col_idx], chunk.size());
 	}
 	return new_row_group;
 }
@@ -335,7 +363,7 @@ void RowGroupCollection::FinalizeAppend(TransactionData transaction, TableAppend
 		auto append_count = MinValue<idx_t>(remaining, RowGroup::ROW_GROUP_SIZE - row_group->count);
 		row_group->AppendVersionInfo(transaction, append_count);
 		remaining -= append_count;
-		row_group = (RowGroup *)row_group->Next();
+		row_group = row_groups->GetNextSegment(row_group);
 	}
 	total_rows += state.total_append_count;
 
@@ -346,7 +374,7 @@ void RowGroupCollection::FinalizeAppend(TransactionData transaction, TableAppend
 }
 
 void RowGroupCollection::CommitAppend(transaction_t commit_id, idx_t row_start, idx_t count) {
-	auto row_group = (RowGroup *)row_groups->GetSegment(row_start);
+	auto row_group = row_groups->GetSegment(row_start);
 	D_ASSERT(row_group);
 	idx_t current_row = row_start;
 	idx_t remaining = count;
@@ -361,7 +389,7 @@ void RowGroupCollection::CommitAppend(transaction_t commit_id, idx_t row_start,
 		if (remaining == 0) {
 			break;
 		}
-		row_group = (RowGroup *)row_group->Next();
+		row_group = row_groups->GetNextSegment(row_group);
 	}
 }
 
@@ -375,7 +403,7 @@ void RowGroupCollection::RevertAppendInternal(idx_t start_row, idx_t count) {
 	// find the segment index that the current row belongs to
 	idx_t segment_index = row_groups->GetSegmentIndex(l, start_row);
 	auto segment = row_groups->GetSegmentByIndex(l, segment_index);
-	auto &info = (RowGroup &)*segment;
+	auto &info = *segment;
 
 	// remove any segments AFTER this segment: they should be deleted entirely
 	row_groups->EraseSegments(l, segment_index);
@@ -387,8 +415,7 @@ void RowGroupCollection::RevertAppendInternal(idx_t start_row, idx_t count) {
 void RowGroupCollection::MergeStorage(RowGroupCollection &data) {
 	D_ASSERT(data.types == types);
 	auto index = row_start + total_rows.load();
-	for (auto segment = data.row_groups->GetRootSegment(); segment; segment = segment->Next()) {
-		auto &row_group = (RowGroup &)*segment;
+	for (auto &row_group : data.row_groups->Segments()) {
 		auto new_group = make_unique<RowGroup>(row_group, index);
 		index += new_group->count;
 		row_groups->AppendSegment(std::move(new_group));
@@ -409,7 +436,7 @@ idx_t RowGroupCollection::Delete(TransactionData transaction, DataTable *table,
 	idx_t pos = 0;
 	do {
 		idx_t start = pos;
-		auto row_group = (RowGroup *)row_groups->GetSegment(ids[start]);
+		auto row_group = row_groups->GetSegment(ids[start]);
 		for (pos++; pos < count; pos++) {
 			D_ASSERT(ids[pos] >= 0);
 			// check if this id still belongs to this row group
@@ -435,7 +462,7 @@ void RowGroupCollection::Update(TransactionData transaction, row_t *ids, const v
 	idx_t pos = 0;
 	do {
 		idx_t start = pos;
-		auto row_group = (RowGroup *)row_groups->GetSegment(ids[pos]);
+		auto row_group = row_groups->GetSegment(ids[pos]);
 		row_t base_id =
 		    row_group->start + ((ids[pos] - row_group->start) / STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE);
 		row_t max_id = MinValue<row_t>(base_id + STANDARD_VECTOR_SIZE, row_group->start + row_group->count);
@@ -465,7 +492,7 @@ void RowGroupCollection::RemoveFromIndexes(TableIndexList &indexes, Vector &row_
 	auto row_ids = FlatVector::GetData<row_t>(row_identifiers);
 
 	// figure out which row_group to fetch from
-	auto row_group = (RowGroup *)row_groups->GetSegment(row_ids[0]);
+	auto row_group = row_groups->GetSegment(row_ids[0]);
 	auto row_group_vector_idx = (row_ids[0] - row_group->start) / STANDARD_VECTOR_SIZE;
 	auto base_row_id = row_group_vector_idx * STANDARD_VECTOR_SIZE + row_group->start;
 
@@ -510,20 +537,19 @@ void RowGroupCollection::UpdateColumn(TransactionData transaction, Vector &row_i
 	}
 	// find the row_group this id belongs to
 	auto primary_column_idx = column_path[0];
-	auto row_group = (RowGroup *)row_groups->GetSegment(first_id);
+	auto row_group = row_groups->GetSegment(first_id);
 	row_group->UpdateColumn(transaction, updates, row_ids, column_path);
 
-	row_group->MergeIntoStatistics(primary_column_idx, *stats.GetStats(primary_column_idx).stats);
+	row_group->MergeIntoStatistics(primary_column_idx, stats.GetStats(primary_column_idx).Statistics());
 }
 
 //===--------------------------------------------------------------------===//
 // Checkpoint
 //===--------------------------------------------------------------------===//
-void RowGroupCollection::Checkpoint(TableDataWriter &writer, vector<unique_ptr<BaseStatistics>> &global_stats) {
-	for (auto row_group = (RowGroup *)row_groups->GetRootSegment(); row_group;
-	     row_group = (RowGroup *)row_group->Next()) {
-		auto rowg_writer = writer.GetRowGroupWriter(*row_group);
-		auto pointer = row_group->Checkpoint(*rowg_writer, global_stats);
+void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &global_stats) {
+	for (auto &row_group : row_groups->Segments()) {
+		auto rowg_writer = writer.GetRowGroupWriter(row_group);
+		auto pointer = row_group.Checkpoint(*rowg_writer, global_stats);
 		writer.AddRowGroup(std::move(pointer), std::move(rowg_writer));
 	}
 }
@@ -532,18 +558,14 @@ void RowGroupCollection::Checkpoint(TableDataWriter &writer, vector<unique_ptr<B
 // CommitDrop
 //===--------------------------------------------------------------------===//
 void RowGroupCollection::CommitDropColumn(idx_t index) {
-	auto segment = (RowGroup *)row_groups->GetRootSegment();
-	while (segment) {
-		segment->CommitDropColumn(index);
-		segment = (RowGroup *)segment->Next();
+	for (auto &row_group : row_groups->Segments()) {
+		row_group.CommitDropColumn(index);
 	}
 }
 
 void RowGroupCollection::CommitDropTable() {
-	auto segment = (RowGroup *)row_groups->GetRootSegment();
-	while (segment) {
-		segment->CommitDrop();
-		segment = (RowGroup *)segment->Next();
+	for (auto &row_group : row_groups->Segments()) {
+		row_group.CommitDrop();
 	}
 }
 
@@ -551,13 +573,8 @@ void RowGroupCollection::CommitDropTable() {
 // GetStorageInfo
 //===--------------------------------------------------------------------===//
 void RowGroupCollection::GetStorageInfo(TableStorageInfo &result) {
-	auto row_group = (RowGroup *)row_groups->GetRootSegment();
-	idx_t row_group_index = 0;
-	while (row_group) {
-		row_group->GetStorageInfo(row_group_index, result);
-		row_group_index++;
-
-		row_group = (RowGroup *)row_group->Next();
+	for (auto &row_group : row_groups->Segments()) {
+		row_group.GetStorageInfo(row_group.index, result);
 	}
 }
 
@@ -586,14 +603,12 @@ shared_ptr<RowGroupCollection> RowGroupCollection::AddColumn(ClientContext &cont
 
 	// fill the column with its DEFAULT value, or NULL if none is specified
 	auto new_stats = make_unique<SegmentStatistics>(new_column.GetType());
-	auto current_row_group = (RowGroup *)row_groups->GetRootSegment();
-	while (current_row_group) {
-		auto new_row_group = current_row_group->AddColumn(new_column, executor, default_value, default_vector);
+	for (auto &current_row_group : row_groups->Segments()) {
+		auto new_row_group = current_row_group.AddColumn(new_column, executor, default_value, default_vector);
 		// merge in the statistics
-		new_row_group->MergeIntoStatistics(new_column_idx, *new_column_stats.stats);
+		new_row_group->MergeIntoStatistics(new_column_idx, new_column_stats.Statistics());
 
 		result->row_groups->AppendSegment(std::move(new_row_group));
-		current_row_group = (RowGroup *)current_row_group->Next();
 	}
 	return result;
 }
@@ -607,11 +622,9 @@ shared_ptr<RowGroupCollection> RowGroupCollection::RemoveColumn(idx_t col_idx) {
 	    make_shared<RowGroupCollection>(info, block_manager, std::move(new_types), row_start, total_rows.load());
 	result->stats.InitializeRemoveColumn(stats, col_idx);
 
-	auto current_row_group = (RowGroup *)row_groups->GetRootSegment();
-	while (current_row_group) {
-		auto new_row_group = current_row_group->RemoveColumn(col_idx);
+	for (auto &current_row_group : row_groups->Segments()) {
+		auto new_row_group = current_row_group.RemoveColumn(col_idx);
 		result->row_groups->AppendSegment(std::move(new_row_group));
-		current_row_group = (RowGroup *)current_row_group->Next();
 	}
 	return result;
 }
@@ -646,14 +659,12 @@ shared_ptr<RowGroupCollection> RowGroupCollection::AlterType(ClientContext &cont
 	scan_state.table_state.max_row = row_start + total_rows;
 
 	// now alter the type of the column within all of the row_groups individually
-	auto current_row_group = (RowGroup *)row_groups->GetRootSegment();
 	auto &changed_stats = result->stats.GetStats(changed_idx);
-	while (current_row_group) {
-		auto new_row_group = current_row_group->AlterType(target_type, changed_idx, executor,
-		                                                  scan_state.table_state.row_group_state, scan_chunk);
-		new_row_group->MergeIntoStatistics(changed_idx, *changed_stats.stats);
+	for (auto &current_row_group : row_groups->Segments()) {
+		auto new_row_group = current_row_group.AlterType(target_type, changed_idx, executor,
+		                                                 scan_state.table_state.row_group_state, scan_chunk);
+		new_row_group->MergeIntoStatistics(changed_idx, changed_stats.Statistics());
 		result->row_groups->AppendSegment(std::move(new_row_group));
-		current_row_group = (RowGroup *)current_row_group->Next();
 	}
 
 	return result;
@@ -681,7 +692,8 @@ void RowGroupCollection::VerifyNewConstraint(DataTable &parent, const BoundConst
 	InitializeCreateIndexScan(state);
 	while (true) {
 		scan_chunk.Reset();
-		state.table_state.ScanCommitted(scan_chunk, TableScanType::TABLE_SCAN_COMMITTED_ROWS_OMIT_PERMANENTLY_DELETED);
+		state.table_state.ScanCommitted(scan_chunk, state.segment_lock,
+		                                TableScanType::TABLE_SCAN_COMMITTED_ROWS_OMIT_PERMANENTLY_DELETED);
 		if (scan_chunk.size() == 0) {
 			break;
 		}
@@ -696,14 +708,18 @@ void RowGroupCollection::VerifyNewConstraint(DataTable &parent, const BoundConst
 //===--------------------------------------------------------------------===//
 // Statistics
 //===--------------------------------------------------------------------===//
+void RowGroupCollection::CopyStats(TableStatistics &other_stats) {
+	stats.CopyStats(other_stats);
+}
+
 unique_ptr<BaseStatistics> RowGroupCollection::CopyStats(column_t column_id) {
 	return stats.CopyStats(column_id);
 }
 
-void RowGroupCollection::SetStatistics(column_t column_id, const std::function<void(BaseStatistics &)> &set_fun) {
+void RowGroupCollection::SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats) {
 	D_ASSERT(column_id != COLUMN_IDENTIFIER_ROW_ID);
 	auto stats_guard = stats.GetLock();
-	set_fun(*stats.GetStats(column_id).stats);
+	stats.GetStats(column_id).SetDistinct(std::move(distinct_stats));
 }
 
 } // namespace duckdb