duckdb 0.7.2-dev16.0 → 0.7.2-dev314.0

package/src/duckdb/src/storage/table/list_column_data.cpp

@@ -1,5 +1,5 @@
 #include "duckdb/storage/table/list_column_data.hpp"
-#include "duckdb/storage/statistics/list_statistics.hpp"
+#include "duckdb/storage/statistics/list_stats.hpp"
 #include "duckdb/transaction/transaction.hpp"
 
 namespace duckdb {
@@ -39,15 +39,14 @@ void ListColumnData::InitializeScan(ColumnScanState &state) {
 	state.child_states.push_back(std::move(child_state));
 }
 
-list_entry_t ListColumnData::FetchListEntry(idx_t row_idx) {
+uint64_t ListColumnData::FetchListOffset(idx_t row_idx) {
 	auto segment = (ColumnSegment *)data.GetSegment(row_idx);
 	ColumnFetchState fetch_state;
 	Vector result(type, 1);
 	segment->FetchRow(fetch_state, row_idx, result, 0);
 
 	// initialize the child scan with the required offset
-	auto list_data = FlatVector::GetData<list_entry_t>(result);
-	return list_data[0];
+	return FlatVector::GetData<uint64_t>(result)[0];
 }
 
 void ListColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) {
@@ -63,8 +62,7 @@ void ListColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_
 	state.child_states.push_back(std::move(validity_state));
 
 	// we need to read the list at position row_idx to get the correct row offset of the child
-	auto list_entry = FetchListEntry(row_idx);
-	auto child_offset = list_entry.offset;
+	auto child_offset = row_idx == start ? 0 : FetchListOffset(row_idx - 1);
 
 	D_ASSERT(child_offset <= child_column->GetMaxEntry());
 	ColumnScanState child_state;
@@ -89,26 +87,26 @@ idx_t ListColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t co
 	// updates not supported for lists
 	D_ASSERT(!updates);
 
-	idx_t scan_count = ScanVector(state, result, count);
+	Vector offset_vector(LogicalType::UBIGINT, count);
+	idx_t scan_count = ScanVector(state, offset_vector, count);
 	D_ASSERT(scan_count > 0);
 	validity.ScanCount(state.child_states[0], result, count);
 
-	auto data = FlatVector::GetData<list_entry_t>(result);
-	auto first_entry = data[0];
+	auto data = FlatVector::GetData<uint64_t>(offset_vector);
 	auto last_entry = data[scan_count - 1];
 
-#ifdef DEBUG
-	for (idx_t i = 1; i < scan_count; i++) {
-		D_ASSERT(data[i].offset == data[i - 1].offset + data[i - 1].length);
-	}
-#endif
 	// shift all offsets so they are 0 at the first entry
+	auto result_data = FlatVector::GetData<list_entry_t>(result);
+	auto base_offset = state.last_offset;
+	idx_t current_offset = 0;
 	for (idx_t i = 0; i < scan_count; i++) {
-		data[i].offset -= first_entry.offset;
+		result_data[i].offset = current_offset;
+		result_data[i].length = data[i] - current_offset - base_offset;
+		current_offset += result_data[i].length;
 	}
 
-	D_ASSERT(last_entry.offset >= first_entry.offset);
-	idx_t child_scan_count = last_entry.offset + last_entry.length - first_entry.offset;
+	D_ASSERT(last_entry >= base_offset);
+	idx_t child_scan_count = last_entry - base_offset;
 	ListVector::Reserve(result, child_scan_count);
 
 	if (child_scan_count > 0) {
@@ -118,6 +116,7 @@ idx_t ListColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t co
 		             child_column->start + child_column->GetMaxEntry());
 		child_column->ScanCount(state.child_states[1], child_entry, child_scan_count);
 	}
+	state.last_offset = last_entry;
 
 	ListVector::SetListSize(result, child_scan_count);
 	return scan_count;
@@ -130,19 +129,19 @@ void ListColumnData::Skip(ColumnScanState &state, idx_t count) {
 	// we need to read the list entries/offsets to figure out how much to skip
 	// note that we only need to read the first and last entry
 	// however, let's just read all "count" entries for now
-	auto data = unique_ptr<list_entry_t[]>(new list_entry_t[count]);
-	Vector result(type, (data_ptr_t)data.get());
+	Vector result(LogicalType::UBIGINT, count);
 	idx_t scan_count = ScanVector(state, result, count);
 	if (scan_count == 0) {
 		return;
 	}
 
-	auto &first_entry = data[0];
-	auto &last_entry = data[scan_count - 1];
-	idx_t child_scan_count = last_entry.offset + last_entry.length - first_entry.offset;
+	auto data = FlatVector::GetData<uint64_t>(result);
+	auto last_entry = data[scan_count - 1];
+	idx_t child_scan_count = last_entry - state.last_offset;
 	if (child_scan_count == 0) {
 		return;
 	}
+	state.last_offset = last_entry;
 
 	// skip the child state forward by the child_scan_count
 	child_column->Skip(state.child_states[1], child_scan_count);
@@ -163,10 +162,8 @@ void ListColumnData::InitializeAppend(ColumnAppendState &state) {
 	state.child_appends.push_back(std::move(child_append_state));
 }
 
-void ListColumnData::Append(BaseStatistics &stats_p, ColumnAppendState &state, Vector &vector, idx_t count) {
+void ListColumnData::Append(BaseStatistics &stats, ColumnAppendState &state, Vector &vector, idx_t count) {
 	D_ASSERT(count > 0);
-	auto &stats = (ListStatistics &)stats_p;
-
 	UnifiedVectorFormat list_data;
 	vector.ToUnifiedFormat(count, list_data);
 	auto &list_validity = list_data.validity;
@@ -177,8 +174,8 @@ void ListColumnData::Append(BaseStatistics &stats_p, ColumnAppendState &state, V
 	idx_t child_count = 0;
 
 	ValidityMask append_mask(count);
-	auto append_offsets = unique_ptr<list_entry_t[]>(new list_entry_t[count]);
-	bool child_contiguous = false;
+	auto append_offsets = unique_ptr<uint64_t[]>(new uint64_t[count]);
+	bool child_contiguous = true;
 	for (idx_t i = 0; i < count; i++) {
 		auto input_idx = list_data.sel->get_index(i);
 		if (list_validity.RowIsValid(input_idx)) {
@@ -186,17 +183,11 @@ void ListColumnData::Append(BaseStatistics &stats_p, ColumnAppendState &state, V
 			if (input_list.offset != child_count) {
 				child_contiguous = false;
 			}
-			append_offsets[i].offset = start_offset + child_count;
-			append_offsets[i].length = input_list.length;
+			append_offsets[i] = start_offset + child_count + input_list.length;
 			child_count += input_list.length;
 		} else {
 			append_mask.SetInvalid(i);
-			if (i > 0) {
-				append_offsets[i].offset = append_offsets[i - 1].offset + append_offsets[i - 1].length;
-			} else {
-				append_offsets[i].offset = start_offset;
-			}
-			append_offsets[i].length = 0;
+			append_offsets[i] = start_offset + child_count;
 		}
 	}
 	auto &list_child = ListVector::GetEntry(vector);
@@ -218,27 +209,19 @@ void ListColumnData::Append(BaseStatistics &stats_p, ColumnAppendState &state, V
 		D_ASSERT(current_count == child_count);
 		child_vector.Slice(list_child, child_sel, child_count);
 	}
-#ifdef DEBUG
-	D_ASSERT(append_offsets[0].offset == start_offset);
-	for (idx_t i = 1; i < count; i++) {
-		D_ASSERT(append_offsets[i].offset == append_offsets[i - 1].offset + append_offsets[i - 1].length);
-	}
-	D_ASSERT(append_offsets[count - 1].offset + append_offsets[count - 1].length - append_offsets[0].offset ==
-	         child_count);
-#endif
 
 	UnifiedVectorFormat vdata;
-	vdata.validity = append_mask;
 	vdata.sel = FlatVector::IncrementalSelectionVector();
 	vdata.data = (data_ptr_t)append_offsets.get();
 
 	// append the list offsets
 	ColumnData::AppendData(stats, state, vdata, count);
 	// append the validity data
-	validity.AppendData(*stats.validity_stats, state.child_appends[0], vdata, count);
+	vdata.validity = append_mask;
+	validity.AppendData(stats, state.child_appends[0], vdata, count);
 	// append the child vector
 	if (child_count > 0) {
-		child_column->Append(*stats.child_stats, state.child_appends[1], child_vector, child_count);
+		child_column->Append(ListStats::GetChildStats(stats), state.child_appends[1], child_vector, child_count);
 	}
 }
 
@@ -248,8 +231,8 @@ void ListColumnData::RevertAppend(row_t start_row) {
 	auto column_count = GetMaxEntry();
 	if (column_count > start) {
 		// revert append in the child column
-		auto list_entry = FetchListEntry(column_count - 1);
-		child_column->RevertAppend(list_entry.offset + list_entry.length);
+		auto list_offset = FetchListOffset(column_count - 1);
+		child_column->RevertAppend(list_offset);
 	}
 }
 
@@ -281,19 +264,18 @@ void ListColumnData::FetchRow(TransactionData transaction, ColumnFetchState &sta
 		auto child_state = make_unique<ColumnFetchState>();
 		state.child_states.push_back(std::move(child_state));
 	}
-	// fetch the list_entry_t and the validity mask for that list
-	auto segment = (ColumnSegment *)data.GetSegment(row_id);
 
 	// now perform the fetch within the segment
-	segment->FetchRow(state, row_id, result, result_idx);
+	auto start_offset = idx_t(row_id) == this->start ? 0 : FetchListOffset(row_id - 1);
+	auto end_offset = FetchListOffset(row_id);
 	validity.FetchRow(transaction, *state.child_states[0], row_id, result, result_idx);
 
 	auto &validity = FlatVector::Validity(result);
 	auto list_data = FlatVector::GetData<list_entry_t>(result);
 	auto &list_entry = list_data[result_idx];
-	auto original_offset = list_entry.offset;
 	// set the list entry offset to the size of the current list
 	list_entry.offset = ListVector::GetListSize(result);
+	list_entry.length = end_offset - start_offset;
 	if (!validity.RowIsValid(result_idx)) {
 		// the list is NULL! no need to fetch the child
 		D_ASSERT(list_entry.length == 0);
@@ -307,7 +289,7 @@ void ListColumnData::FetchRow(TransactionData transaction, ColumnFetchState &sta
 		auto &child_type = ListType::GetChildType(result.GetType());
 		Vector child_scan(child_type, child_scan_count);
 		// seek the scan towards the specified position and read [length] entries
-		child_column->InitializeScanWithOffset(*child_state, start + original_offset);
+		child_column->InitializeScanWithOffset(*child_state, start + start_offset);
 		D_ASSERT(child_type.InternalType() == PhysicalType::STRUCT ||
 		         child_state->row_index + child_scan_count - this->start <= child_column->GetMaxEntry());
 		child_column->ScanCount(*child_state, child_scan, child_scan_count);
@@ -324,7 +306,7 @@ void ListColumnData::CommitDropColumn() {
 struct ListColumnCheckpointState : public ColumnCheckpointState {
 	ListColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, PartialBlockManager &partial_block_manager)
 	    : ColumnCheckpointState(row_group, column_data, partial_block_manager) {
-		global_stats = make_unique<ListStatistics>(column_data.type);
+		global_stats = ListStats::CreateEmpty(column_data.type).ToUnique();
 	}
 
 	unique_ptr<ColumnCheckpointState> validity_state;
@@ -333,10 +315,8 @@ struct ListColumnCheckpointState : public ColumnCheckpointState {
 public:
 	unique_ptr<BaseStatistics> GetStatistics() override {
 		auto stats = global_stats->Copy();
-		auto &list_stats = (ListStatistics &)*stats;
-		stats->validity_stats = validity_state->GetStatistics();
-		list_stats.child_stats = child_state->GetStatistics();
-		return stats;
+		ListStats::SetChildStats(stats, child_state->GetStatistics());
+		return stats.ToUnique();
 	}
 
 	void WriteDataPointers(RowGroupWriter &writer) override {
@@ -376,6 +356,7 @@ void ListColumnData::DeserializeColumn(Deserializer &source) {
 }
 
 void ListColumnData::GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, TableStorageInfo &result) {
+	ColumnData::GetStorageInfo(row_group_index, col_path, result);
 	col_path.push_back(0);
 	validity.GetStorageInfo(row_group_index, col_path, result);
 	col_path.back() = 1;

package/src/duckdb/src/storage/table/row_group.cpp

@@ -46,8 +46,7 @@ RowGroup::RowGroup(AttachedDatabase &db, BlockManager &block_manager, DataTableI
 
 	// set up the statistics
 	for (auto &stats : pointer.statistics) {
-		auto stats_type = stats->type;
-		this->stats.push_back(make_shared<SegmentStatistics>(stats_type, std::move(stats)));
+		this->stats.emplace_back(std::move(stats));
 	}
 	this->version_info = std::move(pointer.versions);
 
@@ -88,7 +87,7 @@ void RowGroup::InitializeEmpty(const vector<LogicalType> &types) {
 	// set up the segment trees for the column segments
 	for (idx_t i = 0; i < types.size(); i++) {
 		auto column_data = ColumnData::CreateColumn(block_manager, GetTableInfo(), i, start, types[i]);
-		stats.push_back(make_shared<SegmentStatistics>(types[i]));
+		stats.emplace_back(types[i]);
 		columns.push_back(std::move(column_data));
 	}
 }
@@ -158,7 +157,7 @@ unique_ptr<RowGroup> RowGroup::AlterType(const LogicalType &target_type, idx_t c
 	InitializeScan(scan_state);
 
 	Vector append_vector(target_type);
-	auto altered_col_stats = make_shared<SegmentStatistics>(target_type);
+	SegmentStatistics altered_col_stats(target_type);
 	while (true) {
 		// scan the table
 		scan_chunk.Reset();
@@ -168,7 +167,7 @@ unique_ptr<RowGroup> RowGroup::AlterType(const LogicalType &target_type, idx_t c
 		}
 		// execute the expression
 		executor.ExecuteExpression(scan_chunk, append_vector);
-		column_data->Append(*altered_col_stats->statistics, append_state, append_vector, scan_chunk.size());
+		column_data->Append(altered_col_stats.statistics, append_state, append_vector, scan_chunk.size());
 	}
 
 	// set up the row_group based on this row_group
@@ -178,11 +177,11 @@ unique_ptr<RowGroup> RowGroup::AlterType(const LogicalType &target_type, idx_t c
 		if (i == changed_idx) {
 			// this is the altered column: use the new column
 			row_group->columns.push_back(std::move(column_data));
-			row_group->stats.push_back(std::move(altered_col_stats));
+			row_group->stats.push_back(std::move(altered_col_stats)); // NOLINT: false positive
 		} else {
 			// this column was not altered: use the data directly
 			row_group->columns.push_back(columns[i]);
-			row_group->stats.push_back(stats[i]);
+			row_group->stats.emplace_back(stats[i].statistics.Copy());
 		}
 	}
 	row_group->Verify();
@@ -196,8 +195,7 @@ unique_ptr<RowGroup> RowGroup::AddColumn(ColumnDefinition &new_column, Expressio
 	// construct a new column data for the new column
 	auto added_column =
 	    ColumnData::CreateColumn(block_manager, GetTableInfo(), columns.size(), start, new_column.Type());
-	auto added_col_stats = make_shared<SegmentStatistics>(
-	    new_column.Type(), BaseStatistics::CreateEmpty(new_column.Type(), StatisticsType::LOCAL_STATS));
+	SegmentStatistics added_col_stats(new_column.Type());
 
 	idx_t rows_to_write = this->count;
 	if (rows_to_write > 0) {
@@ -211,7 +209,7 @@ unique_ptr<RowGroup> RowGroup::AddColumn(ColumnDefinition &new_column, Expressio
 				dummy_chunk.SetCardinality(rows_in_this_vector);
 				executor.ExecuteExpression(dummy_chunk, result);
 			}
-			added_column->Append(*added_col_stats->statistics, state, result, rows_in_this_vector);
+			added_column->Append(added_col_stats.statistics, state, result, rows_in_this_vector);
 		}
 	}
 
@@ -219,7 +217,9 @@ unique_ptr<RowGroup> RowGroup::AddColumn(ColumnDefinition &new_column, Expressio
 	auto row_group = make_unique<RowGroup>(db, block_manager, table_info, this->start, this->count);
 	row_group->version_info = version_info;
 	row_group->columns = columns;
-	row_group->stats = stats;
+	for (auto &stat : stats) {
+		row_group->stats.emplace_back(stat.statistics.Copy());
+	}
 	// now add the new column
 	row_group->columns.push_back(std::move(added_column));
 	row_group->stats.push_back(std::move(added_col_stats));
@@ -236,7 +236,9 @@ unique_ptr<RowGroup> RowGroup::RemoveColumn(idx_t removed_column) {
 	auto row_group = make_unique<RowGroup>(db, block_manager, table_info, this->start, this->count);
 	row_group->version_info = version_info;
 	row_group->columns = columns;
-	row_group->stats = stats;
+	for (auto &stat : stats) {
+		row_group->stats.emplace_back(stat.statistics.Copy());
+	}
 	// now remove the column
 	row_group->columns.erase(row_group->columns.begin() + removed_column);
 	row_group->stats.erase(row_group->stats.begin() + removed_column);
@@ -275,7 +277,7 @@ bool RowGroup::CheckZonemap(TableFilterSet &filters, const vector<column_t> &col
 		auto &filter = entry.second;
 		auto base_column_index = column_ids[column_index];
 
-		auto propagate_result = filter->CheckStatistics(*stats[base_column_index]->statistics);
+		auto propagate_result = filter->CheckStatistics(stats[base_column_index].statistics);
 		if (propagate_result == FilterPropagateResult::FILTER_ALWAYS_FALSE ||
 		    propagate_result == FilterPropagateResult::FILTER_FALSE_OR_NULL) {
 			return false;
@@ -628,7 +630,7 @@ void RowGroup::InitializeAppend(RowGroupAppendState &append_state) {
 void RowGroup::Append(RowGroupAppendState &state, DataChunk &chunk, idx_t append_count) {
 	// append to the current row_group
 	for (idx_t i = 0; i < columns.size(); i++) {
-		columns[i]->Append(*stats[i]->statistics, state.states[i], chunk.data[i], append_count);
+		columns[i]->Append(stats[i].statistics, state.states[i], chunk.data[i], append_count);
 	}
 	state.offset_in_row_group += append_count;
 }
@@ -671,21 +673,21 @@ unique_ptr<BaseStatistics> RowGroup::GetStatistics(idx_t column_idx) {
 	D_ASSERT(column_idx < stats.size());
 
 	lock_guard<mutex> slock(stats_lock);
-	return stats[column_idx]->statistics->Copy();
+	return stats[column_idx].statistics.ToUnique();
 }
 
 void RowGroup::MergeStatistics(idx_t column_idx, const BaseStatistics &other) {
 	D_ASSERT(column_idx < stats.size());
 
 	lock_guard<mutex> slock(stats_lock);
-	stats[column_idx]->statistics->Merge(other);
+	stats[column_idx].statistics.Merge(other);
 }
 
 void RowGroup::MergeIntoStatistics(idx_t column_idx, BaseStatistics &other) {
 	D_ASSERT(column_idx < stats.size());
 
 	lock_guard<mutex> slock(stats_lock);
-	other.Merge(*stats[column_idx]->statistics);
+	other.Merge(stats[column_idx].statistics);
 }
 
 RowGroupWriteData RowGroup::WriteToDisk(PartialBlockManager &manager,
@@ -711,14 +713,14 @@ RowGroupWriteData RowGroup::WriteToDisk(PartialBlockManager &manager,
 		auto stats = checkpoint_state->GetStatistics();
 		D_ASSERT(stats);
 
-		result.statistics.push_back(std::move(stats));
+		result.statistics.push_back(stats->Copy());
 		result.states.push_back(std::move(checkpoint_state));
 	}
 	D_ASSERT(result.states.size() == result.statistics.size());
 	return result;
 }
 
-RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, vector<unique_ptr<BaseStatistics>> &global_stats) {
+RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, TableStatistics &global_stats) {
 	RowGroupPointer row_group_pointer;
 
 	vector<CompressionType> compression_types;
@@ -728,7 +730,7 @@ RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, vector<unique_ptr<B
 	}
 	auto result = WriteToDisk(writer.GetPartialBlockManager(), compression_types);
 	for (idx_t column_idx = 0; column_idx < columns.size(); column_idx++) {
-		global_stats[column_idx]->Merge(*result.statistics[column_idx]);
+		global_stats.GetStats(column_idx).Statistics().Merge(result.statistics[column_idx]);
 	}
 	row_group_pointer.statistics = std::move(result.statistics);
 
@@ -805,7 +807,7 @@ void RowGroup::Serialize(RowGroupPointer &pointer, Serializer &main_serializer)
 	writer.WriteField<uint64_t>(pointer.tuple_count);
 	auto &serializer = writer.GetSerializer();
 	for (auto &stats : pointer.statistics) {
-		stats->Serialize(serializer);
+		stats.Serialize(serializer);
 	}
 	for (auto &data_pointer : pointer.data_pointers) {
 		serializer.Write<block_id_t>(data_pointer.block_id);
@@ -828,8 +830,7 @@ RowGroupPointer RowGroup::Deserialize(Deserializer &main_source, const ColumnLis
 
 	auto &source = reader.GetSource();
 	for (auto &col : columns.Physical()) {
-		auto stats = BaseStatistics::Deserialize(source, col.Type());
-		result.statistics.push_back(std::move(stats));
+		result.statistics.push_back(BaseStatistics::Deserialize(source, col.Type()));
 	}
 	for (idx_t i = 0; i < columns.PhysicalColumnCount(); i++) {
 		BlockPointer pointer;

package/src/duckdb/src/storage/table/row_group_collection.cpp

@@ -280,7 +280,7 @@ bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state) {
 			// merge the stats
 			auto stats_lock = stats.GetLock();
 			for (idx_t i = 0; i < types.size(); i++) {
-				current_row_group->MergeIntoStatistics(i, *stats.GetStats(i).stats);
+				current_row_group->MergeIntoStatistics(i, stats.GetStats(i).Statistics());
 			}
 		}
 		remaining -= append_count;
@@ -319,11 +319,7 @@ bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state) {
 	state.current_row += append_count;
 	auto stats_lock = stats.GetLock();
 	for (idx_t col_idx = 0; col_idx < types.size(); col_idx++) {
-		auto type = types[col_idx].InternalType();
-		if (type == PhysicalType::LIST || type == PhysicalType::STRUCT) {
-			continue;
-		}
-		stats.GetStats(col_idx).stats->UpdateDistinctStatistics(chunk.data[col_idx], chunk.size());
+		stats.GetStats(col_idx).UpdateDistinctStatistics(chunk.data[col_idx], chunk.size());
 	}
 	return new_row_group;
 }
@@ -513,13 +509,13 @@ void RowGroupCollection::UpdateColumn(TransactionData transaction, Vector &row_i
 	auto row_group = (RowGroup *)row_groups->GetSegment(first_id);
 	row_group->UpdateColumn(transaction, updates, row_ids, column_path);
 
-	row_group->MergeIntoStatistics(primary_column_idx, *stats.GetStats(primary_column_idx).stats);
+	row_group->MergeIntoStatistics(primary_column_idx, stats.GetStats(primary_column_idx).Statistics());
 }
 
 //===--------------------------------------------------------------------===//
 // Checkpoint
 //===--------------------------------------------------------------------===//
-void RowGroupCollection::Checkpoint(TableDataWriter &writer, vector<unique_ptr<BaseStatistics>> &global_stats) {
+void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &global_stats) {
 	for (auto row_group = (RowGroup *)row_groups->GetRootSegment(); row_group;
 	     row_group = (RowGroup *)row_group->Next()) {
 		auto rowg_writer = writer.GetRowGroupWriter(*row_group);
@@ -590,7 +586,7 @@ shared_ptr<RowGroupCollection> RowGroupCollection::AddColumn(ClientContext &cont
 	while (current_row_group) {
 		auto new_row_group = current_row_group->AddColumn(new_column, executor, default_value, default_vector);
 		// merge in the statistics
-		new_row_group->MergeIntoStatistics(new_column_idx, *new_column_stats.stats);
+		new_row_group->MergeIntoStatistics(new_column_idx, new_column_stats.Statistics());
 
 		result->row_groups->AppendSegment(std::move(new_row_group));
 		current_row_group = (RowGroup *)current_row_group->Next();
@@ -651,7 +647,7 @@ shared_ptr<RowGroupCollection> RowGroupCollection::AlterType(ClientContext &cont
 	while (current_row_group) {
 		auto new_row_group = current_row_group->AlterType(target_type, changed_idx, executor,
 		                                                  scan_state.table_state.row_group_state, scan_chunk);
-		new_row_group->MergeIntoStatistics(changed_idx, *changed_stats.stats);
+		new_row_group->MergeIntoStatistics(changed_idx, changed_stats.Statistics());
 		result->row_groups->AppendSegment(std::move(new_row_group));
 		current_row_group = (RowGroup *)current_row_group->Next();
 	}
@@ -696,14 +692,18 @@ void RowGroupCollection::VerifyNewConstraint(DataTable &parent, const BoundConst
 //===--------------------------------------------------------------------===//
 // Statistics
 //===--------------------------------------------------------------------===//
+void RowGroupCollection::CopyStats(TableStatistics &other_stats) {
+	stats.CopyStats(other_stats);
+}
+
 unique_ptr<BaseStatistics> RowGroupCollection::CopyStats(column_t column_id) {
 	return stats.CopyStats(column_id);
 }
 
-void RowGroupCollection::SetStatistics(column_t column_id, const std::function<void(BaseStatistics &)> &set_fun) {
+void RowGroupCollection::SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats) {
 	D_ASSERT(column_id != COLUMN_IDENTIFIER_ROW_ID);
 	auto stats_guard = stats.GetLock();
-	set_fun(*stats.GetStats(column_id).stats);
+	stats.GetStats(column_id).SetDistinct(std::move(distinct_stats));
 }
 
 } // namespace duckdb

package/src/duckdb/src/storage/table/standard_column_data.cpp

@@ -24,7 +24,7 @@ bool StandardColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filte
 			return true;
 		}
 		state.segment_checked = true;
-		auto prune_result = filter.CheckStatistics(*state.current->stats.statistics);
+		auto prune_result = filter.CheckStatistics(state.current->stats.statistics);
 		if (prune_result != FilterPropagateResult::FILTER_ALWAYS_FALSE) {
 			return true;
 		}
@@ -91,8 +91,7 @@ void StandardColumnData::InitializeAppend(ColumnAppendState &state) {
 void StandardColumnData::AppendData(BaseStatistics &stats, ColumnAppendState &state, UnifiedVectorFormat &vdata,
                                     idx_t count) {
 	ColumnData::AppendData(stats, state, vdata, count);
-
-	validity.AppendData(*stats.validity_stats, state.child_appends[0], vdata, count);
+	validity.AppendData(stats, state.child_appends[0], vdata, count);
 }
 
 void StandardColumnData::RevertAppend(row_t start_row) {
@@ -136,9 +135,11 @@ unique_ptr<BaseStatistics> StandardColumnData::GetUpdateStatistics() {
 		return nullptr;
 	}
 	if (!stats) {
-		stats = BaseStatistics::CreateEmpty(type, StatisticsType::GLOBAL_STATS);
+		stats = BaseStatistics::CreateEmpty(type).ToUnique();
+	}
+	if (validity_stats) {
+		stats->Merge(*validity_stats);
 	}
-	stats->validity_stats = std::move(validity_stats);
 	return stats;
 }
 
@@ -169,7 +170,6 @@ struct StandardColumnCheckpointState : public ColumnCheckpointState {
 public:
 	unique_ptr<BaseStatistics> GetStatistics() override {
 		D_ASSERT(global_stats);
-		global_stats->validity_stats = validity_state->GetStatistics();
 		return std::move(global_stats);
 	}
 

package/src/duckdb/src/storage/table/struct_column_data.cpp

@@ -1,5 +1,5 @@
 #include "duckdb/storage/table/struct_column_data.hpp"
-#include "duckdb/storage/statistics/struct_statistics.hpp"
+#include "duckdb/storage/statistics/struct_stats.hpp"
 #include "duckdb/transaction/transaction.hpp"
 
 namespace duckdb {
@@ -127,12 +127,12 @@ void StructColumnData::Append(BaseStatistics &stats, ColumnAppendState &state, V
 	vector.Flatten(count);
 
 	// append the null values
-	validity.Append(*stats.validity_stats, state.child_appends[0], vector, count);
+	validity.Append(stats, state.child_appends[0], vector, count);
 
-	auto &struct_stats = (StructStatistics &)stats;
 	auto &child_entries = StructVector::GetEntries(vector);
 	for (idx_t i = 0; i < child_entries.size(); i++) {
-		sub_columns[i]->Append(*struct_stats.child_stats[i], state.child_appends[i + 1], *child_entries[i], count);
+		sub_columns[i]->Append(StructStats::GetChildStats(stats, i), state.child_appends[i + 1], *child_entries[i],
+		                       count);
 	}
 }
 
@@ -190,16 +190,18 @@ void StructColumnData::UpdateColumn(TransactionData transaction, const vector<co
 
 unique_ptr<BaseStatistics> StructColumnData::GetUpdateStatistics() {
 	// check if any child column has updates
-	auto stats = BaseStatistics::CreateEmpty(type, StatisticsType::GLOBAL_STATS);
-	auto &struct_stats = (StructStatistics &)*stats;
-	stats->validity_stats = validity.GetUpdateStatistics();
+	auto stats = BaseStatistics::CreateEmpty(type);
+	auto validity_stats = validity.GetUpdateStatistics();
+	if (validity_stats) {
+		stats.Merge(*validity_stats);
+	}
 	for (idx_t i = 0; i < sub_columns.size(); i++) {
 		auto child_stats = sub_columns[i]->GetUpdateStatistics();
 		if (child_stats) {
-			struct_stats.child_stats[i] = std::move(child_stats);
+			StructStats::SetChildStats(stats, i, std::move(child_stats));
 		}
 	}
-	return stats;
+	return stats.ToUnique();
 }
 
 void StructColumnData::FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result,
@@ -230,7 +232,7 @@ struct StructColumnCheckpointState : public ColumnCheckpointState {
 	StructColumnCheckpointState(RowGroup &row_group, ColumnData &column_data,
 	                            PartialBlockManager &partial_block_manager)
 	    : ColumnCheckpointState(row_group, column_data, partial_block_manager) {
-		global_stats = make_unique<StructStatistics>(column_data.type);
+		global_stats = StructStats::CreateEmpty(column_data.type).ToUnique();
 	}
 
 	unique_ptr<ColumnCheckpointState> validity_state;
@@ -238,14 +240,11 @@ struct StructColumnCheckpointState : public ColumnCheckpointState {
 
 public:
 	unique_ptr<BaseStatistics> GetStatistics() override {
-		auto stats = make_unique<StructStatistics>(column_data.type);
-		D_ASSERT(stats->child_stats.size() == child_states.size());
-		stats->validity_stats = validity_state->GetStatistics();
+		auto stats = StructStats::CreateEmpty(column_data.type);
 		for (idx_t i = 0; i < child_states.size(); i++) {
-			stats->child_stats[i] = child_states[i]->GetStatistics();
-			D_ASSERT(stats->child_stats[i]);
+			StructStats::SetChildStats(stats, i, child_states[i]->GetStatistics());
 		}
-		return std::move(stats);
+		return stats.ToUnique();
 	}
 
 	void WriteDataPointers(RowGroupWriter &writer) override {