duckdb 0.7.2-dev225.0 → 0.7.2-dev314.0

package/src/duckdb/src/storage/statistics/string_stats.cpp

@@ -0,0 +1,273 @@
+#include "duckdb/storage/statistics/string_stats.hpp"
+#include "duckdb/storage/statistics/base_statistics.hpp"
+#include "duckdb/common/field_writer.hpp"
+#include "utf8proc_wrapper.hpp"
+#include "duckdb/common/string_util.hpp"
+#include "duckdb/common/types/vector.hpp"
+#include "duckdb/main/error_manager.hpp"
+
+namespace duckdb {
+
+BaseStatistics StringStats::CreateUnknown(LogicalType type) {
+	BaseStatistics result(std::move(type));
+	result.InitializeUnknown();
+	auto &string_data = StringStats::GetDataUnsafe(result);
+	for (idx_t i = 0; i < StringStatsData::MAX_STRING_MINMAX_SIZE; i++) {
+		string_data.min[i] = 0;
+		string_data.max[i] = 0xFF;
+	}
+	string_data.max_string_length = 0;
+	string_data.has_max_string_length = false;
+	string_data.has_unicode = true;
+	return result;
+}
+
+BaseStatistics StringStats::CreateEmpty(LogicalType type) {
+	BaseStatistics result(std::move(type));
+	result.InitializeEmpty();
+	auto &string_data = StringStats::GetDataUnsafe(result);
+	for (idx_t i = 0; i < StringStatsData::MAX_STRING_MINMAX_SIZE; i++) {
+		string_data.min[i] = 0xFF;
+		string_data.max[i] = 0;
+	}
+	string_data.max_string_length = 0;
+	string_data.has_max_string_length = true;
+	string_data.has_unicode = false;
+	return result;
+}
+
+StringStatsData &StringStats::GetDataUnsafe(BaseStatistics &stats) {
+	D_ASSERT(stats.GetStatsType() == StatisticsType::STRING_STATS);
+	return stats.stats_union.string_data;
+}
+
+const StringStatsData &StringStats::GetDataUnsafe(const BaseStatistics &stats) {
+	D_ASSERT(stats.GetStatsType() == StatisticsType::STRING_STATS);
+	return stats.stats_union.string_data;
+}
+
+bool StringStats::HasMaxStringLength(const BaseStatistics &stats) {
+	if (stats.GetType().id() == LogicalTypeId::SQLNULL) {
+		return false;
+	}
+	return StringStats::GetDataUnsafe(stats).has_max_string_length;
+}
+
+uint32_t StringStats::MaxStringLength(const BaseStatistics &stats) {
+	if (!HasMaxStringLength(stats)) {
+		throw InternalException("MaxStringLength called on statistics that does not have a max string length");
+	}
+	return StringStats::GetDataUnsafe(stats).max_string_length;
+}
+
+bool StringStats::CanContainUnicode(const BaseStatistics &stats) {
+	if (stats.GetType().id() == LogicalTypeId::SQLNULL) {
+		return true;
+	}
+	return StringStats::GetDataUnsafe(stats).has_unicode;
+}
+
+void StringStats::ResetMaxStringLength(BaseStatistics &stats) {
+	StringStats::GetDataUnsafe(stats).has_max_string_length = false;
+}
+
+void StringStats::SetContainsUnicode(BaseStatistics &stats) {
+	StringStats::GetDataUnsafe(stats).has_unicode = true;
+}
+
+void StringStats::Serialize(const BaseStatistics &stats, FieldWriter &writer) {
+	auto &string_data = StringStats::GetDataUnsafe(stats);
+	writer.WriteBlob(string_data.min, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	writer.WriteBlob(string_data.max, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	writer.WriteField<bool>(string_data.has_unicode);
+	writer.WriteField<bool>(string_data.has_max_string_length);
+	writer.WriteField<uint32_t>(string_data.max_string_length);
+}
+
+BaseStatistics StringStats::Deserialize(FieldReader &reader, LogicalType type) {
+	BaseStatistics result(std::move(type));
+	auto &string_data = StringStats::GetDataUnsafe(result);
+	reader.ReadBlob(string_data.min, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	reader.ReadBlob(string_data.max, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	string_data.has_unicode = reader.ReadRequired<bool>();
+	string_data.has_max_string_length = reader.ReadRequired<bool>();
+	string_data.max_string_length = reader.ReadRequired<uint32_t>();
+	return result;
+}
+
+static int StringValueComparison(const_data_ptr_t data, idx_t len, const_data_ptr_t comparison) {
+	D_ASSERT(len <= StringStatsData::MAX_STRING_MINMAX_SIZE);
+	for (idx_t i = 0; i < len; i++) {
+		if (data[i] < comparison[i]) {
+			return -1;
+		} else if (data[i] > comparison[i]) {
+			return 1;
+		}
+	}
+	return 0;
+}
+
+static void ConstructValue(const_data_ptr_t data, idx_t size, data_t target[]) {
+	idx_t value_size = size > StringStatsData::MAX_STRING_MINMAX_SIZE ? StringStatsData::MAX_STRING_MINMAX_SIZE : size;
+	memcpy(target, data, value_size);
+	for (idx_t i = value_size; i < StringStatsData::MAX_STRING_MINMAX_SIZE; i++) {
+		target[i] = '\0';
+	}
+}
+
+void StringStats::Update(BaseStatistics &stats, const string_t &value) {
+	auto data = (const_data_ptr_t)value.GetDataUnsafe();
+	auto size = value.GetSize();
+
+	//! we can only fit 8 bytes, so we might need to trim our string
+	// construct the value
+	data_t target[StringStatsData::MAX_STRING_MINMAX_SIZE];
+	ConstructValue(data, size, target);
+
+	// update the min and max
+	auto &string_data = StringStats::GetDataUnsafe(stats);
+	if (StringValueComparison(target, StringStatsData::MAX_STRING_MINMAX_SIZE, string_data.min) < 0) {
+		memcpy(string_data.min, target, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	}
+	if (StringValueComparison(target, StringStatsData::MAX_STRING_MINMAX_SIZE, string_data.max) > 0) {
+		memcpy(string_data.max, target, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	}
+	if (size > string_data.max_string_length) {
+		string_data.max_string_length = size;
+	}
+	if (stats.GetType().id() == LogicalTypeId::VARCHAR && !string_data.has_unicode) {
+		auto unicode = Utf8Proc::Analyze((const char *)data, size);
+		if (unicode == UnicodeType::UNICODE) {
+			string_data.has_unicode = true;
+		} else if (unicode == UnicodeType::INVALID) {
+			throw InternalException(
+			    ErrorManager::InvalidUnicodeError(string((char *)data, size), "segment statistics update"));
+		}
+	}
+}
+
+void StringStats::Merge(BaseStatistics &stats, const BaseStatistics &other) {
+	if (other.GetType().id() == LogicalTypeId::VALIDITY) {
+		return;
+	}
+	auto &string_data = StringStats::GetDataUnsafe(stats);
+	auto &other_data = StringStats::GetDataUnsafe(other);
+	if (StringValueComparison(other_data.min, StringStatsData::MAX_STRING_MINMAX_SIZE, string_data.min) < 0) {
+		memcpy(string_data.min, other_data.min, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	}
+	if (StringValueComparison(other_data.max, StringStatsData::MAX_STRING_MINMAX_SIZE, string_data.max) > 0) {
+		memcpy(string_data.max, other_data.max, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	}
+	string_data.has_unicode = string_data.has_unicode || other_data.has_unicode;
+	string_data.has_max_string_length = string_data.has_max_string_length && other_data.has_max_string_length;
+	string_data.max_string_length = MaxValue<uint32_t>(string_data.max_string_length, other_data.max_string_length);
+}
+
+FilterPropagateResult StringStats::CheckZonemap(const BaseStatistics &stats, ExpressionType comparison_type,
+                                                const string &constant) {
+	auto &string_data = StringStats::GetDataUnsafe(stats);
+	auto data = (const_data_ptr_t)constant.c_str();
+	auto size = constant.size();
+
+	idx_t value_size = size > StringStatsData::MAX_STRING_MINMAX_SIZE ? StringStatsData::MAX_STRING_MINMAX_SIZE : size;
+	int min_comp = StringValueComparison(data, value_size, string_data.min);
+	int max_comp = StringValueComparison(data, value_size, string_data.max);
+	switch (comparison_type) {
+	case ExpressionType::COMPARE_EQUAL:
+		if (min_comp >= 0 && max_comp <= 0) {
+			return FilterPropagateResult::NO_PRUNING_POSSIBLE;
+		} else {
+			return FilterPropagateResult::FILTER_ALWAYS_FALSE;
+		}
+	case ExpressionType::COMPARE_NOTEQUAL:
+		if (min_comp < 0 || max_comp > 0) {
+			return FilterPropagateResult::FILTER_ALWAYS_TRUE;
+		}
+		return FilterPropagateResult::NO_PRUNING_POSSIBLE;
+	case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
+	case ExpressionType::COMPARE_GREATERTHAN:
+		if (max_comp <= 0) {
+			return FilterPropagateResult::NO_PRUNING_POSSIBLE;
+		} else {
+			return FilterPropagateResult::FILTER_ALWAYS_FALSE;
+		}
+	case ExpressionType::COMPARE_LESSTHAN:
+	case ExpressionType::COMPARE_LESSTHANOREQUALTO:
+		if (min_comp >= 0) {
+			return FilterPropagateResult::NO_PRUNING_POSSIBLE;
+		} else {
+			return FilterPropagateResult::FILTER_ALWAYS_FALSE;
+		}
+	default:
+		throw InternalException("Expression type not implemented for string statistics zone map");
+	}
+}
+
+static idx_t GetValidMinMaxSubstring(const_data_ptr_t data) {
+	for (idx_t i = 0; i < StringStatsData::MAX_STRING_MINMAX_SIZE; i++) {
+		if (data[i] == '\0') {
+			return i;
+		}
+		if ((data[i] & 0x80) != 0) {
+			return i;
+		}
+	}
+	return StringStatsData::MAX_STRING_MINMAX_SIZE;
+}
+
+string StringStats::ToString(const BaseStatistics &stats) {
+	auto &string_data = StringStats::GetDataUnsafe(stats);
+	idx_t min_len = GetValidMinMaxSubstring(string_data.min);
+	idx_t max_len = GetValidMinMaxSubstring(string_data.max);
+	return StringUtil::Format(
+	    "[Min: %s, Max: %s, Has Unicode: %s, Max String Length: %s]", string((const char *)string_data.min, min_len),
+	    string((const char *)string_data.max, max_len), string_data.has_unicode ? "true" : "false",
+	    string_data.has_max_string_length ? to_string(string_data.max_string_length) : "?");
+}
+
+void StringStats::Verify(const BaseStatistics &stats, Vector &vector, const SelectionVector &sel, idx_t count) {
+	auto &string_data = StringStats::GetDataUnsafe(stats);
+
+	UnifiedVectorFormat vdata;
+	vector.ToUnifiedFormat(count, vdata);
+	auto data = (string_t *)vdata.data;
+	for (idx_t i = 0; i < count; i++) {
+		auto idx = sel.get_index(i);
+		auto index = vdata.sel->get_index(idx);
+		if (!vdata.validity.RowIsValid(index)) {
+			continue;
+		}
+		auto value = data[index];
+		auto data = value.GetDataUnsafe();
+		auto len = value.GetSize();
+		// LCOV_EXCL_START
+		if (string_data.has_max_string_length && len > string_data.max_string_length) {
+			throw InternalException(
+			    "Statistics mismatch: string value exceeds maximum string length.\nStatistics: %s\nVector: %s",
+			    stats.ToString(), vector.ToString(count));
+		}
+		if (stats.GetType().id() == LogicalTypeId::VARCHAR && !string_data.has_unicode) {
+			auto unicode = Utf8Proc::Analyze(data, len);
+			if (unicode == UnicodeType::UNICODE) {
+				throw InternalException("Statistics mismatch: string value contains unicode, but statistics says it "
+				                        "shouldn't.\nStatistics: %s\nVector: %s",
+				                        stats.ToString(), vector.ToString(count));
+			} else if (unicode == UnicodeType::INVALID) {
+				throw InternalException("Invalid unicode detected in vector: %s", vector.ToString(count));
+			}
+		}
+		if (StringValueComparison((const_data_ptr_t)data, MinValue<idx_t>(len, StringStatsData::MAX_STRING_MINMAX_SIZE),
+		                          string_data.min) < 0) {
+			throw InternalException("Statistics mismatch: value is smaller than min.\nStatistics: %s\nVector: %s",
+			                        stats.ToString(), vector.ToString(count));
+		}
+		if (StringValueComparison((const_data_ptr_t)data, MinValue<idx_t>(len, StringStatsData::MAX_STRING_MINMAX_SIZE),
+		                          string_data.max) > 0) {
+			throw InternalException("Statistics mismatch: value is bigger than max.\nStatistics: %s\nVector: %s",
+			                        stats.ToString(), vector.ToString(count));
+		}
+		// LCOV_EXCL_STOP
+	}
+}
+
+} // namespace duckdb

package/src/duckdb/src/storage/statistics/struct_stats.cpp

@@ -0,0 +1,131 @@
+#include "duckdb/storage/statistics/struct_stats.hpp"
+#include "duckdb/storage/statistics/base_statistics.hpp"
+#include "duckdb/common/field_writer.hpp"
+#include "duckdb/common/types/vector.hpp"
+
+namespace duckdb {
+
+void StructStats::Construct(BaseStatistics &stats) {
+	auto &child_types = StructType::GetChildTypes(stats.GetType());
+	stats.child_stats = unique_ptr<BaseStatistics[]>(new BaseStatistics[child_types.size()]);
+	for (idx_t i = 0; i < child_types.size(); i++) {
+		BaseStatistics::Construct(stats.child_stats[i], child_types[i].second);
+	}
+}
+
+BaseStatistics StructStats::CreateUnknown(LogicalType type) {
+	auto &child_types = StructType::GetChildTypes(type);
+	BaseStatistics result(std::move(type));
+	result.InitializeUnknown();
+	for (idx_t i = 0; i < child_types.size(); i++) {
+		result.child_stats[i].Copy(BaseStatistics::CreateUnknown(child_types[i].second));
+	}
+	return result;
+}
+
+BaseStatistics StructStats::CreateEmpty(LogicalType type) {
+	auto &child_types = StructType::GetChildTypes(type);
+	BaseStatistics result(std::move(type));
+	result.InitializeEmpty();
+	for (idx_t i = 0; i < child_types.size(); i++) {
+		result.child_stats[i].Copy(BaseStatistics::CreateEmpty(child_types[i].second));
+	}
+	return result;
+}
+
+const BaseStatistics *StructStats::GetChildStats(const BaseStatistics &stats) {
+	D_ASSERT(stats.GetStatsType() == StatisticsType::STRUCT_STATS);
+	return stats.child_stats.get();
+}
+
+const BaseStatistics &StructStats::GetChildStats(const BaseStatistics &stats, idx_t i) {
+	D_ASSERT(stats.GetStatsType() == StatisticsType::STRUCT_STATS);
+	if (i >= StructType::GetChildCount(stats.GetType())) {
+		throw InternalException("Calling StructStats::GetChildStats but there are no stats for this index");
+	}
+	return stats.child_stats[i];
+}
+
+BaseStatistics &StructStats::GetChildStats(BaseStatistics &stats, idx_t i) {
+	D_ASSERT(stats.GetStatsType() == StatisticsType::STRUCT_STATS);
+	if (i >= StructType::GetChildCount(stats.GetType())) {
+		throw InternalException("Calling StructStats::GetChildStats but there are no stats for this index");
+	}
+	return stats.child_stats[i];
+}
+
+void StructStats::SetChildStats(BaseStatistics &stats, idx_t i, const BaseStatistics &new_stats) {
+	D_ASSERT(stats.GetStatsType() == StatisticsType::STRUCT_STATS);
+	D_ASSERT(i < StructType::GetChildCount(stats.GetType()));
+	stats.child_stats[i].Copy(new_stats);
+}
+
+void StructStats::SetChildStats(BaseStatistics &stats, idx_t i, unique_ptr<BaseStatistics> new_stats) {
+	D_ASSERT(stats.GetStatsType() == StatisticsType::STRUCT_STATS);
+	if (!new_stats) {
+		StructStats::SetChildStats(stats, i,
+		                           BaseStatistics::CreateUnknown(StructType::GetChildType(stats.GetType(), i)));
+	} else {
+		StructStats::SetChildStats(stats, i, *new_stats);
+	}
+}
+
+void StructStats::Copy(BaseStatistics &stats, const BaseStatistics &other) {
+	auto count = StructType::GetChildCount(stats.GetType());
+	for (idx_t i = 0; i < count; i++) {
+		stats.child_stats[i].Copy(other.child_stats[i]);
+	}
+}
+
+void StructStats::Merge(BaseStatistics &stats, const BaseStatistics &other) {
+	if (other.GetType().id() == LogicalTypeId::VALIDITY) {
+		return;
+	}
+	D_ASSERT(stats.GetType() == other.GetType());
+	auto child_count = StructType::GetChildCount(stats.GetType());
+	for (idx_t i = 0; i < child_count; i++) {
+		stats.child_stats[i].Merge(other.child_stats[i]);
+	}
+}
+
+void StructStats::Serialize(const BaseStatistics &stats, FieldWriter &writer) {
+	auto child_stats = StructStats::GetChildStats(stats);
+	auto child_count = StructType::GetChildCount(stats.GetType());
+	for (idx_t i = 0; i < child_count; i++) {
+		writer.WriteSerializable(child_stats[i]);
+	}
+}
+
+BaseStatistics StructStats::Deserialize(FieldReader &reader, LogicalType type) {
+	D_ASSERT(type.InternalType() == PhysicalType::STRUCT);
+	auto &child_types = StructType::GetChildTypes(type);
+	BaseStatistics result(std::move(type));
+	for (idx_t i = 0; i < child_types.size(); i++) {
+		result.child_stats[i].Copy(
+		    reader.ReadRequiredSerializable<BaseStatistics, BaseStatistics>(child_types[i].second));
+	}
+	return result;
+}
+
+string StructStats::ToString(const BaseStatistics &stats) {
+	string result;
+	result += " {";
+	auto &child_types = StructType::GetChildTypes(stats.GetType());
+	for (idx_t i = 0; i < child_types.size(); i++) {
+		if (i > 0) {
+			result += ", ";
+		}
+		result += child_types[i].first + ": " + stats.child_stats[i].ToString();
+	}
+	result += "}";
+	return result;
+}
+
+void StructStats::Verify(const BaseStatistics &stats, Vector &vector, const SelectionVector &sel, idx_t count) {
+	auto &child_entries = StructVector::GetEntries(vector);
+	for (idx_t i = 0; i < child_entries.size(); i++) {
+		stats.child_stats[i].Verify(*child_entries[i], sel, count);
+	}
+}
+
+} // namespace duckdb

package/src/duckdb/src/storage/storage_info.cpp

@@ -2,7 +2,7 @@
 
 namespace duckdb {
 
-const uint64_t VERSION_NUMBER = 44;
+const uint64_t VERSION_NUMBER = 45;
 
 struct StorageVersionInfo {
 	const char *version_name;

package/src/duckdb/src/storage/table/column_checkpoint_state.cpp

@@ -96,7 +96,7 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 	} // LCOV_EXCL_STOP
 
 	// merge the segment stats into the global stats
-	global_stats->Merge(*segment->stats.statistics);
+	global_stats->Merge(segment->stats.statistics);
 
 	// get the buffer of the segment and pin it
 	auto &db = column_data.GetDatabase();
@@ -104,7 +104,7 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 	block_id_t block_id = INVALID_BLOCK;
 	uint32_t offset_in_block = 0;
 
-	if (!segment->stats.statistics->IsConstant()) {
+	if (!segment->stats.statistics.IsConstant()) {
 		// non-constant block
 		PartialBlockAllocation allocation = partial_block_manager.GetBlockAllocation(segment_size);
 		block_id = allocation.state.block_id;
@@ -145,7 +145,7 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 	}
 
 	// construct the data pointer
-	DataPointer data_pointer;
+	DataPointer data_pointer(segment->stats.statistics.Copy());
 	data_pointer.block_pointer.block_id = block_id;
 	data_pointer.block_pointer.offset = offset_in_block;
 	data_pointer.row_start = row_group.start;
@@ -155,7 +155,6 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 	}
 	data_pointer.tuple_count = tuple_count;
 	data_pointer.compression_type = segment->function->type;
-	data_pointer.statistics = segment->stats.statistics->Copy();
 
 	// append the segment to the new segment tree
 	new_tree.AppendSegment(std::move(segment));

package/src/duckdb/src/storage/table/column_data.cpp

@@ -254,7 +254,7 @@ void ColumnData::AppendData(BaseStatistics &stats, ColumnAppendState &state, Uni
 	while (true) {
 		// append the data from the vector
 		idx_t copied_elements = state.current->Append(state, vdata, offset, count);
-		stats.Merge(*state.current->stats.statistics);
+		stats.Merge(state.current->stats.statistics);
 		if (copied_elements == count) {
 			// finished copying everything
 			break;
@@ -389,7 +389,7 @@ unique_ptr<ColumnCheckpointState> ColumnData::Checkpoint(RowGroup &row_group,
 	// scan the segments of the column data
 	// set up the checkpoint state
 	auto checkpoint_state = CreateCheckpointState(row_group, partial_block_manager);
-	checkpoint_state->global_stats = BaseStatistics::CreateEmpty(type, StatisticsType::LOCAL_STATS);
+	checkpoint_state->global_stats = BaseStatistics::CreateEmpty(type).ToUnique();
 
 	auto l = data.Lock();
 	auto nodes = data.MoveSegments(l);
@@ -414,13 +414,19 @@ void ColumnData::DeserializeColumn(Deserializer &source) {
 	idx_t data_pointer_count = source.Read<idx_t>();
 	for (idx_t data_ptr = 0; data_ptr < data_pointer_count; data_ptr++) {
 		// read the data pointer
-		DataPointer data_pointer;
-		data_pointer.row_start = source.Read<idx_t>();
-		data_pointer.tuple_count = source.Read<idx_t>();
-		data_pointer.block_pointer.block_id = source.Read<block_id_t>();
-		data_pointer.block_pointer.offset = source.Read<uint32_t>();
-		data_pointer.compression_type = source.Read<CompressionType>();
-		data_pointer.statistics = BaseStatistics::Deserialize(source, type);
+		auto row_start = source.Read<idx_t>();
+		auto tuple_count = source.Read<idx_t>();
+		auto block_pointer_block_id = source.Read<block_id_t>();
+		auto block_pointer_offset = source.Read<uint32_t>();
+		auto compression_type = source.Read<CompressionType>();
+		auto stats = BaseStatistics::Deserialize(source, type);
+
+		DataPointer data_pointer(std::move(stats));
+		data_pointer.row_start = row_start;
+		data_pointer.tuple_count = tuple_count;
+		data_pointer.block_pointer.block_id = block_pointer_block_id;
+		data_pointer.block_pointer.offset = block_pointer_offset;
+		data_pointer.compression_type = compression_type;
 
 		// create a persistent segment
 		auto segment = ColumnSegment::CreatePersistentSegment(
@@ -466,11 +472,7 @@ void ColumnData::GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, T
 		column_info.segment_start = segment->start;
 		column_info.segment_count = segment->count;
 		column_info.compression_type = CompressionTypeToString(segment->function->type);
-		if (!segment->stats.statistics || type.id() == LogicalTypeId::LIST) {
-			column_info.segment_stats = string("No Stats");
-		} else {
-			column_info.segment_stats = segment->stats.statistics->ToString();
-		}
+		column_info.segment_stats = segment->stats.statistics.ToString();
 		column_info.has_updates = updates ? true : false;
 		// persistent
 		// block_id

package/src/duckdb/src/storage/table/column_data_checkpointer.cpp

@@ -218,16 +218,15 @@ void ColumnDataCheckpointer::WritePersistentSegments() {
 		D_ASSERT(segment->segment_type == ColumnSegmentType::PERSISTENT);
 
 		// set up the data pointer directly using the data from the persistent segment
-		DataPointer pointer;
+		DataPointer pointer(segment->stats.statistics.Copy());
 		pointer.block_pointer.block_id = segment->GetBlockId();
 		pointer.block_pointer.offset = segment->GetBlockOffset();
 		pointer.row_start = segment->start;
 		pointer.tuple_count = segment->count;
 		pointer.compression_type = segment->function->type;
-		pointer.statistics = segment->stats.statistics->Copy();
 
 		// merge the persistent stats into the global column stats
-		state.global_stats->Merge(*segment->stats.statistics);
+		state.global_stats->Merge(segment->stats.statistics);
 
 		// directly append the current segment to the new tree
 		state.new_tree.AppendSegment(std::move(nodes[segment_idx].node));

package/src/duckdb/src/storage/table/column_segment.cpp

@@ -19,7 +19,7 @@ unique_ptr<ColumnSegment> ColumnSegment::CreatePersistentSegment(DatabaseInstanc
                                                                  block_id_t block_id, idx_t offset,
                                                                  const LogicalType &type, idx_t start, idx_t count,
                                                                  CompressionType compression_type,
-                                                                 unique_ptr<BaseStatistics> statistics) {
+                                                                 BaseStatistics statistics) {
 	auto &config = DBConfig::GetConfig(db);
 	CompressionFunction *function;
 	shared_ptr<BlockHandle> block;
@@ -48,7 +48,7 @@ unique_ptr<ColumnSegment> ColumnSegment::CreateTransientSegment(DatabaseInstance
 		buffer_manager.Allocate(segment_size, false, &block);
 	}
 	return make_unique<ColumnSegment>(db, std::move(block), type, ColumnSegmentType::TRANSIENT, start, 0, function,
-	                                  nullptr, INVALID_BLOCK, 0, segment_size);
+	                                  BaseStatistics::CreateEmpty(type), INVALID_BLOCK, 0, segment_size);
 }
 
 unique_ptr<ColumnSegment> ColumnSegment::CreateSegment(ColumnSegment &other, idx_t start) {
@@ -57,10 +57,9 @@ unique_ptr<ColumnSegment> ColumnSegment::CreateSegment(ColumnSegment &other, idx
 
 ColumnSegment::ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block, LogicalType type_p,
                              ColumnSegmentType segment_type, idx_t start, idx_t count, CompressionFunction *function_p,
-                             unique_ptr<BaseStatistics> statistics, block_id_t block_id_p, idx_t offset_p,
-                             idx_t segment_size_p)
+                             BaseStatistics statistics, block_id_t block_id_p, idx_t offset_p, idx_t segment_size_p)
     : SegmentBase(start, count), db(db), type(std::move(type_p)), type_size(GetTypeIdSize(type.InternalType())),
-      segment_type(segment_type), function(function_p), stats(type, std::move(statistics)), block(std::move(block)),
+      segment_type(segment_type), function(function_p), stats(std::move(statistics)), block(std::move(block)),
       block_id(block_id_p), offset(offset_p), segment_size(segment_size_p) {
 	D_ASSERT(function);
 	if (function->init_segment) {
@@ -181,13 +180,12 @@ void ColumnSegment::ConvertToPersistent(BlockManager *block_manager, block_id_t
 	block_id = block_id_p;
 	offset = 0;
 
-	D_ASSERT(stats.statistics);
 	if (block_id == INVALID_BLOCK) {
 		// constant block: reset the block buffer
-		D_ASSERT(stats.statistics->IsConstant());
+		D_ASSERT(stats.statistics.IsConstant());
 		block.reset();
 	} else {
-		D_ASSERT(!stats.statistics->IsConstant());
+		D_ASSERT(!stats.statistics.IsConstant());
 		// non-constant block: write the block to disk
 		// the data for the block already exists in-memory of our block
 		// instead of copying the data we alter some metadata so the buffer points to an on-disk block

package/src/duckdb/src/storage/table/list_column_data.cpp

@@ -1,5 +1,5 @@
 #include "duckdb/storage/table/list_column_data.hpp"
-#include "duckdb/storage/statistics/list_statistics.hpp"
+#include "duckdb/storage/statistics/list_stats.hpp"
 #include "duckdb/transaction/transaction.hpp"
 
 namespace duckdb {
@@ -162,10 +162,8 @@ void ListColumnData::InitializeAppend(ColumnAppendState &state) {
 	state.child_appends.push_back(std::move(child_append_state));
 }
 
-void ListColumnData::Append(BaseStatistics &stats_p, ColumnAppendState &state, Vector &vector, idx_t count) {
+void ListColumnData::Append(BaseStatistics &stats, ColumnAppendState &state, Vector &vector, idx_t count) {
 	D_ASSERT(count > 0);
-	auto &stats = (ListStatistics &)stats_p;
-
 	UnifiedVectorFormat list_data;
 	vector.ToUnifiedFormat(count, list_data);
 	auto &list_validity = list_data.validity;
@@ -220,10 +218,10 @@ void ListColumnData::Append(BaseStatistics &stats_p, ColumnAppendState &state, V
 	ColumnData::AppendData(stats, state, vdata, count);
 	// append the validity data
 	vdata.validity = append_mask;
-	validity.AppendData(*stats.validity_stats, state.child_appends[0], vdata, count);
+	validity.AppendData(stats, state.child_appends[0], vdata, count);
 	// append the child vector
 	if (child_count > 0) {
-		child_column->Append(*stats.child_stats, state.child_appends[1], child_vector, child_count);
+		child_column->Append(ListStats::GetChildStats(stats), state.child_appends[1], child_vector, child_count);
 	}
 }
 
@@ -308,7 +306,7 @@ void ListColumnData::CommitDropColumn() {
 struct ListColumnCheckpointState : public ColumnCheckpointState {
 	ListColumnCheckpointState(RowGroup &row_group, ColumnData &column_data, PartialBlockManager &partial_block_manager)
 	    : ColumnCheckpointState(row_group, column_data, partial_block_manager) {
-		global_stats = make_unique<ListStatistics>(column_data.type);
+		global_stats = ListStats::CreateEmpty(column_data.type).ToUnique();
 	}
 
 	unique_ptr<ColumnCheckpointState> validity_state;
@@ -317,10 +315,8 @@ struct ListColumnCheckpointState : public ColumnCheckpointState {
 public:
 	unique_ptr<BaseStatistics> GetStatistics() override {
 		auto stats = global_stats->Copy();
-		auto &list_stats = (ListStatistics &)*stats;
-		stats->validity_stats = validity_state->GetStatistics();
-		list_stats.child_stats = child_state->GetStatistics();
-		return stats;
+		ListStats::SetChildStats(stats, child_state->GetStatistics());
+		return stats.ToUnique();
 	}
 
 	void WriteDataPointers(RowGroupWriter &writer) override {