duckdb 0.7.2-dev12.0 → 0.7.2-dev1238.0

package/src/duckdb/src/storage/statistics/string_stats.cpp

@@ -0,0 +1,273 @@
+#include "duckdb/storage/statistics/string_stats.hpp"
+#include "duckdb/storage/statistics/base_statistics.hpp"
+#include "duckdb/common/field_writer.hpp"
+#include "utf8proc_wrapper.hpp"
+#include "duckdb/common/string_util.hpp"
+#include "duckdb/common/types/vector.hpp"
+#include "duckdb/main/error_manager.hpp"
+
+namespace duckdb {
+
+BaseStatistics StringStats::CreateUnknown(LogicalType type) {
+	BaseStatistics result(std::move(type));
+	result.InitializeUnknown();
+	auto &string_data = StringStats::GetDataUnsafe(result);
+	for (idx_t i = 0; i < StringStatsData::MAX_STRING_MINMAX_SIZE; i++) {
+		string_data.min[i] = 0;
+		string_data.max[i] = 0xFF;
+	}
+	string_data.max_string_length = 0;
+	string_data.has_max_string_length = false;
+	string_data.has_unicode = true;
+	return result;
+}
+
+BaseStatistics StringStats::CreateEmpty(LogicalType type) {
+	BaseStatistics result(std::move(type));
+	result.InitializeEmpty();
+	auto &string_data = StringStats::GetDataUnsafe(result);
+	for (idx_t i = 0; i < StringStatsData::MAX_STRING_MINMAX_SIZE; i++) {
+		string_data.min[i] = 0xFF;
+		string_data.max[i] = 0;
+	}
+	string_data.max_string_length = 0;
+	string_data.has_max_string_length = true;
+	string_data.has_unicode = false;
+	return result;
+}
+
+StringStatsData &StringStats::GetDataUnsafe(BaseStatistics &stats) {
+	D_ASSERT(stats.GetStatsType() == StatisticsType::STRING_STATS);
+	return stats.stats_union.string_data;
+}
+
+const StringStatsData &StringStats::GetDataUnsafe(const BaseStatistics &stats) {
+	D_ASSERT(stats.GetStatsType() == StatisticsType::STRING_STATS);
+	return stats.stats_union.string_data;
+}
+
+bool StringStats::HasMaxStringLength(const BaseStatistics &stats) {
+	if (stats.GetType().id() == LogicalTypeId::SQLNULL) {
+		return false;
+	}
+	return StringStats::GetDataUnsafe(stats).has_max_string_length;
+}
+
+uint32_t StringStats::MaxStringLength(const BaseStatistics &stats) {
+	if (!HasMaxStringLength(stats)) {
+		throw InternalException("MaxStringLength called on statistics that does not have a max string length");
+	}
+	return StringStats::GetDataUnsafe(stats).max_string_length;
+}
+
+bool StringStats::CanContainUnicode(const BaseStatistics &stats) {
+	if (stats.GetType().id() == LogicalTypeId::SQLNULL) {
+		return true;
+	}
+	return StringStats::GetDataUnsafe(stats).has_unicode;
+}
+
+void StringStats::ResetMaxStringLength(BaseStatistics &stats) {
+	StringStats::GetDataUnsafe(stats).has_max_string_length = false;
+}
+
+void StringStats::SetContainsUnicode(BaseStatistics &stats) {
+	StringStats::GetDataUnsafe(stats).has_unicode = true;
+}
+
+void StringStats::Serialize(const BaseStatistics &stats, FieldWriter &writer) {
+	auto &string_data = StringStats::GetDataUnsafe(stats);
+	writer.WriteBlob(string_data.min, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	writer.WriteBlob(string_data.max, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	writer.WriteField<bool>(string_data.has_unicode);
+	writer.WriteField<bool>(string_data.has_max_string_length);
+	writer.WriteField<uint32_t>(string_data.max_string_length);
+}
+
+BaseStatistics StringStats::Deserialize(FieldReader &reader, LogicalType type) {
+	BaseStatistics result(std::move(type));
+	auto &string_data = StringStats::GetDataUnsafe(result);
+	reader.ReadBlob(string_data.min, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	reader.ReadBlob(string_data.max, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	string_data.has_unicode = reader.ReadRequired<bool>();
+	string_data.has_max_string_length = reader.ReadRequired<bool>();
+	string_data.max_string_length = reader.ReadRequired<uint32_t>();
+	return result;
+}
+
+static int StringValueComparison(const_data_ptr_t data, idx_t len, const_data_ptr_t comparison) {
+	D_ASSERT(len <= StringStatsData::MAX_STRING_MINMAX_SIZE);
+	for (idx_t i = 0; i < len; i++) {
+		if (data[i] < comparison[i]) {
+			return -1;
+		} else if (data[i] > comparison[i]) {
+			return 1;
+		}
+	}
+	return 0;
+}
+
+static void ConstructValue(const_data_ptr_t data, idx_t size, data_t target[]) {
+	idx_t value_size = size > StringStatsData::MAX_STRING_MINMAX_SIZE ? StringStatsData::MAX_STRING_MINMAX_SIZE : size;
+	memcpy(target, data, value_size);
+	for (idx_t i = value_size; i < StringStatsData::MAX_STRING_MINMAX_SIZE; i++) {
+		target[i] = '\0';
+	}
+}
+
+void StringStats::Update(BaseStatistics &stats, const string_t &value) {
+	auto data = (const_data_ptr_t)value.GetDataUnsafe();
+	auto size = value.GetSize();
+
+	//! we can only fit 8 bytes, so we might need to trim our string
+	// construct the value
+	data_t target[StringStatsData::MAX_STRING_MINMAX_SIZE];
+	ConstructValue(data, size, target);
+
+	// update the min and max
+	auto &string_data = StringStats::GetDataUnsafe(stats);
+	if (StringValueComparison(target, StringStatsData::MAX_STRING_MINMAX_SIZE, string_data.min) < 0) {
+		memcpy(string_data.min, target, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	}
+	if (StringValueComparison(target, StringStatsData::MAX_STRING_MINMAX_SIZE, string_data.max) > 0) {
+		memcpy(string_data.max, target, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	}
+	if (size > string_data.max_string_length) {
+		string_data.max_string_length = size;
+	}
+	if (stats.GetType().id() == LogicalTypeId::VARCHAR && !string_data.has_unicode) {
+		auto unicode = Utf8Proc::Analyze((const char *)data, size);
+		if (unicode == UnicodeType::UNICODE) {
+			string_data.has_unicode = true;
+		} else if (unicode == UnicodeType::INVALID) {
+			throw InternalException(
+			    ErrorManager::InvalidUnicodeError(string((char *)data, size), "segment statistics update"));
+		}
+	}
+}
+
+void StringStats::Merge(BaseStatistics &stats, const BaseStatistics &other) {
+	if (other.GetType().id() == LogicalTypeId::VALIDITY) {
+		return;
+	}
+	auto &string_data = StringStats::GetDataUnsafe(stats);
+	auto &other_data = StringStats::GetDataUnsafe(other);
+	if (StringValueComparison(other_data.min, StringStatsData::MAX_STRING_MINMAX_SIZE, string_data.min) < 0) {
+		memcpy(string_data.min, other_data.min, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	}
+	if (StringValueComparison(other_data.max, StringStatsData::MAX_STRING_MINMAX_SIZE, string_data.max) > 0) {
+		memcpy(string_data.max, other_data.max, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	}
+	string_data.has_unicode = string_data.has_unicode || other_data.has_unicode;
+	string_data.has_max_string_length = string_data.has_max_string_length && other_data.has_max_string_length;
+	string_data.max_string_length = MaxValue<uint32_t>(string_data.max_string_length, other_data.max_string_length);
+}
+
+FilterPropagateResult StringStats::CheckZonemap(const BaseStatistics &stats, ExpressionType comparison_type,
+                                                const string &constant) {
+	auto &string_data = StringStats::GetDataUnsafe(stats);
+	auto data = (const_data_ptr_t)constant.c_str();
+	auto size = constant.size();
+
+	idx_t value_size = size > StringStatsData::MAX_STRING_MINMAX_SIZE ? StringStatsData::MAX_STRING_MINMAX_SIZE : size;
+	int min_comp = StringValueComparison(data, value_size, string_data.min);
+	int max_comp = StringValueComparison(data, value_size, string_data.max);
+	switch (comparison_type) {
+	case ExpressionType::COMPARE_EQUAL:
+		if (min_comp >= 0 && max_comp <= 0) {
+			return FilterPropagateResult::NO_PRUNING_POSSIBLE;
+		} else {
+			return FilterPropagateResult::FILTER_ALWAYS_FALSE;
+		}
+	case ExpressionType::COMPARE_NOTEQUAL:
+		if (min_comp < 0 || max_comp > 0) {
+			return FilterPropagateResult::FILTER_ALWAYS_TRUE;
+		}
+		return FilterPropagateResult::NO_PRUNING_POSSIBLE;
+	case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
+	case ExpressionType::COMPARE_GREATERTHAN:
+		if (max_comp <= 0) {
+			return FilterPropagateResult::NO_PRUNING_POSSIBLE;
+		} else {
+			return FilterPropagateResult::FILTER_ALWAYS_FALSE;
+		}
+	case ExpressionType::COMPARE_LESSTHAN:
+	case ExpressionType::COMPARE_LESSTHANOREQUALTO:
+		if (min_comp >= 0) {
+			return FilterPropagateResult::NO_PRUNING_POSSIBLE;
+		} else {
+			return FilterPropagateResult::FILTER_ALWAYS_FALSE;
+		}
+	default:
+		throw InternalException("Expression type not implemented for string statistics zone map");
+	}
+}
+
+static idx_t GetValidMinMaxSubstring(const_data_ptr_t data) {
+	for (idx_t i = 0; i < StringStatsData::MAX_STRING_MINMAX_SIZE; i++) {
+		if (data[i] == '\0') {
+			return i;
+		}
+		if ((data[i] & 0x80) != 0) {
+			return i;
+		}
+	}
+	return StringStatsData::MAX_STRING_MINMAX_SIZE;
+}
+
+string StringStats::ToString(const BaseStatistics &stats) {
+	auto &string_data = StringStats::GetDataUnsafe(stats);
+	idx_t min_len = GetValidMinMaxSubstring(string_data.min);
+	idx_t max_len = GetValidMinMaxSubstring(string_data.max);
+	return StringUtil::Format(
+	    "[Min: %s, Max: %s, Has Unicode: %s, Max String Length: %s]", string((const char *)string_data.min, min_len),
+	    string((const char *)string_data.max, max_len), string_data.has_unicode ? "true" : "false",
+	    string_data.has_max_string_length ? to_string(string_data.max_string_length) : "?");
+}
+
+void StringStats::Verify(const BaseStatistics &stats, Vector &vector, const SelectionVector &sel, idx_t count) {
+	auto &string_data = StringStats::GetDataUnsafe(stats);
+
+	UnifiedVectorFormat vdata;
+	vector.ToUnifiedFormat(count, vdata);
+	auto data = (string_t *)vdata.data;
+	for (idx_t i = 0; i < count; i++) {
+		auto idx = sel.get_index(i);
+		auto index = vdata.sel->get_index(idx);
+		if (!vdata.validity.RowIsValid(index)) {
+			continue;
+		}
+		auto value = data[index];
+		auto data = value.GetDataUnsafe();
+		auto len = value.GetSize();
+		// LCOV_EXCL_START
+		if (string_data.has_max_string_length && len > string_data.max_string_length) {
+			throw InternalException(
+			    "Statistics mismatch: string value exceeds maximum string length.\nStatistics: %s\nVector: %s",
+			    stats.ToString(), vector.ToString(count));
+		}
+		if (stats.GetType().id() == LogicalTypeId::VARCHAR && !string_data.has_unicode) {
+			auto unicode = Utf8Proc::Analyze(data, len);
+			if (unicode == UnicodeType::UNICODE) {
+				throw InternalException("Statistics mismatch: string value contains unicode, but statistics says it "
+				                        "shouldn't.\nStatistics: %s\nVector: %s",
+				                        stats.ToString(), vector.ToString(count));
+			} else if (unicode == UnicodeType::INVALID) {
+				throw InternalException("Invalid unicode detected in vector: %s", vector.ToString(count));
+			}
+		}
+		if (StringValueComparison((const_data_ptr_t)data, MinValue<idx_t>(len, StringStatsData::MAX_STRING_MINMAX_SIZE),
+		                          string_data.min) < 0) {
+			throw InternalException("Statistics mismatch: value is smaller than min.\nStatistics: %s\nVector: %s",
+			                        stats.ToString(), vector.ToString(count));
+		}
+		if (StringValueComparison((const_data_ptr_t)data, MinValue<idx_t>(len, StringStatsData::MAX_STRING_MINMAX_SIZE),
+		                          string_data.max) > 0) {
+			throw InternalException("Statistics mismatch: value is bigger than max.\nStatistics: %s\nVector: %s",
+			                        stats.ToString(), vector.ToString(count));
+		}
+		// LCOV_EXCL_STOP
+	}
+}
+
+} // namespace duckdb

package/src/duckdb/src/storage/statistics/struct_stats.cpp

@@ -0,0 +1,133 @@
+#include "duckdb/storage/statistics/struct_stats.hpp"
+#include "duckdb/storage/statistics/base_statistics.hpp"
+#include "duckdb/common/field_writer.hpp"
+#include "duckdb/common/types/vector.hpp"
+
+namespace duckdb {
+
+void StructStats::Construct(BaseStatistics &stats) {
+	auto &child_types = StructType::GetChildTypes(stats.GetType());
+	stats.child_stats = unique_ptr<BaseStatistics[]>(new BaseStatistics[child_types.size()]);
+	for (idx_t i = 0; i < child_types.size(); i++) {
+		BaseStatistics::Construct(stats.child_stats[i], child_types[i].second);
+	}
+}
+
+BaseStatistics StructStats::CreateUnknown(LogicalType type) {
+	auto &child_types = StructType::GetChildTypes(type);
+	BaseStatistics result(std::move(type));
+	result.InitializeUnknown();
+	for (idx_t i = 0; i < child_types.size(); i++) {
+		result.child_stats[i].Copy(BaseStatistics::CreateUnknown(child_types[i].second));
+	}
+	return result;
+}
+
+BaseStatistics StructStats::CreateEmpty(LogicalType type) {
+	auto &child_types = StructType::GetChildTypes(type);
+	BaseStatistics result(std::move(type));
+	result.InitializeEmpty();
+	for (idx_t i = 0; i < child_types.size(); i++) {
+		result.child_stats[i].Copy(BaseStatistics::CreateEmpty(child_types[i].second));
+	}
+	return result;
+}
+
+const BaseStatistics *StructStats::GetChildStats(const BaseStatistics &stats) {
+	if (stats.GetStatsType() != StatisticsType::STRUCT_STATS) {
+		throw InternalException("Calling StructStats::GetChildStats on stats that is not a struct");
+	}
+	return stats.child_stats.get();
+}
+
+const BaseStatistics &StructStats::GetChildStats(const BaseStatistics &stats, idx_t i) {
+	D_ASSERT(stats.GetStatsType() == StatisticsType::STRUCT_STATS);
+	if (i >= StructType::GetChildCount(stats.GetType())) {
+		throw InternalException("Calling StructStats::GetChildStats but there are no stats for this index");
+	}
+	return stats.child_stats[i];
+}
+
+BaseStatistics &StructStats::GetChildStats(BaseStatistics &stats, idx_t i) {
+	D_ASSERT(stats.GetStatsType() == StatisticsType::STRUCT_STATS);
+	if (i >= StructType::GetChildCount(stats.GetType())) {
+		throw InternalException("Calling StructStats::GetChildStats but there are no stats for this index");
+	}
+	return stats.child_stats[i];
+}
+
+void StructStats::SetChildStats(BaseStatistics &stats, idx_t i, const BaseStatistics &new_stats) {
+	D_ASSERT(stats.GetStatsType() == StatisticsType::STRUCT_STATS);
+	D_ASSERT(i < StructType::GetChildCount(stats.GetType()));
+	stats.child_stats[i].Copy(new_stats);
+}
+
+void StructStats::SetChildStats(BaseStatistics &stats, idx_t i, unique_ptr<BaseStatistics> new_stats) {
+	D_ASSERT(stats.GetStatsType() == StatisticsType::STRUCT_STATS);
+	if (!new_stats) {
+		StructStats::SetChildStats(stats, i,
+		                           BaseStatistics::CreateUnknown(StructType::GetChildType(stats.GetType(), i)));
+	} else {
+		StructStats::SetChildStats(stats, i, *new_stats);
+	}
+}
+
+void StructStats::Copy(BaseStatistics &stats, const BaseStatistics &other) {
+	auto count = StructType::GetChildCount(stats.GetType());
+	for (idx_t i = 0; i < count; i++) {
+		stats.child_stats[i].Copy(other.child_stats[i]);
+	}
+}
+
+void StructStats::Merge(BaseStatistics &stats, const BaseStatistics &other) {
+	if (other.GetType().id() == LogicalTypeId::VALIDITY) {
+		return;
+	}
+	D_ASSERT(stats.GetType() == other.GetType());
+	auto child_count = StructType::GetChildCount(stats.GetType());
+	for (idx_t i = 0; i < child_count; i++) {
+		stats.child_stats[i].Merge(other.child_stats[i]);
+	}
+}
+
+void StructStats::Serialize(const BaseStatistics &stats, FieldWriter &writer) {
+	auto child_stats = StructStats::GetChildStats(stats);
+	auto child_count = StructType::GetChildCount(stats.GetType());
+	for (idx_t i = 0; i < child_count; i++) {
+		writer.WriteSerializable(child_stats[i]);
+	}
+}
+
+BaseStatistics StructStats::Deserialize(FieldReader &reader, LogicalType type) {
+	D_ASSERT(type.InternalType() == PhysicalType::STRUCT);
+	auto &child_types = StructType::GetChildTypes(type);
+	BaseStatistics result(std::move(type));
+	for (idx_t i = 0; i < child_types.size(); i++) {
+		result.child_stats[i].Copy(
+		    reader.ReadRequiredSerializable<BaseStatistics, BaseStatistics>(child_types[i].second));
+	}
+	return result;
+}
+
+string StructStats::ToString(const BaseStatistics &stats) {
+	string result;
+	result += " {";
+	auto &child_types = StructType::GetChildTypes(stats.GetType());
+	for (idx_t i = 0; i < child_types.size(); i++) {
+		if (i > 0) {
+			result += ", ";
+		}
+		result += child_types[i].first + ": " + stats.child_stats[i].ToString();
+	}
+	result += "}";
+	return result;
+}
+
+void StructStats::Verify(const BaseStatistics &stats, Vector &vector, const SelectionVector &sel, idx_t count) {
+	auto &child_entries = StructVector::GetEntries(vector);
+	for (idx_t i = 0; i < child_entries.size(); i++) {
+		stats.child_stats[i].Verify(*child_entries[i], sel, count);
+	}
+}
+
+} // namespace duckdb

package/src/duckdb/src/storage/storage_info.cpp

@@ -2,14 +2,14 @@
 
 namespace duckdb {
 
-const uint64_t VERSION_NUMBER = 43;
+const uint64_t VERSION_NUMBER = 49;
 
 struct StorageVersionInfo {
 	const char *version_name;
 	idx_t storage_version;
 };
 
-static StorageVersionInfo storage_version_info[] = {{"v0.7.0", 43},
+static StorageVersionInfo storage_version_info[] = {{"v0.7.0 or v0.7.1", 43},
                                                     {"v0.6.0 or v0.6.1", 39},
                                                     {"v0.5.0 or v0.5.1", 38},
                                                     {"v0.3.3, v0.3.4 or v0.4.0", 33},

package/src/duckdb/src/storage/table/column_checkpoint_state.cpp

@@ -1,12 +1,7 @@
-
 #include "duckdb/storage/table/column_data.hpp"
-
+#include "duckdb/storage/table/column_checkpoint_state.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/storage/checkpoint/write_overflow_strings_to_disk.hpp"
-#include "duckdb/storage/table/validity_column_data.hpp"
-#include "duckdb/storage/table/standard_column_data.hpp"
-#include "duckdb/storage/table/list_column_data.hpp"
-#include "duckdb/transaction/transaction.hpp"
 #include "duckdb/storage/table/row_group.hpp"
 #include "duckdb/storage/checkpoint/table_data_writer.hpp"
 
@@ -96,7 +91,7 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 	} // LCOV_EXCL_STOP
 
 	// merge the segment stats into the global stats
-	global_stats->Merge(*segment->stats.statistics);
+	global_stats->Merge(segment->stats.statistics);
 
 	// get the buffer of the segment and pin it
 	auto &db = column_data.GetDatabase();
@@ -104,7 +99,7 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 	block_id_t block_id = INVALID_BLOCK;
 	uint32_t offset_in_block = 0;
 
-	if (!segment->stats.statistics->IsConstant()) {
+	if (!segment->stats.statistics.IsConstant()) {
 		// non-constant block
 		PartialBlockAllocation allocation = partial_block_manager.GetBlockAllocation(segment_size);
 		block_id = allocation.state.block_id;
@@ -145,7 +140,7 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 	}
 
 	// construct the data pointer
-	DataPointer data_pointer;
+	DataPointer data_pointer(segment->stats.statistics.Copy());
 	data_pointer.block_pointer.block_id = block_id;
 	data_pointer.block_pointer.offset = offset_in_block;
 	data_pointer.row_start = row_group.start;
@@ -155,7 +150,6 @@ void ColumnCheckpointState::FlushSegment(unique_ptr<ColumnSegment> segment, idx_
 	}
 	data_pointer.tuple_count = tuple_count;
 	data_pointer.compression_type = segment->function->type;
-	data_pointer.statistics = segment->stats.statistics->Copy();
 
 	// append the segment to the new segment tree
 	new_tree.AppendSegment(std::move(segment));