duckdb 0.7.2-dev225.0 → 0.7.2-dev314.0

package/src/duckdb/src/storage/statistics/string_statistics.cpp

@@ -1,220 +0,0 @@
-#include "duckdb/storage/statistics/string_statistics.hpp"
-#include "duckdb/common/field_writer.hpp"
-#include "utf8proc_wrapper.hpp"
-#include "duckdb/common/string_util.hpp"
-#include "duckdb/common/types/vector.hpp"
-#include "duckdb/main/error_manager.hpp"
-
-namespace duckdb {
-
-StringStatistics::StringStatistics(LogicalType type_p, StatisticsType stats_type)
-    : BaseStatistics(std::move(type_p), stats_type) {
-	InitializeBase();
-	for (idx_t i = 0; i < MAX_STRING_MINMAX_SIZE; i++) {
-		min[i] = 0xFF;
-		max[i] = 0;
-	}
-	max_string_length = 0;
-	has_unicode = false;
-	has_overflow_strings = false;
-}
-
-unique_ptr<BaseStatistics> StringStatistics::Copy() const {
-	auto result = make_unique<StringStatistics>(type, stats_type);
-	result->CopyBase(*this);
-
-	memcpy(result->min, min, MAX_STRING_MINMAX_SIZE);
-	memcpy(result->max, max, MAX_STRING_MINMAX_SIZE);
-	result->has_unicode = has_unicode;
-	result->max_string_length = max_string_length;
-	return std::move(result);
-}
-
-void StringStatistics::Serialize(FieldWriter &writer) const {
-	writer.WriteBlob(min, MAX_STRING_MINMAX_SIZE);
-	writer.WriteBlob(max, MAX_STRING_MINMAX_SIZE);
-	writer.WriteField<bool>(has_unicode);
-	writer.WriteField<uint32_t>(max_string_length);
-	writer.WriteField<bool>(has_overflow_strings);
-}
-
-unique_ptr<BaseStatistics> StringStatistics::Deserialize(FieldReader &reader, LogicalType type) {
-	auto stats = make_unique<StringStatistics>(std::move(type), StatisticsType::LOCAL_STATS);
-	reader.ReadBlob(stats->min, MAX_STRING_MINMAX_SIZE);
-	reader.ReadBlob(stats->max, MAX_STRING_MINMAX_SIZE);
-	stats->has_unicode = reader.ReadRequired<bool>();
-	stats->max_string_length = reader.ReadRequired<uint32_t>();
-	stats->has_overflow_strings = reader.ReadRequired<bool>();
-	return std::move(stats);
-}
-
-static int StringValueComparison(const_data_ptr_t data, idx_t len, const_data_ptr_t comparison) {
-	D_ASSERT(len <= StringStatistics::MAX_STRING_MINMAX_SIZE);
-	for (idx_t i = 0; i < len; i++) {
-		if (data[i] < comparison[i]) {
-			return -1;
-		} else if (data[i] > comparison[i]) {
-			return 1;
-		}
-	}
-	return 0;
-}
-
-static void ConstructValue(const_data_ptr_t data, idx_t size, data_t target[]) {
-	idx_t value_size =
-	    size > StringStatistics::MAX_STRING_MINMAX_SIZE ? StringStatistics::MAX_STRING_MINMAX_SIZE : size;
-	memcpy(target, data, value_size);
-	for (idx_t i = value_size; i < StringStatistics::MAX_STRING_MINMAX_SIZE; i++) {
-		target[i] = '\0';
-	}
-}
-
-void StringStatistics::Update(const string_t &value) {
-	auto data = (const_data_ptr_t)value.GetDataUnsafe();
-	auto size = value.GetSize();
-
-	//! we can only fit 8 bytes, so we might need to trim our string
-	// construct the value
-	data_t target[MAX_STRING_MINMAX_SIZE];
-	ConstructValue(data, size, target);
-
-	// update the min and max
-	if (StringValueComparison(target, MAX_STRING_MINMAX_SIZE, min) < 0) {
-		memcpy(min, target, MAX_STRING_MINMAX_SIZE);
-	}
-	if (StringValueComparison(target, MAX_STRING_MINMAX_SIZE, max) > 0) {
-		memcpy(max, target, MAX_STRING_MINMAX_SIZE);
-	}
-	if (size > max_string_length) {
-		max_string_length = size;
-	}
-	if (type.id() == LogicalTypeId::VARCHAR && !has_unicode) {
-		auto unicode = Utf8Proc::Analyze((const char *)data, size);
-		if (unicode == UnicodeType::UNICODE) {
-			has_unicode = true;
-		} else if (unicode == UnicodeType::INVALID) {
-			throw InternalException(
-			    ErrorManager::InvalidUnicodeError(string((char *)data, size), "segment statistics update"));
-		}
-	}
-}
-
-void StringStatistics::Merge(const BaseStatistics &other_p) {
-	BaseStatistics::Merge(other_p);
-	auto &other = (const StringStatistics &)other_p;
-	if (StringValueComparison(other.min, MAX_STRING_MINMAX_SIZE, min) < 0) {
-		memcpy(min, other.min, MAX_STRING_MINMAX_SIZE);
-	}
-	if (StringValueComparison(other.max, MAX_STRING_MINMAX_SIZE, max) > 0) {
-		memcpy(max, other.max, MAX_STRING_MINMAX_SIZE);
-	}
-	has_unicode = has_unicode || other.has_unicode;
-	max_string_length = MaxValue<uint32_t>(max_string_length, other.max_string_length);
-	has_overflow_strings = has_overflow_strings || other.has_overflow_strings;
-}
-
-FilterPropagateResult StringStatistics::CheckZonemap(ExpressionType comparison_type, const string &constant) const {
-	auto data = (const_data_ptr_t)constant.c_str();
-	auto size = constant.size();
-
-	idx_t value_size = size > MAX_STRING_MINMAX_SIZE ? MAX_STRING_MINMAX_SIZE : size;
-	int min_comp = StringValueComparison(data, value_size, min);
-	int max_comp = StringValueComparison(data, value_size, max);
-	switch (comparison_type) {
-	case ExpressionType::COMPARE_EQUAL:
-		if (min_comp >= 0 && max_comp <= 0) {
-			return FilterPropagateResult::NO_PRUNING_POSSIBLE;
-		} else {
-			return FilterPropagateResult::FILTER_ALWAYS_FALSE;
-		}
-	case ExpressionType::COMPARE_NOTEQUAL:
-		if (min_comp < 0 || max_comp > 0) {
-			return FilterPropagateResult::FILTER_ALWAYS_TRUE;
-		}
-		return FilterPropagateResult::NO_PRUNING_POSSIBLE;
-	case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
-	case ExpressionType::COMPARE_GREATERTHAN:
-		if (max_comp <= 0) {
-			return FilterPropagateResult::NO_PRUNING_POSSIBLE;
-		} else {
-			return FilterPropagateResult::FILTER_ALWAYS_FALSE;
-		}
-	case ExpressionType::COMPARE_LESSTHAN:
-	case ExpressionType::COMPARE_LESSTHANOREQUALTO:
-		if (min_comp >= 0) {
-			return FilterPropagateResult::NO_PRUNING_POSSIBLE;
-		} else {
-			return FilterPropagateResult::FILTER_ALWAYS_FALSE;
-		}
-	default:
-		throw InternalException("Expression type not implemented for string statistics zone map");
-	}
-}
-
-static idx_t GetValidMinMaxSubstring(const_data_ptr_t data) {
-	for (idx_t i = 0; i < StringStatistics::MAX_STRING_MINMAX_SIZE; i++) {
-		if (data[i] == '\0') {
-			return i;
-		}
-		if ((data[i] & 0x80) != 0) {
-			return i;
-		}
-	}
-	return StringStatistics::MAX_STRING_MINMAX_SIZE;
-}
-
-string StringStatistics::ToString() const {
-	idx_t min_len = GetValidMinMaxSubstring(min);
-	idx_t max_len = GetValidMinMaxSubstring(max);
-	return StringUtil::Format("[Min: %s, Max: %s, Has Unicode: %s, Max String Length: %lld]%s",
-	                          string((const char *)min, min_len), string((const char *)max, max_len),
-	                          has_unicode ? "true" : "false", max_string_length, BaseStatistics::ToString());
-}
-
-void StringStatistics::Verify(Vector &vector, const SelectionVector &sel, idx_t count) const {
-	BaseStatistics::Verify(vector, sel, count);
-
-	string_t min_string((const char *)min, MAX_STRING_MINMAX_SIZE);
-	string_t max_string((const char *)max, MAX_STRING_MINMAX_SIZE);
-
-	UnifiedVectorFormat vdata;
-	vector.ToUnifiedFormat(count, vdata);
-	auto data = (string_t *)vdata.data;
-	for (idx_t i = 0; i < count; i++) {
-		auto idx = sel.get_index(i);
-		auto index = vdata.sel->get_index(idx);
-		if (!vdata.validity.RowIsValid(index)) {
-			continue;
-		}
-		auto value = data[index];
-		auto data = value.GetDataUnsafe();
-		auto len = value.GetSize();
-		// LCOV_EXCL_START
-		if (len > max_string_length) {
-			throw InternalException(
-			    "Statistics mismatch: string value exceeds maximum string length.\nStatistics: %s\nVector: %s",
-			    ToString(), vector.ToString(count));
-		}
-		if (type.id() == LogicalTypeId::VARCHAR && !has_unicode) {
-			auto unicode = Utf8Proc::Analyze(data, len);
-			if (unicode == UnicodeType::UNICODE) {
-				throw InternalException("Statistics mismatch: string value contains unicode, but statistics says it "
-				                        "shouldn't.\nStatistics: %s\nVector: %s",
-				                        ToString(), vector.ToString(count));
-			} else if (unicode == UnicodeType::INVALID) {
-				throw InternalException("Invalid unicode detected in vector: %s", vector.ToString(count));
-			}
-		}
-		if (StringValueComparison((const_data_ptr_t)data, MinValue<idx_t>(len, MAX_STRING_MINMAX_SIZE), min) < 0) {
-			throw InternalException("Statistics mismatch: value is smaller than min.\nStatistics: %s\nVector: %s",
-			                        ToString(), vector.ToString(count));
-		}
-		if (StringValueComparison((const_data_ptr_t)data, MinValue<idx_t>(len, MAX_STRING_MINMAX_SIZE), max) > 0) {
-			throw InternalException("Statistics mismatch: value is bigger than max.\nStatistics: %s\nVector: %s",
-			                        ToString(), vector.ToString(count));
-		}
-		// LCOV_EXCL_STOP
-	}
-}
-
-} // namespace duckdb

package/src/duckdb/src/storage/statistics/struct_statistics.cpp

@@ -1,108 +0,0 @@
-#include "duckdb/storage/statistics/struct_statistics.hpp"
-
-#include "duckdb/common/field_writer.hpp"
-#include "duckdb/common/types/vector.hpp"
-
-namespace duckdb {
-
-StructStatistics::StructStatistics(LogicalType type_p)
-    : BaseStatistics(std::move(type_p), StatisticsType::LOCAL_STATS) {
-	D_ASSERT(type.InternalType() == PhysicalType::STRUCT);
-	InitializeBase();
-
-	auto &child_types = StructType::GetChildTypes(type);
-	child_stats.resize(child_types.size());
-	for (idx_t i = 0; i < child_types.size(); i++) {
-		child_stats[i] = BaseStatistics::CreateEmpty(child_types[i].second, StatisticsType::LOCAL_STATS);
-	}
-}
-
-void StructStatistics::Merge(const BaseStatistics &other_p) {
-	BaseStatistics::Merge(other_p);
-
-	auto &other = (const StructStatistics &)other_p;
-	D_ASSERT(other.child_stats.size() == child_stats.size());
-	for (idx_t i = 0; i < child_stats.size(); i++) {
-		if (child_stats[i] && other.child_stats[i]) {
-			child_stats[i]->Merge(*other.child_stats[i]);
-		} else {
-			child_stats[i].reset();
-		}
-	}
-}
-
-// LCOV_EXCL_START
-FilterPropagateResult StructStatistics::CheckZonemap(ExpressionType comparison_type, const Value &constant) const {
-	throw InternalException("Struct zonemaps are not supported yet");
-}
-// LCOV_EXCL_STOP
-
-unique_ptr<BaseStatistics> StructStatistics::Copy() const {
-	auto result = make_unique<StructStatistics>(type);
-	result->CopyBase(*this);
-
-	for (idx_t i = 0; i < child_stats.size(); i++) {
-		result->child_stats[i] = child_stats[i] ? child_stats[i]->Copy() : nullptr;
-	}
-	return std::move(result);
-}
-
-void StructStatistics::Serialize(FieldWriter &writer) const {
-	writer.WriteField<uint32_t>(child_stats.size());
-	auto &serializer = writer.GetSerializer();
-	for (idx_t i = 0; i < child_stats.size(); i++) {
-		serializer.Write<bool>(child_stats[i] ? true : false);
-		if (child_stats[i]) {
-			child_stats[i]->Serialize(serializer);
-		}
-	}
-}
-
-unique_ptr<BaseStatistics> StructStatistics::Deserialize(FieldReader &reader, LogicalType type) {
-	D_ASSERT(type.InternalType() == PhysicalType::STRUCT);
-	auto result = make_unique<StructStatistics>(std::move(type));
-	auto &child_types = StructType::GetChildTypes(result->type);
-
-	auto child_type_count = reader.ReadRequired<uint32_t>();
-	if (child_types.size() != child_type_count) {
-		throw InternalException("Struct stats deserialization failure: child count does not match type count!");
-	}
-	auto &source = reader.GetSource();
-	for (idx_t i = 0; i < child_types.size(); i++) {
-		auto has_child = source.Read<bool>();
-		if (has_child) {
-			result->child_stats[i] = BaseStatistics::Deserialize(source, child_types[i].second);
-		} else {
-			result->child_stats[i].reset();
-		}
-	}
-	return std::move(result);
-}
-
-string StructStatistics::ToString() const {
-	string result;
-	result += " {";
-	auto &child_types = StructType::GetChildTypes(type);
-	for (idx_t i = 0; i < child_types.size(); i++) {
-		if (i > 0) {
-			result += ", ";
-		}
-		result += child_types[i].first + ": " + (child_stats[i] ? child_stats[i]->ToString() : "No Stats");
-	}
-	result += "}";
-	result += BaseStatistics::ToString();
-	return result;
-}
-
-void StructStatistics::Verify(Vector &vector, const SelectionVector &sel, idx_t count) const {
-	BaseStatistics::Verify(vector, sel, count);
-
-	auto &child_entries = StructVector::GetEntries(vector);
-	for (idx_t i = 0; i < child_entries.size(); i++) {
-		if (child_stats[i]) {
-			child_stats[i]->Verify(*child_entries[i], sel, count);
-		}
-	}
-}
-
-} // namespace duckdb

package/src/duckdb/src/storage/statistics/validity_statistics.cpp

@@ -1,91 +0,0 @@
-#include "duckdb/storage/statistics/validity_statistics.hpp"
-
-#include "duckdb/common/exception.hpp"
-#include "duckdb/common/field_writer.hpp"
-#include "duckdb/common/string_util.hpp"
-#include "duckdb/common/vector_operations/vector_operations.hpp"
-
-namespace duckdb {
-
-ValidityStatistics::ValidityStatistics(bool has_null, bool has_no_null)
-    : BaseStatistics(LogicalType(LogicalTypeId::VALIDITY), StatisticsType::LOCAL_STATS), has_null(has_null),
-      has_no_null(has_no_null) {
-}
-
-unique_ptr<BaseStatistics> ValidityStatistics::Combine(const unique_ptr<BaseStatistics> &lstats,
-                                                       const unique_ptr<BaseStatistics> &rstats) {
-	if (!lstats && !rstats) {
-		return nullptr;
-	} else if (!lstats) {
-		return rstats->Copy();
-	} else if (!rstats) {
-		return lstats->Copy();
-	} else {
-		auto &l = (ValidityStatistics &)*lstats;
-		auto &r = (ValidityStatistics &)*rstats;
-		return make_unique<ValidityStatistics>(l.has_null || r.has_null, l.has_no_null || r.has_no_null);
-	}
-}
-
-bool ValidityStatistics::IsConstant() const {
-	if (!has_null) {
-		return true;
-	}
-	if (!has_no_null) {
-		return true;
-	}
-	return false;
-}
-
-void ValidityStatistics::Merge(const BaseStatistics &other_p) {
-	auto &other = (ValidityStatistics &)other_p;
-	has_null = has_null || other.has_null;
-	has_no_null = has_no_null || other.has_no_null;
-}
-
-unique_ptr<BaseStatistics> ValidityStatistics::Copy() const {
-	return make_unique<ValidityStatistics>(has_null, has_no_null);
-}
-
-void ValidityStatistics::Serialize(FieldWriter &writer) const {
-	writer.WriteField<bool>(has_null);
-	writer.WriteField<bool>(has_no_null);
-}
-
-unique_ptr<ValidityStatistics> ValidityStatistics::Deserialize(FieldReader &reader) {
-	bool has_null = reader.ReadRequired<bool>();
-	bool has_no_null = reader.ReadRequired<bool>();
-	return make_unique<ValidityStatistics>(has_null, has_no_null);
-}
-
-void ValidityStatistics::Verify(Vector &vector, const SelectionVector &sel, idx_t count) const {
-	if (has_null && has_no_null) {
-		// nothing to verify
-		return;
-	}
-	UnifiedVectorFormat vdata;
-	vector.ToUnifiedFormat(count, vdata);
-	for (idx_t i = 0; i < count; i++) {
-		auto idx = sel.get_index(i);
-		auto index = vdata.sel->get_index(idx);
-		bool row_is_valid = vdata.validity.RowIsValid(index);
-		if (row_is_valid && !has_no_null) {
-			throw InternalException(
-			    "Statistics mismatch: vector labeled as having only NULL values, but vector contains valid values: %s",
-			    vector.ToString(count));
-		}
-		if (!row_is_valid && !has_null) {
-			throw InternalException(
-			    "Statistics mismatch: vector labeled as not having NULL values, but vector contains null values: %s",
-			    vector.ToString(count));
-		}
-	}
-}
-
-string ValidityStatistics::ToString() const {
-	auto has_n = has_null ? "true" : "false";
-	auto has_n_n = has_no_null ? "true" : "false";
-	return StringUtil::Format("[Has Null: %s, Has No Null: %s]", has_n, has_n_n);
-}
-
-} // namespace duckdb