duckdb 0.7.2-dev717.0 → 0.7.2-dev865.0

package/src/duckdb/src/common/types.cpp

@@ -27,6 +27,10 @@
 #include "duckdb/parser/keyword_helper.hpp"
 #include "duckdb/parser/parser.hpp"
 
+#include "duckdb/common/serializer/format_deserializer.hpp"
+#include "duckdb/common/serializer/enum_serializer.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
+
 #include <cmath>
 
 namespace duckdb {
@@ -327,95 +331,8 @@ bool TypeIsInteger(PhysicalType type) {
 	return (type >= PhysicalType::UINT8 && type <= PhysicalType::INT64) || type == PhysicalType::INT128;
 }
 
-// LCOV_EXCL_START
 string LogicalTypeIdToString(LogicalTypeId id) {
-	switch (id) {
-	case LogicalTypeId::BOOLEAN:
-		return "BOOLEAN";
-	case LogicalTypeId::TINYINT:
-		return "TINYINT";
-	case LogicalTypeId::SMALLINT:
-		return "SMALLINT";
-	case LogicalTypeId::INTEGER:
-		return "INTEGER";
-	case LogicalTypeId::BIGINT:
-		return "BIGINT";
-	case LogicalTypeId::HUGEINT:
-		return "HUGEINT";
-	case LogicalTypeId::UUID:
-		return "UUID";
-	case LogicalTypeId::UTINYINT:
-		return "UTINYINT";
-	case LogicalTypeId::USMALLINT:
-		return "USMALLINT";
-	case LogicalTypeId::UINTEGER:
-		return "UINTEGER";
-	case LogicalTypeId::UBIGINT:
-		return "UBIGINT";
-	case LogicalTypeId::DATE:
-		return "DATE";
-	case LogicalTypeId::TIME:
-		return "TIME";
-	case LogicalTypeId::TIMESTAMP:
-		return "TIMESTAMP";
-	case LogicalTypeId::TIMESTAMP_MS:
-		return "TIMESTAMP_MS";
-	case LogicalTypeId::TIMESTAMP_NS:
-		return "TIMESTAMP_NS";
-	case LogicalTypeId::TIMESTAMP_SEC:
-		return "TIMESTAMP_S";
-	case LogicalTypeId::TIMESTAMP_TZ:
-		return "TIMESTAMP WITH TIME ZONE";
-	case LogicalTypeId::TIME_TZ:
-		return "TIME WITH TIME ZONE";
-	case LogicalTypeId::FLOAT:
-		return "FLOAT";
-	case LogicalTypeId::DOUBLE:
-		return "DOUBLE";
-	case LogicalTypeId::DECIMAL:
-		return "DECIMAL";
-	case LogicalTypeId::VARCHAR:
-		return "VARCHAR";
-	case LogicalTypeId::BLOB:
-		return "BLOB";
-	case LogicalTypeId::CHAR:
-		return "CHAR";
-	case LogicalTypeId::INTERVAL:
-		return "INTERVAL";
-	case LogicalTypeId::SQLNULL:
-		return "NULL";
-	case LogicalTypeId::ANY:
-		return "ANY";
-	case LogicalTypeId::VALIDITY:
-		return "VALIDITY";
-	case LogicalTypeId::STRUCT:
-		return "STRUCT";
-	case LogicalTypeId::LIST:
-		return "LIST";
-	case LogicalTypeId::MAP:
-		return "MAP";
-	case LogicalTypeId::POINTER:
-		return "POINTER";
-	case LogicalTypeId::TABLE:
-		return "TABLE";
-	case LogicalTypeId::LAMBDA:
-		return "LAMBDA";
-	case LogicalTypeId::INVALID:
-		return "INVALID";
-	case LogicalTypeId::UNION:
-		return "UNION";
-	case LogicalTypeId::UNKNOWN:
-		return "UNKNOWN";
-	case LogicalTypeId::ENUM:
-		return "ENUM";
-	case LogicalTypeId::AGGREGATE_STATE:
-		return "AGGREGATE_STATE";
-	case LogicalTypeId::USER:
-		return "USER";
-	case LogicalTypeId::BIT:
-		return "BIT";
-	}
-	return "UNDEFINED";
+	return EnumSerializer::EnumToString(id);
 }
 
 string LogicalType::ToString() const {
@@ -883,12 +800,16 @@ public:
 		return alias == other_p->alias && EqualsInternal(other_p);
 	}
 	//! Serializes a ExtraTypeInfo to a stand-alone binary blob
-	virtual void Serialize(FieldWriter &writer) const {};
+	virtual void Serialize(FieldWriter &writer) const {
+	}
 	//! Serializes a ExtraTypeInfo to a stand-alone binary blob
 	static void Serialize(ExtraTypeInfo *info, FieldWriter &writer);
 	//! Deserializes a blob back into an ExtraTypeInfo
 	static shared_ptr<ExtraTypeInfo> Deserialize(FieldReader &reader);
 
+	virtual void FormatSerialize(FormatSerializer &serializer) const;
+	static shared_ptr<ExtraTypeInfo> FormatDeserialize(FormatDeserializer &source);
+
 protected:
 	virtual bool EqualsInternal(ExtraTypeInfo *other_p) const {
 		// Do nothing
@@ -959,6 +880,18 @@ public:
 		writer.WriteField<uint8_t>(scale);
 	}
 
+	void FormatSerialize(FormatSerializer &serializer) const override {
+		ExtraTypeInfo::FormatSerialize(serializer);
+		serializer.WriteProperty("width", width);
+		serializer.WriteProperty("scale", scale);
+	}
+
+	static shared_ptr<ExtraTypeInfo> FormatDeserialize(FormatDeserializer &source) {
+		auto width = source.ReadProperty<uint8_t>("width");
+		auto scale = source.ReadProperty<uint8_t>("scale");
+		return make_shared<DecimalTypeInfo>(width, scale);
+	}
+
 	static shared_ptr<ExtraTypeInfo> Deserialize(FieldReader &reader) {
 		auto width = reader.ReadRequired<uint8_t>();
 		auto scale = reader.ReadRequired<uint8_t>();
@@ -1016,6 +949,16 @@ public:
 		return make_shared<StringTypeInfo>(std::move(collation));
 	}
 
+	void FormatSerialize(FormatSerializer &serializer) const override {
+		ExtraTypeInfo::FormatSerialize(serializer);
+		serializer.WriteProperty("collation", collation);
+	}
+
+	static shared_ptr<ExtraTypeInfo> FormatDeserialize(FormatDeserializer &source) {
+		auto collation = source.ReadProperty<string>("collation");
+		return make_shared<StringTypeInfo>(std::move(collation));
+	}
+
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override {
 		// collation info has no impact on equality
@@ -1057,11 +1000,21 @@ public:
 		writer.WriteSerializable(child_type);
 	}
 
+	void FormatSerialize(FormatSerializer &serializer) const override {
+		ExtraTypeInfo::FormatSerialize(serializer);
+		serializer.WriteProperty("child_type", child_type);
+	}
+
 	static shared_ptr<ExtraTypeInfo> Deserialize(FieldReader &reader) {
 		auto child_type = reader.ReadRequiredSerializable<LogicalType, LogicalType>();
 		return make_shared<ListTypeInfo>(std::move(child_type));
 	}
 
+	static shared_ptr<ExtraTypeInfo> FormatDeserialize(FormatDeserializer &source) {
+		auto child_type = source.ReadProperty<LogicalType>("child_type");
+		return make_shared<ListTypeInfo>(std::move(child_type));
+	}
+
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override {
 		auto &other = (ListTypeInfo &)*other_p;
@@ -1101,6 +1054,11 @@ public:
 		}
 	}
 
+	void FormatSerialize(FormatSerializer &serializer) const override {
+		ExtraTypeInfo::FormatSerialize(serializer);
+		serializer.WriteProperty("child_types", child_types);
+	}
+
 	static shared_ptr<ExtraTypeInfo> Deserialize(FieldReader &reader) {
 		child_list_t<LogicalType> child_list;
 		auto child_types_size = reader.ReadRequired<uint32_t>();
@@ -1113,6 +1071,11 @@ public:
 		return make_shared<StructTypeInfo>(std::move(child_list));
 	}
 
+	static shared_ptr<ExtraTypeInfo> FormatDeserialize(FormatDeserializer &deserializer) {
+		auto child_types = deserializer.ReadProperty<child_list_t<LogicalType>>("child_types");
+		return make_shared<StructTypeInfo>(std::move(child_types));
+	}
+
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override {
 		auto &other = (StructTypeInfo &)*other_p;
@@ -1138,6 +1101,21 @@ public:
 		}
 	}
 
+	void FormatSerialize(FormatSerializer &serializer) const override {
+		ExtraTypeInfo::FormatSerialize(serializer);
+		serializer.WriteProperty("function_name", state_type.function_name);
+		serializer.WriteProperty("return_type", state_type.return_type);
+		serializer.WriteProperty("bound_argument_types", state_type.bound_argument_types);
+	}
+
+	static shared_ptr<ExtraTypeInfo> FormatDeserialize(FormatDeserializer &source) {
+		auto function_name = source.ReadProperty<string>("function_name");
+		auto return_type = source.ReadProperty<LogicalType>("return_type");
+		auto bound_argument_types = source.ReadProperty<vector<LogicalType>>("bound_argument_types");
+		return make_shared<AggregateStateTypeInfo>(
+		    aggregate_state_t(std::move(function_name), std::move(return_type), std::move(bound_argument_types)));
+	}
+
 	static shared_ptr<ExtraTypeInfo> Deserialize(FieldReader &reader) {
 		auto &source = reader.GetSource();
 
@@ -1294,11 +1272,21 @@ public:
 		writer.WriteString(user_type_name);
 	}
 
+	void FormatSerialize(FormatSerializer &serializer) const override {
+		ExtraTypeInfo::FormatSerialize(serializer);
+		serializer.WriteProperty("user_type_name", user_type_name);
+	}
+
 	static shared_ptr<ExtraTypeInfo> Deserialize(FieldReader &reader) {
 		auto enum_name = reader.ReadRequired<string>();
 		return make_shared<UserTypeInfo>(std::move(enum_name));
 	}
 
+	static shared_ptr<ExtraTypeInfo> FormatDeserialize(FormatDeserializer &source) {
+		auto enum_name = source.ReadProperty<string>("user_type_name");
+		return make_shared<UserTypeInfo>(std::move(enum_name));
+	}
+
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override {
 		auto &other = (UserTypeInfo &)*other_p;
@@ -1380,6 +1368,12 @@ protected:
 		EnumType::Serialize(writer, *this, serialize_internals);
 	}
 
+	void FormatSerialize(FormatSerializer &serializer) const override {
+		ExtraTypeInfo::FormatSerialize(serializer);
+		serializer.WriteProperty("dict_size", dict_size);
+		serializer.WriteProperty("enum_name", enum_name);
+		((Vector &)values_insert_order).FormatSerialize(serializer, dict_size);
+	}
 	Vector values_insert_order;
 
 private:
@@ -1439,6 +1433,13 @@ struct EnumTypeInfoTemplated : public EnumTypeInfo {
 		return make_shared<EnumTypeInfoTemplated>(std::move(enum_name), values_insert_order, size);
 	}
 
+	static shared_ptr<EnumTypeInfoTemplated> FormatDeserialize(FormatDeserializer &source, uint32_t size) {
+		auto enum_name = source.ReadProperty<string>("enum_name");
+		Vector values_insert_order(LogicalType::VARCHAR, size);
+		values_insert_order.FormatDeserialize(source, size);
+		return make_shared<EnumTypeInfoTemplated>(std::move(enum_name), values_insert_order, size);
+	}
+
 	string_map_t<T> &GetValues() {
 		return values;
 	}
@@ -1573,6 +1574,70 @@ void ExtraTypeInfo::Serialize(ExtraTypeInfo *info, FieldWriter &writer) {
 		writer.WriteString(info->alias);
 	}
 }
+void ExtraTypeInfo::FormatSerialize(FormatSerializer &serializer) const {
+	serializer.WriteProperty("type", type);
+	serializer.WriteProperty("alias", alias);
+}
+
+shared_ptr<ExtraTypeInfo> ExtraTypeInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto type = deserializer.ReadProperty<ExtraTypeInfoType>("type");
+
+	shared_ptr<ExtraTypeInfo> result;
+	switch (type) {
+	case ExtraTypeInfoType::INVALID_TYPE_INFO: {
+		string alias;
+		deserializer.ReadOptionalProperty("alias", alias);
+		if (!alias.empty()) {
+			return make_shared<ExtraTypeInfo>(type, alias);
+		}
+		return nullptr;
+	}
+	case ExtraTypeInfoType::GENERIC_TYPE_INFO: {
+		result = make_shared<ExtraTypeInfo>(type);
+	} break;
+	case ExtraTypeInfoType::DECIMAL_TYPE_INFO:
+		result = DecimalTypeInfo::FormatDeserialize(deserializer);
+		break;
+	case ExtraTypeInfoType::STRING_TYPE_INFO:
+		result = StringTypeInfo::FormatDeserialize(deserializer);
+		break;
+	case ExtraTypeInfoType::LIST_TYPE_INFO:
+		result = ListTypeInfo::FormatDeserialize(deserializer);
+		break;
+	case ExtraTypeInfoType::STRUCT_TYPE_INFO:
+		result = StructTypeInfo::FormatDeserialize(deserializer);
+		break;
+	case ExtraTypeInfoType::USER_TYPE_INFO:
+		result = UserTypeInfo::FormatDeserialize(deserializer);
+		break;
+	case ExtraTypeInfoType::ENUM_TYPE_INFO: {
+		auto enum_size = deserializer.ReadProperty<uint32_t>("enum_size");
+		auto enum_internal_type = EnumVectorDictType(enum_size);
+		switch (enum_internal_type) {
+		case PhysicalType::UINT8:
+			result = EnumTypeInfoTemplated<uint8_t>::FormatDeserialize(deserializer, enum_size);
+			break;
+		case PhysicalType::UINT16:
+			result = EnumTypeInfoTemplated<uint16_t>::FormatDeserialize(deserializer, enum_size);
+			break;
+		case PhysicalType::UINT32:
+			result = EnumTypeInfoTemplated<uint32_t>::FormatDeserialize(deserializer, enum_size);
+			break;
+		default:
+			throw InternalException("Invalid Physical Type for ENUMs");
+		}
+	} break;
+	case ExtraTypeInfoType::AGGREGATE_STATE_TYPE_INFO:
+		result = AggregateStateTypeInfo::FormatDeserialize(deserializer);
+		break;
+
+	default:
+		throw InternalException("Unimplemented type info in ExtraTypeInfo::Deserialize");
+	}
+	deserializer.ReadOptionalPropertyOrDefault("alias", result->alias, string());
+	return result;
+}
+
 shared_ptr<ExtraTypeInfo> ExtraTypeInfo::Deserialize(FieldReader &reader) {
 	auto type = reader.ReadRequired<ExtraTypeInfoType>();
 	shared_ptr<ExtraTypeInfo> extra_info;
@@ -1684,6 +1749,18 @@ LogicalType LogicalType::Deserialize(Deserializer &source) {
 	return LogicalType(id, std::move(info));
 }
 
+void LogicalType::FormatSerialize(FormatSerializer &serializer) const {
+	serializer.WriteProperty("id", id_);
+	serializer.WriteOptionalProperty("type_info", type_info_.get());
+}
+
+LogicalType LogicalType::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto id = deserializer.ReadProperty<LogicalTypeId>("id");
+	auto info = deserializer.ReadOptionalProperty<shared_ptr<ExtraTypeInfo>>("type_info");
+
+	return LogicalType(id, std::move(info));
+}
+
 bool LogicalType::EqualTypeInfo(const LogicalType &rhs) const {
 	if (type_info_.get() == rhs.type_info_.get()) {
 		return true;

package/src/duckdb/src/execution/operator/helper/physical_limit.cpp

@@ -111,6 +111,9 @@ SinkResultType PhysicalLimit::Sink(ExecutionContext &context, GlobalSinkState &g
 	}
 	state.data.Append(input, lstate.batch_index);
 	state.current_offset += input.size();
+	if (state.current_offset == max_element) {
+		return SinkResultType::FINISHED;
+	}
 	return SinkResultType::NEED_MORE_INPUT;
 }
 

package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp

@@ -73,20 +73,17 @@ static bool CanUsePerfectHashAggregate(ClientContext &context, LogicalAggregate
 		int64_t range;
 		switch (group_type.InternalType()) {
 		case PhysicalType::INT8:
-			range = int64_t(NumericStats::GetMaxUnsafe<int8_t>(nstats)) -
-			        int64_t(NumericStats::GetMinUnsafe<int8_t>(nstats));
+			range = int64_t(NumericStats::GetMax<int8_t>(nstats)) - int64_t(NumericStats::GetMin<int8_t>(nstats));
 			break;
 		case PhysicalType::INT16:
-			range = int64_t(NumericStats::GetMaxUnsafe<int16_t>(nstats)) -
-			        int64_t(NumericStats::GetMinUnsafe<int16_t>(nstats));
+			range = int64_t(NumericStats::GetMax<int16_t>(nstats)) - int64_t(NumericStats::GetMin<int16_t>(nstats));
 			break;
 		case PhysicalType::INT32:
-			range = int64_t(NumericStats::GetMaxUnsafe<int32_t>(nstats)) -
-			        int64_t(NumericStats::GetMinUnsafe<int32_t>(nstats));
+			range = int64_t(NumericStats::GetMax<int32_t>(nstats)) - int64_t(NumericStats::GetMin<int32_t>(nstats));
 			break;
 		case PhysicalType::INT64:
-			if (!TrySubtractOperator::Operation(NumericStats::GetMaxUnsafe<int64_t>(nstats),
-			                                    NumericStats::GetMinUnsafe<int64_t>(nstats), range)) {
+			if (!TrySubtractOperator::Operation(NumericStats::GetMax<int64_t>(nstats),
+			                                    NumericStats::GetMin<int64_t>(nstats), range)) {
 				return false;
 			}
 			break;

package/src/duckdb/src/function/scalar/date/date_part.cpp

@@ -180,8 +180,8 @@ struct DatePart {
 			return nullptr;
 		}
 		// run the operator on both the min and the max, this gives us the [min, max] bound
-		auto min = NumericStats::GetMinUnsafe<T>(nstats);
-		auto max = NumericStats::GetMaxUnsafe<T>(nstats);
+		auto min = NumericStats::GetMin<T>(nstats);
+		auto max = NumericStats::GetMax<T>(nstats);
 		if (min > max) {
 			return nullptr;
 		}

package/src/duckdb/src/function/scalar/date/date_trunc.cpp

@@ -594,8 +594,8 @@ static unique_ptr<BaseStatistics> DateTruncStatistics(vector<BaseStatistics> &ch
 		return nullptr;
 	}
 	// run the operator on both the min and the max, this gives us the [min, max] bound
-	auto min = NumericStats::GetMinUnsafe<TA>(nstats);
-	auto max = NumericStats::GetMaxUnsafe<TA>(nstats);
+	auto min = NumericStats::GetMin<TA>(nstats);
+	auto max = NumericStats::GetMax<TA>(nstats);
 	if (min > max) {
 		return nullptr;
 	}

package/src/duckdb/src/function/scalar/list/list_aggregates.cpp

@@ -507,7 +507,7 @@ ScalarFunction ListUniqueFun::GetFunction() {
 }
 
 void ListAggregateFun::RegisterFunction(BuiltinFunctions &set) {
-	set.AddFunction({"list_aggregate", "array_aggregate", "list_aggr", "array_aggr"}, GetFunction());
+	set.AddFunction({"list_aggregate", "array_aggregate", "list_aggr", "array_aggr", "aggregate"}, GetFunction());
 }
 
 void ListDistinctFun::RegisterFunction(BuiltinFunctions &set) {

package/src/duckdb/src/function/scalar/list/list_lambdas.cpp

@@ -392,6 +392,8 @@ void ListTransformFun::RegisterFunction(BuiltinFunctions &set) {
 	set.AddFunction(fun);
 	fun.name = "array_apply";
 	set.AddFunction(fun);
+	fun.name = "apply";
+	set.AddFunction(fun);
 }
 
 void ListFilterFun::RegisterFunction(BuiltinFunctions &set) {
@@ -405,6 +407,8 @@ void ListFilterFun::RegisterFunction(BuiltinFunctions &set) {
 
 	fun.name = "array_filter";
 	set.AddFunction(fun);
+	fun.name = "filter";
+	set.AddFunction(fun);
 }
 
 } // namespace duckdb

package/src/duckdb/src/function/scalar/operators/arithmetic.cpp

@@ -81,11 +81,11 @@ struct AddPropagateStatistics {
 	                      Value &new_max) {
 		T min, max;
 		// new min is min+min
-		if (!OP::Operation(NumericStats::GetMinUnsafe<T>(lstats), NumericStats::GetMinUnsafe<T>(rstats), min)) {
+		if (!OP::Operation(NumericStats::GetMin<T>(lstats), NumericStats::GetMin<T>(rstats), min)) {
 			return true;
 		}
 		// new max is max+max
-		if (!OP::Operation(NumericStats::GetMaxUnsafe<T>(lstats), NumericStats::GetMaxUnsafe<T>(rstats), max)) {
+		if (!OP::Operation(NumericStats::GetMax<T>(lstats), NumericStats::GetMax<T>(rstats), max)) {
 			return true;
 		}
 		new_min = Value::Numeric(type, min);
@@ -99,10 +99,10 @@ struct SubtractPropagateStatistics {
 	static bool Operation(LogicalType type, BaseStatistics &lstats, BaseStatistics &rstats, Value &new_min,
 	                      Value &new_max) {
 		T min, max;
-		if (!OP::Operation(NumericStats::GetMinUnsafe<T>(lstats), NumericStats::GetMaxUnsafe<T>(rstats), min)) {
+		if (!OP::Operation(NumericStats::GetMin<T>(lstats), NumericStats::GetMax<T>(rstats), min)) {
 			return true;
 		}
-		if (!OP::Operation(NumericStats::GetMaxUnsafe<T>(lstats), NumericStats::GetMinUnsafe<T>(rstats), max)) {
+		if (!OP::Operation(NumericStats::GetMax<T>(lstats), NumericStats::GetMin<T>(rstats), max)) {
 			return true;
 		}
 		new_min = Value::Numeric(type, min);
@@ -489,8 +489,8 @@ unique_ptr<FunctionData> DecimalNegateBind(ClientContext &context, ScalarFunctio
 struct NegatePropagateStatistics {
 	template <class T>
 	static bool Operation(LogicalType type, BaseStatistics &istats, Value &new_min, Value &new_max) {
-		auto max_value = NumericStats::GetMaxUnsafe<T>(istats);
-		auto min_value = NumericStats::GetMinUnsafe<T>(istats);
+		auto max_value = NumericStats::GetMax<T>(istats);
+		auto min_value = NumericStats::GetMin<T>(istats);
 		if (!NegateOperator::CanNegate<T>(min_value) || !NegateOperator::CanNegate<T>(max_value)) {
 			return true;
 		}
@@ -664,8 +664,8 @@ struct MultiplyPropagateStatistics {
 		// etc
 		// rather than doing all this switcheroo we just multiply all combinations of lmin/lmax with rmin/rmax
 		// and check what the minimum/maximum value is
-		T lvals[] {NumericStats::GetMinUnsafe<T>(lstats), NumericStats::GetMaxUnsafe<T>(lstats)};
-		T rvals[] {NumericStats::GetMinUnsafe<T>(rstats), NumericStats::GetMaxUnsafe<T>(rstats)};
+		T lvals[] {NumericStats::GetMin<T>(lstats), NumericStats::GetMax<T>(lstats)};
+		T rvals[] {NumericStats::GetMin<T>(rstats), NumericStats::GetMax<T>(rstats)};
 		T min = NumericLimits<T>::Maximum();
 		T max = NumericLimits<T>::Minimum();
 		// multiplications