duckdb 0.8.2-dev3989.0 → 0.8.2-dev4126.0

package/src/duckdb/src/common/sort/partition_state.cpp

@@ -429,7 +429,11 @@ bool PartitionGlobalMergeState::TryPrepareNextStage() {
 
 	switch (stage) {
 	case PartitionSortStage::INIT:
-		total_tasks = num_threads;
+		//	If the partitions are unordered, don't scan in parallel
+		//	because it produces non-deterministic orderings.
+		//	This can theoretically happen with ORDER BY,
+		//	but that is something the query should be explicit about.
+		total_tasks = sink.orders.size() > sink.partitions.size() ? num_threads : 1;
 		stage = PartitionSortStage::SCAN;
 		return true;
 

package/src/duckdb/src/core_functions/aggregate/holistic/mode.cpp

@@ -220,7 +220,7 @@ struct ModeFunction {
 			state.frequency_map = new typename STATE::Counts;
 		}
 		const double tau = .25;
-		if (state.nonzero <= tau * state.frequency_map->size()) {
+		if (state.nonzero <= tau * state.frequency_map->size() || prev.end <= frame.start || frame.end <= prev.start) {
 			state.Reset();
 			// for f ∈ F do
 			for (auto f = frame.start; f < frame.end; ++f) {

package/src/duckdb/src/core_functions/function_list.cpp

@@ -49,7 +49,10 @@ static StaticFunctionDefinition internal_functions[] = {
 	DUCKDB_SCALAR_FUNCTION(FactorialOperatorFun),
 	DUCKDB_SCALAR_FUNCTION_SET(BitwiseAndFun),
 	DUCKDB_SCALAR_FUNCTION(PowOperatorFun),
+	DUCKDB_SCALAR_FUNCTION_SET_ALIAS(ListInnerProductFunAlias),
+	DUCKDB_SCALAR_FUNCTION_SET_ALIAS(ListDistanceFunAlias),
 	DUCKDB_SCALAR_FUNCTION_SET(LeftShiftFun),
+	DUCKDB_SCALAR_FUNCTION_SET_ALIAS(ListCosineSimilarityFunAlias),
 	DUCKDB_SCALAR_FUNCTION_SET(RightShiftFun),
 	DUCKDB_SCALAR_FUNCTION_SET(AbsOperatorFun),
 	DUCKDB_SCALAR_FUNCTION_ALIAS(PowOperatorFunAlias),
@@ -197,8 +200,12 @@ static StaticFunctionDefinition internal_functions[] = {
 	DUCKDB_SCALAR_FUNCTION_ALIAS(ListAggrFun),
 	DUCKDB_SCALAR_FUNCTION(ListAggregateFun),
 	DUCKDB_SCALAR_FUNCTION_ALIAS(ListApplyFun),
+	DUCKDB_SCALAR_FUNCTION_SET(ListCosineSimilarityFun),
+	DUCKDB_SCALAR_FUNCTION_SET(ListDistanceFun),
 	DUCKDB_SCALAR_FUNCTION(ListDistinctFun),
+	DUCKDB_SCALAR_FUNCTION_SET_ALIAS(ListDotProductFun),
 	DUCKDB_SCALAR_FUNCTION(ListFilterFun),
+	DUCKDB_SCALAR_FUNCTION_SET(ListInnerProductFun),
 	DUCKDB_SCALAR_FUNCTION_ALIAS(ListPackFun),
 	DUCKDB_SCALAR_FUNCTION_SET(ListReverseSortFun),
 	DUCKDB_SCALAR_FUNCTION_SET(ListSliceFun),
@@ -281,6 +288,7 @@ static StaticFunctionDefinition internal_functions[] = {
 	DUCKDB_AGGREGATE_FUNCTION(StandardErrorOfTheMeanFun),
 	DUCKDB_SCALAR_FUNCTION(SetBitFun),
 	DUCKDB_SCALAR_FUNCTION(SetseedFun),
+	DUCKDB_SCALAR_FUNCTION(SHA256Fun),
 	DUCKDB_SCALAR_FUNCTION_SET(SignFun),
 	DUCKDB_SCALAR_FUNCTION_SET(SignBitFun),
 	DUCKDB_SCALAR_FUNCTION(SinFun),

package/src/duckdb/src/core_functions/scalar/list/list_cosine_similarity.cpp

@@ -0,0 +1,78 @@
+#include "duckdb/core_functions/scalar/list_functions.hpp"
+#include <cmath>
+#include <algorithm>
+
+namespace duckdb {
+
+template <class NUMERIC_TYPE>
+static void ListCosineSimilarity(DataChunk &args, ExpressionState &, Vector &result) {
+	D_ASSERT(args.ColumnCount() == 2);
+
+	auto count = args.size();
+	auto &left = args.data[0];
+	auto &right = args.data[1];
+	auto left_count = ListVector::GetListSize(left);
+	auto right_count = ListVector::GetListSize(right);
+
+	auto &left_child = ListVector::GetEntry(left);
+	auto &right_child = ListVector::GetEntry(right);
+
+	D_ASSERT(left_child.GetVectorType() == VectorType::FLAT_VECTOR);
+	D_ASSERT(right_child.GetVectorType() == VectorType::FLAT_VECTOR);
+
+	if (!FlatVector::Validity(left_child).CheckAllValid(left_count)) {
+		throw InvalidInputException("list_cosine_similarity: left argument can not contain NULL values");
+	}
+
+	if (!FlatVector::Validity(right_child).CheckAllValid(right_count)) {
+		throw InvalidInputException("list_cosine_similarity: right argument can not contain NULL values");
+	}
+
+	auto left_data = FlatVector::GetData<NUMERIC_TYPE>(left_child);
+	auto right_data = FlatVector::GetData<NUMERIC_TYPE>(right_child);
+
+	BinaryExecutor::Execute<list_entry_t, list_entry_t, NUMERIC_TYPE>(
+	    left, right, result, count, [&](list_entry_t left, list_entry_t right) {
+		    if (left.length != right.length) {
+			    throw InvalidInputException(StringUtil::Format(
+			        "list_cosine_similarity: list dimensions must be equal, got left length %d and right length %d",
+			        left.length, right.length));
+		    }
+
+		    auto dimensions = left.length;
+
+		    NUMERIC_TYPE distance = 0;
+		    NUMERIC_TYPE norm_l = 0;
+		    NUMERIC_TYPE norm_r = 0;
+
+		    auto l_ptr = left_data + left.offset;
+		    auto r_ptr = right_data + right.offset;
+		    for (idx_t i = 0; i < dimensions; i++) {
+			    auto x = *l_ptr++;
+			    auto y = *r_ptr++;
+			    distance += x * y;
+			    norm_l += x * x;
+			    norm_r += y * y;
+		    }
+
+		    auto similarity = distance / (std::sqrt(norm_l) * std::sqrt(norm_r));
+
+		    // clamp to [-1, 1] to avoid floating point errors
+		    return std::max(static_cast<NUMERIC_TYPE>(-1), std::min(similarity, static_cast<NUMERIC_TYPE>(1)));
+	    });
+
+	if (args.AllConstant()) {
+		result.SetVectorType(VectorType::CONSTANT_VECTOR);
+	}
+}
+
+ScalarFunctionSet ListCosineSimilarityFun::GetFunctions() {
+	ScalarFunctionSet set("list_cosine_similarity");
+	set.AddFunction(ScalarFunction({LogicalType::LIST(LogicalType::FLOAT), LogicalType::LIST(LogicalType::FLOAT)},
+	                               LogicalType::FLOAT, ListCosineSimilarity<float>));
+	set.AddFunction(ScalarFunction({LogicalType::LIST(LogicalType::DOUBLE), LogicalType::LIST(LogicalType::DOUBLE)},
+	                               LogicalType::DOUBLE, ListCosineSimilarity<double>));
+	return set;
+}
+
+} // namespace duckdb

package/src/duckdb/src/core_functions/scalar/list/list_distance.cpp

@@ -0,0 +1,72 @@
+#include "duckdb/core_functions/scalar/list_functions.hpp"
+#include <cmath>
+
+namespace duckdb {
+
+template <class NUMERIC_TYPE>
+static void ListDistance(DataChunk &args, ExpressionState &, Vector &result) {
+	D_ASSERT(args.ColumnCount() == 2);
+
+	auto count = args.size();
+	auto &left = args.data[0];
+	auto &right = args.data[1];
+	auto left_count = ListVector::GetListSize(left);
+	auto right_count = ListVector::GetListSize(right);
+
+	auto &left_child = ListVector::GetEntry(left);
+	auto &right_child = ListVector::GetEntry(right);
+
+	D_ASSERT(left_child.GetVectorType() == VectorType::FLAT_VECTOR);
+	D_ASSERT(right_child.GetVectorType() == VectorType::FLAT_VECTOR);
+
+	if (!FlatVector::Validity(left_child).CheckAllValid(left_count)) {
+		throw InvalidInputException("list_distance: left argument can not contain NULL values");
+	}
+
+	if (!FlatVector::Validity(right_child).CheckAllValid(right_count)) {
+		throw InvalidInputException("list_distance: right argument can not contain NULL values");
+	}
+
+	auto left_data = FlatVector::GetData<NUMERIC_TYPE>(left_child);
+	auto right_data = FlatVector::GetData<NUMERIC_TYPE>(right_child);
+
+	BinaryExecutor::Execute<list_entry_t, list_entry_t, NUMERIC_TYPE>(
+	    left, right, result, count, [&](list_entry_t left, list_entry_t right) {
+		    if (left.length != right.length) {
+			    throw InvalidInputException(StringUtil::Format(
+			        "list_distance: list dimensions must be equal, got left length %d and right length %d", left.length,
+			        right.length));
+		    }
+
+		    auto dimensions = left.length;
+
+		    NUMERIC_TYPE distance = 0;
+
+		    auto l_ptr = left_data + left.offset;
+		    auto r_ptr = right_data + right.offset;
+
+		    for (idx_t i = 0; i < dimensions; i++) {
+			    auto x = *l_ptr++;
+			    auto y = *r_ptr++;
+			    auto diff = x - y;
+			    distance += diff * diff;
+		    }
+
+		    return std::sqrt(distance);
+	    });
+
+	if (args.AllConstant()) {
+		result.SetVectorType(VectorType::CONSTANT_VECTOR);
+	}
+}
+
+ScalarFunctionSet ListDistanceFun::GetFunctions() {
+	ScalarFunctionSet set("list_distance");
+	set.AddFunction(ScalarFunction({LogicalType::LIST(LogicalType::FLOAT), LogicalType::LIST(LogicalType::FLOAT)},
+	                               LogicalType::FLOAT, ListDistance<float>));
+	set.AddFunction(ScalarFunction({LogicalType::LIST(LogicalType::DOUBLE), LogicalType::LIST(LogicalType::DOUBLE)},
+	                               LogicalType::DOUBLE, ListDistance<double>));
+	return set;
+}
+
+} // namespace duckdb

package/src/duckdb/src/core_functions/scalar/list/list_inner_product.cpp

@@ -0,0 +1,70 @@
+#include "duckdb/core_functions/scalar/list_functions.hpp"
+
+namespace duckdb {
+
+template <class NUMERIC_TYPE>
+static void ListInnerProduct(DataChunk &args, ExpressionState &, Vector &result) {
+	D_ASSERT(args.ColumnCount() == 2);
+
+	auto count = args.size();
+	auto &left = args.data[0];
+	auto &right = args.data[1];
+	auto left_count = ListVector::GetListSize(left);
+	auto right_count = ListVector::GetListSize(right);
+
+	auto &left_child = ListVector::GetEntry(left);
+	auto &right_child = ListVector::GetEntry(right);
+
+	D_ASSERT(left_child.GetVectorType() == VectorType::FLAT_VECTOR);
+	D_ASSERT(right_child.GetVectorType() == VectorType::FLAT_VECTOR);
+
+	if (!FlatVector::Validity(left_child).CheckAllValid(left_count)) {
+		throw InvalidInputException("list_inner_product: left argument can not contain NULL values");
+	}
+
+	if (!FlatVector::Validity(right_child).CheckAllValid(right_count)) {
+		throw InvalidInputException("list_inner_product: right argument can not contain NULL values");
+	}
+
+	auto left_data = FlatVector::GetData<NUMERIC_TYPE>(left_child);
+	auto right_data = FlatVector::GetData<NUMERIC_TYPE>(right_child);
+
+	BinaryExecutor::Execute<list_entry_t, list_entry_t, NUMERIC_TYPE>(
+	    left, right, result, count, [&](list_entry_t left, list_entry_t right) {
+		    if (left.length != right.length) {
+			    throw InvalidInputException(StringUtil::Format(
+			        "list_inner_product: list dimensions must be equal, got left length %d and right length %d",
+			        left.length, right.length));
+		    }
+
+		    auto dimensions = left.length;
+
+		    NUMERIC_TYPE distance = 0;
+
+		    auto l_ptr = left_data + left.offset;
+		    auto r_ptr = right_data + right.offset;
+
+		    for (idx_t i = 0; i < dimensions; i++) {
+			    auto x = *l_ptr++;
+			    auto y = *r_ptr++;
+			    distance += x * y;
+		    }
+
+		    return distance;
+	    });
+
+	if (args.AllConstant()) {
+		result.SetVectorType(VectorType::CONSTANT_VECTOR);
+	}
+}
+
+ScalarFunctionSet ListInnerProductFun::GetFunctions() {
+	ScalarFunctionSet set("list_inner_product");
+	set.AddFunction(ScalarFunction({LogicalType::LIST(LogicalType::FLOAT), LogicalType::LIST(LogicalType::FLOAT)},
+	                               LogicalType::FLOAT, ListInnerProduct<float>));
+	set.AddFunction(ScalarFunction({LogicalType::LIST(LogicalType::DOUBLE), LogicalType::LIST(LogicalType::DOUBLE)},
+	                               LogicalType::DOUBLE, ListInnerProduct<double>));
+	return set;
+}
+
+} // namespace duckdb

package/src/duckdb/src/core_functions/scalar/string/sha256.cpp

@@ -0,0 +1,32 @@
+#include "duckdb/common/exception.hpp"
+#include "duckdb/common/vector_operations/unary_executor.hpp"
+#include "duckdb/core_functions/scalar/string_functions.hpp"
+#include "mbedtls_wrapper.hpp"
+
+namespace duckdb {
+
+struct SHA256Operator {
+	template <class INPUT_TYPE, class RESULT_TYPE>
+	static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) {
+		auto hash = StringVector::EmptyString(result, duckdb_mbedtls::MbedTlsWrapper::SHA256_HASH_LENGTH_TEXT);
+
+		duckdb_mbedtls::MbedTlsWrapper::SHA256State state;
+		state.AddString(input.GetString());
+		state.FinishHex(hash.GetDataWriteable());
+
+		hash.Finalize();
+		return hash;
+	}
+};
+
+static void SHA256Function(DataChunk &args, ExpressionState &state, Vector &result) {
+	auto &input = args.data[0];
+
+	UnaryExecutor::ExecuteString<string_t, string_t, SHA256Operator>(input, result, args.size());
+}
+
+ScalarFunction SHA256Fun::GetFunction() {
+	return ScalarFunction({LogicalType::VARCHAR}, LogicalType::VARCHAR, SHA256Function);
+}
+
+} // namespace duckdb