duckdb 0.3.3-dev1312.0 → 0.3.3-dev1350.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.3-dev1312.0",
+  "version": "0.3.3-dev1350.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -4681,7 +4681,44 @@ static DefaultMacro internal_macros[] = {
 	{DEFAULT_SCHEMA, "generate_subscripts", {"arr", "dim", nullptr}, "unnest(generate_series(1, array_length(arr, dim)))"},
 	{DEFAULT_SCHEMA, "fdiv", {"x", "y", nullptr}, "floor(x/y)"},
 	{DEFAULT_SCHEMA, "fmod", {"x", "y", nullptr}, "(x-y*floor(x/y))"},
-	{nullptr, nullptr, {nullptr}, nullptr}};
+
+	// algebraic list aggregates
+	{DEFAULT_SCHEMA, "list_avg", {"l", nullptr}, "list_aggr(l, 'avg')"},
+	{DEFAULT_SCHEMA, "list_var_samp", {"l", nullptr}, "list_aggr(l, 'var_samp')"},
+	{DEFAULT_SCHEMA, "list_var_pop", {"l", nullptr}, "list_aggr(l, 'var_pop')"},
+	{DEFAULT_SCHEMA, "list_stddev_pop", {"l", nullptr}, "list_aggr(l, 'stddev_pop')"},
+	{DEFAULT_SCHEMA, "list_stddev_samp", {"l", nullptr}, "list_aggr(l, 'stddev_samp')"},
+	{DEFAULT_SCHEMA, "list_sem", {"l", nullptr}, "list_aggr(l, 'sem')"},
+
+	// distributive list aggregates
+	{DEFAULT_SCHEMA, "list_approx_count_distinct", {"l", nullptr}, "list_aggr(l, 'approx_count_distinct')"},
+	{DEFAULT_SCHEMA, "list_bit_xor", {"l", nullptr}, "list_aggr(l, 'bit_xor')"},
+	{DEFAULT_SCHEMA, "list_bit_or", {"l", nullptr}, "list_aggr(l, 'bit_or')"},
+	{DEFAULT_SCHEMA, "list_bit_and", {"l", nullptr}, "list_aggr(l, 'bit_and')"},
+	{DEFAULT_SCHEMA, "list_bool_and", {"l", nullptr}, "list_aggr(l, 'bool_and')"},
+	{DEFAULT_SCHEMA, "list_bool_or", {"l", nullptr}, "list_aggr(l, 'bool_or')"},
+	{DEFAULT_SCHEMA, "list_count", {"l", nullptr}, "list_aggr(l, 'count')"},
+	{DEFAULT_SCHEMA, "list_entropy", {"l", nullptr}, "list_aggr(l, 'entropy')"},
+	{DEFAULT_SCHEMA, "list_last", {"l", nullptr}, "list_aggr(l, 'last')"},
+	{DEFAULT_SCHEMA, "list_first", {"l", nullptr}, "list_aggr(l, 'first')"},
+	{DEFAULT_SCHEMA, "list_kurtosis", {"l", nullptr}, "list_aggr(l, 'kurtosis')"},
+	{DEFAULT_SCHEMA, "list_min", {"l", nullptr}, "list_aggr(l, 'min')"},
+	{DEFAULT_SCHEMA, "list_max", {"l", nullptr}, "list_aggr(l, 'max')"},
+	{DEFAULT_SCHEMA, "list_product", {"l", nullptr}, "list_aggr(l, 'product')"},
+	{DEFAULT_SCHEMA, "list_skewness", {"l", nullptr}, "list_aggr(l, 'skewness')"},
+	{DEFAULT_SCHEMA, "list_sum", {"l", nullptr}, "list_aggr(l, 'sum')"},
+	{DEFAULT_SCHEMA, "list_string_agg", {"l", nullptr}, "list_aggr(l, 'string_agg')"},
+
+	// holistic list aggregates
+	{DEFAULT_SCHEMA, "list_mode", {"l", nullptr}, "list_aggr(l, 'mode')"},
+	{DEFAULT_SCHEMA, "list_median", {"l", nullptr}, "list_aggr(l, 'median')"},
+	{DEFAULT_SCHEMA, "list_mad", {"l", nullptr}, "list_aggr(l, 'mad')"},
+
+	// nested list aggregates
+	{DEFAULT_SCHEMA, "list_histogram", {"l", nullptr}, "list_aggr(l, 'histogram')"},
+
+	{nullptr, nullptr, {nullptr}, nullptr}
+	};
 
 unique_ptr<CreateMacroInfo> DefaultFunctionGenerator::CreateInternalMacroInfo(DefaultMacro &default_macro) {
 	// parse the expression
@@ -78309,6 +78346,11 @@ struct ListPositionFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
 
+struct ListAggregateFun {
+	static ScalarFunction GetFunction();
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
 struct CardinalityFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
@@ -86746,6 +86788,227 @@ void ListFlattenFun::RegisterFunction(BuiltinFunctions &set) {
 
 
 
+
+namespace duckdb {
+
+// FIXME: use a local state for each thread to increase performance?
+// FIXME: benchmark the use of simple_update against using update (if applicable)
+
+struct ListAggregatesBindData : public FunctionData {
+	ListAggregatesBindData(const LogicalType &stype_p, unique_ptr<Expression> aggr_expr_p);
+	~ListAggregatesBindData() override;
+
+	LogicalType stype;
+	unique_ptr<Expression> aggr_expr;
+
+	unique_ptr<FunctionData> Copy() override;
+};
+
+ListAggregatesBindData::ListAggregatesBindData(const LogicalType &stype_p, unique_ptr<Expression> aggr_expr_p)
+    : stype(stype_p), aggr_expr(move(aggr_expr_p)) {
+}
+
+unique_ptr<FunctionData> ListAggregatesBindData::Copy() {
+	return make_unique<ListAggregatesBindData>(stype, aggr_expr->Copy());
+}
+
+ListAggregatesBindData::~ListAggregatesBindData() {
+}
+
+struct StateVector {
+	StateVector(idx_t count_p, unique_ptr<Expression> aggr_expr_p)
+	    : count(count_p), aggr_expr(move(aggr_expr_p)), state_vector(Vector(LogicalType::POINTER, count_p)) {
+	}
+
+	~StateVector() {
+		// destroy objects within the aggregate states
+		auto &aggr = (BoundAggregateExpression &)*aggr_expr;
+		if (aggr.function.destructor) {
+			aggr.function.destructor(state_vector, count);
+		}
+	}
+
+	idx_t count;
+	unique_ptr<Expression> aggr_expr;
+	Vector state_vector;
+};
+
+static void ListAggregateFunction(DataChunk &args, ExpressionState &state, Vector &result) {
+
+	D_ASSERT(args.ColumnCount() == 2);
+	auto count = args.size();
+	Vector &lists = args.data[0];
+
+	// set the result vector
+	result.SetVectorType(VectorType::FLAT_VECTOR);
+	auto &result_validity = FlatVector::Validity(result);
+
+	if (lists.GetType().id() == LogicalTypeId::SQLNULL) {
+		result_validity.SetInvalid(0);
+		return;
+	}
+
+	// get the aggregate function
+	auto &func_expr = (BoundFunctionExpression &)state.expr;
+	auto &info = (ListAggregatesBindData &)*func_expr.bind_info;
+	auto &aggr = (BoundAggregateExpression &)*info.aggr_expr;
+
+	D_ASSERT(aggr.function.update);
+
+	auto lists_size = ListVector::GetListSize(lists);
+	auto &child_vector = ListVector::GetEntry(lists);
+
+	VectorData child_data;
+	child_vector.Orrify(lists_size, child_data);
+
+	VectorData lists_data;
+	lists.Orrify(count, lists_data);
+	auto list_entries = (list_entry_t *)lists_data.data;
+
+	// state_buffer holds the state for each list of this chunk
+	idx_t size = aggr.function.state_size();
+	auto state_buffer = unique_ptr<data_t[]>(new data_t[size * count]);
+
+	// state vector for initialize and finalize
+	StateVector state_vector(count, info.aggr_expr->Copy());
+	auto states = FlatVector::GetData<data_ptr_t>(state_vector.state_vector);
+
+	// state vector of STANDARD_VECTOR_SIZE holds the pointers to the states
+	Vector state_vector_update = Vector(LogicalType::POINTER);
+	auto states_update = FlatVector::GetData<data_ptr_t>(state_vector_update);
+
+	// selection vector pointing to the data
+	SelectionVector sel_vector(STANDARD_VECTOR_SIZE);
+	idx_t states_idx = 0;
+
+	for (idx_t i = 0; i < count; i++) {
+
+		// initialize the state for this list
+		auto state_ptr = state_buffer.get() + size * i;
+		states[i] = state_ptr;
+		aggr.function.initialize(states[i]);
+
+		auto lists_index = lists_data.sel->get_index(i);
+		const auto &list_entry = list_entries[lists_index];
+
+		// nothing to do for this list
+		if (!lists_data.validity.RowIsValid(lists_index)) {
+			result_validity.SetInvalid(i);
+			continue;
+		}
+
+		// skip empty list
+		if (list_entry.length == 0) {
+			continue;
+		}
+
+		auto source_idx = child_data.sel->get_index(list_entry.offset);
+		idx_t child_idx = 0;
+
+		while (child_idx < list_entry.length) {
+
+			// states vector is full, update
+			if (states_idx == STANDARD_VECTOR_SIZE) {
+
+				// update the aggregate state(s)
+				Vector slice = Vector(child_vector, sel_vector, states_idx);
+				aggr.function.update(&slice, aggr.bind_info.get(), 1, state_vector_update, states_idx);
+
+				// reset values
+				states_idx = 0;
+			}
+
+			sel_vector.set_index(states_idx, source_idx + child_idx);
+			states_update[states_idx] = state_ptr;
+			states_idx++;
+			child_idx++;
+		}
+	}
+
+	// update the remaining elements of the last list(s)
+	if (states_idx != 0) {
+		Vector slice = Vector(child_vector, sel_vector, states_idx);
+		aggr.function.update(&slice, aggr.bind_info.get(), 1, state_vector_update, states_idx);
+	}
+
+	// finalize all the aggregate states
+	aggr.function.finalize(state_vector.state_vector, aggr.bind_info.get(), result, count, 0);
+}
+
+static unique_ptr<FunctionData> ListAggregateBind(ClientContext &context, ScalarFunction &bound_function,
+                                                  vector<unique_ptr<Expression>> &arguments) {
+
+	// the list column and the name of the aggregate function
+	D_ASSERT(bound_function.arguments.size() == 2);
+	D_ASSERT(arguments.size() == 2);
+
+	if (arguments[0]->return_type.id() == LogicalTypeId::SQLNULL) {
+		bound_function.arguments[0] = LogicalType::SQLNULL;
+		bound_function.return_type = LogicalType::SQLNULL;
+		return make_unique<VariableReturnBindData>(bound_function.return_type);
+	}
+
+	D_ASSERT(LogicalTypeId::LIST == arguments[0]->return_type.id());
+	auto list_child_type = ListType::GetChildType(arguments[0]->return_type);
+	bound_function.return_type = list_child_type;
+
+	if (!arguments[1]->IsFoldable()) {
+		throw InvalidInputException("Aggregate function name must be a constant");
+	}
+
+	// get the function name
+	Value function_value = ExpressionExecutor::EvaluateScalar(*arguments[1]);
+	auto function_name = StringValue::Get(function_value);
+
+	vector<LogicalType> types;
+	types.push_back(list_child_type);
+
+	// create the child expression and its type
+	vector<unique_ptr<Expression>> children;
+	auto expr = make_unique<BoundConstantExpression>(Value(LogicalType::SQLNULL));
+	expr->return_type = list_child_type;
+	children.push_back(move(expr));
+
+	// look up the aggregate function in the catalog
+	QueryErrorContext error_context(nullptr, 0);
+	auto func = (AggregateFunctionCatalogEntry *)Catalog::GetCatalog(context).GetEntry<AggregateFunctionCatalogEntry>(
+	    context, DEFAULT_SCHEMA, function_name, false, error_context);
+	D_ASSERT(func->type == CatalogType::AGGREGATE_FUNCTION_ENTRY);
+
+	// find a matching aggregate function
+	string error;
+	auto best_function_idx = Function::BindFunction(func->name, func->functions, types, error);
+	if (best_function_idx == DConstants::INVALID_INDEX) {
+		throw BinderException("No matching aggregate function");
+	}
+
+	// found a matching function, bind it as an aggregate
+	auto &best_function = func->functions[best_function_idx];
+	auto bound_aggr_function = AggregateFunction::BindAggregateFunction(context, best_function, move(children));
+
+	bound_function.arguments[0] =
+	    LogicalType::LIST(bound_aggr_function->function.arguments[0]); // for proper casting of the vectors
+	bound_function.return_type = bound_aggr_function->function.return_type;
+	return make_unique<ListAggregatesBindData>(bound_function.return_type, move(bound_aggr_function));
+}
+
+ScalarFunction ListAggregateFun::GetFunction() {
+	return ScalarFunction({LogicalType::LIST(LogicalType::ANY), LogicalType::VARCHAR}, LogicalType::ANY,
+	                      ListAggregateFunction, false, ListAggregateBind, nullptr, nullptr, nullptr);
+}
+
+void ListAggregateFun::RegisterFunction(BuiltinFunctions &set) {
+	set.AddFunction({"list_aggregate", "array_aggregate", "list_aggr", "array_aggr"}, GetFunction());
+}
+
+} // namespace duckdb
+
+
+
+
+
+
+
 namespace duckdb {
 
 static void ListConcatFunction(DataChunk &args, ExpressionState &state, Vector &result) {
@@ -89027,6 +89290,7 @@ void BuiltinFunctions::RegisterNestedFunctions() {
 	Register<ListConcatFun>();
 	Register<ListContainsFun>();
 	Register<ListPositionFun>();
+	Register<ListAggregateFun>();
 	Register<ListValueFun>();
 	Register<ListExtractFun>();
 	Register<ListRangeFun>();

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "452b83160"
-#define DUCKDB_VERSION "v0.3.3-dev1312"
+#define DUCKDB_SOURCE_ID "9d561d4d9"
+#define DUCKDB_VERSION "v0.3.3-dev1350"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //