duckdb 0.8.2-dev3007.0 → 0.8.2-dev3079.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.8.2-dev3007.0",
+  "version": "0.8.2-dev3079.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp

@@ -468,11 +468,12 @@ struct SortedAggregateFunction {
 
 					// These are all simple updates, so use it if available
 					if (simple_update) {
-						simple_update(sliced.data.data(), aggr_bind_info, 1, agg_state.data(), sliced.size());
+						simple_update(sliced.data.data(), aggr_bind_info, sliced.data.size(), agg_state.data(),
+						              sliced.size());
 					} else {
 						// We are only updating a constant state
 						agg_state_vec.SetVectorType(VectorType::CONSTANT_VECTOR);
-						update(sliced.data.data(), aggr_bind_info, 1, agg_state_vec, sliced.size());
+						update(sliced.data.data(), aggr_bind_info, sliced.data.size(), agg_state_vec, sliced.size());
 					}
 
 					consumed += input_count;

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.8.2-dev3007"
+#define DUCKDB_VERSION "0.8.2-dev3079"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "dd7f0c0870"
+#define DUCKDB_SOURCE_ID "3cc87593cf"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/main/extension_util.hpp

@@ -23,6 +23,10 @@ public:
 	DUCKDB_API static void RegisterFunction(DatabaseInstance &db, ScalarFunction function);
 	//! Register a new scalar function set - throw an exception if the function already exists
 	DUCKDB_API static void RegisterFunction(DatabaseInstance &db, ScalarFunctionSet function);
+	//! Register a new aggregate function - throw an exception if the function already exists
+	DUCKDB_API static void RegisterFunction(DatabaseInstance &db, AggregateFunction function);
+	//! Register a new aggregate function set - throw an exception if the function already exists
+	DUCKDB_API static void RegisterFunction(DatabaseInstance &db, AggregateFunctionSet function);
 	//! Register a new table function - throw an exception if the function already exists
 	DUCKDB_API static void RegisterFunction(DatabaseInstance &db, TableFunction function);
 	//! Register a new table function set - throw an exception if the function already exists

package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp

@@ -70,6 +70,10 @@ private:
 	unique_ptr<LogicalOperator> PushdownSingleJoin(unique_ptr<LogicalOperator> op, unordered_set<idx_t> &left_bindings,
 	                                               unordered_set<idx_t> &right_bindings);
 
+	// AddLogicalFilter used to add an extra LogicalFilter at this level,
+	// because in some cases, some expressions can not be pushed down.
+	unique_ptr<LogicalOperator> AddLogicalFilter(unique_ptr<LogicalOperator> op,
+	                                             vector<unique_ptr<Expression>> expressions);
 	//! Push any remaining filters into a LogicalFilter at this level
 	unique_ptr<LogicalOperator> PushFinalFilters(unique_ptr<LogicalOperator> op);
 	// Finish pushing down at this operator, creating a LogicalFilter to store any of the stored filters and recursively

package/src/duckdb/src/main/extension/extension_util.cpp

@@ -1,5 +1,6 @@
 #include "duckdb/main/extension_util.hpp"
 #include "duckdb/function/scalar_function.hpp"
+#include "duckdb/parser/parsed_data/create_aggregate_function_info.hpp"
 #include "duckdb/parser/parsed_data/create_type_info.hpp"
 #include "duckdb/parser/parsed_data/create_copy_function_info.hpp"
 #include "duckdb/parser/parsed_data/create_pragma_function_info.hpp"
@@ -26,6 +27,21 @@ void ExtensionUtil::RegisterFunction(DatabaseInstance &db, ScalarFunction functi
 	RegisterFunction(db, std::move(set));
 }
 
+void ExtensionUtil::RegisterFunction(DatabaseInstance &db, AggregateFunction function) {
+	D_ASSERT(!function.name.empty());
+	AggregateFunctionSet set(function.name);
+	set.AddFunction(std::move(function));
+	RegisterFunction(db, std::move(set));
+}
+
+void ExtensionUtil::RegisterFunction(DatabaseInstance &db, AggregateFunctionSet set) {
+	D_ASSERT(!set.name.empty());
+	CreateAggregateFunctionInfo info(std::move(set));
+	auto &system_catalog = Catalog::GetSystemCatalog(db);
+	auto data = CatalogTransaction::GetSystemTransaction(db);
+	system_catalog.CreateFunction(data, info);
+}
+
 void ExtensionUtil::RegisterFunction(DatabaseInstance &db, TableFunction function) {
 	D_ASSERT(!function.name.empty());
 	TableFunctionSet set(function.name);

package/src/duckdb/src/optimizer/filter_pushdown.cpp

@@ -116,19 +116,27 @@ void FilterPushdown::GenerateFilters() {
 	});
 }
 
-unique_ptr<LogicalOperator> FilterPushdown::PushFinalFilters(unique_ptr<LogicalOperator> op) {
-	if (filters.empty()) {
-		// no filters to push
+unique_ptr<LogicalOperator> FilterPushdown::AddLogicalFilter(unique_ptr<LogicalOperator> op,
+                                                             vector<unique_ptr<Expression>> expressions) {
+	if (expressions.empty()) {
+		// No left expressions, so needn't to add an extra filter operator.
 		return op;
 	}
 	auto filter = make_uniq<LogicalFilter>();
-	for (auto &f : filters) {
-		filter->expressions.push_back(std::move(f->filter));
-	}
+	filter->expressions = std::move(expressions);
 	filter->children.push_back(std::move(op));
 	return std::move(filter);
 }
 
+unique_ptr<LogicalOperator> FilterPushdown::PushFinalFilters(unique_ptr<LogicalOperator> op) {
+	vector<unique_ptr<Expression>> expressions;
+	for (auto &f : filters) {
+		expressions.push_back(std::move(f->filter));
+	}
+
+	return AddLogicalFilter(std::move(op), std::move(expressions));
+}
+
 unique_ptr<LogicalOperator> FilterPushdown::FinishPushdown(unique_ptr<LogicalOperator> op) {
 	// unhandled type, first perform filter pushdown in its children
 	for (auto &child : op->children) {

package/src/duckdb/src/optimizer/pushdown/pushdown_projection.cpp

@@ -6,6 +6,23 @@
 
 namespace duckdb {
 
+static bool HasSideEffects(LogicalProjection &proj, const unique_ptr<Expression> &expr) {
+	if (expr->type == ExpressionType::BOUND_COLUMN_REF) {
+		auto &colref = expr->Cast<BoundColumnRefExpression>();
+		D_ASSERT(colref.binding.table_index == proj.table_index);
+		D_ASSERT(colref.binding.column_index < proj.expressions.size());
+		D_ASSERT(colref.depth == 0);
+		if (proj.expressions[colref.binding.column_index]->HasSideEffects()) {
+			return true;
+		}
+		return false;
+	}
+	bool has_side_effects = false;
+	ExpressionIterator::EnumerateChildren(
+	    *expr, [&](unique_ptr<Expression> &child) { has_side_effects |= HasSideEffects(proj, child); });
+	return has_side_effects;
+}
+
 static unique_ptr<Expression> ReplaceProjectionBindings(LogicalProjection &proj, unique_ptr<Expression> expr) {
 	if (expr->type == ExpressionType::BOUND_COLUMN_REF) {
 		auto &colref = expr->Cast<BoundColumnRefExpression>();
@@ -27,15 +44,25 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownProjection(unique_ptr<Logica
 	// all the BoundColumnRefExpressions in the filter should refer to the LogicalProjection
 	// we can rewrite them by replacing those references with the expression of the LogicalProjection node
 	FilterPushdown child_pushdown(optimizer);
+	// There are some expressions can not be pushed down. We should keep them
+	// and add an extra filter operator.
+	vector<unique_ptr<Expression>> remain_expressions;
 	for (auto &filter : filters) {
 		auto &f = *filter;
 		D_ASSERT(f.bindings.size() <= 1);
-		// rewrite the bindings within this subquery
-		f.filter = ReplaceProjectionBindings(proj, std::move(f.filter));
-		// add the filter to the child pushdown
-		if (child_pushdown.AddFilter(std::move(f.filter)) == FilterResult::UNSATISFIABLE) {
-			// filter statically evaluates to false, strip tree
-			return make_uniq<LogicalEmptyResult>(std::move(op));
+		bool has_side_effects = HasSideEffects(proj, f.filter);
+		if (has_side_effects) {
+			// We can't push down related expressions if the column in the
+			// expression is generated by the functions which have side effects
+			remain_expressions.push_back(std::move(f.filter));
+		} else {
+			// rewrite the bindings within this subquery
+			f.filter = ReplaceProjectionBindings(proj, std::move(f.filter));
+			// add the filter to the child pushdown
+			if (child_pushdown.AddFilter(std::move(f.filter)) == FilterResult::UNSATISFIABLE) {
+				// filter statically evaluates to false, strip tree
+				return make_uniq<LogicalEmptyResult>(std::move(op));
+			}
 		}
 	}
 	child_pushdown.GenerateFilters();
@@ -45,7 +72,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownProjection(unique_ptr<Logica
 		// child returns an empty result: generate an empty result here too
 		return make_uniq<LogicalEmptyResult>(std::move(op));
 	}
-	return op;
+	return AddLogicalFilter(std::move(op), std::move(remain_expressions));
 }
 
 } // namespace duckdb

package/test/extension.test.ts

@@ -37,7 +37,7 @@ const test_httpfs = async function (db: duckdb.Database) {
         }));
     await chai.assert.isRejected(promise, 'IO Error: Connection error for HTTP HEAD');
 
-    await new Promise<void>((resolve, reject) => db.all("SELECT id, first_name, last_name FROM PARQUET_SCAN('https://raw.githubusercontent.com/cwida/duckdb/master/data/parquet-testing/userdata1.parquet') LIMIT 3;", function (err: null | Error, rows: TableData) {
+    await new Promise<void>((resolve, reject) => db.all("SELECT id, first_name, last_name FROM PARQUET_SCAN('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/userdata1.parquet') LIMIT 3;", function (err: null | Error, rows: TableData) {
         if (err) {
             if (err.message.startsWith("Unable to connect to URL")) {
                 console.warn("Warning: HTTP request failed in extension.test.js");