duckdb 0.8.2-dev3055.0 → 0.8.2-dev3079.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.8.2-dev3055.0",
+  "version": "0.8.2-dev3079.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.8.2-dev3055"
+#define DUCKDB_VERSION "0.8.2-dev3079"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "b105af71bb"
+#define DUCKDB_SOURCE_ID "3cc87593cf"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp

@@ -70,6 +70,10 @@ private:
 	unique_ptr<LogicalOperator> PushdownSingleJoin(unique_ptr<LogicalOperator> op, unordered_set<idx_t> &left_bindings,
 	                                               unordered_set<idx_t> &right_bindings);
 
+	// AddLogicalFilter used to add an extra LogicalFilter at this level,
+	// because in some cases, some expressions can not be pushed down.
+	unique_ptr<LogicalOperator> AddLogicalFilter(unique_ptr<LogicalOperator> op,
+	                                             vector<unique_ptr<Expression>> expressions);
 	//! Push any remaining filters into a LogicalFilter at this level
 	unique_ptr<LogicalOperator> PushFinalFilters(unique_ptr<LogicalOperator> op);
 	// Finish pushing down at this operator, creating a LogicalFilter to store any of the stored filters and recursively

package/src/duckdb/src/optimizer/filter_pushdown.cpp

@@ -116,19 +116,27 @@ void FilterPushdown::GenerateFilters() {
 	});
 }
 
-unique_ptr<LogicalOperator> FilterPushdown::PushFinalFilters(unique_ptr<LogicalOperator> op) {
-	if (filters.empty()) {
-		// no filters to push
+unique_ptr<LogicalOperator> FilterPushdown::AddLogicalFilter(unique_ptr<LogicalOperator> op,
+                                                             vector<unique_ptr<Expression>> expressions) {
+	if (expressions.empty()) {
+		// No left expressions, so needn't to add an extra filter operator.
 		return op;
 	}
 	auto filter = make_uniq<LogicalFilter>();
-	for (auto &f : filters) {
-		filter->expressions.push_back(std::move(f->filter));
-	}
+	filter->expressions = std::move(expressions);
 	filter->children.push_back(std::move(op));
 	return std::move(filter);
 }
 
+unique_ptr<LogicalOperator> FilterPushdown::PushFinalFilters(unique_ptr<LogicalOperator> op) {
+	vector<unique_ptr<Expression>> expressions;
+	for (auto &f : filters) {
+		expressions.push_back(std::move(f->filter));
+	}
+
+	return AddLogicalFilter(std::move(op), std::move(expressions));
+}
+
 unique_ptr<LogicalOperator> FilterPushdown::FinishPushdown(unique_ptr<LogicalOperator> op) {
 	// unhandled type, first perform filter pushdown in its children
 	for (auto &child : op->children) {

package/src/duckdb/src/optimizer/pushdown/pushdown_projection.cpp

@@ -6,6 +6,23 @@
 
 namespace duckdb {
 
+static bool HasSideEffects(LogicalProjection &proj, const unique_ptr<Expression> &expr) {
+	if (expr->type == ExpressionType::BOUND_COLUMN_REF) {
+		auto &colref = expr->Cast<BoundColumnRefExpression>();
+		D_ASSERT(colref.binding.table_index == proj.table_index);
+		D_ASSERT(colref.binding.column_index < proj.expressions.size());
+		D_ASSERT(colref.depth == 0);
+		if (proj.expressions[colref.binding.column_index]->HasSideEffects()) {
+			return true;
+		}
+		return false;
+	}
+	bool has_side_effects = false;
+	ExpressionIterator::EnumerateChildren(
+	    *expr, [&](unique_ptr<Expression> &child) { has_side_effects |= HasSideEffects(proj, child); });
+	return has_side_effects;
+}
+
 static unique_ptr<Expression> ReplaceProjectionBindings(LogicalProjection &proj, unique_ptr<Expression> expr) {
 	if (expr->type == ExpressionType::BOUND_COLUMN_REF) {
 		auto &colref = expr->Cast<BoundColumnRefExpression>();
@@ -27,15 +44,25 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownProjection(unique_ptr<Logica
 	// all the BoundColumnRefExpressions in the filter should refer to the LogicalProjection
 	// we can rewrite them by replacing those references with the expression of the LogicalProjection node
 	FilterPushdown child_pushdown(optimizer);
+	// There are some expressions can not be pushed down. We should keep them
+	// and add an extra filter operator.
+	vector<unique_ptr<Expression>> remain_expressions;
 	for (auto &filter : filters) {
 		auto &f = *filter;
 		D_ASSERT(f.bindings.size() <= 1);
-		// rewrite the bindings within this subquery
-		f.filter = ReplaceProjectionBindings(proj, std::move(f.filter));
-		// add the filter to the child pushdown
-		if (child_pushdown.AddFilter(std::move(f.filter)) == FilterResult::UNSATISFIABLE) {
-			// filter statically evaluates to false, strip tree
-			return make_uniq<LogicalEmptyResult>(std::move(op));
+		bool has_side_effects = HasSideEffects(proj, f.filter);
+		if (has_side_effects) {
+			// We can't push down related expressions if the column in the
+			// expression is generated by the functions which have side effects
+			remain_expressions.push_back(std::move(f.filter));
+		} else {
+			// rewrite the bindings within this subquery
+			f.filter = ReplaceProjectionBindings(proj, std::move(f.filter));
+			// add the filter to the child pushdown
+			if (child_pushdown.AddFilter(std::move(f.filter)) == FilterResult::UNSATISFIABLE) {
+				// filter statically evaluates to false, strip tree
+				return make_uniq<LogicalEmptyResult>(std::move(op));
+			}
 		}
 	}
 	child_pushdown.GenerateFilters();
@@ -45,7 +72,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownProjection(unique_ptr<Logica
 		// child returns an empty result: generate an empty result here too
 		return make_uniq<LogicalEmptyResult>(std::move(op));
 	}
-	return op;
+	return AddLogicalFilter(std::move(op), std::move(remain_expressions));
 }
 
 } // namespace duckdb

package/test/extension.test.ts

@@ -37,7 +37,7 @@ const test_httpfs = async function (db: duckdb.Database) {
         }));
     await chai.assert.isRejected(promise, 'IO Error: Connection error for HTTP HEAD');
 
-    await new Promise<void>((resolve, reject) => db.all("SELECT id, first_name, last_name FROM PARQUET_SCAN('https://raw.githubusercontent.com/cwida/duckdb/master/data/parquet-testing/userdata1.parquet') LIMIT 3;", function (err: null | Error, rows: TableData) {
+    await new Promise<void>((resolve, reject) => db.all("SELECT id, first_name, last_name FROM PARQUET_SCAN('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/userdata1.parquet') LIMIT 3;", function (err: null | Error, rows: TableData) {
         if (err) {
             if (err.message.startsWith("Unable to connect to URL")) {
                 console.warn("Warning: HTTP request failed in extension.test.js");