duckdb 0.8.2-dev3055.0 → 0.8.2-dev3092.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.8.2-dev3055.0",
+  "version": "0.8.2-dev3092.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/function/scalar/operators/add.cpp

@@ -160,6 +160,15 @@ bool TryAddOperator::Operation(int64_t left, int64_t right, int64_t &result) {
 	return true;
 }
 
+template <>
+bool TryAddOperator::Operation(hugeint_t left, hugeint_t right, hugeint_t &result) {
+	if (!Hugeint::AddInPlace(left, right)) {
+		return false;
+	}
+	result = left;
+	return true;
+}
+
 //===--------------------------------------------------------------------===//
 // add decimal with overflow check
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/function/scalar/operators/arithmetic.cpp

@@ -35,6 +35,9 @@ static scalar_function_t GetScalarIntegerFunction(PhysicalType type) {
 	case PhysicalType::INT64:
 		function = &ScalarFunction::BinaryFunction<int64_t, int64_t, int64_t, OP>;
 		break;
+	case PhysicalType::INT128:
+		function = &ScalarFunction::BinaryFunction<hugeint_t, hugeint_t, hugeint_t, OP>;
+		break;
 	case PhysicalType::UINT8:
 		function = &ScalarFunction::BinaryFunction<uint8_t, uint8_t, uint8_t, OP>;
 		break;
@@ -302,7 +305,7 @@ ScalarFunction AddFun::GetFunction(const LogicalType &left_type, const LogicalTy
 			function.serialize = SerializeDecimalArithmetic;
 			function.deserialize = DeserializeDecimalArithmetic<AddOperator, DecimalAddOverflowCheck>;
 			return function;
-		} else if (left_type.IsIntegral() && left_type.id() != LogicalTypeId::HUGEINT) {
+		} else if (left_type.IsIntegral()) {
 			return ScalarFunction("+", {left_type, right_type}, left_type,
 			                      GetScalarIntegerFunction<AddOperatorOverflowCheck>(left_type.InternalType()), nullptr,
 			                      nullptr, PropagateNumericStats<TryAddOperator, AddPropagateStatistics, AddOperator>);
@@ -564,7 +567,7 @@ ScalarFunction SubtractFun::GetFunction(const LogicalType &left_type, const Logi
 			function.serialize = SerializeDecimalArithmetic;
 			function.deserialize = DeserializeDecimalArithmetic<SubtractOperator, DecimalSubtractOverflowCheck>;
 			return function;
-		} else if (left_type.IsIntegral() && left_type.id() != LogicalTypeId::HUGEINT) {
+		} else if (left_type.IsIntegral()) {
 			return ScalarFunction(
 			    "-", {left_type, right_type}, left_type,
 			    GetScalarIntegerFunction<SubtractOperatorOverflowCheck>(left_type.InternalType()), nullptr, nullptr,
@@ -768,7 +771,7 @@ void MultiplyFun::RegisterFunction(BuiltinFunctions &set) {
 			function.serialize = SerializeDecimalArithmetic;
 			function.deserialize = DeserializeDecimalArithmetic<MultiplyOperator, DecimalMultiplyOverflowCheck>;
 			functions.AddFunction(function);
-		} else if (TypeIsIntegral(type.InternalType()) && type.id() != LogicalTypeId::HUGEINT) {
+		} else if (TypeIsIntegral(type.InternalType())) {
 			functions.AddFunction(ScalarFunction(
 			    {type, type}, type, GetScalarIntegerFunction<MultiplyOperatorOverflowCheck>(type.InternalType()),
 			    nullptr, nullptr,

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.8.2-dev3055"
+#define DUCKDB_VERSION "0.8.2-dev3092"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "b105af71bb"
+#define DUCKDB_SOURCE_ID "1f17e1cec6"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/operator/add.hpp

@@ -11,6 +11,7 @@
 #include "duckdb/common/types.hpp"
 #include "duckdb/common/type_util.hpp"
 #include "duckdb/common/exception.hpp"
+#include "duckdb/common/types/cast_helpers.hpp"
 
 namespace duckdb {
 
@@ -68,14 +69,16 @@ template <>
 bool TryAddOperator::Operation(int32_t left, int32_t right, int32_t &result);
 template <>
 DUCKDB_API bool TryAddOperator::Operation(int64_t left, int64_t right, int64_t &result);
+template <>
+bool TryAddOperator::Operation(hugeint_t left, hugeint_t right, hugeint_t &result);
 
 struct AddOperatorOverflowCheck {
 	template <class TA, class TB, class TR>
 	static inline TR Operation(TA left, TB right) {
 		TR result;
 		if (!TryAddOperator::Operation(left, right, result)) {
-			throw OutOfRangeException("Overflow in addition of %s (%d + %d)!", TypeIdToString(GetTypeId<TA>()), left,
-			                          right);
+			throw OutOfRangeException("Overflow in addition of %s (%s + %s)!", TypeIdToString(GetTypeId<TA>()),
+			                          NumericHelper::ToString(left), NumericHelper::ToString(right));
 		}
 		return result;
 	}

package/src/duckdb/src/include/duckdb/common/operator/multiply.hpp

@@ -11,6 +11,7 @@
 #include "duckdb/common/types.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/type_util.hpp"
+#include "duckdb/common/types/cast_helpers.hpp"
 
 namespace duckdb {
 
@@ -64,8 +65,8 @@ struct MultiplyOperatorOverflowCheck {
 	static inline TR Operation(TA left, TB right) {
 		TR result;
 		if (!TryMultiplyOperator::Operation(left, right, result)) {
-			throw OutOfRangeException("Overflow in multiplication of %s (%d * %d)!", TypeIdToString(GetTypeId<TA>()),
-			                          left, right);
+			throw OutOfRangeException("Overflow in multiplication of %s (%s * %s)!", TypeIdToString(GetTypeId<TA>()),
+			                          NumericHelper::ToString(left), NumericHelper::ToString(right));
 		}
 		return result;
 	}

package/src/duckdb/src/include/duckdb/common/operator/subtract.hpp

@@ -11,6 +11,7 @@
 #include "duckdb/common/types.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/type_util.hpp"
+#include "duckdb/common/types/cast_helpers.hpp"
 
 namespace duckdb {
 
@@ -75,8 +76,8 @@ struct SubtractOperatorOverflowCheck {
 	static inline TR Operation(TA left, TB right) {
 		TR result;
 		if (!TrySubtractOperator::Operation(left, right, result)) {
-			throw OutOfRangeException("Overflow in subtraction of %s (%d - %d)!", TypeIdToString(GetTypeId<TA>()), left,
-			                          right);
+			throw OutOfRangeException("Overflow in subtraction of %s (%s - %s)!", TypeIdToString(GetTypeId<TA>()),
+			                          NumericHelper::ToString(left), NumericHelper::ToString(right));
 		}
 		return result;
 	}

package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp

@@ -70,6 +70,10 @@ private:
 	unique_ptr<LogicalOperator> PushdownSingleJoin(unique_ptr<LogicalOperator> op, unordered_set<idx_t> &left_bindings,
 	                                               unordered_set<idx_t> &right_bindings);
 
+	// AddLogicalFilter used to add an extra LogicalFilter at this level,
+	// because in some cases, some expressions can not be pushed down.
+	unique_ptr<LogicalOperator> AddLogicalFilter(unique_ptr<LogicalOperator> op,
+	                                             vector<unique_ptr<Expression>> expressions);
 	//! Push any remaining filters into a LogicalFilter at this level
 	unique_ptr<LogicalOperator> PushFinalFilters(unique_ptr<LogicalOperator> op);
 	// Finish pushing down at this operator, creating a LogicalFilter to store any of the stored filters and recursively

package/src/duckdb/src/optimizer/filter_pushdown.cpp

@@ -116,19 +116,27 @@ void FilterPushdown::GenerateFilters() {
 	});
 }
 
-unique_ptr<LogicalOperator> FilterPushdown::PushFinalFilters(unique_ptr<LogicalOperator> op) {
-	if (filters.empty()) {
-		// no filters to push
+unique_ptr<LogicalOperator> FilterPushdown::AddLogicalFilter(unique_ptr<LogicalOperator> op,
+                                                             vector<unique_ptr<Expression>> expressions) {
+	if (expressions.empty()) {
+		// No left expressions, so needn't to add an extra filter operator.
 		return op;
 	}
 	auto filter = make_uniq<LogicalFilter>();
-	for (auto &f : filters) {
-		filter->expressions.push_back(std::move(f->filter));
-	}
+	filter->expressions = std::move(expressions);
 	filter->children.push_back(std::move(op));
 	return std::move(filter);
 }
 
+unique_ptr<LogicalOperator> FilterPushdown::PushFinalFilters(unique_ptr<LogicalOperator> op) {
+	vector<unique_ptr<Expression>> expressions;
+	for (auto &f : filters) {
+		expressions.push_back(std::move(f->filter));
+	}
+
+	return AddLogicalFilter(std::move(op), std::move(expressions));
+}
+
 unique_ptr<LogicalOperator> FilterPushdown::FinishPushdown(unique_ptr<LogicalOperator> op) {
 	// unhandled type, first perform filter pushdown in its children
 	for (auto &child : op->children) {

package/src/duckdb/src/optimizer/pushdown/pushdown_projection.cpp

@@ -6,6 +6,23 @@
 
 namespace duckdb {
 
+static bool HasSideEffects(LogicalProjection &proj, const unique_ptr<Expression> &expr) {
+	if (expr->type == ExpressionType::BOUND_COLUMN_REF) {
+		auto &colref = expr->Cast<BoundColumnRefExpression>();
+		D_ASSERT(colref.binding.table_index == proj.table_index);
+		D_ASSERT(colref.binding.column_index < proj.expressions.size());
+		D_ASSERT(colref.depth == 0);
+		if (proj.expressions[colref.binding.column_index]->HasSideEffects()) {
+			return true;
+		}
+		return false;
+	}
+	bool has_side_effects = false;
+	ExpressionIterator::EnumerateChildren(
+	    *expr, [&](unique_ptr<Expression> &child) { has_side_effects |= HasSideEffects(proj, child); });
+	return has_side_effects;
+}
+
 static unique_ptr<Expression> ReplaceProjectionBindings(LogicalProjection &proj, unique_ptr<Expression> expr) {
 	if (expr->type == ExpressionType::BOUND_COLUMN_REF) {
 		auto &colref = expr->Cast<BoundColumnRefExpression>();
@@ -27,15 +44,25 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownProjection(unique_ptr<Logica
 	// all the BoundColumnRefExpressions in the filter should refer to the LogicalProjection
 	// we can rewrite them by replacing those references with the expression of the LogicalProjection node
 	FilterPushdown child_pushdown(optimizer);
+	// There are some expressions can not be pushed down. We should keep them
+	// and add an extra filter operator.
+	vector<unique_ptr<Expression>> remain_expressions;
 	for (auto &filter : filters) {
 		auto &f = *filter;
 		D_ASSERT(f.bindings.size() <= 1);
-		// rewrite the bindings within this subquery
-		f.filter = ReplaceProjectionBindings(proj, std::move(f.filter));
-		// add the filter to the child pushdown
-		if (child_pushdown.AddFilter(std::move(f.filter)) == FilterResult::UNSATISFIABLE) {
-			// filter statically evaluates to false, strip tree
-			return make_uniq<LogicalEmptyResult>(std::move(op));
+		bool has_side_effects = HasSideEffects(proj, f.filter);
+		if (has_side_effects) {
+			// We can't push down related expressions if the column in the
+			// expression is generated by the functions which have side effects
+			remain_expressions.push_back(std::move(f.filter));
+		} else {
+			// rewrite the bindings within this subquery
+			f.filter = ReplaceProjectionBindings(proj, std::move(f.filter));
+			// add the filter to the child pushdown
+			if (child_pushdown.AddFilter(std::move(f.filter)) == FilterResult::UNSATISFIABLE) {
+				// filter statically evaluates to false, strip tree
+				return make_uniq<LogicalEmptyResult>(std::move(op));
+			}
 		}
 	}
 	child_pushdown.GenerateFilters();
@@ -45,7 +72,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownProjection(unique_ptr<Logica
 		// child returns an empty result: generate an empty result here too
 		return make_uniq<LogicalEmptyResult>(std::move(op));
 	}
-	return op;
+	return AddLogicalFilter(std::move(op), std::move(remain_expressions));
 }
 
 } // namespace duckdb

package/test/extension.test.ts

@@ -37,7 +37,7 @@ const test_httpfs = async function (db: duckdb.Database) {
         }));
     await chai.assert.isRejected(promise, 'IO Error: Connection error for HTTP HEAD');
 
-    await new Promise<void>((resolve, reject) => db.all("SELECT id, first_name, last_name FROM PARQUET_SCAN('https://raw.githubusercontent.com/cwida/duckdb/master/data/parquet-testing/userdata1.parquet') LIMIT 3;", function (err: null | Error, rows: TableData) {
+    await new Promise<void>((resolve, reject) => db.all("SELECT id, first_name, last_name FROM PARQUET_SCAN('https://raw.githubusercontent.com/duckdb/duckdb/main/data/parquet-testing/userdata1.parquet') LIMIT 3;", function (err: null | Error, rows: TableData) {
         if (err) {
             if (err.message.startsWith("Unable to connect to URL")) {
                 console.warn("Warning: HTTP request failed in extension.test.js");