duckdb 0.5.2-dev833.0 → 0.5.2-dev841.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.5.2-dev833.0",
+  "version": "0.5.2-dev841.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -146404,12 +146404,11 @@ private:
 	//! Find Joins with a DelimGet that can be removed
 	void FindCandidates(unique_ptr<LogicalOperator> *op_ptr, vector<unique_ptr<LogicalOperator> *> &candidates);
 	//! Try to remove a Join with a DelimGet, returns true if it was successful
-	bool RemoveCandidate(unique_ptr<LogicalOperator> *op_ptr, DeliminatorPlanUpdater &updater);
-	//! Replace references to a removed DelimGet, remove DelimJoins if all their DelimGets are gone
-	void UpdatePlan(LogicalOperator &op, expression_map_t<Expression *> &expr_map,
-	                column_binding_map_t<bool> &projection_map);
-	//! Whether the operator has one or more children of type DELIM_GET
-	bool HasChildDelimGet(LogicalOperator &op);
+	bool RemoveCandidate(unique_ptr<LogicalOperator> *plan, unique_ptr<LogicalOperator> *candidate,
+	                     DeliminatorPlanUpdater &updater);
+	//! Try to remove an inequality Join with a DelimGet, returns true if it was successful
+	bool RemoveInequalityCandidate(unique_ptr<LogicalOperator> *plan, unique_ptr<LogicalOperator> *candidate,
+	                               DeliminatorPlanUpdater &updater);
 };
 
 } // namespace duckdb
@@ -146433,31 +146432,64 @@ public:
 	//! Update the plan after a DelimGet has been removed
 	void VisitOperator(LogicalOperator &op) override;
 	void VisitExpression(unique_ptr<Expression> *expression) override;
-	//! Whether the operator has one or more children of type DELIM_GET
-	bool HasChildDelimGet(LogicalOperator &op);
+
+public:
 	expression_map_t<Expression *> expr_map;
 	column_binding_map_t<bool> projection_map;
+	column_binding_map_t<Expression *> reverse_proj_or_agg_map;
 	unique_ptr<LogicalOperator> temp_ptr;
 };
 
+static idx_t DelimGetCount(LogicalOperator &op) {
+	if (op.type == LogicalOperatorType::LOGICAL_DELIM_GET) {
+		return 1;
+	}
+	idx_t child_count = 0;
+	for (auto &child : op.children) {
+		child_count += DelimGetCount(*child);
+	}
+	return child_count;
+}
+
+static bool IsEqualityJoinCondition(JoinCondition &cond) {
+	switch (cond.comparison) {
+	case ExpressionType::COMPARE_EQUAL:
+	case ExpressionType::COMPARE_NOT_DISTINCT_FROM:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool InequalityDelimJoinCanBeEliminated(JoinType &join_type) {
+	switch (join_type) {
+	case JoinType::ANTI:
+	case JoinType::MARK:
+	case JoinType::SEMI:
+	case JoinType::SINGLE:
+		return true;
+	default:
+		return false;
+	}
+}
+
 void DeliminatorPlanUpdater::VisitOperator(LogicalOperator &op) {
 	VisitOperatorChildren(op);
 	VisitOperatorExpressions(op);
-	if (op.type == LogicalOperatorType::LOGICAL_DELIM_JOIN && !HasChildDelimGet(op)) {
+	if (op.type == LogicalOperatorType::LOGICAL_DELIM_JOIN && DelimGetCount(op) == 0) {
 		auto &delim_join = (LogicalDelimJoin &)op;
 		auto decs = &delim_join.duplicate_eliminated_columns;
 		for (auto &cond : delim_join.conditions) {
-			if (cond.comparison != ExpressionType::COMPARE_EQUAL &&
-			    cond.comparison != ExpressionType::COMPARE_NOT_DISTINCT_FROM) {
+			if (!IsEqualityJoinCondition(cond)) {
 				continue;
 			}
-			Expression *rhs = cond.right.get();
+			auto rhs = cond.right.get();
 			while (rhs->type == ExpressionType::OPERATOR_CAST) {
 				auto &cast = (BoundCastExpression &)*rhs;
 				rhs = cast.child.get();
 			}
 			if (rhs->type != ExpressionType::BOUND_COLUMN_REF) {
-				throw InternalException("Erorr in deliminator: expected a bound column reference");
+				throw InternalException("Error in Deliminator: expected a bound column reference");
 			}
 			auto &colref = (BoundColumnRefExpression &)*rhs;
 			if (projection_map.find(colref.binding) != projection_map.end()) {
@@ -146488,25 +146520,13 @@ void DeliminatorPlanUpdater::VisitExpression(unique_ptr<Expression> *expression)
 	}
 }
 
-bool DeliminatorPlanUpdater::HasChildDelimGet(LogicalOperator &op) {
-	if (op.type == LogicalOperatorType::LOGICAL_DELIM_GET) {
-		return true;
-	}
-	for (auto &child : op.children) {
-		if (HasChildDelimGet(*child)) {
-			return true;
-		}
-	}
-	return false;
-}
-
 unique_ptr<LogicalOperator> Deliminator::Optimize(unique_ptr<LogicalOperator> op) {
 	vector<unique_ptr<LogicalOperator> *> candidates;
 	FindCandidates(&op, candidates);
 
-	for (auto candidate : candidates) {
+	for (auto &candidate : candidates) {
 		DeliminatorPlanUpdater updater;
-		if (RemoveCandidate(candidate, updater)) {
+		if (RemoveCandidate(&op, candidate, updater)) {
 			updater.VisitOperator(*op);
 		}
 	}
@@ -146561,10 +146581,21 @@ static bool OperatorIsDelimGet(LogicalOperator &op) {
 	return false;
 }
 
-bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *op_ptr, DeliminatorPlanUpdater &updater) {
-	auto &proj_or_agg = **op_ptr;
+static bool ChildJoinTypeCanBeDeliminated(JoinType &join_type) {
+	switch (join_type) {
+	case JoinType::INNER:
+	case JoinType::SEMI:
+		return true;
+	default:
+		return false;
+	}
+}
+
+bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *plan, unique_ptr<LogicalOperator> *candidate,
+                                  DeliminatorPlanUpdater &updater) {
+	auto &proj_or_agg = **candidate;
 	auto &join = (LogicalComparisonJoin &)*proj_or_agg.children[0];
-	if (join.join_type != JoinType::INNER && join.join_type != JoinType::SEMI) {
+	if (!ChildJoinTypeCanBeDeliminated(join.join_type)) {
 		return false;
 	}
 
@@ -146582,13 +146613,10 @@ bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *op_ptr, Deliminat
 		return false;
 	}
 	// check if joining with the DelimGet is redundant, and collect relevant column information
+	bool all_equality_conditions = true;
 	vector<Expression *> nulls_are_not_equal_exprs;
 	for (auto &cond : join.conditions) {
-		if (cond.comparison != ExpressionType::COMPARE_EQUAL &&
-		    cond.comparison != ExpressionType::COMPARE_NOT_DISTINCT_FROM) {
-			// non-equality join condition
-			return false;
-		}
+		all_equality_conditions = all_equality_conditions && IsEqualityJoinCondition(cond);
 		auto delim_side = delim_idx == 0 ? cond.left.get() : cond.right.get();
 		auto other_side = delim_idx == 0 ? cond.right.get() : cond.left.get();
 		if (delim_side->type != ExpressionType::BOUND_COLUMN_REF) {
@@ -146601,10 +146629,12 @@ bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *op_ptr, Deliminat
 			nulls_are_not_equal_exprs.push_back(other_side);
 		}
 	}
+
 	// removed DelimGet columns are assigned a new ColumnBinding by Projection/Aggregation, keep track here
 	if (proj_or_agg.type == LogicalOperatorType::LOGICAL_PROJECTION) {
 		for (auto &cb : proj_or_agg.GetColumnBindings()) {
 			updater.projection_map[cb] = true;
+			updater.reverse_proj_or_agg_map[cb] = proj_or_agg.expressions[cb.column_index].get();
 			for (auto &expr : nulls_are_not_equal_exprs) {
 				if (proj_or_agg.expressions[cb.column_index]->Equals(expr)) {
 					updater.projection_map[cb] = false;
@@ -146614,8 +146644,19 @@ bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *op_ptr, Deliminat
 		}
 	} else {
 		auto &agg = (LogicalAggregate &)proj_or_agg;
+
+		// Create a vector of all exprs in the agg
+		vector<Expression *> all_agg_exprs;
+		for (auto &expr : agg.groups) {
+			all_agg_exprs.push_back(expr.get());
+		}
+		for (auto &expr : agg.expressions) {
+			all_agg_exprs.push_back(expr.get());
+		}
+
 		for (auto &cb : agg.GetColumnBindings()) {
 			updater.projection_map[cb] = true;
+			updater.reverse_proj_or_agg_map[cb] = all_agg_exprs[cb.column_index];
 			for (auto &expr : nulls_are_not_equal_exprs) {
 				if ((cb.table_index == agg.group_index && agg.groups[cb.column_index]->Equals(expr)) ||
 				    (cb.table_index == agg.aggregate_index && agg.expressions[cb.column_index]->Equals(expr))) {
@@ -146625,6 +146666,14 @@ bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *op_ptr, Deliminat
 			}
 		}
 	}
+
+	if (!all_equality_conditions) {
+		// we can get rid of an inequality join with a DelimGet, but only under specific circumstances
+		if (!RemoveInequalityCandidate(plan, candidate, updater)) {
+			return false;
+		}
+	}
+
 	// make a filter if needed
 	if (!nulls_are_not_equal_exprs.empty() || filter != nullptr) {
 		auto filter_op = make_unique<LogicalFilter>();
@@ -146652,6 +146701,146 @@ bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *op_ptr, Deliminat
 	return true;
 }
 
+static void GetDelimJoins(LogicalOperator &op, vector<LogicalOperator *> &delim_joins) {
+	for (auto &child : op.children) {
+		GetDelimJoins(*child, delim_joins);
+	}
+	if (op.type == LogicalOperatorType::LOGICAL_DELIM_JOIN) {
+		delim_joins.push_back(&op);
+	}
+}
+
+static bool HasChild(LogicalOperator *haystack, LogicalOperator *needle, idx_t &side) {
+	if (haystack == needle) {
+		return true;
+	}
+	for (idx_t i = 0; i < haystack->children.size(); i++) {
+		auto &child = haystack->children[i];
+		idx_t dummy_side;
+		if (HasChild(child.get(), needle, dummy_side)) {
+			side = i;
+			return true;
+		}
+	}
+	return false;
+}
+
+bool Deliminator::RemoveInequalityCandidate(unique_ptr<LogicalOperator> *plan, unique_ptr<LogicalOperator> *candidate,
+                                            DeliminatorPlanUpdater &updater) {
+	auto &proj_or_agg = **candidate;
+	// first, we find a DelimJoin in "plan" that has only one DelimGet as a child, which is in "candidate"
+	if (DelimGetCount(proj_or_agg) != 1) {
+		// the candidate therefore must have only a single DelimGet in its children
+		return false;
+	}
+
+	vector<LogicalOperator *> delim_joins;
+	GetDelimJoins(**plan, delim_joins);
+
+	LogicalOperator *parent = nullptr;
+	idx_t parent_delim_get_side;
+	for (auto dj : delim_joins) {
+		D_ASSERT(dj->type == LogicalOperatorType::LOGICAL_DELIM_JOIN);
+		if (!HasChild(dj, &proj_or_agg, parent_delim_get_side)) {
+			continue;
+		}
+		// we found a parent DelimJoin
+		if (DelimGetCount(*dj) != 1) {
+			// it has more than one DelimGet children
+			continue;
+		}
+
+		// we can only remove inequality join with a DelimGet if the parent DelimJoin has one of these join types
+		auto &delim_join = (LogicalDelimJoin &)*dj;
+		if (!InequalityDelimJoinCanBeEliminated(delim_join.join_type)) {
+			continue;
+		}
+
+		parent = dj;
+		break;
+	}
+	if (!parent) {
+		return false;
+	}
+
+	// we found the parent delim join, and we may be able to remove the child DelimGet join
+	// but we need to make sure that their conditions refer to exactly the same columns
+	auto &parent_delim_join = (LogicalDelimJoin &)*parent;
+	auto &join = (LogicalComparisonJoin &)*proj_or_agg.children[0];
+	if (parent_delim_join.conditions.size() != join.conditions.size()) {
+		// different number of conditions, can't replace
+		return false;
+	}
+
+	// we can only do this optimization under the following conditions:
+	// 1. all join expressions coming from the DelimGet side are colrefs
+	// 2. these expressions refer to colrefs coming from the proj/agg on top of the child DelimGet join
+	// 3. the expression (before it was proj/agg) can be found in the conditions of the child DelimGet join
+	for (auto &parent_cond : parent_delim_join.conditions) {
+		auto &parent_expr = parent_delim_get_side == 0 ? parent_cond.left : parent_cond.right;
+		if (parent_expr->type != ExpressionType::BOUND_COLUMN_REF) {
+			// can only deal with colrefs
+			return false;
+		}
+		auto &parent_colref = (BoundColumnRefExpression &)*parent_expr;
+		auto it = updater.reverse_proj_or_agg_map.find(parent_colref.binding);
+		if (it == updater.reverse_proj_or_agg_map.end()) {
+			// refers to a column that was not in the child DelimGet join
+			return false;
+		}
+		// try to find the corresponding child condition
+		// TODO: can be more flexible - allow CAST
+		auto child_expr = it->second;
+		bool found = false;
+		for (auto &child_cond : join.conditions) {
+			if (child_cond.left->Equals(child_expr) || child_cond.right->Equals(child_expr)) {
+				found = true;
+				break;
+			}
+		}
+		if (!found) {
+			// could not find the mapped expression in the child condition expressions
+			return false;
+		}
+	}
+
+	// TODO: we cannot perform the optimization here because our pure inequality joins don't implement
+	//  JoinType::SINGLE yet
+	if (parent_delim_join.join_type == JoinType::SINGLE) {
+		bool has_one_equality = false;
+		for (auto &cond : join.conditions) {
+			has_one_equality = has_one_equality || IsEqualityJoinCondition(cond);
+		}
+		if (!has_one_equality) {
+			return false;
+		}
+	}
+
+	// we are now sure that we can remove the child DelimGet join, so we basically do the same loop as above
+	// this time without checks because we already did them, and replace the expressions
+	for (auto &parent_cond : parent_delim_join.conditions) {
+		auto &parent_expr = parent_delim_get_side == 0 ? parent_cond.left : parent_cond.right;
+		auto &parent_colref = (BoundColumnRefExpression &)*parent_expr;
+		auto it = updater.reverse_proj_or_agg_map.find(parent_colref.binding);
+		auto child_expr = it->second;
+		for (auto &child_cond : join.conditions) {
+			if (!child_cond.left->Equals(child_expr) && !child_cond.right->Equals(child_expr)) {
+				continue;
+			}
+			parent_expr =
+			    make_unique<BoundColumnRefExpression>(parent_expr->alias, parent_expr->return_type, it->first);
+			parent_cond.comparison = child_cond.comparison;
+			break;
+		}
+	}
+
+	// no longer needs to be a delim join
+	parent_delim_join.duplicate_eliminated_columns.clear();
+	parent_delim_join.type = LogicalOperatorType::LOGICAL_COMPARISON_JOIN;
+
+	return true;
+}
+
 } // namespace duckdb
 
 

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "788ebe9a0"
-#define DUCKDB_VERSION "v0.5.2-dev833"
+#define DUCKDB_SOURCE_ID "c2ca23eef"
+#define DUCKDB_VERSION "v0.5.2-dev841"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //