duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/src/optimizer/join_order/query_graph_manager.cpp

@@ -2,9 +2,8 @@
 
 #include "duckdb/common/assert.hpp"
 #include "duckdb/common/enums/join_type.hpp"
-#include "duckdb/common/string_util.hpp"
-#include "duckdb/execution/physical_plan_generator.hpp"
 #include "duckdb/optimizer/join_order/join_relation.hpp"
+#include "duckdb/planner/expression/bound_conjunction_expression.hpp"
 #include "duckdb/planner/expression/bound_comparison_expression.hpp"
 #include "duckdb/planner/expression_iterator.hpp"
 #include "duckdb/planner/logical_operator.hpp"
@@ -20,10 +19,10 @@ static bool Disjoint(const unordered_set<T> &a, const unordered_set<T> &b) {
 	});
 }
 
-bool QueryGraphManager::Build(LogicalOperator &op) {
+bool QueryGraphManager::Build(JoinOrderOptimizer &optimizer, LogicalOperator &op) {
 	// have the relation manager extract the join relations and create a reference list of all the
 	// filter operators.
-	auto can_reorder = relation_manager.ExtractJoinRelations(op, filter_operators);
+	auto can_reorder = relation_manager.ExtractJoinRelations(optimizer, op, filter_operators);
 	auto num_relations = relation_manager.NumRelations();
 	if (num_relations <= 1 || !can_reorder) {
 		// nothing to optimize/reorder
@@ -57,6 +56,14 @@ const vector<unique_ptr<FilterInfo>> &QueryGraphManager::GetFilterBindings() con
 	return filters_and_bindings;
 }
 
+void FilterInfo::SetLeftSet(optional_ptr<JoinRelationSet> left_set_new) {
+	left_set = left_set_new;
+}
+
+void FilterInfo::SetRightSet(optional_ptr<JoinRelationSet> right_set_new) {
+	right_set = right_set_new;
+}
+
 static unique_ptr<LogicalOperator> PushFilter(unique_ptr<LogicalOperator> node, unique_ptr<Expression> expr) {
 	// push an expression into a filter
 	// first check if we have any filter to push it into
@@ -89,8 +96,12 @@ void QueryGraphManager::CreateHyperGraphEdges() {
 			if (!left_bindings.empty() && !right_bindings.empty()) {
 				// both the left and the right side have bindings
 				// first create the relation sets, if they do not exist
-				filter_info->left_set = &set_manager.GetJoinRelation(left_bindings);
-				filter_info->right_set = &set_manager.GetJoinRelation(right_bindings);
+				if (!filter_info->left_set) {
+					filter_info->left_set = &set_manager.GetJoinRelation(left_bindings);
+				}
+				if (!filter_info->right_set) {
+					filter_info->right_set = &set_manager.GetJoinRelation(right_bindings);
+				}
 				// we can only create a meaningful edge if the sets are not exactly the same
 				if (filter_info->left_set != filter_info->right_set) {
 					// check if the sets are disjoint
@@ -98,11 +109,50 @@ void QueryGraphManager::CreateHyperGraphEdges() {
 						// they are disjoint, we only need to create one set of edges in the join graph
 						query_graph.CreateEdge(*filter_info->left_set, *filter_info->right_set, filter_info);
 						query_graph.CreateEdge(*filter_info->right_set, *filter_info->left_set, filter_info);
-					} else {
-						continue;
 					}
+				}
+			}
+		} else if (filter->GetExpressionClass() == ExpressionClass::BOUND_CONJUNCTION) {
+			auto &conjunction = filter->Cast<BoundConjunctionExpression>();
+			if (conjunction.type == ExpressionType::CONJUNCTION_OR || filter_info->join_type == JoinType::INNER ||
+			    filter_info->join_type == JoinType::INVALID) {
+				// Currently we do not interpret Conjunction expressions as INNER joins
+				// for hyper graph edges. These are most likely OR conjunctions, and
+				// will be pushed down into a join later in the optimizer.
+				// Conjunction filters are mostly to help plan semi and anti joins at the moment.
+				continue;
+			}
+			unordered_set<idx_t> left_bindings, right_bindings;
+			D_ASSERT(filter_info->left_set);
+			D_ASSERT(filter_info->right_set);
+			D_ASSERT(filter_info->join_type == JoinType::SEMI || filter_info->join_type == JoinType::ANTI);
+			for (auto &child_comp : conjunction.children) {
+				if (child_comp->expression_class != ExpressionClass::BOUND_COMPARISON) {
 					continue;
 				}
+				auto &comparison = child_comp->Cast<BoundComparisonExpression>();
+				// extract the bindings that are required for the left and right side of the comparison
+				relation_manager.ExtractBindings(*comparison.left, left_bindings);
+				relation_manager.ExtractBindings(*comparison.right, right_bindings);
+				if (filter_info->left_binding.table_index == DConstants::INVALID_INDEX &&
+				    filter_info->left_binding.column_index == DConstants::INVALID_INDEX) {
+					GetColumnBinding(*comparison.left, filter_info->left_binding);
+				}
+				if (filter_info->right_binding.table_index == DConstants::INVALID_INDEX &&
+				    filter_info->right_binding.column_index == DConstants::INVALID_INDEX) {
+					GetColumnBinding(*comparison.right, filter_info->right_binding);
+				}
+			}
+			if (!left_bindings.empty() && !right_bindings.empty()) {
+				// we can only create a meaningful edge if the sets are not exactly the same
+				if (filter_info->left_set != filter_info->right_set) {
+					// check if the sets are disjoint
+					if (Disjoint(left_bindings, right_bindings)) {
+						// they are disjoint, we only need to create one set of edges in the join graph
+						query_graph.CreateEdge(*filter_info->left_set, *filter_info->right_set, filter_info);
+						query_graph.CreateEdge(*filter_info->right_set, *filter_info->left_set, filter_info);
+					}
+				}
 			}
 		}
 	}
@@ -171,6 +221,19 @@ unique_ptr<LogicalOperator> QueryGraphManager::Reconstruct(unique_ptr<LogicalOpe
 	return plan;
 }
 
+static JoinCondition MaybeInvertConditions(unique_ptr<Expression> condition, bool invert) {
+	auto &comparison = condition->Cast<BoundComparisonExpression>();
+	JoinCondition cond;
+	cond.left = !invert ? std::move(comparison.left) : std::move(comparison.right);
+	cond.right = !invert ? std::move(comparison.right) : std::move(comparison.left);
+	cond.comparison = condition->type;
+	if (invert) {
+		// reverse comparison expression if we reverse the order of the children
+		cond.comparison = FlipComparisonExpression(cond.comparison);
+	}
+	return cond;
+}
+
 GenerateJoinRelation QueryGraphManager::GenerateJoins(vector<unique_ptr<LogicalOperator>> &extracted_relations,
                                                       JoinRelationSet &set) {
 	optional_ptr<JoinRelationSet> left_node;
@@ -193,7 +256,15 @@ GenerateJoinRelation QueryGraphManager::GenerateJoins(vector<unique_ptr<LogicalO
 			result_operator = LogicalCrossProduct::Create(std::move(left.op), std::move(right.op));
 		} else {
 			// we have filters, create a join node
-			auto join = make_uniq<LogicalComparisonJoin>(JoinType::INNER);
+			auto chosen_filter = node->info->filters.at(0);
+			for (idx_t i = 0; i < node->info->filters.size(); i++) {
+				if (node->info->filters.at(i)->join_type == JoinType::INNER) {
+					chosen_filter = node->info->filters.at(i);
+					break;
+				}
+			}
+
+			auto join = make_uniq<LogicalComparisonJoin>(chosen_filter->join_type);
 			// Here we optimize build side probe side. Our build side is the right side
 			// So the right plans should have lower cardinalities.
 			join->children.push_back(std::move(left.op));
@@ -211,21 +282,26 @@ GenerateJoinRelation QueryGraphManager::GenerateJoins(vector<unique_ptr<LogicalO
 				          JoinRelationSet::IsSubset(*right.set, *f->right_set)) ||
 				         (JoinRelationSet::IsSubset(*left.set, *f->right_set) &&
 				          JoinRelationSet::IsSubset(*right.set, *f->left_set)));
-				JoinCondition cond;
-				D_ASSERT(condition->GetExpressionClass() == ExpressionClass::BOUND_COMPARISON);
-				auto &comparison = condition->Cast<BoundComparisonExpression>();
 
-				// we need to figure out which side is which by looking at the relations available to us
 				bool invert = !JoinRelationSet::IsSubset(*left.set, *f->left_set);
-				cond.left = !invert ? std::move(comparison.left) : std::move(comparison.right);
-				cond.right = !invert ? std::move(comparison.right) : std::move(comparison.left);
-				cond.comparison = condition->type;
+				// If the left and right set are inverted AND it is a semi or anti join
+				// swap left and right children back.
+				if (invert && (f->join_type == JoinType::SEMI || f->join_type == JoinType::ANTI)) {
+					std::swap(left, right);
+					invert = false;
+				}
 
-				if (invert) {
-					// reverse comparison expression if we reverse the order of the children
-					cond.comparison = FlipComparisonExpression(cond.comparison);
+				if (condition->GetExpressionClass() == ExpressionClass::BOUND_COMPARISON) {
+					auto cond = MaybeInvertConditions(std::move(condition), invert);
+					join->conditions.push_back(std::move(cond));
+				} else if (condition->GetExpressionClass() == ExpressionClass::BOUND_CONJUNCTION) {
+					auto &conjunction = condition->Cast<BoundConjunctionExpression>();
+					for (auto &child : conjunction.children) {
+						D_ASSERT(child->GetExpressionClass() == ExpressionClass::BOUND_COMPARISON);
+						auto cond = MaybeInvertConditions(std::move(child), invert);
+						join->conditions.push_back(std::move(cond));
+					}
 				}
-				join->conditions.push_back(std::move(cond));
 			}
 			D_ASSERT(!join->conditions.empty());
 			result_operator = std::move(join);
@@ -246,14 +322,6 @@ GenerateJoinRelation QueryGraphManager::GenerateJoins(vector<unique_ptr<LogicalO
 	//	result_operator->estimated_props = node.estimated_props->Copy();
 	result_operator->estimated_cardinality = node->cardinality;
 	result_operator->has_estimated_cardinality = true;
-	if (result_operator->type == LogicalOperatorType::LOGICAL_FILTER &&
-	    result_operator->children[0]->type == LogicalOperatorType::LOGICAL_GET) {
-		// FILTER on top of GET, add estimated properties to both
-		// auto &filter_props = *result_operator->estimated_props;
-		auto &child_operator = *result_operator->children[0];
-		child_operator.estimated_cardinality = node->cardinality;
-		child_operator.has_estimated_cardinality = true;
-	}
 	// check if we should do a pushdown on this node
 	// basically, any remaining filter that is a subset of the current relation will no longer be used in joins
 	// hence we should push it here
@@ -263,7 +331,7 @@ GenerateJoinRelation QueryGraphManager::GenerateJoins(vector<unique_ptr<LogicalO
 		if (filters_and_bindings[info.filter_index]->filter) {
 			// now check if the filter is a subset of the current relation
 			// note that infos with an empty relation set are a special case and we do not push them down
-			if (info.set.count > 0 && JoinRelationSet::IsSubset(*result_relation, info.set)) {
+			if (info.set.get().count > 0 && JoinRelationSet::IsSubset(*result_relation, info.set)) {
 				auto &filter_and_binding = filters_and_bindings[info.filter_index];
 				auto filter = std::move(filter_and_binding->filter);
 				// if it is, we can push the filter
@@ -342,106 +410,4 @@ void QueryGraphManager::CreateQueryGraphCrossProduct(JoinRelationSet &left, Join
 	query_graph.CreateEdge(right, left, nullptr);
 }
 
-static void FlipChildren(LogicalOperator &op) {
-	std::swap(op.children[0], op.children[1]);
-	if (op.type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN || op.type == LogicalOperatorType::LOGICAL_DELIM_JOIN) {
-		auto &join = op.Cast<LogicalComparisonJoin>();
-		join.join_type = InverseJoinType(join.join_type);
-		for (auto &cond : join.conditions) {
-			std::swap(cond.left, cond.right);
-			cond.comparison = FlipComparisonExpression(cond.comparison);
-		}
-	}
-	if (op.type == LogicalOperatorType::LOGICAL_ANY_JOIN) {
-		auto &join = op.Cast<LogicalAnyJoin>();
-		join.join_type = InverseJoinType(join.join_type);
-	}
-}
-
-void QueryGraphManager::TryFlipChildren(LogicalOperator &op, idx_t cardinality_ratio) {
-	auto &left_child = op.children[0];
-	auto &right_child = op.children[1];
-	auto lhs_cardinality = left_child->has_estimated_cardinality ? left_child->estimated_cardinality
-	                                                             : left_child->EstimateCardinality(context);
-	auto rhs_cardinality = right_child->has_estimated_cardinality ? right_child->estimated_cardinality
-	                                                              : right_child->EstimateCardinality(context);
-	if (rhs_cardinality < lhs_cardinality * cardinality_ratio) {
-		return;
-	}
-	FlipChildren(op);
-}
-
-unique_ptr<LogicalOperator> QueryGraphManager::LeftRightOptimizations(unique_ptr<LogicalOperator> input_op) {
-	auto op = input_op.get();
-	// pass through single child operators
-	while (!op->children.empty()) {
-		if (op->children.size() == 2) {
-			switch (op->type) {
-			case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: {
-				auto &join = op->Cast<LogicalComparisonJoin>();
-
-				switch (join.join_type) {
-				case JoinType::INNER:
-				case JoinType::OUTER:
-					TryFlipChildren(join);
-					break;
-				case JoinType::LEFT:
-				case JoinType::RIGHT:
-					if (join.right_projection_map.empty()) {
-						TryFlipChildren(join, 2);
-					}
-					break;
-				case JoinType::SEMI:
-				case JoinType::ANTI: {
-					idx_t has_range = 0;
-					if (!PhysicalPlanGenerator::HasEquality(join.conditions, has_range)) {
-						// if the conditions have no equality, do not flip the children.
-						// There is no physical join operator (yet) that can do a right_semi/anti join.
-						break;
-					}
-					TryFlipChildren(join, 2);
-					break;
-				}
-				default:
-					break;
-				}
-				break;
-			}
-			case LogicalOperatorType::LOGICAL_CROSS_PRODUCT: {
-				// cross product not a comparison join so JoinType::INNER will get ignored
-				TryFlipChildren(*op, 1);
-				break;
-			}
-			case LogicalOperatorType::LOGICAL_ANY_JOIN: {
-				auto &join = op->Cast<LogicalAnyJoin>();
-				if (join.join_type == JoinType::LEFT && join.right_projection_map.empty()) {
-					TryFlipChildren(join, 2);
-				} else if (join.join_type == JoinType::INNER) {
-					TryFlipChildren(join, 1);
-				}
-				break;
-			}
-			case LogicalOperatorType::LOGICAL_DELIM_JOIN: {
-				auto &join = op->Cast<LogicalComparisonJoin>();
-				if (HasInverseJoinType(join.join_type) && join.right_projection_map.empty()) {
-					FlipChildren(join);
-					join.delim_flipped = true;
-				}
-				break;
-			}
-			default:
-				break;
-			}
-			op->children[0] = LeftRightOptimizations(std::move(op->children[0]));
-			op->children[1] = LeftRightOptimizations(std::move(op->children[1]));
-			// break from while loop
-			break;
-		}
-		if (op->children.size() == 1) {
-			op = op->children[0].get();
-		}
-	}
-	return input_op;
-}
-
 } // namespace duckdb