duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/src/optimizer/join_order/cardinality_estimator.cpp

@@ -1,14 +1,13 @@
+#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
+#include "duckdb/common/enums/join_type.hpp"
+#include "duckdb/common/limits.hpp"
+#include "duckdb/common/printer.hpp"
+#include "duckdb/planner/expression_iterator.hpp"
 #include "duckdb/function/table/table_scan.hpp"
 #include "duckdb/optimizer/join_order/join_node.hpp"
-#include "duckdb/optimizer/join_order/join_order_optimizer.hpp"
-#include "duckdb/planner/filter/conjunction_filter.hpp"
+#include "duckdb/optimizer/join_order/query_graph_manager.hpp"
 #include "duckdb/planner/operator/logical_comparison_join.hpp"
-#include "duckdb/planner/operator/logical_get.hpp"
 #include "duckdb/storage/data_table.hpp"
-#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
-#include "duckdb/common/printer.hpp"
-#include "duckdb/common/enums/join_type.hpp"
-#include "duckdb/common/limits.hpp"
 
 namespace duckdb {
 
@@ -22,7 +21,7 @@ bool CardinalityEstimator::EmptyFilter(FilterInfo &filter_info) {
 }
 
 void CardinalityEstimator::AddRelationTdom(FilterInfo &filter_info) {
-	D_ASSERT(filter_info.set.count >= 1);
+	D_ASSERT(filter_info.set.get().count >= 1);
 	for (const RelationsToTDom &r2tdom : relations_to_tdoms) {
 		auto &i_set = r2tdom.equivalent_relations;
 		if (i_set.find(filter_info.left_binding) != i_set.end()) {
@@ -37,18 +36,21 @@ void CardinalityEstimator::AddRelationTdom(FilterInfo &filter_info) {
 	relations_to_tdoms.emplace_back(new_r2tdom);
 }
 
-bool CardinalityEstimator::SingleRelationFilter(FilterInfo &filter_info) {
-	if (filter_info.left_set && filter_info.right_set && filter_info.set.count > 1) {
+bool CardinalityEstimator::SingleColumnFilter(duckdb::FilterInfo &filter_info) {
+	if (filter_info.left_set && filter_info.right_set && filter_info.set.get().count > 1) {
 		// Both set and are from different relations
 		return false;
 	}
 	if (EmptyFilter(filter_info)) {
 		return false;
 	}
+	if (filter_info.join_type == JoinType::SEMI || filter_info.join_type == JoinType::ANTI) {
+		return false;
+	}
 	return true;
 }
 
-vector<idx_t> CardinalityEstimator::DetermineMatchingEquivalentSets(FilterInfo *filter_info) {
+vector<idx_t> CardinalityEstimator::DetermineMatchingEquivalentSets(optional_ptr<FilterInfo> filter_info) {
 	vector<idx_t> matching_equivalent_sets;
 	idx_t equivalent_relation_index = 0;
 
@@ -66,7 +68,8 @@ vector<idx_t> CardinalityEstimator::DetermineMatchingEquivalentSets(FilterInfo *
 	return matching_equivalent_sets;
 }
 
-void CardinalityEstimator::AddToEquivalenceSets(FilterInfo *filter_info, vector<idx_t> matching_equivalent_sets) {
+void CardinalityEstimator::AddToEquivalenceSets(optional_ptr<FilterInfo> filter_info,
+                                                vector<idx_t> matching_equivalent_sets) {
 	D_ASSERT(matching_equivalent_sets.size() <= 2);
 	if (matching_equivalent_sets.size() > 1) {
 		// an equivalence relation is connecting two sets of equivalence relations
@@ -100,8 +103,8 @@ void CardinalityEstimator::InitEquivalentRelations(const vector<unique_ptr<Filte
 	// For each filter, we fill keep track of the index of the equivalent relation set
 	// the left and right relation needs to be added to.
 	for (auto &filter : filter_infos) {
-		if (SingleRelationFilter(*filter)) {
-			// Filter on one relation, (i.e string or range filter on a column).
+		if (SingleColumnFilter(*filter)) {
+			// Filter on one relation, (i.e. string or range filter on a column).
 			// Grab the first relation and add it to  the equivalence_relations
 			AddRelationTdom(*filter);
 			continue;
@@ -123,44 +126,157 @@ void CardinalityEstimator::RemoveEmptyTotalDomains() {
 	relations_to_tdoms.erase(remove_start, relations_to_tdoms.end());
 }
 
-void UpdateDenom(Subgraph2Denominator &relation_2_denom, RelationsToTDom &relation_to_tdom) {
-	relation_2_denom.denom *= relation_to_tdom.has_tdom_hll ? relation_to_tdom.tdom_hll : relation_to_tdom.tdom_no_hll;
+double CardinalityEstimator::GetNumerator(JoinRelationSet &set) {
+	double numerator = 1;
+	for (idx_t i = 0; i < set.count; i++) {
+		auto &single_node_set = set_manager.GetJoinRelation(set.relations[i]);
+		auto card_helper = relation_set_2_cardinality[single_node_set.ToString()];
+		numerator *= card_helper.cardinality_before_filters == 0 ? 1 : card_helper.cardinality_before_filters;
+	}
+	return numerator;
+}
+
+bool EdgeConnects(FilterInfoWithTotalDomains &edge, Subgraph2Denominator &subgraph) {
+	if (edge.filter_info->left_set) {
+		if (JoinRelationSet::IsSubset(*subgraph.relations, *edge.filter_info->left_set)) {
+			// cool
+			return true;
+		}
+	}
+	if (edge.filter_info->right_set) {
+		if (JoinRelationSet::IsSubset(*subgraph.relations, *edge.filter_info->right_set)) {
+			return true;
+		}
+	}
+	return false;
 }
 
-void FindSubgraphMatchAndMerge(Subgraph2Denominator &merge_to, idx_t find_me,
-                               vector<Subgraph2Denominator>::iterator subgraph,
-                               vector<Subgraph2Denominator>::iterator end) {
-	for (; subgraph != end; subgraph++) {
-		if (subgraph->relations.count(find_me) >= 1) {
-			for (auto &relation : subgraph->relations) {
-				merge_to.relations.insert(relation);
+vector<FilterInfoWithTotalDomains> GetEdges(vector<RelationsToTDom> &relations_to_tdom,
+                                            JoinRelationSet &requested_set) {
+	vector<FilterInfoWithTotalDomains> res;
+	for (auto &relation_2_tdom : relations_to_tdom) {
+		for (auto &filter : relation_2_tdom.filters) {
+			if (JoinRelationSet::IsSubset(requested_set, filter->set)) {
+				FilterInfoWithTotalDomains new_edge(filter, relation_2_tdom);
+				res.push_back(new_edge);
 			}
-			subgraph->relations.clear();
-			merge_to.denom *= subgraph->denom;
-			return;
 		}
 	}
+	return res;
 }
 
-template <>
-double CardinalityEstimator::EstimateCardinalityWithSet(JoinRelationSet &new_set) {
+vector<idx_t> SubgraphsConnectedByEdge(FilterInfoWithTotalDomains &edge, vector<Subgraph2Denominator> &subgraphs) {
+	vector<idx_t> res;
+	if (subgraphs.empty()) {
+		return res;
+	} else {
+		// check the combinations of subgraphs and see if the edge connects two of them,
+		// if so, return the indexes of the two subgraphs within the vector
+		for (idx_t outer = 0; outer != subgraphs.size(); outer++) {
+			// check if the edge connects two subgraphs.
+			for (idx_t inner = outer + 1; inner != subgraphs.size(); inner++) {
+				if (EdgeConnects(edge, subgraphs.at(outer)) && EdgeConnects(edge, subgraphs.at(inner))) {
+					// order is important because we will delete the inner subgraph later
+					res.push_back(outer);
+					res.push_back(inner);
+					return res;
+				}
+			}
+			// if the edge does not connect two subgraphs, see if the edge connects with just outer
+			// merge subgraph.at(outer) with the RelationSet(s) that edge connects
+			if (EdgeConnects(edge, subgraphs.at(outer))) {
+				res.push_back(outer);
+				return res;
+			}
+		}
+	}
+	// this edge connects only the relations it connects. Return an empty result so a new subgraph is created.
+	return res;
+}
 
-	if (relation_set_2_cardinality.find(new_set.ToString()) != relation_set_2_cardinality.end()) {
-		return relation_set_2_cardinality[new_set.ToString()].cardinality_before_filters;
+JoinRelationSet &CardinalityEstimator::UpdateNumeratorRelations(Subgraph2Denominator left, Subgraph2Denominator right,
+                                                                FilterInfoWithTotalDomains &filter) {
+	switch (filter.filter_info->join_type) {
+	case JoinType::SEMI:
+	case JoinType::ANTI: {
+		if (JoinRelationSet::IsSubset(*left.relations, *filter.filter_info->left_set) &&
+		    JoinRelationSet::IsSubset(*right.relations, *filter.filter_info->right_set)) {
+			return *left.numerator_relations;
+		}
+		return *right.numerator_relations;
 	}
-	double numerator = 1;
-	unordered_set<idx_t> actual_set;
+	default:
+		// cross product or inner join
+		return set_manager.Union(*left.numerator_relations, *right.numerator_relations);
+	}
+}
 
-	for (idx_t i = 0; i < new_set.count; i++) {
-		auto &single_node_set = set_manager.GetJoinRelation(new_set.relations[i]);
-		auto card_helper = relation_set_2_cardinality[single_node_set.ToString()];
-		numerator *= card_helper.cardinality_before_filters == 0 ? 1 : card_helper.cardinality_before_filters;
-		actual_set.insert(new_set.relations[i]);
+double CardinalityEstimator::CalculateUpdatedDenom(Subgraph2Denominator left, Subgraph2Denominator right,
+                                                   FilterInfoWithTotalDomains &filter) {
+	double new_denom = left.denom * right.denom;
+	switch (filter.filter_info->join_type) {
+	case JoinType::INNER: {
+		bool set = false;
+		ExpressionType comparison_type = ExpressionType::COMPARE_EQUAL;
+		ExpressionIterator::EnumerateExpression(filter.filter_info->filter, [&](Expression &expr) {
+			if (expr.expression_class == ExpressionClass::BOUND_COMPARISON) {
+				comparison_type = expr.type;
+				set = true;
+				return;
+			}
+		});
+		if (!set) {
+			new_denom *= filter.has_tdom_hll ? (double)filter.tdom_hll : (double)filter.tdom_no_hll;
+			// no comparison is taking place, so the denominator is just the product of the left and right
+			return new_denom;
+		}
+		// extra_ratio helps represents how many tuples will be filtered out if the comparison evaluates to
+		// false. set to 1 to assume cross product.
+		double extra_ratio = 1;
+		switch (comparison_type) {
+		case ExpressionType::COMPARE_EQUAL:
+		case ExpressionType::COMPARE_NOT_DISTINCT_FROM:
+			// extra ration stays 1
+			extra_ratio = filter.has_tdom_hll ? (double)filter.tdom_hll : (double)filter.tdom_no_hll;
+			break;
+		case ExpressionType::COMPARE_LESSTHANOREQUALTO:
+		case ExpressionType::COMPARE_LESSTHAN:
+		case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
+		case ExpressionType::COMPARE_GREATERTHAN:
+			// start with the selectivity of equality
+			extra_ratio = filter.has_tdom_hll ? (double)filter.tdom_hll : (double)filter.tdom_no_hll;
+			// now assume every tuple will match 2.5 times (on average)
+			extra_ratio *= static_cast<double>(1) / CardinalityEstimator::DEFAULT_LT_GT_MULTIPLIER;
+			break;
+		case ExpressionType::COMPARE_NOTEQUAL:
+		case ExpressionType::COMPARE_DISTINCT_FROM:
+			// basically assume cross product.
+			extra_ratio = 1;
+			break;
+		default:
+			break;
+		}
+		new_denom *= extra_ratio;
+		return new_denom;
+	}
+	case JoinType::SEMI:
+	case JoinType::ANTI: {
+		if (JoinRelationSet::IsSubset(*left.relations, *filter.filter_info->left_set) &&
+		    JoinRelationSet::IsSubset(*right.relations, *filter.filter_info->right_set)) {
+			new_denom = left.denom * CardinalityEstimator::DEFAULT_SEMI_ANTI_SELECTIVITY;
+			return new_denom;
+		}
+		new_denom = right.denom * CardinalityEstimator::DEFAULT_SEMI_ANTI_SELECTIVITY;
+		return new_denom;
 	}
+	default:
+		// cross product
+		return new_denom;
+	}
+}
 
+DenomInfo CardinalityEstimator::GetDenominator(JoinRelationSet &set) {
 	vector<Subgraph2Denominator> subgraphs;
-	bool done = false;
-	bool found_match = false;
 
 	// Finding the denominator is tricky. You need to go through the tdoms in decreasing order
 	// Then loop through all filters in the equivalence set of the tdom to see if both the
@@ -170,87 +286,103 @@ double CardinalityEstimator::EstimateCardinalityWithSet(JoinRelationSet &new_set
 	// TODO: Implement a method to cache subgraphs so you don't have to build them up every
 	// time the cardinality of a new set is requested
 
-	// relations_to_tdoms has already been sorted.
-	for (auto &relation_2_tdom : relations_to_tdoms) {
-		// loop through each filter in the tdom.
-		if (done) {
-			break;
-		}
-		for (auto &filter : relation_2_tdom.filters) {
-			if (actual_set.count(filter->left_binding.table_index) == 0 ||
-			    actual_set.count(filter->right_binding.table_index) == 0) {
-				continue;
-			}
-			// the join filter is on relations in the new set.
-			found_match = false;
-			vector<Subgraph2Denominator>::iterator it;
-			for (it = subgraphs.begin(); it != subgraphs.end(); it++) {
-				auto left_in = it->relations.count(filter->left_binding.table_index);
-				auto right_in = it->relations.count(filter->right_binding.table_index);
-				if (left_in && right_in) {
-					// if both left and right bindings are in the subgraph, continue.
-					// This means another filter is connecting relations already in the
-					// subgraph it, but it has a tdom that is less, and we don't care.
-					found_match = true;
-					continue;
-				}
-				if (!left_in && !right_in) {
-					// if both left and right bindings are *not* in the subgraph, continue
-					// without finding a match. This will trigger the process to add a new
-					// subgraph
-					continue;
-				}
-				idx_t find_table;
-				if (left_in) {
-					find_table = filter->right_binding.table_index;
-				} else {
-					D_ASSERT(right_in);
-					find_table = filter->left_binding.table_index;
-				}
-				auto next_subgraph = it + 1;
-				// iterate through other subgraphs and merge.
-				FindSubgraphMatchAndMerge(*it, find_table, next_subgraph, subgraphs.end());
-				// Now insert the right binding and update denominator with the
-				// tdom of the filter
-				it->relations.insert(find_table);
-				UpdateDenom(*it, relation_2_tdom);
-				found_match = true;
-				break;
+	// relations_to_tdoms has already been sorted by largest to smallest total domain
+	// then we look through the filters for the relations_to_tdoms,
+	// and we start to choose the filters that join relations in the set.
+
+	// edges are guaranteed to be in order of largest tdom to smallest tdom.
+	auto edges = GetEdges(relations_to_tdoms, set);
+	for (auto &edge : edges) {
+		auto subgraph_connections = SubgraphsConnectedByEdge(edge, subgraphs);
+
+		if (subgraph_connections.empty()) {
+			// create a subgraph out of left and right, then merge right into left and add left to subgraphs.
+			// this helps cover a case where there are no subgraphs yet, and the only join filter is a SEMI JOIN
+			auto left_subgraph = Subgraph2Denominator();
+			auto right_subgraph = Subgraph2Denominator();
+			left_subgraph.relations = edge.filter_info->left_set;
+			left_subgraph.numerator_relations = edge.filter_info->left_set;
+			right_subgraph.relations = edge.filter_info->right_set;
+			right_subgraph.numerator_relations = edge.filter_info->right_set;
+			left_subgraph.numerator_relations = &UpdateNumeratorRelations(left_subgraph, right_subgraph, edge);
+			left_subgraph.relations = edge.filter_info->set.get();
+			left_subgraph.denom = CalculateUpdatedDenom(left_subgraph, right_subgraph, edge);
+			subgraphs.push_back(left_subgraph);
+		} else if (subgraph_connections.size() == 1) {
+			auto left_subgraph = &subgraphs.at(subgraph_connections.at(0));
+			auto right_subgraph = Subgraph2Denominator();
+			right_subgraph.relations = edge.filter_info->right_set;
+			right_subgraph.numerator_relations = edge.filter_info->right_set;
+			if (JoinRelationSet::IsSubset(*left_subgraph->relations, *right_subgraph.relations)) {
+				right_subgraph.relations = edge.filter_info->left_set;
+				right_subgraph.numerator_relations = edge.filter_info->left_set;
 			}
-			// means that the filter joins relations in the given set, but there is no
-			// connection to any subgraph in subgraphs. Add a new subgraph, and maybe later there will be
-			// a connection.
-			if (!found_match) {
-				subgraphs.emplace_back();
-				auto &subgraph = subgraphs.back();
-				subgraph.relations.insert(filter->left_binding.table_index);
-				subgraph.relations.insert(filter->right_binding.table_index);
-				UpdateDenom(subgraph, relation_2_tdom);
+
+			if (JoinRelationSet::IsSubset(*left_subgraph->relations, *edge.filter_info->left_set) &&
+			    JoinRelationSet::IsSubset(*left_subgraph->relations, *edge.filter_info->right_set)) {
+				// here we have an edge that connects the same subgraph to the same subgraph. Just continue. no need to
+				// update the denom
+				continue;
 			}
+			left_subgraph->numerator_relations = &UpdateNumeratorRelations(*left_subgraph, right_subgraph, edge);
+			left_subgraph->relations = &set_manager.Union(*left_subgraph->relations, *right_subgraph.relations);
+			left_subgraph->denom = CalculateUpdatedDenom(*left_subgraph, right_subgraph, edge);
+		} else if (subgraph_connections.size() == 2) {
+			// The two subgraphs in the subgraph_connections can be merged by this edge.
+			D_ASSERT(subgraph_connections.at(0) < subgraph_connections.at(1));
+			auto subgraph_to_merge_into = &subgraphs.at(subgraph_connections.at(0));
+			auto subgraph_to_delete = &subgraphs.at(subgraph_connections.at(1));
+			subgraph_to_merge_into->relations =
+			    &set_manager.Union(*subgraph_to_merge_into->relations, *subgraph_to_delete->relations);
+			subgraph_to_merge_into->numerator_relations =
+			    &UpdateNumeratorRelations(*subgraph_to_merge_into, *subgraph_to_delete, edge);
+			subgraph_to_delete->relations = nullptr;
+			subgraph_to_merge_into->denom = CalculateUpdatedDenom(*subgraph_to_merge_into, *subgraph_to_delete, edge);
 			auto remove_start = std::remove_if(subgraphs.begin(), subgraphs.end(),
-			                                   [](Subgraph2Denominator &s) { return s.relations.empty(); });
+			                                   [](Subgraph2Denominator &s) { return !s.relations; });
 			subgraphs.erase(remove_start, subgraphs.end());
+		}
+		if (subgraphs.size() == 1 && subgraphs.at(0).relations->ToString() == set.ToString()) {
+			// the first subgraph has connected all the desired relations, no need to iterate
+			// through the rest of the edges.
+			break;
+		}
+	}
 
-			if (subgraphs.size() == 1 && subgraphs.at(0).relations.size() == new_set.count) {
-				// You have found enough filters to connect the relations. These are guaranteed
-				// to be the filters with the highest Tdoms.
-				done = true;
-				break;
-			}
+	// It's possible cross-products were added and are not present in the filters in the relation_2_tdom
+	// structures. When that's the case, merge all remaining subgraphs.
+	if (subgraphs.size() > 1) {
+		auto final_subgraph = subgraphs.at(0);
+		for (auto merge_with = subgraphs.begin() + 1; merge_with != subgraphs.end(); merge_with++) {
+			D_ASSERT(final_subgraph.relations && merge_with->relations);
+			final_subgraph.relations = &set_manager.Union(*final_subgraph.relations, *merge_with->relations);
+			D_ASSERT(final_subgraph.numerator_relations && merge_with->numerator_relations);
+			final_subgraph.numerator_relations =
+			    &set_manager.Union(*final_subgraph.numerator_relations, *merge_with->numerator_relations);
+			final_subgraph.denom *= merge_with->denom;
 		}
 	}
-	double denom = 1;
-	// TODO: It's possible cross-products were added and are not present in the filters in the relation_2_tdom
-	//       structures. When that's the case, multiply the denom structures that have no intersection
-	for (auto &match : subgraphs) {
-		denom *= match.denom;
+	// can happen if a table has cardinality 0, a tdom is set to 0, or if a cross product is used.
+	if (subgraphs.empty() || subgraphs.at(0).denom == 0) {
+		// denominator is 1 and numerators are a cross product of cardinalities.
+		return DenomInfo(set, 1, 1);
 	}
-	// can happen if a table has cardinality 0, or a tdom is set to 0
-	if (denom <= 1) {
-		denom = 1;
+	return DenomInfo(*subgraphs.at(0).numerator_relations, 1, subgraphs.at(0).denom);
+}
+
+template <>
+double CardinalityEstimator::EstimateCardinalityWithSet(JoinRelationSet &new_set) {
+
+	if (relation_set_2_cardinality.find(new_set.ToString()) != relation_set_2_cardinality.end()) {
+		return relation_set_2_cardinality[new_set.ToString()].cardinality_before_filters;
 	}
-	auto result = numerator / denom;
-	auto new_entry = CardinalityHelper((double)result, 1);
+
+	// can happen if a table has cardinality 0, or a tdom is set to 0
+	auto denom = GetDenominator(new_set);
+	auto numerator = GetNumerator(denom.numerator_relations);
+
+	double result = numerator / denom.denominator;
+	auto new_entry = CardinalityHelper(result);
 	relation_set_2_cardinality[new_set.ToString()] = new_entry;
 	return result;
 }
@@ -259,7 +391,7 @@ template <>
 idx_t CardinalityEstimator::EstimateCardinalityWithSet(JoinRelationSet &new_set) {
 	auto cardinality_as_double = EstimateCardinalityWithSet<double>(new_set);
 	auto max = NumericLimits<idx_t>::Maximum();
-	if (cardinality_as_double >= max) {
+	if (cardinality_as_double >= (double)max) {
 		return max;
 	}
 	return (idx_t)cardinality_as_double;
@@ -282,9 +414,8 @@ void CardinalityEstimator::InitCardinalityEstimatorProps(optional_ptr<JoinRelati
 	// Get the join relation set
 	D_ASSERT(stats.stats_initialized);
 	auto relation_cardinality = stats.cardinality;
-	auto relation_filter = stats.filter_strength;
 
-	auto card_helper = CardinalityHelper(relation_cardinality, relation_filter);
+	auto card_helper = CardinalityHelper((double)relation_cardinality);
 	relation_set_2_cardinality[set->ToString()] = card_helper;
 
 	UpdateTotalDomains(set, stats);

package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp

@@ -1,24 +1,23 @@
 #include "duckdb/optimizer/join_order/join_order_optimizer.hpp"
-#include "duckdb/optimizer/join_order/cost_model.hpp"
-#include "duckdb/optimizer/join_order/plan_enumerator.hpp"
+
 #include "duckdb/common/enums/join_type.hpp"
 #include "duckdb/common/limits.hpp"
 #include "duckdb/common/pair.hpp"
+#include "duckdb/optimizer/join_order/cost_model.hpp"
+#include "duckdb/optimizer/join_order/plan_enumerator.hpp"
 #include "duckdb/planner/expression/list.hpp"
 #include "duckdb/planner/operator/list.hpp"
 
 namespace duckdb {
 
-static bool HasJoin(LogicalOperator *op) {
-	while (!op->children.empty()) {
-		if (op->children.size() == 1) {
-			op = op->children[0].get();
-		}
-		if (op->children.size() == 2) {
-			return true;
-		}
-	}
-	return false;
+JoinOrderOptimizer::JoinOrderOptimizer(ClientContext &context) : context(context), query_graph_manager(context) {
+}
+
+JoinOrderOptimizer JoinOrderOptimizer::CreateChildOptimizer() {
+	JoinOrderOptimizer child_optimizer(context);
+	child_optimizer.materialized_cte_stats = materialized_cte_stats;
+	child_optimizer.delim_scan_stats = delim_scan_stats;
+	return child_optimizer;
 }
 
 unique_ptr<LogicalOperator> JoinOrderOptimizer::Optimize(unique_ptr<LogicalOperator> plan,
@@ -29,7 +28,7 @@ unique_ptr<LogicalOperator> JoinOrderOptimizer::Optimize(unique_ptr<LogicalOpera
 
 	// extract the relations that go into the hyper graph.
 	// We optimize the children of any non-reorderable operations we come across.
-	bool reorderable = query_graph_manager.Build(*op);
+	bool reorderable = query_graph_manager.Build(*this, *op);
 
 	// get relation_stats here since the reconstruction process will move all relations.
 	auto relation_stats = query_graph_manager.relation_manager.GetRelationStats();
@@ -58,13 +57,6 @@ unique_ptr<LogicalOperator> JoinOrderOptimizer::Optimize(unique_ptr<LogicalOpera
 		}
 	}
 
-	// only perform left right optimizations when stats is null (means we have the top level optimize call)
-	// Don't check reorderability because non-reorderable joins will result in 1 relation, but we can
-	// still switch the children.
-	if (stats == nullptr && HasJoin(new_logical_plan.get())) {
-		new_logical_plan = query_graph_manager.LeftRightOptimizations(std::move(new_logical_plan));
-	}
-
 	// Propagate up a stats object from the top of the new_logical_plan if stats exist.
 	if (stats) {
 		auto cardinality = new_logical_plan->EstimateCardinality(context);
@@ -72,9 +64,39 @@ unique_ptr<LogicalOperator> JoinOrderOptimizer::Optimize(unique_ptr<LogicalOpera
 		auto new_stats = RelationStatisticsHelper::CombineStatsOfReorderableOperator(bindings, relation_stats);
 		new_stats.cardinality = cardinality;
 		RelationStatisticsHelper::CopyRelationStats(*stats, new_stats);
+	} else {
+		// starts recursively setting cardinality
+		new_logical_plan->EstimateCardinality(context);
+	}
+
+	if (new_logical_plan->type == LogicalOperatorType::LOGICAL_EXPLAIN) {
+		new_logical_plan->SetEstimatedCardinality(3);
 	}
 
 	return new_logical_plan;
 }
 
+void JoinOrderOptimizer::AddMaterializedCTEStats(idx_t index, RelationStats &&stats) {
+	materialized_cte_stats.emplace(index, std::move(stats));
+}
+
+RelationStats JoinOrderOptimizer::GetMaterializedCTEStats(idx_t index) {
+	auto it = materialized_cte_stats.find(index);
+	if (it == materialized_cte_stats.end()) {
+		throw InternalException("Unable to find materialized CTE stats with index %llu", index);
+	}
+	return it->second;
+}
+
+void JoinOrderOptimizer::AddDelimScanStats(RelationStats &stats) {
+	delim_scan_stats = &stats;
+}
+
+RelationStats JoinOrderOptimizer::GetDelimScanStats() {
+	if (!delim_scan_stats) {
+		throw InternalException("Unable to find delim scan stats!");
+	}
+	return *delim_scan_stats;
+}
+
 } // namespace duckdb

package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp

@@ -99,11 +99,15 @@ JoinRelationSet &JoinRelationSetManager::Union(JoinRelationSet &left, JoinRelati
 			// left is smaller, progress left and add it to the set
 			relations[count++] = left.relations[i];
 			i++;
-		} else {
-			D_ASSERT(left.relations[i] > right.relations[j]);
+		} else if (left.relations[i] > right.relations[j]) {
 			// right is smaller, progress right and add it to the set
 			relations[count++] = right.relations[j];
 			j++;
+		} else {
+			D_ASSERT(left.relations[i] == right.relations[j]);
+			relations[count++] = left.relations[i];
+			i++;
+			j++;
 		}
 	}
 	return GetJoinRelation(std::move(relations), count);

package/src/duckdb/src/optimizer/join_order/plan_enumerator.cpp

@@ -80,7 +80,9 @@ void PlanEnumerator::GenerateCrossProducts() {
 	for (idx_t i = 0; i < query_graph_manager.relation_manager.NumRelations(); i++) {
 		auto &left = query_graph_manager.set_manager.GetJoinRelation(i);
 		for (idx_t j = 0; j < query_graph_manager.relation_manager.NumRelations(); j++) {
-			if (i != j) {
+			auto cross_product_allowed = query_graph_manager.relation_manager.CrossProductWithRelationAllowed(i) &&
+			                             query_graph_manager.relation_manager.CrossProductWithRelationAllowed(j);
+			if (i != j && cross_product_allowed) {
 				auto &right = query_graph_manager.set_manager.GetJoinRelation(j);
 				query_graph_manager.CreateQueryGraphCrossProduct(left, right);
 			}
@@ -99,17 +101,37 @@ const reference_map_t<JoinRelationSet, unique_ptr<DPJoinNode>> &PlanEnumerator::
 unique_ptr<DPJoinNode> PlanEnumerator::CreateJoinTree(JoinRelationSet &set,
                                                       const vector<reference<NeighborInfo>> &possible_connections,
                                                       DPJoinNode &left, DPJoinNode &right) {
-	// for the hash join we want the right side (build side) to have the smallest cardinality
-	// also just a heuristic but for now...
-	// FIXME: we should probably actually benchmark that as well
-	// FIXME: should consider different join algorithms, should we pick a join algorithm here as well? (probably)
-	optional_ptr<NeighborInfo> best_connection = nullptr;
 
-	// cross products are techincally still connections, but the filter expression is a null_ptr
-	if (!possible_connections.empty()) {
-		best_connection = &possible_connections.back().get();
+	// FIXME: should consider different join algorithms, should we pick a join algorithm here as well? (probably)
+	optional_ptr<NeighborInfo> best_connection = possible_connections.back().get();
+	// cross products are technically still connections, but the filter expression is a null_ptr
+	bool found_non_cross_product_connection = false;
+	for (auto &connection : possible_connections) {
+		for (auto &filter : connection.get().filters) {
+			if (filter->join_type != JoinType::INVALID) {
+				best_connection = connection.get();
+				found_non_cross_product_connection = true;
+				break;
+			}
+		}
+		if (found_non_cross_product_connection) {
+			break;
+		}
 	}
+	auto join_type = JoinType::INVALID;
+	for (auto &filter_binding : best_connection->filters) {
+		if (!filter_binding->left_set || !filter_binding->right_set) {
+			continue;
+		}
 
+		join_type = filter_binding->join_type;
+		// prefer joining on semi and anti joins as they have a higher chance of being more
+		// selective
+		if (join_type == JoinType::SEMI || join_type == JoinType::ANTI) {
+			break;
+		}
+	}
+	// need the filter info from the Neighborhood info.
 	auto cost = cost_model.ComputeCost(left, right);
 	auto result = make_uniq<DPJoinNode>(set, best_connection, left.set, right.set, cost);
 	result->cardinality = cost_model.cardinality_estimator.EstimateCardinalityWithSet<idx_t>(set);
@@ -176,7 +198,7 @@ bool PlanEnumerator::EmitCSG(JoinRelationSet &node) {
 	}
 
 	//! Neighbors should be reversed when iterating over them.
-	std::sort(neighbors.begin(), neighbors.end(), std::greater_equal<idx_t>());
+	std::sort(neighbors.begin(), neighbors.end(), std::greater<idx_t>());
 	for (idx_t i = 0; i < neighbors.size() - 1; i++) {
 		D_ASSERT(neighbors[i] > neighbors[i + 1]);
 	}