duckdb 0.10.3-dev3.0 → 0.10.3-dev6.0

package/src/duckdb/src/optimizer/filter_combiner.cpp

@@ -67,7 +67,7 @@ FilterResult FilterCombiner::AddConstantComparison(vector<ExpressionValueInforma
 		switch (comparison) {
 		case ValueComparisonResult::PRUNE_LEFT:
 			// prune the entry from the info list
-			info_list.erase(info_list.begin() + i);
+			info_list.erase_at(i);
 			i--;
 			break;
 		case ValueComparisonResult::PRUNE_RIGHT:
@@ -607,7 +607,7 @@ TableFilterSet FilterCombiner::GenerateTableScanFilters(vector<idx_t> &column_id
 			table_filters.PushFilter(column_index, std::move(upper_bound));
 			table_filters.PushFilter(column_index, make_uniq<IsNotNullFilter>());
 
-			remaining_filters.erase(remaining_filters.begin() + rem_fil_idx);
+			remaining_filters.erase_at(rem_fil_idx);
 		}
 	}
 
@@ -971,7 +971,7 @@ unique_ptr<Expression> FilterCombiner::FindTransitiveFilter(Expression &expr) {
 			auto &comparison = remaining_filters[i]->Cast<BoundComparisonExpression>();
 			if (expr.Equals(*comparison.right) && comparison.type != ExpressionType::COMPARE_NOTEQUAL) {
 				auto filter = std::move(remaining_filters[i]);
-				remaining_filters.erase(remaining_filters.begin() + i);
+				remaining_filters.erase_at(i);
 				return filter;
 			}
 		}

package/src/duckdb/src/optimizer/filter_pushdown.cpp

@@ -9,7 +9,8 @@ namespace duckdb {
 
 using Filter = FilterPushdown::Filter;
 
-FilterPushdown::FilterPushdown(Optimizer &optimizer) : optimizer(optimizer), combiner(optimizer.context) {
+FilterPushdown::FilterPushdown(Optimizer &optimizer, bool convert_mark_joins)
+    : optimizer(optimizer), combiner(optimizer.context), convert_mark_joins(convert_mark_joins) {
 }
 
 unique_ptr<LogicalOperator> FilterPushdown::Rewrite(unique_ptr<LogicalOperator> op) {
@@ -144,7 +145,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushFinalFilters(unique_ptr<LogicalO
 unique_ptr<LogicalOperator> FilterPushdown::FinishPushdown(unique_ptr<LogicalOperator> op) {
 	// unhandled type, first perform filter pushdown in its children
 	for (auto &child : op->children) {
-		FilterPushdown pushdown(optimizer);
+		FilterPushdown pushdown(optimizer, convert_mark_joins);
 		child = pushdown.Rewrite(std::move(child));
 	}
 	// now push any existing filters

package/src/duckdb/src/optimizer/join_order/cardinality_estimator.cpp

@@ -50,7 +50,7 @@ bool CardinalityEstimator::SingleRelationFilter(FilterInfo &filter_info) {
 
 vector<idx_t> CardinalityEstimator::DetermineMatchingEquivalentSets(FilterInfo *filter_info) {
 	vector<idx_t> matching_equivalent_sets;
-	auto equivalent_relation_index = 0;
+	idx_t equivalent_relation_index = 0;
 
 	for (const RelationsToTDom &r2tdom : relations_to_tdoms) {
 		auto &i_set = r2tdom.equivalent_relations;
@@ -246,7 +246,7 @@ double CardinalityEstimator::EstimateCardinalityWithSet(JoinRelationSet &new_set
 		denom *= match.denom;
 	}
 	// can happen if a table has cardinality 0, or a tdom is set to 0
-	if (denom == 0) {
+	if (denom <= 1) {
 		denom = 1;
 	}
 	auto result = numerator / denom;
@@ -259,7 +259,7 @@ template <>
 idx_t CardinalityEstimator::EstimateCardinalityWithSet(JoinRelationSet &new_set) {
 	auto cardinality_as_double = EstimateCardinalityWithSet<double>(new_set);
 	auto max = NumericLimits<idx_t>::Maximum();
-	if (cardinality_as_double > max) {
+	if (cardinality_as_double >= max) {
 		return max;
 	}
 	return (idx_t)cardinality_as_double;

package/src/duckdb/src/optimizer/join_order/cost_model.cpp

@@ -8,7 +8,7 @@ CostModel::CostModel(QueryGraphManager &query_graph_manager)
     : query_graph_manager(query_graph_manager), cardinality_estimator() {
 }
 
-double CostModel::ComputeCost(JoinNode &left, JoinNode &right) {
+double CostModel::ComputeCost(DPJoinNode &left, DPJoinNode &right) {
 	auto &combination = query_graph_manager.set_manager.Union(left.set, right.set);
 	auto join_card = cardinality_estimator.EstimateCardinalityWithSet<double>(combination);
 	auto join_cost = join_card;

package/src/duckdb/src/optimizer/join_order/join_node.cpp

@@ -6,35 +6,12 @@
 
 namespace duckdb {
 
-JoinNode::JoinNode(JoinRelationSet &set) : set(set) {
+DPJoinNode::DPJoinNode(JoinRelationSet &set) : set(set), info(nullptr), is_leaf(true), left_set(set), right_set(set) {
 }
 
-JoinNode::JoinNode(JoinRelationSet &set, optional_ptr<NeighborInfo> info, JoinNode &left, JoinNode &right, double cost)
-    : set(set), info(info), left(&left), right(&right), cost(cost) {
-}
-
-unique_ptr<EstimatedProperties> EstimatedProperties::Copy() {
-	auto result = make_uniq<EstimatedProperties>(cardinality, cost);
-	return result;
-}
-
-string JoinNode::ToString() {
-	string result = "-------------------------------\n";
-	result += set.ToString() + "\n";
-	result += "cost = " + to_string(cost) + "\n";
-	result += "left = \n";
-	if (left) {
-		result += left->ToString();
-	}
-	result += "right = \n";
-	if (right) {
-		result += right->ToString();
-	}
-	return result;
-}
-
-void JoinNode::Print() {
-	Printer::Print(ToString());
+DPJoinNode::DPJoinNode(JoinRelationSet &set, optional_ptr<NeighborInfo> info, JoinRelationSet &left,
+                       JoinRelationSet &right, double cost)
+    : set(set), info(info), is_leaf(false), left_set(left), right_set(right), cost(cost) {
 }
 
 } // namespace duckdb

package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp

@@ -5,7 +5,6 @@
 #include "duckdb/common/limits.hpp"
 #include "duckdb/common/pair.hpp"
 #include "duckdb/planner/expression/list.hpp"
-#include "duckdb/planner/expression_iterator.hpp"
 #include "duckdb/planner/operator/list.hpp"
 
 namespace duckdb {
@@ -32,7 +31,7 @@ unique_ptr<LogicalOperator> JoinOrderOptimizer::Optimize(unique_ptr<LogicalOpera
 	// We optimize the children of any non-reorderable operations we come across.
 	bool reorderable = query_graph_manager.Build(*op);
 
-	// get relation_stats here since the reconstruction process will move all of the relations.
+	// get relation_stats here since the reconstruction process will move all relations.
 	auto relation_stats = query_graph_manager.relation_manager.GetRelationStats();
 	unique_ptr<LogicalOperator> new_logical_plan = nullptr;
 
@@ -46,13 +45,11 @@ unique_ptr<LogicalOperator> JoinOrderOptimizer::Optimize(unique_ptr<LogicalOpera
 
 		// Initialize the leaf/single node plans
 		plan_enumerator.InitLeafPlans();
-
-		// Ask the plan enumerator to enumerate a number of join orders
-		auto final_plan = plan_enumerator.SolveJoinOrder();
-		// TODO: add in the check that if no plan exists, you have to add a cross product.
-
+		plan_enumerator.SolveJoinOrder();
 		// now reconstruct a logical plan from the query graph plan
-		new_logical_plan = query_graph_manager.Reconstruct(std::move(plan), *final_plan);
+		query_graph_manager.plans = &plan_enumerator.GetPlans();
+
+		new_logical_plan = query_graph_manager.Reconstruct(std::move(plan));
 	} else {
 		new_logical_plan = std::move(plan);
 		if (relation_stats.size() == 1) {

package/src/duckdb/src/optimizer/join_order/plan_enumerator.cpp

@@ -8,25 +8,6 @@
 
 namespace duckdb {
 
-bool PlanEnumerator::NodeInFullPlan(JoinNode &node) {
-	return join_nodes_in_full_plan.find(node.set.ToString()) != join_nodes_in_full_plan.end();
-}
-
-void PlanEnumerator::UpdateJoinNodesInFullPlan(JoinNode &node) {
-	if (node.set.count == query_graph_manager.relation_manager.NumRelations()) {
-		join_nodes_in_full_plan.clear();
-	}
-	if (node.set.count < query_graph_manager.relation_manager.NumRelations()) {
-		join_nodes_in_full_plan.insert(node.set.ToString());
-	}
-	if (node.left) {
-		UpdateJoinNodesInFullPlan(*node.left);
-	}
-	if (node.right) {
-		UpdateJoinNodesInFullPlan(*node.right);
-	}
-}
-
 static vector<unordered_set<idx_t>> AddSuperSets(const vector<unordered_set<idx_t>> &current,
                                                  const vector<idx_t> &all_neighbors) {
 	vector<unordered_set<idx_t>> ret;
@@ -110,10 +91,14 @@ void PlanEnumerator::GenerateCrossProducts() {
 	// query_graph = query_graph_manager.GetQueryGraph();
 }
 
+const reference_map_t<JoinRelationSet, unique_ptr<DPJoinNode>> &PlanEnumerator::GetPlans() const {
+	return plans;
+}
+
 //! Create a new JoinTree node by joining together two previous JoinTree nodes
-unique_ptr<JoinNode> PlanEnumerator::CreateJoinTree(JoinRelationSet &set,
-                                                    const vector<reference<NeighborInfo>> &possible_connections,
-                                                    JoinNode &left, JoinNode &right) {
+unique_ptr<DPJoinNode> PlanEnumerator::CreateJoinTree(JoinRelationSet &set,
+                                                      const vector<reference<NeighborInfo>> &possible_connections,
+                                                      DPJoinNode &left, DPJoinNode &right) {
 	// for the hash join we want the right side (build side) to have the smallest cardinality
 	// also just a heuristic but for now...
 	// FIXME: we should probably actually benchmark that as well
@@ -126,13 +111,13 @@ unique_ptr<JoinNode> PlanEnumerator::CreateJoinTree(JoinRelationSet &set,
 	}
 
 	auto cost = cost_model.ComputeCost(left, right);
-	auto result = make_uniq<JoinNode>(set, best_connection, left, right, cost);
+	auto result = make_uniq<DPJoinNode>(set, best_connection, left.set, right.set, cost);
 	result->cardinality = cost_model.cardinality_estimator.EstimateCardinalityWithSet<idx_t>(set);
 	return result;
 }
 
-JoinNode &PlanEnumerator::EmitPair(JoinRelationSet &left, JoinRelationSet &right,
-                                   const vector<reference<NeighborInfo>> &info) {
+DPJoinNode &PlanEnumerator::EmitPair(JoinRelationSet &left, JoinRelationSet &right,
+                                     const vector<reference<NeighborInfo>> &info) {
 	// get the left and right join plans
 	auto left_plan = plans.find(left);
 	auto right_plan = plans.find(right);
@@ -150,33 +135,11 @@ JoinNode &PlanEnumerator::EmitPair(JoinRelationSet &left, JoinRelationSet &right
 		old_cost = entry->second->cost;
 	}
 	if (entry == plans.end() || new_cost < old_cost) {
-		// the new plan costs less than the old plan. Update our DP tree and cost tree
-		auto &result = *new_plan;
-
-		if (full_plan_found &&
-		    join_nodes_in_full_plan.find(new_plan->set.ToString()) != join_nodes_in_full_plan.end()) {
-			must_update_full_plan = true;
-		}
-		if (new_set.count == query_graph_manager.relation_manager.NumRelations()) {
-			full_plan_found = true;
-			// If we find a full plan, we need to keep track of which nodes are in the full plan.
-			// It's possible the DP algorithm updates a node in the current full plan, then moves on
-			// to the SolveApproximately. SolveApproximately may find a full plan with a higher cost than
-			// what SolveExactly found. In this case, we revert to the SolveExactly plan, but it is
-			// possible to get use-after-free errors if the SolveApproximately algorithm updated some (but not all)
-			// nodes in the SolveExactly plan
-			// If we know a node in the full plan is updated, we can prevent ourselves from exiting the
-			// DP algorithm until the last plan updated is a full plan
-			UpdateJoinNodesInFullPlan(result);
-			if (must_update_full_plan) {
-				must_update_full_plan = false;
-			}
-		}
-
-		D_ASSERT(new_plan);
+		// the new plan costs less than the old plan. Update our DP table.
 		plans[new_set] = std::move(new_plan);
-		return result;
+		return *plans[new_set];
 	}
+	// Create join node from the plan currently in the DP table.
 	return *entry->second;
 }
 
@@ -186,7 +149,7 @@ bool PlanEnumerator::TryEmitPair(JoinRelationSet &left, JoinRelationSet &right,
 	// If a full plan is created, it's possible a node in the plan gets updated. When this happens, make sure you keep
 	// emitting pairs until you emit another final plan. Another final plan is guaranteed to be produced because of
 	// our symmetry guarantees.
-	if (pairs >= 10000 && !must_update_full_plan) {
+	if (pairs >= 10000) {
 		// when the amount of pairs gets too large we exit the dynamic programming and resort to a greedy algorithm
 		// FIXME: simple heuristic currently
 		// at 10K pairs stop searching exactly and switch to heuristic
@@ -218,7 +181,7 @@ bool PlanEnumerator::EmitCSG(JoinRelationSet &node) {
 		D_ASSERT(neighbors[i] > neighbors[i + 1]);
 	}
 
-	// Dphyp paper missiing this.
+	// Dphyp paper missing this.
 	// Because we are traversing in reverse order, we need to add neighbors whose number is smaller than the current
 	// node to exclusion_set
 	// This avoids duplicated enumeration
@@ -353,46 +316,6 @@ bool PlanEnumerator::SolveJoinOrderExactly() {
 	return true;
 }
 
-void PlanEnumerator::UpdateDPTree(JoinNode &new_plan) {
-	if (!NodeInFullPlan(new_plan)) {
-		// if the new node is not in the full plan, feel free to return
-		// because you won't be updating the full plan.
-		return;
-	}
-	auto &new_set = new_plan.set;
-	// now update every plan that uses this plan
-	unordered_set<idx_t> exclusion_set;
-	for (idx_t i = 0; i < new_set.count; i++) {
-		exclusion_set.insert(new_set.relations[i]);
-	}
-	auto neighbors = query_graph.GetNeighbors(new_set, exclusion_set);
-	auto all_neighbors = GetAllNeighborSets(neighbors);
-	for (const auto &neighbor : all_neighbors) {
-		auto &neighbor_relation = query_graph_manager.set_manager.GetJoinRelation(neighbor);
-		auto &combined_set = query_graph_manager.set_manager.Union(new_set, neighbor_relation);
-
-		auto combined_set_plan = plans.find(combined_set);
-		if (combined_set_plan == plans.end()) {
-			continue;
-		}
-
-		double combined_set_plan_cost = combined_set_plan->second->cost; // combined_set_plan->second->GetCost();
-		auto connections = query_graph.GetConnections(new_set, neighbor_relation);
-		// recurse and update up the tree if the combined set produces a plan with a lower cost
-		// only recurse on neighbor relations that have plans.
-		auto right_plan = plans.find(neighbor_relation);
-		if (right_plan == plans.end()) {
-			continue;
-		}
-		auto &updated_plan = EmitPair(new_set, neighbor_relation, connections);
-		// <= because the child node has already been replaced. You need to
-		// replace the parent node as well in this case
-		if (updated_plan.cost < combined_set_plan_cost) {
-			UpdateDPTree(updated_plan);
-		}
-	}
-}
-
 void PlanEnumerator::SolveJoinOrderApproximately() {
 	// at this point, we exited the dynamic programming but did not compute the final join order because it took too
 	// long instead, we use a greedy heuristic to obtain a join ordering now we use Greedy Operator Ordering to
@@ -405,8 +328,10 @@ void PlanEnumerator::SolveJoinOrderApproximately() {
 		// now in every step of the algorithm, we greedily pick the join between the to-be-joined relations that has the
 		// smallest cost. This is O(r^2) per step, and every step will reduce the total amount of relations to-be-joined
 		// by 1, so the total cost is O(r^3) in the amount of relations
+		// long is needed to prevent clang-tidy complaints. (idx_t) cannot be added to an iterator position because it
+		// is unsigned.
 		idx_t best_left = 0, best_right = 0;
-		optional_ptr<JoinNode> best_connection;
+		optional_ptr<DPJoinNode> best_connection;
 		for (idx_t i = 0; i < join_relations.size(); i++) {
 			auto left = join_relations[i];
 			for (idx_t j = i + 1; j < join_relations.size(); j++) {
@@ -415,17 +340,16 @@ void PlanEnumerator::SolveJoinOrderApproximately() {
 				auto connection = query_graph.GetConnections(left, right);
 				if (!connection.empty()) {
 					// we can check the cost of this connection
-					auto &node = EmitPair(left, right, connection);
+					auto node = EmitPair(left, right, connection);
 
 					// update the DP tree in case a plan created by the DP algorithm uses the node
 					// that was potentially just updated by EmitPair. You will get a use-after-free
 					// error if future plans rely on the old node that was just replaced.
 					// if node in FullPath, then updateDP tree.
-					UpdateDPTree(node);
 
 					if (!best_connection || node.cost < best_connection->cost) {
 						// best pair found so far
-						best_connection = &node;
+						best_connection = &EmitPair(left, right, connection);
 						best_left = i;
 						best_right = j;
 					}
@@ -435,14 +359,14 @@ void PlanEnumerator::SolveJoinOrderApproximately() {
 		if (!best_connection) {
 			// could not find a connection, but we were not done with finding a completed plan
 			// we have to add a cross product; we add it between the two smallest relations
-			optional_ptr<JoinNode> smallest_plans[2];
-			idx_t smallest_index[2];
+			optional_ptr<DPJoinNode> smallest_plans[2];
+			size_t smallest_index[2];
 			D_ASSERT(join_relations.size() >= 2);
 
 			// first just add the first two join relations. It doesn't matter the cost as the JOO
 			// will swap them on estimated cardinality anyway.
 			for (idx_t i = 0; i < 2; i++) {
-				auto current_plan = plans[join_relations[i]].get();
+				optional_ptr<DPJoinNode> current_plan = plans[join_relations[i]];
 				smallest_plans[i] = current_plan;
 				smallest_index[i] = i;
 			}
@@ -451,7 +375,7 @@ void PlanEnumerator::SolveJoinOrderApproximately() {
 			// add them if they have lower estimated cardinality.
 			for (idx_t i = 2; i < join_relations.size(); i++) {
 				// get the plan for this relation
-				auto current_plan = plans[join_relations[i].get()].get();
+				optional_ptr<DPJoinNode> current_plan = plans[join_relations[i]];
 				// check if the cardinality is smaller than the smallest two found so far
 				for (idx_t j = 0; j < 2; j++) {
 					if (!smallest_plans[j] || smallest_plans[j]->cost > current_plan->cost) {
@@ -478,7 +402,6 @@ void PlanEnumerator::SolveJoinOrderApproximately() {
 			best_left = smallest_index[0];
 			best_right = smallest_index[1];
 
-			UpdateDPTree(*best_connection);
 			// the code below assumes best_right > best_left
 			if (best_left > best_right) {
 				std::swap(best_left, best_right);
@@ -489,10 +412,12 @@ void PlanEnumerator::SolveJoinOrderApproximately() {
 
 		// important to erase the biggest element first
 		// if we erase the smallest element first the index of the biggest element changes
+		auto &new_set = query_graph_manager.set_manager.Union(join_relations.at(best_left).get(),
+		                                                      join_relations.at(best_right).get());
 		D_ASSERT(best_right > best_left);
-		join_relations.erase(join_relations.begin() + best_right);
-		join_relations.erase(join_relations.begin() + best_left);
-		join_relations.push_back(best_connection->set);
+		join_relations.erase(join_relations.begin() + (int64_t)best_right);
+		join_relations.erase(join_relations.begin() + (int64_t)best_left);
+		join_relations.push_back(new_set);
 	}
 }
 
@@ -510,9 +435,10 @@ void PlanEnumerator::InitLeafPlans() {
 	for (idx_t i = 0; i < relation_stats.size(); i++) {
 		auto stats = relation_stats.at(i);
 		auto &relation_set = query_graph_manager.set_manager.GetJoinRelation(i);
-		auto join_node = make_uniq<JoinNode>(relation_set);
+		auto join_node = make_uniq<DPJoinNode>(relation_set);
 		join_node->cost = 0;
 		join_node->cardinality = stats.cardinality;
+		D_ASSERT(join_node->set.count == 1);
 		plans[relation_set] = std::move(join_node);
 		cost_model.cardinality_estimator.InitCardinalityEstimatorProps(&relation_set, stats);
 	}
@@ -521,7 +447,7 @@ void PlanEnumerator::InitLeafPlans() {
 // the plan enumeration is a straight implementation of the paper "Dynamic Programming Strikes Back" by Guido
 // Moerkotte and Thomas Neumannn, see that paper for additional info/documentation bonus slides:
 // https://db.in.tum.de/teaching/ws1415/queryopt/chapter3.pdf?lang=de
-unique_ptr<JoinNode> PlanEnumerator::SolveJoinOrder() {
+void PlanEnumerator::SolveJoinOrder() {
 	bool force_no_cross_product = query_graph_manager.context.config.force_no_cross_product;
 	// first try to solve the join order exactly
 	if (!SolveJoinOrderExactly()) {
@@ -549,7 +475,6 @@ unique_ptr<JoinNode> PlanEnumerator::SolveJoinOrder() {
 		//! solve the join order again, returning the final plan
 		return SolveJoinOrder();
 	}
-	return std::move(final_plan->second);
 }
 
 } // namespace duckdb

package/src/duckdb/src/optimizer/join_order/query_graph_manager.cpp

@@ -2,7 +2,6 @@
 
 #include "duckdb/common/assert.hpp"
 #include "duckdb/common/enums/join_type.hpp"
-#include "duckdb/common/printer.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/execution/physical_plan_generator.hpp"
 #include "duckdb/optimizer/join_order/join_relation.hpp"
@@ -22,7 +21,6 @@ static bool Disjoint(const unordered_set<T> &a, const unordered_set<T> &b) {
 }
 
 bool QueryGraphManager::Build(LogicalOperator &op) {
-	vector<reference<LogicalOperator>> filter_operators;
 	// have the relation manager extract the join relations and create a reference list of all the
 	// filter operators.
 	auto can_reorder = relation_manager.ExtractJoinRelations(op, filter_operators);
@@ -116,29 +114,81 @@ static unique_ptr<LogicalOperator> ExtractJoinRelation(unique_ptr<SingleJoinRela
 		if (children[i].get() == &rel->op) {
 			// found it! take ownership o/**/f it from the parent
 			auto result = std::move(children[i]);
-			children.erase(children.begin() + i);
+			children.erase_at(i);
 			return result;
 		}
 	}
 	throw InternalException("Could not find relation in parent node (?)");
 }
 
-unique_ptr<LogicalOperator> QueryGraphManager::Reconstruct(unique_ptr<LogicalOperator> plan, JoinNode &node) {
-	return RewritePlan(std::move(plan), node);
+unique_ptr<LogicalOperator> QueryGraphManager::Reconstruct(unique_ptr<LogicalOperator> plan) {
+	// now we have to rewrite the plan
+	bool root_is_join = plan->children.size() > 1;
+
+	unordered_set<idx_t> bindings;
+	for (idx_t i = 0; i < relation_manager.NumRelations(); i++) {
+		bindings.insert(i);
+	}
+	auto &total_relation = set_manager.GetJoinRelation(bindings);
+
+	// first we will extract all relations from the main plan
+	vector<unique_ptr<LogicalOperator>> extracted_relations;
+	extracted_relations.reserve(relation_manager.NumRelations());
+	for (auto &relation : relation_manager.GetRelations()) {
+		extracted_relations.push_back(ExtractJoinRelation(relation));
+	}
+
+	// now we generate the actual joins
+	auto join_tree = GenerateJoins(extracted_relations, total_relation);
+
+	// perform the final pushdown of remaining filters
+	for (auto &filter : filters_and_bindings) {
+		// check if the filter has already been extracted
+		if (filter->filter) {
+			// if not we need to push it
+			join_tree.op = PushFilter(std::move(join_tree.op), std::move(filter->filter));
+		}
+	}
+
+	// find the first join in the relation to know where to place this node
+	if (root_is_join) {
+		// first node is the join, return it immediately
+		return std::move(join_tree.op);
+	}
+	D_ASSERT(plan->children.size() == 1);
+	// have to move up through the relations
+	auto op = plan.get();
+	auto parent = plan.get();
+	while (op->type != LogicalOperatorType::LOGICAL_CROSS_PRODUCT &&
+	       op->type != LogicalOperatorType::LOGICAL_COMPARISON_JOIN &&
+	       op->type != LogicalOperatorType::LOGICAL_ASOF_JOIN) {
+		D_ASSERT(op->children.size() == 1);
+		parent = op;
+		op = op->children[0].get();
+	}
+	// have to replace at this node
+	parent->children[0] = std::move(join_tree.op);
+	return plan;
 }
 
 GenerateJoinRelation QueryGraphManager::GenerateJoins(vector<unique_ptr<LogicalOperator>> &extracted_relations,
-                                                      JoinNode &node) {
+                                                      JoinRelationSet &set) {
 	optional_ptr<JoinRelationSet> left_node;
 	optional_ptr<JoinRelationSet> right_node;
 	optional_ptr<JoinRelationSet> result_relation;
 	unique_ptr<LogicalOperator> result_operator;
-	if (node.left && node.right && node.info) {
-		// generate the left and right children
-		auto left = GenerateJoins(extracted_relations, *node.left);
-		auto right = GenerateJoins(extracted_relations, *node.right);
 
-		if (node.info->filters.empty()) {
+	auto dp_entry = plans->find(set);
+	if (dp_entry == plans->end()) {
+		throw InternalException("Join Order Optimizer Error: No full plan was created");
+	}
+	auto &node = dp_entry->second;
+	if (!dp_entry->second->is_leaf) {
+
+		// generate the left and right children
+		auto left = GenerateJoins(extracted_relations, node->left_set);
+		auto right = GenerateJoins(extracted_relations, node->right_set);
+		if (dp_entry->second->info->filters.empty()) {
 			// no filters, create a cross product
 			result_operator = LogicalCrossProduct::Create(std::move(left.op), std::move(right.op));
 		} else {
@@ -150,7 +200,7 @@ GenerateJoinRelation QueryGraphManager::GenerateJoins(vector<unique_ptr<LogicalO
 			join->children.push_back(std::move(right.op));
 
 			// set the join conditions from the join node
-			for (auto &filter_ref : node.info->filters) {
+			for (auto &filter_ref : node->info->filters) {
 				auto f = filter_ref.get();
 				// extract the filter from the operator it originally belonged to
 				D_ASSERT(filters_and_bindings[f->filter_index]->filter);
@@ -185,23 +235,23 @@ GenerateJoinRelation QueryGraphManager::GenerateJoins(vector<unique_ptr<LogicalO
 		result_relation = &set_manager.Union(*left.set, *right.set);
 	} else {
 		// base node, get the entry from the list of extracted relations
-		D_ASSERT(node.set.count == 1);
-		D_ASSERT(extracted_relations[node.set.relations[0]]);
-		result_relation = &node.set;
-		result_operator = std::move(extracted_relations[node.set.relations[0]]);
+		D_ASSERT(node->set.count == 1);
+		D_ASSERT(extracted_relations[node->set.relations[0]]);
+		result_relation = &node->set;
+		result_operator = std::move(extracted_relations[result_relation->relations[0]]);
 	}
 	// TODO: this is where estimated properties start coming into play.
 	//  when creating the result operator, we should ask the cost model and cardinality estimator what
 	//  the cost and cardinality are
 	//	result_operator->estimated_props = node.estimated_props->Copy();
-	result_operator->estimated_cardinality = node.cardinality;
+	result_operator->estimated_cardinality = node->cardinality;
 	result_operator->has_estimated_cardinality = true;
 	if (result_operator->type == LogicalOperatorType::LOGICAL_FILTER &&
 	    result_operator->children[0]->type == LogicalOperatorType::LOGICAL_GET) {
 		// FILTER on top of GET, add estimated properties to both
 		// auto &filter_props = *result_operator->estimated_props;
 		auto &child_operator = *result_operator->children[0];
-		child_operator.estimated_cardinality = node.cardinality;
+		child_operator.estimated_cardinality = node->cardinality;
 		child_operator.has_estimated_cardinality = true;
 	}
 	// check if we should do a pushdown on this node
@@ -292,49 +342,6 @@ void QueryGraphManager::CreateQueryGraphCrossProduct(JoinRelationSet &left, Join
 	query_graph.CreateEdge(right, left, nullptr);
 }
 
-unique_ptr<LogicalOperator> QueryGraphManager::RewritePlan(unique_ptr<LogicalOperator> plan, JoinNode &node) {
-	// now we have to rewrite the plan
-	bool root_is_join = plan->children.size() > 1;
-
-	// first we will extract all relations from the main plan
-	vector<unique_ptr<LogicalOperator>> extracted_relations;
-	extracted_relations.reserve(relation_manager.NumRelations());
-	for (auto &relation : relation_manager.GetRelations()) {
-		extracted_relations.push_back(ExtractJoinRelation(relation));
-	}
-
-	// now we generate the actual joins
-	auto join_tree = GenerateJoins(extracted_relations, node);
-	// perform the final pushdown of remaining filters
-	for (auto &filter : filters_and_bindings) {
-		// check if the filter has already been extracted
-		if (filter->filter) {
-			// if not we need to push it
-			join_tree.op = PushFilter(std::move(join_tree.op), std::move(filter->filter));
-		}
-	}
-
-	// find the first join in the relation to know where to place this node
-	if (root_is_join) {
-		// first node is the join, return it immediately
-		return std::move(join_tree.op);
-	}
-	D_ASSERT(plan->children.size() == 1);
-	// have to move up through the relations
-	auto op = plan.get();
-	auto parent = plan.get();
-	while (op->type != LogicalOperatorType::LOGICAL_CROSS_PRODUCT &&
-	       op->type != LogicalOperatorType::LOGICAL_COMPARISON_JOIN &&
-	       op->type != LogicalOperatorType::LOGICAL_ASOF_JOIN) {
-		D_ASSERT(op->children.size() == 1);
-		parent = op;
-		op = op->children[0].get();
-	}
-	// have to replace at this node
-	parent->children[0] = std::move(join_tree.op);
-	return plan;
-}
-
 static void FlipChildren(LogicalOperator &op) {
 	std::swap(op.children[0], op.children[1]);
 	if (op.type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN || op.type == LogicalOperatorType::LOGICAL_DELIM_JOIN) {

package/src/duckdb/src/optimizer/join_order/relation_manager.cpp

@@ -183,7 +183,8 @@ bool RelationManager::ExtractJoinRelations(LogicalOperator &input_op,
 		auto &aggr = op->Cast<LogicalAggregate>();
 		auto operator_stats = RelationStatisticsHelper::ExtractAggregationStats(aggr, child_stats);
 		if (!datasource_filters.empty()) {
-			operator_stats.cardinality *= RelationStatisticsHelper::DEFAULT_SELECTIVITY;
+			operator_stats.cardinality = NumericCast<idx_t>(static_cast<double>(operator_stats.cardinality) *
+			                                                RelationStatisticsHelper::DEFAULT_SELECTIVITY);
 		}
 		AddAggregateOrWindowRelation(input_op, parent, operator_stats, op->type);
 		return true;
@@ -196,7 +197,8 @@ bool RelationManager::ExtractJoinRelations(LogicalOperator &input_op,
 		auto &window = op->Cast<LogicalWindow>();
 		auto operator_stats = RelationStatisticsHelper::ExtractWindowStats(window, child_stats);
 		if (!datasource_filters.empty()) {
-			operator_stats.cardinality *= RelationStatisticsHelper::DEFAULT_SELECTIVITY;
+			operator_stats.cardinality = NumericCast<idx_t>(static_cast<double>(operator_stats.cardinality) *
+			                                                RelationStatisticsHelper::DEFAULT_SELECTIVITY);
 		}
 		AddAggregateOrWindowRelation(input_op, parent, operator_stats, op->type);
 		return true;

package/src/duckdb/src/optimizer/join_order/relation_statistics_helper.cpp

@@ -121,8 +121,8 @@ RelationStats RelationStatisticsHelper::ExtractGetStats(LogicalGet &get, ClientC
 		// and there are other table filters (i.e cost > 50), use default selectivity.
 		bool has_equality_filter = (cardinality_after_filters != base_table_cardinality);
 		if (!has_equality_filter && !get.table_filters.filters.empty()) {
-			cardinality_after_filters =
-			    MaxValue<idx_t>(base_table_cardinality * RelationStatisticsHelper::DEFAULT_SELECTIVITY, 1);
+			cardinality_after_filters = MaxValue<idx_t>(
+			    NumericCast<idx_t>(base_table_cardinality * RelationStatisticsHelper::DEFAULT_SELECTIVITY), 1U);
 		}
 		if (base_table_cardinality == 0) {
 			cardinality_after_filters = 0;
@@ -345,7 +345,7 @@ RelationStats RelationStatisticsHelper::ExtractAggregationStats(LogicalAggregate
 		// most likely we are running on parquet files. Therefore we divide by 2.
 		new_card = (double)child_stats.cardinality / 2;
 	}
-	stats.cardinality = new_card;
+	stats.cardinality = NumericCast<idx_t>(new_card);
 	stats.column_names = child_stats.column_names;
 	stats.stats_initialized = true;
 	auto num_child_columns = aggr.GetColumnBindings().size();