duckdb 0.10.3-dev0.0 → 0.10.3-dev13.0

package/src/duckdb/src/optimizer/join_order/plan_enumerator.cpp

@@ -8,25 +8,6 @@
 
 namespace duckdb {
 
-bool PlanEnumerator::NodeInFullPlan(JoinNode &node) {
-	return join_nodes_in_full_plan.find(node.set.ToString()) != join_nodes_in_full_plan.end();
-}
-
-void PlanEnumerator::UpdateJoinNodesInFullPlan(JoinNode &node) {
-	if (node.set.count == query_graph_manager.relation_manager.NumRelations()) {
-		join_nodes_in_full_plan.clear();
-	}
-	if (node.set.count < query_graph_manager.relation_manager.NumRelations()) {
-		join_nodes_in_full_plan.insert(node.set.ToString());
-	}
-	if (node.left) {
-		UpdateJoinNodesInFullPlan(*node.left);
-	}
-	if (node.right) {
-		UpdateJoinNodesInFullPlan(*node.right);
-	}
-}
-
 static vector<unordered_set<idx_t>> AddSuperSets(const vector<unordered_set<idx_t>> &current,
                                                  const vector<idx_t> &all_neighbors) {
 	vector<unordered_set<idx_t>> ret;
@@ -110,10 +91,14 @@ void PlanEnumerator::GenerateCrossProducts() {
 	// query_graph = query_graph_manager.GetQueryGraph();
 }
 
+const reference_map_t<JoinRelationSet, unique_ptr<DPJoinNode>> &PlanEnumerator::GetPlans() const {
+	return plans;
+}
+
 //! Create a new JoinTree node by joining together two previous JoinTree nodes
-unique_ptr<JoinNode> PlanEnumerator::CreateJoinTree(JoinRelationSet &set,
-                                                    const vector<reference<NeighborInfo>> &possible_connections,
-                                                    JoinNode &left, JoinNode &right) {
+unique_ptr<DPJoinNode> PlanEnumerator::CreateJoinTree(JoinRelationSet &set,
+                                                      const vector<reference<NeighborInfo>> &possible_connections,
+                                                      DPJoinNode &left, DPJoinNode &right) {
 	// for the hash join we want the right side (build side) to have the smallest cardinality
 	// also just a heuristic but for now...
 	// FIXME: we should probably actually benchmark that as well
@@ -126,13 +111,13 @@ unique_ptr<JoinNode> PlanEnumerator::CreateJoinTree(JoinRelationSet &set,
 	}
 
 	auto cost = cost_model.ComputeCost(left, right);
-	auto result = make_uniq<JoinNode>(set, best_connection, left, right, cost);
+	auto result = make_uniq<DPJoinNode>(set, best_connection, left.set, right.set, cost);
 	result->cardinality = cost_model.cardinality_estimator.EstimateCardinalityWithSet<idx_t>(set);
 	return result;
 }
 
-JoinNode &PlanEnumerator::EmitPair(JoinRelationSet &left, JoinRelationSet &right,
-                                   const vector<reference<NeighborInfo>> &info) {
+DPJoinNode &PlanEnumerator::EmitPair(JoinRelationSet &left, JoinRelationSet &right,
+                                     const vector<reference<NeighborInfo>> &info) {
 	// get the left and right join plans
 	auto left_plan = plans.find(left);
 	auto right_plan = plans.find(right);
@@ -150,33 +135,11 @@ JoinNode &PlanEnumerator::EmitPair(JoinRelationSet &left, JoinRelationSet &right
 		old_cost = entry->second->cost;
 	}
 	if (entry == plans.end() || new_cost < old_cost) {
-		// the new plan costs less than the old plan. Update our DP tree and cost tree
-		auto &result = *new_plan;
-
-		if (full_plan_found &&
-		    join_nodes_in_full_plan.find(new_plan->set.ToString()) != join_nodes_in_full_plan.end()) {
-			must_update_full_plan = true;
-		}
-		if (new_set.count == query_graph_manager.relation_manager.NumRelations()) {
-			full_plan_found = true;
-			// If we find a full plan, we need to keep track of which nodes are in the full plan.
-			// It's possible the DP algorithm updates a node in the current full plan, then moves on
-			// to the SolveApproximately. SolveApproximately may find a full plan with a higher cost than
-			// what SolveExactly found. In this case, we revert to the SolveExactly plan, but it is
-			// possible to get use-after-free errors if the SolveApproximately algorithm updated some (but not all)
-			// nodes in the SolveExactly plan
-			// If we know a node in the full plan is updated, we can prevent ourselves from exiting the
-			// DP algorithm until the last plan updated is a full plan
-			UpdateJoinNodesInFullPlan(result);
-			if (must_update_full_plan) {
-				must_update_full_plan = false;
-			}
-		}
-
-		D_ASSERT(new_plan);
+		// the new plan costs less than the old plan. Update our DP table.
 		plans[new_set] = std::move(new_plan);
-		return result;
+		return *plans[new_set];
 	}
+	// Create join node from the plan currently in the DP table.
 	return *entry->second;
 }
 
@@ -186,7 +149,7 @@ bool PlanEnumerator::TryEmitPair(JoinRelationSet &left, JoinRelationSet &right,
 	// If a full plan is created, it's possible a node in the plan gets updated. When this happens, make sure you keep
 	// emitting pairs until you emit another final plan. Another final plan is guaranteed to be produced because of
 	// our symmetry guarantees.
-	if (pairs >= 10000 && !must_update_full_plan) {
+	if (pairs >= 10000) {
 		// when the amount of pairs gets too large we exit the dynamic programming and resort to a greedy algorithm
 		// FIXME: simple heuristic currently
 		// at 10K pairs stop searching exactly and switch to heuristic
@@ -218,7 +181,7 @@ bool PlanEnumerator::EmitCSG(JoinRelationSet &node) {
 		D_ASSERT(neighbors[i] > neighbors[i + 1]);
 	}
 
-	// Dphyp paper missiing this.
+	// Dphyp paper missing this.
 	// Because we are traversing in reverse order, we need to add neighbors whose number is smaller than the current
 	// node to exclusion_set
 	// This avoids duplicated enumeration
@@ -353,46 +316,6 @@ bool PlanEnumerator::SolveJoinOrderExactly() {
 	return true;
 }
 
-void PlanEnumerator::UpdateDPTree(JoinNode &new_plan) {
-	if (!NodeInFullPlan(new_plan)) {
-		// if the new node is not in the full plan, feel free to return
-		// because you won't be updating the full plan.
-		return;
-	}
-	auto &new_set = new_plan.set;
-	// now update every plan that uses this plan
-	unordered_set<idx_t> exclusion_set;
-	for (idx_t i = 0; i < new_set.count; i++) {
-		exclusion_set.insert(new_set.relations[i]);
-	}
-	auto neighbors = query_graph.GetNeighbors(new_set, exclusion_set);
-	auto all_neighbors = GetAllNeighborSets(neighbors);
-	for (const auto &neighbor : all_neighbors) {
-		auto &neighbor_relation = query_graph_manager.set_manager.GetJoinRelation(neighbor);
-		auto &combined_set = query_graph_manager.set_manager.Union(new_set, neighbor_relation);
-
-		auto combined_set_plan = plans.find(combined_set);
-		if (combined_set_plan == plans.end()) {
-			continue;
-		}
-
-		double combined_set_plan_cost = combined_set_plan->second->cost; // combined_set_plan->second->GetCost();
-		auto connections = query_graph.GetConnections(new_set, neighbor_relation);
-		// recurse and update up the tree if the combined set produces a plan with a lower cost
-		// only recurse on neighbor relations that have plans.
-		auto right_plan = plans.find(neighbor_relation);
-		if (right_plan == plans.end()) {
-			continue;
-		}
-		auto &updated_plan = EmitPair(new_set, neighbor_relation, connections);
-		// <= because the child node has already been replaced. You need to
-		// replace the parent node as well in this case
-		if (updated_plan.cost < combined_set_plan_cost) {
-			UpdateDPTree(updated_plan);
-		}
-	}
-}
-
 void PlanEnumerator::SolveJoinOrderApproximately() {
 	// at this point, we exited the dynamic programming but did not compute the final join order because it took too
 	// long instead, we use a greedy heuristic to obtain a join ordering now we use Greedy Operator Ordering to
@@ -405,8 +328,10 @@ void PlanEnumerator::SolveJoinOrderApproximately() {
 		// now in every step of the algorithm, we greedily pick the join between the to-be-joined relations that has the
 		// smallest cost. This is O(r^2) per step, and every step will reduce the total amount of relations to-be-joined
 		// by 1, so the total cost is O(r^3) in the amount of relations
+		// long is needed to prevent clang-tidy complaints. (idx_t) cannot be added to an iterator position because it
+		// is unsigned.
 		idx_t best_left = 0, best_right = 0;
-		optional_ptr<JoinNode> best_connection;
+		optional_ptr<DPJoinNode> best_connection;
 		for (idx_t i = 0; i < join_relations.size(); i++) {
 			auto left = join_relations[i];
 			for (idx_t j = i + 1; j < join_relations.size(); j++) {
@@ -415,17 +340,16 @@ void PlanEnumerator::SolveJoinOrderApproximately() {
 				auto connection = query_graph.GetConnections(left, right);
 				if (!connection.empty()) {
 					// we can check the cost of this connection
-					auto &node = EmitPair(left, right, connection);
+					auto node = EmitPair(left, right, connection);
 
 					// update the DP tree in case a plan created by the DP algorithm uses the node
 					// that was potentially just updated by EmitPair. You will get a use-after-free
 					// error if future plans rely on the old node that was just replaced.
 					// if node in FullPath, then updateDP tree.
-					UpdateDPTree(node);
 
 					if (!best_connection || node.cost < best_connection->cost) {
 						// best pair found so far
-						best_connection = &node;
+						best_connection = &EmitPair(left, right, connection);
 						best_left = i;
 						best_right = j;
 					}
@@ -435,14 +359,14 @@ void PlanEnumerator::SolveJoinOrderApproximately() {
 		if (!best_connection) {
 			// could not find a connection, but we were not done with finding a completed plan
 			// we have to add a cross product; we add it between the two smallest relations
-			optional_ptr<JoinNode> smallest_plans[2];
-			idx_t smallest_index[2];
+			optional_ptr<DPJoinNode> smallest_plans[2];
+			size_t smallest_index[2];
 			D_ASSERT(join_relations.size() >= 2);
 
 			// first just add the first two join relations. It doesn't matter the cost as the JOO
 			// will swap them on estimated cardinality anyway.
 			for (idx_t i = 0; i < 2; i++) {
-				auto current_plan = plans[join_relations[i]].get();
+				optional_ptr<DPJoinNode> current_plan = plans[join_relations[i]];
 				smallest_plans[i] = current_plan;
 				smallest_index[i] = i;
 			}
@@ -451,7 +375,7 @@ void PlanEnumerator::SolveJoinOrderApproximately() {
 			// add them if they have lower estimated cardinality.
 			for (idx_t i = 2; i < join_relations.size(); i++) {
 				// get the plan for this relation
-				auto current_plan = plans[join_relations[i].get()].get();
+				optional_ptr<DPJoinNode> current_plan = plans[join_relations[i]];
 				// check if the cardinality is smaller than the smallest two found so far
 				for (idx_t j = 0; j < 2; j++) {
 					if (!smallest_plans[j] || smallest_plans[j]->cost > current_plan->cost) {
@@ -478,7 +402,6 @@ void PlanEnumerator::SolveJoinOrderApproximately() {
 			best_left = smallest_index[0];
 			best_right = smallest_index[1];
 
-			UpdateDPTree(*best_connection);
 			// the code below assumes best_right > best_left
 			if (best_left > best_right) {
 				std::swap(best_left, best_right);
@@ -489,10 +412,12 @@ void PlanEnumerator::SolveJoinOrderApproximately() {
 
 		// important to erase the biggest element first
 		// if we erase the smallest element first the index of the biggest element changes
+		auto &new_set = query_graph_manager.set_manager.Union(join_relations.at(best_left).get(),
+		                                                      join_relations.at(best_right).get());
 		D_ASSERT(best_right > best_left);
-		join_relations.erase(join_relations.begin() + best_right);
-		join_relations.erase(join_relations.begin() + best_left);
-		join_relations.push_back(best_connection->set);
+		join_relations.erase(join_relations.begin() + (int64_t)best_right);
+		join_relations.erase(join_relations.begin() + (int64_t)best_left);
+		join_relations.push_back(new_set);
 	}
 }
 
@@ -510,9 +435,10 @@ void PlanEnumerator::InitLeafPlans() {
 	for (idx_t i = 0; i < relation_stats.size(); i++) {
 		auto stats = relation_stats.at(i);
 		auto &relation_set = query_graph_manager.set_manager.GetJoinRelation(i);
-		auto join_node = make_uniq<JoinNode>(relation_set);
+		auto join_node = make_uniq<DPJoinNode>(relation_set);
 		join_node->cost = 0;
 		join_node->cardinality = stats.cardinality;
+		D_ASSERT(join_node->set.count == 1);
 		plans[relation_set] = std::move(join_node);
 		cost_model.cardinality_estimator.InitCardinalityEstimatorProps(&relation_set, stats);
 	}
@@ -521,7 +447,7 @@ void PlanEnumerator::InitLeafPlans() {
 // the plan enumeration is a straight implementation of the paper "Dynamic Programming Strikes Back" by Guido
 // Moerkotte and Thomas Neumannn, see that paper for additional info/documentation bonus slides:
 // https://db.in.tum.de/teaching/ws1415/queryopt/chapter3.pdf?lang=de
-unique_ptr<JoinNode> PlanEnumerator::SolveJoinOrder() {
+void PlanEnumerator::SolveJoinOrder() {
 	bool force_no_cross_product = query_graph_manager.context.config.force_no_cross_product;
 	// first try to solve the join order exactly
 	if (!SolveJoinOrderExactly()) {
@@ -549,7 +475,6 @@ unique_ptr<JoinNode> PlanEnumerator::SolveJoinOrder() {
 		//! solve the join order again, returning the final plan
 		return SolveJoinOrder();
 	}
-	return std::move(final_plan->second);
 }
 
 } // namespace duckdb

package/src/duckdb/src/optimizer/join_order/query_graph_manager.cpp

@@ -2,7 +2,6 @@
 
 #include "duckdb/common/assert.hpp"
 #include "duckdb/common/enums/join_type.hpp"
-#include "duckdb/common/printer.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/execution/physical_plan_generator.hpp"
 #include "duckdb/optimizer/join_order/join_relation.hpp"
@@ -22,7 +21,6 @@ static bool Disjoint(const unordered_set<T> &a, const unordered_set<T> &b) {
 }
 
 bool QueryGraphManager::Build(LogicalOperator &op) {
-	vector<reference<LogicalOperator>> filter_operators;
 	// have the relation manager extract the join relations and create a reference list of all the
 	// filter operators.
 	auto can_reorder = relation_manager.ExtractJoinRelations(op, filter_operators);
@@ -116,29 +114,81 @@ static unique_ptr<LogicalOperator> ExtractJoinRelation(unique_ptr<SingleJoinRela
 		if (children[i].get() == &rel->op) {
 			// found it! take ownership o/**/f it from the parent
 			auto result = std::move(children[i]);
-			children.erase(children.begin() + i);
+			children.erase_at(i);
 			return result;
 		}
 	}
 	throw InternalException("Could not find relation in parent node (?)");
 }
 
-unique_ptr<LogicalOperator> QueryGraphManager::Reconstruct(unique_ptr<LogicalOperator> plan, JoinNode &node) {
-	return RewritePlan(std::move(plan), node);
+unique_ptr<LogicalOperator> QueryGraphManager::Reconstruct(unique_ptr<LogicalOperator> plan) {
+	// now we have to rewrite the plan
+	bool root_is_join = plan->children.size() > 1;
+
+	unordered_set<idx_t> bindings;
+	for (idx_t i = 0; i < relation_manager.NumRelations(); i++) {
+		bindings.insert(i);
+	}
+	auto &total_relation = set_manager.GetJoinRelation(bindings);
+
+	// first we will extract all relations from the main plan
+	vector<unique_ptr<LogicalOperator>> extracted_relations;
+	extracted_relations.reserve(relation_manager.NumRelations());
+	for (auto &relation : relation_manager.GetRelations()) {
+		extracted_relations.push_back(ExtractJoinRelation(relation));
+	}
+
+	// now we generate the actual joins
+	auto join_tree = GenerateJoins(extracted_relations, total_relation);
+
+	// perform the final pushdown of remaining filters
+	for (auto &filter : filters_and_bindings) {
+		// check if the filter has already been extracted
+		if (filter->filter) {
+			// if not we need to push it
+			join_tree.op = PushFilter(std::move(join_tree.op), std::move(filter->filter));
+		}
+	}
+
+	// find the first join in the relation to know where to place this node
+	if (root_is_join) {
+		// first node is the join, return it immediately
+		return std::move(join_tree.op);
+	}
+	D_ASSERT(plan->children.size() == 1);
+	// have to move up through the relations
+	auto op = plan.get();
+	auto parent = plan.get();
+	while (op->type != LogicalOperatorType::LOGICAL_CROSS_PRODUCT &&
+	       op->type != LogicalOperatorType::LOGICAL_COMPARISON_JOIN &&
+	       op->type != LogicalOperatorType::LOGICAL_ASOF_JOIN) {
+		D_ASSERT(op->children.size() == 1);
+		parent = op;
+		op = op->children[0].get();
+	}
+	// have to replace at this node
+	parent->children[0] = std::move(join_tree.op);
+	return plan;
 }
 
 GenerateJoinRelation QueryGraphManager::GenerateJoins(vector<unique_ptr<LogicalOperator>> &extracted_relations,
-                                                      JoinNode &node) {
+                                                      JoinRelationSet &set) {
 	optional_ptr<JoinRelationSet> left_node;
 	optional_ptr<JoinRelationSet> right_node;
 	optional_ptr<JoinRelationSet> result_relation;
 	unique_ptr<LogicalOperator> result_operator;
-	if (node.left && node.right && node.info) {
-		// generate the left and right children
-		auto left = GenerateJoins(extracted_relations, *node.left);
-		auto right = GenerateJoins(extracted_relations, *node.right);
 
-		if (node.info->filters.empty()) {
+	auto dp_entry = plans->find(set);
+	if (dp_entry == plans->end()) {
+		throw InternalException("Join Order Optimizer Error: No full plan was created");
+	}
+	auto &node = dp_entry->second;
+	if (!dp_entry->second->is_leaf) {
+
+		// generate the left and right children
+		auto left = GenerateJoins(extracted_relations, node->left_set);
+		auto right = GenerateJoins(extracted_relations, node->right_set);
+		if (dp_entry->second->info->filters.empty()) {
 			// no filters, create a cross product
 			result_operator = LogicalCrossProduct::Create(std::move(left.op), std::move(right.op));
 		} else {
@@ -150,7 +200,7 @@ GenerateJoinRelation QueryGraphManager::GenerateJoins(vector<unique_ptr<LogicalO
 			join->children.push_back(std::move(right.op));
 
 			// set the join conditions from the join node
-			for (auto &filter_ref : node.info->filters) {
+			for (auto &filter_ref : node->info->filters) {
 				auto f = filter_ref.get();
 				// extract the filter from the operator it originally belonged to
 				D_ASSERT(filters_and_bindings[f->filter_index]->filter);
@@ -185,23 +235,23 @@ GenerateJoinRelation QueryGraphManager::GenerateJoins(vector<unique_ptr<LogicalO
 		result_relation = &set_manager.Union(*left.set, *right.set);
 	} else {
 		// base node, get the entry from the list of extracted relations
-		D_ASSERT(node.set.count == 1);
-		D_ASSERT(extracted_relations[node.set.relations[0]]);
-		result_relation = &node.set;
-		result_operator = std::move(extracted_relations[node.set.relations[0]]);
+		D_ASSERT(node->set.count == 1);
+		D_ASSERT(extracted_relations[node->set.relations[0]]);
+		result_relation = &node->set;
+		result_operator = std::move(extracted_relations[result_relation->relations[0]]);
 	}
 	// TODO: this is where estimated properties start coming into play.
 	//  when creating the result operator, we should ask the cost model and cardinality estimator what
 	//  the cost and cardinality are
 	//	result_operator->estimated_props = node.estimated_props->Copy();
-	result_operator->estimated_cardinality = node.cardinality;
+	result_operator->estimated_cardinality = node->cardinality;
 	result_operator->has_estimated_cardinality = true;
 	if (result_operator->type == LogicalOperatorType::LOGICAL_FILTER &&
 	    result_operator->children[0]->type == LogicalOperatorType::LOGICAL_GET) {
 		// FILTER on top of GET, add estimated properties to both
 		// auto &filter_props = *result_operator->estimated_props;
 		auto &child_operator = *result_operator->children[0];
-		child_operator.estimated_cardinality = node.cardinality;
+		child_operator.estimated_cardinality = node->cardinality;
 		child_operator.has_estimated_cardinality = true;
 	}
 	// check if we should do a pushdown on this node
@@ -292,49 +342,6 @@ void QueryGraphManager::CreateQueryGraphCrossProduct(JoinRelationSet &left, Join
 	query_graph.CreateEdge(right, left, nullptr);
 }
 
-unique_ptr<LogicalOperator> QueryGraphManager::RewritePlan(unique_ptr<LogicalOperator> plan, JoinNode &node) {
-	// now we have to rewrite the plan
-	bool root_is_join = plan->children.size() > 1;
-
-	// first we will extract all relations from the main plan
-	vector<unique_ptr<LogicalOperator>> extracted_relations;
-	extracted_relations.reserve(relation_manager.NumRelations());
-	for (auto &relation : relation_manager.GetRelations()) {
-		extracted_relations.push_back(ExtractJoinRelation(relation));
-	}
-
-	// now we generate the actual joins
-	auto join_tree = GenerateJoins(extracted_relations, node);
-	// perform the final pushdown of remaining filters
-	for (auto &filter : filters_and_bindings) {
-		// check if the filter has already been extracted
-		if (filter->filter) {
-			// if not we need to push it
-			join_tree.op = PushFilter(std::move(join_tree.op), std::move(filter->filter));
-		}
-	}
-
-	// find the first join in the relation to know where to place this node
-	if (root_is_join) {
-		// first node is the join, return it immediately
-		return std::move(join_tree.op);
-	}
-	D_ASSERT(plan->children.size() == 1);
-	// have to move up through the relations
-	auto op = plan.get();
-	auto parent = plan.get();
-	while (op->type != LogicalOperatorType::LOGICAL_CROSS_PRODUCT &&
-	       op->type != LogicalOperatorType::LOGICAL_COMPARISON_JOIN &&
-	       op->type != LogicalOperatorType::LOGICAL_ASOF_JOIN) {
-		D_ASSERT(op->children.size() == 1);
-		parent = op;
-		op = op->children[0].get();
-	}
-	// have to replace at this node
-	parent->children[0] = std::move(join_tree.op);
-	return plan;
-}
-
 static void FlipChildren(LogicalOperator &op) {
 	std::swap(op.children[0], op.children[1]);
 	if (op.type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN || op.type == LogicalOperatorType::LOGICAL_DELIM_JOIN) {

package/src/duckdb/src/optimizer/join_order/relation_manager.cpp

@@ -183,7 +183,8 @@ bool RelationManager::ExtractJoinRelations(LogicalOperator &input_op,
 		auto &aggr = op->Cast<LogicalAggregate>();
 		auto operator_stats = RelationStatisticsHelper::ExtractAggregationStats(aggr, child_stats);
 		if (!datasource_filters.empty()) {
-			operator_stats.cardinality *= RelationStatisticsHelper::DEFAULT_SELECTIVITY;
+			operator_stats.cardinality = NumericCast<idx_t>(static_cast<double>(operator_stats.cardinality) *
+			                                                RelationStatisticsHelper::DEFAULT_SELECTIVITY);
 		}
 		AddAggregateOrWindowRelation(input_op, parent, operator_stats, op->type);
 		return true;
@@ -196,7 +197,8 @@ bool RelationManager::ExtractJoinRelations(LogicalOperator &input_op,
 		auto &window = op->Cast<LogicalWindow>();
 		auto operator_stats = RelationStatisticsHelper::ExtractWindowStats(window, child_stats);
 		if (!datasource_filters.empty()) {
-			operator_stats.cardinality *= RelationStatisticsHelper::DEFAULT_SELECTIVITY;
+			operator_stats.cardinality = NumericCast<idx_t>(static_cast<double>(operator_stats.cardinality) *
+			                                                RelationStatisticsHelper::DEFAULT_SELECTIVITY);
 		}
 		AddAggregateOrWindowRelation(input_op, parent, operator_stats, op->type);
 		return true;

package/src/duckdb/src/optimizer/join_order/relation_statistics_helper.cpp

@@ -121,8 +121,8 @@ RelationStats RelationStatisticsHelper::ExtractGetStats(LogicalGet &get, ClientC
 		// and there are other table filters (i.e cost > 50), use default selectivity.
 		bool has_equality_filter = (cardinality_after_filters != base_table_cardinality);
 		if (!has_equality_filter && !get.table_filters.filters.empty()) {
-			cardinality_after_filters =
-			    MaxValue<idx_t>(base_table_cardinality * RelationStatisticsHelper::DEFAULT_SELECTIVITY, 1);
+			cardinality_after_filters = MaxValue<idx_t>(
+			    NumericCast<idx_t>(base_table_cardinality * RelationStatisticsHelper::DEFAULT_SELECTIVITY), 1U);
 		}
 		if (base_table_cardinality == 0) {
 			cardinality_after_filters = 0;
@@ -345,7 +345,7 @@ RelationStats RelationStatisticsHelper::ExtractAggregationStats(LogicalAggregate
 		// most likely we are running on parquet files. Therefore we divide by 2.
 		new_card = (double)child_stats.cardinality / 2;
 	}
-	stats.cardinality = new_card;
+	stats.cardinality = NumericCast<idx_t>(new_card);
 	stats.column_names = child_stats.column_names;
 	stats.stats_initialized = true;
 	auto num_child_columns = aggr.GetColumnBindings().size();

package/src/duckdb/src/optimizer/optimizer.cpp

@@ -43,6 +43,7 @@ Optimizer::Optimizer(Binder &binder, ClientContext &context) : context(context),
 	rewriter.rules.push_back(make_uniq<RegexOptimizationRule>(rewriter));
 	rewriter.rules.push_back(make_uniq<EmptyNeedleRemovalRule>(rewriter));
 	rewriter.rules.push_back(make_uniq<EnumComparisonRule>(rewriter));
+	rewriter.rules.push_back(make_uniq<TimeStampComparison>(context, rewriter));
 
 #ifdef DEBUG
 	for (auto &rule : rewriter.rules) {
@@ -193,7 +194,8 @@ unique_ptr<LogicalOperator> Optimizer::Optimize(unique_ptr<LogicalOperator> plan
 
 	for (auto &optimizer_extension : DBConfig::GetConfig(context).optimizer_extensions) {
 		RunOptimizer(OptimizerType::EXTENSION, [&]() {
-			optimizer_extension.optimize_function(context, optimizer_extension.optimizer_info.get(), plan);
+			OptimizerExtensionInput input {GetContext(), *this, optimizer_extension.optimizer_info.get()};
+			optimizer_extension.optimize_function(input, plan);
 		});
 	}
 

package/src/duckdb/src/optimizer/pushdown/pushdown_aggregate.cpp

@@ -37,7 +37,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownAggregate(unique_ptr<Logical
 
 	// pushdown into AGGREGATE and GROUP BY
 	// we cannot push expressions that refer to the aggregate
-	FilterPushdown child_pushdown(optimizer);
+	FilterPushdown child_pushdown(optimizer, convert_mark_joins);
 	for (idx_t i = 0; i < filters.size(); i++) {
 		auto &f = *filters[i];
 		if (f.bindings.find(aggr.aggregate_index) != f.bindings.end()) {
@@ -87,7 +87,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownAggregate(unique_ptr<Logical
 			return make_uniq<LogicalEmptyResult>(std::move(op));
 		}
 		// erase the filter from here
-		filters.erase(filters.begin() + i);
+		filters.erase_at(i);
 		i--;
 	}
 	child_pushdown.GenerateFilters();

package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp

@@ -8,7 +8,7 @@ using Filter = FilterPushdown::Filter;
 
 unique_ptr<LogicalOperator> FilterPushdown::PushdownCrossProduct(unique_ptr<LogicalOperator> op) {
 	D_ASSERT(op->children.size() > 1);
-	FilterPushdown left_pushdown(optimizer), right_pushdown(optimizer);
+	FilterPushdown left_pushdown(optimizer, convert_mark_joins), right_pushdown(optimizer, convert_mark_joins);
 	vector<unique_ptr<Expression>> join_expressions;
 	auto join_ref_type = JoinRefType::REGULAR;
 	switch (op->type) {
@@ -52,9 +52,9 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownCrossProduct(unique_ptr<Logi
 		vector<JoinCondition> conditions;
 		vector<unique_ptr<Expression>> arbitrary_expressions;
 		const auto join_type = JoinType::INNER;
-		LogicalComparisonJoin::ExtractJoinConditions(GetContext(), join_type, op->children[0], op->children[1],
-		                                             left_bindings, right_bindings, join_expressions, conditions,
-		                                             arbitrary_expressions);
+		LogicalComparisonJoin::ExtractJoinConditions(GetContext(), join_type, join_ref_type, op->children[0],
+		                                             op->children[1], left_bindings, right_bindings, join_expressions,
+		                                             conditions, arbitrary_expressions);
 		// create the join from the join conditions
 		return LogicalComparisonJoin::CreateJoin(GetContext(), join_type, join_ref_type, std::move(op->children[0]),
 		                                         std::move(op->children[1]), std::move(conditions),

package/src/duckdb/src/optimizer/pushdown/pushdown_left_join.cpp

@@ -64,7 +64,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownLeftJoin(unique_ptr<LogicalO
 	if (op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN) {
 		return FinishPushdown(std::move(op));
 	}
-	FilterPushdown left_pushdown(optimizer), right_pushdown(optimizer);
+	FilterPushdown left_pushdown(optimizer, convert_mark_joins), right_pushdown(optimizer, convert_mark_joins);
 	// for a comparison join we create a FilterCombiner that checks if we can push conditions on LHS join conditions
 	// into the RHS of the join
 	FilterCombiner filter_combiner(optimizer);
@@ -91,7 +91,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownLeftJoin(unique_ptr<LogicalO
 			}
 			left_pushdown.filters.push_back(std::move(filters[i]));
 			// erase the filter from the list of filters
-			filters.erase(filters.begin() + i);
+			filters.erase_at(i);
 			i--;
 		} else {
 			// bindings match right side or both sides: we cannot directly push it into the right

package/src/duckdb/src/optimizer/pushdown/pushdown_mark_join.cpp

@@ -16,7 +16,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownMarkJoin(unique_ptr<LogicalO
 	         op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN || op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN);
 
 	right_bindings.insert(comp_join.mark_index);
-	FilterPushdown left_pushdown(optimizer), right_pushdown(optimizer);
+	FilterPushdown left_pushdown(optimizer, convert_mark_joins), right_pushdown(optimizer, convert_mark_joins);
 #ifdef DEBUG
 	bool simplified_mark_join = false;
 #endif
@@ -27,7 +27,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownMarkJoin(unique_ptr<LogicalO
 			// bindings match left side: push into left
 			left_pushdown.filters.push_back(std::move(filters[i]));
 			// erase the filter from the list of filters
-			filters.erase(filters.begin() + i);
+			filters.erase_at(i);
 			i--;
 		} else if (side == JoinSide::RIGHT) {
 #ifdef DEBUG
@@ -35,13 +35,13 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownMarkJoin(unique_ptr<LogicalO
 #endif
 			// this filter references the marker
 			// we can turn this into a SEMI join if the filter is on only the marker
-			if (filters[i]->filter->type == ExpressionType::BOUND_COLUMN_REF) {
+			if (filters[i]->filter->type == ExpressionType::BOUND_COLUMN_REF && convert_mark_joins) {
 				// filter just references the marker: turn into semi join
 #ifdef DEBUG
 				simplified_mark_join = true;
 #endif
 				join.join_type = JoinType::SEMI;
-				filters.erase(filters.begin() + i);
+				filters.erase_at(i);
 				i--;
 				continue;
 			}
@@ -61,13 +61,13 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownMarkJoin(unique_ptr<LogicalO
 							break;
 						}
 					}
-					if (all_null_values_are_equal) {
+					if (all_null_values_are_equal && convert_mark_joins) {
 #ifdef DEBUG
 						simplified_mark_join = true;
 #endif
 						// all null values are equal, convert to ANTI join
 						join.join_type = JoinType::ANTI;
-						filters.erase(filters.begin() + i);
+						filters.erase_at(i);
 						i--;
 						continue;
 					}

package/src/duckdb/src/optimizer/pushdown/pushdown_projection.cpp

@@ -43,7 +43,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownProjection(unique_ptr<Logica
 	// push filter through logical projection
 	// all the BoundColumnRefExpressions in the filter should refer to the LogicalProjection
 	// we can rewrite them by replacing those references with the expression of the LogicalProjection node
-	FilterPushdown child_pushdown(optimizer);
+	FilterPushdown child_pushdown(optimizer, convert_mark_joins);
 	// There are some expressions can not be pushed down. We should keep them
 	// and add an extra filter operator.
 	vector<unique_ptr<Expression>> remain_expressions;

package/src/duckdb/src/optimizer/pushdown/pushdown_semi_anti_join.cpp

@@ -17,7 +17,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownSemiAntiJoin(unique_ptr<Logi
 
 	// push all current filters down the left side
 	op->children[0] = Rewrite(std::move(op->children[0]));
-	FilterPushdown right_pushdown(optimizer);
+	FilterPushdown right_pushdown(optimizer, convert_mark_joins);
 	op->children[1] = right_pushdown.Rewrite(std::move(op->children[1]));
 
 	bool left_empty = op->children[0]->type == LogicalOperatorType::LOGICAL_EMPTY_RESULT;