duckdb 1.0.1-dev21.0 → 1.0.1-dev27.0

package/src/duckdb/src/optimizer/join_order/relation_manager.cpp

@@ -1,16 +1,14 @@
 #include "duckdb/optimizer/join_order/relation_manager.hpp"
+
+#include "duckdb/common/enums/join_type.hpp"
+#include "duckdb/common/string_util.hpp"
 #include "duckdb/optimizer/join_order/join_order_optimizer.hpp"
 #include "duckdb/optimizer/join_order/relation_statistics_helper.hpp"
-#include "duckdb/common/printer.hpp"
-#include "duckdb/common/string_util.hpp"
-#include "duckdb/common/enums/join_type.hpp"
 #include "duckdb/parser/expression_map.hpp"
-#include "duckdb/planner/expression_iterator.hpp"
 #include "duckdb/planner/expression/list.hpp"
+#include "duckdb/planner/expression_iterator.hpp"
 #include "duckdb/planner/operator/list.hpp"
 
-#include <cmath>
-
 namespace duckdb {
 
 const vector<RelationStats> RelationManager::GetRelationStats() {
@@ -41,6 +39,8 @@ void RelationManager::AddAggregateOrWindowRelation(LogicalOperator &op, optional
 		}
 	}
 	relations.push_back(std::move(relation));
+	op.estimated_cardinality = stats.cardinality;
+	op.has_estimated_cardinality = true;
 }
 
 void RelationManager::AddRelation(LogicalOperator &op, optional_ptr<LogicalOperator> parent,
@@ -72,6 +72,12 @@ void RelationManager::AddRelation(LogicalOperator &op, optional_ptr<LogicalOpera
 		relation_mapping[table_index] = relation_id;
 	}
 	relations.push_back(std::move(relation));
+	op.estimated_cardinality = stats.cardinality;
+	op.has_estimated_cardinality = true;
+}
+
+bool RelationManager::CrossProductWithRelationAllowed(idx_t relation_id) {
+	return no_cross_product_relations.find(relation_id) == no_cross_product_relations.end();
 }
 
 static bool OperatorNeedsRelation(LogicalOperatorType op_type) {
@@ -93,7 +99,6 @@ static bool OperatorIsNonReorderable(LogicalOperatorType op_type) {
 	case LogicalOperatorType::LOGICAL_UNION:
 	case LogicalOperatorType::LOGICAL_EXCEPT:
 	case LogicalOperatorType::LOGICAL_INTERSECT:
-	case LogicalOperatorType::LOGICAL_DELIM_JOIN:
 	case LogicalOperatorType::LOGICAL_ANY_JOIN:
 	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
 		return true;
@@ -102,6 +107,48 @@ static bool OperatorIsNonReorderable(LogicalOperatorType op_type) {
 	}
 }
 
+bool ExpressionContainsColumnRef(Expression &expression) {
+	if (expression.type == ExpressionType::BOUND_COLUMN_REF) {
+		// Here you have a filter on a single column in a table. Return a binding for the column
+		// being filtered on so the filter estimator knows what HLL count to pull
+#ifdef DEBUG
+		auto &colref = expression.Cast<BoundColumnRefExpression>();
+		(void)colref.depth;
+		D_ASSERT(colref.depth == 0);
+		D_ASSERT(colref.binding.table_index != DConstants::INVALID_INDEX);
+#endif
+		// map the base table index to the relation index used by the JoinOrderOptimizer
+		return true;
+	}
+	// TODO: handle inequality filters with functions.
+	auto children_ret = false;
+	ExpressionIterator::EnumerateChildren(expression,
+	                                      [&](Expression &expr) { children_ret = ExpressionContainsColumnRef(expr); });
+	return children_ret;
+}
+
+static bool JoinIsReorderable(LogicalOperator &op) {
+	if (op.type == LogicalOperatorType::LOGICAL_CROSS_PRODUCT) {
+		return true;
+	} else if (op.type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) {
+		auto &join = op.Cast<LogicalComparisonJoin>();
+		switch (join.join_type) {
+		case JoinType::INNER:
+		case JoinType::SEMI:
+		case JoinType::ANTI:
+			for (auto &cond : join.conditions) {
+				if (ExpressionContainsColumnRef(*cond.left) && ExpressionContainsColumnRef(*cond.right)) {
+					return true;
+				}
+			}
+			return false;
+		default:
+			return false;
+		}
+	}
+	return false;
+}
+
 static bool HasNonReorderableChild(LogicalOperator &op) {
 	LogicalOperator *tmp = &op;
 	while (tmp->children.size() == 1) {
@@ -110,8 +157,7 @@ static bool HasNonReorderableChild(LogicalOperator &op) {
 		}
 		tmp = tmp->children[0].get();
 		if (tmp->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) {
-			auto &join = tmp->Cast<LogicalComparisonJoin>();
-			if (join.join_type != JoinType::INNER) {
+			if (!JoinIsReorderable(*tmp)) {
 				return true;
 			}
 		}
@@ -119,11 +165,22 @@ static bool HasNonReorderableChild(LogicalOperator &op) {
 	return tmp->children.empty();
 }
 
-bool RelationManager::ExtractJoinRelations(LogicalOperator &input_op,
+static void ModifyStatsIfLimit(optional_ptr<LogicalOperator> limit_op, RelationStats &stats) {
+	if (!limit_op) {
+		return;
+	}
+	auto &limit = limit_op->Cast<LogicalLimit>();
+	if (limit.limit_val.Type() == LimitNodeType::CONSTANT_VALUE) {
+		stats.cardinality = MinValue(limit.limit_val.GetConstantValue(), stats.cardinality);
+	}
+}
+
+bool RelationManager::ExtractJoinRelations(JoinOrderOptimizer &optimizer, LogicalOperator &input_op,
                                            vector<reference<LogicalOperator>> &filter_operators,
                                            optional_ptr<LogicalOperator> parent) {
-	LogicalOperator *op = &input_op;
+	optional_ptr<LogicalOperator> op = &input_op;
 	vector<reference<LogicalOperator>> datasource_filters;
+	optional_ptr<LogicalOperator> limit_op = nullptr;
 	// pass through single child operators
 	while (op->children.size() == 1 && !OperatorNeedsRelation(op->type)) {
 		if (op->type == LogicalOperatorType::LOGICAL_FILTER) {
@@ -132,6 +189,9 @@ bool RelationManager::ExtractJoinRelations(LogicalOperator &input_op,
 			}
 			filter_operators.push_back(*op);
 		}
+		if (op->type == LogicalOperatorType::LOGICAL_LIMIT) {
+			limit_op = op;
+		}
 		op = op->children[0].get();
 	}
 	bool non_reorderable_operation = false;
@@ -141,8 +201,7 @@ bool RelationManager::ExtractJoinRelations(LogicalOperator &input_op,
 	}
 
 	if (op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) {
-		auto &join = op->Cast<LogicalComparisonJoin>();
-		if (join.join_type == JoinType::INNER) {
+		if (JoinIsReorderable(*op)) {
 			// extract join conditions from inner join
 			filter_operators.push_back(*op);
 		} else {
@@ -160,15 +219,16 @@ bool RelationManager::ExtractJoinRelations(LogicalOperator &input_op,
 		vector<RelationStats> children_stats;
 		for (auto &child : op->children) {
 			auto stats = RelationStats();
-			JoinOrderOptimizer optimizer(context);
-			child = optimizer.Optimize(std::move(child), &stats);
+			auto child_optimizer = optimizer.CreateChildOptimizer();
+			child = child_optimizer.Optimize(std::move(child), &stats);
 			children_stats.push_back(stats);
 		}
 
 		auto combined_stats = RelationStatisticsHelper::CombineStatsOfNonReorderableOperator(*op, children_stats);
+		op->SetEstimatedCardinality(combined_stats.cardinality);
 		if (!datasource_filters.empty()) {
-			combined_stats.cardinality =
-			    (idx_t)MaxValue(combined_stats.cardinality * RelationStatisticsHelper::DEFAULT_SELECTIVITY, (double)1);
+			combined_stats.cardinality = (idx_t)MaxValue(
+			    double(combined_stats.cardinality) * RelationStatisticsHelper::DEFAULT_SELECTIVITY, (double)1);
 		}
 		AddRelation(input_op, parent, combined_stats);
 		return true;
@@ -178,36 +238,73 @@ bool RelationManager::ExtractJoinRelations(LogicalOperator &input_op,
 	case LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY: {
 		// optimize children
 		RelationStats child_stats;
-		JoinOrderOptimizer optimizer(context);
-		op->children[0] = optimizer.Optimize(std::move(op->children[0]), &child_stats);
+		auto child_optimizer = optimizer.CreateChildOptimizer();
+		op->children[0] = child_optimizer.Optimize(std::move(op->children[0]), &child_stats);
 		auto &aggr = op->Cast<LogicalAggregate>();
 		auto operator_stats = RelationStatisticsHelper::ExtractAggregationStats(aggr, child_stats);
+		// the extracted cardinality should be set for aggregate
+		aggr.SetEstimatedCardinality(operator_stats.cardinality);
 		if (!datasource_filters.empty()) {
-			operator_stats.cardinality = NumericCast<idx_t>(static_cast<double>(operator_stats.cardinality) *
-			                                                RelationStatisticsHelper::DEFAULT_SELECTIVITY);
+			operator_stats.cardinality = LossyNumericCast<idx_t>(static_cast<double>(operator_stats.cardinality) *
+			                                                     RelationStatisticsHelper::DEFAULT_SELECTIVITY);
 		}
+		ModifyStatsIfLimit(limit_op.get(), child_stats);
 		AddAggregateOrWindowRelation(input_op, parent, operator_stats, op->type);
 		return true;
 	}
 	case LogicalOperatorType::LOGICAL_WINDOW: {
 		// optimize children
 		RelationStats child_stats;
-		JoinOrderOptimizer optimizer(context);
-		op->children[0] = optimizer.Optimize(std::move(op->children[0]), &child_stats);
+		auto child_optimizer = optimizer.CreateChildOptimizer();
+		op->children[0] = child_optimizer.Optimize(std::move(op->children[0]), &child_stats);
 		auto &window = op->Cast<LogicalWindow>();
 		auto operator_stats = RelationStatisticsHelper::ExtractWindowStats(window, child_stats);
+		// the extracted cardinality should be set for window
+		window.SetEstimatedCardinality(operator_stats.cardinality);
 		if (!datasource_filters.empty()) {
-			operator_stats.cardinality = NumericCast<idx_t>(static_cast<double>(operator_stats.cardinality) *
-			                                                RelationStatisticsHelper::DEFAULT_SELECTIVITY);
+			operator_stats.cardinality = LossyNumericCast<idx_t>(static_cast<double>(operator_stats.cardinality) *
+			                                                     RelationStatisticsHelper::DEFAULT_SELECTIVITY);
 		}
+		ModifyStatsIfLimit(limit_op.get(), child_stats);
 		AddAggregateOrWindowRelation(input_op, parent, operator_stats, op->type);
 		return true;
 	}
-	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
+	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: {
+		auto &join = op->Cast<LogicalComparisonJoin>();
+		// Adding relations of the left side to the current join order optimizer
+		bool can_reorder_left = ExtractJoinRelations(optimizer, *op->children[0], filter_operators, op);
+		bool can_reorder_right = true;
+		// For semi & anti joins, you only reorder relations in the left side of the join.
+		// We do not want to reorder a relation A into the right side because then all column bindings A from A will be
+		// lost after the semi or anti join
+
+		// We cannot reorder a relation B out of the right side because any filter/join in the right side
+		// between a relation B and another RHS relation will be invalid. The semi join will remove
+		// all right column bindings,
+
+		// So we treat the right side of left join as its own relation so no relations
+		// are pushed into the right side, or taken out of the right side.
+		if (join.join_type == JoinType::SEMI || join.join_type == JoinType::ANTI) {
+			RelationStats child_stats;
+			// optimize the child and copy the stats
+			auto child_optimizer = optimizer.CreateChildOptimizer();
+			op->children[1] = child_optimizer.Optimize(std::move(op->children[1]), &child_stats);
+			AddRelation(*op->children[1], op, child_stats);
+			// remember that if a cross product needs to be forced, it cannot be forced
+			// across the children of a semi or anti join
+			no_cross_product_relations.insert(relations.size() - 1);
+			auto right_child_bindings = op->children[1]->GetColumnBindings();
+			for (auto &bindings : right_child_bindings) {
+				relation_mapping[bindings.table_index] = relations.size() - 1;
+			}
+		} else {
+			can_reorder_right = ExtractJoinRelations(optimizer, *op->children[1], filter_operators, op);
+		}
+		return can_reorder_left && can_reorder_right;
+	}
 	case LogicalOperatorType::LOGICAL_CROSS_PRODUCT: {
-		// Adding relations to the current join order optimizer
-		bool can_reorder_left = ExtractJoinRelations(*op->children[0], filter_operators, op);
-		bool can_reorder_right = ExtractJoinRelations(*op->children[1], filter_operators, op);
+		bool can_reorder_right = ExtractJoinRelations(optimizer, *op->children[1], filter_operators, op);
+		bool can_reorder_left = ExtractJoinRelations(optimizer, *op->children[0], filter_operators, op);
 		return can_reorder_left && can_reorder_right;
 	}
 	case LogicalOperatorType::LOGICAL_DUMMY_SCAN: {
@@ -230,28 +327,25 @@ bool RelationManager::ExtractJoinRelations(LogicalOperator &input_op,
 		auto stats = RelationStatisticsHelper::ExtractGetStats(get, context);
 		// if there is another logical filter that could not be pushed down into the
 		// table scan, apply another selectivity.
+		get.SetEstimatedCardinality(stats.cardinality);
 		if (!datasource_filters.empty()) {
 			stats.cardinality =
-			    (idx_t)MaxValue(stats.cardinality * RelationStatisticsHelper::DEFAULT_SELECTIVITY, (double)1);
+			    (idx_t)MaxValue(double(stats.cardinality) * RelationStatisticsHelper::DEFAULT_SELECTIVITY, (double)1);
 		}
+		ModifyStatsIfLimit(limit_op.get(), stats);
 		AddRelation(input_op, parent, stats);
 		return true;
 	}
-	case LogicalOperatorType::LOGICAL_DELIM_GET: {
-		//      Removed until we can extract better stats from delim gets. See #596
-		//		auto &delim_get = op->Cast<LogicalDelimGet>();
-		//		auto stats = RelationStatisticsHelper::ExtractDelimGetStats(delim_get, context);
-		//		AddRelation(input_op, parent, stats);
-		return false;
-	}
 	case LogicalOperatorType::LOGICAL_PROJECTION: {
-		auto child_stats = RelationStats();
+		RelationStats child_stats;
 		// optimize the child and copy the stats
-		JoinOrderOptimizer optimizer(context);
-		op->children[0] = optimizer.Optimize(std::move(op->children[0]), &child_stats);
+		auto child_optimizer = optimizer.CreateChildOptimizer();
+		op->children[0] = child_optimizer.Optimize(std::move(op->children[0]), &child_stats);
 		auto &proj = op->Cast<LogicalProjection>();
 		// Projection can create columns so we need to add them here
 		auto proj_stats = RelationStatisticsHelper::ExtractProjectionStats(proj, child_stats);
+		proj.SetEstimatedCardinality(proj_stats.cardinality);
+		ModifyStatsIfLimit(limit_op.get(), proj_stats);
 		AddRelation(input_op, parent, proj_stats);
 		return true;
 	}
@@ -260,9 +354,68 @@ bool RelationManager::ExtractJoinRelations(LogicalOperator &input_op,
 		auto &empty_result = op->Cast<LogicalEmptyResult>();
 		// Projection can create columns so we need to add them here
 		auto stats = RelationStatisticsHelper::ExtractEmptyResultStats(empty_result);
+		empty_result.SetEstimatedCardinality(stats.cardinality);
 		AddRelation(input_op, parent, stats);
 		return true;
 	}
+	case LogicalOperatorType::LOGICAL_MATERIALIZED_CTE:
+	case LogicalOperatorType::LOGICAL_RECURSIVE_CTE: {
+		RelationStats lhs_stats;
+		// optimize the lhs child and copy the stats
+		auto lhs_optimizer = optimizer.CreateChildOptimizer();
+		op->children[0] = lhs_optimizer.Optimize(std::move(op->children[0]), &lhs_stats);
+		// optimize the rhs child
+		auto rhs_optimizer = optimizer.CreateChildOptimizer();
+		auto table_index = op->type == LogicalOperatorType::LOGICAL_MATERIALIZED_CTE
+		                       ? op->Cast<LogicalMaterializedCTE>().table_index
+		                       : op->Cast<LogicalRecursiveCTE>().table_index;
+		rhs_optimizer.AddMaterializedCTEStats(table_index, std::move(lhs_stats));
+		op->children[1] = rhs_optimizer.Optimize(std::move(op->children[1]));
+		return false;
+	}
+	case LogicalOperatorType::LOGICAL_CTE_REF: {
+		auto &cte_ref = op->Cast<LogicalCTERef>();
+		if (cte_ref.materialized_cte != CTEMaterialize::CTE_MATERIALIZE_ALWAYS) {
+			return false;
+		}
+		auto cte_stats = optimizer.GetMaterializedCTEStats(cte_ref.cte_index);
+		cte_ref.SetEstimatedCardinality(cte_stats.cardinality);
+		AddRelation(input_op, parent, cte_stats);
+		return true;
+	}
+	case LogicalOperatorType::LOGICAL_DELIM_JOIN: {
+		auto &delim_join = op->Cast<LogicalComparisonJoin>();
+
+		// optimize LHS (duplicate-eliminated) child
+		RelationStats lhs_stats;
+		auto lhs_optimizer = optimizer.CreateChildOptimizer();
+		op->children[0] = lhs_optimizer.Optimize(std::move(op->children[0]), &lhs_stats);
+
+		// create dummy aggregation for the duplicate elimination
+		auto dummy_aggr = make_uniq<LogicalAggregate>(DConstants::INVALID_INDEX - 1, DConstants::INVALID_INDEX,
+		                                              vector<unique_ptr<Expression>>());
+		for (auto &delim_col : delim_join.duplicate_eliminated_columns) {
+			dummy_aggr->groups.push_back(delim_col->Copy());
+		}
+		auto lhs_delim_stats = RelationStatisticsHelper::ExtractAggregationStats(*dummy_aggr, lhs_stats);
+
+		// optimize the other child, which will now have access to the stats
+		RelationStats rhs_stats;
+		auto rhs_optimizer = optimizer.CreateChildOptimizer();
+		rhs_optimizer.AddDelimScanStats(lhs_delim_stats);
+		op->children[1] = rhs_optimizer.Optimize(std::move(op->children[1]), rhs_stats);
+
+		return false;
+	}
+	case LogicalOperatorType::LOGICAL_DELIM_GET: {
+		// Used to not be possible to reorder these. We added reordering (without stats) before,
+		// but ran into terrible join orders (see internal issue #596), so we removed it again
+		// We now have proper statistics for DelimGets, and get an even better query plan for #596
+		auto delim_scan_stats = optimizer.GetDelimScanStats();
+		op->SetEstimatedCardinality(delim_scan_stats.cardinality);
+		AddAggregateOrWindowRelation(input_op, parent, delim_scan_stats, op->type);
+		return true;
+	}
 	default:
 		return false;
 	}
@@ -314,18 +467,79 @@ vector<unique_ptr<FilterInfo>> RelationManager::ExtractEdges(LogicalOperator &op
 		if (f_op.type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN ||
 		    f_op.type == LogicalOperatorType::LOGICAL_ASOF_JOIN) {
 			auto &join = f_op.Cast<LogicalComparisonJoin>();
-			D_ASSERT(join.join_type == JoinType::INNER);
 			D_ASSERT(join.expressions.empty());
-			for (auto &cond : join.conditions) {
-				auto comparison =
-				    make_uniq<BoundComparisonExpression>(cond.comparison, std::move(cond.left), std::move(cond.right));
-				if (filter_set.find(*comparison) == filter_set.end()) {
-					filter_set.insert(*comparison);
-					unordered_set<idx_t> bindings;
-					ExtractBindings(*comparison, bindings);
-					auto &set = set_manager.GetJoinRelation(bindings);
-					auto filter_info = make_uniq<FilterInfo>(std::move(comparison), set, filters_and_bindings.size());
-					filters_and_bindings.push_back(std::move(filter_info));
+			if (join.join_type == JoinType::SEMI || join.join_type == JoinType::ANTI) {
+
+				auto conjunction_expression = make_uniq<BoundConjunctionExpression>(ExpressionType::CONJUNCTION_AND);
+				// create a conjunction expression for the semi join.
+				// It's possible multiple LHS relations have a condition in
+				// this semi join. Suppose we have ((A ⨝ B) ⋉ C). (example in test_4950.test)
+				// If the semi join condition has A.x = C.y AND B.x = C.z then we need to prevent a reordering
+				// that looks like ((A ⋉ C) ⨝ B)), since all columns from C will be lost after it joins with A,
+				// and the condition B.x = C.z will no longer be possible.
+				// if we make a conjunction expressions and populate the left set and right set with all
+				// the relations from the conditions in the conjunction expression, we can prevent invalid
+				// reordering.
+				for (auto &cond : join.conditions) {
+					auto comparison = make_uniq<BoundComparisonExpression>(cond.comparison, std::move(cond.left),
+					                                                       std::move(cond.right));
+					conjunction_expression->children.push_back(std::move(comparison));
+				}
+
+				// create the filter info so all required LHS relations are present when reconstructing the
+				// join
+				optional_ptr<JoinRelationSet> left_set;
+				optional_ptr<JoinRelationSet> right_set;
+				optional_ptr<JoinRelationSet> full_set;
+				// here we create a left_set that unions all relations from the left side of
+				// every expression and a right_set that unions all relations frmo the right side of a
+				// every expression (although this should always be 1).
+				for (auto &bound_expr : conjunction_expression->children) {
+					D_ASSERT(bound_expr->GetExpressionClass() == ExpressionClass::BOUND_COMPARISON);
+					auto &comp = bound_expr->Cast<BoundComparisonExpression>();
+					unordered_set<idx_t> right_bindings, left_bindings;
+					ExtractBindings(*comp.right, right_bindings);
+					ExtractBindings(*comp.left, left_bindings);
+
+					if (!left_set) {
+						left_set = set_manager.GetJoinRelation(left_bindings);
+					} else {
+						left_set = set_manager.Union(set_manager.GetJoinRelation(left_bindings), *left_set);
+					}
+					if (!right_set) {
+						right_set = set_manager.GetJoinRelation(right_bindings);
+					} else {
+						right_set = set_manager.Union(set_manager.GetJoinRelation(right_bindings), *right_set);
+					}
+				}
+				full_set = set_manager.Union(*left_set, *right_set);
+				D_ASSERT(left_set && left_set->count > 0);
+				D_ASSERT(right_set && right_set->count == 1);
+				D_ASSERT(full_set && full_set->count > 0);
+
+				// now we push the conjunction expressions
+				// In QueryGraphManager::GenerateJoins we extract each condition again and create a standalone join
+				// condition.
+				auto filter_info = make_uniq<FilterInfo>(std::move(conjunction_expression), *full_set,
+				                                         filters_and_bindings.size(), join.join_type);
+				filter_info->SetLeftSet(left_set);
+				filter_info->SetRightSet(right_set);
+
+				filters_and_bindings.push_back(std::move(filter_info));
+			} else {
+				// can extract every inner join condition individually.
+				for (auto &cond : join.conditions) {
+					auto comparison = make_uniq<BoundComparisonExpression>(cond.comparison, std::move(cond.left),
+					                                                       std::move(cond.right));
+					if (filter_set.find(*comparison) == filter_set.end()) {
+						filter_set.insert(*comparison);
+						unordered_set<idx_t> bindings;
+						ExtractBindings(*comparison, bindings);
+						auto &set = set_manager.GetJoinRelation(bindings);
+						auto filter_info = make_uniq<FilterInfo>(std::move(comparison), set,
+						                                         filters_and_bindings.size(), join.join_type);
+						filters_and_bindings.push_back(std::move(filter_info));
+					}
 				}
 			}
 			join.conditions.clear();

package/src/duckdb/src/optimizer/join_order/relation_statistics_helper.cpp

@@ -59,10 +59,10 @@ RelationStats RelationStatisticsHelper::ExtractGetStats(LogicalGet &get, ClientC
 	auto cardinality_after_filters = base_table_cardinality;
 	unique_ptr<BaseStatistics> column_statistics;
 
-	auto table_thing = get.GetTable();
+	auto catalog_table = get.GetTable();
 	auto name = string("some table");
-	if (table_thing) {
-		name = table_thing->name;
+	if (catalog_table) {
+		name = catalog_table->name;
 		return_stats.table_name = name;
 	}
 
@@ -77,14 +77,16 @@ RelationStats RelationStatisticsHelper::ExtractGetStats(LogicalGet &get, ClientC
 	}
 
 	// first push back basic distinct counts for each column (if we have them).
-	for (idx_t i = 0; i < get.column_ids.size(); i++) {
+	auto &column_ids = get.GetColumnIds();
+	for (idx_t i = 0; i < column_ids.size(); i++) {
 		bool have_distinct_count_stats = false;
 		if (get.function.statistics) {
-			column_statistics = get.function.statistics(context, get.bind_data.get(), get.column_ids[i]);
+			column_statistics = get.function.statistics(context, get.bind_data.get(), column_ids[i]);
 			if (column_statistics && have_catalog_table_statistics) {
-				auto column_distinct_count = DistinctCount({column_statistics->GetDistinctCount(), true});
+				auto distinct_count = MaxValue((idx_t)1, column_statistics->GetDistinctCount());
+				auto column_distinct_count = DistinctCount({distinct_count, true});
 				return_stats.column_distinct_count.push_back(column_distinct_count);
-				return_stats.column_names.push_back(name + "." + get.names.at(get.column_ids.at(i)));
+				return_stats.column_names.push_back(name + "." + get.names.at(column_ids.at(i)));
 				have_distinct_count_stats = true;
 			}
 		}
@@ -95,8 +97,8 @@ RelationStats RelationStatisticsHelper::ExtractGetStats(LogicalGet &get, ClientC
 			auto column_distinct_count = DistinctCount({cardinality_after_filters, false});
 			return_stats.column_distinct_count.push_back(column_distinct_count);
 			auto column_name = string("column");
-			if (get.column_ids.at(i) < get.names.size()) {
-				column_name = get.names.at(get.column_ids.at(i));
+			if (column_ids.at(i) < get.names.size()) {
+				column_name = get.names.at(column_ids.at(i));
 			}
 			return_stats.column_names.push_back(get.GetName() + "." + column_name);
 		}
@@ -105,9 +107,8 @@ RelationStats RelationStatisticsHelper::ExtractGetStats(LogicalGet &get, ClientC
 	if (!get.table_filters.filters.empty()) {
 		column_statistics = nullptr;
 		for (auto &it : get.table_filters.filters) {
-			if (get.bind_data && get.function.name.compare("seq_scan") == 0) {
-				auto &table_scan_bind_data = get.bind_data->Cast<TableScanBindData>();
-				column_statistics = get.function.statistics(context, &table_scan_bind_data, it.first);
+			if (get.bind_data && get.function.statistics) {
+				column_statistics = get.function.statistics(context, get.bind_data.get(), it.first);
 			}
 
 			if (column_statistics && it.second->filter_type == TableFilterType::CONJUNCTION_AND) {
@@ -122,7 +123,8 @@ RelationStats RelationStatisticsHelper::ExtractGetStats(LogicalGet &get, ClientC
 		bool has_equality_filter = (cardinality_after_filters != base_table_cardinality);
 		if (!has_equality_filter && !get.table_filters.filters.empty()) {
 			cardinality_after_filters = MaxValue<idx_t>(
-			    NumericCast<idx_t>(base_table_cardinality * RelationStatisticsHelper::DEFAULT_SELECTIVITY), 1U);
+			    LossyNumericCast<idx_t>(double(base_table_cardinality) * RelationStatisticsHelper::DEFAULT_SELECTIVITY),
+			    1U);
 		}
 		if (base_table_cardinality == 0) {
 			cardinality_after_filters = 0;
@@ -237,6 +239,7 @@ RelationStats RelationStatisticsHelper::CombineStatsOfNonReorderableOperator(Log
 		case JoinType::ANTI:
 		case JoinType::SEMI:
 		case JoinType::SINGLE:
+		case JoinType::MARK:
 			ret.cardinality = child_1_card;
 			break;
 		default:
@@ -335,17 +338,18 @@ RelationStats RelationStatisticsHelper::ExtractAggregationStats(LogicalAggregate
 				// be grouped by. Hopefully this can be fixed with duckdb-internal#606
 				continue;
 			}
-			if (new_card < child_stats.column_distinct_count[col_index].distinct_count) {
-				new_card = child_stats.column_distinct_count[col_index].distinct_count;
+			double distinct_count = double(child_stats.column_distinct_count[col_index].distinct_count);
+			if (new_card < distinct_count) {
+				new_card = distinct_count;
 			}
 		}
 	}
-	if (new_card < 0 || new_card >= child_stats.cardinality) {
+	if (new_card < 0 || new_card >= double(child_stats.cardinality)) {
 		// We have no good statistics on distinct count.
 		// most likely we are running on parquet files. Therefore we divide by 2.
 		new_card = (double)child_stats.cardinality / 2;
 	}
-	stats.cardinality = NumericCast<idx_t>(new_card);
+	stats.cardinality = LossyNumericCast<idx_t>(new_card);
 	stats.column_names = child_stats.column_names;
 	stats.stats_initialized = true;
 	auto num_child_columns = aggr.GetColumnBindings().size();

package/src/duckdb/src/optimizer/limit_pushdown.cpp

@@ -0,0 +1,42 @@
+#include "duckdb/optimizer/limit_pushdown.hpp"
+
+#include "duckdb/planner/operator/logical_limit.hpp"
+#include "duckdb/planner/operator/logical_projection.hpp"
+
+namespace duckdb {
+
+bool LimitPushdown::CanOptimize(duckdb::LogicalOperator &op) {
+	if (op.type == LogicalOperatorType::LOGICAL_LIMIT &&
+	    op.children[0]->type == LogicalOperatorType::LOGICAL_PROJECTION) {
+		auto &limit = op.Cast<LogicalLimit>();
+
+		if (limit.offset_val.Type() == LimitNodeType::EXPRESSION_PERCENTAGE ||
+		    limit.offset_val.Type() == LimitNodeType::EXPRESSION_VALUE) {
+			// Offset cannot be an expression
+			return false;
+		}
+
+		if (limit.limit_val.Type() == LimitNodeType::CONSTANT_VALUE && limit.limit_val.GetConstantValue() < 8192) {
+			// Push down only when limit value is smaller than 8192.
+			// when physical_limit is introduced, it will end a parallel pipeline
+			// restrict the limit value to be small so that remaining operations run fast without parallelization.
+			return true;
+		}
+	}
+	return false;
+}
+
+unique_ptr<LogicalOperator> LimitPushdown::Optimize(unique_ptr<LogicalOperator> op) {
+	if (CanOptimize(*op)) {
+		auto projection = std::move(op->children[0]);
+		op->children[0] = std::move(projection->children[0]);
+		projection->children[0] = std::move(op);
+		swap(projection, op);
+	}
+	for (auto &child : op->children) {
+		child = Optimize(std::move(child));
+	}
+	return op;
+}
+
+} // namespace duckdb