duckdb 1.3.2 → 1.3.3-dev4.0

package/src/duckdb/src/optimizer/compressed_materialization.cpp

@@ -47,15 +47,10 @@ CompressedMaterialization::CompressedMaterialization(Optimizer &optimizer_p, Log
     : optimizer(optimizer_p), context(optimizer.context), root(&root_p), statistics_map(statistics_map_p) {
 }
 
-void CompressedMaterialization::GetReferencedBindings(const Expression &expression,
+void CompressedMaterialization::GetReferencedBindings(const Expression &root_expr,
                                                       column_binding_set_t &referenced_bindings) {
-	if (expression.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) {
-		const auto &col_ref = expression.Cast<BoundColumnRefExpression>();
-		referenced_bindings.insert(col_ref.binding);
-	} else {
-		ExpressionIterator::EnumerateChildren(
-		    expression, [&](const Expression &child) { GetReferencedBindings(child, referenced_bindings); });
-	}
+	ExpressionIterator::VisitExpression<BoundColumnRefExpression>(
+	    root_expr, [&](const BoundColumnRefExpression &col_ref) { referenced_bindings.insert(col_ref.binding); });
 }
 
 void CompressedMaterialization::UpdateBindingInfo(CompressedMaterializationInfo &info, const ColumnBinding &binding,
@@ -350,28 +345,42 @@ static Value GetIntegralRangeValue(ClientContext &context, const LogicalType &ty
 unique_ptr<CompressExpression> CompressedMaterialization::GetIntegralCompress(unique_ptr<Expression> input,
                                                                               const BaseStatistics &stats) {
 	const auto &type = input->return_type;
-	if (GetTypeIdSize(type.InternalType()) == 1 || !NumericStats::HasMinMax(stats)) {
-		return nullptr;
-	}
-
-	// Get range and cast to UBIGINT (might fail for UHUGEINT, in which case we just return)
-	Value range_value = GetIntegralRangeValue(context, type, stats);
-	if (!range_value.DefaultTryCastAs(LogicalType::UBIGINT)) {
+	if (GetTypeIdSize(type.InternalType()) == 1) {
 		return nullptr;
 	}
 
-	// Get the smallest type that the range can fit into
-	const auto range = UBigIntValue::Get(range_value);
 	LogicalType cast_type;
-	if (range <= NumericLimits<uint8_t>().Maximum()) {
+	Value range_value;
+	Value min;
+	if (!stats.CanHaveNoNull()) {
+		// All NULL
 		cast_type = LogicalType::UTINYINT;
-	} else if (range <= NumericLimits<uint16_t>().Maximum()) {
-		cast_type = LogicalType::USMALLINT;
-	} else if (range <= NumericLimits<uint32_t>().Maximum()) {
-		cast_type = LogicalType::UINTEGER;
+		range_value = Value::UTINYINT(0);
+		min = Value(input->return_type);
+	} else if (NumericStats::HasMinMax(stats)) {
+		// Get range and cast to UBIGINT (might fail for HUGEINT, in which case we just return)
+		range_value = GetIntegralRangeValue(context, type, stats);
+		if (!range_value.DefaultTryCastAs(LogicalType::UBIGINT)) {
+			return nullptr;
+		}
+
+		// Get the smallest type that the range can fit into
+		const auto range = UBigIntValue::Get(range_value);
+		if (range <= NumericLimits<uint8_t>().Maximum()) {
+			cast_type = LogicalType::UTINYINT;
+		} else if (range <= NumericLimits<uint16_t>().Maximum()) {
+			cast_type = LogicalType::USMALLINT;
+		} else if (range <= NumericLimits<uint32_t>().Maximum()) {
+			cast_type = LogicalType::UINTEGER;
+		} else {
+			D_ASSERT(range <= NumericLimits<uint64_t>().Maximum());
+			cast_type = LogicalType::UBIGINT;
+		}
+
+		min = NumericStats::Min(stats);
 	} else {
-		D_ASSERT(range <= NumericLimits<uint64_t>().Maximum());
-		cast_type = LogicalType::UBIGINT;
+		// We don't have enough stats to do anything
+		return nullptr;
 	}
 
 	// Check if type that fits the range is smaller than the input type
@@ -384,7 +393,7 @@ unique_ptr<CompressExpression> CompressedMaterialization::GetIntegralCompress(un
 	auto compress_function = CMIntegralCompressFun::GetFunction(type, cast_type);
 	vector<unique_ptr<Expression>> arguments;
 	arguments.emplace_back(std::move(input));
-	arguments.emplace_back(make_uniq<BoundConstantExpression>(NumericStats::Min(stats)));
+	arguments.emplace_back(make_uniq<BoundConstantExpression>(min));
 	auto compress_expr =
 	    make_uniq<BoundFunctionExpression>(cast_type, compress_function, std::move(arguments), nullptr);
 
@@ -398,19 +407,25 @@ unique_ptr<CompressExpression> CompressedMaterialization::GetIntegralCompress(un
 
 unique_ptr<CompressExpression> CompressedMaterialization::GetStringCompress(unique_ptr<Expression> input,
                                                                             const BaseStatistics &stats) {
-	if (!StringStats::HasMaxStringLength(stats)) {
-		return nullptr;
-	}
-
-	const auto max_string_length = StringStats::MaxStringLength(stats);
 	LogicalType cast_type = LogicalType::INVALID;
-	for (const auto &compressed_type : CMUtils::StringTypes()) {
-		if (max_string_length < GetTypeIdSize(compressed_type.InternalType())) {
-			cast_type = compressed_type;
-			break;
+	uint32_t max_string_length;
+	if (!stats.CanHaveNoNull()) {
+		// All NULL
+		cast_type = LogicalType::UTINYINT;
+		max_string_length = 0;
+	} else if (StringStats::HasMaxStringLength(stats)) {
+		max_string_length = StringStats::MaxStringLength(stats);
+		for (const auto &compressed_type : CMUtils::StringTypes()) {
+			if (max_string_length < GetTypeIdSize(compressed_type.InternalType())) {
+				cast_type = compressed_type;
+				break;
+			}
 		}
-	}
-	if (cast_type == LogicalType::INVALID) {
+		if (cast_type == LogicalType::INVALID) {
+			return nullptr;
+		}
+	} else {
+		// We don't have enough stats to do anything
 		return nullptr;
 	}
 
@@ -467,18 +482,20 @@ unique_ptr<Expression> CompressedMaterialization::GetDecompressExpression(unique
 unique_ptr<Expression> CompressedMaterialization::GetIntegralDecompress(unique_ptr<Expression> input,
                                                                         const LogicalType &result_type,
                                                                         const BaseStatistics &stats) {
-	D_ASSERT(NumericStats::HasMinMax(stats));
+	D_ASSERT(!stats.CanHaveNoNull() || NumericStats::HasMinMax(stats));
 	auto decompress_function = CMIntegralDecompressFun::GetFunction(input->return_type, result_type);
+	const auto min = !stats.CanHaveNoNull() ? Value(result_type) : NumericStats::Min(stats);
+
 	vector<unique_ptr<Expression>> arguments;
 	arguments.emplace_back(std::move(input));
-	arguments.emplace_back(make_uniq<BoundConstantExpression>(NumericStats::Min(stats)));
+	arguments.emplace_back(make_uniq<BoundConstantExpression>(min));
 	return make_uniq<BoundFunctionExpression>(result_type, decompress_function, std::move(arguments), nullptr);
 }
 
 unique_ptr<Expression> CompressedMaterialization::GetStringDecompress(unique_ptr<Expression> input,
                                                                       const LogicalType &result_type,
                                                                       const BaseStatistics &stats) {
-	D_ASSERT(StringStats::HasMaxStringLength(stats));
+	D_ASSERT(!stats.CanHaveNoNull() || StringStats::HasMaxStringLength(stats));
 	auto decompress_function = CMStringDecompressFun::GetFunction(input->return_type);
 	vector<unique_ptr<Expression>> arguments;
 	arguments.emplace_back(std::move(input));

package/src/duckdb/src/optimizer/cte_inlining.cpp

@@ -0,0 +1,225 @@
+#include "duckdb/optimizer/cte_inlining.hpp"
+
+#include "duckdb/optimizer/column_binding_replacer.hpp"
+#include "duckdb/optimizer/filter_pushdown.hpp"
+#include "duckdb/planner/expression/bound_conjunction_expression.hpp"
+#include "duckdb/planner/expression/list.hpp"
+#include "duckdb/planner/operator/logical_cteref.hpp"
+#include "duckdb/planner/operator/logical_filter.hpp"
+#include "duckdb/planner/operator/logical_projection.hpp"
+#include "duckdb/planner/operator/logical_materialized_cte.hpp"
+
+#include "duckdb/optimizer/optimizer.hpp"
+#include "duckdb/planner/logical_operator_deep_copy.hpp"
+#include "duckdb/planner/operator/logical_prepare.hpp"
+
+#include "duckdb/function/scalar/generic_functions.hpp"
+
+namespace duckdb {
+
+CTEInlining::CTEInlining(Optimizer &optimizer_p) : optimizer(optimizer_p) {
+}
+
+unique_ptr<LogicalOperator> CTEInlining::Optimize(unique_ptr<LogicalOperator> op) {
+	TryInlining(op);
+	return op;
+}
+
+static idx_t CountCTEReferences(const LogicalOperator &op, idx_t cte_index) {
+	if (op.type == LogicalOperatorType::LOGICAL_CTE_REF) {
+		auto &cte = op.Cast<LogicalCTERef>();
+		if (cte.cte_index == cte_index) {
+			return 1;
+		}
+	}
+	idx_t number_of_references = 0;
+	for (auto &child : op.children) {
+		number_of_references += CountCTEReferences(*child, cte_index);
+	}
+
+	return number_of_references;
+}
+
+static bool ContainsLimit(const LogicalOperator &op) {
+	if (op.type == LogicalOperatorType::LOGICAL_LIMIT || op.type == LogicalOperatorType::LOGICAL_TOP_N) {
+		return true;
+	}
+	if (op.children.size() != 1) {
+		return false;
+	}
+	for (auto &child : op.children) {
+		if (ContainsLimit(*child)) {
+			return true;
+		}
+	}
+	return false;
+}
+
+static bool EndsInAggregateOrDistinct(const LogicalOperator &op) {
+	if (op.type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY ||
+	    op.type == LogicalOperatorType::LOGICAL_DISTINCT) {
+		return true;
+	}
+	if (op.children.size() != 1) {
+		return false;
+	}
+	for (auto &child : op.children) {
+		if (EndsInAggregateOrDistinct(*child)) {
+			return true;
+		}
+	}
+	return false;
+}
+
+void CTEInlining::TryInlining(unique_ptr<LogicalOperator> &op) {
+	if (op->type == LogicalOperatorType::LOGICAL_PREPARE) {
+		// we are in a prepare statement, if we have to copy an operator during inlining,
+		// we have to be careful to use the correct parameter data
+		auto &prepare = op->Cast<LogicalPrepare>();
+		parameter_data = prepare.prepared->value_map;
+	}
+
+	// traverse children first, so we can inline the deepest CTEs first
+	for (auto &child : op->children) {
+		TryInlining(child);
+	}
+
+	if (op->type == LogicalOperatorType::LOGICAL_MATERIALIZED_CTE) {
+		auto &cte = op->Cast<LogicalMaterializedCTE>();
+		auto ref_count = CountCTEReferences(*op, cte.table_index);
+		if (ref_count == 0) {
+			// this CTE is not referenced, we can remove it
+			op = std::move(op->children[1]);
+			return;
+		}
+		if (cte.materialize == CTEMaterialize::CTE_MATERIALIZE_ALWAYS) {
+			// This CTE is always materialized, we cannot inline it
+			return;
+		}
+		if (ref_count == 1) {
+			// this CTE is only referenced once, we can inline it directly without copying
+			bool success = Inline(op->children[1], *op, false);
+			if (success) {
+				op = std::move(op->children[1]);
+			}
+			return;
+		}
+		if (ref_count > 1) {
+			if (cte.materialize == CTEMaterialize::CTE_MATERIALIZE_NEVER) {
+				// this CTE is referenced multiple times, but we are not allowed to materialize it
+				// we have to inline it if possible
+				bool success = Inline(op->children[1], *op, true);
+				if (success) {
+					op = std::move(op->children[1]);
+				}
+				return;
+			}
+			// check if we can inline this CTE
+			PreventInlining prevent_inlining;
+			prevent_inlining.VisitOperator(*op->children[0]);
+
+			if (prevent_inlining.prevent_inlining) {
+				// we cannot inline this CTE, we have to keep it materialized
+				return;
+			}
+
+			// Prevent inlining if the CTE ends in an aggregate or distinct operator
+			// This mimics the behavior of the CTE materialization in the binder
+			if (EndsInAggregateOrDistinct(*op->children[0])) {
+				return;
+			}
+
+			// CTEs require full materialization before the CTE scans begin,
+			// LIMIT and TOP_N operators cannot abort the materialization,
+			// even if only a part of the CTE result is needed.
+			// Therefore, we check if the CTE Scans are below the LIMIT or TOP_N operator
+			// and if so, we try to inline the CTE definition.
+			if (ContainsLimit(*op->children[1]) || op->children[0]->type == LogicalOperatorType::LOGICAL_EMPTY_RESULT) {
+				// this CTE is referenced multiple times and has a limit, we want to inline it
+				bool success = Inline(op->children[1], *op, true);
+				if (success) {
+					op = std::move(op->children[1]);
+				}
+				return;
+			}
+		}
+	}
+}
+
+bool CTEInlining::Inline(unique_ptr<duckdb::LogicalOperator> &op, LogicalOperator &materialized_cte,
+                         bool requires_copy) {
+	if (op->type == LogicalOperatorType::LOGICAL_CTE_REF) {
+		auto &cteref = op->Cast<LogicalCTERef>();
+		auto &cte = materialized_cte.Cast<LogicalCTE>();
+		if (cteref.cte_index == cte.table_index) {
+			unique_ptr<LogicalOperator> &definition = cte.children[0];
+			unique_ptr<LogicalOperator> copy;
+			if (requires_copy) {
+				// there are multiple references to the CTE, we need to copy it
+				LogicalOperatorDeepCopy deep_copy(optimizer.binder, parameter_data);
+				try {
+					copy = deep_copy.DeepCopy(definition);
+				} catch (NotImplementedException &ex) {
+					// if we have to copy the lhs of a CTE, but we cannot copy the operator, we have to
+					// stop inlining and keep the materialized CTE instead
+					return false;
+				}
+			}
+			vector<unique_ptr<Expression>> proj_expressions;
+			definition->ResolveOperatorTypes();
+			vector<LogicalType> types = definition->types;
+			vector<ColumnBinding> bindings =
+			    requires_copy ? copy->GetColumnBindings() : definition->GetColumnBindings();
+
+			idx_t col_idx = 0;
+			for (auto &col : bindings) {
+				proj_expressions.push_back(make_uniq<BoundColumnRefExpression>(types[col_idx], col));
+				col_idx++;
+			}
+			auto proj = make_uniq<LogicalProjection>(cteref.table_index, std::move(proj_expressions));
+
+			if (requires_copy) {
+				proj->children.push_back(std::move(copy));
+			} else {
+				proj->children.push_back(std::move(definition));
+			}
+			op = std::move(proj);
+			return true;
+		}
+		return true;
+	} else {
+		bool success = true;
+		for (auto &child : op->children) {
+			success &= Inline(child, materialized_cte, requires_copy);
+		}
+		return success;
+	}
+}
+
+void PreventInlining::VisitOperator(LogicalOperator &op) {
+	VisitOperatorExpressions(op);
+	// We can stop checking early if we already know that inlining is not possible
+	if (!prevent_inlining) {
+		VisitOperatorChildren(op);
+	}
+}
+
+void PreventInlining::VisitExpression(unique_ptr<Expression> *expression) {
+	auto &expr = *expression;
+
+	if (expr->GetExpressionClass() == ExpressionClass::BOUND_FUNCTION) {
+		auto &bound_function = expr->Cast<BoundFunctionExpression>();
+		// if we encounter the ErrorFun function, we still want to inline
+		if (bound_function.function == ErrorFun::GetFunction()) {
+			return;
+		}
+
+		if (expr->IsVolatile()) {
+			prevent_inlining = true;
+			return;
+		}
+	}
+	VisitExpressionChildren(**expression);
+}
+
+} // namespace duckdb

package/src/duckdb/src/optimizer/empty_result_pullup.cpp

@@ -66,7 +66,6 @@ unique_ptr<LogicalOperator> EmptyResultPullup::Optimize(unique_ptr<LogicalOperat
 	case LogicalOperatorType::LOGICAL_FILTER:
 	case LogicalOperatorType::LOGICAL_DISTINCT:
 	case LogicalOperatorType::LOGICAL_WINDOW:
-	case LogicalOperatorType::LOGICAL_MATERIALIZED_CTE:
 	case LogicalOperatorType::LOGICAL_GET:
 	case LogicalOperatorType::LOGICAL_INTERSECT:
 	case LogicalOperatorType::LOGICAL_PIVOT:
@@ -80,6 +79,14 @@ unique_ptr<LogicalOperator> EmptyResultPullup::Optimize(unique_ptr<LogicalOperat
 		}
 		return op;
 	}
+	case LogicalOperatorType::LOGICAL_MATERIALIZED_CTE: {
+		D_ASSERT(op->children.size() == 2);
+		if (op->children[1]->type == LogicalOperatorType::LOGICAL_EMPTY_RESULT) {
+			op = make_uniq<LogicalEmptyResult>(std::move(op));
+			break;
+		}
+		return op;
+	}
 	case LogicalOperatorType::LOGICAL_EXCEPT:
 	case LogicalOperatorType::LOGICAL_ANY_JOIN:
 	case LogicalOperatorType::LOGICAL_DELIM_JOIN:

package/src/duckdb/src/optimizer/filter_combiner.cpp

@@ -332,13 +332,11 @@ FilterPushdownResult FilterCombiner::TryPushdownConstantFilter(TableFilterSet &t
 	return FilterPushdownResult::PUSHED_DOWN_FULLY;
 }
 
-void ReplaceWithBoundReference(unique_ptr<Expression> &expr) {
-	if (expr->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) {
-		expr = make_uniq<BoundReferenceExpression>(expr->return_type, 0ULL);
-		return;
-	}
-	ExpressionIterator::EnumerateChildren(*expr,
-	                                      [&](unique_ptr<Expression> &child) { ReplaceWithBoundReference(child); });
+void ReplaceWithBoundReference(unique_ptr<Expression> &root_expr) {
+	ExpressionIterator::VisitExpressionMutable<BoundColumnRefExpression>(
+	    root_expr, [&](BoundColumnRefExpression &col_ref, unique_ptr<Expression> &expr) {
+		    expr = make_uniq<BoundReferenceExpression>(col_ref.return_type, 0ULL);
+	    });
 }
 
 FilterPushdownResult FilterCombiner::TryPushdownGenericExpression(LogicalGet &get, Expression &expr) {

package/src/duckdb/src/optimizer/filter_pushdown.cpp

@@ -160,6 +160,9 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownJoin(unique_ptr<LogicalOpera
 
 	unique_ptr<LogicalOperator> result;
 	switch (join.join_type) {
+	case JoinType::OUTER:
+		result = PushdownOuterJoin(std::move(op), left_bindings, right_bindings);
+		break;
 	case JoinType::INNER:
 		//	AsOf joins can't push anything into the RHS, so treat it as a left join
 		if (op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN) {

package/src/duckdb/src/optimizer/join_order/cardinality_estimator.cpp

@@ -355,7 +355,7 @@ DenomInfo CardinalityEstimator::GetDenominator(JoinRelationSet &set) {
 	auto denom_multiplier = 1.0 + static_cast<double>(unused_edge_tdoms.size());
 
 	// It's possible cross-products were added and are not present in the filters in the relation_2_tdom
-	// structures. When that's the case, merge all remaining subgraphs.
+	// structures. When that's the case, merge all remaining subgraphs as if they are connected by a cross product
 	if (subgraphs.size() > 1) {
 		auto final_subgraph = subgraphs.at(0);
 		for (auto merge_with = subgraphs.begin() + 1; merge_with != subgraphs.end(); merge_with++) {
@@ -367,6 +367,23 @@ DenomInfo CardinalityEstimator::GetDenominator(JoinRelationSet &set) {
 			final_subgraph.denom *= merge_with->denom;
 		}
 	}
+	if (!subgraphs.empty()) {
+		// Some relations are connected by cross products and will not end up in a subgraph
+		// Check and make sure all relations were considered, if not, they are connected to the graph by cross products
+		auto &returning_subgraph = subgraphs.at(0);
+		if (returning_subgraph.relations->count != set.count) {
+			for (idx_t rel_index = 0; rel_index < set.count; rel_index++) {
+				auto relation_id = set.relations[rel_index];
+				auto &rel = set_manager.GetJoinRelation(relation_id);
+				if (!JoinRelationSet::IsSubset(*returning_subgraph.relations, rel)) {
+					returning_subgraph.numerator_relations =
+					    &set_manager.Union(*returning_subgraph.numerator_relations, rel);
+					returning_subgraph.relations = &set_manager.Union(*returning_subgraph.relations, rel);
+				}
+			}
+		}
+	}
+
 	// can happen if a table has cardinality 0, a tdom is set to 0, or if a cross product is used.
 	if (subgraphs.empty() || subgraphs.at(0).denom == 0) {
 		// denominator is 1 and numerators are a cross product of cardinalities.
@@ -377,7 +394,6 @@ DenomInfo CardinalityEstimator::GetDenominator(JoinRelationSet &set) {
 
 template <>
 double CardinalityEstimator::EstimateCardinalityWithSet(JoinRelationSet &new_set) {
-
 	if (relation_set_2_cardinality.find(new_set.ToString()) != relation_set_2_cardinality.end()) {
 		return relation_set_2_cardinality[new_set.ToString()].cardinality_before_filters;
 	}

package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp

@@ -19,6 +19,7 @@ JoinOrderOptimizer JoinOrderOptimizer::CreateChildOptimizer() {
 	child_optimizer.materialized_cte_stats = materialized_cte_stats;
 	child_optimizer.delim_scan_stats = delim_scan_stats;
 	child_optimizer.depth = depth + 1;
+	child_optimizer.recursive_cte_indexes = recursive_cte_indexes;
 	return child_optimizer;
 }
 

package/src/duckdb/src/optimizer/join_order/plan_enumerator.cpp

@@ -3,6 +3,7 @@
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/optimizer/join_order/join_node.hpp"
 #include "duckdb/optimizer/join_order/query_graph_manager.hpp"
+#include "duckdb/main/settings.hpp"
 
 #include <cmath>
 
@@ -60,7 +61,7 @@ static vector<unordered_set<idx_t>> GetAllNeighborSets(vector<idx_t> neighbors)
 	// drive by test to make sure we have an accurate amount of
 	// subsets, and that each neighbor is in a correct amount
 	// of those subsets.
-	D_ASSERT(ret.size() == std::pow(2, neighbors.size()) - 1);
+	D_ASSERT(ret.size() == static_cast<size_t>(std::pow(2, neighbors.size())) - 1);
 	for (auto &n : neighbors) {
 		idx_t count = 0;
 		for (auto &set : ret) {
@@ -68,7 +69,7 @@ static vector<unordered_set<idx_t>> GetAllNeighborSets(vector<idx_t> neighbors)
 				count += 1;
 			}
 		}
-		D_ASSERT(count == std::pow(2, neighbors.size() - 1));
+		D_ASSERT(count == static_cast<size_t>(std::pow(2, neighbors.size() - 1)));
 	}
 #endif
 	return ret;
@@ -470,7 +471,7 @@ void PlanEnumerator::InitLeafPlans() {
 // Moerkotte and Thomas Neumannn, see that paper for additional info/documentation bonus slides:
 // https://db.in.tum.de/teaching/ws1415/queryopt/chapter3.pdf?lang=de
 void PlanEnumerator::SolveJoinOrder() {
-	bool force_no_cross_product = query_graph_manager.context.config.force_no_cross_product;
+	bool force_no_cross_product = DBConfig::GetSetting<DebugForceNoCrossProductSetting>(query_graph_manager.context);
 	// first try to solve the join order exactly
 	if (query_graph_manager.relation_manager.NumRelations() >= THRESHOLD_TO_SWAP_TO_APPROXIMATE) {
 		SolveJoinOrderApproximately();

package/src/duckdb/src/optimizer/join_order/query_graph_manager.cpp

@@ -35,21 +35,17 @@ bool QueryGraphManager::Build(JoinOrderOptimizer &optimizer, LogicalOperator &op
 	return true;
 }
 
-void QueryGraphManager::GetColumnBinding(Expression &expression, ColumnBinding &binding) {
-	if (expression.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) {
-		// Here you have a filter on a single column in a table. Return a binding for the column
-		// being filtered on so the filter estimator knows what HLL count to pull
-		auto &colref = expression.Cast<BoundColumnRefExpression>();
-		D_ASSERT(colref.depth == 0);
-		D_ASSERT(colref.binding.table_index != DConstants::INVALID_INDEX);
-		// map the base table index to the relation index used by the JoinOrderOptimizer
-		D_ASSERT(relation_manager.relation_mapping.find(colref.binding.table_index) !=
-		         relation_manager.relation_mapping.end());
-		binding =
-		    ColumnBinding(relation_manager.relation_mapping[colref.binding.table_index], colref.binding.column_index);
-	}
-	// TODO: handle inequality filters with functions.
-	ExpressionIterator::EnumerateChildren(expression, [&](Expression &expr) { GetColumnBinding(expr, binding); });
+void QueryGraphManager::GetColumnBinding(Expression &root_expr, ColumnBinding &binding) {
+	ExpressionIterator::VisitExpression<BoundColumnRefExpression>(
+	    root_expr, [&](const BoundColumnRefExpression &colref) {
+		    D_ASSERT(colref.depth == 0);
+		    D_ASSERT(colref.binding.table_index != DConstants::INVALID_INDEX);
+		    // map the base table index to the relation index used by the JoinOrderOptimizer
+		    D_ASSERT(relation_manager.relation_mapping.find(colref.binding.table_index) !=
+		             relation_manager.relation_mapping.end());
+		    binding = ColumnBinding(relation_manager.relation_mapping[colref.binding.table_index],
+		                            colref.binding.column_index);
+	    });
 }
 
 const vector<unique_ptr<FilterInfo>> &QueryGraphManager::GetFilterBindings() const {

package/src/duckdb/src/optimizer/join_order/relation_manager.cpp

@@ -117,24 +117,21 @@ static bool OperatorIsNonReorderable(LogicalOperatorType op_type) {
 	}
 }
 
-bool ExpressionContainsColumnRef(Expression &expression) {
-	if (expression.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) {
-		// Here you have a filter on a single column in a table. Return a binding for the column
-		// being filtered on so the filter estimator knows what HLL count to pull
+bool ExpressionContainsColumnRef(const Expression &root_expr) {
+	bool contains_column_ref = false;
+	ExpressionIterator::VisitExpression<BoundColumnRefExpression>(
+	    root_expr, [&](const BoundColumnRefExpression &colref) {
+	// Here you have a filter on a single column in a table. Return a binding for the column
+	// being filtered on so the filter estimator knows what HLL count to pull
 #ifdef DEBUG
-		auto &colref = expression.Cast<BoundColumnRefExpression>();
-		(void)colref.depth;
-		D_ASSERT(colref.depth == 0);
-		D_ASSERT(colref.binding.table_index != DConstants::INVALID_INDEX);
+		    (void)colref.depth;
+		    D_ASSERT(colref.depth == 0);
+		    D_ASSERT(colref.binding.table_index != DConstants::INVALID_INDEX);
 #endif
-		// map the base table index to the relation index used by the JoinOrderOptimizer
-		return true;
-	}
-	// TODO: handle inequality filters with functions.
-	auto children_ret = false;
-	ExpressionIterator::EnumerateChildren(expression,
-	                                      [&](Expression &expr) { children_ret = ExpressionContainsColumnRef(expr); });
-	return children_ret;
+		    // map the base table index to the relation index used by the JoinOrderOptimizer
+		    contains_column_ref = true;
+	    });
+	return contains_column_ref;
 }
 
 static bool JoinIsReorderable(LogicalOperator &op) {
@@ -390,21 +387,41 @@ bool RelationManager::ExtractJoinRelations(JoinOrderOptimizer &optimizer, Logica
 		op->children[0] = lhs_optimizer.Optimize(std::move(op->children[0]), &lhs_stats);
 		// optimize the rhs child
 		auto rhs_optimizer = optimizer.CreateChildOptimizer();
-		auto table_index = op->type == LogicalOperatorType::LOGICAL_MATERIALIZED_CTE
-		                       ? op->Cast<LogicalMaterializedCTE>().table_index
-		                       : op->Cast<LogicalRecursiveCTE>().table_index;
+		auto table_index = op->Cast<LogicalCTE>().table_index;
+
+		auto child_1_card = lhs_stats.stats_initialized ? lhs_stats.cardinality : 0;
 		rhs_optimizer.AddMaterializedCTEStats(table_index, std::move(lhs_stats));
-		op->children[1] = rhs_optimizer.Optimize(std::move(op->children[1]));
+		if (op->type == LogicalOperatorType::LOGICAL_RECURSIVE_CTE) {
+			rhs_optimizer.recursive_cte_indexes.insert(op->Cast<LogicalCTE>().table_index);
+		}
+		RelationStats rhs_stats;
+		op->children[1] = rhs_optimizer.Optimize(std::move(op->children[1]), &rhs_stats);
+
+		// create the stats for the CTE
+		auto child_2_card = rhs_stats.stats_initialized ? rhs_stats.cardinality : 0;
+
+		if (op->type == LogicalOperatorType::LOGICAL_RECURSIVE_CTE) {
+			// we cannot really estimate the cardinality of a recursive CTE
+			// because we don't know how many times it will be executed
+			// we just assume it will be executed 1000 times
+			op->SetEstimatedCardinality(child_1_card + child_2_card * 1000);
+		} else if (op->type == LogicalOperatorType::LOGICAL_MATERIALIZED_CTE) {
+			// for a materialized CTE, we just take the cardinality of the right children
+			op->SetEstimatedCardinality(child_2_card);
+		}
+
 		return false;
 	}
 	case LogicalOperatorType::LOGICAL_CTE_REF: {
 		auto &cte_ref = op->Cast<LogicalCTERef>();
-		if (cte_ref.materialized_cte != CTEMaterialize::CTE_MATERIALIZE_ALWAYS) {
-			return false;
-		}
 		auto cte_stats = optimizer.GetMaterializedCTEStats(cte_ref.cte_index);
 		cte_ref.SetEstimatedCardinality(cte_stats.cardinality);
 		AddRelation(input_op, parent, cte_stats);
+
+		auto is_recursive = optimizer.recursive_cte_indexes.find(cte_ref.cte_index);
+		if (is_recursive != optimizer.recursive_cte_indexes.end()) {
+			return false;
+		}
 		return true;
 	}
 	case LogicalOperatorType::LOGICAL_DELIM_JOIN: {