duckdb 0.8.2-dev11.0 → 0.8.2-dev1212.0

package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp

@@ -14,6 +14,10 @@
 #include "duckdb/planner/operator/logical_window.hpp"
 #include "duckdb/function/function_binder.hpp"
 #include "duckdb/planner/subquery/flatten_dependent_join.hpp"
+#include "duckdb/common/enums/logical_operator_type.hpp"
+#include "duckdb/planner/operator/logical_dependent_join.hpp"
+#include "duckdb/planner/expression_binder/lateral_binder.hpp"
+#include "duckdb/planner/subquery/recursive_dependent_join_planner.hpp"
 
 namespace duckdb {
 
@@ -327,43 +331,43 @@ static unique_ptr<Expression> PlanCorrelatedSubquery(Binder &binder, BoundSubque
 	}
 }
 
-class RecursiveSubqueryPlanner : public LogicalOperatorVisitor {
-public:
-	explicit RecursiveSubqueryPlanner(Binder &binder) : binder(binder) {
-	}
-	void VisitOperator(LogicalOperator &op) override {
-		if (!op.children.empty()) {
-			root = std::move(op.children[0]);
-			D_ASSERT(root);
-			VisitOperatorExpressions(op);
-			op.children[0] = std::move(root);
-			for (idx_t i = 0; i < op.children.size(); i++) {
-				D_ASSERT(op.children[i]);
-				VisitOperator(*op.children[i]);
-			}
+void RecursiveDependentJoinPlanner::VisitOperator(LogicalOperator &op) {
+	if (!op.children.empty()) {
+		root = std::move(op.children[0]);
+		D_ASSERT(root);
+		if (root->type == LogicalOperatorType::LOGICAL_DEPENDENT_JOIN) {
+			// Found a dependent join, flatten it
+			auto &new_root = root->Cast<LogicalDependentJoin>();
+			root = binder.PlanLateralJoin(std::move(new_root.children[0]), std::move(new_root.children[1]),
+			                              new_root.correlated_columns, new_root.join_type,
+			                              std::move(new_root.join_condition));
+		}
+		VisitOperatorExpressions(op);
+		op.children[0] = std::move(root);
+		for (idx_t i = 0; i < op.children.size(); i++) {
+			D_ASSERT(op.children[i]);
+			VisitOperator(*op.children[i]);
 		}
 	}
+}
 
-	unique_ptr<Expression> VisitReplace(BoundSubqueryExpression &expr, unique_ptr<Expression> *expr_ptr) override {
-		return binder.PlanSubquery(expr, root);
-	}
-
-private:
-	unique_ptr<LogicalOperator> root;
-	Binder &binder;
-};
+unique_ptr<Expression> RecursiveDependentJoinPlanner::VisitReplace(BoundSubqueryExpression &expr,
+                                                                   unique_ptr<Expression> *expr_ptr) {
+	return binder.PlanSubquery(expr, root);
+}
 
 unique_ptr<Expression> Binder::PlanSubquery(BoundSubqueryExpression &expr, unique_ptr<LogicalOperator> &root) {
 	D_ASSERT(root);
 	// first we translate the QueryNode of the subquery into a logical plan
 	// note that we do not plan nested subqueries yet
 	auto sub_binder = Binder::CreateBinder(context, this);
-	sub_binder->plan_subquery = false;
+	sub_binder->is_outside_flattened = false;
 	auto subquery_root = sub_binder->CreatePlan(*expr.subquery);
 	D_ASSERT(subquery_root);
 
 	// now we actually flatten the subquery
 	auto plan = std::move(subquery_root);
+
 	unique_ptr<Expression> result_expression;
 	if (!expr.IsCorrelated()) {
 		result_expression = PlanUncorrelatedSubquery(*this, expr, root, std::move(plan));
@@ -371,8 +375,8 @@ unique_ptr<Expression> Binder::PlanSubquery(BoundSubqueryExpression &expr, uniqu
 		result_expression = PlanCorrelatedSubquery(*this, expr, root, std::move(plan));
 	}
 	// finally, we recursively plan the nested subqueries (if there are any)
-	if (sub_binder->has_unplanned_subqueries) {
-		RecursiveSubqueryPlanner plan(*this);
+	if (sub_binder->has_unplanned_dependent_joins) {
+		RecursiveDependentJoinPlanner plan(*this);
 		plan.VisitOperator(*root);
 	}
 	return result_expression;
@@ -383,7 +387,6 @@ void Binder::PlanSubqueries(unique_ptr<Expression> &expr_ptr, unique_ptr<Logical
 		return;
 	}
 	auto &expr = *expr_ptr;
-
 	// first visit the children of the node, if any
 	ExpressionIterator::EnumerateChildren(expr, [&](unique_ptr<Expression> &expr) { PlanSubqueries(expr, root); });
 
@@ -391,11 +394,11 @@ void Binder::PlanSubqueries(unique_ptr<Expression> &expr_ptr, unique_ptr<Logical
 	if (expr.expression_class == ExpressionClass::BOUND_SUBQUERY) {
 		auto &subquery = expr.Cast<BoundSubqueryExpression>();
 		// subquery node! plan it
-		if (subquery.IsCorrelated() && !plan_subquery) {
+		if (subquery.IsCorrelated() && !is_outside_flattened) {
 			// detected a nested correlated subquery
 			// we don't plan it yet here, we are currently planning a subquery
 			// nested subqueries will only be planned AFTER the current subquery has been flattened entirely
-			has_unplanned_subqueries = true;
+			has_unplanned_dependent_joins = true;
 			return;
 		}
 		expr_ptr = PlanSubquery(subquery, root);
@@ -411,7 +414,7 @@ unique_ptr<LogicalOperator> Binder::PlanLateralJoin(unique_ptr<LogicalOperator>
 	vector<unique_ptr<Expression>> arbitrary_expressions;
 	if (condition) {
 		// extract join conditions, if there are any
-		LogicalComparisonJoin::ExtractJoinConditions(join_type, left, right, std::move(condition), conditions,
+		LogicalComparisonJoin::ExtractJoinConditions(context, join_type, left, right, std::move(condition), conditions,
 		                                             arbitrary_expressions);
 	}
 

package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp

@@ -45,9 +45,18 @@ unique_ptr<BoundTableRef> Binder::Bind(BaseTableRef &ref) {
 			return Bind(subquery, found_cte);
 		} else {
 			// There is a CTE binding in the BindContext.
-			// This can only be the case if there is a recursive CTE present.
+			// This can only be the case if there is a recursive CTE,
+			// or a materialized CTE present.
 			auto index = GenerateTableIndex();
-			auto result = make_uniq<BoundCTERef>(index, ctebinding->index);
+			auto materialized = cte.materialized;
+			if (materialized == CTEMaterialize::CTE_MATERIALIZE_DEFAULT) {
+#ifdef DUCKDB_ALTERNATIVE_VERIFY
+				materialized = CTEMaterialize::CTE_MATERIALIZE_ALWAYS;
+#else
+				materialized = CTEMaterialize::CTE_MATERIALIZE_NEVER;
+#endif
+			}
+			auto result = make_uniq<BoundCTERef>(index, ctebinding->index, materialized);
 			auto b = ctebinding;
 			auto alias = ref.alias.empty() ? ref.table_name : ref.alias;
 			auto names = BindContext::AliasColumnNames(alias, b->names, ref.column_name_alias);

package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp

@@ -130,9 +130,19 @@ unique_ptr<BoundTableRef> Binder::Bind(JoinRef &ref) {
 	{
 		LateralBinder binder(left_binder, context);
 		result->right = right_binder.Bind(*ref.right);
-		result->correlated_columns = binder.ExtractCorrelatedColumns(right_binder);
-
-		result->lateral = binder.HasCorrelatedColumns();
+		bool is_lateral = false;
+		// Store the correlated columns in the right binder in bound ref for planning of LATERALs
+		// Ignore the correlated columns in the left binder, flattening handles those correlations
+		result->correlated_columns = right_binder.correlated_columns;
+		// Find correlations for the current join
+		for (auto &cor_col : result->correlated_columns) {
+			if (cor_col.depth == 1) {
+				// Depth 1 indicates columns binding from the left indicating a lateral join
+				is_lateral = true;
+				break;
+			}
+		}
+		result->lateral = is_lateral;
 		if (result->lateral) {
 			// lateral join: can only be an INNER or LEFT join
 			if (ref.type != JoinType::INNER && ref.type != JoinType::LEFT) {
@@ -210,6 +220,7 @@ unique_ptr<BoundTableRef> Binder::Bind(JoinRef &ref) {
 
 	case JoinRefType::CROSS:
 	case JoinRefType::POSITIONAL:
+	case JoinRefType::DEPENDENT:
 		break;
 	}
 	extra_using_columns = RemoveDuplicateUsingColumns(extra_using_columns);
@@ -268,8 +279,24 @@ unique_ptr<BoundTableRef> Binder::Bind(JoinRef &ref) {
 
 	bind_context.AddContext(std::move(left_binder.bind_context));
 	bind_context.AddContext(std::move(right_binder.bind_context));
-	MoveCorrelatedExpressions(left_binder);
-	MoveCorrelatedExpressions(right_binder);
+
+	// Update the correlated columns for the parent binder
+	// For the left binder, depth >= 1 indicates correlations from the parent binder
+	for (const auto &col : left_binder.correlated_columns) {
+		if (col.depth >= 1) {
+			AddCorrelatedColumn(col);
+		}
+	}
+	// For the right binder, depth > 1 indicates correlations from the parent binder
+	// (depth = 1 indicates correlations from the left side of the join)
+	for (auto col : right_binder.correlated_columns) {
+		if (col.depth > 1) {
+			// Decrement the depth to account for the effect of the lateral binder
+			col.depth--;
+			AddCorrelatedColumn(col);
+		}
+	}
+
 	for (auto &condition : extra_conditions) {
 		if (ref.condition) {
 			ref.condition = make_uniq<ConjunctionExpression>(ExpressionType::CONJUNCTION_AND, std::move(ref.condition),

package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp

@@ -18,6 +18,7 @@
 #include "duckdb/planner/tableref/bound_pivotref.hpp"
 #include "duckdb/planner/expression/bound_aggregate_expression.hpp"
 #include "duckdb/main/client_config.hpp"
+#include "duckdb/catalog/catalog_entry/aggregate_function_catalog_entry.hpp"
 
 namespace duckdb {
 
@@ -66,18 +67,10 @@ static void ExtractPivotExpressions(ParsedExpression &expr, case_insensitive_set
 	    expr, [&](ParsedExpression &child) { ExtractPivotExpressions(child, handled_columns); });
 }
 
-struct PivotBindState {
-	vector<string> internal_group_names;
-	vector<string> group_names;
-	vector<string> aggregate_names;
-	vector<string> internal_aggregate_names;
-};
-
-static unique_ptr<SelectNode> PivotInitialAggregate(PivotBindState &bind_state, PivotRef &ref,
-                                                    vector<unique_ptr<ParsedExpression>> all_columns,
-                                                    const case_insensitive_set_t &handled_columns) {
-	auto subquery_stage1 = make_uniq<SelectNode>();
-	subquery_stage1->from_table = std::move(ref.source);
+static unique_ptr<SelectNode> ConstructInitialGrouping(PivotRef &ref, vector<unique_ptr<ParsedExpression>> all_columns,
+                                                       const case_insensitive_set_t &handled_columns) {
+	auto subquery = make_uniq<SelectNode>();
+	subquery->from_table = std::move(ref.source);
 	if (ref.groups.empty()) {
 		// if rows are not specified any columns that are not pivoted/aggregated on are added to the GROUP BY clause
 		for (auto &entry : all_columns) {
@@ -87,19 +80,74 @@ static unique_ptr<SelectNode> PivotInitialAggregate(PivotBindState &bind_state,
 			auto &columnref = entry->Cast<ColumnRefExpression>();
 			if (handled_columns.find(columnref.GetColumnName()) == handled_columns.end()) {
 				// not handled - add to grouping set
-				subquery_stage1->groups.group_expressions.push_back(
-				    make_uniq<ConstantExpression>(Value::INTEGER(subquery_stage1->select_list.size() + 1)));
-				subquery_stage1->select_list.push_back(make_uniq<ColumnRefExpression>(columnref.GetColumnName()));
+				subquery->groups.group_expressions.push_back(
+				    make_uniq<ConstantExpression>(Value::INTEGER(subquery->select_list.size() + 1)));
+				subquery->select_list.push_back(make_uniq<ColumnRefExpression>(columnref.GetColumnName()));
 			}
 		}
 	} else {
 		// if rows are specified only the columns mentioned in rows are added as groups
 		for (auto &row : ref.groups) {
-			subquery_stage1->groups.group_expressions.push_back(
-			    make_uniq<ConstantExpression>(Value::INTEGER(subquery_stage1->select_list.size() + 1)));
-			subquery_stage1->select_list.push_back(make_uniq<ColumnRefExpression>(row));
+			subquery->groups.group_expressions.push_back(
+			    make_uniq<ConstantExpression>(Value::INTEGER(subquery->select_list.size() + 1)));
+			subquery->select_list.push_back(make_uniq<ColumnRefExpression>(row));
 		}
 	}
+	return subquery;
+}
+
+static unique_ptr<SelectNode> PivotFilteredAggregate(PivotRef &ref, vector<unique_ptr<ParsedExpression>> all_columns,
+                                                     const case_insensitive_set_t &handled_columns,
+                                                     vector<PivotValueElement> pivot_values) {
+	auto subquery = ConstructInitialGrouping(ref, std::move(all_columns), handled_columns);
+
+	// push the filtered aggregates
+	for (auto &pivot_value : pivot_values) {
+		unique_ptr<ParsedExpression> filter;
+		idx_t pivot_value_idx = 0;
+		for (auto &pivot_column : ref.pivots) {
+			for (auto &pivot_expr : pivot_column.pivot_expressions) {
+				auto column_ref = make_uniq<CastExpression>(LogicalType::VARCHAR, pivot_expr->Copy());
+				auto constant_value = make_uniq<ConstantExpression>(pivot_value.values[pivot_value_idx++]);
+				auto comp_expr = make_uniq<ComparisonExpression>(ExpressionType::COMPARE_NOT_DISTINCT_FROM,
+				                                                 std::move(column_ref), std::move(constant_value));
+				if (filter) {
+					filter = make_uniq<ConjunctionExpression>(ExpressionType::CONJUNCTION_AND, std::move(filter),
+					                                          std::move(comp_expr));
+				} else {
+					filter = std::move(comp_expr);
+				}
+			}
+		}
+		for (auto &aggregate : ref.aggregates) {
+			auto copied_aggr = aggregate->Copy();
+			auto &aggr = copied_aggr->Cast<FunctionExpression>();
+			aggr.filter = filter->Copy();
+			auto &aggr_name = aggregate->alias;
+			auto name = pivot_value.name;
+			if (ref.aggregates.size() > 1 || !aggr_name.empty()) {
+				// if there are multiple aggregates specified we add the name of the aggregate as well
+				name += "_" + (aggr_name.empty() ? aggregate->GetName() : aggr_name);
+			}
+			aggr.alias = name;
+			subquery->select_list.push_back(std::move(copied_aggr));
+		}
+	}
+	return subquery;
+}
+
+struct PivotBindState {
+	vector<string> internal_group_names;
+	vector<string> group_names;
+	vector<string> aggregate_names;
+	vector<string> internal_aggregate_names;
+};
+
+static unique_ptr<SelectNode> PivotInitialAggregate(PivotBindState &bind_state, PivotRef &ref,
+                                                    vector<unique_ptr<ParsedExpression>> all_columns,
+                                                    const case_insensitive_set_t &handled_columns) {
+	auto subquery_stage1 = ConstructInitialGrouping(ref, std::move(all_columns), handled_columns);
+
 	idx_t group_count = 0;
 	for (auto &expr : subquery_stage1->select_list) {
 		bind_state.group_names.push_back(expr->GetName());
@@ -134,6 +182,15 @@ static unique_ptr<SelectNode> PivotInitialAggregate(PivotBindState &bind_state,
 	return subquery_stage1;
 }
 
+unique_ptr<ParsedExpression> ConstructPivotExpression(unique_ptr<ParsedExpression> pivot_expr) {
+	auto cast = make_uniq<CastExpression>(LogicalType::VARCHAR, std::move(pivot_expr));
+	vector<unique_ptr<ParsedExpression>> coalesce_children;
+	coalesce_children.push_back(std::move(cast));
+	coalesce_children.push_back(make_uniq<ConstantExpression>(Value("NULL")));
+	auto coalesce = make_uniq<OperatorExpression>(ExpressionType::OPERATOR_COALESCE, std::move(coalesce_children));
+	return std::move(coalesce);
+}
+
 static unique_ptr<SelectNode> PivotListAggregate(PivotBindState &bind_state, PivotRef &ref,
                                                  unique_ptr<SelectNode> subquery_stage1) {
 	auto subquery_stage2 = make_uniq<SelectNode>();
@@ -166,13 +223,7 @@ static unique_ptr<SelectNode> PivotListAggregate(PivotBindState &bind_state, Piv
 	for (auto &pivot : ref.pivots) {
 		for (auto &pivot_expr : pivot.pivot_expressions) {
 			// coalesce(pivot::VARCHAR, 'NULL')
-			auto cast = make_uniq<CastExpression>(LogicalType::VARCHAR, std::move(pivot_expr));
-			vector<unique_ptr<ParsedExpression>> coalesce_children;
-			coalesce_children.push_back(std::move(cast));
-			coalesce_children.push_back(make_uniq<ConstantExpression>(Value("NULL")));
-			auto coalesce =
-			    make_uniq<OperatorExpression>(ExpressionType::OPERATOR_COALESCE, std::move(coalesce_children));
-
+			auto coalesce = ConstructPivotExpression(std::move(pivot_expr));
 			if (!expr) {
 				expr = std::move(coalesce);
 			} else {
@@ -250,9 +301,8 @@ unique_ptr<BoundTableRef> Binder::BindBoundPivot(PivotRef &ref) {
 	auto &aggregates = result->bound_pivot.aggregates;
 	ExtractPivotAggregates(*result->child, aggregates);
 	if (aggregates.size() != ref.bound_aggregate_names.size()) {
-		throw BinderException("Pivot aggregate count mismatch. Expected %llu aggregates but found %llu. Are all pivot "
-		                      "expressions aggregate functions?",
-		                      ref.bound_aggregate_names.size(), aggregates.size());
+		throw InternalException("Pivot aggregate count mismatch (expected %llu, found %llu)",
+		                        ref.bound_aggregate_names.size(), aggregates.size());
 	}
 
 	vector<string> child_names;
@@ -313,9 +363,12 @@ unique_ptr<SelectNode> Binder::BindPivot(PivotRef &ref, vector<unique_ptr<Parsed
 		if (aggr->IsWindow()) {
 			throw BinderException(FormatError(*aggr, "Pivot expression cannot contain window functions"));
 		}
+		// bind the function as an aggregate to ensure it is an aggregate and not a scalar function
+		auto &aggr_function = aggr->Cast<FunctionExpression>();
+		(void)Catalog::GetEntry<AggregateFunctionCatalogEntry>(context, aggr_function.catalog, aggr_function.schema,
+		                                                       aggr_function.function_name);
 		ExtractPivotExpressions(*aggr, handled_columns);
 	}
-	value_set_t pivots;
 
 	// first add all pivots to the set of handled columns, and check for duplicates
 	idx_t total_pivots = 1;
@@ -362,40 +415,53 @@ unique_ptr<SelectNode> Binder::BindPivot(PivotRef &ref, vector<unique_ptr<Parsed
 			pivots.insert(val);
 		}
 	}
-	auto pivot_limit = ClientConfig::GetConfig(context).pivot_limit;
+	auto &client_config = ClientConfig::GetConfig(context);
+	auto pivot_limit = client_config.pivot_limit;
 	if (total_pivots >= pivot_limit) {
 		throw BinderException("Pivot column limit of %llu exceeded. Use SET pivot_limit=X to increase the limit.",
-		                      ClientConfig::GetConfig(context).pivot_limit);
+		                      client_config.pivot_limit);
 	}
 
 	// construct the required pivot values recursively
 	vector<PivotValueElement> pivot_values;
 	ConstructPivots(ref, pivot_values);
 
+	unique_ptr<SelectNode> pivot_node;
 	// pivots have three components
 	// - the pivots (i.e. future column names)
 	// - the groups (i.e. the future row names
 	// - the aggregates (i.e. the values of the pivot columns)
 
-	// executing a pivot statement happens in three stages
-	// 1) execute the query "SELECT {groups}, {pivots}, {aggregates} FROM {from_clause} GROUP BY {groups}, {pivots}
-	// this computes all values that are required in the final result, but not yet in the correct orientation
-	// 2) execute the query "SELECT {groups}, LIST({pivots}), LIST({aggregates}) FROM [Q1] GROUP BY {groups}
-	// this pushes all pivots and aggregates that belong to a specific group together in an aligned manner
-	// 3) push a PIVOT operator, that performs the actual pivoting of the values into the different columns
-
-	PivotBindState bind_state;
-	// Pivot Stage 1
-	// SELECT {groups}, {pivots}, {aggregates} FROM {from_clause} GROUP BY {groups}, {pivots}
-	auto subquery_stage1 = PivotInitialAggregate(bind_state, ref, std::move(all_columns), handled_columns);
-
-	// Pivot stage 2
-	// SELECT {groups}, LIST({pivots}), LIST({aggregates}) FROM [Q1] GROUP BY {groups}
-	auto subquery_stage2 = PivotListAggregate(bind_state, ref, std::move(subquery_stage1));
-
-	// Pivot stage 3
-	// construct the final pivot operator
-	auto pivot_node = PivotFinalOperator(bind_state, ref, std::move(subquery_stage2), std::move(pivot_values));
+	// we have two ways of executing a pivot statement
+	// (1) the straightforward manner of filtered aggregates SUM(..) FILTER (pivot_value=X)
+	// (2) computing the aggregates once, then using LIST to group the aggregates together with the PIVOT operator
+	// -> filtered aggregates are faster when there are FEW pivot values
+	// -> LIST is faster when there are MANY pivot values
+	// we switch dynamically based on the number of pivots to compute
+	if (pivot_values.size() <= client_config.pivot_filter_threshold) {
+		// use a set of filtered aggregates
+		pivot_node = PivotFilteredAggregate(ref, std::move(all_columns), handled_columns, std::move(pivot_values));
+	} else {
+		// executing a pivot statement happens in three stages
+		// 1) execute the query "SELECT {groups}, {pivots}, {aggregates} FROM {from_clause} GROUP BY {groups}, {pivots}
+		// this computes all values that are required in the final result, but not yet in the correct orientation
+		// 2) execute the query "SELECT {groups}, LIST({pivots}), LIST({aggregates}) FROM [Q1] GROUP BY {groups}
+		// this pushes all pivots and aggregates that belong to a specific group together in an aligned manner
+		// 3) push a PIVOT operator, that performs the actual pivoting of the values into the different columns
+
+		PivotBindState bind_state;
+		// Pivot Stage 1
+		// SELECT {groups}, {pivots}, {aggregates} FROM {from_clause} GROUP BY {groups}, {pivots}
+		auto subquery_stage1 = PivotInitialAggregate(bind_state, ref, std::move(all_columns), handled_columns);
+
+		// Pivot stage 2
+		// SELECT {groups}, LIST({pivots}), LIST({aggregates}) FROM [Q1] GROUP BY {groups}
+		auto subquery_stage2 = PivotListAggregate(bind_state, ref, std::move(subquery_stage1));
+
+		// Pivot stage 3
+		// construct the final pivot operator
+		pivot_node = PivotFinalOperator(bind_state, ref, std::move(subquery_stage2), std::move(pivot_values));
+	}
 	return pivot_node;
 }
 

package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp

@@ -13,7 +13,7 @@ unique_ptr<LogicalOperator> Binder::CreatePlan(BoundCTERef &ref) {
 		types.push_back(type);
 	}
 
-	return make_uniq<LogicalCTERef>(index, ref.cte_index, types, ref.bound_columns);
+	return make_uniq<LogicalCTERef>(index, ref.cte_index, types, ref.bound_columns, ref.materialized_cte);
 }
 
 } // namespace duckdb

package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp

@@ -10,11 +10,14 @@
 #include "duckdb/planner/operator/logical_asof_join.hpp"
 #include "duckdb/planner/operator/logical_comparison_join.hpp"
 #include "duckdb/planner/operator/logical_cross_product.hpp"
+#include "duckdb/planner/operator/logical_dependent_join.hpp"
 #include "duckdb/planner/operator/logical_filter.hpp"
 #include "duckdb/planner/operator/logical_positional_join.hpp"
 #include "duckdb/planner/tableref/bound_joinref.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/planner/expression_binder/lateral_binder.hpp"
+#include "duckdb/planner/subquery/recursive_dependent_join_planner.hpp"
+#include "duckdb/execution/expression_executor.hpp"
 
 namespace duckdb {
 
@@ -44,13 +47,11 @@ static bool CreateJoinCondition(Expression &expr, const unordered_set<idx_t> &le
 	return false;
 }
 
-void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<LogicalOperator> &left_child,
-                                                  unique_ptr<LogicalOperator> &right_child,
-                                                  const unordered_set<idx_t> &left_bindings,
-                                                  const unordered_set<idx_t> &right_bindings,
-                                                  vector<unique_ptr<Expression>> &expressions,
-                                                  vector<JoinCondition> &conditions,
-                                                  vector<unique_ptr<Expression>> &arbitrary_expressions) {
+void LogicalComparisonJoin::ExtractJoinConditions(
+    ClientContext &context, JoinType type, unique_ptr<LogicalOperator> &left_child,
+    unique_ptr<LogicalOperator> &right_child, const unordered_set<idx_t> &left_bindings,
+    const unordered_set<idx_t> &right_bindings, vector<unique_ptr<Expression>> &expressions,
+    vector<JoinCondition> &conditions, vector<unique_ptr<Expression>> &arbitrary_expressions) {
 	for (auto &expr : expressions) {
 		auto total_side = JoinSide::GetJoinSide(*expr, left_bindings, right_bindings);
 		if (total_side != JoinSide::BOTH) {
@@ -68,6 +69,15 @@ void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<Logi
 				filter.expressions.push_back(std::move(expr));
 				continue;
 			}
+			// if the join is a LEFT JOIN and the join expression constantly evaluates to TRUE,
+			// then we do not add it to the arbitrary expressions
+			if (type == JoinType::LEFT && expr->IsFoldable()) {
+				Value result;
+				ExpressionExecutor::TryEvaluateScalar(context, *expr, result);
+				if (!result.IsNull() && result == Value(true)) {
+					continue;
+				}
+			}
 		} else if ((expr->type >= ExpressionType::COMPARE_EQUAL &&
 		            expr->type <= ExpressionType::COMPARE_GREATERTHANOREQUALTO) ||
 		           expr->type == ExpressionType::COMPARE_DISTINCT_FROM ||
@@ -82,7 +92,8 @@ void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<Logi
 	}
 }
 
-void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<LogicalOperator> &left_child,
+void LogicalComparisonJoin::ExtractJoinConditions(ClientContext &context, JoinType type,
+                                                  unique_ptr<LogicalOperator> &left_child,
                                                   unique_ptr<LogicalOperator> &right_child,
                                                   vector<unique_ptr<Expression>> &expressions,
                                                   vector<JoinCondition> &conditions,
@@ -90,11 +101,12 @@ void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<Logi
 	unordered_set<idx_t> left_bindings, right_bindings;
 	LogicalJoin::GetTableReferences(*left_child, left_bindings);
 	LogicalJoin::GetTableReferences(*right_child, right_bindings);
-	return ExtractJoinConditions(type, left_child, right_child, left_bindings, right_bindings, expressions, conditions,
-	                             arbitrary_expressions);
+	return ExtractJoinConditions(context, type, left_child, right_child, left_bindings, right_bindings, expressions,
+	                             conditions, arbitrary_expressions);
 }
 
-void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<LogicalOperator> &left_child,
+void LogicalComparisonJoin::ExtractJoinConditions(ClientContext &context, JoinType type,
+                                                  unique_ptr<LogicalOperator> &left_child,
                                                   unique_ptr<LogicalOperator> &right_child,
                                                   unique_ptr<Expression> condition, vector<JoinCondition> &conditions,
                                                   vector<unique_ptr<Expression>> &arbitrary_expressions) {
@@ -102,10 +114,12 @@ void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<Logi
 	vector<unique_ptr<Expression>> expressions;
 	expressions.push_back(std::move(condition));
 	LogicalFilter::SplitPredicates(expressions);
-	return ExtractJoinConditions(type, left_child, right_child, expressions, conditions, arbitrary_expressions);
+	return ExtractJoinConditions(context, type, left_child, right_child, expressions, conditions,
+	                             arbitrary_expressions);
 }
 
-unique_ptr<LogicalOperator> LogicalComparisonJoin::CreateJoin(JoinType type, JoinRefType reftype,
+unique_ptr<LogicalOperator> LogicalComparisonJoin::CreateJoin(ClientContext &context, JoinType type,
+                                                              JoinRefType reftype,
                                                               unique_ptr<LogicalOperator> left_child,
                                                               unique_ptr<LogicalOperator> right_child,
                                                               vector<JoinCondition> conditions,
@@ -216,29 +230,37 @@ static bool HasCorrelatedColumns(Expression &expression) {
 	return has_correlated_columns;
 }
 
-unique_ptr<LogicalOperator> LogicalComparisonJoin::CreateJoin(JoinType type, JoinRefType reftype,
+unique_ptr<LogicalOperator> LogicalComparisonJoin::CreateJoin(ClientContext &context, JoinType type,
+                                                              JoinRefType reftype,
                                                               unique_ptr<LogicalOperator> left_child,
                                                               unique_ptr<LogicalOperator> right_child,
                                                               unique_ptr<Expression> condition) {
 	vector<JoinCondition> conditions;
 	vector<unique_ptr<Expression>> arbitrary_expressions;
-	LogicalComparisonJoin::ExtractJoinConditions(type, left_child, right_child, std::move(condition), conditions,
-	                                             arbitrary_expressions);
-	return LogicalComparisonJoin::CreateJoin(type, reftype, std::move(left_child), std::move(right_child),
+	LogicalComparisonJoin::ExtractJoinConditions(context, type, left_child, right_child, std::move(condition),
+	                                             conditions, arbitrary_expressions);
+	return LogicalComparisonJoin::CreateJoin(context, type, reftype, std::move(left_child), std::move(right_child),
 	                                         std::move(conditions), std::move(arbitrary_expressions));
 }
 
 unique_ptr<LogicalOperator> Binder::CreatePlan(BoundJoinRef &ref) {
+	auto old_is_outside_flattened = is_outside_flattened;
+	// Plan laterals from outermost to innermost
+	if (ref.lateral) {
+		// Set the flag to ensure that children do not flatten before the root
+		is_outside_flattened = false;
+	}
 	auto left = CreatePlan(*ref.left);
 	auto right = CreatePlan(*ref.right);
+	is_outside_flattened = old_is_outside_flattened;
+
+	// For joins, depth of the bindings will be one higher on the right because of the lateral binder
+	// If the current join does not have correlations between left and right, then the right bindings
+	// have depth 1 too high and can be reduced by 1 throughout
 	if (!ref.lateral && !ref.correlated_columns.empty()) {
-		// non-lateral join with correlated columns
-		// this happens if there is a join (or cross product) in a correlated subquery
-		// due to the lateral binder the expression depth of all correlated columns in the "ref.correlated_columns" set
-		// is 1 too high
-		// we reduce expression depth of all columns in the "ref.correlated_columns" set by 1
 		LateralBinder::ReduceExpressionDepth(*right, ref.correlated_columns);
 	}
+
 	if (ref.type == JoinType::RIGHT && ref.ref_type != JoinRefType::ASOF &&
 	    ClientConfig::GetConfig(context).enable_optimizer) {
 		// we turn any right outer joins into left outer joins for optimization purposes
@@ -247,9 +269,22 @@ unique_ptr<LogicalOperator> Binder::CreatePlan(BoundJoinRef &ref) {
 		std::swap(left, right);
 	}
 	if (ref.lateral) {
-		// lateral join
-		return PlanLateralJoin(std::move(left), std::move(right), ref.correlated_columns, ref.type,
-		                       std::move(ref.condition));
+		if (!is_outside_flattened) {
+			// If outer dependent joins is yet to be flattened, only plan the lateral
+			has_unplanned_dependent_joins = true;
+			return LogicalDependentJoin::Create(std::move(left), std::move(right), ref.correlated_columns, ref.type,
+			                                    std::move(ref.condition));
+		} else {
+			// All outer dependent joins have been planned and flattened, so plan and flatten lateral and recursively
+			// plan the children
+			auto new_plan = PlanLateralJoin(std::move(left), std::move(right), ref.correlated_columns, ref.type,
+			                                std::move(ref.condition));
+			if (has_unplanned_dependent_joins) {
+				RecursiveDependentJoinPlanner plan(*this);
+				plan.VisitOperator(*new_plan);
+			}
+			return new_plan;
+		}
 	}
 	switch (ref.ref_type) {
 	case JoinRefType::CROSS:
@@ -275,7 +310,7 @@ unique_ptr<LogicalOperator> Binder::CreatePlan(BoundJoinRef &ref) {
 	}
 
 	// now create the join operator from the join condition
-	auto result = LogicalComparisonJoin::CreateJoin(ref.type, ref.ref_type, std::move(left), std::move(right),
+	auto result = LogicalComparisonJoin::CreateJoin(context, ref.type, ref.ref_type, std::move(left), std::move(right),
 	                                                std::move(ref.condition));
 
 	optional_ptr<LogicalOperator> join;

package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp

@@ -6,10 +6,10 @@ namespace duckdb {
 unique_ptr<LogicalOperator> Binder::CreatePlan(BoundSubqueryRef &ref) {
 	// generate the logical plan for the subquery
 	// this happens separately from the current LogicalPlan generation
-	ref.binder->plan_subquery = plan_subquery;
+	ref.binder->is_outside_flattened = is_outside_flattened;
 	auto subquery = ref.binder->CreatePlan(*ref.subquery);
-	if (ref.binder->has_unplanned_subqueries) {
-		has_unplanned_subqueries = true;
+	if (ref.binder->has_unplanned_dependent_joins) {
+		has_unplanned_dependent_joins = true;
 	}
 	return subquery;
 }