duckdb 0.8.2-dev33.0 → 0.8.2-dev3334.0

package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp

@@ -18,6 +18,7 @@
 #include "duckdb/planner/tableref/bound_pivotref.hpp"
 #include "duckdb/planner/expression/bound_aggregate_expression.hpp"
 #include "duckdb/main/client_config.hpp"
+#include "duckdb/catalog/catalog_entry/aggregate_function_catalog_entry.hpp"
 
 namespace duckdb {
 
@@ -66,18 +67,10 @@ static void ExtractPivotExpressions(ParsedExpression &expr, case_insensitive_set
 	    expr, [&](ParsedExpression &child) { ExtractPivotExpressions(child, handled_columns); });
 }
 
-struct PivotBindState {
-	vector<string> internal_group_names;
-	vector<string> group_names;
-	vector<string> aggregate_names;
-	vector<string> internal_aggregate_names;
-};
-
-static unique_ptr<SelectNode> PivotInitialAggregate(PivotBindState &bind_state, PivotRef &ref,
-                                                    vector<unique_ptr<ParsedExpression>> all_columns,
-                                                    const case_insensitive_set_t &handled_columns) {
-	auto subquery_stage1 = make_uniq<SelectNode>();
-	subquery_stage1->from_table = std::move(ref.source);
+static unique_ptr<SelectNode> ConstructInitialGrouping(PivotRef &ref, vector<unique_ptr<ParsedExpression>> all_columns,
+                                                       const case_insensitive_set_t &handled_columns) {
+	auto subquery = make_uniq<SelectNode>();
+	subquery->from_table = std::move(ref.source);
 	if (ref.groups.empty()) {
 		// if rows are not specified any columns that are not pivoted/aggregated on are added to the GROUP BY clause
 		for (auto &entry : all_columns) {
@@ -87,19 +80,74 @@ static unique_ptr<SelectNode> PivotInitialAggregate(PivotBindState &bind_state,
 			auto &columnref = entry->Cast<ColumnRefExpression>();
 			if (handled_columns.find(columnref.GetColumnName()) == handled_columns.end()) {
 				// not handled - add to grouping set
-				subquery_stage1->groups.group_expressions.push_back(
-				    make_uniq<ConstantExpression>(Value::INTEGER(subquery_stage1->select_list.size() + 1)));
-				subquery_stage1->select_list.push_back(make_uniq<ColumnRefExpression>(columnref.GetColumnName()));
+				subquery->groups.group_expressions.push_back(
+				    make_uniq<ConstantExpression>(Value::INTEGER(subquery->select_list.size() + 1)));
+				subquery->select_list.push_back(make_uniq<ColumnRefExpression>(columnref.GetColumnName()));
 			}
 		}
 	} else {
 		// if rows are specified only the columns mentioned in rows are added as groups
 		for (auto &row : ref.groups) {
-			subquery_stage1->groups.group_expressions.push_back(
-			    make_uniq<ConstantExpression>(Value::INTEGER(subquery_stage1->select_list.size() + 1)));
-			subquery_stage1->select_list.push_back(make_uniq<ColumnRefExpression>(row));
+			subquery->groups.group_expressions.push_back(
+			    make_uniq<ConstantExpression>(Value::INTEGER(subquery->select_list.size() + 1)));
+			subquery->select_list.push_back(make_uniq<ColumnRefExpression>(row));
 		}
 	}
+	return subquery;
+}
+
+static unique_ptr<SelectNode> PivotFilteredAggregate(PivotRef &ref, vector<unique_ptr<ParsedExpression>> all_columns,
+                                                     const case_insensitive_set_t &handled_columns,
+                                                     vector<PivotValueElement> pivot_values) {
+	auto subquery = ConstructInitialGrouping(ref, std::move(all_columns), handled_columns);
+
+	// push the filtered aggregates
+	for (auto &pivot_value : pivot_values) {
+		unique_ptr<ParsedExpression> filter;
+		idx_t pivot_value_idx = 0;
+		for (auto &pivot_column : ref.pivots) {
+			for (auto &pivot_expr : pivot_column.pivot_expressions) {
+				auto column_ref = make_uniq<CastExpression>(LogicalType::VARCHAR, pivot_expr->Copy());
+				auto constant_value = make_uniq<ConstantExpression>(pivot_value.values[pivot_value_idx++]);
+				auto comp_expr = make_uniq<ComparisonExpression>(ExpressionType::COMPARE_NOT_DISTINCT_FROM,
+				                                                 std::move(column_ref), std::move(constant_value));
+				if (filter) {
+					filter = make_uniq<ConjunctionExpression>(ExpressionType::CONJUNCTION_AND, std::move(filter),
+					                                          std::move(comp_expr));
+				} else {
+					filter = std::move(comp_expr);
+				}
+			}
+		}
+		for (auto &aggregate : ref.aggregates) {
+			auto copied_aggr = aggregate->Copy();
+			auto &aggr = copied_aggr->Cast<FunctionExpression>();
+			aggr.filter = filter->Copy();
+			auto &aggr_name = aggregate->alias;
+			auto name = pivot_value.name;
+			if (ref.aggregates.size() > 1 || !aggr_name.empty()) {
+				// if there are multiple aggregates specified we add the name of the aggregate as well
+				name += "_" + (aggr_name.empty() ? aggregate->GetName() : aggr_name);
+			}
+			aggr.alias = name;
+			subquery->select_list.push_back(std::move(copied_aggr));
+		}
+	}
+	return subquery;
+}
+
+struct PivotBindState {
+	vector<string> internal_group_names;
+	vector<string> group_names;
+	vector<string> aggregate_names;
+	vector<string> internal_aggregate_names;
+};
+
+static unique_ptr<SelectNode> PivotInitialAggregate(PivotBindState &bind_state, PivotRef &ref,
+                                                    vector<unique_ptr<ParsedExpression>> all_columns,
+                                                    const case_insensitive_set_t &handled_columns) {
+	auto subquery_stage1 = ConstructInitialGrouping(ref, std::move(all_columns), handled_columns);
+
 	idx_t group_count = 0;
 	for (auto &expr : subquery_stage1->select_list) {
 		bind_state.group_names.push_back(expr->GetName());
@@ -134,6 +182,15 @@ static unique_ptr<SelectNode> PivotInitialAggregate(PivotBindState &bind_state,
 	return subquery_stage1;
 }
 
+unique_ptr<ParsedExpression> ConstructPivotExpression(unique_ptr<ParsedExpression> pivot_expr) {
+	auto cast = make_uniq<CastExpression>(LogicalType::VARCHAR, std::move(pivot_expr));
+	vector<unique_ptr<ParsedExpression>> coalesce_children;
+	coalesce_children.push_back(std::move(cast));
+	coalesce_children.push_back(make_uniq<ConstantExpression>(Value("NULL")));
+	auto coalesce = make_uniq<OperatorExpression>(ExpressionType::OPERATOR_COALESCE, std::move(coalesce_children));
+	return std::move(coalesce);
+}
+
 static unique_ptr<SelectNode> PivotListAggregate(PivotBindState &bind_state, PivotRef &ref,
                                                  unique_ptr<SelectNode> subquery_stage1) {
 	auto subquery_stage2 = make_uniq<SelectNode>();
@@ -166,13 +223,7 @@ static unique_ptr<SelectNode> PivotListAggregate(PivotBindState &bind_state, Piv
 	for (auto &pivot : ref.pivots) {
 		for (auto &pivot_expr : pivot.pivot_expressions) {
 			// coalesce(pivot::VARCHAR, 'NULL')
-			auto cast = make_uniq<CastExpression>(LogicalType::VARCHAR, std::move(pivot_expr));
-			vector<unique_ptr<ParsedExpression>> coalesce_children;
-			coalesce_children.push_back(std::move(cast));
-			coalesce_children.push_back(make_uniq<ConstantExpression>(Value("NULL")));
-			auto coalesce =
-			    make_uniq<OperatorExpression>(ExpressionType::OPERATOR_COALESCE, std::move(coalesce_children));
-
+			auto coalesce = ConstructPivotExpression(std::move(pivot_expr));
 			if (!expr) {
 				expr = std::move(coalesce);
 			} else {
@@ -250,9 +301,8 @@ unique_ptr<BoundTableRef> Binder::BindBoundPivot(PivotRef &ref) {
 	auto &aggregates = result->bound_pivot.aggregates;
 	ExtractPivotAggregates(*result->child, aggregates);
 	if (aggregates.size() != ref.bound_aggregate_names.size()) {
-		throw BinderException("Pivot aggregate count mismatch. Expected %llu aggregates but found %llu. Are all pivot "
-		                      "expressions aggregate functions?",
-		                      ref.bound_aggregate_names.size(), aggregates.size());
+		throw InternalException("Pivot aggregate count mismatch (expected %llu, found %llu)",
+		                        ref.bound_aggregate_names.size(), aggregates.size());
 	}
 
 	vector<string> child_names;
@@ -313,9 +363,12 @@ unique_ptr<SelectNode> Binder::BindPivot(PivotRef &ref, vector<unique_ptr<Parsed
 		if (aggr->IsWindow()) {
 			throw BinderException(FormatError(*aggr, "Pivot expression cannot contain window functions"));
 		}
+		// bind the function as an aggregate to ensure it is an aggregate and not a scalar function
+		auto &aggr_function = aggr->Cast<FunctionExpression>();
+		(void)Catalog::GetEntry<AggregateFunctionCatalogEntry>(context, aggr_function.catalog, aggr_function.schema,
+		                                                       aggr_function.function_name);
 		ExtractPivotExpressions(*aggr, handled_columns);
 	}
-	value_set_t pivots;
 
 	// first add all pivots to the set of handled columns, and check for duplicates
 	idx_t total_pivots = 1;
@@ -362,40 +415,53 @@ unique_ptr<SelectNode> Binder::BindPivot(PivotRef &ref, vector<unique_ptr<Parsed
 			pivots.insert(val);
 		}
 	}
-	auto pivot_limit = ClientConfig::GetConfig(context).pivot_limit;
+	auto &client_config = ClientConfig::GetConfig(context);
+	auto pivot_limit = client_config.pivot_limit;
 	if (total_pivots >= pivot_limit) {
 		throw BinderException("Pivot column limit of %llu exceeded. Use SET pivot_limit=X to increase the limit.",
-		                      ClientConfig::GetConfig(context).pivot_limit);
+		                      client_config.pivot_limit);
 	}
 
 	// construct the required pivot values recursively
 	vector<PivotValueElement> pivot_values;
 	ConstructPivots(ref, pivot_values);
 
+	unique_ptr<SelectNode> pivot_node;
 	// pivots have three components
 	// - the pivots (i.e. future column names)
 	// - the groups (i.e. the future row names
 	// - the aggregates (i.e. the values of the pivot columns)
 
-	// executing a pivot statement happens in three stages
-	// 1) execute the query "SELECT {groups}, {pivots}, {aggregates} FROM {from_clause} GROUP BY {groups}, {pivots}
-	// this computes all values that are required in the final result, but not yet in the correct orientation
-	// 2) execute the query "SELECT {groups}, LIST({pivots}), LIST({aggregates}) FROM [Q1] GROUP BY {groups}
-	// this pushes all pivots and aggregates that belong to a specific group together in an aligned manner
-	// 3) push a PIVOT operator, that performs the actual pivoting of the values into the different columns
-
-	PivotBindState bind_state;
-	// Pivot Stage 1
-	// SELECT {groups}, {pivots}, {aggregates} FROM {from_clause} GROUP BY {groups}, {pivots}
-	auto subquery_stage1 = PivotInitialAggregate(bind_state, ref, std::move(all_columns), handled_columns);
-
-	// Pivot stage 2
-	// SELECT {groups}, LIST({pivots}), LIST({aggregates}) FROM [Q1] GROUP BY {groups}
-	auto subquery_stage2 = PivotListAggregate(bind_state, ref, std::move(subquery_stage1));
-
-	// Pivot stage 3
-	// construct the final pivot operator
-	auto pivot_node = PivotFinalOperator(bind_state, ref, std::move(subquery_stage2), std::move(pivot_values));
+	// we have two ways of executing a pivot statement
+	// (1) the straightforward manner of filtered aggregates SUM(..) FILTER (pivot_value=X)
+	// (2) computing the aggregates once, then using LIST to group the aggregates together with the PIVOT operator
+	// -> filtered aggregates are faster when there are FEW pivot values
+	// -> LIST is faster when there are MANY pivot values
+	// we switch dynamically based on the number of pivots to compute
+	if (pivot_values.size() <= client_config.pivot_filter_threshold) {
+		// use a set of filtered aggregates
+		pivot_node = PivotFilteredAggregate(ref, std::move(all_columns), handled_columns, std::move(pivot_values));
+	} else {
+		// executing a pivot statement happens in three stages
+		// 1) execute the query "SELECT {groups}, {pivots}, {aggregates} FROM {from_clause} GROUP BY {groups}, {pivots}
+		// this computes all values that are required in the final result, but not yet in the correct orientation
+		// 2) execute the query "SELECT {groups}, LIST({pivots}), LIST({aggregates}) FROM [Q1] GROUP BY {groups}
+		// this pushes all pivots and aggregates that belong to a specific group together in an aligned manner
+		// 3) push a PIVOT operator, that performs the actual pivoting of the values into the different columns
+
+		PivotBindState bind_state;
+		// Pivot Stage 1
+		// SELECT {groups}, {pivots}, {aggregates} FROM {from_clause} GROUP BY {groups}, {pivots}
+		auto subquery_stage1 = PivotInitialAggregate(bind_state, ref, std::move(all_columns), handled_columns);
+
+		// Pivot stage 2
+		// SELECT {groups}, LIST({pivots}), LIST({aggregates}) FROM [Q1] GROUP BY {groups}
+		auto subquery_stage2 = PivotListAggregate(bind_state, ref, std::move(subquery_stage1));
+
+		// Pivot stage 3
+		// construct the final pivot operator
+		pivot_node = PivotFinalOperator(bind_state, ref, std::move(subquery_stage2), std::move(pivot_values));
+	}
 	return pivot_node;
 }
 

package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp

@@ -13,7 +13,7 @@ unique_ptr<LogicalOperator> Binder::CreatePlan(BoundCTERef &ref) {
 		types.push_back(type);
 	}
 
-	return make_uniq<LogicalCTERef>(index, ref.cte_index, types, ref.bound_columns);
+	return make_uniq<LogicalCTERef>(index, ref.cte_index, types, ref.bound_columns, ref.materialized_cte);
 }
 
 } // namespace duckdb

package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp

@@ -7,14 +7,16 @@
 #include "duckdb/planner/expression_iterator.hpp"
 #include "duckdb/planner/binder.hpp"
 #include "duckdb/planner/operator/logical_any_join.hpp"
-#include "duckdb/planner/operator/logical_asof_join.hpp"
 #include "duckdb/planner/operator/logical_comparison_join.hpp"
 #include "duckdb/planner/operator/logical_cross_product.hpp"
+#include "duckdb/planner/operator/logical_dependent_join.hpp"
 #include "duckdb/planner/operator/logical_filter.hpp"
 #include "duckdb/planner/operator/logical_positional_join.hpp"
 #include "duckdb/planner/tableref/bound_joinref.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/planner/expression_binder/lateral_binder.hpp"
+#include "duckdb/planner/subquery/recursive_dependent_join_planner.hpp"
+#include "duckdb/execution/expression_executor.hpp"
 
 namespace duckdb {
 
@@ -44,13 +46,11 @@ static bool CreateJoinCondition(Expression &expr, const unordered_set<idx_t> &le
 	return false;
 }
 
-void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<LogicalOperator> &left_child,
-                                                  unique_ptr<LogicalOperator> &right_child,
-                                                  const unordered_set<idx_t> &left_bindings,
-                                                  const unordered_set<idx_t> &right_bindings,
-                                                  vector<unique_ptr<Expression>> &expressions,
-                                                  vector<JoinCondition> &conditions,
-                                                  vector<unique_ptr<Expression>> &arbitrary_expressions) {
+void LogicalComparisonJoin::ExtractJoinConditions(
+    ClientContext &context, JoinType type, unique_ptr<LogicalOperator> &left_child,
+    unique_ptr<LogicalOperator> &right_child, const unordered_set<idx_t> &left_bindings,
+    const unordered_set<idx_t> &right_bindings, vector<unique_ptr<Expression>> &expressions,
+    vector<JoinCondition> &conditions, vector<unique_ptr<Expression>> &arbitrary_expressions) {
 	for (auto &expr : expressions) {
 		auto total_side = JoinSide::GetJoinSide(*expr, left_bindings, right_bindings);
 		if (total_side != JoinSide::BOTH) {
@@ -68,6 +68,15 @@ void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<Logi
 				filter.expressions.push_back(std::move(expr));
 				continue;
 			}
+			// if the join is a LEFT JOIN and the join expression constantly evaluates to TRUE,
+			// then we do not add it to the arbitrary expressions
+			if (type == JoinType::LEFT && expr->IsFoldable()) {
+				Value result;
+				ExpressionExecutor::TryEvaluateScalar(context, *expr, result);
+				if (!result.IsNull() && result == Value(true)) {
+					continue;
+				}
+			}
 		} else if ((expr->type >= ExpressionType::COMPARE_EQUAL &&
 		            expr->type <= ExpressionType::COMPARE_GREATERTHANOREQUALTO) ||
 		           expr->type == ExpressionType::COMPARE_DISTINCT_FROM ||
@@ -82,7 +91,8 @@ void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<Logi
 	}
 }
 
-void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<LogicalOperator> &left_child,
+void LogicalComparisonJoin::ExtractJoinConditions(ClientContext &context, JoinType type,
+                                                  unique_ptr<LogicalOperator> &left_child,
                                                   unique_ptr<LogicalOperator> &right_child,
                                                   vector<unique_ptr<Expression>> &expressions,
                                                   vector<JoinCondition> &conditions,
@@ -90,11 +100,12 @@ void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<Logi
 	unordered_set<idx_t> left_bindings, right_bindings;
 	LogicalJoin::GetTableReferences(*left_child, left_bindings);
 	LogicalJoin::GetTableReferences(*right_child, right_bindings);
-	return ExtractJoinConditions(type, left_child, right_child, left_bindings, right_bindings, expressions, conditions,
-	                             arbitrary_expressions);
+	return ExtractJoinConditions(context, type, left_child, right_child, left_bindings, right_bindings, expressions,
+	                             conditions, arbitrary_expressions);
 }
 
-void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<LogicalOperator> &left_child,
+void LogicalComparisonJoin::ExtractJoinConditions(ClientContext &context, JoinType type,
+                                                  unique_ptr<LogicalOperator> &left_child,
                                                   unique_ptr<LogicalOperator> &right_child,
                                                   unique_ptr<Expression> condition, vector<JoinCondition> &conditions,
                                                   vector<unique_ptr<Expression>> &arbitrary_expressions) {
@@ -102,10 +113,12 @@ void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<Logi
 	vector<unique_ptr<Expression>> expressions;
 	expressions.push_back(std::move(condition));
 	LogicalFilter::SplitPredicates(expressions);
-	return ExtractJoinConditions(type, left_child, right_child, expressions, conditions, arbitrary_expressions);
+	return ExtractJoinConditions(context, type, left_child, right_child, expressions, conditions,
+	                             arbitrary_expressions);
 }
 
-unique_ptr<LogicalOperator> LogicalComparisonJoin::CreateJoin(JoinType type, JoinRefType reftype,
+unique_ptr<LogicalOperator> LogicalComparisonJoin::CreateJoin(ClientContext &context, JoinType type,
+                                                              JoinRefType reftype,
                                                               unique_ptr<LogicalOperator> left_child,
                                                               unique_ptr<LogicalOperator> right_child,
                                                               vector<JoinCondition> conditions,
@@ -123,6 +136,9 @@ unique_ptr<LogicalOperator> LogicalComparisonJoin::CreateJoin(JoinType type, Joi
 			case ExpressionType::COMPARE_NOT_DISTINCT_FROM:
 				break;
 			case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
+			case ExpressionType::COMPARE_GREATERTHAN:
+			case ExpressionType::COMPARE_LESSTHANOREQUALTO:
+			case ExpressionType::COMPARE_LESSTHAN:
 				if (asof_idx < conditions.size()) {
 					throw BinderException("Multiple ASOF JOIN inequalities");
 				}
@@ -176,12 +192,11 @@ unique_ptr<LogicalOperator> LogicalComparisonJoin::CreateJoin(JoinType type, Joi
 	} else {
 		// we successfully converted expressions into JoinConditions
 		// create a LogicalComparisonJoin
-		unique_ptr<LogicalComparisonJoin> comp_join;
+		auto logical_type = LogicalOperatorType::LOGICAL_COMPARISON_JOIN;
 		if (reftype == JoinRefType::ASOF) {
-			comp_join = make_uniq<LogicalAsOfJoin>(type);
-		} else {
-			comp_join = make_uniq<LogicalComparisonJoin>(type);
+			logical_type = LogicalOperatorType::LOGICAL_ASOF_JOIN;
 		}
+		auto comp_join = make_uniq<LogicalComparisonJoin>(type, logical_type);
 		comp_join->conditions = std::move(conditions);
 		comp_join->children.push_back(std::move(left_child));
 		comp_join->children.push_back(std::move(right_child));
@@ -216,29 +231,37 @@ static bool HasCorrelatedColumns(Expression &expression) {
 	return has_correlated_columns;
 }
 
-unique_ptr<LogicalOperator> LogicalComparisonJoin::CreateJoin(JoinType type, JoinRefType reftype,
+unique_ptr<LogicalOperator> LogicalComparisonJoin::CreateJoin(ClientContext &context, JoinType type,
+                                                              JoinRefType reftype,
                                                               unique_ptr<LogicalOperator> left_child,
                                                               unique_ptr<LogicalOperator> right_child,
                                                               unique_ptr<Expression> condition) {
 	vector<JoinCondition> conditions;
 	vector<unique_ptr<Expression>> arbitrary_expressions;
-	LogicalComparisonJoin::ExtractJoinConditions(type, left_child, right_child, std::move(condition), conditions,
-	                                             arbitrary_expressions);
-	return LogicalComparisonJoin::CreateJoin(type, reftype, std::move(left_child), std::move(right_child),
+	LogicalComparisonJoin::ExtractJoinConditions(context, type, left_child, right_child, std::move(condition),
+	                                             conditions, arbitrary_expressions);
+	return LogicalComparisonJoin::CreateJoin(context, type, reftype, std::move(left_child), std::move(right_child),
 	                                         std::move(conditions), std::move(arbitrary_expressions));
 }
 
 unique_ptr<LogicalOperator> Binder::CreatePlan(BoundJoinRef &ref) {
+	auto old_is_outside_flattened = is_outside_flattened;
+	// Plan laterals from outermost to innermost
+	if (ref.lateral) {
+		// Set the flag to ensure that children do not flatten before the root
+		is_outside_flattened = false;
+	}
 	auto left = CreatePlan(*ref.left);
 	auto right = CreatePlan(*ref.right);
+	is_outside_flattened = old_is_outside_flattened;
+
+	// For joins, depth of the bindings will be one higher on the right because of the lateral binder
+	// If the current join does not have correlations between left and right, then the right bindings
+	// have depth 1 too high and can be reduced by 1 throughout
 	if (!ref.lateral && !ref.correlated_columns.empty()) {
-		// non-lateral join with correlated columns
-		// this happens if there is a join (or cross product) in a correlated subquery
-		// due to the lateral binder the expression depth of all correlated columns in the "ref.correlated_columns" set
-		// is 1 too high
-		// we reduce expression depth of all columns in the "ref.correlated_columns" set by 1
 		LateralBinder::ReduceExpressionDepth(*right, ref.correlated_columns);
 	}
+
 	if (ref.type == JoinType::RIGHT && ref.ref_type != JoinRefType::ASOF &&
 	    ClientConfig::GetConfig(context).enable_optimizer) {
 		// we turn any right outer joins into left outer joins for optimization purposes
@@ -247,9 +270,22 @@ unique_ptr<LogicalOperator> Binder::CreatePlan(BoundJoinRef &ref) {
 		std::swap(left, right);
 	}
 	if (ref.lateral) {
-		// lateral join
-		return PlanLateralJoin(std::move(left), std::move(right), ref.correlated_columns, ref.type,
-		                       std::move(ref.condition));
+		if (!is_outside_flattened) {
+			// If outer dependent joins is yet to be flattened, only plan the lateral
+			has_unplanned_dependent_joins = true;
+			return LogicalDependentJoin::Create(std::move(left), std::move(right), ref.correlated_columns, ref.type,
+			                                    std::move(ref.condition));
+		} else {
+			// All outer dependent joins have been planned and flattened, so plan and flatten lateral and recursively
+			// plan the children
+			auto new_plan = PlanLateralJoin(std::move(left), std::move(right), ref.correlated_columns, ref.type,
+			                                std::move(ref.condition));
+			if (has_unplanned_dependent_joins) {
+				RecursiveDependentJoinPlanner plan(*this);
+				plan.VisitOperator(*new_plan);
+			}
+			return new_plan;
+		}
 	}
 	switch (ref.ref_type) {
 	case JoinRefType::CROSS:
@@ -275,7 +311,7 @@ unique_ptr<LogicalOperator> Binder::CreatePlan(BoundJoinRef &ref) {
 	}
 
 	// now create the join operator from the join condition
-	auto result = LogicalComparisonJoin::CreateJoin(ref.type, ref.ref_type, std::move(left), std::move(right),
+	auto result = LogicalComparisonJoin::CreateJoin(context, ref.type, ref.ref_type, std::move(left), std::move(right),
 	                                                std::move(ref.condition));
 
 	optional_ptr<LogicalOperator> join;

package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp

@@ -6,10 +6,10 @@ namespace duckdb {
 unique_ptr<LogicalOperator> Binder::CreatePlan(BoundSubqueryRef &ref) {
 	// generate the logical plan for the subquery
 	// this happens separately from the current LogicalPlan generation
-	ref.binder->plan_subquery = plan_subquery;
+	ref.binder->is_outside_flattened = is_outside_flattened;
 	auto subquery = ref.binder->CreatePlan(*ref.subquery);
-	if (ref.binder->has_unplanned_subqueries) {
-		has_unplanned_subqueries = true;
+	if (ref.binder->has_unplanned_dependent_joins) {
+		has_unplanned_dependent_joins = true;
 	}
 	return subquery;
 }

package/src/duckdb/src/planner/binder.cpp

@@ -21,7 +21,31 @@
 
 namespace duckdb {
 
+Binder *Binder::GetRootBinder() {
+	Binder *root = this;
+	while (root->parent) {
+		root = root->parent.get();
+	}
+	return root;
+}
+
+idx_t Binder::GetBinderDepth() const {
+	const Binder *root = this;
+	idx_t depth = 1;
+	while (root->parent) {
+		depth++;
+		root = root->parent.get();
+	}
+	return depth;
+}
+
 shared_ptr<Binder> Binder::CreateBinder(ClientContext &context, optional_ptr<Binder> parent, bool inherit_ctes) {
+	auto depth = parent ? parent->GetBinderDepth() : 0;
+	if (depth > context.config.max_expression_depth) {
+		throw BinderException("Max expression depth limit of %lld exceeded. Use \"SET max_expression_depth TO x\" to "
+		                      "increase the maximum expression depth.",
+		                      context.config.max_expression_depth);
+	}
 	return make_shared<Binder>(true, context, parent ? parent->shared_from_this() : nullptr, inherit_ctes);
 }
 
@@ -117,6 +141,9 @@ unique_ptr<BoundQueryNode> Binder::BindNode(QueryNode &node) {
 	case QueryNodeType::RECURSIVE_CTE_NODE:
 		result = BindNode(node.Cast<RecursiveCTENode>());
 		break;
+	case QueryNodeType::CTE_NODE:
+		result = BindNode(node.Cast<CTENode>());
+		break;
 	default:
 		D_ASSERT(node.type == QueryNodeType::SET_OPERATION_NODE);
 		result = BindNode(node.Cast<SetOperationNode>());
@@ -145,6 +172,8 @@ unique_ptr<LogicalOperator> Binder::CreatePlan(BoundQueryNode &node) {
 		return CreatePlan(node.Cast<BoundSetOperationNode>());
 	case QueryNodeType::RECURSIVE_CTE_NODE:
 		return CreatePlan(node.Cast<BoundRecursiveCTENode>());
+	case QueryNodeType::CTE_NODE:
+		return CreatePlan(node.Cast<BoundCTENode>());
 	default:
 		throw InternalException("Unsupported bound query node type");
 	}
@@ -266,11 +295,8 @@ void Binder::AddBoundView(ViewCatalogEntry &view) {
 }
 
 idx_t Binder::GenerateTableIndex() {
-	D_ASSERT(parent.get() != this);
-	if (parent) {
-		return parent->GenerateTableIndex();
-	}
-	return bound_tables++;
+	auto root_binder = GetRootBinder();
+	return root_binder->bound_tables++;
 }
 
 void Binder::PushExpressionBinder(ExpressionBinder &binder) {
@@ -296,18 +322,13 @@ bool Binder::HasActiveBinder() {
 }
 
 vector<reference<ExpressionBinder>> &Binder::GetActiveBinders() {
-	if (parent) {
-		return parent->GetActiveBinders();
-	}
-	return active_binders;
+	auto root_binder = GetRootBinder();
+	return root_binder->active_binders;
 }
 
 void Binder::AddUsingBindingSet(unique_ptr<UsingColumnSet> set) {
-	if (parent) {
-		parent->AddUsingBindingSet(std::move(set));
-		return;
-	}
-	bind_context.AddUsingBindingSet(std::move(set));
+	auto root_binder = GetRootBinder();
+	root_binder->bind_context.AddUsingBindingSet(std::move(set));
 }
 
 void Binder::MoveCorrelatedExpressions(Binder &other) {
@@ -376,17 +397,14 @@ bool Binder::HasMatchingBinding(const string &catalog_name, const string &schema
 }
 
 void Binder::SetBindingMode(BindingMode mode) {
-	if (parent) {
-		parent->SetBindingMode(mode);
-	}
-	this->mode = mode;
+	auto root_binder = GetRootBinder();
+	// FIXME: this used to also set the 'mode' for the current binder, was that necessary?
+	root_binder->mode = mode;
 }
 
 BindingMode Binder::GetBindingMode() {
-	if (parent) {
-		return parent->GetBindingMode();
-	}
-	return mode;
+	auto root_binder = GetRootBinder();
+	return root_binder->mode;
 }
 
 void Binder::SetCanContainNulls(bool can_contain_nulls_p) {
@@ -394,18 +412,13 @@ void Binder::SetCanContainNulls(bool can_contain_nulls_p) {
 }
 
 void Binder::AddTableName(string table_name) {
-	if (parent) {
-		parent->AddTableName(std::move(table_name));
-		return;
-	}
-	table_names.insert(std::move(table_name));
+	auto root_binder = GetRootBinder();
+	root_binder->table_names.insert(std::move(table_name));
 }
 
 const unordered_set<string> &Binder::GetTableNames() {
-	if (parent) {
-		return parent->GetTableNames();
-	}
-	return table_names;
+	auto root_binder = GetRootBinder();
+	return root_binder->table_names;
 }
 
 string Binder::FormatError(ParsedExpression &expr_context, const string &message) {

package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp

@@ -15,7 +15,7 @@ BoundAggregateExpression::BoundAggregateExpression(AggregateFunction function, v
     : Expression(ExpressionType::BOUND_AGGREGATE, ExpressionClass::BOUND_AGGREGATE, function.return_type),
       function(std::move(function)), children(std::move(children)), bind_info(std::move(bind_info)),
       aggr_type(aggr_type), filter(std::move(filter)) {
-	D_ASSERT(!function.name.empty());
+	D_ASSERT(!this->function.name.empty());
 }
 
 string BoundAggregateExpression::ToString() const {
@@ -104,4 +104,27 @@ unique_ptr<Expression> BoundAggregateExpression::Deserialize(ExpressionDeseriali
 	return std::move(x);
 }
 
+void BoundAggregateExpression::FormatSerialize(FormatSerializer &serializer) const {
+	Expression::FormatSerialize(serializer);
+	serializer.WriteProperty(200, "return_type", return_type);
+	serializer.WriteProperty(201, "children", children);
+	FunctionSerializer::FormatSerialize(serializer, function, bind_info.get());
+	serializer.WriteProperty(203, "aggregate_type", aggr_type);
+	serializer.WriteOptionalProperty(204, "filter", filter);
+	serializer.WriteOptionalProperty(205, "order_bys", order_bys);
+}
+
+unique_ptr<Expression> BoundAggregateExpression::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto return_type = deserializer.ReadProperty<LogicalType>(200, "return_type");
+	auto children = deserializer.ReadProperty<vector<unique_ptr<Expression>>>(201, "children");
+	auto entry = FunctionSerializer::FormatDeserialize<AggregateFunction, AggregateFunctionCatalogEntry>(
+	    deserializer, CatalogType::AGGREGATE_FUNCTION_ENTRY, children);
+	auto aggregate_type = deserializer.ReadProperty<AggregateType>(203, "aggregate_type");
+	auto filter = deserializer.ReadOptionalProperty<unique_ptr<Expression>>(204, "filter");
+	auto result = make_uniq<BoundAggregateExpression>(std::move(entry.first), std::move(children), std::move(filter),
+	                                                  std::move(entry.second), aggregate_type);
+	deserializer.ReadOptionalProperty(205, "order_bys", result->order_bys);
+	return std::move(result);
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/expression/bound_between_expression.cpp

@@ -4,6 +4,10 @@
 
 namespace duckdb {
 
+BoundBetweenExpression::BoundBetweenExpression()
+    : Expression(ExpressionType::COMPARE_BETWEEN, ExpressionClass::BOUND_BETWEEN, LogicalType::BOOLEAN) {
+}
+
 BoundBetweenExpression::BoundBetweenExpression(unique_ptr<Expression> input, unique_ptr<Expression> lower,
                                                unique_ptr<Expression> upper, bool lower_inclusive, bool upper_inclusive)
     : Expression(ExpressionType::COMPARE_BETWEEN, ExpressionClass::BOUND_BETWEEN, LogicalType::BOOLEAN),