duckdb 0.8.2-dev150.0 → 0.8.2-dev1549.0

package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp

@@ -18,6 +18,7 @@
 #include "duckdb/planner/tableref/bound_pivotref.hpp"
 #include "duckdb/planner/expression/bound_aggregate_expression.hpp"
 #include "duckdb/main/client_config.hpp"
+#include "duckdb/catalog/catalog_entry/aggregate_function_catalog_entry.hpp"
 
 namespace duckdb {
 
@@ -66,18 +67,10 @@ static void ExtractPivotExpressions(ParsedExpression &expr, case_insensitive_set
 	    expr, [&](ParsedExpression &child) { ExtractPivotExpressions(child, handled_columns); });
 }
 
-struct PivotBindState {
-	vector<string> internal_group_names;
-	vector<string> group_names;
-	vector<string> aggregate_names;
-	vector<string> internal_aggregate_names;
-};
-
-static unique_ptr<SelectNode> PivotInitialAggregate(PivotBindState &bind_state, PivotRef &ref,
-                                                    vector<unique_ptr<ParsedExpression>> all_columns,
-                                                    const case_insensitive_set_t &handled_columns) {
-	auto subquery_stage1 = make_uniq<SelectNode>();
-	subquery_stage1->from_table = std::move(ref.source);
+static unique_ptr<SelectNode> ConstructInitialGrouping(PivotRef &ref, vector<unique_ptr<ParsedExpression>> all_columns,
+                                                       const case_insensitive_set_t &handled_columns) {
+	auto subquery = make_uniq<SelectNode>();
+	subquery->from_table = std::move(ref.source);
 	if (ref.groups.empty()) {
 		// if rows are not specified any columns that are not pivoted/aggregated on are added to the GROUP BY clause
 		for (auto &entry : all_columns) {
@@ -87,19 +80,74 @@ static unique_ptr<SelectNode> PivotInitialAggregate(PivotBindState &bind_state,
 			auto &columnref = entry->Cast<ColumnRefExpression>();
 			if (handled_columns.find(columnref.GetColumnName()) == handled_columns.end()) {
 				// not handled - add to grouping set
-				subquery_stage1->groups.group_expressions.push_back(
-				    make_uniq<ConstantExpression>(Value::INTEGER(subquery_stage1->select_list.size() + 1)));
-				subquery_stage1->select_list.push_back(make_uniq<ColumnRefExpression>(columnref.GetColumnName()));
+				subquery->groups.group_expressions.push_back(
+				    make_uniq<ConstantExpression>(Value::INTEGER(subquery->select_list.size() + 1)));
+				subquery->select_list.push_back(make_uniq<ColumnRefExpression>(columnref.GetColumnName()));
 			}
 		}
 	} else {
 		// if rows are specified only the columns mentioned in rows are added as groups
 		for (auto &row : ref.groups) {
-			subquery_stage1->groups.group_expressions.push_back(
-			    make_uniq<ConstantExpression>(Value::INTEGER(subquery_stage1->select_list.size() + 1)));
-			subquery_stage1->select_list.push_back(make_uniq<ColumnRefExpression>(row));
+			subquery->groups.group_expressions.push_back(
+			    make_uniq<ConstantExpression>(Value::INTEGER(subquery->select_list.size() + 1)));
+			subquery->select_list.push_back(make_uniq<ColumnRefExpression>(row));
 		}
 	}
+	return subquery;
+}
+
+static unique_ptr<SelectNode> PivotFilteredAggregate(PivotRef &ref, vector<unique_ptr<ParsedExpression>> all_columns,
+                                                     const case_insensitive_set_t &handled_columns,
+                                                     vector<PivotValueElement> pivot_values) {
+	auto subquery = ConstructInitialGrouping(ref, std::move(all_columns), handled_columns);
+
+	// push the filtered aggregates
+	for (auto &pivot_value : pivot_values) {
+		unique_ptr<ParsedExpression> filter;
+		idx_t pivot_value_idx = 0;
+		for (auto &pivot_column : ref.pivots) {
+			for (auto &pivot_expr : pivot_column.pivot_expressions) {
+				auto column_ref = make_uniq<CastExpression>(LogicalType::VARCHAR, pivot_expr->Copy());
+				auto constant_value = make_uniq<ConstantExpression>(pivot_value.values[pivot_value_idx++]);
+				auto comp_expr = make_uniq<ComparisonExpression>(ExpressionType::COMPARE_NOT_DISTINCT_FROM,
+				                                                 std::move(column_ref), std::move(constant_value));
+				if (filter) {
+					filter = make_uniq<ConjunctionExpression>(ExpressionType::CONJUNCTION_AND, std::move(filter),
+					                                          std::move(comp_expr));
+				} else {
+					filter = std::move(comp_expr);
+				}
+			}
+		}
+		for (auto &aggregate : ref.aggregates) {
+			auto copied_aggr = aggregate->Copy();
+			auto &aggr = copied_aggr->Cast<FunctionExpression>();
+			aggr.filter = filter->Copy();
+			auto &aggr_name = aggregate->alias;
+			auto name = pivot_value.name;
+			if (ref.aggregates.size() > 1 || !aggr_name.empty()) {
+				// if there are multiple aggregates specified we add the name of the aggregate as well
+				name += "_" + (aggr_name.empty() ? aggregate->GetName() : aggr_name);
+			}
+			aggr.alias = name;
+			subquery->select_list.push_back(std::move(copied_aggr));
+		}
+	}
+	return subquery;
+}
+
+struct PivotBindState {
+	vector<string> internal_group_names;
+	vector<string> group_names;
+	vector<string> aggregate_names;
+	vector<string> internal_aggregate_names;
+};
+
+static unique_ptr<SelectNode> PivotInitialAggregate(PivotBindState &bind_state, PivotRef &ref,
+                                                    vector<unique_ptr<ParsedExpression>> all_columns,
+                                                    const case_insensitive_set_t &handled_columns) {
+	auto subquery_stage1 = ConstructInitialGrouping(ref, std::move(all_columns), handled_columns);
+
 	idx_t group_count = 0;
 	for (auto &expr : subquery_stage1->select_list) {
 		bind_state.group_names.push_back(expr->GetName());
@@ -134,6 +182,15 @@ static unique_ptr<SelectNode> PivotInitialAggregate(PivotBindState &bind_state,
 	return subquery_stage1;
 }
 
+unique_ptr<ParsedExpression> ConstructPivotExpression(unique_ptr<ParsedExpression> pivot_expr) {
+	auto cast = make_uniq<CastExpression>(LogicalType::VARCHAR, std::move(pivot_expr));
+	vector<unique_ptr<ParsedExpression>> coalesce_children;
+	coalesce_children.push_back(std::move(cast));
+	coalesce_children.push_back(make_uniq<ConstantExpression>(Value("NULL")));
+	auto coalesce = make_uniq<OperatorExpression>(ExpressionType::OPERATOR_COALESCE, std::move(coalesce_children));
+	return std::move(coalesce);
+}
+
 static unique_ptr<SelectNode> PivotListAggregate(PivotBindState &bind_state, PivotRef &ref,
                                                  unique_ptr<SelectNode> subquery_stage1) {
 	auto subquery_stage2 = make_uniq<SelectNode>();
@@ -166,13 +223,7 @@ static unique_ptr<SelectNode> PivotListAggregate(PivotBindState &bind_state, Piv
 	for (auto &pivot : ref.pivots) {
 		for (auto &pivot_expr : pivot.pivot_expressions) {
 			// coalesce(pivot::VARCHAR, 'NULL')
-			auto cast = make_uniq<CastExpression>(LogicalType::VARCHAR, std::move(pivot_expr));
-			vector<unique_ptr<ParsedExpression>> coalesce_children;
-			coalesce_children.push_back(std::move(cast));
-			coalesce_children.push_back(make_uniq<ConstantExpression>(Value("NULL")));
-			auto coalesce =
-			    make_uniq<OperatorExpression>(ExpressionType::OPERATOR_COALESCE, std::move(coalesce_children));
-
+			auto coalesce = ConstructPivotExpression(std::move(pivot_expr));
 			if (!expr) {
 				expr = std::move(coalesce);
 			} else {
@@ -250,9 +301,8 @@ unique_ptr<BoundTableRef> Binder::BindBoundPivot(PivotRef &ref) {
 	auto &aggregates = result->bound_pivot.aggregates;
 	ExtractPivotAggregates(*result->child, aggregates);
 	if (aggregates.size() != ref.bound_aggregate_names.size()) {
-		throw BinderException("Pivot aggregate count mismatch. Expected %llu aggregates but found %llu. Are all pivot "
-		                      "expressions aggregate functions?",
-		                      ref.bound_aggregate_names.size(), aggregates.size());
+		throw InternalException("Pivot aggregate count mismatch (expected %llu, found %llu)",
+		                        ref.bound_aggregate_names.size(), aggregates.size());
 	}
 
 	vector<string> child_names;
@@ -313,6 +363,10 @@ unique_ptr<SelectNode> Binder::BindPivot(PivotRef &ref, vector<unique_ptr<Parsed
 		if (aggr->IsWindow()) {
 			throw BinderException(FormatError(*aggr, "Pivot expression cannot contain window functions"));
 		}
+		// bind the function as an aggregate to ensure it is an aggregate and not a scalar function
+		auto &aggr_function = aggr->Cast<FunctionExpression>();
+		(void)Catalog::GetEntry<AggregateFunctionCatalogEntry>(context, aggr_function.catalog, aggr_function.schema,
+		                                                       aggr_function.function_name);
 		ExtractPivotExpressions(*aggr, handled_columns);
 	}
 
@@ -361,40 +415,53 @@ unique_ptr<SelectNode> Binder::BindPivot(PivotRef &ref, vector<unique_ptr<Parsed
 			pivots.insert(val);
 		}
 	}
-	auto pivot_limit = ClientConfig::GetConfig(context).pivot_limit;
+	auto &client_config = ClientConfig::GetConfig(context);
+	auto pivot_limit = client_config.pivot_limit;
 	if (total_pivots >= pivot_limit) {
 		throw BinderException("Pivot column limit of %llu exceeded. Use SET pivot_limit=X to increase the limit.",
-		                      ClientConfig::GetConfig(context).pivot_limit);
+		                      client_config.pivot_limit);
 	}
 
 	// construct the required pivot values recursively
 	vector<PivotValueElement> pivot_values;
 	ConstructPivots(ref, pivot_values);
 
+	unique_ptr<SelectNode> pivot_node;
 	// pivots have three components
 	// - the pivots (i.e. future column names)
 	// - the groups (i.e. the future row names
 	// - the aggregates (i.e. the values of the pivot columns)
 
-	// executing a pivot statement happens in three stages
-	// 1) execute the query "SELECT {groups}, {pivots}, {aggregates} FROM {from_clause} GROUP BY {groups}, {pivots}
-	// this computes all values that are required in the final result, but not yet in the correct orientation
-	// 2) execute the query "SELECT {groups}, LIST({pivots}), LIST({aggregates}) FROM [Q1] GROUP BY {groups}
-	// this pushes all pivots and aggregates that belong to a specific group together in an aligned manner
-	// 3) push a PIVOT operator, that performs the actual pivoting of the values into the different columns
-
-	PivotBindState bind_state;
-	// Pivot Stage 1
-	// SELECT {groups}, {pivots}, {aggregates} FROM {from_clause} GROUP BY {groups}, {pivots}
-	auto subquery_stage1 = PivotInitialAggregate(bind_state, ref, std::move(all_columns), handled_columns);
-
-	// Pivot stage 2
-	// SELECT {groups}, LIST({pivots}), LIST({aggregates}) FROM [Q1] GROUP BY {groups}
-	auto subquery_stage2 = PivotListAggregate(bind_state, ref, std::move(subquery_stage1));
-
-	// Pivot stage 3
-	// construct the final pivot operator
-	auto pivot_node = PivotFinalOperator(bind_state, ref, std::move(subquery_stage2), std::move(pivot_values));
+	// we have two ways of executing a pivot statement
+	// (1) the straightforward manner of filtered aggregates SUM(..) FILTER (pivot_value=X)
+	// (2) computing the aggregates once, then using LIST to group the aggregates together with the PIVOT operator
+	// -> filtered aggregates are faster when there are FEW pivot values
+	// -> LIST is faster when there are MANY pivot values
+	// we switch dynamically based on the number of pivots to compute
+	if (pivot_values.size() <= client_config.pivot_filter_threshold) {
+		// use a set of filtered aggregates
+		pivot_node = PivotFilteredAggregate(ref, std::move(all_columns), handled_columns, std::move(pivot_values));
+	} else {
+		// executing a pivot statement happens in three stages
+		// 1) execute the query "SELECT {groups}, {pivots}, {aggregates} FROM {from_clause} GROUP BY {groups}, {pivots}
+		// this computes all values that are required in the final result, but not yet in the correct orientation
+		// 2) execute the query "SELECT {groups}, LIST({pivots}), LIST({aggregates}) FROM [Q1] GROUP BY {groups}
+		// this pushes all pivots and aggregates that belong to a specific group together in an aligned manner
+		// 3) push a PIVOT operator, that performs the actual pivoting of the values into the different columns
+
+		PivotBindState bind_state;
+		// Pivot Stage 1
+		// SELECT {groups}, {pivots}, {aggregates} FROM {from_clause} GROUP BY {groups}, {pivots}
+		auto subquery_stage1 = PivotInitialAggregate(bind_state, ref, std::move(all_columns), handled_columns);
+
+		// Pivot stage 2
+		// SELECT {groups}, LIST({pivots}), LIST({aggregates}) FROM [Q1] GROUP BY {groups}
+		auto subquery_stage2 = PivotListAggregate(bind_state, ref, std::move(subquery_stage1));
+
+		// Pivot stage 3
+		// construct the final pivot operator
+		pivot_node = PivotFinalOperator(bind_state, ref, std::move(subquery_stage2), std::move(pivot_values));
+	}
 	return pivot_node;
 }
 

package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp

@@ -13,7 +13,7 @@ unique_ptr<LogicalOperator> Binder::CreatePlan(BoundCTERef &ref) {
 		types.push_back(type);
 	}
 
-	return make_uniq<LogicalCTERef>(index, ref.cte_index, types, ref.bound_columns);
+	return make_uniq<LogicalCTERef>(index, ref.cte_index, types, ref.bound_columns, ref.materialized_cte);
 }
 
 } // namespace duckdb

package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp

@@ -10,11 +10,14 @@
 #include "duckdb/planner/operator/logical_asof_join.hpp"
 #include "duckdb/planner/operator/logical_comparison_join.hpp"
 #include "duckdb/planner/operator/logical_cross_product.hpp"
+#include "duckdb/planner/operator/logical_dependent_join.hpp"
 #include "duckdb/planner/operator/logical_filter.hpp"
 #include "duckdb/planner/operator/logical_positional_join.hpp"
 #include "duckdb/planner/tableref/bound_joinref.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/planner/expression_binder/lateral_binder.hpp"
+#include "duckdb/planner/subquery/recursive_dependent_join_planner.hpp"
+#include "duckdb/execution/expression_executor.hpp"
 
 namespace duckdb {
 
@@ -44,13 +47,11 @@ static bool CreateJoinCondition(Expression &expr, const unordered_set<idx_t> &le
 	return false;
 }
 
-void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<LogicalOperator> &left_child,
-                                                  unique_ptr<LogicalOperator> &right_child,
-                                                  const unordered_set<idx_t> &left_bindings,
-                                                  const unordered_set<idx_t> &right_bindings,
-                                                  vector<unique_ptr<Expression>> &expressions,
-                                                  vector<JoinCondition> &conditions,
-                                                  vector<unique_ptr<Expression>> &arbitrary_expressions) {
+void LogicalComparisonJoin::ExtractJoinConditions(
+    ClientContext &context, JoinType type, unique_ptr<LogicalOperator> &left_child,
+    unique_ptr<LogicalOperator> &right_child, const unordered_set<idx_t> &left_bindings,
+    const unordered_set<idx_t> &right_bindings, vector<unique_ptr<Expression>> &expressions,
+    vector<JoinCondition> &conditions, vector<unique_ptr<Expression>> &arbitrary_expressions) {
 	for (auto &expr : expressions) {
 		auto total_side = JoinSide::GetJoinSide(*expr, left_bindings, right_bindings);
 		if (total_side != JoinSide::BOTH) {
@@ -68,6 +69,15 @@ void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<Logi
 				filter.expressions.push_back(std::move(expr));
 				continue;
 			}
+			// if the join is a LEFT JOIN and the join expression constantly evaluates to TRUE,
+			// then we do not add it to the arbitrary expressions
+			if (type == JoinType::LEFT && expr->IsFoldable()) {
+				Value result;
+				ExpressionExecutor::TryEvaluateScalar(context, *expr, result);
+				if (!result.IsNull() && result == Value(true)) {
+					continue;
+				}
+			}
 		} else if ((expr->type >= ExpressionType::COMPARE_EQUAL &&
 		            expr->type <= ExpressionType::COMPARE_GREATERTHANOREQUALTO) ||
 		           expr->type == ExpressionType::COMPARE_DISTINCT_FROM ||
@@ -82,7 +92,8 @@ void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<Logi
 	}
 }
 
-void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<LogicalOperator> &left_child,
+void LogicalComparisonJoin::ExtractJoinConditions(ClientContext &context, JoinType type,
+                                                  unique_ptr<LogicalOperator> &left_child,
                                                   unique_ptr<LogicalOperator> &right_child,
                                                   vector<unique_ptr<Expression>> &expressions,
                                                   vector<JoinCondition> &conditions,
@@ -90,11 +101,12 @@ void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<Logi
 	unordered_set<idx_t> left_bindings, right_bindings;
 	LogicalJoin::GetTableReferences(*left_child, left_bindings);
 	LogicalJoin::GetTableReferences(*right_child, right_bindings);
-	return ExtractJoinConditions(type, left_child, right_child, left_bindings, right_bindings, expressions, conditions,
-	                             arbitrary_expressions);
+	return ExtractJoinConditions(context, type, left_child, right_child, left_bindings, right_bindings, expressions,
+	                             conditions, arbitrary_expressions);
 }
 
-void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<LogicalOperator> &left_child,
+void LogicalComparisonJoin::ExtractJoinConditions(ClientContext &context, JoinType type,
+                                                  unique_ptr<LogicalOperator> &left_child,
                                                   unique_ptr<LogicalOperator> &right_child,
                                                   unique_ptr<Expression> condition, vector<JoinCondition> &conditions,
                                                   vector<unique_ptr<Expression>> &arbitrary_expressions) {
@@ -102,10 +114,12 @@ void LogicalComparisonJoin::ExtractJoinConditions(JoinType type, unique_ptr<Logi
 	vector<unique_ptr<Expression>> expressions;
 	expressions.push_back(std::move(condition));
 	LogicalFilter::SplitPredicates(expressions);
-	return ExtractJoinConditions(type, left_child, right_child, expressions, conditions, arbitrary_expressions);
+	return ExtractJoinConditions(context, type, left_child, right_child, expressions, conditions,
+	                             arbitrary_expressions);
 }
 
-unique_ptr<LogicalOperator> LogicalComparisonJoin::CreateJoin(JoinType type, JoinRefType reftype,
+unique_ptr<LogicalOperator> LogicalComparisonJoin::CreateJoin(ClientContext &context, JoinType type,
+                                                              JoinRefType reftype,
                                                               unique_ptr<LogicalOperator> left_child,
                                                               unique_ptr<LogicalOperator> right_child,
                                                               vector<JoinCondition> conditions,
@@ -216,29 +230,37 @@ static bool HasCorrelatedColumns(Expression &expression) {
 	return has_correlated_columns;
 }
 
-unique_ptr<LogicalOperator> LogicalComparisonJoin::CreateJoin(JoinType type, JoinRefType reftype,
+unique_ptr<LogicalOperator> LogicalComparisonJoin::CreateJoin(ClientContext &context, JoinType type,
+                                                              JoinRefType reftype,
                                                               unique_ptr<LogicalOperator> left_child,
                                                               unique_ptr<LogicalOperator> right_child,
                                                               unique_ptr<Expression> condition) {
 	vector<JoinCondition> conditions;
 	vector<unique_ptr<Expression>> arbitrary_expressions;
-	LogicalComparisonJoin::ExtractJoinConditions(type, left_child, right_child, std::move(condition), conditions,
-	                                             arbitrary_expressions);
-	return LogicalComparisonJoin::CreateJoin(type, reftype, std::move(left_child), std::move(right_child),
+	LogicalComparisonJoin::ExtractJoinConditions(context, type, left_child, right_child, std::move(condition),
+	                                             conditions, arbitrary_expressions);
+	return LogicalComparisonJoin::CreateJoin(context, type, reftype, std::move(left_child), std::move(right_child),
 	                                         std::move(conditions), std::move(arbitrary_expressions));
 }
 
 unique_ptr<LogicalOperator> Binder::CreatePlan(BoundJoinRef &ref) {
+	auto old_is_outside_flattened = is_outside_flattened;
+	// Plan laterals from outermost to innermost
+	if (ref.lateral) {
+		// Set the flag to ensure that children do not flatten before the root
+		is_outside_flattened = false;
+	}
 	auto left = CreatePlan(*ref.left);
 	auto right = CreatePlan(*ref.right);
+	is_outside_flattened = old_is_outside_flattened;
+
+	// For joins, depth of the bindings will be one higher on the right because of the lateral binder
+	// If the current join does not have correlations between left and right, then the right bindings
+	// have depth 1 too high and can be reduced by 1 throughout
 	if (!ref.lateral && !ref.correlated_columns.empty()) {
-		// non-lateral join with correlated columns
-		// this happens if there is a join (or cross product) in a correlated subquery
-		// due to the lateral binder the expression depth of all correlated columns in the "ref.correlated_columns" set
-		// is 1 too high
-		// we reduce expression depth of all columns in the "ref.correlated_columns" set by 1
 		LateralBinder::ReduceExpressionDepth(*right, ref.correlated_columns);
 	}
+
 	if (ref.type == JoinType::RIGHT && ref.ref_type != JoinRefType::ASOF &&
 	    ClientConfig::GetConfig(context).enable_optimizer) {
 		// we turn any right outer joins into left outer joins for optimization purposes
@@ -247,9 +269,22 @@ unique_ptr<LogicalOperator> Binder::CreatePlan(BoundJoinRef &ref) {
 		std::swap(left, right);
 	}
 	if (ref.lateral) {
-		// lateral join
-		return PlanLateralJoin(std::move(left), std::move(right), ref.correlated_columns, ref.type,
-		                       std::move(ref.condition));
+		if (!is_outside_flattened) {
+			// If outer dependent joins is yet to be flattened, only plan the lateral
+			has_unplanned_dependent_joins = true;
+			return LogicalDependentJoin::Create(std::move(left), std::move(right), ref.correlated_columns, ref.type,
+			                                    std::move(ref.condition));
+		} else {
+			// All outer dependent joins have been planned and flattened, so plan and flatten lateral and recursively
+			// plan the children
+			auto new_plan = PlanLateralJoin(std::move(left), std::move(right), ref.correlated_columns, ref.type,
+			                                std::move(ref.condition));
+			if (has_unplanned_dependent_joins) {
+				RecursiveDependentJoinPlanner plan(*this);
+				plan.VisitOperator(*new_plan);
+			}
+			return new_plan;
+		}
 	}
 	switch (ref.ref_type) {
 	case JoinRefType::CROSS:
@@ -275,7 +310,7 @@ unique_ptr<LogicalOperator> Binder::CreatePlan(BoundJoinRef &ref) {
 	}
 
 	// now create the join operator from the join condition
-	auto result = LogicalComparisonJoin::CreateJoin(ref.type, ref.ref_type, std::move(left), std::move(right),
+	auto result = LogicalComparisonJoin::CreateJoin(context, ref.type, ref.ref_type, std::move(left), std::move(right),
 	                                                std::move(ref.condition));
 
 	optional_ptr<LogicalOperator> join;

package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp

@@ -6,10 +6,10 @@ namespace duckdb {
 unique_ptr<LogicalOperator> Binder::CreatePlan(BoundSubqueryRef &ref) {
 	// generate the logical plan for the subquery
 	// this happens separately from the current LogicalPlan generation
-	ref.binder->plan_subquery = plan_subquery;
+	ref.binder->is_outside_flattened = is_outside_flattened;
 	auto subquery = ref.binder->CreatePlan(*ref.subquery);
-	if (ref.binder->has_unplanned_subqueries) {
-		has_unplanned_subqueries = true;
+	if (ref.binder->has_unplanned_dependent_joins) {
+		has_unplanned_dependent_joins = true;
 	}
 	return subquery;
 }

package/src/duckdb/src/planner/binder.cpp

@@ -117,6 +117,9 @@ unique_ptr<BoundQueryNode> Binder::BindNode(QueryNode &node) {
 	case QueryNodeType::RECURSIVE_CTE_NODE:
 		result = BindNode(node.Cast<RecursiveCTENode>());
 		break;
+	case QueryNodeType::CTE_NODE:
+		result = BindNode(node.Cast<CTENode>());
+		break;
 	default:
 		D_ASSERT(node.type == QueryNodeType::SET_OPERATION_NODE);
 		result = BindNode(node.Cast<SetOperationNode>());
@@ -145,6 +148,8 @@ unique_ptr<LogicalOperator> Binder::CreatePlan(BoundQueryNode &node) {
 		return CreatePlan(node.Cast<BoundSetOperationNode>());
 	case QueryNodeType::RECURSIVE_CTE_NODE:
 		return CreatePlan(node.Cast<BoundRecursiveCTENode>());
+	case QueryNodeType::CTE_NODE:
+		return CreatePlan(node.Cast<BoundCTENode>());
 	default:
 		throw InternalException("Unsupported bound query node type");
 	}

package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp

@@ -15,7 +15,7 @@ BoundAggregateExpression::BoundAggregateExpression(AggregateFunction function, v
     : Expression(ExpressionType::BOUND_AGGREGATE, ExpressionClass::BOUND_AGGREGATE, function.return_type),
       function(std::move(function)), children(std::move(children)), bind_info(std::move(bind_info)),
       aggr_type(aggr_type), filter(std::move(filter)) {
-	D_ASSERT(!function.name.empty());
+	D_ASSERT(!this->function.name.empty());
 }
 
 string BoundAggregateExpression::ToString() const {

package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp

@@ -31,41 +31,10 @@ BindResult LateralBinder::BindColumnRef(unique_ptr<ParsedExpression> &expr_ptr,
 	if (result.HasError()) {
 		return result;
 	}
-	if (depth > 1) {
-		throw BinderException("Nested lateral joins are not supported yet");
-	}
 	ExtractCorrelatedColumns(*result.expression);
 	return result;
 }
 
-vector<CorrelatedColumnInfo> LateralBinder::ExtractCorrelatedColumns(Binder &binder) {
-
-	if (correlated_columns.empty()) {
-		return binder.correlated_columns;
-	}
-
-	// clear outer
-	correlated_columns.clear();
-	auto all_correlated_columns = binder.correlated_columns;
-
-	// remove outer from inner
-	for (auto &corr_column : correlated_columns) {
-		auto entry = std::find(binder.correlated_columns.begin(), binder.correlated_columns.end(), corr_column);
-		if (entry != binder.correlated_columns.end()) {
-			binder.correlated_columns.erase(entry);
-		}
-	}
-
-	// add inner to outer
-	for (auto &corr_column : binder.correlated_columns) {
-		correlated_columns.push_back(corr_column);
-	}
-
-	// clear inner
-	binder.correlated_columns.clear();
-	return all_correlated_columns;
-}
-
 BindResult LateralBinder::BindExpression(unique_ptr<ParsedExpression> &expr_ptr, idx_t depth, bool root_expression) {
 	auto &expr = *expr_ptr;
 	switch (expr.GetExpressionClass()) {
@@ -125,7 +94,11 @@ protected:
 			ReduceColumnRefDepth(expr.Cast<BoundColumnRefExpression>());
 		}
 		if (expr.GetExpressionClass() == ExpressionClass::BOUND_SUBQUERY) {
+			auto &subquery_ref = expr.Cast<BoundSubqueryExpression>();
 			ReduceExpressionSubquery(expr.Cast<BoundSubqueryExpression>());
+			// Recursively update the depth in the bindings of the children nodes
+			ExpressionIterator::EnumerateQueryNodeChildren(
+			    *subquery_ref.subquery, [&](Expression &child_expr) { ReduceExpressionDepth(child_expr); });
 		}
 	}
 

package/src/duckdb/src/planner/expression_binder.cpp

@@ -79,9 +79,12 @@ bool ExpressionBinder::BindCorrelatedColumns(unique_ptr<ParsedExpression> &expr)
 	auto &active_binders = binder.GetActiveBinders();
 	// make a copy of the set of binders, so we can restore it later
 	auto binders = active_binders;
+
+	// we already failed with the current binder
 	active_binders.pop_back();
 	idx_t depth = 1;
 	bool success = false;
+
 	while (!active_binders.empty()) {
 		auto &next_binder = active_binders.back().get();
 		ExpressionBinder::QualifyColumnNames(next_binder.binder, expr);

package/src/duckdb/src/planner/expression_iterator.cpp

@@ -5,6 +5,7 @@
 #include "duckdb/planner/query_node/bound_select_node.hpp"
 #include "duckdb/planner/query_node/bound_set_operation_node.hpp"
 #include "duckdb/planner/query_node/bound_recursive_cte_node.hpp"
+#include "duckdb/planner/query_node/bound_cte_node.hpp"
 #include "duckdb/planner/tableref/list.hpp"
 
 namespace duckdb {
@@ -198,6 +199,11 @@ void ExpressionIterator::EnumerateQueryNodeChildren(BoundQueryNode &node,
 		EnumerateQueryNodeChildren(*cte_node.right, callback);
 		break;
 	}
+	case QueryNodeType::CTE_NODE: {
+		auto &cte_node = node.Cast<BoundCTENode>();
+		EnumerateQueryNodeChildren(*cte_node.child, callback);
+		break;
+	}
 	case QueryNodeType::SELECT_NODE: {
 		auto &bound_select = node.Cast<BoundSelectNode>();
 		for (auto &expr : bound_select.select_list) {

package/src/duckdb/src/planner/logical_operator.cpp

@@ -8,6 +8,7 @@
 #include "duckdb/parser/parser.hpp"
 #include "duckdb/planner/operator/list.hpp"
 #include "duckdb/planner/operator/logical_extension_operator.hpp"
+#include "duckdb/planner/operator/logical_dependent_join.hpp"
 
 namespace duckdb {
 
@@ -283,6 +284,9 @@ unique_ptr<LogicalOperator> LogicalOperator::Deserialize(Deserializer &deseriali
 	case LogicalOperatorType::LOGICAL_RECURSIVE_CTE:
 		result = LogicalRecursiveCTE::Deserialize(state, reader);
 		break;
+	case LogicalOperatorType::LOGICAL_MATERIALIZED_CTE:
+		result = LogicalMaterializedCTE::Deserialize(state, reader);
+		break;
 	case LogicalOperatorType::LOGICAL_INSERT:
 		result = LogicalInsert::Deserialize(state, reader);
 		break;
@@ -352,6 +356,7 @@ unique_ptr<LogicalOperator> LogicalOperator::Deserialize(Deserializer &deseriali
 	case LogicalOperatorType::LOGICAL_PIVOT:
 		result = LogicalPivot::Deserialize(state, reader);
 		break;
+	case LogicalOperatorType::LOGICAL_DEPENDENT_JOIN:
 	case LogicalOperatorType::LOGICAL_INVALID:
 		/* no default here to trigger a warning if we forget to implement deserialize for a new operator */
 		throw SerializationException("Invalid type for operator deserialization");

package/src/duckdb/src/planner/logical_operator_visitor.cpp

@@ -19,6 +19,7 @@ void LogicalOperatorVisitor::VisitOperatorChildren(LogicalOperator &op) {
 
 void LogicalOperatorVisitor::EnumerateExpressions(LogicalOperator &op,
                                                   const std::function<void(unique_ptr<Expression> *child)> &callback) {
+
 	switch (op.type) {
 	case LogicalOperatorType::LOGICAL_EXPRESSION_GET: {
 		auto &get = op.Cast<LogicalExpressionGet>();
@@ -67,6 +68,7 @@ void LogicalOperatorVisitor::EnumerateExpressions(LogicalOperator &op,
 	}
 	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
 	case LogicalOperatorType::LOGICAL_DELIM_JOIN:
+	case LogicalOperatorType::LOGICAL_DEPENDENT_JOIN:
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: {
 		if (op.type == LogicalOperatorType::LOGICAL_DELIM_JOIN) {
 			auto &delim_join = op.Cast<LogicalDelimJoin>();

package/src/duckdb/src/planner/operator/logical_cteref.cpp

@@ -10,6 +10,7 @@ void LogicalCTERef::Serialize(FieldWriter &writer) const {
 	writer.WriteField(cte_index);
 	writer.WriteRegularSerializableList(chunk_types);
 	writer.WriteList<string>(bound_columns);
+	writer.WriteField(materialized_cte);
 }
 
 unique_ptr<LogicalOperator> LogicalCTERef::Deserialize(LogicalDeserializationState &state, FieldReader &reader) {
@@ -17,7 +18,8 @@ unique_ptr<LogicalOperator> LogicalCTERef::Deserialize(LogicalDeserializationSta
 	auto cte_index = reader.ReadRequired<idx_t>();
 	auto chunk_types = reader.ReadRequiredSerializableList<LogicalType, LogicalType>();
 	auto bound_columns = reader.ReadRequiredList<string>();
-	return make_uniq<LogicalCTERef>(table_index, cte_index, chunk_types, bound_columns);
+	auto materialized_cte = reader.ReadField<CTEMaterialize>(CTEMaterialize::CTE_MATERIALIZE_DEFAULT);
+	return make_uniq<LogicalCTERef>(table_index, cte_index, chunk_types, bound_columns, materialized_cte);
 }
 
 vector<idx_t> LogicalCTERef::GetTableIndex() const {

package/src/duckdb/src/planner/operator/logical_dependent_join.cpp

@@ -0,0 +1,26 @@
+#include "duckdb/planner/operator/logical_dependent_join.hpp"
+
+namespace duckdb {
+
+LogicalDependentJoin::LogicalDependentJoin(unique_ptr<LogicalOperator> left, unique_ptr<LogicalOperator> right,
+                                           vector<CorrelatedColumnInfo> correlated_columns, JoinType type,
+                                           unique_ptr<Expression> condition)
+    : LogicalComparisonJoin(type, LogicalOperatorType::LOGICAL_DEPENDENT_JOIN), join_condition(std::move(condition)),
+      correlated_columns(std::move(correlated_columns)) {
+	children.push_back(std::move(left));
+	children.push_back(std::move(right));
+}
+
+unique_ptr<LogicalOperator> LogicalDependentJoin::Create(unique_ptr<LogicalOperator> left,
+                                                         unique_ptr<LogicalOperator> right,
+                                                         vector<CorrelatedColumnInfo> correlated_columns, JoinType type,
+                                                         unique_ptr<Expression> condition) {
+	return make_uniq<LogicalDependentJoin>(std::move(left), std::move(right), std::move(correlated_columns), type,
+	                                       std::move(condition));
+}
+
+void LogicalDependentJoin::Serialize(FieldWriter &writer) const {
+	throw InternalException("LogicalDependentJoin should not exist in any plan");
+}
+
+} // namespace duckdb