duckdb 0.8.2-dev157.0 → 0.8.2-dev1573.0

package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp

@@ -17,6 +17,9 @@ class SubqueryRef : public TableRef {
 public:
 	static constexpr const TableReferenceType TYPE = TableReferenceType::SUBQUERY;
 
+private:
+	SubqueryRef();
+
 public:
 	DUCKDB_API explicit SubqueryRef(unique_ptr<SelectStatement> subquery, string alias = string());
 

package/src/duckdb/src/include/duckdb/parser/tokens.hpp

@@ -50,6 +50,7 @@ class QueryNode;
 class SelectNode;
 class SetOperationNode;
 class RecursiveCTENode;
+class CTENode;
 
 //===--------------------------------------------------------------------===//
 // Expressions

package/src/duckdb/src/include/duckdb/parser/transformer.hpp

@@ -8,20 +8,20 @@
 
 #pragma once
 
+#include "duckdb/common/case_insensitive_map.hpp"
 #include "duckdb/common/constants.hpp"
 #include "duckdb/common/enums/expression_type.hpp"
 #include "duckdb/common/types.hpp"
 #include "duckdb/common/unordered_map.hpp"
-#include "duckdb/parser/qualified_name.hpp"
-#include "duckdb/parser/tokens.hpp"
-#include "duckdb/parser/parsed_data/create_info.hpp"
 #include "duckdb/parser/group_by_node.hpp"
+#include "duckdb/parser/parsed_data/create_info.hpp"
+#include "duckdb/parser/qualified_name.hpp"
 #include "duckdb/parser/query_node.hpp"
-#include "duckdb/common/case_insensitive_map.hpp"
-
-#include "pg_definitions.hpp"
+#include "duckdb/parser/query_node/cte_node.hpp"
+#include "duckdb/parser/tokens.hpp"
 #include "nodes/parsenodes.hpp"
 #include "nodes/primnodes.hpp"
+#include "pg_definitions.hpp"
 
 namespace duckdb {
 
@@ -232,6 +232,9 @@ private:
 	unique_ptr<ParsedExpression> TransformParamRef(duckdb_libpgquery::PGParamRef &node);
 	unique_ptr<ParsedExpression> TransformNamedArg(duckdb_libpgquery::PGNamedArgExpr &root);
 
+	//! Transform multi assignment reference into an Expression
+	unique_ptr<ParsedExpression> TransformMultiAssignRef(duckdb_libpgquery::PGMultiAssignRef &root);
+
 	unique_ptr<ParsedExpression> TransformSQLValueFunction(duckdb_libpgquery::PGSQLValueFunction &node);
 
 	unique_ptr<ParsedExpression> TransformSubquery(duckdb_libpgquery::PGSubLink &root);
@@ -269,13 +272,17 @@ private:
 	OnCreateConflict TransformOnConflict(duckdb_libpgquery::PGOnCreateConflict conflict);
 	string TransformAlias(duckdb_libpgquery::PGAlias *root, vector<string> &column_name_alias);
 	vector<string> TransformStringList(duckdb_libpgquery::PGList *list);
-	void TransformCTE(duckdb_libpgquery::PGWithClause &de_with_clause, CommonTableExpressionMap &cte_map);
-	unique_ptr<SelectStatement> TransformRecursiveCTE(duckdb_libpgquery::PGCommonTableExpr &cte,
+	void TransformCTE(duckdb_libpgquery::PGWithClause &de_with_clause, CommonTableExpressionMap &cte_map,
+	                  vector<unique_ptr<CTENode>> &materialized_ctes);
+	static unique_ptr<QueryNode> TransformMaterializedCTE(unique_ptr<QueryNode> root,
+	                                                      vector<unique_ptr<CTENode>> &materialized_ctes);
+	unique_ptr<SelectStatement> TransformRecursiveCTE(duckdb_libpgquery::PGCommonTableExpr &node,
 	                                                  CommonTableExpressionInfo &info);
 
 	unique_ptr<ParsedExpression> TransformUnaryOperator(const string &op, unique_ptr<ParsedExpression> child);
 	unique_ptr<ParsedExpression> TransformBinaryOperator(string op, unique_ptr<ParsedExpression> left,
 	                                                     unique_ptr<ParsedExpression> right);
+	static bool ConstructConstantFromExpression(const ParsedExpression &expr, Value &value);
 	//===--------------------------------------------------------------------===//
 	// TableRef transform
 	//===--------------------------------------------------------------------===//

package/src/duckdb/src/include/duckdb/planner/binder.hpp

@@ -21,6 +21,7 @@
 #include "duckdb/planner/bound_tokens.hpp"
 #include "duckdb/planner/expression/bound_columnref_expression.hpp"
 #include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/planner/joinside.hpp"
 #include "duckdb/common/reference_map.hpp"
 
 namespace duckdb {
@@ -75,7 +76,7 @@ struct CorrelatedColumnInfo {
 */
 class Binder : public std::enable_shared_from_this<Binder> {
 	friend class ExpressionBinder;
-	friend class RecursiveSubqueryPlanner;
+	friend class RecursiveDependentJoinPlanner;
 
 public:
 	DUCKDB_API static shared_ptr<Binder> CreateBinder(ClientContext &context, optional_ptr<Binder> parent = nullptr,
@@ -199,10 +200,10 @@ private:
 	vector<reference<ExpressionBinder>> active_binders;
 	//! The count of bound_tables
 	idx_t bound_tables;
-	//! Whether or not the binder has any unplanned subqueries that still need to be planned
-	bool has_unplanned_subqueries = false;
-	//! Whether or not subqueries should be planned already
-	bool plan_subquery = true;
+	//! Whether or not the binder has any unplanned dependent joins that still need to be planned/flattened
+	bool has_unplanned_dependent_joins = false;
+	//! Whether or not outside dependent joins have been planned and flattened
+	bool is_outside_flattened = true;
 	//! Whether CTEs should reference the parent binder (if it exists)
 	bool inherit_ctes = true;
 	//! Whether or not the binder can contain NULLs as the root of expressions
@@ -217,6 +218,10 @@ private:
 	reference_set_t<ViewCatalogEntry> bound_views;
 
 private:
+	//! Get the root binder (binder with no parent)
+	Binder *GetRootBinder();
+	//! Determine the depth of the binder
+	idx_t GetBinderDepth() const;
 	//! Bind the expressions of generated columns to check for errors
 	void BindGeneratedColumns(BoundCreateTableInfo &info);
 	//! Bind the default values of the columns of a table
@@ -265,10 +270,12 @@ private:
 	unique_ptr<BoundQueryNode> BindNode(SelectNode &node);
 	unique_ptr<BoundQueryNode> BindNode(SetOperationNode &node);
 	unique_ptr<BoundQueryNode> BindNode(RecursiveCTENode &node);
+	unique_ptr<BoundQueryNode> BindNode(CTENode &node);
 	unique_ptr<BoundQueryNode> BindNode(QueryNode &node);
 
 	unique_ptr<LogicalOperator> VisitQueryNode(BoundQueryNode &node, unique_ptr<LogicalOperator> root);
 	unique_ptr<LogicalOperator> CreatePlan(BoundRecursiveCTENode &node);
+	unique_ptr<LogicalOperator> CreatePlan(BoundCTENode &node);
 	unique_ptr<LogicalOperator> CreatePlan(BoundSelectNode &statement);
 	unique_ptr<LogicalOperator> CreatePlan(BoundSetOperationNode &node);
 	unique_ptr<LogicalOperator> CreatePlan(BoundQueryNode &node);

package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp

@@ -17,6 +17,7 @@ class BoundQueryNode;
 class BoundSelectNode;
 class BoundSetOperationNode;
 class BoundRecursiveCTENode;
+class BoundCTENode;
 
 //===--------------------------------------------------------------------===//
 // Expressions

package/src/duckdb/src/include/duckdb/planner/column_binding.hpp

@@ -32,6 +32,10 @@ struct ColumnBinding {
 	bool operator==(const ColumnBinding &rhs) const {
 		return table_index == rhs.table_index && column_index == rhs.column_index;
 	}
+
+	bool operator!=(const ColumnBinding &rhs) const {
+		return !(*this == rhs);
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp

@@ -23,9 +23,9 @@ public:
 	    : BoundConstraint(ConstraintType::UNIQUE), keys(std::move(keys)), key_set(std::move(key_set)),
 	      is_primary_key(is_primary_key) {
 #ifdef DEBUG
-		D_ASSERT(keys.size() == key_set.size());
-		for (auto &key : keys) {
-			D_ASSERT(key_set.find(key) != key_set.end());
+		D_ASSERT(this->keys.size() == this->key_set.size());
+		for (auto &key : this->keys) {
+			D_ASSERT(this->key_set.find(key) != this->key_set.end());
 		}
 #endif
 	}

package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp

@@ -20,8 +20,6 @@ class LateralBinder : public ExpressionBinder {
 public:
 	LateralBinder(Binder &binder, ClientContext &context);
 
-	//! Extract the correlated lateral join columns and remove them from the targeted binder
-	vector<CorrelatedColumnInfo> ExtractCorrelatedColumns(Binder &binder);
 	bool HasCorrelatedColumns() const {
 		return !correlated_columns.empty();
 	}

package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp

@@ -46,6 +46,7 @@ class LogicalPragma;
 class LogicalPrepare;
 class LogicalProjection;
 class LogicalRecursiveCTE;
+class LogicalMaterializedCTE;
 class LogicalSetOperation;
 class LogicalSample;
 class LogicalShow;

package/src/duckdb/src/include/duckdb/planner/operator/list.hpp

@@ -25,6 +25,7 @@
 #include "duckdb/planner/operator/logical_join.hpp"
 #include "duckdb/planner/operator/logical_limit.hpp"
 #include "duckdb/planner/operator/logical_limit_percent.hpp"
+#include "duckdb/planner/operator/logical_materialized_cte.hpp"
 #include "duckdb/planner/operator/logical_order.hpp"
 #include "duckdb/planner/operator/logical_pivot.hpp"
 #include "duckdb/planner/operator/logical_positional_join.hpp"
@@ -32,9 +33,9 @@
 #include "duckdb/planner/operator/logical_prepare.hpp"
 #include "duckdb/planner/operator/logical_projection.hpp"
 #include "duckdb/planner/operator/logical_recursive_cte.hpp"
+#include "duckdb/planner/operator/logical_reset.hpp"
 #include "duckdb/planner/operator/logical_sample.hpp"
 #include "duckdb/planner/operator/logical_set.hpp"
-#include "duckdb/planner/operator/logical_reset.hpp"
 #include "duckdb/planner/operator/logical_set_operation.hpp"
 #include "duckdb/planner/operator/logical_show.hpp"
 #include "duckdb/planner/operator/logical_simple.hpp"

package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp

@@ -38,25 +38,25 @@ public:
 	                        FieldReader &reader);
 
 public:
-	static unique_ptr<LogicalOperator> CreateJoin(JoinType type, JoinRefType ref_type,
+	static unique_ptr<LogicalOperator> CreateJoin(ClientContext &context, JoinType type, JoinRefType ref_type,
 	                                              unique_ptr<LogicalOperator> left_child,
 	                                              unique_ptr<LogicalOperator> right_child,
 	                                              unique_ptr<Expression> condition);
-	static unique_ptr<LogicalOperator> CreateJoin(JoinType type, JoinRefType ref_type,
+	static unique_ptr<LogicalOperator> CreateJoin(ClientContext &context, JoinType type, JoinRefType ref_type,
 	                                              unique_ptr<LogicalOperator> left_child,
 	                                              unique_ptr<LogicalOperator> right_child,
 	                                              vector<JoinCondition> conditions,
 	                                              vector<unique_ptr<Expression>> arbitrary_expressions);
 
-	static void ExtractJoinConditions(JoinType type, unique_ptr<LogicalOperator> &left_child,
+	static void ExtractJoinConditions(ClientContext &context, JoinType type, unique_ptr<LogicalOperator> &left_child,
 	                                  unique_ptr<LogicalOperator> &right_child, unique_ptr<Expression> condition,
 	                                  vector<JoinCondition> &conditions,
 	                                  vector<unique_ptr<Expression>> &arbitrary_expressions);
-	static void ExtractJoinConditions(JoinType type, unique_ptr<LogicalOperator> &left_child,
+	static void ExtractJoinConditions(ClientContext &context, JoinType type, unique_ptr<LogicalOperator> &left_child,
 	                                  unique_ptr<LogicalOperator> &right_child,
 	                                  vector<unique_ptr<Expression>> &expressions, vector<JoinCondition> &conditions,
 	                                  vector<unique_ptr<Expression>> &arbitrary_expressions);
-	static void ExtractJoinConditions(JoinType type, unique_ptr<LogicalOperator> &left_child,
+	static void ExtractJoinConditions(ClientContext &context, JoinType type, unique_ptr<LogicalOperator> &left_child,
 	                                  unique_ptr<LogicalOperator> &right_child,
 	                                  const unordered_set<idx_t> &left_bindings,
 	                                  const unordered_set<idx_t> &right_bindings,

package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/common/types/chunk_collection.hpp"
 #include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/common/enums/cte_materialize.hpp"
 
 namespace duckdb {
 
@@ -19,8 +20,10 @@ public:
 	static constexpr const LogicalOperatorType TYPE = LogicalOperatorType::LOGICAL_CTE_REF;
 
 public:
-	LogicalCTERef(idx_t table_index, idx_t cte_index, vector<LogicalType> types, vector<string> colnames)
-	    : LogicalOperator(LogicalOperatorType::LOGICAL_CTE_REF), table_index(table_index), cte_index(cte_index) {
+	LogicalCTERef(idx_t table_index, idx_t cte_index, vector<LogicalType> types, vector<string> colnames,
+	              CTEMaterialize materialized_cte)
+	    : LogicalOperator(LogicalOperatorType::LOGICAL_CTE_REF), table_index(table_index), cte_index(cte_index),
+	      materialized_cte(materialized_cte) {
 		D_ASSERT(types.size() > 0);
 		chunk_types = types;
 		bound_columns = colnames;
@@ -33,6 +36,8 @@ public:
 	idx_t cte_index;
 	//! The types of the chunk
 	vector<LogicalType> chunk_types;
+	//! Does this operator read a materialized CTE?
+	CTEMaterialize materialized_cte;
 
 public:
 	vector<ColumnBinding> GetColumnBindings() override {

package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp

@@ -0,0 +1,43 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/planner/operator/logical_dependent_join.hpp
+//
+// logical_dependent_join represents a logical operator for lateral joins that
+// is planned but not yet flattened
+//
+// This construct only exists during planning and should not exist in the plan
+// once flattening is complete. Although the same information can be kept in the
+// join itself, creating a new construct makes the code cleaner and easier to
+// understand.
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/planner/binder.hpp"
+#include "duckdb/planner/operator/logical_comparison_join.hpp"
+
+namespace duckdb {
+
+class LogicalDependentJoin : public LogicalComparisonJoin {
+public:
+	static constexpr const LogicalOperatorType TYPE = LogicalOperatorType::LOGICAL_DEPENDENT_JOIN;
+
+public:
+	explicit LogicalDependentJoin(unique_ptr<LogicalOperator> left, unique_ptr<LogicalOperator> right,
+	                              vector<CorrelatedColumnInfo> correlated_columns, JoinType type,
+	                              unique_ptr<Expression> condition);
+
+	//! The conditions of the join
+	unique_ptr<Expression> join_condition;
+	//! The list of columns that have correlations with the right
+	vector<CorrelatedColumnInfo> correlated_columns;
+
+public:
+	static unique_ptr<LogicalOperator> Create(unique_ptr<LogicalOperator> left, unique_ptr<LogicalOperator> right,
+	                                          vector<CorrelatedColumnInfo> correlated_columns, JoinType type,
+	                                          unique_ptr<Expression> condition);
+	void Serialize(FieldWriter &writer) const override;
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp

@@ -11,6 +11,7 @@
 #include "duckdb/function/table_function.hpp"
 #include "duckdb/planner/logical_operator.hpp"
 #include "duckdb/planner/table_filter.hpp"
+#include "duckdb/common/extra_operator_info.hpp"
 
 namespace duckdb {
 
@@ -49,6 +50,9 @@ public:
 	vector<string> input_table_names;
 	//! For a table-in-out function, the set of projected input columns
 	vector<column_t> projected_input;
+	//! Currently stores File Filters (as strings) applied by hive partitioning/complex filter pushdown
+	//! Stored so the can be included in explain output
+	ExtraOperatorInfo extra_info;
 
 	string GetName() const override;
 	string ParamsToString() const override;

package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp

@@ -0,0 +1,49 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/planner/operator/logical_materialized_cte.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/planner/logical_operator.hpp"
+
+namespace duckdb {
+
+class LogicalMaterializedCTE : public LogicalOperator {
+	LogicalMaterializedCTE(idx_t table_index)
+	    : LogicalOperator(LogicalOperatorType::LOGICAL_MATERIALIZED_CTE), table_index(table_index) {
+	}
+
+public:
+	static constexpr const LogicalOperatorType TYPE = LogicalOperatorType::LOGICAL_MATERIALIZED_CTE;
+
+public:
+	LogicalMaterializedCTE(string ctename, idx_t table_index, idx_t column_count, unique_ptr<LogicalOperator> cte,
+	                       unique_ptr<LogicalOperator> child)
+	    : LogicalOperator(LogicalOperatorType::LOGICAL_MATERIALIZED_CTE), table_index(table_index),
+	      column_count(column_count), ctename(ctename) {
+		children.push_back(std::move(cte));
+		children.push_back(std::move(child));
+	}
+
+	idx_t table_index;
+	idx_t column_count;
+	string ctename;
+
+public:
+	vector<ColumnBinding> GetColumnBindings() override {
+		return children[1]->GetColumnBindings();
+	}
+	void Serialize(FieldWriter &writer) const override;
+	static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
+	vector<idx_t> GetTableIndex() const override;
+
+protected:
+	void ResolveTypes() override {
+		types = children[1]->types;
+	}
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp

@@ -22,15 +22,16 @@ public:
 	static constexpr const LogicalOperatorType TYPE = LogicalOperatorType::LOGICAL_RECURSIVE_CTE;
 
 public:
-	LogicalRecursiveCTE(idx_t table_index, idx_t column_count, bool union_all, unique_ptr<LogicalOperator> top,
-	                    unique_ptr<LogicalOperator> bottom)
-	    : LogicalOperator(LogicalOperatorType::LOGICAL_RECURSIVE_CTE), union_all(union_all), table_index(table_index),
-	      column_count(column_count) {
+	LogicalRecursiveCTE(string ctename, idx_t table_index, idx_t column_count, bool union_all,
+	                    unique_ptr<LogicalOperator> top, unique_ptr<LogicalOperator> bottom)
+	    : LogicalOperator(LogicalOperatorType::LOGICAL_RECURSIVE_CTE), union_all(union_all), ctename(ctename),
+	      table_index(table_index), column_count(column_count) {
 		children.push_back(std::move(top));
 		children.push_back(std::move(bottom));
 	}
 
 	bool union_all;
+	string ctename;
 	idx_t table_index;
 	idx_t column_count;
 

package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp

@@ -0,0 +1,44 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/planner/query_node/bound_cte_node.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/planner/binder.hpp"
+#include "duckdb/planner/bound_query_node.hpp"
+
+namespace duckdb {
+
+class BoundCTENode : public BoundQueryNode {
+public:
+	static constexpr const QueryNodeType TYPE = QueryNodeType::CTE_NODE;
+
+public:
+	BoundCTENode() : BoundQueryNode(QueryNodeType::CTE_NODE) {
+	}
+
+	//! Keep track of the CTE name this node represents
+	string ctename;
+
+	//! The cte node
+	unique_ptr<BoundQueryNode> query;
+	//! The child node
+	unique_ptr<BoundQueryNode> child;
+	//! Index used by the set operation
+	idx_t setop_index;
+	//! The binder used by the query side of the CTE
+	shared_ptr<Binder> query_binder;
+	//! The binder used by the child side of the CTE
+	shared_ptr<Binder> child_binder;
+
+public:
+	idx_t GetRootIndex() override {
+		return child->GetRootIndex();
+	}
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp

@@ -1,3 +1,4 @@
 #include "duckdb/planner/query_node/bound_recursive_cte_node.hpp"
+#include "duckdb/planner/query_node/bound_cte_node.hpp"
 #include "duckdb/planner/query_node/bound_select_node.hpp"
 #include "duckdb/planner/query_node/bound_set_operation_node.hpp"

package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp

@@ -23,7 +23,7 @@ struct FlattenDependentJoins {
 
 	//! Detects which Logical Operators have correlated expressions that they are dependent upon, filling the
 	//! has_correlated_expressions map.
-	bool DetectCorrelatedExpressions(LogicalOperator *op, bool lateral = false);
+	bool DetectCorrelatedExpressions(LogicalOperator *op, bool lateral = false, idx_t lateral_depth = 0);
 
 	//! Push the dependent join down a LogicalOperator
 	unique_ptr<LogicalOperator> PushDownDependentJoin(unique_ptr<LogicalOperator> plan);
@@ -43,7 +43,7 @@ struct FlattenDependentJoins {
 
 private:
 	unique_ptr<LogicalOperator> PushDownDependentJoinInternal(unique_ptr<LogicalOperator> plan,
-	                                                          bool &parent_propagate_null_values);
+	                                                          bool &parent_propagate_null_values, idx_t lateral_depth);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp

@@ -16,7 +16,8 @@ namespace duckdb {
 //! Helper class to recursively detect correlated expressions inside a single LogicalOperator
 class HasCorrelatedExpressions : public LogicalOperatorVisitor {
 public:
-	explicit HasCorrelatedExpressions(const vector<CorrelatedColumnInfo> &correlated, bool lateral = false);
+	explicit HasCorrelatedExpressions(const vector<CorrelatedColumnInfo> &correlated, bool lateral = false,
+	                                  idx_t lateral_depth = 0);
 
 	void VisitOperator(LogicalOperator &op) override;
 
@@ -28,6 +29,8 @@ protected:
 	unique_ptr<Expression> VisitReplace(BoundSubqueryExpression &expr, unique_ptr<Expression> *expr_ptr) override;
 
 	const vector<CorrelatedColumnInfo> &correlated_columns;
+	// Tracks number of nested laterals
+	idx_t lateral_depth;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp

@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/planner/subquery/recursive_dependent_join_planner.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/planner/logical_operator_visitor.hpp"
+
+namespace duckdb {
+
+class Binder;
+
+/*
+ * Recursively plan subqueries and flatten dependent joins from outermost to innermost (like peeling an onion).
+ */
+class RecursiveDependentJoinPlanner : public LogicalOperatorVisitor {
+public:
+	explicit RecursiveDependentJoinPlanner(Binder &binder) : binder(binder) {
+	}
+	void VisitOperator(LogicalOperator &op) override;
+	unique_ptr<Expression> VisitReplace(BoundSubqueryExpression &expr, unique_ptr<Expression> *expr_ptr) override;
+
+private:
+	unique_ptr<LogicalOperator> root;
+	Binder &binder;
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp

@@ -17,7 +17,8 @@ namespace duckdb {
 //! Helper class to rewrite correlated expressions within a single LogicalOperator
 class RewriteCorrelatedExpressions : public LogicalOperatorVisitor {
 public:
-	RewriteCorrelatedExpressions(ColumnBinding base_binding, column_binding_map_t<idx_t> &correlated_map);
+	RewriteCorrelatedExpressions(ColumnBinding base_binding, column_binding_map_t<idx_t> &correlated_map,
+	                             idx_t lateral_depth, bool recursive_rewrite = false);
 
 	void VisitOperator(LogicalOperator &op) override;
 
@@ -31,7 +32,7 @@ private:
 	public:
 		RewriteCorrelatedRecursive(BoundSubqueryExpression &parent, ColumnBinding base_binding,
 		                           column_binding_map_t<idx_t> &correlated_map);
-
+		void RewriteJoinRefRecursive(BoundTableRef &ref);
 		void RewriteCorrelatedSubquery(BoundSubqueryExpression &expr);
 		void RewriteCorrelatedExpressions(Expression &child);
 
@@ -43,6 +44,11 @@ private:
 private:
 	ColumnBinding base_binding;
 	column_binding_map_t<idx_t> &correlated_map;
+	// To keep track of the number of dependent joins encountered
+	idx_t lateral_depth;
+	// This flag is used to determine if the rewrite should recursively update the bindings for all
+	// bound columns ref in the plan, and update the depths to match the new source
+	bool recursive_rewrite;
 };
 
 //! Helper class that rewrites COUNT aggregates into a CASE expression turning NULL into 0 after a LEFT OUTER JOIN

package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp

@@ -17,8 +17,9 @@ public:
 	static constexpr const TableReferenceType TYPE = TableReferenceType::CTE;
 
 public:
-	BoundCTERef(idx_t bind_index, idx_t cte_index)
-	    : BoundTableRef(TableReferenceType::CTE), bind_index(bind_index), cte_index(cte_index) {
+	BoundCTERef(idx_t bind_index, idx_t cte_index, CTEMaterialize materialized_cte)
+	    : BoundTableRef(TableReferenceType::CTE), bind_index(bind_index), cte_index(cte_index),
+	      materialized_cte(materialized_cte) {
 	}
 
 	//! The set of columns bound to this base table reference
@@ -29,5 +30,7 @@ public:
 	idx_t bind_index;
 	//! The index of the cte
 	idx_t cte_index;
+	//! Is this a reference to a materialized CTE?
+	CTEMaterialize materialized_cte;
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp

@@ -45,7 +45,7 @@ public:
 	DUCKDB_API ArenaChunk *GetHead();
 	DUCKDB_API ArenaChunk *GetTail();
 
-	DUCKDB_API bool IsEmpty();
+	DUCKDB_API bool IsEmpty() const;
 
 	//! Returns an "Allocator" wrapper for this arena allocator
 	Allocator &GetAllocator() {

package/src/duckdb/src/include/duckdb/storage/block_manager.hpp

@@ -65,6 +65,9 @@ public:
 	//! Returns the number of free blocks
 	virtual idx_t FreeBlocks() = 0;
 
+	//! Truncate the underlying database file after a checkpoint
+	virtual void Truncate();
+
 	//! Register a block with the given block id in the base file
 	shared_ptr<BlockHandle> RegisterBlock(block_id_t block_id, bool is_meta_block = false);
 	//! Clear cached handles for meta blocks
@@ -74,9 +77,6 @@ public:
 
 	void UnregisterBlock(block_id_t block_id, bool can_destroy);
 
-	static BlockManager &GetBlockManager(ClientContext &context);
-	static BlockManager &GetBlockManager(DatabaseInstance &db);
-
 private:
 	//! The lock for the set of blocks
 	mutex blocks_lock;

package/src/duckdb/src/include/duckdb/storage/data_table.hpp

@@ -51,7 +51,7 @@ public:
 	          const string &table, vector<ColumnDefinition> column_definitions_p,
 	          unique_ptr<PersistentTableData> data = nullptr);
 	//! Constructs a DataTable as a delta on an existing data table with a newly added column
-	DataTable(ClientContext &context, DataTable &parent, ColumnDefinition &new_column, Expression *default_value);
+	DataTable(ClientContext &context, DataTable &parent, ColumnDefinition &new_column, Expression &default_value);
 	//! Constructs a DataTable as a delta on an existing data table but with one column removed
 	DataTable(ClientContext &context, DataTable &parent, idx_t removed_column);
 	//! Constructs a DataTable as a delta on an existing data table but with one column changed type

package/src/duckdb/src/include/duckdb/storage/object_cache.hpp

@@ -47,11 +47,33 @@ public:
 		return std::static_pointer_cast<T, ObjectCacheEntry>(object);
 	}
 
+	template <class T, class... Args>
+	shared_ptr<T> GetOrCreate(const string &key, Args &&...args) {
+		lock_guard<mutex> glock(lock);
+
+		auto entry = cache.find(key);
+		if (entry == cache.end()) {
+			auto value = make_shared<T>(args...);
+			cache[key] = value;
+			return value;
+		}
+		auto object = entry->second;
+		if (!object || object->GetObjectType() != T::ObjectType()) {
+			return nullptr;
+		}
+		return std::static_pointer_cast<T, ObjectCacheEntry>(object);
+	}
+
 	void Put(string key, shared_ptr<ObjectCacheEntry> value) {
 		lock_guard<mutex> glock(lock);
 		cache[key] = std::move(value);
 	}
 
+	void Delete(const string &key) {
+		lock_guard<mutex> glock(lock);
+		cache.erase(key);
+	}
+
 	DUCKDB_API static ObjectCache &GetObjectCache(ClientContext &context);
 	DUCKDB_API static bool ObjectCacheEnabled(ClientContext &context);