duckdb 0.8.2-dev33.0 → 0.8.2-dev3300.0

package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp

@@ -10,6 +10,7 @@
 #include "duckdb/planner/operator/list.hpp"
 #include "duckdb/planner/subquery/has_correlated_expressions.hpp"
 #include "duckdb/planner/subquery/rewrite_correlated_expressions.hpp"
+#include "duckdb/planner/operator/logical_dependent_join.hpp"
 
 namespace duckdb {
 
@@ -24,20 +25,32 @@ FlattenDependentJoins::FlattenDependentJoins(Binder &binder, const vector<Correl
 	}
 }
 
-bool FlattenDependentJoins::DetectCorrelatedExpressions(LogicalOperator *op, bool lateral) {
+bool FlattenDependentJoins::DetectCorrelatedExpressions(LogicalOperator *op, bool lateral, idx_t lateral_depth) {
+
+	bool is_lateral_join = false;
+
 	D_ASSERT(op);
 	// check if this entry has correlated expressions
-	HasCorrelatedExpressions visitor(correlated_columns, lateral);
+	if (op->type == LogicalOperatorType::LOGICAL_DEPENDENT_JOIN) {
+		is_lateral_join = true;
+	}
+	HasCorrelatedExpressions visitor(correlated_columns, lateral, lateral_depth);
 	visitor.VisitOperator(*op);
 	bool has_correlation = visitor.has_correlated_expressions;
+	int child_idx = 0;
 	// now visit the children of this entry and check if they have correlated expressions
 	for (auto &child : op->children) {
+		auto new_lateral_depth = lateral_depth;
+		if (is_lateral_join && child_idx == 1) {
+			new_lateral_depth = lateral_depth + 1;
+		}
 		// we OR the property with its children such that has_correlation is true if either
 		// (1) this node has a correlated expression or
 		// (2) one of its children has a correlated expression
-		if (DetectCorrelatedExpressions(child.get(), lateral)) {
+		if (DetectCorrelatedExpressions(child.get(), lateral, new_lateral_depth)) {
 			has_correlation = true;
 		}
+		child_idx++;
 	}
 	// set the entry in the map
 	has_correlated_expressions[op] = has_correlation;
@@ -46,7 +59,7 @@ bool FlattenDependentJoins::DetectCorrelatedExpressions(LogicalOperator *op, boo
 
 unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoin(unique_ptr<LogicalOperator> plan) {
 	bool propagate_null_values = true;
-	auto result = PushDownDependentJoinInternal(std::move(plan), propagate_null_values);
+	auto result = PushDownDependentJoinInternal(std::move(plan), propagate_null_values, 0);
 	if (!replacement_map.empty()) {
 		// check if we have to replace any COUNT aggregates into "CASE WHEN X IS NULL THEN 0 ELSE COUNT END"
 		RewriteCountAggregates aggr(replacement_map);
@@ -70,8 +83,10 @@ bool SubqueryDependentFilter(Expression *expr) {
 	}
 	return false;
 }
+
 unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal(unique_ptr<LogicalOperator> plan,
-                                                                                 bool &parent_propagate_null_values) {
+                                                                                 bool &parent_propagate_null_values,
+                                                                                 idx_t lateral_depth) {
 	// first check if the logical operator has correlated expressions
 	auto entry = has_correlated_expressions.find(plan.get());
 	D_ASSERT(entry != has_correlated_expressions.end());
@@ -95,10 +110,11 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		for (auto &expr : plan->expressions) {
 			any_join |= SubqueryDependentFilter(expr.get());
 		}
-		plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+		plan->children[0] =
+		    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
 
 		// then we replace any correlated expressions with the corresponding entry in the correlated_map
-		RewriteCorrelatedExpressions rewriter(base_binding, correlated_map);
+		RewriteCorrelatedExpressions rewriter(base_binding, correlated_map, lateral_depth);
 		rewriter.VisitOperator(*plan);
 		return plan;
 	}
@@ -108,10 +124,11 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		for (auto &expr : plan->expressions) {
 			parent_propagate_null_values &= expr->PropagatesNullValues();
 		}
-		plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+		plan->children[0] =
+		    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
 
 		// then we replace any correlated expressions with the corresponding entry in the correlated_map
-		RewriteCorrelatedExpressions rewriter(base_binding, correlated_map);
+		RewriteCorrelatedExpressions rewriter(base_binding, correlated_map, lateral_depth);
 		rewriter.VisitOperator(*plan);
 		// now we add all the columns of the delim_scan to the projection list
 		auto &proj = plan->Cast<LogicalProjection>();
@@ -134,9 +151,10 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		for (auto &expr : plan->expressions) {
 			parent_propagate_null_values &= expr->PropagatesNullValues();
 		}
-		plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+		plan->children[0] =
+		    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
 		// then we replace any correlated expressions with the corresponding entry in the correlated_map
-		RewriteCorrelatedExpressions rewriter(base_binding, correlated_map);
+		RewriteCorrelatedExpressions rewriter(base_binding, correlated_map, lateral_depth);
 		rewriter.VisitOperator(*plan);
 		// now we add all the columns of the delim_scan to the grouping operators AND the projection list
 		idx_t delim_table_index;
@@ -231,22 +249,24 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		bool right_has_correlation = has_correlated_expressions.find(plan->children[1].get())->second;
 		if (!right_has_correlation) {
 			// only left has correlation: push into left
-			plan->children[0] =
-			    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+			plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]),
+			                                                  parent_propagate_null_values, lateral_depth);
 			return plan;
 		}
 		if (!left_has_correlation) {
 			// only right has correlation: push into right
-			plan->children[1] =
-			    PushDownDependentJoinInternal(std::move(plan->children[1]), parent_propagate_null_values);
+			plan->children[1] = PushDownDependentJoinInternal(std::move(plan->children[1]),
+			                                                  parent_propagate_null_values, lateral_depth);
 			return plan;
 		}
 		// both sides have correlation
 		// turn into an inner join
 		auto join = make_uniq<LogicalComparisonJoin>(JoinType::INNER);
-		plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+		plan->children[0] =
+		    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
 		auto left_binding = this->base_binding;
-		plan->children[1] = PushDownDependentJoinInternal(std::move(plan->children[1]), parent_propagate_null_values);
+		plan->children[1] =
+		    PushDownDependentJoinInternal(std::move(plan->children[1]), parent_propagate_null_values, lateral_depth);
 		// add the correlated columns to the join conditions
 		for (idx_t i = 0; i < correlated_columns.size(); i++) {
 			JoinCondition cond;
@@ -261,6 +281,26 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		join->children.push_back(std::move(plan->children[1]));
 		return std::move(join);
 	}
+	case LogicalOperatorType::LOGICAL_DEPENDENT_JOIN: {
+		auto &dependent_join = plan->Cast<LogicalJoin>();
+		if (!((dependent_join.join_type == JoinType::INNER) || (dependent_join.join_type == JoinType::LEFT))) {
+			throw Exception("Dependent join can only be INNER or LEFT type");
+		}
+		D_ASSERT(plan->children.size() == 2);
+		// Push all the bindings down to the left side so the right side knows where to refer DELIM_GET from
+		plan->children[0] =
+		    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
+
+		// Normal rewriter like in other joins
+		RewriteCorrelatedExpressions rewriter(this->base_binding, correlated_map, lateral_depth);
+		rewriter.VisitOperator(*plan);
+
+		// Recursive rewriter to visit right side of lateral join and update bindings from left
+		RewriteCorrelatedExpressions recursive_rewriter(this->base_binding, correlated_map, lateral_depth + 1, true);
+		recursive_rewriter.VisitOperator(*plan->children[1]);
+
+		return plan;
+	}
 	case LogicalOperatorType::LOGICAL_ANY_JOIN:
 	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: {
@@ -274,30 +314,33 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 			// inner join
 			if (!right_has_correlation) {
 				// only left has correlation: push into left
-				plan->children[0] =
-				    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+				plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]),
+				                                                  parent_propagate_null_values, lateral_depth);
+				// Remove the correlated columns coming from outside for current join node
 				return plan;
 			}
 			if (!left_has_correlation) {
 				// only right has correlation: push into right
-				plan->children[1] =
-				    PushDownDependentJoinInternal(std::move(plan->children[1]), parent_propagate_null_values);
+				plan->children[1] = PushDownDependentJoinInternal(std::move(plan->children[1]),
+				                                                  parent_propagate_null_values, lateral_depth);
+				// Remove the correlated columns coming from outside for current join node
 				return plan;
 			}
 		} else if (join.join_type == JoinType::LEFT) {
 			// left outer join
 			if (!right_has_correlation) {
 				// only left has correlation: push into left
-				plan->children[0] =
-				    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+				plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]),
+				                                                  parent_propagate_null_values, lateral_depth);
+				// Remove the correlated columns coming from outside for current join node
 				return plan;
 			}
 		} else if (join.join_type == JoinType::RIGHT) {
 			// left outer join
 			if (!left_has_correlation) {
 				// only right has correlation: push into right
-				plan->children[1] =
-				    PushDownDependentJoinInternal(std::move(plan->children[1]), parent_propagate_null_values);
+				plan->children[1] = PushDownDependentJoinInternal(std::move(plan->children[1]),
+				                                                  parent_propagate_null_values, lateral_depth);
 				return plan;
 			}
 		} else if (join.join_type == JoinType::MARK) {
@@ -305,10 +348,10 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 				throw Exception("MARK join with correlation in RHS not supported");
 			}
 			// push the child into the LHS
-			plan->children[0] =
-			    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+			plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]),
+			                                                  parent_propagate_null_values, lateral_depth);
 			// rewrite expressions in the join conditions
-			RewriteCorrelatedExpressions rewriter(base_binding, correlated_map);
+			RewriteCorrelatedExpressions rewriter(base_binding, correlated_map, lateral_depth);
 			rewriter.VisitOperator(*plan);
 			return plan;
 		} else {
@@ -316,9 +359,11 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		}
 		// both sides have correlation
 		// push into both sides
-		plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+		plan->children[0] =
+		    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
 		auto left_binding = this->base_binding;
-		plan->children[1] = PushDownDependentJoinInternal(std::move(plan->children[1]), parent_propagate_null_values);
+		plan->children[1] =
+		    PushDownDependentJoinInternal(std::move(plan->children[1]), parent_propagate_null_values, lateral_depth);
 		auto right_binding = this->base_binding;
 		// NOTE: for OUTER JOINS it matters what the BASE BINDING is after the join
 		// for the LEFT OUTER JOIN, we want the LEFT side to be the base binding after we push
@@ -354,7 +399,7 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 			}
 		}
 		// then we replace any correlated expressions with the corresponding entry in the correlated_map
-		RewriteCorrelatedExpressions rewriter(right_binding, correlated_map);
+		RewriteCorrelatedExpressions rewriter(right_binding, correlated_map, lateral_depth);
 		rewriter.VisitOperator(*plan);
 		return plan;
 	}
@@ -371,9 +416,11 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		// this is done for an optimization to avoid having to compute the total order
 		if (plan->children[0]->type == LogicalOperatorType::LOGICAL_ORDER_BY) {
 			order_by = unique_ptr_cast<LogicalOperator, LogicalOrder>(std::move(plan->children[0]));
-			child = PushDownDependentJoinInternal(std::move(order_by->children[0]), parent_propagate_null_values);
+			child = PushDownDependentJoinInternal(std::move(order_by->children[0]), parent_propagate_null_values,
+			                                      lateral_depth);
 		} else {
-			child = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+			child = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values,
+			                                      lateral_depth);
 		}
 		auto child_column_count = child->GetColumnBindings().size();
 		// we push a row_number() OVER (PARTITION BY [correlated columns])
@@ -437,7 +484,8 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 	case LogicalOperatorType::LOGICAL_WINDOW: {
 		auto &window = plan->Cast<LogicalWindow>();
 		// push into children
-		plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+		plan->children[0] =
+		    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
 		// add the correlated columns to the PARTITION BY clauses in the Window
 		for (auto &expr : window.expressions) {
 			D_ASSERT(expr->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);
@@ -488,9 +536,10 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 	case LogicalOperatorType::LOGICAL_EXPRESSION_GET: {
 		// expression get
 		// first we flatten the dependent join in the child
-		plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+		plan->children[0] =
+		    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
 		// then we replace any correlated expressions with the corresponding entry in the correlated_map
-		RewriteCorrelatedExpressions rewriter(base_binding, correlated_map);
+		RewriteCorrelatedExpressions rewriter(base_binding, correlated_map, lateral_depth);
 		rewriter.VisitOperator(*plan);
 		// now we add all the correlated columns to each of the expressions of the expression scan
 		auto &expr_get = plan->Cast<LogicalExpressionGet>();
@@ -519,7 +568,7 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 			throw InternalException("Flatten dependent joins - logical get encountered without children");
 		}
 		plan->children[0] = PushDownDependentJoin(std::move(plan->children[0]));
-		for (idx_t i = 0; i < (perform_delim ? correlated_columns.size() : 1); i++) {
+		for (idx_t i = 0; i < correlated_columns.size(); i++) {
 			get.projected_input.push_back(this->delim_offset + i);
 		}
 		this->delim_offset = get.returned_types.size();
@@ -527,7 +576,10 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		return plan;
 	}
 	case LogicalOperatorType::LOGICAL_RECURSIVE_CTE: {
-		throw BinderException("Recursive CTEs not supported in correlated subquery");
+		throw BinderException("Recursive CTEs not (yet) supported in correlated subquery");
+	}
+	case LogicalOperatorType::LOGICAL_MATERIALIZED_CTE: {
+		throw BinderException("Materialized CTEs not (yet) supported in correlated subquery");
 	}
 	case LogicalOperatorType::LOGICAL_DELIM_JOIN: {
 		throw BinderException("Nested lateral joins or lateral joins in correlated subqueries are not (yet) supported");

package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp

@@ -7,8 +7,10 @@
 
 namespace duckdb {
 
-HasCorrelatedExpressions::HasCorrelatedExpressions(const vector<CorrelatedColumnInfo> &correlated, bool lateral)
-    : has_correlated_expressions(false), lateral(lateral), correlated_columns(correlated) {
+HasCorrelatedExpressions::HasCorrelatedExpressions(const vector<CorrelatedColumnInfo> &correlated, bool lateral,
+                                                   idx_t lateral_depth)
+    : has_correlated_expressions(false), lateral(lateral), correlated_columns(correlated),
+      lateral_depth(lateral_depth) {
 }
 
 void HasCorrelatedExpressions::VisitOperator(LogicalOperator &op) {
@@ -17,18 +19,31 @@ void HasCorrelatedExpressions::VisitOperator(LogicalOperator &op) {
 
 unique_ptr<Expression> HasCorrelatedExpressions::VisitReplace(BoundColumnRefExpression &expr,
                                                               unique_ptr<Expression> *expr_ptr) {
-	if (expr.depth == 0) {
+	// Indicates local correlations (all correlations within a child) for the root
+	if (expr.depth <= lateral_depth) {
 		return nullptr;
 	}
-	if (expr.depth > 1) {
+
+	// Should never happen
+	if (expr.depth > 1 + lateral_depth) {
 		if (lateral) {
-			throw BinderException("Nested lateral joins are not (yet) supported");
+			throw BinderException("Invalid lateral depth encountered for an expression");
 		}
 		throw InternalException("Expression with depth > 1 detected in non-lateral join");
 	}
+	// Note: This is added, since we only want to set has_correlated_expressions to true when the
+	// BoundSubqueryExpression has the same bindings as one of the correlated_columns from the left hand side
+	// (correlated_columns is the correlated_columns from left hand side)
+	bool found_match = false;
+	for (idx_t i = 0; i < correlated_columns.size(); i++) {
+		if (correlated_columns[i].binding == expr.binding) {
+			found_match = true;
+			break;
+		}
+	}
 	// correlated column reference
-	D_ASSERT(expr.depth == 1);
-	has_correlated_expressions = true;
+	D_ASSERT(expr.depth == lateral_depth + 1);
+	has_correlated_expressions = found_match;
 	return nullptr;
 }
 

package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp

@@ -5,34 +5,67 @@
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
 #include "duckdb/planner/expression/bound_operator_expression.hpp"
 #include "duckdb/planner/expression/bound_subquery_expression.hpp"
+#include "duckdb/planner/query_node/bound_select_node.hpp"
 #include "duckdb/planner/expression_iterator.hpp"
+#include "duckdb/planner/tableref/bound_joinref.hpp"
+#include "duckdb/planner/operator/logical_dependent_join.hpp"
 
 namespace duckdb {
 
 RewriteCorrelatedExpressions::RewriteCorrelatedExpressions(ColumnBinding base_binding,
-                                                           column_binding_map_t<idx_t> &correlated_map)
-    : base_binding(base_binding), correlated_map(correlated_map) {
+                                                           column_binding_map_t<idx_t> &correlated_map,
+                                                           idx_t lateral_depth, bool recursive_rewrite)
+    : base_binding(base_binding), correlated_map(correlated_map), lateral_depth(lateral_depth),
+      recursive_rewrite(recursive_rewrite) {
 }
 
 void RewriteCorrelatedExpressions::VisitOperator(LogicalOperator &op) {
+	if (recursive_rewrite) {
+		// Update column bindings from left child of lateral to right child
+		if (op.type == LogicalOperatorType::LOGICAL_DEPENDENT_JOIN) {
+			D_ASSERT(op.children.size() == 2);
+			VisitOperator(*op.children[0]);
+			lateral_depth++;
+			VisitOperator(*op.children[1]);
+			lateral_depth--;
+		} else {
+			VisitOperatorChildren(op);
+		}
+	}
+	// update the bindings in the correlated columns of the dependendent join
+	if (op.type == LogicalOperatorType::LOGICAL_DEPENDENT_JOIN) {
+		auto &plan = op.Cast<LogicalDependentJoin>();
+		for (auto &corr : plan.correlated_columns) {
+			auto entry = correlated_map.find(corr.binding);
+			if (entry != correlated_map.end()) {
+				corr.binding = ColumnBinding(base_binding.table_index, base_binding.column_index + entry->second);
+			}
+		}
+	}
 	VisitOperatorExpressions(op);
 }
 
 unique_ptr<Expression> RewriteCorrelatedExpressions::VisitReplace(BoundColumnRefExpression &expr,
                                                                   unique_ptr<Expression> *expr_ptr) {
-	if (expr.depth == 0) {
+	if (expr.depth <= lateral_depth) {
+		// Indicates local correlations not relevant for the current the rewrite
 		return nullptr;
 	}
 	// correlated column reference
 	// replace with the entry referring to the duplicate eliminated scan
-	// if this assertion occurs it generally means the correlated expressions were not propagated correctly
-	// through different binders
-	D_ASSERT(expr.depth == 1);
+	// if this assertion occurs it generally means the bindings are inappropriate set in the binder or
+	// we either missed to account for lateral binder or over-counted for the lateral binder
+	D_ASSERT(expr.depth == 1 + lateral_depth);
 	auto entry = correlated_map.find(expr.binding);
 	D_ASSERT(entry != correlated_map.end());
 
 	expr.binding = ColumnBinding(base_binding.table_index, base_binding.column_index + entry->second);
-	expr.depth = 0;
+	if (recursive_rewrite) {
+		D_ASSERT(expr.depth > 1);
+		expr.depth--;
+	} else {
+		expr.depth = 0;
+	}
 	return nullptr;
 }
 
@@ -53,6 +86,21 @@ RewriteCorrelatedExpressions::RewriteCorrelatedRecursive::RewriteCorrelatedRecur
     : parent(parent), base_binding(base_binding), correlated_map(correlated_map) {
 }
 
+void RewriteCorrelatedExpressions::RewriteCorrelatedRecursive::RewriteJoinRefRecursive(BoundTableRef &ref) {
+	// recursively rewrite bindings in the correlated columns for the table ref and all the children
+	if (ref.type == TableReferenceType::JOIN) {
+		auto &bound_join = ref.Cast<BoundJoinRef>();
+		for (auto &corr : bound_join.correlated_columns) {
+			auto entry = correlated_map.find(corr.binding);
+			if (entry != correlated_map.end()) {
+				corr.binding = ColumnBinding(base_binding.table_index, base_binding.column_index + entry->second);
+			}
+		}
+		RewriteJoinRefRecursive(*bound_join.left);
+		RewriteJoinRefRecursive(*bound_join.right);
+	}
+}
+
 void RewriteCorrelatedExpressions::RewriteCorrelatedRecursive::RewriteCorrelatedSubquery(
     BoundSubqueryExpression &expr) {
 	// rewrite the binding in the correlated list of the subquery)
@@ -62,6 +110,16 @@ void RewriteCorrelatedExpressions::RewriteCorrelatedRecursive::RewriteCorrelated
 			corr.binding = ColumnBinding(base_binding.table_index, base_binding.column_index + entry->second);
 		}
 	}
+	// TODO: Cleanup and find a better way to do this
+	auto &node = *expr.subquery;
+	if (node.type == QueryNodeType::SELECT_NODE) {
+		// Found an unplanned select node, need to update column bindings correlated columns in the from tables
+		auto &bound_select = node.Cast<BoundSelectNode>();
+		if (bound_select.from_table) {
+			BoundTableRef &table_ref = *bound_select.from_table;
+			RewriteJoinRefRecursive(table_ref);
+		}
+	}
 	// now rewrite any correlated BoundColumnRef expressions inside the subquery
 	ExpressionIterator::EnumerateQueryNodeChildren(*expr.subquery,
 	                                               [&](Expression &child) { RewriteCorrelatedExpressions(child); });

package/src/duckdb/src/storage/arena_allocator.cpp

@@ -109,7 +109,6 @@ data_ptr_t ArenaAllocator::ReallocateAligned(data_ptr_t pointer, idx_t old_size,
 }
 
 void ArenaAllocator::Reset() {
-
 	if (head) {
 		// destroy all chunks except the current one
 		if (head->next) {
@@ -148,8 +147,20 @@ ArenaChunk *ArenaAllocator::GetTail() {
 	return tail;
 }
 
-bool ArenaAllocator::IsEmpty() {
+bool ArenaAllocator::IsEmpty() const {
 	return head == nullptr;
 }
 
+idx_t ArenaAllocator::SizeInBytes() const {
+	idx_t total_size = 0;
+	if (!IsEmpty()) {
+		auto current = head.get();
+		while (current != nullptr) {
+			total_size += current->current_position;
+			current = current->next.get();
+		}
+	}
+	return total_size;
+}
+
 } // namespace duckdb

package/src/duckdb/src/storage/buffer/block_manager.cpp

@@ -2,10 +2,15 @@
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/storage/buffer/block_handle.hpp"
 #include "duckdb/storage/buffer/buffer_pool.hpp"
+#include "duckdb/storage/metadata/metadata_manager.hpp"
 
 namespace duckdb {
 
-shared_ptr<BlockHandle> BlockManager::RegisterBlock(block_id_t block_id, bool is_meta_block) {
+BlockManager::BlockManager(BufferManager &buffer_manager)
+    : buffer_manager(buffer_manager), metadata_manager(make_uniq<MetadataManager>(*this, buffer_manager)) {
+}
+
+shared_ptr<BlockHandle> BlockManager::RegisterBlock(block_id_t block_id) {
 	lock_guard<mutex> lock(blocks_lock);
 	// check if the block already exists
 	auto entry = blocks.find(block_id);
@@ -19,19 +24,11 @@ shared_ptr<BlockHandle> BlockManager::RegisterBlock(block_id_t block_id, bool is
 	}
 	// create a new block pointer for this block
 	auto result = make_shared<BlockHandle>(*this, block_id);
-	// for meta block, cache the handle in meta_blocks
-	if (is_meta_block) {
-		meta_blocks[block_id] = result;
-	}
 	// register the block pointer in the set of blocks as a weak pointer
 	blocks[block_id] = weak_ptr<BlockHandle>(result);
 	return result;
 }
 
-void BlockManager::ClearMetaBlockHandles() {
-	meta_blocks.clear();
-}
-
 shared_ptr<BlockHandle> BlockManager::ConvertToPersistent(block_id_t block_id, shared_ptr<BlockHandle> old_block) {
 	// pin the old block to ensure we have it loaded in memory
 	auto old_handle = buffer_manager.Pin(old_block);
@@ -79,4 +76,11 @@ void BlockManager::UnregisterBlock(block_id_t block_id, bool can_destroy) {
 	}
 }
 
+MetadataManager &BlockManager::GetMetadataManager() {
+	return *metadata_manager;
+}
+
+void BlockManager::Truncate() {
+}
+
 } // namespace duckdb

package/src/duckdb/src/storage/checkpoint/row_group_writer.cpp

@@ -36,7 +36,7 @@ void SingleFileRowGroupWriter::WriteColumnDataPointers(ColumnCheckpointState &co
 	}
 }
 
-MetaBlockWriter &SingleFileRowGroupWriter::GetPayloadWriter() {
+MetadataWriter &SingleFileRowGroupWriter::GetPayloadWriter() {
 	return table_data_writer;
 }
 

package/src/duckdb/src/storage/checkpoint/table_data_reader.cpp

@@ -1,5 +1,5 @@
 #include "duckdb/storage/checkpoint/table_data_reader.hpp"
-#include "duckdb/storage/meta_block_reader.hpp"
+#include "duckdb/storage/metadata/metadata_reader.hpp"
 #include "duckdb/common/types/null_value.hpp"
 
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
@@ -10,7 +10,7 @@
 
 namespace duckdb {
 
-TableDataReader::TableDataReader(MetaBlockReader &reader, BoundCreateTableInfo &info) : reader(reader), info(info) {
+TableDataReader::TableDataReader(MetadataReader &reader, BoundCreateTableInfo &info) : reader(reader), info(info) {
 	info.data = make_uniq<PersistentTableData>(info.Base().columns.LogicalColumnCount());
 }
 
@@ -23,8 +23,7 @@ void TableDataReader::ReadTableData() {
 
 	// deserialize each of the individual row groups
 	info.data->row_group_count = reader.Read<uint64_t>();
-	info.data->block_id = reader.block->BlockId();
-	info.data->offset = reader.offset;
+	info.data->block_pointer = reader.GetMetaBlockPointer();
 }
 
 } // namespace duckdb

package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp

@@ -31,8 +31,8 @@ void TableDataWriter::AddRowGroup(RowGroupPointer &&row_group_pointer, unique_pt
 }
 
 SingleFileTableDataWriter::SingleFileTableDataWriter(SingleFileCheckpointWriter &checkpoint_manager,
-                                                     TableCatalogEntry &table, MetaBlockWriter &table_data_writer,
-                                                     MetaBlockWriter &meta_data_writer)
+                                                     TableCatalogEntry &table, MetadataWriter &table_data_writer,
+                                                     MetadataWriter &meta_data_writer)
     : TableDataWriter(table), checkpoint_manager(checkpoint_manager), table_data_writer(table_data_writer),
       meta_data_writer(meta_data_writer) {
 }
@@ -44,7 +44,7 @@ unique_ptr<RowGroupWriter> SingleFileTableDataWriter::GetRowGroupWriter(RowGroup
 void SingleFileTableDataWriter::FinalizeTable(TableStatistics &&global_stats, DataTableInfo *info) {
 	// store the current position in the metadata writer
 	// this is where the row groups for this table start
-	auto pointer = table_data_writer.GetBlockPointer();
+	auto pointer = table_data_writer.GetMetaBlockPointer();
 
 	global_stats.Serialize(table_data_writer);
 
@@ -60,18 +60,18 @@ void SingleFileTableDataWriter::FinalizeTable(TableStatistics &&global_stats, Da
 	}
 
 	// Pointer to the table itself goes to the metadata stream.
-	meta_data_writer.Write<block_id_t>(pointer.block_id);
+	meta_data_writer.Write<idx_t>(pointer.block_pointer);
 	meta_data_writer.Write<uint64_t>(pointer.offset);
 	meta_data_writer.Write<idx_t>(total_rows);
 
 	// Now we serialize indexes in the table_metadata_writer
-	vector<BlockPointer> index_pointers = info->indexes.SerializeIndexes(table_data_writer);
+	auto index_pointers = info->indexes.SerializeIndexes(table_data_writer);
 
 	// Write-off to metadata block ids and offsets of indexes
 	meta_data_writer.Write<idx_t>(index_pointers.size());
 	for (auto &block_info : index_pointers) {
-		meta_data_writer.Write<idx_t>(block_info.block_id);
-		meta_data_writer.Write<idx_t>(block_info.offset);
+		meta_data_writer.Write<block_id_t>(block_info.block_id);
+		meta_data_writer.Write<uint32_t>(block_info.offset);
 	}
 }