duckdb 0.8.2-dev145.0 → 0.8.2-dev1493.0

package/src/duckdb/src/planner/operator/logical_get.cpp

@@ -14,7 +14,8 @@ namespace duckdb {
 LogicalGet::LogicalGet(idx_t table_index, TableFunction function, unique_ptr<FunctionData> bind_data,
                        vector<LogicalType> returned_types, vector<string> returned_names)
     : LogicalOperator(LogicalOperatorType::LOGICAL_GET), table_index(table_index), function(std::move(function)),
-      bind_data(std::move(bind_data)), returned_types(std::move(returned_types)), names(std::move(returned_names)) {
+      bind_data(std::move(bind_data)), returned_types(std::move(returned_types)), names(std::move(returned_names)),
+      extra_info() {
 }
 
 optional_ptr<TableCatalogEntry> LogicalGet::GetTable() const {
@@ -22,7 +23,7 @@ optional_ptr<TableCatalogEntry> LogicalGet::GetTable() const {
 }
 
 string LogicalGet::ParamsToString() const {
-	string result;
+	string result = "";
 	for (auto &kv : table_filters.filters) {
 		auto &column_index = kv.first;
 		auto &filter = kv.second;
@@ -31,10 +32,14 @@ string LogicalGet::ParamsToString() const {
 		}
 		result += "\n";
 	}
+	if (!extra_info.file_filters.empty()) {
+		result += "\n[INFOSEPARATOR]\n";
+		result += "File Filters: " + extra_info.file_filters;
+	}
 	if (!function.to_string) {
-		return string();
+		return result;
 	}
-	return function.to_string(bind_data.get());
+	return result + "\n" + function.to_string(bind_data.get());
 }
 
 vector<ColumnBinding> LogicalGet::GetColumnBindings() {

package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp

@@ -0,0 +1,21 @@
+#include "duckdb/planner/operator/logical_materialized_cte.hpp"
+
+#include "duckdb/common/field_writer.hpp"
+
+namespace duckdb {
+
+void LogicalMaterializedCTE::Serialize(FieldWriter &writer) const {
+	writer.WriteField(table_index);
+}
+
+unique_ptr<LogicalOperator> LogicalMaterializedCTE::Deserialize(LogicalDeserializationState &state,
+                                                                FieldReader &reader) {
+	auto table_index = reader.ReadRequired<idx_t>();
+	return unique_ptr<LogicalMaterializedCTE>(new LogicalMaterializedCTE(table_index));
+}
+
+vector<idx_t> LogicalMaterializedCTE::GetTableIndex() const {
+	return vector<idx_t> {table_index};
+}
+
+} // namespace duckdb

package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp

@@ -10,6 +10,7 @@
 #include "duckdb/planner/operator/list.hpp"
 #include "duckdb/planner/subquery/has_correlated_expressions.hpp"
 #include "duckdb/planner/subquery/rewrite_correlated_expressions.hpp"
+#include "duckdb/planner/operator/logical_dependent_join.hpp"
 
 namespace duckdb {
 
@@ -24,20 +25,32 @@ FlattenDependentJoins::FlattenDependentJoins(Binder &binder, const vector<Correl
 	}
 }
 
-bool FlattenDependentJoins::DetectCorrelatedExpressions(LogicalOperator *op, bool lateral) {
+bool FlattenDependentJoins::DetectCorrelatedExpressions(LogicalOperator *op, bool lateral, idx_t lateral_depth) {
+
+	bool is_lateral_join = false;
+
 	D_ASSERT(op);
 	// check if this entry has correlated expressions
-	HasCorrelatedExpressions visitor(correlated_columns, lateral);
+	if (op->type == LogicalOperatorType::LOGICAL_DEPENDENT_JOIN) {
+		is_lateral_join = true;
+	}
+	HasCorrelatedExpressions visitor(correlated_columns, lateral, lateral_depth);
 	visitor.VisitOperator(*op);
 	bool has_correlation = visitor.has_correlated_expressions;
+	int child_idx = 0;
 	// now visit the children of this entry and check if they have correlated expressions
 	for (auto &child : op->children) {
+		auto new_lateral_depth = lateral_depth;
+		if (is_lateral_join && child_idx == 1) {
+			new_lateral_depth = lateral_depth + 1;
+		}
 		// we OR the property with its children such that has_correlation is true if either
 		// (1) this node has a correlated expression or
 		// (2) one of its children has a correlated expression
-		if (DetectCorrelatedExpressions(child.get(), lateral)) {
+		if (DetectCorrelatedExpressions(child.get(), lateral, new_lateral_depth)) {
 			has_correlation = true;
 		}
+		child_idx++;
 	}
 	// set the entry in the map
 	has_correlated_expressions[op] = has_correlation;
@@ -46,7 +59,7 @@ bool FlattenDependentJoins::DetectCorrelatedExpressions(LogicalOperator *op, boo
 
 unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoin(unique_ptr<LogicalOperator> plan) {
 	bool propagate_null_values = true;
-	auto result = PushDownDependentJoinInternal(std::move(plan), propagate_null_values);
+	auto result = PushDownDependentJoinInternal(std::move(plan), propagate_null_values, 0);
 	if (!replacement_map.empty()) {
 		// check if we have to replace any COUNT aggregates into "CASE WHEN X IS NULL THEN 0 ELSE COUNT END"
 		RewriteCountAggregates aggr(replacement_map);
@@ -70,8 +83,10 @@ bool SubqueryDependentFilter(Expression *expr) {
 	}
 	return false;
 }
+
 unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal(unique_ptr<LogicalOperator> plan,
-                                                                                 bool &parent_propagate_null_values) {
+                                                                                 bool &parent_propagate_null_values,
+                                                                                 idx_t lateral_depth) {
 	// first check if the logical operator has correlated expressions
 	auto entry = has_correlated_expressions.find(plan.get());
 	D_ASSERT(entry != has_correlated_expressions.end());
@@ -95,10 +110,11 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		for (auto &expr : plan->expressions) {
 			any_join |= SubqueryDependentFilter(expr.get());
 		}
-		plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+		plan->children[0] =
+		    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
 
 		// then we replace any correlated expressions with the corresponding entry in the correlated_map
-		RewriteCorrelatedExpressions rewriter(base_binding, correlated_map);
+		RewriteCorrelatedExpressions rewriter(base_binding, correlated_map, lateral_depth);
 		rewriter.VisitOperator(*plan);
 		return plan;
 	}
@@ -108,10 +124,11 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		for (auto &expr : plan->expressions) {
 			parent_propagate_null_values &= expr->PropagatesNullValues();
 		}
-		plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+		plan->children[0] =
+		    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
 
 		// then we replace any correlated expressions with the corresponding entry in the correlated_map
-		RewriteCorrelatedExpressions rewriter(base_binding, correlated_map);
+		RewriteCorrelatedExpressions rewriter(base_binding, correlated_map, lateral_depth);
 		rewriter.VisitOperator(*plan);
 		// now we add all the columns of the delim_scan to the projection list
 		auto &proj = plan->Cast<LogicalProjection>();
@@ -134,9 +151,10 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		for (auto &expr : plan->expressions) {
 			parent_propagate_null_values &= expr->PropagatesNullValues();
 		}
-		plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+		plan->children[0] =
+		    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
 		// then we replace any correlated expressions with the corresponding entry in the correlated_map
-		RewriteCorrelatedExpressions rewriter(base_binding, correlated_map);
+		RewriteCorrelatedExpressions rewriter(base_binding, correlated_map, lateral_depth);
 		rewriter.VisitOperator(*plan);
 		// now we add all the columns of the delim_scan to the grouping operators AND the projection list
 		idx_t delim_table_index;
@@ -231,22 +249,24 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		bool right_has_correlation = has_correlated_expressions.find(plan->children[1].get())->second;
 		if (!right_has_correlation) {
 			// only left has correlation: push into left
-			plan->children[0] =
-			    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+			plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]),
+			                                                  parent_propagate_null_values, lateral_depth);
 			return plan;
 		}
 		if (!left_has_correlation) {
 			// only right has correlation: push into right
-			plan->children[1] =
-			    PushDownDependentJoinInternal(std::move(plan->children[1]), parent_propagate_null_values);
+			plan->children[1] = PushDownDependentJoinInternal(std::move(plan->children[1]),
+			                                                  parent_propagate_null_values, lateral_depth);
 			return plan;
 		}
 		// both sides have correlation
 		// turn into an inner join
 		auto join = make_uniq<LogicalComparisonJoin>(JoinType::INNER);
-		plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+		plan->children[0] =
+		    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
 		auto left_binding = this->base_binding;
-		plan->children[1] = PushDownDependentJoinInternal(std::move(plan->children[1]), parent_propagate_null_values);
+		plan->children[1] =
+		    PushDownDependentJoinInternal(std::move(plan->children[1]), parent_propagate_null_values, lateral_depth);
 		// add the correlated columns to the join conditions
 		for (idx_t i = 0; i < correlated_columns.size(); i++) {
 			JoinCondition cond;
@@ -261,6 +281,26 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		join->children.push_back(std::move(plan->children[1]));
 		return std::move(join);
 	}
+	case LogicalOperatorType::LOGICAL_DEPENDENT_JOIN: {
+		auto &dependent_join = plan->Cast<LogicalJoin>();
+		if (!((dependent_join.join_type == JoinType::INNER) || (dependent_join.join_type == JoinType::LEFT))) {
+			throw Exception("Dependent join can only be INNER or LEFT type");
+		}
+		D_ASSERT(plan->children.size() == 2);
+		// Push all the bindings down to the left side so the right side knows where to refer DELIM_GET from
+		plan->children[0] =
+		    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
+
+		// Normal rewriter like in other joins
+		RewriteCorrelatedExpressions rewriter(this->base_binding, correlated_map, lateral_depth);
+		rewriter.VisitOperator(*plan);
+
+		// Recursive rewriter to visit right side of lateral join and update bindings from left
+		RewriteCorrelatedExpressions recursive_rewriter(this->base_binding, correlated_map, lateral_depth + 1, true);
+		recursive_rewriter.VisitOperator(*plan->children[1]);
+
+		return plan;
+	}
 	case LogicalOperatorType::LOGICAL_ANY_JOIN:
 	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: {
@@ -274,30 +314,33 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 			// inner join
 			if (!right_has_correlation) {
 				// only left has correlation: push into left
-				plan->children[0] =
-				    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+				plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]),
+				                                                  parent_propagate_null_values, lateral_depth);
+				// Remove the correlated columns coming from outside for current join node
 				return plan;
 			}
 			if (!left_has_correlation) {
 				// only right has correlation: push into right
-				plan->children[1] =
-				    PushDownDependentJoinInternal(std::move(plan->children[1]), parent_propagate_null_values);
+				plan->children[1] = PushDownDependentJoinInternal(std::move(plan->children[1]),
+				                                                  parent_propagate_null_values, lateral_depth);
+				// Remove the correlated columns coming from outside for current join node
 				return plan;
 			}
 		} else if (join.join_type == JoinType::LEFT) {
 			// left outer join
 			if (!right_has_correlation) {
 				// only left has correlation: push into left
-				plan->children[0] =
-				    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+				plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]),
+				                                                  parent_propagate_null_values, lateral_depth);
+				// Remove the correlated columns coming from outside for current join node
 				return plan;
 			}
 		} else if (join.join_type == JoinType::RIGHT) {
 			// left outer join
 			if (!left_has_correlation) {
 				// only right has correlation: push into right
-				plan->children[1] =
-				    PushDownDependentJoinInternal(std::move(plan->children[1]), parent_propagate_null_values);
+				plan->children[1] = PushDownDependentJoinInternal(std::move(plan->children[1]),
+				                                                  parent_propagate_null_values, lateral_depth);
 				return plan;
 			}
 		} else if (join.join_type == JoinType::MARK) {
@@ -305,10 +348,10 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 				throw Exception("MARK join with correlation in RHS not supported");
 			}
 			// push the child into the LHS
-			plan->children[0] =
-			    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+			plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]),
+			                                                  parent_propagate_null_values, lateral_depth);
 			// rewrite expressions in the join conditions
-			RewriteCorrelatedExpressions rewriter(base_binding, correlated_map);
+			RewriteCorrelatedExpressions rewriter(base_binding, correlated_map, lateral_depth);
 			rewriter.VisitOperator(*plan);
 			return plan;
 		} else {
@@ -316,9 +359,11 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		}
 		// both sides have correlation
 		// push into both sides
-		plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+		plan->children[0] =
+		    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
 		auto left_binding = this->base_binding;
-		plan->children[1] = PushDownDependentJoinInternal(std::move(plan->children[1]), parent_propagate_null_values);
+		plan->children[1] =
+		    PushDownDependentJoinInternal(std::move(plan->children[1]), parent_propagate_null_values, lateral_depth);
 		auto right_binding = this->base_binding;
 		// NOTE: for OUTER JOINS it matters what the BASE BINDING is after the join
 		// for the LEFT OUTER JOIN, we want the LEFT side to be the base binding after we push
@@ -354,7 +399,7 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 			}
 		}
 		// then we replace any correlated expressions with the corresponding entry in the correlated_map
-		RewriteCorrelatedExpressions rewriter(right_binding, correlated_map);
+		RewriteCorrelatedExpressions rewriter(right_binding, correlated_map, lateral_depth);
 		rewriter.VisitOperator(*plan);
 		return plan;
 	}
@@ -371,9 +416,11 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		// this is done for an optimization to avoid having to compute the total order
 		if (plan->children[0]->type == LogicalOperatorType::LOGICAL_ORDER_BY) {
 			order_by = unique_ptr_cast<LogicalOperator, LogicalOrder>(std::move(plan->children[0]));
-			child = PushDownDependentJoinInternal(std::move(order_by->children[0]), parent_propagate_null_values);
+			child = PushDownDependentJoinInternal(std::move(order_by->children[0]), parent_propagate_null_values,
+			                                      lateral_depth);
 		} else {
-			child = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+			child = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values,
+			                                      lateral_depth);
 		}
 		auto child_column_count = child->GetColumnBindings().size();
 		// we push a row_number() OVER (PARTITION BY [correlated columns])
@@ -437,7 +484,8 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 	case LogicalOperatorType::LOGICAL_WINDOW: {
 		auto &window = plan->Cast<LogicalWindow>();
 		// push into children
-		plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+		plan->children[0] =
+		    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
 		// add the correlated columns to the PARTITION BY clauses in the Window
 		for (auto &expr : window.expressions) {
 			D_ASSERT(expr->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);
@@ -488,9 +536,10 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 	case LogicalOperatorType::LOGICAL_EXPRESSION_GET: {
 		// expression get
 		// first we flatten the dependent join in the child
-		plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values);
+		plan->children[0] =
+		    PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth);
 		// then we replace any correlated expressions with the corresponding entry in the correlated_map
-		RewriteCorrelatedExpressions rewriter(base_binding, correlated_map);
+		RewriteCorrelatedExpressions rewriter(base_binding, correlated_map, lateral_depth);
 		rewriter.VisitOperator(*plan);
 		// now we add all the correlated columns to each of the expressions of the expression scan
 		auto &expr_get = plan->Cast<LogicalExpressionGet>();
@@ -519,7 +568,7 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 			throw InternalException("Flatten dependent joins - logical get encountered without children");
 		}
 		plan->children[0] = PushDownDependentJoin(std::move(plan->children[0]));
-		for (idx_t i = 0; i < (perform_delim ? correlated_columns.size() : 1); i++) {
+		for (idx_t i = 0; i < correlated_columns.size(); i++) {
 			get.projected_input.push_back(this->delim_offset + i);
 		}
 		this->delim_offset = get.returned_types.size();
@@ -527,7 +576,10 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		return plan;
 	}
 	case LogicalOperatorType::LOGICAL_RECURSIVE_CTE: {
-		throw BinderException("Recursive CTEs not supported in correlated subquery");
+		throw BinderException("Recursive CTEs not (yet) supported in correlated subquery");
+	}
+	case LogicalOperatorType::LOGICAL_MATERIALIZED_CTE: {
+		throw BinderException("Materialized CTEs not (yet) supported in correlated subquery");
 	}
 	case LogicalOperatorType::LOGICAL_DELIM_JOIN: {
 		throw BinderException("Nested lateral joins or lateral joins in correlated subqueries are not (yet) supported");

package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp

@@ -7,8 +7,10 @@
 
 namespace duckdb {
 
-HasCorrelatedExpressions::HasCorrelatedExpressions(const vector<CorrelatedColumnInfo> &correlated, bool lateral)
-    : has_correlated_expressions(false), lateral(lateral), correlated_columns(correlated) {
+HasCorrelatedExpressions::HasCorrelatedExpressions(const vector<CorrelatedColumnInfo> &correlated, bool lateral,
+                                                   idx_t lateral_depth)
+    : has_correlated_expressions(false), lateral(lateral), correlated_columns(correlated),
+      lateral_depth(lateral_depth) {
 }
 
 void HasCorrelatedExpressions::VisitOperator(LogicalOperator &op) {
@@ -17,18 +19,31 @@ void HasCorrelatedExpressions::VisitOperator(LogicalOperator &op) {
 
 unique_ptr<Expression> HasCorrelatedExpressions::VisitReplace(BoundColumnRefExpression &expr,
                                                               unique_ptr<Expression> *expr_ptr) {
-	if (expr.depth == 0) {
+	// Indicates local correlations (all correlations within a child) for the root
+	if (expr.depth <= lateral_depth) {
 		return nullptr;
 	}
-	if (expr.depth > 1) {
+
+	// Should never happen
+	if (expr.depth > 1 + lateral_depth) {
 		if (lateral) {
-			throw BinderException("Nested lateral joins are not (yet) supported");
+			throw BinderException("Invalid lateral depth encountered for an expression");
 		}
 		throw InternalException("Expression with depth > 1 detected in non-lateral join");
 	}
+	// Note: This is added, since we only want to set has_correlated_expressions to true when the
+	// BoundSubqueryExpression has the same bindings as one of the correlated_columns from the left hand side
+	// (correlated_columns is the correlated_columns from left hand side)
+	bool found_match = false;
+	for (idx_t i = 0; i < correlated_columns.size(); i++) {
+		if (correlated_columns[i].binding == expr.binding) {
+			found_match = true;
+			break;
+		}
+	}
 	// correlated column reference
-	D_ASSERT(expr.depth == 1);
-	has_correlated_expressions = true;
+	D_ASSERT(expr.depth == lateral_depth + 1);
+	has_correlated_expressions = found_match;
 	return nullptr;
 }
 

package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp

@@ -5,34 +5,67 @@
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
 #include "duckdb/planner/expression/bound_operator_expression.hpp"
 #include "duckdb/planner/expression/bound_subquery_expression.hpp"
+#include "duckdb/planner/query_node/bound_select_node.hpp"
 #include "duckdb/planner/expression_iterator.hpp"
+#include "duckdb/planner/tableref/bound_joinref.hpp"
+#include "duckdb/planner/operator/logical_dependent_join.hpp"
 
 namespace duckdb {
 
 RewriteCorrelatedExpressions::RewriteCorrelatedExpressions(ColumnBinding base_binding,
-                                                           column_binding_map_t<idx_t> &correlated_map)
-    : base_binding(base_binding), correlated_map(correlated_map) {
+                                                           column_binding_map_t<idx_t> &correlated_map,
+                                                           idx_t lateral_depth, bool recursive_rewrite)
+    : base_binding(base_binding), correlated_map(correlated_map), lateral_depth(lateral_depth),
+      recursive_rewrite(recursive_rewrite) {
 }
 
 void RewriteCorrelatedExpressions::VisitOperator(LogicalOperator &op) {
+	if (recursive_rewrite) {
+		// Update column bindings from left child of lateral to right child
+		if (op.type == LogicalOperatorType::LOGICAL_DEPENDENT_JOIN) {
+			D_ASSERT(op.children.size() == 2);
+			VisitOperator(*op.children[0]);
+			lateral_depth++;
+			VisitOperator(*op.children[1]);
+			lateral_depth--;
+		} else {
+			VisitOperatorChildren(op);
+		}
+	}
+	// update the bindings in the correlated columns of the dependendent join
+	if (op.type == LogicalOperatorType::LOGICAL_DEPENDENT_JOIN) {
+		auto &plan = op.Cast<LogicalDependentJoin>();
+		for (auto &corr : plan.correlated_columns) {
+			auto entry = correlated_map.find(corr.binding);
+			if (entry != correlated_map.end()) {
+				corr.binding = ColumnBinding(base_binding.table_index, base_binding.column_index + entry->second);
+			}
+		}
+	}
 	VisitOperatorExpressions(op);
 }
 
 unique_ptr<Expression> RewriteCorrelatedExpressions::VisitReplace(BoundColumnRefExpression &expr,
                                                                   unique_ptr<Expression> *expr_ptr) {
-	if (expr.depth == 0) {
+	if (expr.depth <= lateral_depth) {
+		// Indicates local correlations not relevant for the current the rewrite
 		return nullptr;
 	}
 	// correlated column reference
 	// replace with the entry referring to the duplicate eliminated scan
-	// if this assertion occurs it generally means the correlated expressions were not propagated correctly
-	// through different binders
-	D_ASSERT(expr.depth == 1);
+	// if this assertion occurs it generally means the bindings are inappropriate set in the binder or
+	// we either missed to account for lateral binder or over-counted for the lateral binder
+	D_ASSERT(expr.depth == 1 + lateral_depth);
 	auto entry = correlated_map.find(expr.binding);
 	D_ASSERT(entry != correlated_map.end());
 
 	expr.binding = ColumnBinding(base_binding.table_index, base_binding.column_index + entry->second);
-	expr.depth = 0;
+	if (recursive_rewrite) {
+		D_ASSERT(expr.depth > 1);
+		expr.depth--;
+	} else {
+		expr.depth = 0;
+	}
 	return nullptr;
 }
 
@@ -53,6 +86,21 @@ RewriteCorrelatedExpressions::RewriteCorrelatedRecursive::RewriteCorrelatedRecur
     : parent(parent), base_binding(base_binding), correlated_map(correlated_map) {
 }
 
+void RewriteCorrelatedExpressions::RewriteCorrelatedRecursive::RewriteJoinRefRecursive(BoundTableRef &ref) {
+	// recursively rewrite bindings in the correlated columns for the table ref and all the children
+	if (ref.type == TableReferenceType::JOIN) {
+		auto &bound_join = ref.Cast<BoundJoinRef>();
+		for (auto &corr : bound_join.correlated_columns) {
+			auto entry = correlated_map.find(corr.binding);
+			if (entry != correlated_map.end()) {
+				corr.binding = ColumnBinding(base_binding.table_index, base_binding.column_index + entry->second);
+			}
+		}
+		RewriteJoinRefRecursive(*bound_join.left);
+		RewriteJoinRefRecursive(*bound_join.right);
+	}
+}
+
 void RewriteCorrelatedExpressions::RewriteCorrelatedRecursive::RewriteCorrelatedSubquery(
     BoundSubqueryExpression &expr) {
 	// rewrite the binding in the correlated list of the subquery)
@@ -62,6 +110,16 @@ void RewriteCorrelatedExpressions::RewriteCorrelatedRecursive::RewriteCorrelated
 			corr.binding = ColumnBinding(base_binding.table_index, base_binding.column_index + entry->second);
 		}
 	}
+	// TODO: Cleanup and find a better way to do this
+	auto &node = *expr.subquery;
+	if (node.type == QueryNodeType::SELECT_NODE) {
+		// Found an unplanned select node, need to update column bindings correlated columns in the from tables
+		auto &bound_select = node.Cast<BoundSelectNode>();
+		if (bound_select.from_table) {
+			BoundTableRef &table_ref = *bound_select.from_table;
+			RewriteJoinRefRecursive(table_ref);
+		}
+	}
 	// now rewrite any correlated BoundColumnRef expressions inside the subquery
 	ExpressionIterator::EnumerateQueryNodeChildren(*expr.subquery,
 	                                               [&](Expression &child) { RewriteCorrelatedExpressions(child); });

package/src/duckdb/src/storage/arena_allocator.cpp

@@ -109,7 +109,6 @@ data_ptr_t ArenaAllocator::ReallocateAligned(data_ptr_t pointer, idx_t old_size,
 }
 
 void ArenaAllocator::Reset() {
-
 	if (head) {
 		// destroy all chunks except the current one
 		if (head->next) {
@@ -148,7 +147,7 @@ ArenaChunk *ArenaAllocator::GetTail() {
 	return tail;
 }
 
-bool ArenaAllocator::IsEmpty() {
+bool ArenaAllocator::IsEmpty() const {
 	return head == nullptr;
 }
 

package/src/duckdb/src/storage/buffer/block_manager.cpp

@@ -79,4 +79,7 @@ void BlockManager::UnregisterBlock(block_id_t block_id, bool can_destroy) {
 	}
 }
 
+void BlockManager::Truncate() {
+}
+
 } // namespace duckdb

package/src/duckdb/src/storage/checkpoint_manager.cpp

@@ -111,6 +111,9 @@ void SingleFileCheckpointWriter::CreateCheckpoint() {
 	// truncate the WAL
 	wal->Truncate(0);
 
+	// truncate the file
+	block_manager.Truncate();
+
 	// mark all blocks written as part of the metadata as modified
 	metadata_writer->MarkWrittenBlocks();
 	table_metadata_writer->MarkWrittenBlocks();

package/src/duckdb/src/storage/compression/rle.cpp

@@ -272,7 +272,6 @@ struct RLEScanState : public SegmentScanState {
 	}
 
 	BufferHandle handle;
-	uint32_t rle_offset;
 	idx_t entry_pos;
 	idx_t position_in_entry;
 	uint32_t rle_count_offset;

package/src/duckdb/src/storage/data_table.cpp

@@ -56,7 +56,7 @@ DataTable::DataTable(AttachedDatabase &db, shared_ptr<TableIOManager> table_io_m
 	row_groups->Verify();
 }
 
-DataTable::DataTable(ClientContext &context, DataTable &parent, ColumnDefinition &new_column, Expression *default_value)
+DataTable::DataTable(ClientContext &context, DataTable &parent, ColumnDefinition &new_column, Expression &default_value)
     : info(parent.info), db(parent.db), is_root(true) {
 	// add the column definitions from this DataTable
 	for (auto &column_def : parent.column_definitions) {

package/src/duckdb/src/storage/local_storage.cpp

@@ -61,11 +61,11 @@ LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &paren
 }
 
 LocalTableStorage::LocalTableStorage(ClientContext &context, DataTable &new_dt, LocalTableStorage &parent,
-                                     ColumnDefinition &new_column, optional_ptr<Expression> default_value)
+                                     ColumnDefinition &new_column, Expression &default_value)
     : table_ref(new_dt), allocator(Allocator::Get(new_dt.db)), deleted_rows(parent.deleted_rows),
       optimistic_writer(new_dt, parent.optimistic_writer), optimistic_writers(std::move(parent.optimistic_writers)),
       merged_storage(parent.merged_storage) {
-	row_groups = parent.row_groups->AddColumn(context, new_column, default_value.get());
+	row_groups = parent.row_groups->AddColumn(context, new_column, default_value);
 	parent.row_groups.reset();
 	indexes.Move(parent.indexes);
 }
@@ -508,7 +508,7 @@ void LocalStorage::MoveStorage(DataTable &old_dt, DataTable &new_dt) {
 }
 
 void LocalStorage::AddColumn(DataTable &old_dt, DataTable &new_dt, ColumnDefinition &new_column,
-                             optional_ptr<Expression> default_value) {
+                             Expression &default_value) {
 	// check if there are any pending appends for the old version of the table
 	auto storage = table_manager.MoveEntry(old_dt);
 	if (!storage) {