duckdb 0.4.1-dev308.0 → 0.4.1-dev318.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.4.1-dev308.0",
+  "version": "0.4.1-dev318.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -39446,6 +39446,12 @@ bool ChunkCollection::Equals(ChunkCollection &other) {
 	if (compare_equals) {
 		return true;
 	}
+	for (auto &type : types) {
+		// sort not supported
+		if (type.InternalType() == PhysicalType::LIST || type.InternalType() == PhysicalType::STRUCT) {
+			return false;
+		}
+	}
 	// if the results are not equal,
 	// sort both chunk collections to ensure the comparison is not order insensitive
 	vector<OrderType> desc(ColumnCount(), OrderType::DESCENDING);
@@ -54373,6 +54379,7 @@ public:
 	//! Turns the JoinCondition into an expression; note that this destroys the JoinCondition as the expression inherits
 	//! the left/right expressions
 	static unique_ptr<Expression> CreateExpression(JoinCondition cond);
+	static unique_ptr<Expression> CreateExpression(vector<JoinCondition> conditions);
 
 public:
 	unique_ptr<Expression> left;
@@ -67309,6 +67316,8 @@ public:
 		return true;
 	}
 
+	static bool IsSupported(const vector<JoinCondition> &conditions);
+
 private:
 	// resolve joins that output max N elements (SEMI, ANTI, MARK)
 	void ResolveSimpleJoin(ExecutionContext &context, DataChunk &input, DataChunk &chunk, OperatorState &state) const;
@@ -67425,6 +67434,16 @@ void PhysicalJoin::ConstructMarkJoinResult(DataChunk &join_keys, DataChunk &left
 	}
 }
 
+bool PhysicalNestedLoopJoin::IsSupported(const vector<JoinCondition> &conditions) {
+	for (auto &cond : conditions) {
+		if (cond.left->return_type.InternalType() == PhysicalType::STRUCT ||
+		    cond.left->return_type.InternalType() == PhysicalType::LIST) {
+			return false;
+		}
+	}
+	return true;
+}
+
 //===--------------------------------------------------------------------===//
 // Sink
 //===--------------------------------------------------------------------===//
@@ -76036,6 +76055,44 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalChunkGet &
 
 
 
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/planner/expression_iterator.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+
+#include <functional>
+
+namespace duckdb {
+class BoundQueryNode;
+class BoundTableRef;
+
+class ExpressionIterator {
+public:
+	static void EnumerateChildren(const Expression &expression,
+	                              const std::function<void(const Expression &child)> &callback);
+	static void EnumerateChildren(Expression &expression, const std::function<void(Expression &child)> &callback);
+	static void EnumerateChildren(Expression &expression,
+	                              const std::function<void(unique_ptr<Expression> &child)> &callback);
+
+	static void EnumerateExpression(unique_ptr<Expression> &expr,
+	                                const std::function<void(Expression &child)> &callback);
+
+	static void EnumerateTableRefChildren(BoundTableRef &ref, const std::function<void(Expression &child)> &callback);
+	static void EnumerateQueryNodeChildren(BoundQueryNode &node,
+	                                       const std::function<void(Expression &child)> &callback);
+};
+
+} // namespace duckdb
+
+
 namespace duckdb {
 
 static bool CanPlanIndexJoin(Transaction &transaction, TableScanBindData *bind_data, PhysicalTableScan &scan) {
@@ -76178,6 +76235,14 @@ void TransformIndexJoin(ClientContext &context, LogicalComparisonJoin &op, Index
 	}
 }
 
+static void RewriteJoinCondition(Expression &expr, idx_t offset) {
+	if (expr.type == ExpressionType::BOUND_REF) {
+		auto &ref = (BoundReferenceExpression &)expr;
+		ref.index += offset;
+	}
+	ExpressionIterator::EnumerateChildren(expr, [&](Expression &child) { RewriteJoinCondition(child, offset); });
+}
+
 unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparisonJoin &op) {
 	// now visit the children
 	D_ASSERT(op.children.size() == 2);
@@ -76263,10 +76328,17 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparison
 			// range join: use piecewise merge join
 			plan = make_unique<PhysicalPiecewiseMergeJoin>(op, move(left), move(right), move(op.conditions),
 			                                               op.join_type, op.estimated_cardinality);
-		} else {
+		} else if (PhysicalNestedLoopJoin::IsSupported(op.conditions)) {
 			// inequality join: use nested loop
 			plan = make_unique<PhysicalNestedLoopJoin>(op, move(left), move(right), move(op.conditions), op.join_type,
 			                                           op.estimated_cardinality);
+		} else {
+			for (auto &cond : op.conditions) {
+				RewriteJoinCondition(*cond.right, left->types.size());
+			}
+			auto condition = JoinCondition::CreateExpression(move(op.conditions));
+			plan = make_unique<PhysicalBlockwiseNLJoin>(op, move(left), move(right), move(condition), op.join_type,
+			                                            op.estimated_cardinality);
 		}
 	}
 	return plan;
@@ -76415,42 +76487,6 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCreateInde
 
 
 
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/planner/expression_iterator.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-
-
-
-
-
-
-#include <functional>
-
-namespace duckdb {
-class BoundQueryNode;
-class BoundTableRef;
-
-class ExpressionIterator {
-public:
-	static void EnumerateChildren(const Expression &expression,
-	                              const std::function<void(const Expression &child)> &callback);
-	static void EnumerateChildren(Expression &expression, const std::function<void(Expression &child)> &callback);
-	static void EnumerateChildren(Expression &expression,
-	                              const std::function<void(unique_ptr<Expression> &child)> &callback);
-
-	static void EnumerateExpression(unique_ptr<Expression> &expr,
-	                                const std::function<void(Expression &child)> &callback);
-
-	static void EnumerateTableRefChildren(BoundTableRef &ref, const std::function<void(Expression &child)> &callback);
-	static void EnumerateQueryNodeChildren(BoundQueryNode &node,
-	                                       const std::function<void(Expression &child)> &callback);
-};
-
-} // namespace duckdb
 
 //===----------------------------------------------------------------------===//
 //                         DuckDB
@@ -155632,6 +155668,7 @@ private:
 	unique_ptr<ConstantExpression> TransformValue(duckdb_libpgquery::PGValue val);
 	//! Transform a Postgres operator into an Expression
 	unique_ptr<ParsedExpression> TransformAExpr(duckdb_libpgquery::PGAExpr *root);
+	unique_ptr<ParsedExpression> TransformAExprInternal(duckdb_libpgquery::PGAExpr *root);
 	//! Transform a Postgres abstract expression into an Expression
 	unique_ptr<ParsedExpression> TransformExpression(duckdb_libpgquery::PGNode *node);
 	//! Transform a Postgres function call into an Expression
@@ -159478,7 +159515,7 @@ unique_ptr<ParsedExpression> Transformer::TransformBinaryOperator(const string &
 	}
 }
 
-unique_ptr<ParsedExpression> Transformer::TransformAExpr(duckdb_libpgquery::PGAExpr *root) {
+unique_ptr<ParsedExpression> Transformer::TransformAExprInternal(duckdb_libpgquery::PGAExpr *root) {
 	D_ASSERT(root);
 	auto name = string((reinterpret_cast<duckdb_libpgquery::PGValue *>(root->name->head->data.ptr_value))->val.str);
 
@@ -159626,6 +159663,14 @@ unique_ptr<ParsedExpression> Transformer::TransformAExpr(duckdb_libpgquery::PGAE
 	}
 }
 
+unique_ptr<ParsedExpression> Transformer::TransformAExpr(duckdb_libpgquery::PGAExpr *root) {
+	auto result = TransformAExprInternal(root);
+	if (result) {
+		result->query_location = root->location;
+	}
+	return result;
+}
+
 } // namespace duckdb
 
 
@@ -166900,7 +166945,8 @@ namespace duckdb {
 //! The FlattenDependentJoins class is responsible for pushing the dependent join down into the plan to create a
 //! flattened subquery
 struct FlattenDependentJoins {
-	FlattenDependentJoins(Binder &binder, const vector<CorrelatedColumnInfo> &correlated, bool any_join = false);
+	FlattenDependentJoins(Binder &binder, const vector<CorrelatedColumnInfo> &correlated, bool perform_delim = true,
+	                      bool any_join = false);
 
 	//! Detects which Logical Operators have correlated expressions that they are dependent upon, filling the
 	//! has_correlated_expressions map.
@@ -166919,6 +166965,7 @@ struct FlattenDependentJoins {
 	const vector<CorrelatedColumnInfo> &correlated_columns;
 	vector<LogicalType> delim_types;
 
+	bool perform_delim;
 	bool any_join;
 
 private:
@@ -166930,6 +166977,9 @@ private:
 
 
 
+
+
+
 namespace duckdb {
 
 static unique_ptr<Expression> PlanUncorrelatedSubquery(Binder &binder, BoundSubqueryExpression &expr,
@@ -167051,8 +167101,25 @@ static unique_ptr<Expression> PlanUncorrelatedSubquery(Binder &binder, BoundSubq
 }
 
 static unique_ptr<LogicalDelimJoin> CreateDuplicateEliminatedJoin(vector<CorrelatedColumnInfo> &correlated_columns,
-                                                                  JoinType join_type) {
+                                                                  JoinType join_type,
+                                                                  unique_ptr<LogicalOperator> original_plan,
+                                                                  bool perform_delim) {
 	auto delim_join = make_unique<LogicalDelimJoin>(join_type);
+	if (!perform_delim) {
+		// if we are not performing a delim join, we push a row_number() OVER() window operator on the LHS
+		// and perform all duplicate elimination on that row number instead
+		D_ASSERT(correlated_columns[0].type.id() == LogicalTypeId::BIGINT);
+		auto window = make_unique<LogicalWindow>(correlated_columns[0].binding.table_index);
+		auto row_number = make_unique<BoundWindowExpression>(ExpressionType::WINDOW_ROW_NUMBER, LogicalType::BIGINT,
+		                                                     nullptr, nullptr);
+		row_number->start = WindowBoundary::UNBOUNDED_PRECEDING;
+		row_number->end = WindowBoundary::CURRENT_ROW_ROWS;
+		row_number->alias = "delim_index";
+		window->expressions.push_back(move(row_number));
+		window->AddChild(move(original_plan));
+		original_plan = move(window);
+	}
+	delim_join->AddChild(move(original_plan));
 	for (idx_t i = 0; i < correlated_columns.size(); i++) {
 		auto &col = correlated_columns[i];
 		delim_join->duplicate_eliminated_columns.push_back(
@@ -167063,8 +167130,9 @@ static unique_ptr<LogicalDelimJoin> CreateDuplicateEliminatedJoin(vector<Correla
 }
 
 static void CreateDelimJoinConditions(LogicalDelimJoin &delim_join, vector<CorrelatedColumnInfo> &correlated_columns,
-                                      vector<ColumnBinding> bindings, idx_t base_offset) {
-	for (idx_t i = 0; i < correlated_columns.size(); i++) {
+                                      vector<ColumnBinding> bindings, idx_t base_offset, bool perform_delim) {
+	auto col_count = perform_delim ? correlated_columns.size() : 1;
+	for (idx_t i = 0; i < col_count; i++) {
 		auto &col = correlated_columns[i];
 		JoinCondition cond;
 		cond.left = make_unique<BoundColumnRefExpression>(col.name, col.type, col.binding);
@@ -167074,10 +167142,50 @@ static void CreateDelimJoinConditions(LogicalDelimJoin &delim_join, vector<Corre
 	}
 }
 
+static bool PerformDelimOnType(const LogicalType &type) {
+	if (type.InternalType() == PhysicalType::LIST) {
+		return false;
+	}
+	if (type.InternalType() == PhysicalType::STRUCT) {
+		for (auto &entry : StructType::GetChildTypes(type)) {
+			if (!PerformDelimOnType(entry.second)) {
+				return false;
+			}
+		}
+	}
+	return true;
+}
+
+static bool PerformDuplicateElimination(Binder &binder, vector<CorrelatedColumnInfo> &correlated_columns) {
+	if (!ClientConfig::GetConfig(binder.context).enable_optimizer) {
+		// if optimizations are disabled we always do a delim join
+		return true;
+	}
+	bool perform_delim = true;
+	for (auto &col : correlated_columns) {
+		if (!PerformDelimOnType(col.type)) {
+			perform_delim = false;
+			break;
+		}
+	}
+	if (perform_delim) {
+		return true;
+	}
+	auto binding = ColumnBinding(binder.GenerateTableIndex(), 0);
+	auto type = LogicalType::BIGINT;
+	auto name = "delim_index";
+	CorrelatedColumnInfo info(binding, type, name, 0);
+	correlated_columns.insert(correlated_columns.begin(), move(info));
+	return false;
+}
+
 static unique_ptr<Expression> PlanCorrelatedSubquery(Binder &binder, BoundSubqueryExpression &expr,
                                                      unique_ptr<LogicalOperator> &root,
                                                      unique_ptr<LogicalOperator> plan) {
 	auto &correlated_columns = expr.binder->correlated_columns;
+	// FIXME: there should be a way of disabling decorrelation for ANY queries as well, but not for now...
+	bool perform_delim =
+	    expr.subquery_type == SubqueryType::ANY ? true : PerformDuplicateElimination(binder, correlated_columns);
 	D_ASSERT(expr.IsCorrelated());
 	// correlated subquery
 	// for a more in-depth explanation of this code, read the paper "Unnesting Arbitrary Subqueries"
@@ -167094,15 +167202,15 @@ static unique_ptr<Expression> PlanCorrelatedSubquery(Binder &binder, BoundSubque
 		// NULL values are equal in this join because we join on the correlated columns ONLY
 		// and e.g. in the query: SELECT (SELECT 42 FROM integers WHERE i1.i IS NULL LIMIT 1) FROM integers i1;
 		// the input value NULL will generate the value 42, and we need to join NULL on the LHS with NULL on the RHS
-		auto delim_join = CreateDuplicateEliminatedJoin(correlated_columns, JoinType::SINGLE);
-
 		// the left side is the original plan
 		// this is the side that will be duplicate eliminated and pushed into the RHS
-		delim_join->AddChild(move(root));
+		auto delim_join =
+		    CreateDuplicateEliminatedJoin(correlated_columns, JoinType::SINGLE, move(root), perform_delim);
+
 		// the right side initially is a DEPENDENT join between the duplicate eliminated scan and the subquery
 		// HOWEVER: we do not explicitly create the dependent join
 		// instead, we eliminate the dependent join by pushing it down into the right side of the plan
-		FlattenDependentJoins flatten(binder, correlated_columns);
+		FlattenDependentJoins flatten(binder, correlated_columns, perform_delim);
 
 		// first we check which logical operators have correlated expressions in the first place
 		flatten.DetectCorrelatedExpressions(plan.get());
@@ -167115,7 +167223,7 @@ static unique_ptr<Expression> PlanCorrelatedSubquery(Binder &binder, BoundSubque
 		auto plan_columns = dependent_join->GetColumnBindings();
 
 		// now create the join conditions
-		CreateDelimJoinConditions(*delim_join, correlated_columns, plan_columns, flatten.delim_offset);
+		CreateDelimJoinConditions(*delim_join, correlated_columns, plan_columns, flatten.delim_offset, perform_delim);
 		delim_join->AddChild(move(dependent_join));
 		root = move(delim_join);
 		// finally push the BoundColumnRefExpression referring to the data element returned by the join
@@ -167126,12 +167234,10 @@ static unique_ptr<Expression> PlanCorrelatedSubquery(Binder &binder, BoundSubque
 		// correlated EXISTS query
 		// this query is similar to the correlated SCALAR query, except we use a MARK join here
 		idx_t mark_index = binder.GenerateTableIndex();
-		auto delim_join = CreateDuplicateEliminatedJoin(correlated_columns, JoinType::MARK);
+		auto delim_join = CreateDuplicateEliminatedJoin(correlated_columns, JoinType::MARK, move(root), perform_delim);
 		delim_join->mark_index = mark_index;
-		// LHS
-		delim_join->AddChild(move(root));
 		// RHS
-		FlattenDependentJoins flatten(binder, correlated_columns);
+		FlattenDependentJoins flatten(binder, correlated_columns, perform_delim);
 		flatten.DetectCorrelatedExpressions(plan.get());
 		auto dependent_join = flatten.PushDownDependentJoin(move(plan));
 
@@ -167139,7 +167245,7 @@ static unique_ptr<Expression> PlanCorrelatedSubquery(Binder &binder, BoundSubque
 		auto plan_columns = dependent_join->GetColumnBindings();
 
 		// now we create the join conditions between the dependent join and the original table
-		CreateDelimJoinConditions(*delim_join, correlated_columns, plan_columns, flatten.delim_offset);
+		CreateDelimJoinConditions(*delim_join, correlated_columns, plan_columns, flatten.delim_offset, perform_delim);
 		delim_join->AddChild(move(dependent_join));
 		root = move(delim_join);
 		// finally push the BoundColumnRefExpression referring to the marker
@@ -167155,10 +167261,8 @@ static unique_ptr<Expression> PlanCorrelatedSubquery(Binder &binder, BoundSubque
 		// as the MARK join has one extra join condition (the original condition, of the ANY expression, e.g.
 		// [i=ANY(...)])
 		idx_t mark_index = binder.GenerateTableIndex();
-		auto delim_join = CreateDuplicateEliminatedJoin(correlated_columns, JoinType::MARK);
+		auto delim_join = CreateDuplicateEliminatedJoin(correlated_columns, JoinType::MARK, move(root), perform_delim);
 		delim_join->mark_index = mark_index;
-		// LHS
-		delim_join->AddChild(move(root));
 		// RHS
 		FlattenDependentJoins flatten(binder, correlated_columns, true);
 		flatten.DetectCorrelatedExpressions(plan.get());
@@ -167168,7 +167272,7 @@ static unique_ptr<Expression> PlanCorrelatedSubquery(Binder &binder, BoundSubque
 		auto plan_columns = dependent_join->GetColumnBindings();
 
 		// now we create the join conditions between the dependent join and the original table
-		CreateDelimJoinConditions(*delim_join, correlated_columns, plan_columns, flatten.delim_offset);
+		CreateDelimJoinConditions(*delim_join, correlated_columns, plan_columns, flatten.delim_offset, perform_delim);
 		// add the actual condition based on the ANY/ALL predicate
 		JoinCondition compare_cond;
 		compare_cond.left = move(expr.child);
@@ -167192,9 +167296,11 @@ public:
 	void VisitOperator(LogicalOperator &op) override {
 		if (!op.children.empty()) {
 			root = move(op.children[0]);
+			D_ASSERT(root);
 			VisitOperatorExpressions(op);
 			op.children[0] = move(root);
 			for (idx_t i = 0; i < op.children.size(); i++) {
+				D_ASSERT(op.children[i]);
 				VisitOperator(*op.children[i]);
 			}
 		}
@@ -173807,6 +173913,7 @@ string IsNotNullFilter::ToString(const string &column_name) {
 
 
 
+
 namespace duckdb {
 
 unique_ptr<Expression> JoinCondition::CreateExpression(JoinCondition cond) {
@@ -173814,6 +173921,21 @@ unique_ptr<Expression> JoinCondition::CreateExpression(JoinCondition cond) {
 	return move(bound_comparison);
 }
 
+unique_ptr<Expression> JoinCondition::CreateExpression(vector<JoinCondition> conditions) {
+	unique_ptr<Expression> result;
+	for (auto &cond : conditions) {
+		auto expr = CreateExpression(move(cond));
+		if (!result) {
+			result = move(expr);
+		} else {
+			auto conj =
+			    make_unique<BoundConjunctionExpression>(ExpressionType::CONJUNCTION_AND, move(expr), move(result));
+			result = move(conj);
+		}
+	}
+	return result;
+}
+
 JoinSide JoinSide::CombineJoinSide(JoinSide left, JoinSide right) {
 	if (left == JoinSide::NONE) {
 		return right;
@@ -173896,6 +174018,20 @@ JoinSide JoinSide::GetJoinSide(const unordered_set<idx_t> &bindings, unordered_s
 
 namespace duckdb {
 
+LogicalOperator::LogicalOperator(LogicalOperatorType type) : type(type) {
+}
+
+LogicalOperator::LogicalOperator(LogicalOperatorType type, vector<unique_ptr<Expression>> expressions)
+    : type(type), expressions(move(expressions)) {
+}
+
+LogicalOperator::~LogicalOperator() {
+}
+
+vector<ColumnBinding> LogicalOperator::GetColumnBindings() {
+	return {ColumnBinding(0, 0)};
+}
+
 string LogicalOperator::GetName() const {
 	return LogicalOperatorToString(type);
 }
@@ -173999,6 +174135,20 @@ void LogicalOperator::Verify() {
 #endif
 }
 
+void LogicalOperator::AddChild(unique_ptr<LogicalOperator> child) {
+	D_ASSERT(child);
+	children.push_back(move(child));
+}
+
+idx_t LogicalOperator::EstimateCardinality(ClientContext &context) {
+	// simple estimator, just take the max of the children
+	idx_t max_cardinality = 0;
+	for (auto &child : children) {
+		max_cardinality = MaxValue(child->EstimateCardinality(context), max_cardinality);
+	}
+	return max_cardinality;
+}
+
 void LogicalOperator::Print() {
 	Printer::Print(ToString());
 }
@@ -175100,8 +175250,8 @@ public:
 namespace duckdb {
 
 FlattenDependentJoins::FlattenDependentJoins(Binder &binder, const vector<CorrelatedColumnInfo> &correlated,
-                                             bool any_join)
-    : binder(binder), correlated_columns(correlated), any_join(any_join) {
+                                             bool perform_delim, bool any_join)
+    : binder(binder), correlated_columns(correlated), perform_delim(perform_delim), any_join(any_join) {
 	for (idx_t i = 0; i < correlated_columns.size(); i++) {
 		auto &col = correlated_columns[i];
 		correlated_map[col.binding] = i;
@@ -175201,8 +175351,9 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		// now we add all the columns of the delim_scan to the projection list
 		auto proj = (LogicalProjection *)plan.get();
 		for (idx_t i = 0; i < correlated_columns.size(); i++) {
+			auto &col = correlated_columns[i];
 			auto colref = make_unique<BoundColumnRefExpression>(
-			    correlated_columns[i].type, ColumnBinding(base_binding.table_index, base_binding.column_index + i));
+			    col.name, col.type, ColumnBinding(base_binding.table_index, base_binding.column_index + i));
 			plan->expressions.push_back(move(colref));
 		}
 
@@ -175223,15 +175374,42 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		RewriteCorrelatedExpressions rewriter(base_binding, correlated_map);
 		rewriter.VisitOperator(*plan);
 		// now we add all the columns of the delim_scan to the grouping operators AND the projection list
-		for (idx_t i = 0; i < correlated_columns.size(); i++) {
+		idx_t delim_table_index;
+		idx_t delim_column_offset;
+		idx_t delim_data_offset;
+		auto new_group_count = perform_delim ? correlated_columns.size() : 1;
+		for (idx_t i = 0; i < new_group_count; i++) {
+			auto &col = correlated_columns[i];
 			auto colref = make_unique<BoundColumnRefExpression>(
-			    correlated_columns[i].type, ColumnBinding(base_binding.table_index, base_binding.column_index + i));
+			    col.name, col.type, ColumnBinding(base_binding.table_index, base_binding.column_index + i));
 			for (auto &set : aggr.grouping_sets) {
 				set.insert(aggr.groups.size());
 			}
 			aggr.groups.push_back(move(colref));
 		}
-		if (aggr.groups.size() == correlated_columns.size()) {
+		if (!perform_delim) {
+			// if we are not performing the duplicate elimination, we have only added the row_id column to the grouping
+			// operators in this case, we push a FIRST aggregate for each of the remaining expressions
+			delim_table_index = aggr.aggregate_index;
+			delim_column_offset = aggr.expressions.size();
+			delim_data_offset = aggr.groups.size();
+			for (idx_t i = 0; i < correlated_columns.size(); i++) {
+				auto &col = correlated_columns[i];
+				auto first_aggregate = FirstFun::GetFunction(col.type);
+				auto colref = make_unique<BoundColumnRefExpression>(
+				    col.name, col.type, ColumnBinding(base_binding.table_index, base_binding.column_index + i));
+				vector<unique_ptr<Expression>> aggr_children;
+				aggr_children.push_back(move(colref));
+				auto first_fun = make_unique<BoundAggregateExpression>(move(first_aggregate), move(aggr_children),
+				                                                       nullptr, nullptr, false);
+				aggr.expressions.push_back(move(first_fun));
+			}
+		} else {
+			delim_table_index = aggr.group_index;
+			delim_column_offset = aggr.groups.size() - correlated_columns.size();
+			delim_data_offset = aggr.groups.size();
+		}
+		if (aggr.groups.size() == new_group_count) {
 			// we have to perform a LEFT OUTER JOIN between the result of this aggregate and the delim scan
 			// FIXME: this does not always have to be a LEFT OUTER JOIN, depending on whether aggr.expressions return
 			// NULL or a value
@@ -175247,13 +175425,12 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 			auto delim_scan = make_unique<LogicalDelimGet>(left_index, delim_types);
 			join->children.push_back(move(delim_scan));
 			join->children.push_back(move(plan));
-			for (idx_t i = 0; i < correlated_columns.size(); i++) {
+			for (idx_t i = 0; i < new_group_count; i++) {
+				auto &col = correlated_columns[i];
 				JoinCondition cond;
-				cond.left =
-				    make_unique<BoundColumnRefExpression>(correlated_columns[i].type, ColumnBinding(left_index, i));
+				cond.left = make_unique<BoundColumnRefExpression>(col.name, col.type, ColumnBinding(left_index, i));
 				cond.right = make_unique<BoundColumnRefExpression>(
-				    correlated_columns[i].type,
-				    ColumnBinding(aggr.group_index, (aggr.groups.size() - correlated_columns.size()) + i));
+				    correlated_columns[i].type, ColumnBinding(delim_table_index, delim_column_offset + i));
 				cond.comparison = ExpressionType::COMPARE_NOT_DISTINCT_FROM;
 				join->conditions.push_back(move(cond));
 			}
@@ -175269,16 +175446,15 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 				}
 			}
 			// now we update the delim_index
-
 			base_binding.table_index = left_index;
 			this->delim_offset = base_binding.column_index = 0;
 			this->data_offset = 0;
 			return move(join);
 		} else {
 			// update the delim_index
-			base_binding.table_index = aggr.group_index;
-			this->delim_offset = base_binding.column_index = aggr.groups.size() - correlated_columns.size();
-			this->data_offset = aggr.groups.size();
+			base_binding.table_index = delim_table_index;
+			this->delim_offset = base_binding.column_index = delim_column_offset;
+			this->data_offset = delim_data_offset;
 			return plan;
 		}
 	}

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "453d09706"
-#define DUCKDB_VERSION "v0.4.1-dev308"
+#define DUCKDB_SOURCE_ID "f5d15448b"
+#define DUCKDB_VERSION "v0.4.1-dev318"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -8879,13 +8879,9 @@ namespace duckdb {
 //! logical query tree
 class LogicalOperator {
 public:
-	explicit LogicalOperator(LogicalOperatorType type) : type(type) {
-	}
-	LogicalOperator(LogicalOperatorType type, vector<unique_ptr<Expression>> expressions)
-	    : type(type), expressions(move(expressions)) {
-	}
-	virtual ~LogicalOperator() {
-	}
+	explicit LogicalOperator(LogicalOperatorType type);
+	LogicalOperator(LogicalOperatorType type, vector<unique_ptr<Expression>> expressions);
+	virtual ~LogicalOperator();
 
 	//! The type of the logical operator
 	LogicalOperatorType type;
@@ -8899,9 +8895,7 @@ public:
 	idx_t estimated_cardinality = 0;
 
 public:
-	virtual vector<ColumnBinding> GetColumnBindings() {
-		return {ColumnBinding(0, 0)};
-	}
+	virtual vector<ColumnBinding> GetColumnBindings();
 	static vector<ColumnBinding> GenerateColumnBindings(idx_t table_idx, idx_t column_count);
 	static vector<LogicalType> MapTypes(const vector<LogicalType> &types, const vector<idx_t> &projection_map);
 	static vector<ColumnBinding> MapBindings(const vector<ColumnBinding> &types, const vector<idx_t> &projection_map);
@@ -8916,18 +8910,9 @@ public:
 	//! Debug method: verify that the integrity of expressions & child nodes are maintained
 	virtual void Verify();
 
-	void AddChild(unique_ptr<LogicalOperator> child) {
-		children.push_back(move(child));
-	}
+	void AddChild(unique_ptr<LogicalOperator> child);
 
-	virtual idx_t EstimateCardinality(ClientContext &context) {
-		// simple estimator, just take the max of the children
-		idx_t max_cardinality = 0;
-		for (auto &child : children) {
-			max_cardinality = MaxValue(child->EstimateCardinality(context), max_cardinality);
-		}
-		return max_cardinality;
-	}
+	virtual idx_t EstimateCardinality(ClientContext &context);
 
 protected:
 	//! Resolve types for this specific operator
@@ -14373,8 +14358,11 @@ struct CorrelatedColumnInfo {
 	string name;
 	idx_t depth;
 
+	CorrelatedColumnInfo(ColumnBinding binding, LogicalType type_p, string name_p, idx_t depth)
+	    : binding(binding), type(move(type_p)), name(move(name_p)), depth(depth) {
+	}
 	explicit CorrelatedColumnInfo(BoundColumnRefExpression &expr)
-	    : binding(expr.binding), type(expr.return_type), name(expr.GetName()), depth(expr.depth) {
+	    : CorrelatedColumnInfo(expr.binding, expr.return_type, expr.GetName(), expr.depth) {
 	}
 
 	bool operator==(const CorrelatedColumnInfo &rhs) const {