duckdb 0.7.2-dev402.0 → 0.7.2-dev586.0

package/src/duckdb/src/parser/parsed_expression_iterator.cpp

@@ -96,6 +96,13 @@ void ParsedExpressionIterator::EnumerateChildren(
 		}
 		break;
 	}
+	case ExpressionClass::STAR: {
+		auto &star_expr = (StarExpression &)expr;
+		if (star_expr.expr) {
+			callback(star_expr.expr);
+		}
+		break;
+	}
 	case ExpressionClass::SUBQUERY: {
 		auto &subquery_expr = (SubqueryExpression &)expr;
 		if (subquery_expr.child) {
@@ -135,7 +142,6 @@ void ParsedExpressionIterator::EnumerateChildren(
 	case ExpressionClass::COLUMN_REF:
 	case ExpressionClass::CONSTANT:
 	case ExpressionClass::DEFAULT:
-	case ExpressionClass::STAR:
 	case ExpressionClass::PARAMETER:
 	case ExpressionClass::POSITIONAL_REFERENCE:
 		// these node types have no children

package/src/duckdb/src/parser/transform/expression/transform_columnref.cpp

@@ -1,5 +1,6 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/parser/expression/columnref_expression.hpp"
+#include "duckdb/parser/expression/function_expression.hpp"
 #include "duckdb/parser/expression/star_expression.hpp"
 #include "duckdb/parser/transformer.hpp"
 
@@ -36,13 +37,27 @@ unique_ptr<ParsedExpression> Transformer::TransformStarExpression(duckdb_libpgqu
 			result->replace_list.insert(make_pair(std::move(exclude_entry), std::move(replace_expression)));
 		}
 	}
-	if (star->regex) {
+	if (star->expr) {
+		D_ASSERT(star->columns);
 		D_ASSERT(result->relation_name.empty());
 		D_ASSERT(result->exclude_list.empty());
 		D_ASSERT(result->replace_list.empty());
-		result->regex = star->regex;
+		result->expr = TransformExpression(star->expr);
+		if (result->expr->type == ExpressionType::STAR) {
+			auto child_star = (StarExpression *)result->expr.get();
+			result->exclude_list = std::move(child_star->exclude_list);
+			result->replace_list = std::move(child_star->replace_list);
+			result->expr.reset();
+		} else if (result->expr->type == ExpressionType::LAMBDA) {
+			vector<unique_ptr<ParsedExpression>> children;
+			children.push_back(make_unique<StarExpression>());
+			children.push_back(std::move(result->expr));
+			auto list_filter = make_unique<FunctionExpression>("list_filter", std::move(children));
+			result->expr = std::move(list_filter);
+		}
 	}
 	result->columns = star->columns;
+	result->query_location = star->location;
 	return std::move(result);
 }
 

package/src/duckdb/src/parser/transform/expression/transform_function.cpp

@@ -94,6 +94,17 @@ void Transformer::TransformWindowFrame(duckdb_libpgquery::PGWindowDef *window_sp
 	}
 }
 
+bool Transformer::ExpressionIsEmptyStar(ParsedExpression &expr) {
+	if (expr.expression_class != ExpressionClass::STAR) {
+		return false;
+	}
+	auto &star = (StarExpression &)expr;
+	if (!star.columns && star.exclude_list.empty() && star.replace_list.empty()) {
+		return true;
+	}
+	return false;
+}
+
 unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::PGFuncCall *root) {
 	auto name = root->funcname;
 	string catalog, schema, function_name;
@@ -116,8 +127,17 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
 		throw InternalException("TransformFuncCall - Expected 1, 2 or 3 qualifications");
 	}
 
-	auto lowercase_name = StringUtil::Lower(function_name);
+	//  transform children
+	vector<unique_ptr<ParsedExpression>> children;
+	if (root->args) {
+		TransformExpressionList(*root->args, children);
+	}
+	if (children.size() == 1 && ExpressionIsEmptyStar(*children[0]) && !root->agg_distinct && !root->agg_order) {
+		// COUNT(*) gets translated into COUNT()
+		children.clear();
+	}
 
+	auto lowercase_name = StringUtil::Lower(function_name);
 	if (root->over) {
 		const auto win_fun_type = WindowToExpressionType(lowercase_name);
 		if (win_fun_type == ExpressionType::INVALID) {
@@ -151,39 +171,32 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
 			expr->filter_expr = std::move(filter_expr);
 		}
 
-		if (root->args) {
-			vector<unique_ptr<ParsedExpression>> function_list;
-			TransformExpressionList(*root->args, function_list);
-
-			if (win_fun_type == ExpressionType::WINDOW_AGGREGATE) {
-				for (auto &child : function_list) {
-					expr->children.push_back(std::move(child));
+		if (win_fun_type == ExpressionType::WINDOW_AGGREGATE) {
+			expr->children = std::move(children);
+		} else {
+			if (!children.empty()) {
+				expr->children.push_back(std::move(children[0]));
+			}
+			if (win_fun_type == ExpressionType::WINDOW_LEAD || win_fun_type == ExpressionType::WINDOW_LAG) {
+				if (children.size() > 1) {
+					expr->offset_expr = std::move(children[1]);
 				}
-			} else {
-				if (!function_list.empty()) {
-					expr->children.push_back(std::move(function_list[0]));
+				if (children.size() > 2) {
+					expr->default_expr = std::move(children[2]);
+				}
+				if (children.size() > 3) {
+					throw ParserException("Incorrect number of parameters for function %s", lowercase_name);
+				}
+			} else if (win_fun_type == ExpressionType::WINDOW_NTH_VALUE) {
+				if (children.size() > 1) {
+					expr->children.push_back(std::move(children[1]));
 				}
-				if (win_fun_type == ExpressionType::WINDOW_LEAD || win_fun_type == ExpressionType::WINDOW_LAG) {
-					if (function_list.size() > 1) {
-						expr->offset_expr = std::move(function_list[1]);
-					}
-					if (function_list.size() > 2) {
-						expr->default_expr = std::move(function_list[2]);
-					}
-					if (function_list.size() > 3) {
-						throw ParserException("Incorrect number of parameters for function %s", lowercase_name);
-					}
-				} else if (win_fun_type == ExpressionType::WINDOW_NTH_VALUE) {
-					if (function_list.size() > 1) {
-						expr->children.push_back(std::move(function_list[1]));
-					}
-					if (function_list.size() > 2) {
-						throw ParserException("Incorrect number of parameters for function %s", lowercase_name);
-					}
-				} else {
-					if (function_list.size() > 1) {
-						throw ParserException("Incorrect number of parameters for function %s", lowercase_name);
-					}
+				if (children.size() > 2) {
+					throw ParserException("Incorrect number of parameters for function %s", lowercase_name);
+				}
+			} else {
+				if (children.size() > 1) {
+					throw ParserException("Incorrect number of parameters for function %s", lowercase_name);
 				}
 			}
 		}
@@ -215,14 +228,6 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
 		throw ParserException("IGNORE NULLS is not supported for non-window functions");
 	}
 
-	//  TransformExpressionList??
-	vector<unique_ptr<ParsedExpression>> children;
-	if (root->args != nullptr) {
-		for (auto node = root->args->head; node != nullptr; node = node->next) {
-			auto child_expr = TransformExpression((duckdb_libpgquery::PGNode *)node->data.ptr_value);
-			children.push_back(std::move(child_expr));
-		}
-	}
 	unique_ptr<ParsedExpression> filter_expr;
 	if (root->agg_filter) {
 		filter_expr = TransformExpression(root->agg_filter);

package/src/duckdb/src/parser/transform/helpers/transform_groupby.cpp

@@ -176,6 +176,13 @@ bool Transformer::TransformGroupBy(duckdb_libpgquery::PGList *group, SelectNode
 			result.grouping_sets = std::move(new_sets);
 		}
 	}
+	if (result.group_expressions.size() == 1 && result.grouping_sets.size() == 1 &&
+	    ExpressionIsEmptyStar(*result.group_expressions[0])) {
+		// GROUP BY *
+		result.group_expressions.clear();
+		result.grouping_sets.clear();
+		select_node.aggregate_handling = AggregateHandling::FORCE_AGGREGATES;
+	}
 	return true;
 }
 

package/src/duckdb/src/parser/transform/helpers/transform_orderby.cpp

@@ -36,13 +36,6 @@ bool Transformer::TransformOrderBy(duckdb_libpgquery::PGList *order, vector<Orde
 				throw NotImplementedException("Unimplemented order by type");
 			}
 			auto order_expression = TransformExpression(target);
-			if (order_expression->GetExpressionClass() == ExpressionClass::STAR) {
-				auto &star_expr = (StarExpression &)*order_expression;
-				D_ASSERT(star_expr.relation_name.empty());
-				if (star_expr.columns) {
-					throw ParserException("COLUMNS expr is not supported in ORDER BY");
-				}
-			}
 			result.emplace_back(type, null_order, std::move(order_expression));
 		} else {
 			throw NotImplementedException("ORDER BY list member type %d\n", temp->type);

package/src/duckdb/src/parser/transform/statement/transform_rename.cpp

@@ -6,7 +6,9 @@ namespace duckdb {
 unique_ptr<AlterStatement> Transformer::TransformRename(duckdb_libpgquery::PGNode *node) {
 	auto stmt = reinterpret_cast<duckdb_libpgquery::PGRenameStmt *>(node);
 	D_ASSERT(stmt);
-	D_ASSERT(stmt->relation);
+	if (!stmt->relation) {
+		throw NotImplementedException("Altering schemas is not yet supported");
+	}
 
 	unique_ptr<AlterInfo> info;
 
@@ -17,8 +19,6 @@ unique_ptr<AlterStatement> Transformer::TransformRename(duckdb_libpgquery::PGNod
 	if (stmt->relation->relname) {
 		data.name = stmt->relation->relname;
 	}
-	if (stmt->relation->schemaname) {
-	}
 	// first we check the type of ALTER
 	switch (stmt->renameType) {
 	case duckdb_libpgquery::PG_OBJECT_COLUMN: {
@@ -36,7 +36,6 @@ unique_ptr<AlterStatement> Transformer::TransformRename(duckdb_libpgquery::PGNod
 		info = make_unique<RenameTableInfo>(std::move(data), new_name);
 		break;
 	}
-
 	case duckdb_libpgquery::PG_OBJECT_VIEW: {
 		// change view name
 		string new_name = stmt->newname;

package/src/duckdb/src/planner/bind_context.cpp

@@ -13,7 +13,7 @@
 #include "duckdb/planner/bound_query_node.hpp"
 #include "duckdb/planner/expression/bound_columnref_expression.hpp"
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
-#include "re2/re2.h"
+#include "duckdb/planner/expression_binder/constant_binder.hpp"
 
 #include <algorithm>
 
@@ -319,31 +319,15 @@ bool BindContext::CheckExclusionList(StarExpression &expr, Binding *binding, con
 	return false;
 }
 
-bool CheckRegex(const string &column_name, duckdb_re2::RE2 *regex) {
-	if (!regex) {
-		return true;
-	}
-	return RE2::PartialMatch(column_name, *regex);
-}
-
 void BindContext::GenerateAllColumnExpressions(StarExpression &expr,
                                                vector<unique_ptr<ParsedExpression>> &new_select_list) {
 	if (bindings_list.empty()) {
-		throw BinderException("SELECT * expression without FROM clause!");
+		throw BinderException("* expression without FROM clause!");
 	}
 	case_insensitive_set_t excluded_columns;
 	if (expr.relation_name.empty()) {
 		// SELECT * case
 		// bind all expressions of each table in-order
-		unique_ptr<duckdb_re2::RE2> regex;
-		bool found_match = true;
-		if (!expr.regex.empty()) {
-			regex = make_unique<duckdb_re2::RE2>(expr.regex);
-			if (!regex->error().empty()) {
-				throw BinderException("Failed to compile regex \"%s\": %s", expr.regex, regex->error());
-			}
-			found_match = false;
-		}
 		unordered_set<UsingColumnSet *> handled_using_columns;
 		for (auto &entry : bindings_list) {
 			auto binding = entry.second;
@@ -351,10 +335,6 @@ void BindContext::GenerateAllColumnExpressions(StarExpression &expr,
 				if (CheckExclusionList(expr, binding, column_name, new_select_list, excluded_columns)) {
 					continue;
 				}
-				if (!CheckRegex(column_name, regex.get())) {
-					continue;
-				}
-				found_match = true;
 				// check if this column is a USING column
 				auto using_binding = GetUsingBinding(column_name, binding->alias);
 				if (using_binding) {
@@ -384,9 +364,6 @@ void BindContext::GenerateAllColumnExpressions(StarExpression &expr,
 				new_select_list.push_back(make_unique<ColumnRefExpression>(column_name, binding->alias));
 			}
 		}
-		if (!found_match) {
-			throw BinderException("No matching columns found that match regex \"%s\"", expr.regex);
-		}
 	} else {
 		// SELECT tbl.* case
 		// SELECT struct.* case

package/src/duckdb/src/planner/binder/expression/bind_aggregate_expression.cpp

@@ -208,8 +208,9 @@ BindResult SelectBinder::BindAggregate(FunctionExpression &aggr, AggregateFuncti
 	auto bound_function = func->functions.GetFunctionByOffset(best_function);
 
 	// Bind any sort columns, unless the aggregate is order-insensitive
-	auto order_bys = make_unique<BoundOrderModifier>();
+	unique_ptr<BoundOrderModifier> order_bys;
 	if (!aggr.order_bys->orders.empty()) {
+		order_bys = make_unique<BoundOrderModifier>();
 		auto &config = DBConfig::GetConfig(context);
 		for (auto &order : aggr.order_bys->orders) {
 			auto &order_expr = (BoundExpression &)*order.expression;
@@ -222,12 +223,13 @@ BindResult SelectBinder::BindAggregate(FunctionExpression &aggr, AggregateFuncti
 		}
 	}
 
-	auto aggregate = function_binder.BindAggregateFunction(
-	    bound_function, std::move(children), std::move(bound_filter),
-	    aggr.distinct ? AggregateType::DISTINCT : AggregateType::NON_DISTINCT, std::move(order_bys));
+	auto aggregate =
+	    function_binder.BindAggregateFunction(bound_function, std::move(children), std::move(bound_filter),
+	                                          aggr.distinct ? AggregateType::DISTINCT : AggregateType::NON_DISTINCT);
 	if (aggr.export_state) {
 		aggregate = ExportAggregateFunction::Bind(std::move(aggregate));
 	}
+	aggregate->order_bys = std::move(order_bys);
 
 	// check for all the aggregates if this aggregate already exists
 	idx_t aggr_index;

package/src/duckdb/src/planner/binder/expression/bind_lambda.cpp

@@ -48,7 +48,6 @@ BindResult ExpressionBinder::BindExpression(LambdaExpression &expr, idx_t depth,
 
 	// positional parameters as column references
 	for (idx_t i = 0; i < expr.params.size(); i++) {
-
 		if (expr.params[i]->GetExpressionClass() != ExpressionClass::COLUMN_REF) {
 			throw BinderException("Parameter must be a column name.");
 		}
@@ -80,7 +79,9 @@ BindResult ExpressionBinder::BindExpression(LambdaExpression &expr, idx_t depth,
 	// bind the parameter expressions
 	for (idx_t i = 0; i < expr.params.size(); i++) {
 		auto result = BindExpression(&expr.params[i], depth, false);
-		D_ASSERT(!result.HasError());
+		if (result.HasError()) {
+			throw InternalException("Error during lambda binding: %s", result.error);
+		}
 	}
 
 	auto result = BindExpression(&expr.expr, depth, false);

package/src/duckdb/src/planner/binder/expression/bind_star_expression.cpp

@@ -0,0 +1,176 @@
+#include "duckdb/planner/binder.hpp"
+#include "duckdb/parser/expression/star_expression.hpp"
+#include "duckdb/parser/expression/constant_expression.hpp"
+#include "duckdb/planner/expression_binder/table_function_binder.hpp"
+#include "duckdb/parser/parsed_expression_iterator.hpp"
+#include "duckdb/execution/expression_executor.hpp"
+#include "re2/re2.h"
+
+namespace duckdb {
+
+bool Binder::FindStarExpression(unique_ptr<ParsedExpression> &expr, StarExpression **star, bool is_root,
+                                bool in_columns) {
+	bool has_star = false;
+	if (expr->GetExpressionClass() == ExpressionClass::STAR) {
+		auto current_star = (StarExpression *)expr.get();
+		if (!current_star->columns) {
+			if (is_root) {
+				*star = current_star;
+				return true;
+			}
+			if (!in_columns) {
+				throw BinderException(
+				    "STAR expression is only allowed as the root element of an expression. Use COLUMNS(*) instead.");
+			}
+			// star expression inside a COLUMNS - convert to a constant list
+			if (!current_star->replace_list.empty()) {
+				throw BinderException(
+				    "STAR expression with REPLACE list is only allowed as the root element of COLUMNS");
+			}
+			vector<unique_ptr<ParsedExpression>> star_list;
+			bind_context.GenerateAllColumnExpressions(*current_star, star_list);
+
+			vector<Value> values;
+			values.reserve(star_list.size());
+			for (auto &expr : star_list) {
+				values.emplace_back(expr->ToString());
+			}
+			D_ASSERT(!values.empty());
+
+			expr = make_unique<ConstantExpression>(Value::LIST(LogicalType::VARCHAR, values));
+			return true;
+		}
+		if (in_columns) {
+			throw BinderException("COLUMNS expression is not allowed inside another COLUMNS expression");
+		}
+		in_columns = true;
+		if (*star) {
+			// we can have multiple
+			if (!StarExpression::Equal(*star, current_star)) {
+				throw BinderException(
+				    FormatError(*expr, "Multiple different STAR/COLUMNS in the same expression are not supported"));
+			}
+			return true;
+		}
+		*star = current_star;
+		has_star = true;
+	}
+	ParsedExpressionIterator::EnumerateChildren(*expr, [&](unique_ptr<ParsedExpression> &child_expr) {
+		if (FindStarExpression(child_expr, star, false, in_columns)) {
+			has_star = true;
+		}
+	});
+	return has_star;
+}
+
+void Binder::ReplaceStarExpression(unique_ptr<ParsedExpression> &expr, unique_ptr<ParsedExpression> &replacement) {
+	D_ASSERT(expr);
+	if (expr->GetExpressionClass() == ExpressionClass::STAR) {
+		D_ASSERT(replacement);
+		expr = replacement->Copy();
+		return;
+	}
+	ParsedExpressionIterator::EnumerateChildren(
+	    *expr, [&](unique_ptr<ParsedExpression> &child_expr) { ReplaceStarExpression(child_expr, replacement); });
+}
+
+void Binder::ExpandStarExpression(unique_ptr<ParsedExpression> expr,
+                                  vector<unique_ptr<ParsedExpression>> &new_select_list) {
+	StarExpression *star = nullptr;
+	if (!FindStarExpression(expr, &star, true, false)) {
+		// no star expression: add it as-is
+		D_ASSERT(!star);
+		new_select_list.push_back(std::move(expr));
+		return;
+	}
+	D_ASSERT(star);
+	vector<unique_ptr<ParsedExpression>> star_list;
+	// we have star expressions! expand the list of star expressions
+	bind_context.GenerateAllColumnExpressions(*star, star_list);
+
+	if (star->expr) {
+		// COLUMNS with an expression
+		// two options:
+		// VARCHAR parameter <- this is a regular expression
+		// LIST of VARCHAR parameters <- this is a set of columns
+		TableFunctionBinder binder(*this, context);
+		auto child = star->expr->Copy();
+		auto result = binder.Bind(child);
+		if (!result->IsFoldable()) {
+			// cannot resolve parameters here
+			if (star->expr->HasParameter()) {
+				throw ParameterNotResolvedException();
+			} else {
+				throw BinderException("Unsupported expression in COLUMNS");
+			}
+		}
+		auto val = ExpressionExecutor::EvaluateScalar(context, *result);
+		if (val.type().id() == LogicalTypeId::VARCHAR) {
+			// regex
+			if (val.IsNull()) {
+				throw BinderException("COLUMNS does not support NULL as regex argument");
+			}
+			auto &regex_str = StringValue::Get(val);
+			duckdb_re2::RE2 regex(regex_str);
+			if (!regex.error().empty()) {
+				auto err = StringUtil::Format("Failed to compile regex \"%s\": %s", regex_str, regex.error());
+				throw BinderException(FormatError(*star, err));
+			}
+			vector<unique_ptr<ParsedExpression>> new_list;
+			for (idx_t i = 0; i < star_list.size(); i++) {
+				auto &colref = (ColumnRefExpression &)*star_list[i];
+				if (!RE2::PartialMatch(colref.GetColumnName(), regex)) {
+					continue;
+				}
+				new_list.push_back(std::move(star_list[i]));
+			}
+			if (new_list.empty()) {
+				auto err = StringUtil::Format("No matching columns found that match regex \"%s\"", regex_str);
+				throw BinderException(FormatError(*star, err));
+			}
+			star_list = std::move(new_list);
+		} else if (val.type().id() == LogicalTypeId::LIST &&
+		           ListType::GetChildType(val.type()).id() == LogicalTypeId::VARCHAR) {
+			// list of varchar columns
+			if (val.IsNull() || ListValue::GetChildren(val).empty()) {
+				auto err =
+				    StringUtil::Format("Star expression \"%s\" resulted in an empty set of columns", star->ToString());
+				throw BinderException(FormatError(*star, err));
+			}
+			auto &children = ListValue::GetChildren(val);
+			vector<unique_ptr<ParsedExpression>> new_list;
+			for (auto &child : children) {
+				auto qname = QualifiedName::Parse(StringValue::Get(child));
+				vector<string> names;
+				if (!qname.catalog.empty()) {
+					names.push_back(qname.catalog);
+				}
+				if (!qname.schema.empty()) {
+					names.push_back(qname.schema);
+				}
+				names.push_back(qname.name);
+				new_list.push_back(make_unique<ColumnRefExpression>(std::move(names)));
+			}
+			star_list = std::move(new_list);
+		} else {
+			throw BinderException(FormatError(
+			    *star, "COLUMNS expects either a VARCHAR argument (regex) or a LIST of VARCHAR (list of columns)"));
+		}
+	}
+
+	// now perform the replacement
+	for (idx_t i = 0; i < star_list.size(); i++) {
+		auto new_expr = expr->Copy();
+		ReplaceStarExpression(new_expr, star_list[i]);
+		new_select_list.push_back(std::move(new_expr));
+	}
+}
+
+void Binder::ExpandStarExpressions(vector<unique_ptr<ParsedExpression>> &select_list,
+                                   vector<unique_ptr<ParsedExpression>> &new_select_list) {
+	for (auto &select_element : select_list) {
+		ExpandStarExpression(std::move(select_element), new_select_list);
+	}
+}
+
+} // namespace duckdb