duckdb 0.7.1 → 0.7.2-dev1034.0

package/src/duckdb/src/planner/binder/expression/bind_star_expression.cpp

@@ -0,0 +1,176 @@
+#include "duckdb/planner/binder.hpp"
+#include "duckdb/parser/expression/star_expression.hpp"
+#include "duckdb/parser/expression/constant_expression.hpp"
+#include "duckdb/planner/expression_binder/table_function_binder.hpp"
+#include "duckdb/parser/parsed_expression_iterator.hpp"
+#include "duckdb/execution/expression_executor.hpp"
+#include "re2/re2.h"
+
+namespace duckdb {
+
+bool Binder::FindStarExpression(unique_ptr<ParsedExpression> &expr, StarExpression **star, bool is_root,
+                                bool in_columns) {
+	bool has_star = false;
+	if (expr->GetExpressionClass() == ExpressionClass::STAR) {
+		auto current_star = (StarExpression *)expr.get();
+		if (!current_star->columns) {
+			if (is_root) {
+				*star = current_star;
+				return true;
+			}
+			if (!in_columns) {
+				throw BinderException(
+				    "STAR expression is only allowed as the root element of an expression. Use COLUMNS(*) instead.");
+			}
+			// star expression inside a COLUMNS - convert to a constant list
+			if (!current_star->replace_list.empty()) {
+				throw BinderException(
+				    "STAR expression with REPLACE list is only allowed as the root element of COLUMNS");
+			}
+			vector<unique_ptr<ParsedExpression>> star_list;
+			bind_context.GenerateAllColumnExpressions(*current_star, star_list);
+
+			vector<Value> values;
+			values.reserve(star_list.size());
+			for (auto &expr : star_list) {
+				values.emplace_back(expr->ToString());
+			}
+			D_ASSERT(!values.empty());
+
+			expr = make_unique<ConstantExpression>(Value::LIST(LogicalType::VARCHAR, values));
+			return true;
+		}
+		if (in_columns) {
+			throw BinderException("COLUMNS expression is not allowed inside another COLUMNS expression");
+		}
+		in_columns = true;
+		if (*star) {
+			// we can have multiple
+			if (!StarExpression::Equal(*star, current_star)) {
+				throw BinderException(
+				    FormatError(*expr, "Multiple different STAR/COLUMNS in the same expression are not supported"));
+			}
+			return true;
+		}
+		*star = current_star;
+		has_star = true;
+	}
+	ParsedExpressionIterator::EnumerateChildren(*expr, [&](unique_ptr<ParsedExpression> &child_expr) {
+		if (FindStarExpression(child_expr, star, false, in_columns)) {
+			has_star = true;
+		}
+	});
+	return has_star;
+}
+
+void Binder::ReplaceStarExpression(unique_ptr<ParsedExpression> &expr, unique_ptr<ParsedExpression> &replacement) {
+	D_ASSERT(expr);
+	if (expr->GetExpressionClass() == ExpressionClass::STAR) {
+		D_ASSERT(replacement);
+		expr = replacement->Copy();
+		return;
+	}
+	ParsedExpressionIterator::EnumerateChildren(
+	    *expr, [&](unique_ptr<ParsedExpression> &child_expr) { ReplaceStarExpression(child_expr, replacement); });
+}
+
+void Binder::ExpandStarExpression(unique_ptr<ParsedExpression> expr,
+                                  vector<unique_ptr<ParsedExpression>> &new_select_list) {
+	StarExpression *star = nullptr;
+	if (!FindStarExpression(expr, &star, true, false)) {
+		// no star expression: add it as-is
+		D_ASSERT(!star);
+		new_select_list.push_back(std::move(expr));
+		return;
+	}
+	D_ASSERT(star);
+	vector<unique_ptr<ParsedExpression>> star_list;
+	// we have star expressions! expand the list of star expressions
+	bind_context.GenerateAllColumnExpressions(*star, star_list);
+
+	if (star->expr) {
+		// COLUMNS with an expression
+		// two options:
+		// VARCHAR parameter <- this is a regular expression
+		// LIST of VARCHAR parameters <- this is a set of columns
+		TableFunctionBinder binder(*this, context);
+		auto child = star->expr->Copy();
+		auto result = binder.Bind(child);
+		if (!result->IsFoldable()) {
+			// cannot resolve parameters here
+			if (star->expr->HasParameter()) {
+				throw ParameterNotResolvedException();
+			} else {
+				throw BinderException("Unsupported expression in COLUMNS");
+			}
+		}
+		auto val = ExpressionExecutor::EvaluateScalar(context, *result);
+		if (val.type().id() == LogicalTypeId::VARCHAR) {
+			// regex
+			if (val.IsNull()) {
+				throw BinderException("COLUMNS does not support NULL as regex argument");
+			}
+			auto &regex_str = StringValue::Get(val);
+			duckdb_re2::RE2 regex(regex_str);
+			if (!regex.error().empty()) {
+				auto err = StringUtil::Format("Failed to compile regex \"%s\": %s", regex_str, regex.error());
+				throw BinderException(FormatError(*star, err));
+			}
+			vector<unique_ptr<ParsedExpression>> new_list;
+			for (idx_t i = 0; i < star_list.size(); i++) {
+				auto &colref = (ColumnRefExpression &)*star_list[i];
+				if (!RE2::PartialMatch(colref.GetColumnName(), regex)) {
+					continue;
+				}
+				new_list.push_back(std::move(star_list[i]));
+			}
+			if (new_list.empty()) {
+				auto err = StringUtil::Format("No matching columns found that match regex \"%s\"", regex_str);
+				throw BinderException(FormatError(*star, err));
+			}
+			star_list = std::move(new_list);
+		} else if (val.type().id() == LogicalTypeId::LIST &&
+		           ListType::GetChildType(val.type()).id() == LogicalTypeId::VARCHAR) {
+			// list of varchar columns
+			if (val.IsNull() || ListValue::GetChildren(val).empty()) {
+				auto err =
+				    StringUtil::Format("Star expression \"%s\" resulted in an empty set of columns", star->ToString());
+				throw BinderException(FormatError(*star, err));
+			}
+			auto &children = ListValue::GetChildren(val);
+			vector<unique_ptr<ParsedExpression>> new_list;
+			for (auto &child : children) {
+				auto qname = QualifiedName::Parse(StringValue::Get(child));
+				vector<string> names;
+				if (!qname.catalog.empty()) {
+					names.push_back(qname.catalog);
+				}
+				if (!qname.schema.empty()) {
+					names.push_back(qname.schema);
+				}
+				names.push_back(qname.name);
+				new_list.push_back(make_unique<ColumnRefExpression>(std::move(names)));
+			}
+			star_list = std::move(new_list);
+		} else {
+			throw BinderException(FormatError(
+			    *star, "COLUMNS expects either a VARCHAR argument (regex) or a LIST of VARCHAR (list of columns)"));
+		}
+	}
+
+	// now perform the replacement
+	for (idx_t i = 0; i < star_list.size(); i++) {
+		auto new_expr = expr->Copy();
+		ReplaceStarExpression(new_expr, star_list[i]);
+		new_select_list.push_back(std::move(new_expr));
+	}
+}
+
+void Binder::ExpandStarExpressions(vector<unique_ptr<ParsedExpression>> &select_list,
+                                   vector<unique_ptr<ParsedExpression>> &new_select_list) {
+	for (auto &select_element : select_list) {
+		ExpandStarExpression(std::move(select_element), new_select_list);
+	}
+}
+
+} // namespace duckdb

package/src/duckdb/src/planner/binder/expression/bind_subquery_expression.cpp

@@ -32,6 +32,10 @@ public:
 	void Serialize(FieldWriter &writer) const override {
 		throw InternalException("Cannot serialize bound subquery node");
 	}
+
+	void FormatSerialize(FormatSerializer &serializer) const override {
+		throw InternalException("Cannot serialize bound subquery node");
+	}
 };
 
 BindResult ExpressionBinder::BindExpression(SubqueryExpression &expr, idx_t depth) {

package/src/duckdb/src/planner/binder/expression/bind_unnest_expression.cpp

@@ -3,21 +3,86 @@
 #include "duckdb/parser/expression/function_expression.hpp"
 #include "duckdb/planner/expression/bound_aggregate_expression.hpp"
 #include "duckdb/planner/expression/bound_columnref_expression.hpp"
+#include "duckdb/planner/expression/bound_constant_expression.hpp"
+#include "duckdb/planner/expression/bound_function_expression.hpp"
 #include "duckdb/planner/expression/bound_parameter_expression.hpp"
 #include "duckdb/planner/expression_binder/aggregate_binder.hpp"
 #include "duckdb/planner/expression_binder/select_binder.hpp"
 #include "duckdb/planner/query_node/bound_select_node.hpp"
 #include "duckdb/planner/expression/bound_unnest_expression.hpp"
 #include "duckdb/planner/binder.hpp"
+#include "duckdb/function/scalar/nested_functions.hpp"
+#include "duckdb/execution/expression_executor.hpp"
 
 namespace duckdb {
 
-BindResult SelectBinder::BindUnnest(FunctionExpression &function, idx_t depth) {
+unique_ptr<Expression> CreateBoundStructExtract(ClientContext &context, unique_ptr<Expression> expr, string key) {
+	vector<unique_ptr<Expression>> arguments;
+	arguments.push_back(std::move(expr));
+	arguments.push_back(make_unique<BoundConstantExpression>(Value(key)));
+	auto extract_function = StructExtractFun::GetFunction();
+	auto bind_info = extract_function.bind(context, extract_function, arguments);
+	auto return_type = extract_function.return_type;
+	auto result = make_unique<BoundFunctionExpression>(return_type, std::move(extract_function), std::move(arguments),
+	                                                   std::move(bind_info));
+	result->alias = std::move(key);
+	return std::move(result);
+}
+
+BindResult SelectBinder::BindUnnest(FunctionExpression &function, idx_t depth, bool root_expression) {
 	// bind the children of the function expression
+	if (depth > 0) {
+		return BindResult(binder.FormatError(function, "UNNEST() for correlated expressions is not supported yet"));
+	}
 	string error;
+	if (function.children.empty()) {
+		return BindResult(binder.FormatError(function, "UNNEST() requires a single argument"));
+	}
+	idx_t max_depth = 1;
 	if (function.children.size() != 1) {
-		return BindResult(binder.FormatError(function, "Unnest() needs exactly one child expressions"));
+		bool has_parameter = false;
+		bool supported_argument = false;
+		for (idx_t i = 1; i < function.children.size(); i++) {
+			if (has_parameter) {
+				return BindResult(binder.FormatError(function, "UNNEST() only supports a single additional argument"));
+			}
+			if (function.children[i]->HasParameter()) {
+				throw ParameterNotAllowedException("Parameter not allowed in unnest parameter");
+			}
+			if (!function.children[i]->IsScalar()) {
+				break;
+			}
+			auto alias = function.children[i]->alias;
+			BindChild(function.children[i], depth, error);
+			if (!error.empty()) {
+				return BindResult(error);
+			}
+			auto &const_child = (BoundExpression &)*function.children[i];
+			auto value = ExpressionExecutor::EvaluateScalar(context, *const_child.expr, true);
+			if (alias == "recursive") {
+				auto recursive = value.GetValue<bool>();
+				if (recursive) {
+					max_depth = NumericLimits<idx_t>::Maximum();
+				}
+			} else if (alias == "max_depth") {
+				max_depth = value.GetValue<uint32_t>();
+				if (max_depth == 0) {
+					throw BinderException("UNNEST cannot have a max depth of 0");
+				}
+			} else if (!alias.empty()) {
+				throw BinderException("Unsupported parameter \"%s\" for unnest", alias);
+			} else {
+				break;
+			}
+			has_parameter = true;
+			supported_argument = true;
+		}
+		if (!supported_argument) {
+			return BindResult(binder.FormatError(function, "UNNEST - unsupported extra argument, unnest only supports "
+			                                               "recursive := [true/false] or max_depth := #"));
+		}
 	}
+	unnest_level++;
 	BindChild(function.children[0], depth, error);
 	if (!error.empty()) {
 		// failed to bind
@@ -30,37 +95,111 @@ BindResult SelectBinder::BindUnnest(FunctionExpression &function, idx_t depth) {
 	}
 	auto &child = (BoundExpression &)*function.children[0];
 	auto &child_type = child.expr->return_type;
+	unnest_level--;
 
-	if (child_type.id() != LogicalTypeId::LIST && child_type.id() != LogicalTypeId::SQLNULL &&
-	    child_type.id() != LogicalTypeId::UNKNOWN) {
-		return BindResult(binder.FormatError(function, "Unnest() can only be applied to lists and NULL"));
-	}
-
-	if (depth > 0) {
-		throw BinderException(binder.FormatError(function, "Unnest() for correlated expressions is not supported yet"));
+	if (unnest_level > 0) {
+		throw BinderException(
+		    "Nested UNNEST calls are not supported - use UNNEST(x, recursive := true) to unnest multiple levels");
 	}
 
-	auto return_type = LogicalType(LogicalTypeId::SQLNULL);
-	if (child_type.id() == LogicalTypeId::LIST) {
-		return_type = ListType::GetChildType(child_type);
-	} else if (child_type.id() == LogicalTypeId::UNKNOWN) {
+	switch (child_type.id()) {
+	case LogicalTypeId::UNKNOWN:
 		throw ParameterNotResolvedException();
+	case LogicalTypeId::LIST:
+	case LogicalTypeId::STRUCT:
+	case LogicalTypeId::SQLNULL:
+		break;
+	default:
+		return BindResult(binder.FormatError(function, "UNNEST() can only be applied to lists, structs and NULL"));
 	}
 
-	auto result = make_unique<BoundUnnestExpression>(return_type);
-	result->child = std::move(child.expr);
+	idx_t list_unnests;
+	idx_t struct_unnests = 0;
 
-	auto unnest_index = node.unnests.size();
-	node.unnests.push_back(std::move(result));
-
-	// TODO what if we have multiple unnests in the same projection list? ignore for now
+	auto unnest_expr = std::move(child.expr);
+	if (child_type.id() == LogicalTypeId::SQLNULL) {
+		list_unnests = 1;
+	} else {
+		// first do all of the list unnests
+		auto type = child_type;
+		list_unnests = 0;
+		while (type.id() == LogicalTypeId::LIST) {
+			type = ListType::GetChildType(type);
+			list_unnests++;
+			if (list_unnests >= max_depth) {
+				break;
+			}
+		}
+		// unnest structs all the way afterwards, if there are any
+		if (type.id() == LogicalTypeId::STRUCT) {
+			struct_unnests = max_depth - list_unnests;
+		}
+	}
+	if (struct_unnests > 0 && !root_expression) {
+		return BindResult(binder.FormatError(
+		    function, "UNNEST() on a struct column can only be applied as the root element of a SELECT expression"));
+	}
+	// perform all of the list unnests first
+	auto return_type = child_type;
+	for (idx_t current_depth = 0; current_depth < list_unnests; current_depth++) {
+		if (return_type.id() == LogicalTypeId::LIST) {
+			return_type = ListType::GetChildType(return_type);
+		}
+		auto result = make_unique<BoundUnnestExpression>(return_type);
+		result->child = std::move(unnest_expr);
+		auto alias = function.alias.empty() ? result->ToString() : function.alias;
 
-	// now create a column reference referring to the unnest
-	auto colref = make_unique<BoundColumnRefExpression>(
-	    function.alias.empty() ? node.unnests[unnest_index]->ToString() : function.alias, return_type,
-	    ColumnBinding(node.unnest_index, unnest_index), depth);
+		auto current_level = unnest_level + list_unnests - current_depth - 1;
+		auto entry = node.unnests.find(current_level);
+		idx_t unnest_table_index;
+		idx_t unnest_column_index;
+		if (entry == node.unnests.end()) {
+			BoundUnnestNode unnest_node;
+			unnest_node.index = binder.GenerateTableIndex();
+			unnest_node.expressions.push_back(std::move(result));
+			unnest_table_index = unnest_node.index;
+			unnest_column_index = 0;
+			node.unnests.insert(make_pair(current_level, std::move(unnest_node)));
+		} else {
+			unnest_table_index = entry->second.index;
+			unnest_column_index = entry->second.expressions.size();
+			entry->second.expressions.push_back(std::move(result));
+		}
+		// now create a column reference referring to the unnest
+		unnest_expr = make_unique<BoundColumnRefExpression>(
+		    std::move(alias), return_type, ColumnBinding(unnest_table_index, unnest_column_index), depth);
+	}
+	// now perform struct unnests, if any
+	if (struct_unnests > 0) {
+		vector<unique_ptr<Expression>> struct_expressions;
+		struct_expressions.push_back(std::move(unnest_expr));
 
-	return BindResult(std::move(colref));
+		for (idx_t i = 0; i < struct_unnests; i++) {
+			vector<unique_ptr<Expression>> new_expressions;
+			// check if there are any structs left
+			bool has_structs = false;
+			for (auto &expr : struct_expressions) {
+				if (expr->return_type.id() == LogicalTypeId::STRUCT) {
+					// struct! push a struct_extract
+					auto &child_types = StructType::GetChildTypes(expr->return_type);
+					for (auto &entry : child_types) {
+						new_expressions.push_back(CreateBoundStructExtract(context, expr->Copy(), entry.first));
+					}
+					has_structs = true;
+				} else {
+					// not a struct - push as-is
+					new_expressions.push_back(std::move(expr));
+				}
+			}
+			struct_expressions = std::move(new_expressions);
+			if (!has_structs) {
+				break;
+			}
+		}
+		expanded_expressions = std::move(struct_expressions);
+		unnest_expr = make_unique<BoundConstantExpression>(Value(42));
+	}
+	return BindResult(std::move(unnest_expr));
 }
 
 } // namespace duckdb

package/src/duckdb/src/planner/binder/expression/bind_window_expression.cpp

@@ -119,10 +119,10 @@ static LogicalType BindRangeExpression(ClientContext &context, const string &nam
 	return bound.expr->return_type;
 }
 
-BindResult SelectBinder::BindWindow(WindowExpression &window, idx_t depth) {
+BindResult BaseSelectBinder::BindWindow(WindowExpression &window, idx_t depth) {
 	auto name = window.GetName();
 
-	QueryErrorContext error_context(binder.root_statement, window.query_location);
+	QueryErrorContext error_context(binder.GetRootStatement(), window.query_location);
 	if (inside_window) {
 		throw BinderException(error_context.FormatError("window function calls cannot be nested"));
 	}