duckdb 0.7.2-dev0.0 → 0.7.2-dev1034.0

package/src/duckdb/src/parser/tableref/subqueryref.cpp

@@ -2,6 +2,8 @@
 
 #include "duckdb/common/limits.hpp"
 #include "duckdb/common/field_writer.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
 
 namespace duckdb {
 
@@ -35,6 +37,19 @@ void SubqueryRef::Serialize(FieldWriter &writer) const {
 	writer.WriteList<string>(column_name_alias);
 }
 
+void SubqueryRef::FormatSerialize(FormatSerializer &serializer) const {
+	TableRef::FormatSerialize(serializer);
+	serializer.WriteProperty("subquery", subquery);
+	serializer.WriteProperty("column_name_alias", column_name_alias);
+}
+
+unique_ptr<TableRef> SubqueryRef::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto subquery = deserializer.ReadProperty<unique_ptr<SelectStatement>>("subquery");
+	auto result = make_unique<SubqueryRef>(std::move(subquery));
+	deserializer.ReadProperty("column_name_alias", result->column_name_alias);
+	return std::move(result);
+}
+
 unique_ptr<TableRef> SubqueryRef::Deserialize(FieldReader &reader) {
 	auto subquery = reader.ReadRequiredSerializable<SelectStatement>();
 	auto result = make_unique<SubqueryRef>(std::move(subquery));

package/src/duckdb/src/parser/tableref/table_function.cpp

@@ -1,6 +1,8 @@
 #include "duckdb/parser/tableref/table_function_ref.hpp"
 #include "duckdb/common/vector.hpp"
 #include "duckdb/common/field_writer.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
 
 namespace duckdb {
 
@@ -25,6 +27,21 @@ void TableFunctionRef::Serialize(FieldWriter &writer) const {
 	writer.WriteList<string>(column_name_alias);
 }
 
+void TableFunctionRef::FormatSerialize(FormatSerializer &serializer) const {
+	TableRef::FormatSerialize(serializer);
+	serializer.WriteProperty("function", function);
+	serializer.WriteProperty("alias", alias);
+	serializer.WriteProperty("column_name_alias", column_name_alias);
+}
+
+unique_ptr<TableRef> TableFunctionRef::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto result = make_unique<TableFunctionRef>();
+	deserializer.ReadProperty("function", result->function);
+	deserializer.ReadProperty("alias", result->alias);
+	deserializer.ReadProperty("column_name_alias", result->column_name_alias);
+	return std::move(result);
+}
+
 unique_ptr<TableRef> TableFunctionRef::Deserialize(FieldReader &reader) {
 	auto result = make_unique<TableFunctionRef>();
 	result->function = reader.ReadRequiredSerializable<ParsedExpression>();

package/src/duckdb/src/parser/tableref.cpp

@@ -3,6 +3,8 @@
 #include "duckdb/common/printer.hpp"
 #include "duckdb/common/field_writer.hpp"
 #include "duckdb/parser/tableref/list.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
 #include "duckdb/common/to_string.hpp"
 
 namespace duckdb {
@@ -52,6 +54,50 @@ void TableRef::Serialize(Serializer &serializer) const {
 	writer.Finalize();
 }
 
+void TableRef::FormatSerialize(FormatSerializer &serializer) const {
+	serializer.WriteProperty("type", type);
+	serializer.WriteProperty("alias", alias);
+	serializer.WriteOptionalProperty("sample", sample);
+}
+
+unique_ptr<TableRef> TableRef::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto type = deserializer.ReadProperty<TableReferenceType>("type");
+	auto alias = deserializer.ReadProperty<string>("alias");
+	auto sample = deserializer.ReadOptionalProperty<unique_ptr<SampleOptions>>("sample");
+
+	unique_ptr<TableRef> result;
+
+	switch (type) {
+	case TableReferenceType::BASE_TABLE:
+		result = BaseTableRef::FormatDeserialize(deserializer);
+		break;
+	case TableReferenceType::JOIN:
+		result = JoinRef::FormatDeserialize(deserializer);
+		break;
+	case TableReferenceType::SUBQUERY:
+		result = SubqueryRef::FormatDeserialize(deserializer);
+		break;
+	case TableReferenceType::TABLE_FUNCTION:
+		result = TableFunctionRef::FormatDeserialize(deserializer);
+		break;
+	case TableReferenceType::EMPTY:
+		result = EmptyTableRef::FormatDeserialize(deserializer);
+		break;
+	case TableReferenceType::EXPRESSION_LIST:
+		result = ExpressionListRef::FormatDeserialize(deserializer);
+		break;
+	case TableReferenceType::PIVOT:
+		result = PivotRef::FormatDeserialize(deserializer);
+		break;
+	case TableReferenceType::CTE:
+	case TableReferenceType::INVALID:
+		throw InternalException("Unsupported type for TableRef::FormatDeserialize");
+	}
+	result->alias = alias;
+	result->sample = std::move(sample);
+	return result;
+}
+
 unique_ptr<TableRef> TableRef::Deserialize(Deserializer &source) {
 	FieldReader reader(source);
 
@@ -78,6 +124,9 @@ unique_ptr<TableRef> TableRef::Deserialize(Deserializer &source) {
 	case TableReferenceType::EXPRESSION_LIST:
 		result = ExpressionListRef::Deserialize(reader);
 		break;
+	case TableReferenceType::PIVOT:
+		result = PivotRef::Deserialize(reader);
+		break;
 	case TableReferenceType::CTE:
 	case TableReferenceType::INVALID:
 		throw InternalException("Unsupported type for TableRef::Deserialize");

package/src/duckdb/src/parser/transform/expression/transform_array_access.cpp

@@ -49,6 +49,17 @@ unique_ptr<ParsedExpression> Transformer::TransformArrayAccess(duckdb_libpgquery
 			result = make_unique<OperatorExpression>(ExpressionType::STRUCT_EXTRACT, std::move(children));
 			break;
 		}
+		case duckdb_libpgquery::T_PGFuncCall: {
+			auto func = (duckdb_libpgquery::PGFuncCall *)target;
+			auto function = TransformFuncCall(func);
+			if (function->type != ExpressionType::FUNCTION) {
+				throw ParserException("%s.%s() call must be a function", result->ToString(), function->ToString());
+			}
+			auto &f = (FunctionExpression &)*function;
+			f.children.insert(f.children.begin(), std::move(result));
+			result = std::move(function);
+			break;
+		}
 		default:
 			throw NotImplementedException("Unimplemented subscript type");
 		}

package/src/duckdb/src/parser/transform/expression/transform_bool_expr.cpp

@@ -37,7 +37,7 @@ unique_ptr<ParsedExpression> Transformer::TransformBoolExpr(duckdb_libpgquery::P
 			           next->type <= ExpressionType::COMPARE_GREATERTHANOREQUALTO) {
 				// NOT on a comparison: we can negate the comparison
 				// e.g. NOT(x > y) is equivalent to x <= y
-				next->type = NegateComparisionExpression(next->type);
+				next->type = NegateComparisonExpression(next->type);
 				result = std::move(next);
 			} else {
 				result = make_unique<OperatorExpression>(ExpressionType::OPERATOR_NOT, std::move(next));

package/src/duckdb/src/parser/transform/expression/transform_columnref.cpp

@@ -1,5 +1,6 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/parser/expression/columnref_expression.hpp"
+#include "duckdb/parser/expression/function_expression.hpp"
 #include "duckdb/parser/expression/star_expression.hpp"
 #include "duckdb/parser/transformer.hpp"
 
@@ -36,13 +37,27 @@ unique_ptr<ParsedExpression> Transformer::TransformStarExpression(duckdb_libpgqu
 			result->replace_list.insert(make_pair(std::move(exclude_entry), std::move(replace_expression)));
 		}
 	}
-	if (star->regex) {
+	if (star->expr) {
+		D_ASSERT(star->columns);
 		D_ASSERT(result->relation_name.empty());
 		D_ASSERT(result->exclude_list.empty());
 		D_ASSERT(result->replace_list.empty());
-		result->regex = star->regex;
+		result->expr = TransformExpression(star->expr);
+		if (result->expr->type == ExpressionType::STAR) {
+			auto child_star = (StarExpression *)result->expr.get();
+			result->exclude_list = std::move(child_star->exclude_list);
+			result->replace_list = std::move(child_star->replace_list);
+			result->expr.reset();
+		} else if (result->expr->type == ExpressionType::LAMBDA) {
+			vector<unique_ptr<ParsedExpression>> children;
+			children.push_back(make_unique<StarExpression>());
+			children.push_back(std::move(result->expr));
+			auto list_filter = make_unique<FunctionExpression>("list_filter", std::move(children));
+			result->expr = std::move(list_filter);
+		}
 	}
 	result->columns = star->columns;
+	result->query_location = star->location;
 	return std::move(result);
 }
 

package/src/duckdb/src/parser/transform/expression/transform_function.cpp

@@ -24,7 +24,7 @@ static ExpressionType WindowToExpressionType(string &fun_name) {
 		return ExpressionType::WINDOW_FIRST_VALUE;
 	} else if (fun_name == "last_value" || fun_name == "last") {
 		return ExpressionType::WINDOW_LAST_VALUE;
-	} else if (fun_name == "nth_value" || fun_name == "last") {
+	} else if (fun_name == "nth_value") {
 		return ExpressionType::WINDOW_NTH_VALUE;
 	} else if (fun_name == "cume_dist") {
 		return ExpressionType::WINDOW_CUME_DIST;
@@ -94,6 +94,27 @@ void Transformer::TransformWindowFrame(duckdb_libpgquery::PGWindowDef *window_sp
 	}
 }
 
+bool Transformer::ExpressionIsEmptyStar(ParsedExpression &expr) {
+	if (expr.expression_class != ExpressionClass::STAR) {
+		return false;
+	}
+	auto &star = (StarExpression &)expr;
+	if (!star.columns && star.exclude_list.empty() && star.replace_list.empty()) {
+		return true;
+	}
+	return false;
+}
+
+bool Transformer::InWindowDefinition() {
+	if (in_window_definition) {
+		return true;
+	}
+	if (parent) {
+		return parent->InWindowDefinition();
+	}
+	return false;
+}
+
 unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::PGFuncCall *root) {
 	auto name = root->funcname;
 	string catalog, schema, function_name;
@@ -113,12 +134,25 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
 		schema = INVALID_SCHEMA;
 		function_name = reinterpret_cast<duckdb_libpgquery::PGValue *>(name->head->data.ptr_value)->val.str;
 	} else {
-		throw InternalException("TransformFuncCall - Expected 1, 2 or 3 qualifications");
+		throw ParserException("TransformFuncCall - Expected 1, 2 or 3 qualifications");
 	}
 
-	auto lowercase_name = StringUtil::Lower(function_name);
+	//  transform children
+	vector<unique_ptr<ParsedExpression>> children;
+	if (root->args) {
+		TransformExpressionList(*root->args, children);
+	}
+	if (children.size() == 1 && ExpressionIsEmptyStar(*children[0]) && !root->agg_distinct && !root->agg_order) {
+		// COUNT(*) gets translated into COUNT()
+		children.clear();
+	}
 
+	auto lowercase_name = StringUtil::Lower(function_name);
 	if (root->over) {
+		if (InWindowDefinition()) {
+			throw ParserException("window functions are not allowed in window definitions");
+		}
+
 		const auto win_fun_type = WindowToExpressionType(lowercase_name);
 		if (win_fun_type == ExpressionType::INVALID) {
 			throw InternalException("Unknown/unsupported window function");
@@ -151,39 +185,32 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
 			expr->filter_expr = std::move(filter_expr);
 		}
 
-		if (root->args) {
-			vector<unique_ptr<ParsedExpression>> function_list;
-			TransformExpressionList(*root->args, function_list);
-
-			if (win_fun_type == ExpressionType::WINDOW_AGGREGATE) {
-				for (auto &child : function_list) {
-					expr->children.push_back(std::move(child));
+		if (win_fun_type == ExpressionType::WINDOW_AGGREGATE) {
+			expr->children = std::move(children);
+		} else {
+			if (!children.empty()) {
+				expr->children.push_back(std::move(children[0]));
+			}
+			if (win_fun_type == ExpressionType::WINDOW_LEAD || win_fun_type == ExpressionType::WINDOW_LAG) {
+				if (children.size() > 1) {
+					expr->offset_expr = std::move(children[1]);
 				}
-			} else {
-				if (!function_list.empty()) {
-					expr->children.push_back(std::move(function_list[0]));
+				if (children.size() > 2) {
+					expr->default_expr = std::move(children[2]);
+				}
+				if (children.size() > 3) {
+					throw ParserException("Incorrect number of parameters for function %s", lowercase_name);
 				}
-				if (win_fun_type == ExpressionType::WINDOW_LEAD || win_fun_type == ExpressionType::WINDOW_LAG) {
-					if (function_list.size() > 1) {
-						expr->offset_expr = std::move(function_list[1]);
-					}
-					if (function_list.size() > 2) {
-						expr->default_expr = std::move(function_list[2]);
-					}
-					if (function_list.size() > 3) {
-						throw ParserException("Incorrect number of parameters for function %s", lowercase_name);
-					}
-				} else if (win_fun_type == ExpressionType::WINDOW_NTH_VALUE) {
-					if (function_list.size() > 1) {
-						expr->children.push_back(std::move(function_list[1]));
-					}
-					if (function_list.size() > 2) {
-						throw ParserException("Incorrect number of parameters for function %s", lowercase_name);
-					}
-				} else {
-					if (function_list.size() > 1) {
-						throw ParserException("Incorrect number of parameters for function %s", lowercase_name);
-					}
+			} else if (win_fun_type == ExpressionType::WINDOW_NTH_VALUE) {
+				if (children.size() > 1) {
+					expr->children.push_back(std::move(children[1]));
+				}
+				if (children.size() > 2) {
+					throw ParserException("Incorrect number of parameters for function %s", lowercase_name);
+				}
+			} else {
+				if (children.size() > 1) {
+					throw ParserException("Incorrect number of parameters for function %s", lowercase_name);
 				}
 			}
 		}
@@ -205,8 +232,10 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
 			window_ref = it->second;
 			D_ASSERT(window_ref);
 		}
+		in_window_definition = true;
 		TransformWindowDef(window_ref, expr.get());
 		TransformWindowFrame(window_spec, expr.get());
+		in_window_definition = false;
 		expr->query_location = root->location;
 		return std::move(expr);
 	}
@@ -215,14 +244,6 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
 		throw ParserException("IGNORE NULLS is not supported for non-window functions");
 	}
 
-	//  TransformExpressionList??
-	vector<unique_ptr<ParsedExpression>> children;
-	if (root->args != nullptr) {
-		for (auto node = root->args->head; node != nullptr; node = node->next) {
-			auto child_expr = TransformExpression((duckdb_libpgquery::PGNode *)node->data.ptr_value);
-			children.push_back(std::move(child_expr));
-		}
-	}
 	unique_ptr<ParsedExpression> filter_expr;
 	if (root->agg_filter) {
 		filter_expr = TransformExpression(root->agg_filter);

package/src/duckdb/src/parser/transform/expression/transform_operator.cpp

@@ -84,7 +84,7 @@ unique_ptr<ParsedExpression> Transformer::TransformAExprInternal(duckdb_libpgque
 			// ALL sublink is equivalent to NOT(ANY) with inverted comparison
 			// e.g. [= ALL()] is equivalent to [NOT(<> ANY())]
 			// first invert the comparison type
-			subquery_expr->comparison_type = NegateComparisionExpression(subquery_expr->comparison_type);
+			subquery_expr->comparison_type = NegateComparisonExpression(subquery_expr->comparison_type);
 			return make_unique<OperatorExpression>(ExpressionType::OPERATOR_NOT, std::move(subquery_expr));
 		}
 		return std::move(subquery_expr);

package/src/duckdb/src/parser/transform/expression/transform_subquery.cpp

@@ -44,7 +44,7 @@ unique_ptr<ParsedExpression> Transformer::TransformSubquery(duckdb_libpgquery::P
 			// ALL sublink is equivalent to NOT(ANY) with inverted comparison
 			// e.g. [= ALL()] is equivalent to [NOT(<> ANY())]
 			// first invert the comparison type
-			subquery_expr->comparison_type = NegateComparisionExpression(subquery_expr->comparison_type);
+			subquery_expr->comparison_type = NegateComparisonExpression(subquery_expr->comparison_type);
 			return make_unique<OperatorExpression>(ExpressionType::OPERATOR_NOT, std::move(subquery_expr));
 		}
 		break;

package/src/duckdb/src/parser/transform/helpers/transform_alias.cpp

@@ -2,16 +2,22 @@
 
 namespace duckdb {
 
+vector<string> Transformer::TransformStringList(duckdb_libpgquery::PGList *list) {
+	vector<string> result;
+	if (!list) {
+		return result;
+	}
+	for (auto node = list->head; node != nullptr; node = node->next) {
+		result.emplace_back(reinterpret_cast<duckdb_libpgquery::PGValue *>(node->data.ptr_value)->val.str);
+	}
+	return result;
+}
+
 string Transformer::TransformAlias(duckdb_libpgquery::PGAlias *root, vector<string> &column_name_alias) {
 	if (!root) {
 		return "";
 	}
-	if (root->colnames) {
-		for (auto node = root->colnames->head; node != nullptr; node = node->next) {
-			column_name_alias.emplace_back(
-			    reinterpret_cast<duckdb_libpgquery::PGValue *>(node->data.ptr_value)->val.str);
-		}
-	}
+	column_name_alias = TransformStringList(root->colnames);
 	return root->aliasname;
 }
 

package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp

@@ -6,9 +6,33 @@
 
 namespace duckdb {
 
+unique_ptr<CommonTableExpressionInfo> CommonTableExpressionInfo::Copy() {
+	auto result = make_unique<CommonTableExpressionInfo>();
+	result->aliases = aliases;
+	result->query = unique_ptr_cast<SQLStatement, SelectStatement>(query->Copy());
+	return result;
+}
+
+void Transformer::ExtractCTEsRecursive(CommonTableExpressionMap &cte_map) {
+	for (auto &cte_entry : stored_cte_map) {
+		for (auto &entry : cte_entry->map) {
+			auto found_entry = cte_map.map.find(entry.first);
+			if (found_entry != cte_map.map.end()) {
+				// entry already present - use top-most entry
+				continue;
+			}
+			cte_map.map[entry.first] = entry.second->Copy();
+		}
+	}
+	if (parent) {
+		parent->ExtractCTEsRecursive(cte_map);
+	}
+}
+
 void Transformer::TransformCTE(duckdb_libpgquery::PGWithClause *de_with_clause, CommonTableExpressionMap &cte_map) {
 	// TODO: might need to update in case of future lawsuit
 	D_ASSERT(de_with_clause);
+	stored_cte_map.push_back(&cte_map);
 
 	D_ASSERT(de_with_clause->ctes);
 	for (auto cte_ele = de_with_clause->ctes->head; cte_ele != nullptr; cte_ele = cte_ele->next) {

package/src/duckdb/src/parser/transform/helpers/transform_groupby.cpp

@@ -176,6 +176,13 @@ bool Transformer::TransformGroupBy(duckdb_libpgquery::PGList *group, SelectNode
 			result.grouping_sets = std::move(new_sets);
 		}
 	}
+	if (result.group_expressions.size() == 1 && result.grouping_sets.size() == 1 &&
+	    ExpressionIsEmptyStar(*result.group_expressions[0])) {
+		// GROUP BY *
+		result.group_expressions.clear();
+		result.grouping_sets.clear();
+		select_node.aggregate_handling = AggregateHandling::FORCE_AGGREGATES;
+	}
 	return true;
 }
 

package/src/duckdb/src/parser/transform/helpers/transform_orderby.cpp

@@ -36,13 +36,6 @@ bool Transformer::TransformOrderBy(duckdb_libpgquery::PGList *order, vector<Orde
 				throw NotImplementedException("Unimplemented order by type");
 			}
 			auto order_expression = TransformExpression(target);
-			if (order_expression->GetExpressionClass() == ExpressionClass::STAR) {
-				auto &star_expr = (StarExpression &)*order_expression;
-				D_ASSERT(star_expr.relation_name.empty());
-				if (star_expr.columns) {
-					throw ParserException("COLUMNS expr is not supported in ORDER BY");
-				}
-			}
 			result.emplace_back(type, null_order, std::move(order_expression));
 		} else {
 			throw NotImplementedException("ORDER BY list member type %d\n", temp->type);

package/src/duckdb/src/parser/transform/helpers/transform_typename.cpp

@@ -52,7 +52,8 @@ LogicalType Transformer::TransformTypeName(duckdb_libpgquery::PGTypeName *type_n
 			children.push_back(make_pair(entry_name, entry_type));
 		}
 		D_ASSERT(!children.empty());
-		result_type = LogicalType::STRUCT(std::move(children));
+		result_type = LogicalType::STRUCT(children);
+
 	} else if (base_type == LogicalTypeId::MAP) {
 
 		if (!type_name->typmods || type_name->typmods->length != 2) {
@@ -207,7 +208,7 @@ LogicalType Transformer::TransformTypeName(duckdb_libpgquery::PGTypeName *type_n
 		// array bounds: turn the type into a list
 		idx_t extra_stack = 0;
 		for (auto cell = type_name->arrayBounds->head; cell != nullptr; cell = cell->next) {
-			result_type = LogicalType::LIST(std::move(result_type));
+			result_type = LogicalType::LIST(result_type);
 			StackCheck(extra_stack++);
 		}
 	}

package/src/duckdb/src/parser/transform/statement/transform_create_function.cpp

@@ -29,6 +29,10 @@ unique_ptr<CreateStatement> Transformer::TransformCreateFunction(duckdb_libpgque
 		auto query_node = TransformSelect(stmt->query, true)->node->Copy();
 		macro_func = make_unique<TableMacroFunction>(std::move(query_node));
 	}
+	if (HasPivotEntries()) {
+		throw ParserException("Cannot use PIVOT statement syntax in a macro. Use the SQL standard PIVOT syntax in the "
+		                      "FROM clause instead.");
+	}
 
 	auto info =
 	    make_unique<CreateMacroInfo>((stmt->function ? CatalogType::MACRO_ENTRY : CatalogType::TABLE_MACRO_ENTRY));

package/src/duckdb/src/parser/transform/statement/transform_create_view.cpp

@@ -26,6 +26,10 @@ unique_ptr<CreateStatement> Transformer::TransformCreateView(duckdb_libpgquery::
 	info->on_conflict = TransformOnConflict(stmt->onconflict);
 
 	info->query = TransformSelect(stmt->query, false);
+	if (HasPivotEntries()) {
+		throw ParserException("Cannot use PIVOT statement syntax in a view. Use the SQL standard PIVOT syntax in the "
+		                      "FROM clause instead.");
+	}
 
 	if (stmt->aliases && stmt->aliases->length > 0) {
 		for (auto c = stmt->aliases->head; c != nullptr; c = lnext(c)) {