duckdb 0.7.2-dev16.0 → 0.7.2-dev314.0

package/src/duckdb/src/parser/query_node.cpp

@@ -174,7 +174,7 @@ unique_ptr<QueryNode> QueryNode::Deserialize(Deserializer &main_source) {
 	// cte_map
 	auto cte_count = reader.ReadRequired<uint32_t>();
 	auto &source = reader.GetSource();
-	unordered_map<string, unique_ptr<CommonTableExpressionInfo>> new_map;
+	case_insensitive_map_t<unique_ptr<CommonTableExpressionInfo>> new_map;
 	for (idx_t i = 0; i < cte_count; i++) {
 		auto name = source.Read<string>();
 		auto info = make_unique<CommonTableExpressionInfo>();

package/src/duckdb/src/parser/statement/multi_statement.cpp

@@ -0,0 +1,18 @@
+#include "duckdb/parser/statement/multi_statement.hpp"
+
+namespace duckdb {
+
+MultiStatement::MultiStatement() : SQLStatement(StatementType::MULTI_STATEMENT) {
+}
+
+MultiStatement::MultiStatement(const MultiStatement &other) : SQLStatement(other) {
+	for (auto &stmt : other.statements) {
+		statements.push_back(stmt->Copy());
+	}
+}
+
+unique_ptr<SQLStatement> MultiStatement::Copy() const {
+	return unique_ptr<MultiStatement>(new MultiStatement(*this));
+}
+
+} // namespace duckdb

package/src/duckdb/src/parser/tableref/pivotref.cpp

@@ -0,0 +1,296 @@
+#include "duckdb/parser/tableref/pivotref.hpp"
+
+#include "duckdb/common/limits.hpp"
+#include "duckdb/common/field_writer.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// PivotColumn
+//===--------------------------------------------------------------------===//
+string PivotColumn::ToString() const {
+	string result;
+	if (names.size() == 1) {
+		result += KeywordHelper::WriteOptionallyQuoted(names[0]);
+	} else {
+		result += "(";
+		for (idx_t n = 0; n < names.size(); n++) {
+			if (n > 0) {
+				result += ", ";
+			}
+			result += KeywordHelper::WriteOptionallyQuoted(names[n]);
+		}
+		result += ")";
+	}
+	result += " IN ";
+	if (pivot_enum.empty()) {
+		result += "(";
+		for (idx_t e = 0; e < entries.size(); e++) {
+			auto &entry = entries[e];
+			if (e > 0) {
+				result += ", ";
+			}
+			if (entry.star_expr) {
+				D_ASSERT(entry.values.empty());
+				result += entry.star_expr->ToString();
+			} else if (entry.values.size() == 1) {
+				result += entry.values[0].ToSQLString();
+			} else {
+				result += "(";
+				for (idx_t v = 0; v < entry.values.size(); v++) {
+					if (v > 0) {
+						result += ", ";
+					}
+					result += entry.values[v].ToSQLString();
+				}
+				result += ")";
+			}
+			if (!entry.alias.empty()) {
+				result += " AS " + KeywordHelper::WriteOptionallyQuoted(entry.alias);
+			}
+		}
+		result += ")";
+	} else {
+		result += KeywordHelper::WriteOptionallyQuoted(pivot_enum);
+	}
+	return result;
+}
+
+bool PivotColumnEntry::Equals(const PivotColumnEntry &other) const {
+	if (alias != other.alias) {
+		return false;
+	}
+	if (values.size() != other.values.size()) {
+		return false;
+	}
+	for (idx_t i = 0; i < values.size(); i++) {
+		if (!Value::NotDistinctFrom(values[i], other.values[i])) {
+			return false;
+		}
+	}
+	return true;
+}
+
+bool PivotColumn::Equals(const PivotColumn &other) const {
+	if (other.names != names) {
+		return false;
+	}
+	if (other.pivot_enum != pivot_enum) {
+		return false;
+	}
+	if (other.entries.size() != entries.size()) {
+		return false;
+	}
+	for (idx_t i = 0; i < entries.size(); i++) {
+		if (!entries[i].Equals(other.entries[i])) {
+			return false;
+		}
+	}
+	return true;
+}
+
+PivotColumn PivotColumn::Copy() const {
+	PivotColumn result;
+	result.names = names;
+	for (auto &entry : entries) {
+		result.entries.push_back(entry.Copy());
+	}
+	result.pivot_enum = pivot_enum;
+	return result;
+}
+
+void PivotColumn::Serialize(Serializer &serializer) const {
+	FieldWriter writer(serializer);
+	writer.WriteList<string>(names);
+	writer.WriteRegularSerializableList(entries);
+	writer.WriteString(pivot_enum);
+	writer.Finalize();
+}
+
+PivotColumn PivotColumn::Deserialize(Deserializer &source) {
+	PivotColumn result;
+	FieldReader reader(source);
+	result.names = reader.ReadRequiredList<string>();
+	result.entries = reader.ReadRequiredSerializableList<PivotColumnEntry, PivotColumnEntry>();
+	result.pivot_enum = reader.ReadRequired<string>();
+	reader.Finalize();
+	return result;
+}
+
+//===--------------------------------------------------------------------===//
+// PivotColumnEntry
+//===--------------------------------------------------------------------===//
+PivotColumnEntry PivotColumnEntry::Copy() const {
+	PivotColumnEntry result;
+	result.values = values;
+	result.star_expr = star_expr ? star_expr->Copy() : nullptr;
+	result.alias = alias;
+	return result;
+}
+
+void PivotColumnEntry::Serialize(Serializer &serializer) const {
+	FieldWriter writer(serializer);
+	writer.WriteRegularSerializableList(values);
+	writer.WriteOptional(star_expr);
+	writer.WriteString(alias);
+	writer.Finalize();
+}
+
+PivotColumnEntry PivotColumnEntry::Deserialize(Deserializer &source) {
+	PivotColumnEntry result;
+	FieldReader reader(source);
+	result.values = reader.ReadRequiredSerializableList<Value, Value>();
+	result.star_expr = reader.ReadOptional<ParsedExpression>(nullptr);
+	result.alias = reader.ReadRequired<string>();
+	reader.Finalize();
+	return result;
+}
+
+//===--------------------------------------------------------------------===//
+// PivotRef
+//===--------------------------------------------------------------------===//
+string PivotRef::ToString() const {
+	string result;
+	result = source->ToString();
+	if (!aggregates.empty()) {
+		// pivot
+		result += " PIVOT (";
+		for (idx_t aggr_idx = 0; aggr_idx < aggregates.size(); aggr_idx++) {
+			if (aggr_idx > 0) {
+				result += ", ";
+			}
+			result += aggregates[aggr_idx]->ToString();
+			if (!aggregates[aggr_idx]->alias.empty()) {
+				result += " AS " + KeywordHelper::WriteOptionallyQuoted(aggregates[aggr_idx]->alias);
+			}
+		}
+	} else {
+		// unpivot
+		result += " UNPIVOT ";
+		if (include_nulls) {
+			result += "INCLUDE NULLS ";
+		}
+		result += "(";
+		if (unpivot_names.size() == 1) {
+			result += KeywordHelper::WriteOptionallyQuoted(unpivot_names[0]);
+		} else {
+			result += "(";
+			for (idx_t n = 0; n < unpivot_names.size(); n++) {
+				if (n > 0) {
+					result += ", ";
+				}
+				result += KeywordHelper::WriteOptionallyQuoted(unpivot_names[n]);
+			}
+			result += ")";
+		}
+	}
+	result += " FOR";
+	for (auto &pivot : pivots) {
+		result += " ";
+		result += pivot.ToString();
+	}
+	if (!groups.empty()) {
+		result += " GROUP BY ";
+		for (idx_t i = 0; i < groups.size(); i++) {
+			if (i > 0) {
+				result += ", ";
+			}
+			result += groups[i];
+		}
+	}
+	result += ")";
+	if (!alias.empty()) {
+		result += " AS " + KeywordHelper::WriteOptionallyQuoted(alias);
+		if (!column_name_alias.empty()) {
+			result += "(";
+			for (idx_t i = 0; i < column_name_alias.size(); i++) {
+				if (i > 0) {
+					result += ", ";
+				}
+				result += KeywordHelper::WriteOptionallyQuoted(column_name_alias[i]);
+			}
+			result += ")";
+		}
+	}
+	return result;
+}
+
+bool PivotRef::Equals(const TableRef *other_p) const {
+	if (!TableRef::Equals(other_p)) {
+		return false;
+	}
+	auto other = (PivotRef *)other_p;
+	if (!source->Equals(other->source.get())) {
+		return false;
+	}
+	if (aggregates.size() != other->aggregates.size()) {
+		return false;
+	}
+	for (idx_t i = 0; i < aggregates.size(); i++) {
+		if (!BaseExpression::Equals(aggregates[i].get(), other->aggregates[i].get())) {
+			return false;
+		}
+	}
+	if (pivots.size() != other->pivots.size()) {
+		return false;
+	}
+	for (idx_t i = 0; i < pivots.size(); i++) {
+		if (!pivots[i].Equals(other->pivots[i])) {
+			return false;
+		}
+	}
+	if (unpivot_names != other->unpivot_names) {
+		return false;
+	}
+	if (alias != other->alias) {
+		return false;
+	}
+	if (groups != other->groups) {
+		return false;
+	}
+	if (include_nulls != other->include_nulls) {
+		return false;
+	}
+	return true;
+}
+
+unique_ptr<TableRef> PivotRef::Copy() {
+	auto copy = make_unique<PivotRef>();
+	copy->source = source->Copy();
+	for (auto &aggr : aggregates) {
+		copy->aggregates.push_back(aggr->Copy());
+	}
+	copy->unpivot_names = unpivot_names;
+	for (auto &entry : pivots) {
+		copy->pivots.push_back(entry.Copy());
+	}
+	copy->groups = groups;
+	copy->column_name_alias = column_name_alias;
+	copy->include_nulls = include_nulls;
+	copy->alias = alias;
+	return std::move(copy);
+}
+
+void PivotRef::Serialize(FieldWriter &writer) const {
+	writer.WriteSerializable(*source);
+	writer.WriteSerializableList(aggregates);
+	writer.WriteList<string>(unpivot_names);
+	writer.WriteRegularSerializableList(pivots);
+	writer.WriteList<string>(groups);
+	writer.WriteList<string>(column_name_alias);
+	writer.WriteField<bool>(include_nulls);
+}
+
+unique_ptr<TableRef> PivotRef::Deserialize(FieldReader &reader) {
+	auto result = make_unique<PivotRef>();
+	result->source = reader.ReadRequiredSerializable<TableRef>();
+	result->aggregates = reader.ReadRequiredSerializableList<ParsedExpression>();
+	result->unpivot_names = reader.ReadRequiredList<string>();
+	result->pivots = reader.ReadRequiredSerializableList<PivotColumn, PivotColumn>();
+	result->groups = reader.ReadRequiredList<string>();
+	result->column_name_alias = reader.ReadRequiredList<string>();
+	result->include_nulls = reader.ReadRequired<bool>();
+	return std::move(result);
+}
+
+} // namespace duckdb

package/src/duckdb/src/parser/tableref.cpp

@@ -78,6 +78,9 @@ unique_ptr<TableRef> TableRef::Deserialize(Deserializer &source) {
 	case TableReferenceType::EXPRESSION_LIST:
 		result = ExpressionListRef::Deserialize(reader);
 		break;
+	case TableReferenceType::PIVOT:
+		result = PivotRef::Deserialize(reader);
+		break;
 	case TableReferenceType::CTE:
 	case TableReferenceType::INVALID:
 		throw InternalException("Unsupported type for TableRef::Deserialize");

package/src/duckdb/src/parser/transform/helpers/transform_alias.cpp

@@ -2,16 +2,22 @@
 
 namespace duckdb {
 
+vector<string> Transformer::TransformStringList(duckdb_libpgquery::PGList *list) {
+	vector<string> result;
+	if (!list) {
+		return result;
+	}
+	for (auto node = list->head; node != nullptr; node = node->next) {
+		result.emplace_back(reinterpret_cast<duckdb_libpgquery::PGValue *>(node->data.ptr_value)->val.str);
+	}
+	return result;
+}
+
 string Transformer::TransformAlias(duckdb_libpgquery::PGAlias *root, vector<string> &column_name_alias) {
 	if (!root) {
 		return "";
 	}
-	if (root->colnames) {
-		for (auto node = root->colnames->head; node != nullptr; node = node->next) {
-			column_name_alias.emplace_back(
-			    reinterpret_cast<duckdb_libpgquery::PGValue *>(node->data.ptr_value)->val.str);
-		}
-	}
+	column_name_alias = TransformStringList(root->colnames);
 	return root->aliasname;
 }
 

package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp

@@ -6,9 +6,33 @@
 
 namespace duckdb {
 
+unique_ptr<CommonTableExpressionInfo> CommonTableExpressionInfo::Copy() {
+	auto result = make_unique<CommonTableExpressionInfo>();
+	result->aliases = aliases;
+	result->query = unique_ptr_cast<SQLStatement, SelectStatement>(query->Copy());
+	return result;
+}
+
+void Transformer::ExtractCTEsRecursive(CommonTableExpressionMap &cte_map) {
+	for (auto &cte_entry : stored_cte_map) {
+		for (auto &entry : cte_entry->map) {
+			auto found_entry = cte_map.map.find(entry.first);
+			if (found_entry != cte_map.map.end()) {
+				// entry already present - use top-most entry
+				continue;
+			}
+			cte_map.map[entry.first] = entry.second->Copy();
+		}
+	}
+	if (parent) {
+		parent->ExtractCTEsRecursive(cte_map);
+	}
+}
+
 void Transformer::TransformCTE(duckdb_libpgquery::PGWithClause *de_with_clause, CommonTableExpressionMap &cte_map) {
 	// TODO: might need to update in case of future lawsuit
 	D_ASSERT(de_with_clause);
+	stored_cte_map.push_back(&cte_map);
 
 	D_ASSERT(de_with_clause->ctes);
 	for (auto cte_ele = de_with_clause->ctes->head; cte_ele != nullptr; cte_ele = cte_ele->next) {

package/src/duckdb/src/parser/transform/statement/transform_create_function.cpp

@@ -29,6 +29,10 @@ unique_ptr<CreateStatement> Transformer::TransformCreateFunction(duckdb_libpgque
 		auto query_node = TransformSelect(stmt->query, true)->node->Copy();
 		macro_func = make_unique<TableMacroFunction>(std::move(query_node));
 	}
+	if (HasPivotEntries()) {
+		throw ParserException("Cannot use PIVOT statement syntax in a macro. Use the SQL standard PIVOT syntax in the "
+		                      "FROM clause instead.");
+	}
 
 	auto info =
 	    make_unique<CreateMacroInfo>((stmt->function ? CatalogType::MACRO_ENTRY : CatalogType::TABLE_MACRO_ENTRY));

package/src/duckdb/src/parser/transform/statement/transform_create_view.cpp

@@ -26,6 +26,10 @@ unique_ptr<CreateStatement> Transformer::TransformCreateView(duckdb_libpgquery::
 	info->on_conflict = TransformOnConflict(stmt->onconflict);
 
 	info->query = TransformSelect(stmt->query, false);
+	if (HasPivotEntries()) {
+		throw ParserException("Cannot use PIVOT statement syntax in a view. Use the SQL standard PIVOT syntax in the "
+		                      "FROM clause instead.");
+	}
 
 	if (stmt->aliases && stmt->aliases->length > 0) {
 		for (auto c = stmt->aliases->head; c != nullptr; c = lnext(c)) {

package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp

@@ -0,0 +1,150 @@
+#include "duckdb/parser/transformer.hpp"
+#include "duckdb/parser/tableref/pivotref.hpp"
+#include "duckdb/parser/parsed_data/create_type_info.hpp"
+#include "duckdb/parser/statement/create_statement.hpp"
+#include "duckdb/parser/statement/select_statement.hpp"
+#include "duckdb/parser/expression/columnref_expression.hpp"
+#include "duckdb/parser/expression/star_expression.hpp"
+#include "duckdb/parser/query_node/select_node.hpp"
+#include "duckdb/parser/statement/multi_statement.hpp"
+#include "duckdb/parser/statement/drop_statement.hpp"
+#include "duckdb/parser/parsed_data/drop_info.hpp"
+#include "duckdb/parser/expression/cast_expression.hpp"
+#include "duckdb/parser/result_modifier.hpp"
+#include "duckdb/parser/tableref/subqueryref.hpp"
+
+namespace duckdb {
+
+void Transformer::AddPivotEntry(string enum_name, unique_ptr<SelectNode> base, string column_name) {
+	if (parent) {
+		parent->AddPivotEntry(std::move(enum_name), std::move(base), std::move(column_name));
+		return;
+	}
+	auto result = make_unique<CreatePivotEntry>();
+	result->enum_name = std::move(enum_name);
+	result->base = std::move(base);
+	result->column_name = std::move(column_name);
+
+	pivot_entries.push_back(std::move(result));
+}
+
+bool Transformer::HasPivotEntries() {
+	if (parent) {
+		return parent->HasPivotEntries();
+	}
+	return !pivot_entries.empty();
+}
+
+idx_t Transformer::PivotEntryCount() {
+	if (parent) {
+		return parent->PivotEntryCount();
+	}
+	return pivot_entries.size();
+}
+
+unique_ptr<SQLStatement> Transformer::GenerateCreateEnumStmt(unique_ptr<CreatePivotEntry> entry) {
+	auto result = make_unique<CreateStatement>();
+	auto info = make_unique<CreateTypeInfo>();
+
+	info->temporary = true;
+	info->internal = false;
+	info->catalog = INVALID_CATALOG;
+	info->schema = INVALID_SCHEMA;
+	info->name = std::move(entry->enum_name);
+	info->on_conflict = OnCreateConflict::REPLACE_ON_CONFLICT;
+
+	// generate the query that will result in the enum creation
+	auto select_node = std::move(entry->base);
+	auto columnref = make_unique<ColumnRefExpression>(std::move(entry->column_name));
+	auto cast = make_unique<CastExpression>(LogicalType::VARCHAR, columnref->Copy());
+	select_node->select_list.push_back(std::move(cast));
+
+	// order by the column
+	auto modifier = make_unique<OrderModifier>();
+	modifier->orders.emplace_back(OrderType::ASCENDING, OrderByNullType::ORDER_DEFAULT, std::move(columnref));
+	select_node->modifiers.push_back(std::move(modifier));
+
+	auto select = make_unique<SelectStatement>();
+	select->node = std::move(select_node);
+	info->query = std::move(select);
+	info->type = LogicalType::INVALID;
+
+	result->info = std::move(info);
+	return std::move(result);
+}
+
+// unique_ptr<SQLStatement> GenerateDropEnumStmt(string enum_name) {
+//	auto result = make_unique<DropStatement>();
+//	result->info->if_exists = true;
+//	result->info->schema = INVALID_SCHEMA;
+//	result->info->catalog = INVALID_CATALOG;
+//	result->info->name = std::move(enum_name);
+//	result->info->type = CatalogType::TYPE_ENTRY;
+//	return std::move(result);
+//}
+
+unique_ptr<SQLStatement> Transformer::CreatePivotStatement(unique_ptr<SQLStatement> statement) {
+	auto result = make_unique<MultiStatement>();
+	for (auto &pivot : pivot_entries) {
+		result->statements.push_back(GenerateCreateEnumStmt(std::move(pivot)));
+	}
+	result->statements.push_back(std::move(statement));
+	// FIXME: drop the types again!?
+	//	for(auto &pivot : pivot_entries) {
+	//		result->statements.push_back(GenerateDropEnumStmt(std::move(pivot->enum_name)));
+	//	}
+	return std::move(result);
+}
+
+unique_ptr<QueryNode> Transformer::TransformPivotStatement(duckdb_libpgquery::PGSelectStmt *stmt) {
+	auto pivot = stmt->pivot;
+	auto source = TransformTableRefNode(pivot->source);
+	auto columns = TransformPivotList(pivot->columns);
+
+	auto select_node = make_unique<SelectNode>();
+	// handle the CTEs
+	if (stmt->withClause) {
+		TransformCTE(reinterpret_cast<duckdb_libpgquery::PGWithClause *>(stmt->withClause), select_node->cte_map);
+	}
+
+	// generate CREATE TYPE statements for each of the columns that do not have an IN list
+	auto pivot_idx = PivotEntryCount();
+	for (idx_t c = 0; c < columns.size(); c++) {
+		auto &col = columns[c];
+		if (!col.pivot_enum.empty() || !col.entries.empty()) {
+			continue;
+		}
+		if (col.names.size() != 1) {
+			throw InternalException("PIVOT statement with multiple names in pivot entry!?");
+		}
+		auto enum_name = "__pivot_enum_" + std::to_string(pivot_idx) + "_" + std::to_string(c);
+
+		auto new_select = make_unique<SelectNode>();
+		ExtractCTEsRecursive(new_select->cte_map);
+		new_select->from_table = source->Copy();
+		AddPivotEntry(enum_name, std::move(new_select), col.names[0]);
+		col.pivot_enum = enum_name;
+	}
+
+	// generate the actual query, including the pivot
+	select_node->select_list.push_back(make_unique<StarExpression>());
+
+	auto pivot_ref = make_unique<PivotRef>();
+	pivot_ref->source = std::move(source);
+	if (pivot->aggrs) {
+		TransformExpressionList(*pivot->aggrs, pivot_ref->aggregates);
+	} else {
+		if (!pivot->unpivots) {
+			throw InternalException("No unpivots and no aggrs");
+		}
+		pivot_ref->unpivot_names = TransformStringList(pivot->unpivots);
+	}
+	if (pivot->groups) {
+		pivot_ref->groups = TransformStringList(pivot->groups);
+	}
+	pivot_ref->pivots = std::move(columns);
+	select_node->from_table = std::move(pivot_ref);
+	return std::move(select_node);
+}
+
+} // namespace duckdb

package/src/duckdb/src/parser/transform/statement/transform_select.cpp

@@ -4,6 +4,14 @@
 
 namespace duckdb {
 
+unique_ptr<QueryNode> Transformer::TransformSelectNode(duckdb_libpgquery::PGSelectStmt *stmt) {
+	if (stmt->pivot) {
+		return TransformPivotStatement(stmt);
+	} else {
+		return TransformSelectInternal(stmt);
+	}
+}
+
 unique_ptr<SelectStatement> Transformer::TransformSelect(duckdb_libpgquery::PGNode *node, bool is_select) {
 	auto stmt = reinterpret_cast<duckdb_libpgquery::PGSelectStmt *>(node);
 	auto result = make_unique<SelectStatement>();

package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp

@@ -8,7 +8,7 @@
 
 namespace duckdb {
 
-unique_ptr<QueryNode> Transformer::TransformSelectNode(duckdb_libpgquery::PGSelectStmt *stmt) {
+unique_ptr<QueryNode> Transformer::TransformSelectInternal(duckdb_libpgquery::PGSelectStmt *stmt) {
 	D_ASSERT(stmt->type == duckdb_libpgquery::T_PGSelectStmt);
 	auto stack_checker = StackCheck();
 

package/src/duckdb/src/parser/transform/tableref/transform_join.cpp

@@ -28,6 +28,10 @@ unique_ptr<TableRef> Transformer::TransformJoin(duckdb_libpgquery::PGJoinExpr *r
 		result->type = JoinType::SEMI;
 		break;
 	}
+	case duckdb_libpgquery::PG_JOIN_ANTI: {
+		result->type = JoinType::ANTI;
+		break;
+	}
 	case duckdb_libpgquery::PG_JOIN_POSITION: {
 		result->ref_type = JoinRefType::POSITIONAL;
 		break;