duckdb 0.7.2-dev886.0 → 0.7.2-dev899.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.7.2-dev886.0",
+  "version": "0.7.2-dev899.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/function/scalar/struct/struct_extract.cpp

@@ -96,7 +96,7 @@ static unique_ptr<FunctionData> StructExtractBind(ClientContext &context, Scalar
 	}
 
 	bound_function.return_type = return_type;
-	return make_unique<StructExtractBindData>(key, key_index, return_type);
+	return make_unique<StructExtractBindData>(std::move(key), key_index, std::move(return_type));
 }
 
 static unique_ptr<BaseStatistics> PropagateStructExtractStats(ClientContext &context, FunctionStatisticsInput &input) {

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.7.2-dev886"
+#define DUCKDB_VERSION "0.7.2-dev899"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "7d73c6e6a8"
+#define DUCKDB_SOURCE_ID "88b1bfa74d"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/parser/tableref/pivotref.hpp

@@ -30,8 +30,10 @@ struct PivotColumnEntry {
 };
 
 struct PivotColumn {
-	//! The column names to (un)pivot
-	vector<string> names;
+	//! The set of expressions to pivot on
+	vector<unique_ptr<ParsedExpression>> pivot_expressions;
+	//! The set of unpivot names
+	vector<string> unpivot_names;
 	//! The set of values to pivot on
 	vector<PivotColumnEntry> entries;
 	//! The enum to read pivot values from (if any)

package/src/duckdb/src/include/duckdb/parser/transformer.hpp

@@ -42,7 +42,7 @@ class Transformer {
 	struct CreatePivotEntry {
 		string enum_name;
 		unique_ptr<SelectNode> base;
-		string column_name;
+		unique_ptr<ParsedExpression> column;
 	};
 
 public:
@@ -108,7 +108,7 @@ private:
 		return parent ? parent->HasNamedParameters() : !named_param_map.empty();
 	}
 
-	void AddPivotEntry(string enum_name, unique_ptr<SelectNode> source, string column_name);
+	void AddPivotEntry(string enum_name, unique_ptr<SelectNode> source, unique_ptr<ParsedExpression> column);
 	unique_ptr<SQLStatement> GenerateCreateEnumStmt(unique_ptr<CreatePivotEntry> entry);
 	bool HasPivotEntries();
 	idx_t PivotEntryCount();

package/src/duckdb/src/include/duckdb/planner/binder.hpp

@@ -73,7 +73,6 @@ struct CorrelatedColumnInfo {
 */
 class Binder : public std::enable_shared_from_this<Binder> {
 	friend class ExpressionBinder;
-	friend class SelectBinder;
 	friend class RecursiveSubqueryPlanner;
 
 public:
@@ -185,6 +184,9 @@ public:
 	BindingMode GetBindingMode();
 	void AddTableName(string table_name);
 	const unordered_set<string> &GetTableNames();
+	SQLStatement *GetRootStatement() {
+		return root_statement;
+	}
 
 	void SetCanContainNulls(bool can_contain_nulls);
 

package/src/duckdb/src/include/duckdb/planner/expression_binder/base_select_binder.hpp

@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/planner/expression_binder/base_select_binder.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/case_insensitive_map.hpp"
+#include "duckdb/parser/expression_map.hpp"
+#include "duckdb/planner/expression_binder.hpp"
+
+namespace duckdb {
+class BoundColumnRefExpression;
+class WindowExpression;
+
+class BoundSelectNode;
+
+struct BoundGroupInformation {
+	expression_map_t<idx_t> map;
+	case_insensitive_map_t<idx_t> alias_map;
+};
+
+//! The BaseSelectBinder is the base binder of the SELECT, HAVING and QUALIFY binders. It can bind aggregates and window
+//! functions.
+class BaseSelectBinder : public ExpressionBinder {
+public:
+	BaseSelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info,
+	                 case_insensitive_map_t<idx_t> alias_map);
+	BaseSelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info);
+
+	bool BoundAggregates() {
+		return bound_aggregate;
+	}
+	void ResetBindings() {
+		this->bound_aggregate = false;
+		this->bound_columns.clear();
+	}
+
+protected:
+	BindResult BindExpression(unique_ptr<ParsedExpression> *expr_ptr, idx_t depth,
+	                          bool root_expression = false) override;
+
+	BindResult BindAggregate(FunctionExpression &expr, AggregateFunctionCatalogEntry *function, idx_t depth) override;
+
+	bool inside_window;
+	bool bound_aggregate = false;
+
+	BoundSelectNode &node;
+	BoundGroupInformation &info;
+	case_insensitive_map_t<idx_t> alias_map;
+
+protected:
+	BindResult BindColumnRef(unique_ptr<ParsedExpression> *expr_ptr, idx_t depth);
+	BindResult BindGroupingFunction(OperatorExpression &op, idx_t depth) override;
+	BindResult BindWindow(WindowExpression &expr, idx_t depth);
+
+	idx_t TryBindGroup(ParsedExpression &expr, idx_t depth);
+	BindResult BindGroup(ParsedExpression &expr, idx_t depth, idx_t group_index);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/expression_binder/having_binder.hpp

@@ -8,14 +8,14 @@
 
 #pragma once
 
-#include "duckdb/planner/expression_binder/select_binder.hpp"
+#include "duckdb/planner/expression_binder/base_select_binder.hpp"
 #include "duckdb/planner/expression_binder/column_alias_binder.hpp"
 #include "duckdb/common/enums/aggregate_handling.hpp"
 
 namespace duckdb {
 
 //! The HAVING binder is responsible for binding an expression within the HAVING clause of a SQL statement
-class HavingBinder : public SelectBinder {
+class HavingBinder : public BaseSelectBinder {
 public:
 	HavingBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info,
 	             case_insensitive_map_t<idx_t> &alias_map, AggregateHandling aggregate_handling);

package/src/duckdb/src/include/duckdb/planner/expression_binder/qualify_binder.hpp

@@ -8,13 +8,13 @@
 
 #pragma once
 
-#include "duckdb/planner/expression_binder/select_binder.hpp"
+#include "duckdb/planner/expression_binder/base_select_binder.hpp"
 #include "duckdb/planner/expression_binder/column_alias_binder.hpp"
 
 namespace duckdb {
 
 //! The QUALIFY binder is responsible for binding an expression within the QUALIFY clause of a SQL statement
-class QualifyBinder : public SelectBinder {
+class QualifyBinder : public BaseSelectBinder {
 public:
 	QualifyBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info,
 	              case_insensitive_map_t<idx_t> &alias_map);

package/src/duckdb/src/include/duckdb/planner/expression_binder/select_binder.hpp

@@ -8,58 +8,29 @@
 
 #pragma once
 
-#include "duckdb/common/case_insensitive_map.hpp"
-#include "duckdb/parser/expression_map.hpp"
-#include "duckdb/planner/expression_binder.hpp"
+#include "duckdb/planner/expression_binder/base_select_binder.hpp"
 
 namespace duckdb {
-class BoundColumnRefExpression;
-class WindowExpression;
-
-class BoundSelectNode;
-
-struct BoundGroupInformation {
-	expression_map_t<idx_t> map;
-	case_insensitive_map_t<idx_t> alias_map;
-};
 
 //! The SELECT binder is responsible for binding an expression within the SELECT clause of a SQL statement
-class SelectBinder : public ExpressionBinder {
+class SelectBinder : public BaseSelectBinder {
 public:
 	SelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info,
 	             case_insensitive_map_t<idx_t> alias_map);
 	SelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info);
 
-	bool BoundAggregates() {
-		return bound_aggregate;
+	bool HasExpandedExpressions() {
+		return !expanded_expressions.empty();
 	}
-	void ResetBindings() {
-		this->bound_aggregate = false;
-		this->bound_columns.clear();
+	vector<unique_ptr<Expression>> &ExpandedExpressions() {
+		return expanded_expressions;
 	}
 
 protected:
-	BindResult BindExpression(unique_ptr<ParsedExpression> *expr_ptr, idx_t depth,
-	                          bool root_expression = false) override;
-
-	BindResult BindAggregate(FunctionExpression &expr, AggregateFunctionCatalogEntry *function, idx_t depth) override;
-
-	BindResult BindUnnest(FunctionExpression &function, idx_t depth) override;
-
-	bool inside_window;
-	bool bound_aggregate = false;
-
-	BoundSelectNode &node;
-	BoundGroupInformation &info;
-	case_insensitive_map_t<idx_t> alias_map;
-
-protected:
-	BindResult BindColumnRef(unique_ptr<ParsedExpression> *expr_ptr, idx_t depth);
-	BindResult BindGroupingFunction(OperatorExpression &op, idx_t depth) override;
-	BindResult BindWindow(WindowExpression &expr, idx_t depth);
+	BindResult BindUnnest(FunctionExpression &function, idx_t depth, bool root_expression) override;
 
-	idx_t TryBindGroup(ParsedExpression &expr, idx_t depth);
-	BindResult BindGroup(ParsedExpression &expr, idx_t depth, idx_t group_index);
+	idx_t unnest_level = 0;
+	vector<unique_ptr<Expression>> expanded_expressions;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/expression_binder.hpp

@@ -135,7 +135,7 @@ protected:
 	virtual BindResult BindFunction(FunctionExpression &expr, ScalarFunctionCatalogEntry *function, idx_t depth);
 	virtual BindResult BindLambdaFunction(FunctionExpression &expr, ScalarFunctionCatalogEntry *function, idx_t depth);
 	virtual BindResult BindAggregate(FunctionExpression &expr, AggregateFunctionCatalogEntry *function, idx_t depth);
-	virtual BindResult BindUnnest(FunctionExpression &expr, idx_t depth);
+	virtual BindResult BindUnnest(FunctionExpression &expr, idx_t depth, bool root_expression);
 	virtual BindResult BindMacro(FunctionExpression &expr, ScalarMacroCatalogEntry *macro, idx_t depth,
 	                             unique_ptr<ParsedExpression> *expr_ptr);
 

package/src/duckdb/src/include/duckdb/planner/query_node/bound_select_node.hpp

@@ -25,6 +25,13 @@ public:
 	vector<GroupingSet> grouping_sets;
 };
 
+struct BoundUnnestNode {
+	//! The index of the UNNEST node
+	idx_t index;
+	//! The set of expressions
+	vector<unique_ptr<Expression>> expressions;
+};
+
 //! Bound equivalent of SelectNode
 class BoundSelectNode : public BoundQueryNode {
 public:
@@ -78,9 +85,8 @@ public:
 	//! Window functions to compute (only used if HasWindow is true)
 	vector<unique_ptr<Expression>> windows;
 
-	idx_t unnest_index;
 	//! Unnest expression
-	vector<unique_ptr<Expression>> unnests;
+	unordered_map<idx_t, BoundUnnestNode> unnests;
 
 	//! Index of pruned node
 	idx_t prune_index;

package/src/duckdb/src/parser/tableref/pivotref.cpp

@@ -12,15 +12,29 @@ namespace duckdb {
 //===--------------------------------------------------------------------===//
 string PivotColumn::ToString() const {
 	string result;
-	if (names.size() == 1) {
-		result += KeywordHelper::WriteOptionallyQuoted(names[0]);
-	} else {
+	if (!unpivot_names.empty()) {
+		D_ASSERT(pivot_expressions.empty());
+		// unpivot
+		if (unpivot_names.size() == 1) {
+			result += KeywordHelper::WriteOptionallyQuoted(unpivot_names[0]);
+		} else {
+			result += "(";
+			for (idx_t n = 0; n < unpivot_names.size(); n++) {
+				if (n > 0) {
+					result += ", ";
+				}
+				result += KeywordHelper::WriteOptionallyQuoted(unpivot_names[n]);
+			}
+			result += ")";
+		}
+	} else if (!pivot_expressions.empty()) {
+		// pivot
 		result += "(";
-		for (idx_t n = 0; n < names.size(); n++) {
+		for (idx_t n = 0; n < pivot_expressions.size(); n++) {
 			if (n > 0) {
 				result += ", ";
 			}
-			result += KeywordHelper::WriteOptionallyQuoted(names[n]);
+			result += pivot_expressions[n]->ToString();
 		}
 		result += ")";
 	}
@@ -74,7 +88,10 @@ bool PivotColumnEntry::Equals(const PivotColumnEntry &other) const {
 }
 
 bool PivotColumn::Equals(const PivotColumn &other) const {
-	if (other.names != names) {
+	if (!ExpressionUtil::ListEquals(pivot_expressions, other.pivot_expressions)) {
+		return false;
+	}
+	if (other.unpivot_names != unpivot_names) {
 		return false;
 	}
 	if (other.pivot_enum != pivot_enum) {
@@ -93,7 +110,10 @@ bool PivotColumn::Equals(const PivotColumn &other) const {
 
 PivotColumn PivotColumn::Copy() const {
 	PivotColumn result;
-	result.names = names;
+	for (auto &expr : pivot_expressions) {
+		result.pivot_expressions.push_back(expr->Copy());
+	}
+	result.unpivot_names = unpivot_names;
 	for (auto &entry : entries) {
 		result.entries.push_back(entry.Copy());
 	}
@@ -103,14 +123,16 @@ PivotColumn PivotColumn::Copy() const {
 
 void PivotColumn::Serialize(Serializer &serializer) const {
 	FieldWriter writer(serializer);
-	writer.WriteList<string>(names);
+	writer.WriteSerializableList(pivot_expressions);
+	writer.WriteList<string>(unpivot_names);
 	writer.WriteRegularSerializableList(entries);
 	writer.WriteString(pivot_enum);
 	writer.Finalize();
 }
 
 void PivotColumn::FormatSerialize(FormatSerializer &serializer) const {
-	serializer.WriteProperty("names", names);
+	serializer.WriteProperty("pivot_expressions", pivot_expressions);
+	serializer.WriteProperty("unpivot_names", unpivot_names);
 	serializer.WriteProperty("entries", entries);
 	serializer.WriteProperty("pivot_enum", pivot_enum);
 }
@@ -118,7 +140,8 @@ void PivotColumn::FormatSerialize(FormatSerializer &serializer) const {
 PivotColumn PivotColumn::Deserialize(Deserializer &source) {
 	PivotColumn result;
 	FieldReader reader(source);
-	result.names = reader.ReadRequiredList<string>();
+	result.pivot_expressions = reader.ReadRequiredSerializableList<ParsedExpression>();
+	result.unpivot_names = reader.ReadRequiredList<string>();
 	result.entries = reader.ReadRequiredSerializableList<PivotColumnEntry, PivotColumnEntry>();
 	result.pivot_enum = reader.ReadRequired<string>();
 	reader.Finalize();
@@ -127,7 +150,8 @@ PivotColumn PivotColumn::Deserialize(Deserializer &source) {
 
 PivotColumn PivotColumn::FormatDeserialize(FormatDeserializer &source) {
 	PivotColumn result;
-	source.ReadProperty("names", result.names);
+	source.ReadProperty("pivot_expressions", result.pivot_expressions);
+	source.ReadProperty("unpivot_names", result.unpivot_names);
 	source.ReadProperty("entries", result.entries);
 	source.ReadProperty("pivot_enum", result.pivot_enum);
 	return result;

package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp

@@ -10,21 +10,22 @@
 #include "duckdb/parser/statement/drop_statement.hpp"
 #include "duckdb/parser/parsed_data/drop_info.hpp"
 #include "duckdb/parser/expression/cast_expression.hpp"
+#include "duckdb/parser/expression/operator_expression.hpp"
 #include "duckdb/parser/expression/function_expression.hpp"
 #include "duckdb/parser/result_modifier.hpp"
 #include "duckdb/parser/tableref/subqueryref.hpp"
 
 namespace duckdb {
 
-void Transformer::AddPivotEntry(string enum_name, unique_ptr<SelectNode> base, string column_name) {
+void Transformer::AddPivotEntry(string enum_name, unique_ptr<SelectNode> base, unique_ptr<ParsedExpression> column) {
 	if (parent) {
-		parent->AddPivotEntry(std::move(enum_name), std::move(base), std::move(column_name));
+		parent->AddPivotEntry(std::move(enum_name), std::move(base), std::move(column));
 		return;
 	}
 	auto result = make_unique<CreatePivotEntry>();
 	result->enum_name = std::move(enum_name);
 	result->base = std::move(base);
-	result->column_name = std::move(column_name);
+	result->column = std::move(column);
 
 	pivot_entries.push_back(std::move(result));
 }
@@ -56,10 +57,13 @@ unique_ptr<SQLStatement> Transformer::GenerateCreateEnumStmt(unique_ptr<CreatePi
 
 	// generate the query that will result in the enum creation
 	auto select_node = std::move(entry->base);
-	auto columnref = make_unique<ColumnRefExpression>(std::move(entry->column_name));
+	auto columnref = entry->column->Copy();
 	auto cast = make_unique<CastExpression>(LogicalType::VARCHAR, columnref->Copy());
 	select_node->select_list.push_back(std::move(cast));
 
+	auto is_not_null = make_unique<OperatorExpression>(ExpressionType::OPERATOR_IS_NOT_NULL, std::move(entry->column));
+	select_node->where_clause = std::move(is_not_null);
+
 	// order by the column
 	auto modifier = make_unique<OrderModifier>();
 	modifier->orders.emplace_back(OrderType::ASCENDING, OrderByNullType::ORDER_DEFAULT, std::move(columnref));
@@ -135,7 +139,7 @@ unique_ptr<QueryNode> Transformer::TransformPivotStatement(duckdb_libpgquery::PG
 		if (!col.pivot_enum.empty() || !col.entries.empty()) {
 			continue;
 		}
-		if (col.names.size() != 1) {
+		if (col.pivot_expressions.size() != 1) {
 			throw InternalException("PIVOT statement with multiple names in pivot entry!?");
 		}
 		auto enum_name = "__pivot_enum_" + std::to_string(pivot_idx) + "_" + std::to_string(c);
@@ -143,7 +147,7 @@ unique_ptr<QueryNode> Transformer::TransformPivotStatement(duckdb_libpgquery::PG
 		auto new_select = make_unique<SelectNode>();
 		ExtractCTEsRecursive(new_select->cte_map);
 		new_select->from_table = source->Copy();
-		AddPivotEntry(enum_name, std::move(new_select), col.names[0]);
+		AddPivotEntry(enum_name, std::move(new_select), col.pivot_expressions[0]->Copy());
 		col.pivot_enum = enum_name;
 	}
 

package/src/duckdb/src/parser/transform/tableref/transform_pivot.cpp

@@ -34,7 +34,21 @@ static void TransformPivotInList(unique_ptr<ParsedExpression> &expr, PivotColumn
 
 PivotColumn Transformer::TransformPivotColumn(duckdb_libpgquery::PGPivot *pivot) {
 	PivotColumn col;
-	col.names = TransformStringList(pivot->pivot_columns);
+	if (pivot->pivot_columns) {
+		TransformExpressionList(*pivot->pivot_columns, col.pivot_expressions);
+		for (auto &expr : col.pivot_expressions) {
+			if (expr->IsScalar()) {
+				throw ParserException("Cannot pivot on constant value \"%s\"", expr->ToString());
+			}
+			if (expr->HasSubquery()) {
+				throw ParserException("Cannot pivot on subquery \"%s\"", expr->ToString());
+			}
+		}
+	} else if (pivot->unpivot_columns) {
+		col.unpivot_names = TransformStringList(pivot->unpivot_columns);
+	} else {
+		throw InternalException("Either pivot_columns or unpivot_columns must be defined");
+	}
 	if (pivot->pivot_value) {
 		for (auto node = pivot->pivot_value->head; node != nullptr; node = node->next) {
 			auto n = (duckdb_libpgquery::PGNode *)node->data.ptr_value;
@@ -78,13 +92,15 @@ unique_ptr<TableRef> Transformer::TransformPivot(duckdb_libpgquery::PGPivotExpr
 		bool is_pivot = result->unpivot_names.empty();
 		if (!result->unpivot_names.empty()) {
 			// unpivot
-			if (pivot.names.size() != 1) {
+			if (pivot.unpivot_names.size() != 1) {
 				throw ParserException("UNPIVOT requires a single column name for the PIVOT IN clause");
 			}
+			D_ASSERT(pivot.pivot_expressions.empty());
 			expected_size = pivot.entries[0].values.size();
 		} else {
 			// pivot
-			expected_size = pivot.names.size();
+			expected_size = pivot.pivot_expressions.size();
+			D_ASSERT(pivot.unpivot_names.empty());
 		}
 		for (auto &entry : pivot.entries) {
 			if (entry.star_expr && is_pivot) {

package/src/duckdb/src/planner/binder/expression/bind_aggregate_expression.cpp

@@ -7,7 +7,7 @@
 #include "duckdb/planner/expression/bound_columnref_expression.hpp"
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
 #include "duckdb/planner/expression_binder/aggregate_binder.hpp"
-#include "duckdb/planner/expression_binder/select_binder.hpp"
+#include "duckdb/planner/expression_binder/base_select_binder.hpp"
 #include "duckdb/planner/query_node/bound_select_node.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/function/scalar/generic_functions.hpp"
@@ -81,7 +81,7 @@ static void NegatePercentileFractions(ClientContext &context, unique_ptr<ParsedE
 	}
 }
 
-BindResult SelectBinder::BindAggregate(FunctionExpression &aggr, AggregateFunctionCatalogEntry *func, idx_t depth) {
+BindResult BaseSelectBinder::BindAggregate(FunctionExpression &aggr, AggregateFunctionCatalogEntry *func, idx_t depth) {
 	// first bind the child of the aggregate expression (if any)
 	this->bound_aggregate = true;
 	unique_ptr<Expression> bound_filter;

package/src/duckdb/src/planner/binder/expression/bind_function_expression.cpp

@@ -18,13 +18,6 @@ BindResult ExpressionBinder::BindExpression(FunctionExpression &function, idx_t
                                             unique_ptr<ParsedExpression> *expr_ptr) {
 	// lookup the function in the catalog
 	QueryErrorContext error_context(binder.root_statement, function.query_location);
-
-	if (function.function_name == "unnest" || function.function_name == "unlist") {
-		// special case, not in catalog
-		// TODO make sure someone does not create such a function OR
-		// have unnest live in catalog, too
-		return BindUnnest(function, depth);
-	}
 	auto func = Catalog::GetEntry(context, CatalogType::SCALAR_FUNCTION_ENTRY, function.catalog, function.schema,
 	                              function.function_name, true, error_context);
 	if (!func) {
@@ -246,7 +239,7 @@ BindResult ExpressionBinder::BindAggregate(FunctionExpression &expr, AggregateFu
 	return BindResult(binder.FormatError(expr, UnsupportedAggregateMessage()));
 }
 
-BindResult ExpressionBinder::BindUnnest(FunctionExpression &expr, idx_t depth) {
+BindResult ExpressionBinder::BindUnnest(FunctionExpression &expr, idx_t depth, bool root_expression) {
 	return BindResult(binder.FormatError(expr, UnsupportedUnnestMessage()));
 }