duckdb 0.7.1 → 0.7.2-dev1034.0

package/src/duckdb/src/include/duckdb/parser/transformer.hpp

@@ -32,15 +32,23 @@ struct CommonTableExpressionInfo;
 struct GroupingExpressionMap;
 class OnConflictInfo;
 class UpdateSetInfo;
+struct PivotColumn;
 
 //! The transformer class is responsible for transforming the internal Postgres
 //! parser representation into the DuckDB representation
 class Transformer {
 	friend class StackChecker;
 
+	struct CreatePivotEntry {
+		string enum_name;
+		unique_ptr<SelectNode> base;
+		unique_ptr<ParsedExpression> column;
+	};
+
 public:
 	explicit Transformer(idx_t max_expression_depth_p);
 	explicit Transformer(Transformer *parent);
+	~Transformer();
 
 	//! Transforms a Postgres parse tree into a set of SQL Statements
 	bool TransformParseTree(duckdb_libpgquery::PGList *tree, vector<unique_ptr<SQLStatement>> &statements);
@@ -59,6 +67,15 @@ private:
 	case_insensitive_map_t<idx_t> named_param_map;
 	//! Holds window expressions defined by name. We need those when transforming the expressions referring to them.
 	unordered_map<string, duckdb_libpgquery::PGWindowDef *> window_clauses;
+	//! The set of pivot entries to create
+	vector<unique_ptr<CreatePivotEntry>> pivot_entries;
+	//! Sets of stored CTEs, if any
+	vector<CommonTableExpressionMap *> stored_cte_map;
+	//! Whether or not we are currently binding a window definition
+	bool in_window_definition = false;
+
+	void Clear();
+	bool InWindowDefinition();
 
 	void SetParamCount(idx_t new_count) {
 		if (parent) {
@@ -91,6 +108,12 @@ private:
 		return parent ? parent->HasNamedParameters() : !named_param_map.empty();
 	}
 
+	void AddPivotEntry(string enum_name, unique_ptr<SelectNode> source, unique_ptr<ParsedExpression> column);
+	unique_ptr<SQLStatement> GenerateCreateEnumStmt(unique_ptr<CreatePivotEntry> entry);
+	bool HasPivotEntries();
+	idx_t PivotEntryCount();
+	void ExtractCTEsRecursive(CommonTableExpressionMap &cte_map);
+
 private:
 	//! Transforms a Postgres statement into a single SQL statement
 	unique_ptr<SQLStatement> TransformStatement(duckdb_libpgquery::PGNode *stmt);
@@ -163,6 +186,10 @@ private:
 	unique_ptr<ExecuteStatement> TransformExecute(duckdb_libpgquery::PGNode *node);
 	unique_ptr<CallStatement> TransformCall(duckdb_libpgquery::PGNode *node);
 	unique_ptr<DropStatement> TransformDeallocate(duckdb_libpgquery::PGNode *node);
+	unique_ptr<QueryNode> TransformPivotStatement(duckdb_libpgquery::PGSelectStmt *stmt);
+	unique_ptr<SQLStatement> CreatePivotStatement(unique_ptr<SQLStatement> statement);
+	PivotColumn TransformPivotColumn(duckdb_libpgquery::PGPivot *pivot);
+	vector<PivotColumn> TransformPivotList(duckdb_libpgquery::PGList *list);
 
 	//===--------------------------------------------------------------------===//
 	// SetStatement Transform
@@ -179,6 +206,7 @@ private:
 	//===--------------------------------------------------------------------===//
 	//! Transform a Postgres duckdb_libpgquery::T_PGSelectStmt node into a QueryNode
 	unique_ptr<QueryNode> TransformSelectNode(duckdb_libpgquery::PGSelectStmt *node);
+	unique_ptr<QueryNode> TransformSelectInternal(duckdb_libpgquery::PGSelectStmt *node);
 
 	//===--------------------------------------------------------------------===//
 	// Expression Transform
@@ -256,6 +284,7 @@ private:
 	//===--------------------------------------------------------------------===//
 	OnCreateConflict TransformOnConflict(duckdb_libpgquery::PGOnCreateConflict conflict);
 	string TransformAlias(duckdb_libpgquery::PGAlias *root, vector<string> &column_name_alias);
+	vector<string> TransformStringList(duckdb_libpgquery::PGList *list);
 	void TransformCTE(duckdb_libpgquery::PGWithClause *de_with_clause, CommonTableExpressionMap &cte_map);
 	unique_ptr<SelectStatement> TransformRecursiveCTE(duckdb_libpgquery::PGCommonTableExpr *node,
 	                                                  CommonTableExpressionInfo &info);
@@ -276,6 +305,8 @@ private:
 	unique_ptr<TableRef> TransformRangeFunction(duckdb_libpgquery::PGRangeFunction *root);
 	//! Transform a Postgres join node into a TableRef
 	unique_ptr<TableRef> TransformJoin(duckdb_libpgquery::PGJoinExpr *root);
+	//! Transform a Postgres pivot node into a TableRef
+	unique_ptr<TableRef> TransformPivot(duckdb_libpgquery::PGPivotExpr *root);
 	//! Transform a table producing subquery into a TableRef
 	unique_ptr<TableRef> TransformRangeSubselect(duckdb_libpgquery::PGRangeSubselect *root);
 	//! Transform a VALUES list into a set of expressions
@@ -307,6 +338,8 @@ private:
 	void TransformWindowFrame(duckdb_libpgquery::PGWindowDef *window_spec, WindowExpression *expr);
 
 	unique_ptr<SampleOptions> TransformSampleOptions(duckdb_libpgquery::PGNode *options);
+	//! Returns true if an expression is only a star (i.e. "*", without any other decorators)
+	bool ExpressionIsEmptyStar(ParsedExpression &expr);
 
 private:
 	//! Current stack depth

package/src/duckdb/src/include/duckdb/planner/bind_context.hpp

@@ -141,6 +141,8 @@ public:
 
 	//! Add all the bindings from a BindContext to this BindContext. The other BindContext is destroyed in the process.
 	void AddContext(BindContext other);
+	//! For semi and anti joins we remove the binding context of the right table after binding the condition.
+	void RemoveContext(vector<std::pair<string, duckdb::Binding *>> &other_bindings_list);
 
 	//! Gets a binding of the specified name. Returns a nullptr and sets the out_error if the binding could not be
 	//! found.

package/src/duckdb/src/include/duckdb/planner/binder.hpp

@@ -73,7 +73,6 @@ struct CorrelatedColumnInfo {
 */
 class Binder : public std::enable_shared_from_this<Binder> {
 	friend class ExpressionBinder;
-	friend class SelectBinder;
 	friend class RecursiveSubqueryPlanner;
 
 public:
@@ -185,6 +184,9 @@ public:
 	BindingMode GetBindingMode();
 	void AddTableName(string table_name);
 	const unordered_set<string> &GetTableNames();
+	SQLStatement *GetRootStatement() {
+		return root_statement;
+	}
 
 	void SetCanContainNulls(bool can_contain_nulls);
 
@@ -253,8 +255,8 @@ private:
 	BoundStatement Bind(DetachStatement &stmt);
 
 	BoundStatement BindReturning(vector<unique_ptr<ParsedExpression>> returning_list, TableCatalogEntry *table,
-	                             idx_t update_table_index, unique_ptr<LogicalOperator> child_operator,
-	                             BoundStatement result);
+	                             const string &alias, idx_t update_table_index,
+	                             unique_ptr<LogicalOperator> child_operator, BoundStatement result);
 
 	unique_ptr<QueryNode> BindTableMacro(FunctionExpression &function, TableMacroCatalogEntry *macro_func, idx_t depth);
 
@@ -275,6 +277,12 @@ private:
 	unique_ptr<BoundTableRef> Bind(TableFunctionRef &ref);
 	unique_ptr<BoundTableRef> Bind(EmptyTableRef &ref);
 	unique_ptr<BoundTableRef> Bind(ExpressionListRef &ref);
+	unique_ptr<BoundTableRef> Bind(PivotRef &expr);
+
+	unique_ptr<SelectNode> BindPivot(PivotRef &expr, vector<unique_ptr<ParsedExpression>> all_columns);
+	unique_ptr<SelectNode> BindUnpivot(Binder &child_binder, PivotRef &expr,
+	                                   vector<unique_ptr<ParsedExpression>> all_columns,
+	                                   unique_ptr<ParsedExpression> &where_clause);
 
 	bool BindTableFunctionParameters(TableFunctionCatalogEntry &table_function,
 	                                 vector<unique_ptr<ParsedExpression>> &expressions, vector<LogicalType> &arguments,
@@ -333,13 +341,16 @@ private:
 	void ExpandStarExpressions(vector<unique_ptr<ParsedExpression>> &select_list,
 	                           vector<unique_ptr<ParsedExpression>> &new_select_list);
 	void ExpandStarExpression(unique_ptr<ParsedExpression> expr, vector<unique_ptr<ParsedExpression>> &new_select_list);
-	bool FindStarExpression(ParsedExpression &expr, StarExpression **star);
+	bool FindStarExpression(unique_ptr<ParsedExpression> &expr, StarExpression **star, bool is_root, bool in_columns);
 	void ReplaceStarExpression(unique_ptr<ParsedExpression> &expr, unique_ptr<ParsedExpression> &replacement);
+	void BindWhereStarExpression(unique_ptr<ParsedExpression> &expr);
 
 	//! If only a schema name is provided (e.g. "a.b") then figure out if "a" is a schema or a catalog name
 	void BindSchemaOrCatalog(string &catalog_name, string &schema_name);
 	SchemaCatalogEntry *BindCreateSchema(CreateInfo &info);
 
+	unique_ptr<BoundQueryNode> BindSelectNode(SelectNode &statement, unique_ptr<BoundTableRef> from_table);
+
 public:
 	// This should really be a private constructor, but make_shared does not allow it...
 	// If you are thinking about calling this, you should probably call Binder::CreateBinder

package/src/duckdb/src/include/duckdb/planner/bound_result_modifier.hpp

@@ -65,6 +65,9 @@ public:
 
 	//! List of order nodes
 	vector<BoundOrderByNode> orders;
+
+	unique_ptr<BoundOrderModifier> Copy() const;
+	static bool Equals(const BoundOrderModifier *left, const BoundOrderModifier *right);
 };
 
 class BoundDistinctModifier : public BoundResultModifier {

package/src/duckdb/src/include/duckdb/planner/expression/bound_aggregate_expression.hpp

@@ -25,10 +25,13 @@ public:
 	vector<unique_ptr<Expression>> children;
 	//! The bound function data (if any)
 	unique_ptr<FunctionData> bind_info;
+	//! The aggregate type (distinct or non-distinct)
 	AggregateType aggr_type;
 
 	//! Filter for this aggregate
 	unique_ptr<Expression> filter;
+	//! The order by expression for this aggregate - if any
+	unique_ptr<BoundOrderModifier> order_bys;
 
 public:
 	bool IsDistinct() const {

package/src/duckdb/src/include/duckdb/planner/expression_binder/base_select_binder.hpp

@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/planner/expression_binder/base_select_binder.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/case_insensitive_map.hpp"
+#include "duckdb/parser/expression_map.hpp"
+#include "duckdb/planner/expression_binder.hpp"
+
+namespace duckdb {
+class BoundColumnRefExpression;
+class WindowExpression;
+
+class BoundSelectNode;
+
+struct BoundGroupInformation {
+	expression_map_t<idx_t> map;
+	case_insensitive_map_t<idx_t> alias_map;
+};
+
+//! The BaseSelectBinder is the base binder of the SELECT, HAVING and QUALIFY binders. It can bind aggregates and window
+//! functions.
+class BaseSelectBinder : public ExpressionBinder {
+public:
+	BaseSelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info,
+	                 case_insensitive_map_t<idx_t> alias_map);
+	BaseSelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info);
+
+	bool BoundAggregates() {
+		return bound_aggregate;
+	}
+	void ResetBindings() {
+		this->bound_aggregate = false;
+		this->bound_columns.clear();
+	}
+
+protected:
+	BindResult BindExpression(unique_ptr<ParsedExpression> *expr_ptr, idx_t depth,
+	                          bool root_expression = false) override;
+
+	BindResult BindAggregate(FunctionExpression &expr, AggregateFunctionCatalogEntry *function, idx_t depth) override;
+
+	bool inside_window;
+	bool bound_aggregate = false;
+
+	BoundSelectNode &node;
+	BoundGroupInformation &info;
+	case_insensitive_map_t<idx_t> alias_map;
+
+protected:
+	BindResult BindColumnRef(unique_ptr<ParsedExpression> *expr_ptr, idx_t depth);
+	BindResult BindGroupingFunction(OperatorExpression &op, idx_t depth) override;
+	BindResult BindWindow(WindowExpression &expr, idx_t depth);
+
+	idx_t TryBindGroup(ParsedExpression &expr, idx_t depth);
+	BindResult BindGroup(ParsedExpression &expr, idx_t depth, idx_t group_index);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/expression_binder/having_binder.hpp

@@ -8,14 +8,14 @@
 
 #pragma once
 
-#include "duckdb/planner/expression_binder/select_binder.hpp"
+#include "duckdb/planner/expression_binder/base_select_binder.hpp"
 #include "duckdb/planner/expression_binder/column_alias_binder.hpp"
 #include "duckdb/common/enums/aggregate_handling.hpp"
 
 namespace duckdb {
 
 //! The HAVING binder is responsible for binding an expression within the HAVING clause of a SQL statement
-class HavingBinder : public SelectBinder {
+class HavingBinder : public BaseSelectBinder {
 public:
 	HavingBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info,
 	             case_insensitive_map_t<idx_t> &alias_map, AggregateHandling aggregate_handling);

package/src/duckdb/src/include/duckdb/planner/expression_binder/order_binder.hpp

@@ -32,10 +32,13 @@ public:
 	idx_t MaxCount() const {
 		return max_count;
 	}
-
 	bool HasExtraList() const {
 		return extra_list;
 	}
+	const vector<Binder *> &GetBinders() const {
+		return binders;
+	}
+
 	unique_ptr<Expression> CreateExtraReference(unique_ptr<ParsedExpression> expr);
 
 private:

package/src/duckdb/src/include/duckdb/planner/expression_binder/qualify_binder.hpp

@@ -8,13 +8,13 @@
 
 #pragma once
 
-#include "duckdb/planner/expression_binder/select_binder.hpp"
+#include "duckdb/planner/expression_binder/base_select_binder.hpp"
 #include "duckdb/planner/expression_binder/column_alias_binder.hpp"
 
 namespace duckdb {
 
 //! The QUALIFY binder is responsible for binding an expression within the QUALIFY clause of a SQL statement
-class QualifyBinder : public SelectBinder {
+class QualifyBinder : public BaseSelectBinder {
 public:
 	QualifyBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info,
 	              case_insensitive_map_t<idx_t> &alias_map);

package/src/duckdb/src/include/duckdb/planner/expression_binder/select_binder.hpp

@@ -8,58 +8,29 @@
 
 #pragma once
 
-#include "duckdb/common/case_insensitive_map.hpp"
-#include "duckdb/parser/expression_map.hpp"
-#include "duckdb/planner/expression_binder.hpp"
+#include "duckdb/planner/expression_binder/base_select_binder.hpp"
 
 namespace duckdb {
-class BoundColumnRefExpression;
-class WindowExpression;
-
-class BoundSelectNode;
-
-struct BoundGroupInformation {
-	expression_map_t<idx_t> map;
-	case_insensitive_map_t<idx_t> alias_map;
-};
 
 //! The SELECT binder is responsible for binding an expression within the SELECT clause of a SQL statement
-class SelectBinder : public ExpressionBinder {
+class SelectBinder : public BaseSelectBinder {
 public:
 	SelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info,
 	             case_insensitive_map_t<idx_t> alias_map);
 	SelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info);
 
-	bool BoundAggregates() {
-		return bound_aggregate;
+	bool HasExpandedExpressions() {
+		return !expanded_expressions.empty();
 	}
-	void ResetBindings() {
-		this->bound_aggregate = false;
-		this->bound_columns.clear();
+	vector<unique_ptr<Expression>> &ExpandedExpressions() {
+		return expanded_expressions;
 	}
 
 protected:
-	BindResult BindExpression(unique_ptr<ParsedExpression> *expr_ptr, idx_t depth,
-	                          bool root_expression = false) override;
-
-	BindResult BindAggregate(FunctionExpression &expr, AggregateFunctionCatalogEntry *function, idx_t depth) override;
-
-	BindResult BindUnnest(FunctionExpression &function, idx_t depth) override;
-
-	bool inside_window;
-	bool bound_aggregate = false;
-
-	BoundSelectNode &node;
-	BoundGroupInformation &info;
-	case_insensitive_map_t<idx_t> alias_map;
-
-protected:
-	BindResult BindColumnRef(unique_ptr<ParsedExpression> *expr_ptr, idx_t depth);
-	BindResult BindGroupingFunction(OperatorExpression &op, idx_t depth) override;
-	BindResult BindWindow(WindowExpression &expr, idx_t depth);
+	BindResult BindUnnest(FunctionExpression &function, idx_t depth, bool root_expression) override;
 
-	idx_t TryBindGroup(ParsedExpression &expr, idx_t depth);
-	BindResult BindGroup(ParsedExpression &expr, idx_t depth, idx_t group_index);
+	idx_t unnest_level = 0;
+	vector<unique_ptr<Expression>> expanded_expressions;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/expression_binder.hpp

@@ -135,7 +135,7 @@ protected:
 	virtual BindResult BindFunction(FunctionExpression &expr, ScalarFunctionCatalogEntry *function, idx_t depth);
 	virtual BindResult BindLambdaFunction(FunctionExpression &expr, ScalarFunctionCatalogEntry *function, idx_t depth);
 	virtual BindResult BindAggregate(FunctionExpression &expr, AggregateFunctionCatalogEntry *function, idx_t depth);
-	virtual BindResult BindUnnest(FunctionExpression &expr, idx_t depth);
+	virtual BindResult BindUnnest(FunctionExpression &expr, idx_t depth, bool root_expression);
 	virtual BindResult BindMacro(FunctionExpression &expr, ScalarMacroCatalogEntry *macro, idx_t depth,
 	                             unique_ptr<ParsedExpression> *expr_ptr);
 

package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp

@@ -14,6 +14,7 @@ class LogicalOperator;
 
 class LogicalAggregate;
 class LogicalAnyJoin;
+class LogicalAsOfJoin;
 class LogicalColumnDataGet;
 class LogicalComparisonJoin;
 class LogicalCopyToFile;

package/src/duckdb/src/include/duckdb/planner/operator/list.hpp

@@ -1,5 +1,6 @@
 #include "duckdb/planner/operator/logical_aggregate.hpp"
 #include "duckdb/planner/operator/logical_any_join.hpp"
+#include "duckdb/planner/operator/logical_asof_join.hpp"
 #include "duckdb/planner/operator/logical_column_data_get.hpp"
 #include "duckdb/planner/operator/logical_comparison_join.hpp"
 #include "duckdb/planner/operator/logical_copy_to_file.hpp"

package/src/duckdb/src/include/duckdb/planner/operator/logical_asof_join.hpp

@@ -0,0 +1,22 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/planner/operator/logical_asof_join.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/planner/operator/logical_comparison_join.hpp"
+
+namespace duckdb {
+
+//! LogicalAsOfJoin represents a temporal-style join with one less-than inequality.
+//! This inequality matches the greatest value on the right that satisfies the condition.
+class LogicalAsOfJoin : public LogicalComparisonJoin {
+public:
+	explicit LogicalAsOfJoin(JoinType type);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/common/constants.hpp"
+#include "duckdb/common/enums/joinref_type.hpp"
 #include "duckdb/common/unordered_set.hpp"
 #include "duckdb/planner/joinside.hpp"
 #include "duckdb/planner/operator/logical_join.hpp"
@@ -34,10 +35,12 @@ public:
 	                        FieldReader &reader);
 
 public:
-	static unique_ptr<LogicalOperator> CreateJoin(JoinType type, unique_ptr<LogicalOperator> left_child,
+	static unique_ptr<LogicalOperator> CreateJoin(JoinType type, JoinRefType ref_type,
+	                                              unique_ptr<LogicalOperator> left_child,
 	                                              unique_ptr<LogicalOperator> right_child,
 	                                              unique_ptr<Expression> condition);
-	static unique_ptr<LogicalOperator> CreateJoin(JoinType type, unique_ptr<LogicalOperator> left_child,
+	static unique_ptr<LogicalOperator> CreateJoin(JoinType type, JoinRefType ref_type,
+	                                              unique_ptr<LogicalOperator> left_child,
 	                                              unique_ptr<LogicalOperator> right_child,
 	                                              vector<JoinCondition> conditions,
 	                                              vector<unique_ptr<Expression>> arbitrary_expressions);

package/src/duckdb/src/include/duckdb/planner/operator/logical_distinct.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/planner/bound_result_modifier.hpp"
 
 namespace duckdb {
 
@@ -22,6 +23,8 @@ public:
 	}
 	//! The set of distinct targets (optional).
 	vector<unique_ptr<Expression>> distinct_targets;
+	//! The order by modifier (optional, only for distinct on)
+	unique_ptr<BoundOrderModifier> order_by;
 
 public:
 	string ParamsToString() const override;

package/src/duckdb/src/include/duckdb/planner/query_node/bound_select_node.hpp

@@ -25,6 +25,13 @@ public:
 	vector<GroupingSet> grouping_sets;
 };
 
+struct BoundUnnestNode {
+	//! The index of the UNNEST node
+	idx_t index;
+	//! The set of expressions
+	vector<unique_ptr<Expression>> expressions;
+};
+
 //! Bound equivalent of SelectNode
 class BoundSelectNode : public BoundQueryNode {
 public:
@@ -78,9 +85,8 @@ public:
 	//! Window functions to compute (only used if HasWindow is true)
 	vector<unique_ptr<Expression>> windows;
 
-	idx_t unnest_index;
 	//! Unnest expression
-	vector<unique_ptr<Expression>> unnests;
+	unordered_map<idx_t, BoundUnnestNode> unnests;
 
 	//! Index of pruned node
 	idx_t prune_index;

package/src/duckdb/src/include/duckdb/storage/buffer/block_handle.hpp

@@ -17,6 +17,7 @@ namespace duckdb {
 class BlockManager;
 class BufferHandle;
 class BufferManager;
+class BufferPool;
 class DatabaseInstance;
 class FileBuffer;
 
@@ -55,6 +56,7 @@ class BlockHandle {
 	friend struct BufferEvictionNode;
 	friend class BufferHandle;
 	friend class BufferManager;
+	friend class BufferPool;
 
 public:
 	BlockHandle(BlockManager &block_manager, block_id_t block_id);