duckdb 0.7.2-dev16.0 → 0.7.2-dev314.0

package/src/duckdb/src/include/duckdb/function/table_function.hpp

@@ -235,6 +235,8 @@ public:
 	bool filter_prune;
 	//! Additional function info, passed to the bind
 	shared_ptr<TableFunctionInfo> function_info;
+
+	DUCKDB_API bool Equal(const TableFunction &rhs) const;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/capi/capi_internal.hpp

@@ -40,6 +40,7 @@ struct ExtractStatementsWrapper {
 
 struct PendingStatementWrapper {
 	unique_ptr<PendingQueryResult> statement;
+	bool allow_streaming;
 };
 
 struct ArrowResultWrapper {
@@ -56,6 +57,7 @@ struct AppenderWrapper {
 enum class CAPIResultSetType : uint8_t {
 	CAPI_RESULT_TYPE_NONE = 0,
 	CAPI_RESULT_TYPE_MATERIALIZED,
+	CAPI_RESULT_TYPE_STREAMING,
 	CAPI_RESULT_TYPE_DEPRECATED
 };
 

package/src/duckdb/src/include/duckdb/main/config.hpp

@@ -30,6 +30,7 @@
 #include "duckdb/planner/operator_extension.hpp"
 
 namespace duckdb {
+class BufferPool;
 class CastFunctionSet;
 class ClientContext;
 class ErrorManager;
@@ -186,6 +187,8 @@ public:
 	vector<std::unique_ptr<OperatorExtension>> operator_extensions;
 	//! Extensions made to storage
 	case_insensitive_map_t<std::unique_ptr<StorageExtension>> storage_extensions;
+	//! A buffer pool can be shared across multiple databases (if desired).
+	shared_ptr<BufferPool> buffer_pool;
 
 public:
 	DUCKDB_API static DBConfig &GetConfig(ClientContext &context);

package/src/duckdb/src/include/duckdb/main/database.hpp

@@ -35,6 +35,7 @@ public:
 	DBConfig config;
 
 public:
+	BufferPool &GetBufferPool();
 	DUCKDB_API BufferManager &GetBufferManager();
 	DUCKDB_API DatabaseManager &GetDatabaseManager();
 	DUCKDB_API FileSystem &GetFileSystem();

package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp

@@ -106,7 +106,7 @@ private:
 	//! If there are multiple equivalence sets, they are merged.
 	void AddToEquivalenceSets(FilterInfo *filter_info, vector<idx_t> matching_equivalent_sets);
 
-	TableFilterSet *GetTableFilters(LogicalOperator *op);
+	TableFilterSet *GetTableFilters(LogicalOperator *op, idx_t table_index);
 
 	void AddRelationTdom(FilterInfo *filter_info);
 	bool EmptyFilter(FilterInfo *filter_info);
@@ -115,7 +115,7 @@ private:
 	                            unique_ptr<BaseStatistics> base_stats);
 	idx_t InspectConjunctionOR(idx_t cardinality, idx_t column_index, ConjunctionOrFilter *fil,
 	                           unique_ptr<BaseStatistics> base_stats);
-	idx_t InspectTableFilters(idx_t cardinality, LogicalOperator *op, TableFilterSet *table_filters);
+	idx_t InspectTableFilters(idx_t cardinality, LogicalOperator *op, TableFilterSet *table_filters, idx_t table_index);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp

@@ -17,6 +17,8 @@ class SelectStatement;
 struct CommonTableExpressionInfo {
 	vector<string> aliases;
 	unique_ptr<SelectStatement> query;
+
+	unique_ptr<CommonTableExpressionInfo> Copy();
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/parser/parsed_data/alter_info.hpp

@@ -20,7 +20,8 @@ enum class AlterType : uint8_t {
 	ALTER_VIEW = 2,
 	ALTER_SEQUENCE = 3,
 	CHANGE_OWNERSHIP = 4,
-	ALTER_FUNCTION = 5
+	ALTER_SCALAR_FUNCTION = 5,
+	ALTER_TABLE_FUNCTION = 6
 };
 
 struct AlterEntryData {

package/src/duckdb/src/include/duckdb/parser/parsed_data/{alter_function_info.hpp → alter_scalar_function_info.hpp}

@@ -1,29 +1,29 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/parser/parsed_data/alter_function_info.hpp
+// duckdb/parser/parsed_data/alter_scalar_function_info.hpp
 //
 //
 //===----------------------------------------------------------------------===//
 
 #pragma once
 
-#include "duckdb/parser/parsed_data/alter_info.hpp"
-#include "duckdb/function/scalar_function.hpp"
 #include "duckdb/function/function_set.hpp"
+#include "duckdb/function/scalar_function.hpp"
+#include "duckdb/parser/parsed_data/alter_info.hpp"
 
 namespace duckdb {
 
 //===--------------------------------------------------------------------===//
-// Alter Table
+// Alter Scalar Function
 //===--------------------------------------------------------------------===//
-enum class AlterFunctionType : uint8_t { INVALID = 0, ADD_FUNCTION_OVERLOADS = 1 };
+enum class AlterScalarFunctionType : uint8_t { INVALID = 0, ADD_FUNCTION_OVERLOADS = 1 };
 
-struct AlterFunctionInfo : public AlterInfo {
-	AlterFunctionInfo(AlterFunctionType type, AlterEntryData data);
-	virtual ~AlterFunctionInfo() override;
+struct AlterScalarFunctionInfo : public AlterInfo {
+	AlterScalarFunctionInfo(AlterScalarFunctionType type, AlterEntryData data);
+	virtual ~AlterScalarFunctionInfo() override;
 
-	AlterFunctionType alter_function_type;
+	AlterScalarFunctionType alter_scalar_function_type;
 
 public:
 	CatalogType GetCatalogType() const override;
@@ -32,11 +32,11 @@ public:
 };
 
 //===--------------------------------------------------------------------===//
-// AddFunctionOverloadInfo
+// AddScalarFunctionOverloadInfo
 //===--------------------------------------------------------------------===//
-struct AddFunctionOverloadInfo : public AlterFunctionInfo {
-	AddFunctionOverloadInfo(AlterEntryData data, ScalarFunctionSet new_overloads);
-	~AddFunctionOverloadInfo() override;
+struct AddScalarFunctionOverloadInfo : public AlterScalarFunctionInfo {
+	AddScalarFunctionOverloadInfo(AlterEntryData data, ScalarFunctionSet new_overloads);
+	~AddScalarFunctionOverloadInfo() override;
 
 	ScalarFunctionSet new_overloads;
 

package/src/duckdb/src/include/duckdb/parser/parsed_data/alter_table_function_info.hpp

@@ -0,0 +1,47 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/parser/parsed_data/alter_scalar_function_info.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/function/function_set.hpp"
+#include "duckdb/function/table_function.hpp"
+#include "duckdb/parser/parsed_data/alter_info.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Alter Table Function
+//===--------------------------------------------------------------------===//
+enum class AlterTableFunctionType : uint8_t { INVALID = 0, ADD_FUNCTION_OVERLOADS = 1 };
+
+struct AlterTableFunctionInfo : public AlterInfo {
+	AlterTableFunctionInfo(AlterTableFunctionType type, AlterEntryData data);
+	virtual ~AlterTableFunctionInfo() override;
+
+	AlterTableFunctionType alter_table_function_type;
+
+public:
+	CatalogType GetCatalogType() const override;
+	void Serialize(FieldWriter &writer) const override;
+	static unique_ptr<AlterInfo> Deserialize(FieldReader &reader);
+};
+
+//===--------------------------------------------------------------------===//
+// AddTableFunctionOverloadInfo
+//===--------------------------------------------------------------------===//
+struct AddTableFunctionOverloadInfo : public AlterTableFunctionInfo {
+	AddTableFunctionOverloadInfo(AlterEntryData data, TableFunctionSet new_overloads);
+	~AddTableFunctionOverloadInfo() override;
+
+	TableFunctionSet new_overloads;
+
+public:
+	unique_ptr<AlterInfo> Copy() const override;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/parser/parsed_data/create_table_function_info.hpp

@@ -21,7 +21,8 @@ struct CreateTableFunctionInfo : public CreateFunctionInfo {
 	TableFunctionSet functions;
 
 public:
-	DUCKDB_API unique_ptr<CreateInfo> Copy() const;
+	DUCKDB_API unique_ptr<CreateInfo> Copy() const override;
+	DUCKDB_API unique_ptr<AlterInfo> GetAlterInfo() const override;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/parser/query_node.hpp

@@ -13,6 +13,7 @@
 #include "duckdb/parser/parsed_expression.hpp"
 #include "duckdb/parser/result_modifier.hpp"
 #include "duckdb/parser/common_table_expression_info.hpp"
+#include "duckdb/common/case_insensitive_map.hpp"
 
 namespace duckdb {
 
@@ -29,7 +30,7 @@ class CommonTableExpressionMap {
 public:
 	CommonTableExpressionMap();
 
-	unordered_map<string, unique_ptr<CommonTableExpressionInfo>> map;
+	case_insensitive_map_t<unique_ptr<CommonTableExpressionInfo>> map;
 
 public:
 	string ToString() const;

package/src/duckdb/src/include/duckdb/parser/statement/multi_statement.hpp

@@ -0,0 +1,28 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/parser/statement/multi_statement.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/parser/sql_statement.hpp"
+
+namespace duckdb {
+
+class MultiStatement : public SQLStatement {
+public:
+	MultiStatement();
+
+	vector<unique_ptr<SQLStatement>> statements;
+
+protected:
+	MultiStatement(const MultiStatement &other);
+
+public:
+	unique_ptr<SQLStatement> Copy() const override;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/parser/tableref/list.hpp

@@ -2,5 +2,6 @@
 #include "duckdb/parser/tableref/emptytableref.hpp"
 #include "duckdb/parser/tableref/expressionlistref.hpp"
 #include "duckdb/parser/tableref/joinref.hpp"
+#include "duckdb/parser/tableref/pivotref.hpp"
 #include "duckdb/parser/tableref/subqueryref.hpp"
 #include "duckdb/parser/tableref/table_function_ref.hpp"

package/src/duckdb/src/include/duckdb/parser/tableref/pivotref.hpp

@@ -0,0 +1,76 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/parser/tableref/pivotref.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/parser/tableref.hpp"
+
+namespace duckdb {
+
+struct PivotColumnEntry {
+	//! The set of values to match on
+	vector<Value> values;
+	//! The star expression (UNPIVOT only)
+	unique_ptr<ParsedExpression> star_expr;
+	//! The alias of the pivot column entry
+	string alias;
+
+	bool Equals(const PivotColumnEntry &other) const;
+	void Serialize(Serializer &serializer) const;
+	PivotColumnEntry Copy() const;
+	static PivotColumnEntry Deserialize(Deserializer &source);
+};
+
+struct PivotColumn {
+	//! The column names to (un)pivot
+	vector<string> names;
+	//! The set of values to pivot on
+	vector<PivotColumnEntry> entries;
+	//! The enum to read pivot values from (if any)
+	string pivot_enum;
+
+	string ToString() const;
+	bool Equals(const PivotColumn &other) const;
+	void Serialize(Serializer &serializer) const;
+	PivotColumn Copy() const;
+	static PivotColumn Deserialize(Deserializer &source);
+};
+
+//! Represents a PIVOT or UNPIVOT expression
+class PivotRef : public TableRef {
+public:
+	explicit PivotRef() : TableRef(TableReferenceType::PIVOT), include_nulls(false) {
+	}
+
+	//! The source table of the pivot
+	unique_ptr<TableRef> source;
+	//! The aggregates to compute over the pivot (PIVOT only)
+	vector<unique_ptr<ParsedExpression>> aggregates;
+	//! The names of the unpivot expressions (UNPIVOT only)
+	vector<string> unpivot_names;
+	//! The set of pivots
+	vector<PivotColumn> pivots;
+	//! The groups to pivot over. If none are specified all columns not included in the pivots/aggregate are chosen.
+	vector<string> groups;
+	//! Aliases for the column names
+	vector<string> column_name_alias;
+	//! Whether or not to include nulls in the result (UNPIVOT only)
+	bool include_nulls;
+
+public:
+	string ToString() const override;
+	bool Equals(const TableRef *other_p) const override;
+
+	unique_ptr<TableRef> Copy() override;
+
+	//! Serializes a blob into a JoinRef
+	void Serialize(FieldWriter &serializer) const override;
+	//! Deserializes a blob back into a JoinRef
+	static unique_ptr<TableRef> Deserialize(FieldReader &source);
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/parser/tokens.hpp

@@ -41,6 +41,7 @@ class SetVariableStatement;
 class ResetVariableStatement;
 class LoadStatement;
 class LogicalPlanStatement;
+class MultiStatement;
 
 //===--------------------------------------------------------------------===//
 // Query Node
@@ -94,6 +95,7 @@ class SubqueryRef;
 class TableFunctionRef;
 class EmptyTableRef;
 class ExpressionListRef;
+class PivotRef;
 
 //===--------------------------------------------------------------------===//
 // Other

package/src/duckdb/src/include/duckdb/parser/transformer.hpp

@@ -32,15 +32,23 @@ struct CommonTableExpressionInfo;
 struct GroupingExpressionMap;
 class OnConflictInfo;
 class UpdateSetInfo;
+struct PivotColumn;
 
 //! The transformer class is responsible for transforming the internal Postgres
 //! parser representation into the DuckDB representation
 class Transformer {
 	friend class StackChecker;
 
+	struct CreatePivotEntry {
+		string enum_name;
+		unique_ptr<SelectNode> base;
+		string column_name;
+	};
+
 public:
 	explicit Transformer(idx_t max_expression_depth_p);
 	explicit Transformer(Transformer *parent);
+	~Transformer();
 
 	//! Transforms a Postgres parse tree into a set of SQL Statements
 	bool TransformParseTree(duckdb_libpgquery::PGList *tree, vector<unique_ptr<SQLStatement>> &statements);
@@ -59,6 +67,12 @@ private:
 	case_insensitive_map_t<idx_t> named_param_map;
 	//! Holds window expressions defined by name. We need those when transforming the expressions referring to them.
 	unordered_map<string, duckdb_libpgquery::PGWindowDef *> window_clauses;
+	//! The set of pivot entries to create
+	vector<unique_ptr<CreatePivotEntry>> pivot_entries;
+	//! Sets of stored CTEs, if any
+	vector<CommonTableExpressionMap *> stored_cte_map;
+
+	void Clear();
 
 	void SetParamCount(idx_t new_count) {
 		if (parent) {
@@ -91,6 +105,12 @@ private:
 		return parent ? parent->HasNamedParameters() : !named_param_map.empty();
 	}
 
+	void AddPivotEntry(string enum_name, unique_ptr<SelectNode> source, string column_name);
+	unique_ptr<SQLStatement> GenerateCreateEnumStmt(unique_ptr<CreatePivotEntry> entry);
+	bool HasPivotEntries();
+	idx_t PivotEntryCount();
+	void ExtractCTEsRecursive(CommonTableExpressionMap &cte_map);
+
 private:
 	//! Transforms a Postgres statement into a single SQL statement
 	unique_ptr<SQLStatement> TransformStatement(duckdb_libpgquery::PGNode *stmt);
@@ -163,6 +183,10 @@ private:
 	unique_ptr<ExecuteStatement> TransformExecute(duckdb_libpgquery::PGNode *node);
 	unique_ptr<CallStatement> TransformCall(duckdb_libpgquery::PGNode *node);
 	unique_ptr<DropStatement> TransformDeallocate(duckdb_libpgquery::PGNode *node);
+	unique_ptr<QueryNode> TransformPivotStatement(duckdb_libpgquery::PGSelectStmt *stmt);
+	unique_ptr<SQLStatement> CreatePivotStatement(unique_ptr<SQLStatement> statement);
+	PivotColumn TransformPivotColumn(duckdb_libpgquery::PGPivot *pivot);
+	vector<PivotColumn> TransformPivotList(duckdb_libpgquery::PGList *list);
 
 	//===--------------------------------------------------------------------===//
 	// SetStatement Transform
@@ -179,6 +203,7 @@ private:
 	//===--------------------------------------------------------------------===//
 	//! Transform a Postgres duckdb_libpgquery::T_PGSelectStmt node into a QueryNode
 	unique_ptr<QueryNode> TransformSelectNode(duckdb_libpgquery::PGSelectStmt *node);
+	unique_ptr<QueryNode> TransformSelectInternal(duckdb_libpgquery::PGSelectStmt *node);
 
 	//===--------------------------------------------------------------------===//
 	// Expression Transform
@@ -256,6 +281,7 @@ private:
 	//===--------------------------------------------------------------------===//
 	OnCreateConflict TransformOnConflict(duckdb_libpgquery::PGOnCreateConflict conflict);
 	string TransformAlias(duckdb_libpgquery::PGAlias *root, vector<string> &column_name_alias);
+	vector<string> TransformStringList(duckdb_libpgquery::PGList *list);
 	void TransformCTE(duckdb_libpgquery::PGWithClause *de_with_clause, CommonTableExpressionMap &cte_map);
 	unique_ptr<SelectStatement> TransformRecursiveCTE(duckdb_libpgquery::PGCommonTableExpr *node,
 	                                                  CommonTableExpressionInfo &info);
@@ -276,6 +302,8 @@ private:
 	unique_ptr<TableRef> TransformRangeFunction(duckdb_libpgquery::PGRangeFunction *root);
 	//! Transform a Postgres join node into a TableRef
 	unique_ptr<TableRef> TransformJoin(duckdb_libpgquery::PGJoinExpr *root);
+	//! Transform a Postgres pivot node into a TableRef
+	unique_ptr<TableRef> TransformPivot(duckdb_libpgquery::PGPivotExpr *root);
 	//! Transform a table producing subquery into a TableRef
 	unique_ptr<TableRef> TransformRangeSubselect(duckdb_libpgquery::PGRangeSubselect *root);
 	//! Transform a VALUES list into a set of expressions

package/src/duckdb/src/include/duckdb/planner/bind_context.hpp

@@ -141,6 +141,8 @@ public:
 
 	//! Add all the bindings from a BindContext to this BindContext. The other BindContext is destroyed in the process.
 	void AddContext(BindContext other);
+	//! For semi and anti joins we remove the binding context of the right table after binding the condition.
+	void RemoveContext(vector<std::pair<string, duckdb::Binding *>> &other_bindings_list);
 
 	//! Gets a binding of the specified name. Returns a nullptr and sets the out_error if the binding could not be
 	//! found.

package/src/duckdb/src/include/duckdb/planner/binder.hpp

@@ -275,6 +275,12 @@ private:
 	unique_ptr<BoundTableRef> Bind(TableFunctionRef &ref);
 	unique_ptr<BoundTableRef> Bind(EmptyTableRef &ref);
 	unique_ptr<BoundTableRef> Bind(ExpressionListRef &ref);
+	unique_ptr<BoundTableRef> Bind(PivotRef &expr);
+
+	unique_ptr<SelectNode> BindPivot(PivotRef &expr, vector<unique_ptr<ParsedExpression>> all_columns);
+	unique_ptr<SelectNode> BindUnpivot(Binder &child_binder, PivotRef &expr,
+	                                   vector<unique_ptr<ParsedExpression>> all_columns,
+	                                   unique_ptr<ParsedExpression> &where_clause);
 
 	bool BindTableFunctionParameters(TableFunctionCatalogEntry &table_function,
 	                                 vector<unique_ptr<ParsedExpression>> &expressions, vector<LogicalType> &arguments,
@@ -340,6 +346,8 @@ private:
 	void BindSchemaOrCatalog(string &catalog_name, string &schema_name);
 	SchemaCatalogEntry *BindCreateSchema(CreateInfo &info);
 
+	unique_ptr<BoundQueryNode> BindSelectNode(SelectNode &statement, unique_ptr<BoundTableRef> from_table);
+
 public:
 	// This should really be a private constructor, but make_shared does not allow it...
 	// If you are thinking about calling this, you should probably call Binder::CreateBinder

package/src/duckdb/src/include/duckdb/storage/buffer/block_handle.hpp

@@ -17,6 +17,7 @@ namespace duckdb {
 class BlockManager;
 class BufferHandle;
 class BufferManager;
+class BufferPool;
 class DatabaseInstance;
 class FileBuffer;
 
@@ -55,6 +56,7 @@ class BlockHandle {
 	friend struct BufferEvictionNode;
 	friend class BufferHandle;
 	friend class BufferManager;
+	friend class BufferPool;
 
 public:
 	BlockHandle(BlockManager &block_manager, block_id_t block_id);