duckdb 0.6.2-dev781.0 → 0.6.2-dev891.0

package/src/duckdb/src/include/duckdb/function/scalar/enum_functions.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/function/scalar_function.hpp"
 #include "duckdb/function/function_set.hpp"
+#include "duckdb/function/built_in_functions.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/function/scalar/generic_functions.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/function/scalar_function.hpp"
 #include "duckdb/function/function_set.hpp"
+#include "duckdb/function/built_in_functions.hpp"
 
 namespace duckdb {
 class BoundFunctionExpression;

package/src/duckdb/src/include/duckdb/function/scalar/math_functions.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/function/scalar_function.hpp"
 #include "duckdb/function/function_set.hpp"
+#include "duckdb/function/built_in_functions.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp

@@ -13,6 +13,7 @@
 #include "duckdb/common/map.hpp"
 #include "duckdb/common/unordered_map.hpp"
 #include "duckdb/common/field_writer.hpp"
+#include "duckdb/function/built_in_functions.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/function/scalar/operators.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/function/function_set.hpp"
 #include "duckdb/function/scalar_function.hpp"
+#include "duckdb/function/built_in_functions.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/function/scalar/regexp.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/function/function_set.hpp"
 #include "re2/re2.h"
+#include "duckdb/function/built_in_functions.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/function/scalar/sequence_functions.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/function/scalar_function.hpp"
 #include "duckdb/function/function_set.hpp"
+#include "duckdb/function/built_in_functions.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/function/function_set.hpp"
 #include "utf8proc.hpp"
+#include "duckdb/function/built_in_functions.hpp"
 
 namespace re2 {
 class RE2;

package/src/duckdb/src/include/duckdb/function/scalar/trigonometric_functions.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/function/scalar_function.hpp"
 #include "duckdb/function/function_set.hpp"
+#include "duckdb/function/built_in_functions.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/function/scalar/uuid_functions.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/function/function_set.hpp"
+#include "duckdb/function/built_in_functions.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/function/scalar_function.hpp

@@ -24,6 +24,7 @@ struct FunctionLocalState {
 
 class Binder;
 class BoundFunctionExpression;
+class DependencyList;
 class ScalarFunctionCatalogEntry;
 
 struct FunctionStatisticsInput {
@@ -48,7 +49,7 @@ typedef unique_ptr<FunctionLocalState> (*init_local_state_t)(ExpressionState &st
                                                              FunctionData *bind_data);
 typedef unique_ptr<BaseStatistics> (*function_statistics_t)(ClientContext &context, FunctionStatisticsInput &input);
 //! Adds the dependencies of this BoundFunctionExpression to the set of dependencies
-typedef void (*dependency_function_t)(BoundFunctionExpression &expr, unordered_set<CatalogEntry *> &dependencies);
+typedef void (*dependency_function_t)(BoundFunctionExpression &expr, DependencyList &dependencies);
 
 typedef void (*function_serialize_t)(FieldWriter &writer, const FunctionData *bind_data,
                                      const ScalarFunction &function);

package/src/duckdb/src/include/duckdb/function/table/arrow.hpp

@@ -15,6 +15,7 @@
 #include "duckdb/common/pair.hpp"
 #include "duckdb/common/thread.hpp"
 #include "duckdb/common/unordered_map.hpp"
+#include "duckdb/function/built_in_functions.hpp"
 
 namespace duckdb {
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/include/duckdb/function/table/range.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/function/table_function.hpp"
+#include "duckdb/function/built_in_functions.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/function/table/read_csv.hpp

@@ -15,6 +15,7 @@
 #include "duckdb/execution/operator/persistent/parallel_csv_reader.hpp"
 #include "duckdb/execution/operator/persistent/csv_file_handle.hpp"
 #include "duckdb/execution/operator/persistent/csv_buffer.hpp"
+#include "duckdb/function/built_in_functions.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/function/table/summary.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/function/table_function.hpp"
+#include "duckdb/function/built_in_functions.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/function/table_function.hpp"
+#include "duckdb/function/built_in_functions.hpp"
 
 namespace duckdb {
 
@@ -16,10 +17,6 @@ struct PragmaCollations {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
 
-struct PragmaFunctionPragma {
-	static void RegisterFunction(BuiltinFunctions &set);
-};
-
 struct PragmaTableInfo {
 	static void RegisterFunction(BuiltinFunctions &set);
 };

package/src/duckdb/src/include/duckdb/function/table/table_scan.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/function/table_function.hpp"
 #include "duckdb/common/atomic.hpp"
+#include "duckdb/function/built_in_functions.hpp"
 
 namespace duckdb {
 class TableCatalogEntry;

package/src/duckdb/src/include/duckdb/function/table_function.hpp

@@ -18,6 +18,7 @@
 namespace duckdb {
 
 class BaseStatistics;
+class DependencyList;
 class LogicalGet;
 class TableFilterSet;
 
@@ -156,7 +157,7 @@ typedef BindInfo (*table_function_get_bind_info)(const FunctionData *bind_data);
 
 typedef double (*table_function_progress_t)(ClientContext &context, const FunctionData *bind_data,
                                             const GlobalTableFunctionState *global_state);
-typedef void (*table_function_dependency_t)(unordered_set<CatalogEntry *> &dependencies, const FunctionData *bind_data);
+typedef void (*table_function_dependency_t)(DependencyList &dependencies, const FunctionData *bind_data);
 typedef unique_ptr<NodeStatistics> (*table_function_cardinality_t)(ClientContext &context,
                                                                    const FunctionData *bind_data);
 typedef void (*table_function_pushdown_complex_filter_t)(ClientContext &context, LogicalGet &get,

package/src/duckdb/src/include/duckdb/main/attached_database.hpp

@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/main/attached_database.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/case_insensitive_map.hpp"
+#include "duckdb/common/mutex.hpp"
+#include "duckdb/main/config.hpp"
+#include "duckdb/catalog/catalog_entry.hpp"
+
+namespace duckdb {
+class Catalog;
+class DatabaseInstance;
+class StorageManager;
+class TransactionManager;
+
+enum class AttachedDatabaseType {
+	READ_WRITE_DATABASE,
+	READ_ONLY_DATABASE,
+	SYSTEM_DATABASE,
+	TEMP_DATABASE,
+};
+
+//! The AttachedDatabase represents an attached database instance
+class AttachedDatabase : public CatalogEntry {
+public:
+	//! Create the built-in system attached database (without storage)
+	explicit AttachedDatabase(DatabaseInstance &db, AttachedDatabaseType type = AttachedDatabaseType::SYSTEM_DATABASE);
+	//! Create an attached database instance with the specified name and storage
+	AttachedDatabase(DatabaseInstance &db, Catalog &catalog, string name, string file_path, AccessMode access_mode);
+	~AttachedDatabase();
+
+	void Initialize();
+
+	StorageManager &GetStorageManager();
+	Catalog &GetCatalog();
+	TransactionManager &GetTransactionManager();
+	DatabaseInstance &GetDatabase() {
+		return db;
+	}
+	const string &GetName() const {
+		return name;
+	}
+	bool IsSystem() const;
+	bool IsTemporary() const;
+	bool IsReadOnly() const;
+
+	static string ExtractDatabaseName(const string &dbpath);
+
+private:
+	DatabaseInstance &db;
+	unique_ptr<StorageManager> storage;
+	unique_ptr<Catalog> catalog;
+	unique_ptr<TransactionManager> transaction_manager;
+	AttachedDatabaseType type;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/main/client_context.hpp

@@ -72,8 +72,6 @@ public:
 
 	//! The database that this client is connected to
 	shared_ptr<DatabaseInstance> db;
-	//! Data for the currently running transaction
-	TransactionContext transaction;
 	//! Whether or not the query is interrupted
 	atomic<bool> interrupted;
 	//! External Objects (e.g., Python objects) that views depend of
@@ -84,9 +82,11 @@ public:
 	ClientConfig config;
 	//! The set of client-specific data
 	unique_ptr<ClientData> client_data;
+	//! Data for the currently running transaction
+	TransactionContext transaction;
 
 public:
-	DUCKDB_API Transaction &ActiveTransaction() {
+	DUCKDB_API MetaTransaction &ActiveTransaction() {
 		return transaction.ActiveTransaction();
 	}
 

package/src/duckdb/src/include/duckdb/main/client_data.hpp

@@ -15,6 +15,7 @@
 #include "duckdb/common/atomic.hpp"
 
 namespace duckdb {
+class AttachedDatabase;
 class BufferedFileWriter;
 class ClientContext;
 class CatalogSearchPath;
@@ -36,7 +37,7 @@ struct ClientData {
 	unique_ptr<QueryProfilerHistory> query_profiler_history;
 
 	//! The set of temporary objects that belong to this client
-	shared_ptr<SchemaCatalogEntry> temporary_objects;
+	shared_ptr<AttachedDatabase> temporary_objects;
 	//! The set of bound prepared statements that belong to this client
 	unordered_map<string, shared_ptr<PreparedStatementData>> prepared_statements;
 

package/src/duckdb/src/include/duckdb/main/config.hpp

@@ -142,6 +142,8 @@ struct DBConfigOptions {
 	bool enable_fsst_vectors = false;
 	//! Experimental parallel CSV reader
 	bool experimental_parallel_csv_reader = false;
+	//! Start transactions immediately in all attached databases - instead of lazily when a database is referenced
+	bool immediate_transaction_mode = false;
 
 	bool operator==(const DBConfigOptions &other) const;
 };
@@ -185,6 +187,7 @@ public:
 public:
 	DUCKDB_API static DBConfig &GetConfig(ClientContext &context);
 	DUCKDB_API static DBConfig &GetConfig(DatabaseInstance &db);
+	DUCKDB_API static DBConfig &Get(AttachedDatabase &db);
 	DUCKDB_API static const DBConfig &GetConfig(const ClientContext &context);
 	DUCKDB_API static const DBConfig &GetConfig(const DatabaseInstance &db);
 	DUCKDB_API static vector<ConfigurationOption> GetOptions();

package/src/duckdb/src/include/duckdb/main/database.hpp

@@ -14,6 +14,8 @@
 #include "duckdb/main/extension.hpp"
 
 namespace duckdb {
+class BufferManager;
+class DatabaseManager;
 class StorageManager;
 class Catalog;
 class TransactionManager;
@@ -32,10 +34,9 @@ public:
 	DBConfig config;
 
 public:
-	DUCKDB_API StorageManager &GetStorageManager();
-	DUCKDB_API Catalog &GetCatalog();
+	DUCKDB_API BufferManager &GetBufferManager();
+	DUCKDB_API DatabaseManager &GetDatabaseManager();
 	DUCKDB_API FileSystem &GetFileSystem();
-	DUCKDB_API TransactionManager &GetTransactionManager();
 	DUCKDB_API TaskScheduler &GetScheduler();
 	DUCKDB_API ObjectCache &GetObjectCache();
 	DUCKDB_API ConnectionManager &GetConnectionManager();
@@ -56,9 +57,8 @@ private:
 	void Configure(DBConfig &config);
 
 private:
-	unique_ptr<StorageManager> storage;
-	unique_ptr<Catalog> catalog;
-	unique_ptr<TransactionManager> transaction_manager;
+	unique_ptr<BufferManager> buffer_manager;
+	unique_ptr<DatabaseManager> db_manager;
 	unique_ptr<TaskScheduler> scheduler;
 	unique_ptr<ObjectCache> object_cache;
 	unique_ptr<ConnectionManager> connection_manager;

package/src/duckdb/src/include/duckdb/main/database_manager.hpp

@@ -0,0 +1,69 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/main/database_manager.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/case_insensitive_map.hpp"
+#include "duckdb/common/mutex.hpp"
+#include "duckdb/common/atomic.hpp"
+
+namespace duckdb {
+class AttachedDatabase;
+class Catalog;
+class CatalogSet;
+class ClientContext;
+class DatabaseInstance;
+
+//! The DatabaseManager is a class that sits at the root of all attached databases
+class DatabaseManager {
+	friend class Catalog;
+
+public:
+	explicit DatabaseManager(DatabaseInstance &db);
+	~DatabaseManager();
+
+public:
+	static DatabaseManager &Get(DatabaseInstance &db);
+	static DatabaseManager &Get(ClientContext &db);
+	static DatabaseManager &Get(AttachedDatabase &db);
+
+	void InitializeSystemCatalog();
+	//! Get an attached database with the given name
+	AttachedDatabase *GetDatabase(ClientContext &context, const string &name);
+	//! Add a new attached database to the database manager
+	void AddDatabase(ClientContext &context, unique_ptr<AttachedDatabase> db);
+	void DetachDatabase(ClientContext &context, const string &name, bool if_exists);
+	//! Returns a reference to the system catalog
+	Catalog &GetSystemCatalog();
+	static const string &GetDefaultDatabase(ClientContext &context);
+
+	AttachedDatabase *GetDatabaseFromPath(ClientContext &context, const string &path);
+	vector<AttachedDatabase *> GetDatabases(ClientContext &context);
+
+	transaction_t GetNewQueryNumber() {
+		return current_query_number++;
+	}
+	transaction_t ActiveQueryNumber() const {
+		return current_query_number;
+	}
+
+private:
+	//! The system database is a special database that holds system entries (e.g. functions)
+	unique_ptr<AttachedDatabase> system;
+	//! The set of attached databases
+	unique_ptr<CatalogSet> databases;
+	//! The global catalog version, incremented whenever anything changes in the catalog
+	atomic<idx_t> catalog_version;
+	//! The current query number
+	atomic<transaction_t> current_query_number;
+	//! The current default database
+	string default_database;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/main/settings.hpp

@@ -262,6 +262,16 @@ struct LogQueryPathSetting {
 	static Value GetSetting(ClientContext &context);
 };
 
+struct ImmediateTransactionModeSetting {
+	static constexpr const char *Name = "immediate_transaction_mode";
+	static constexpr const char *Description =
+	    "Whether transactions should be started lazily when needed, or immediately when BEGIN TRANSACTION is called";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(ClientContext &context);
+};
+
 struct MaximumExpressionDepthSetting {
 	static constexpr const char *Name = "max_expression_depth";
 	static constexpr const char *Description =

package/src/duckdb/src/include/duckdb/main/valid_checker.hpp

@@ -14,14 +14,14 @@
 
 namespace duckdb {
 class DatabaseInstance;
-class Transaction;
+class MetaTransaction;
 
 class ValidChecker {
 public:
 	ValidChecker();
 
 	DUCKDB_API static ValidChecker &Get(DatabaseInstance &db);
-	DUCKDB_API static ValidChecker &Get(Transaction &transaction);
+	DUCKDB_API static ValidChecker &Get(MetaTransaction &transaction);
 
 	DUCKDB_API void Invalidate(string error);
 	DUCKDB_API bool IsInvalidated();

package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp

@@ -16,7 +16,7 @@ namespace duckdb {
 //! Represents a function call
 class FunctionExpression : public ParsedExpression {
 public:
-	DUCKDB_API FunctionExpression(string schema_name, const string &function_name,
+	DUCKDB_API FunctionExpression(string catalog_name, string schema_name, const string &function_name,
 	                              vector<unique_ptr<ParsedExpression>> children,
 	                              unique_ptr<ParsedExpression> filter = nullptr,
 	                              unique_ptr<OrderModifier> order_bys = nullptr, bool distinct = false,
@@ -26,6 +26,8 @@ public:
 	                              unique_ptr<OrderModifier> order_bys = nullptr, bool distinct = false,
 	                              bool is_operator = false, bool export_state = false);
 
+	//! Catalog of the function
+	string catalog;
 	//! Schema of the function
 	string schema;
 	//! Function name

package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp

@@ -29,8 +29,10 @@ enum class WindowBoundary : uint8_t {
 //! they inherit from them.
 class WindowExpression : public ParsedExpression {
 public:
-	WindowExpression(ExpressionType type, string schema_name, const string &function_name);
+	WindowExpression(ExpressionType type, string catalog_name, string schema_name, const string &function_name);
 
+	//! Catalog of the aggregate function
+	string catalog;
 	//! Schema of the aggregate function
 	string schema;
 	//! Name of the aggregate function

package/src/duckdb/src/include/duckdb/parser/parsed_data/alter_function_info.hpp

@@ -20,7 +20,7 @@ namespace duckdb {
 enum class AlterFunctionType : uint8_t { INVALID = 0, ADD_FUNCTION_OVERLOADS = 1 };
 
 struct AlterFunctionInfo : public AlterInfo {
-	AlterFunctionInfo(AlterFunctionType type, string schema, string name, bool if_exists);
+	AlterFunctionInfo(AlterFunctionType type, AlterEntryData data);
 	virtual ~AlterFunctionInfo() override;
 
 	AlterFunctionType alter_function_type;
@@ -35,7 +35,7 @@ public:
 // AddFunctionOverloadInfo
 //===--------------------------------------------------------------------===//
 struct AddFunctionOverloadInfo : public AlterFunctionInfo {
-	AddFunctionOverloadInfo(string schema, string name, bool if_exists, ScalarFunctionSet new_overloads);
+	AddFunctionOverloadInfo(AlterEntryData data, ScalarFunctionSet new_overloads);
 	~AddFunctionOverloadInfo() override;
 
 	ScalarFunctionSet new_overloads;

package/src/duckdb/src/include/duckdb/parser/parsed_data/alter_info.hpp

@@ -23,13 +23,28 @@ enum class AlterType : uint8_t {
 	ALTER_FUNCTION = 5
 };
 
+struct AlterEntryData {
+	AlterEntryData() {
+	}
+	AlterEntryData(string catalog_p, string schema_p, string name_p, bool if_exists)
+	    : catalog(move(catalog_p)), schema(move(schema_p)), name(move(name_p)), if_exists(if_exists) {
+	}
+
+	string catalog;
+	string schema;
+	string name;
+	bool if_exists;
+};
+
 struct AlterInfo : public ParseInfo {
-	AlterInfo(AlterType type, string schema, string name, bool if_exists);
+	AlterInfo(AlterType type, string catalog, string schema, string name, bool if_exists);
 	virtual ~AlterInfo() override;
 
 	AlterType type;
 	//! if exists
 	bool if_exists;
+	//! Catalog name to alter
+	string catalog;
 	//! Schema name to alter
 	string schema;
 	//! Entry name to alter
@@ -43,6 +58,8 @@ public:
 	void Serialize(Serializer &serializer) const;
 	virtual void Serialize(FieldWriter &writer) const = 0;
 	static unique_ptr<AlterInfo> Deserialize(Deserializer &source);
+
+	AlterEntryData GetAlterEntryData() const;
 };
 
 } // namespace duckdb