duckdb 0.9.3-dev0.0 → 0.9.3-dev12.0

package/src/duckdb/src/include/duckdb/main/pending_query_result.hpp

@@ -26,8 +26,9 @@ public:
 public:
 	DUCKDB_API PendingQueryResult(shared_ptr<ClientContext> context, PreparedStatementData &statement,
 	                              vector<LogicalType> types, bool allow_stream_result);
-	DUCKDB_API explicit PendingQueryResult(PreservedError error_message);
-	DUCKDB_API ~PendingQueryResult();
+	DUCKDB_API explicit PendingQueryResult(ErrorData error_message);
+	DUCKDB_API ~PendingQueryResult() override;
+	DUCKDB_API bool AllowStreamResult() const;
 
 public:
 	//! Executes a single task within the query, returning whether or not the query is ready.
@@ -38,6 +39,7 @@ public:
 	//!	    but tasks may become available in the future.
 	//! The error message can be obtained by calling GetError() on the PendingQueryResult.
 	DUCKDB_API PendingExecutionResult ExecuteTask();
+	DUCKDB_API PendingExecutionResult CheckPulse();
 
 	//! Returns the result of the query as an actual query result.
 	//! This returns (mostly) instantly if ExecuteTask has been called until RESULT_READY was returned.
@@ -47,6 +49,7 @@ public:
 
 	//! Function to determine whether execution is considered finished
 	DUCKDB_API static bool IsFinished(PendingExecutionResult result);
+	DUCKDB_API static bool IsFinishedOrBlocked(PendingExecutionResult result);
 
 private:
 	shared_ptr<ClientContext> context;

package/src/duckdb/src/include/duckdb/main/prepared_statement.hpp

@@ -11,7 +11,7 @@
 #include "duckdb/common/winapi.hpp"
 #include "duckdb/main/materialized_query_result.hpp"
 #include "duckdb/main/pending_query_result.hpp"
-#include "duckdb/common/preserved_error.hpp"
+#include "duckdb/common/error_data.hpp"
 #include "duckdb/common/case_insensitive_map.hpp"
 
 namespace duckdb {
@@ -25,7 +25,7 @@ public:
 	DUCKDB_API PreparedStatement(shared_ptr<ClientContext> context, shared_ptr<PreparedStatementData> data,
 	                             string query, idx_t n_param, case_insensitive_map_t<idx_t> named_param_map);
 	//! Create a prepared statement that was not successfully prepared
-	DUCKDB_API explicit PreparedStatement(PreservedError error);
+	DUCKDB_API explicit PreparedStatement(ErrorData error);
 
 	DUCKDB_API ~PreparedStatement();
 
@@ -39,7 +39,7 @@ public:
 	//! Whether or not the statement was successfully prepared
 	bool success;
 	//! The error message (if success = false)
-	PreservedError error;
+	ErrorData error;
 	//! The amount of bound parameters
 	idx_t n_param;
 	//! The (optional) named parameters
@@ -49,7 +49,7 @@ public:
 	//! Returns the stored error message
 	DUCKDB_API const string &GetError();
 	//! Returns the stored error object
-	DUCKDB_API PreservedError &GetErrorObject();
+	DUCKDB_API ErrorData &GetErrorObject();
 	//! Returns whether or not an error occurred
 	DUCKDB_API bool HasError() const;
 	//! Returns the number of columns in the result

package/src/duckdb/src/include/duckdb/main/prepared_statement_data.hpp

@@ -46,6 +46,8 @@ public:
 	idx_t catalog_version;
 	//! The map of parameter index to the actual value entry
 	bound_parameter_map_t value_map;
+	//! Whether we are creating a streaming result or not
+	bool is_streaming = false;
 
 public:
 	void CheckParameterCount(idx_t parameter_count);

package/src/duckdb/src/include/duckdb/main/query_profiler.hpp

@@ -230,36 +230,4 @@ private:
 	bool OperatorRequiresProfiling(PhysicalOperatorType op_type);
 };
 
-//! The QueryProfilerHistory can be used to access the profiler of previous queries
-class QueryProfilerHistory {
-private:
-	static constexpr uint64_t DEFAULT_SIZE = 20;
-
-	//! Previous Query profilers
-	deque<pair<transaction_t, shared_ptr<QueryProfiler>>> prev_profilers;
-	//! Previous Query profilers size
-	uint64_t prev_profilers_size = DEFAULT_SIZE;
-
-public:
-	deque<pair<transaction_t, shared_ptr<QueryProfiler>>> &GetPrevProfilers() {
-		return prev_profilers;
-	}
-	QueryProfilerHistory() {
-	}
-
-	void SetPrevProfilersSize(uint64_t prevProfilersSize) {
-		prev_profilers_size = prevProfilersSize;
-	}
-	uint64_t GetPrevProfilersSize() const {
-		return prev_profilers_size;
-	}
-
-public:
-	void SetProfilerHistorySize(uint64_t size) {
-		this->prev_profilers_size = size;
-	}
-	void ResetProfilerHistorySize() {
-		this->prev_profilers_size = DEFAULT_SIZE;
-	}
-};
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/query_result.hpp

@@ -11,7 +11,7 @@
 #include "duckdb/common/enums/statement_type.hpp"
 #include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/common/winapi.hpp"
-#include "duckdb/common/preserved_error.hpp"
+#include "duckdb/common/error_data.hpp"
 #include "duckdb/main/client_properties.hpp"
 
 namespace duckdb {
@@ -25,7 +25,7 @@ public:
 	DUCKDB_API BaseQueryResult(QueryResultType type, StatementType statement_type, StatementProperties properties,
 	                           vector<LogicalType> types, vector<string> names);
 	//! Creates an unsuccessful query result with error condition
-	DUCKDB_API BaseQueryResult(QueryResultType type, PreservedError error);
+	DUCKDB_API BaseQueryResult(QueryResultType type, ErrorData error);
 	DUCKDB_API virtual ~BaseQueryResult();
 
 	//! The type of the result (MATERIALIZED or STREAMING)
@@ -41,18 +41,18 @@ public:
 
 public:
 	[[noreturn]] DUCKDB_API void ThrowError(const string &prepended_message = "") const;
-	DUCKDB_API void SetError(PreservedError error);
+	DUCKDB_API void SetError(ErrorData error);
 	DUCKDB_API bool HasError() const;
 	DUCKDB_API const ExceptionType &GetErrorType() const;
 	DUCKDB_API const std::string &GetError();
-	DUCKDB_API PreservedError &GetErrorObject();
+	DUCKDB_API ErrorData &GetErrorObject();
 	DUCKDB_API idx_t ColumnCount();
 
 protected:
 	//! Whether or not execution was successful
 	bool success;
 	//! The error (in case execution was not successful)
-	PreservedError error;
+	ErrorData error;
 };
 
 //! The QueryResult object holds the result of a query. It can either be a MaterializedQueryResult, in which case the
@@ -64,7 +64,7 @@ public:
 	DUCKDB_API QueryResult(QueryResultType type, StatementType statement_type, StatementProperties properties,
 	                       vector<LogicalType> types, vector<string> names, ClientProperties client_properties);
 	//! Creates an unsuccessful query result with error condition
-	DUCKDB_API QueryResult(QueryResultType type, PreservedError error);
+	DUCKDB_API QueryResult(QueryResultType type, ErrorData error);
 	DUCKDB_API virtual ~QueryResult() override;
 
 	//! Properties from the client context
@@ -89,6 +89,10 @@ public:
 		return reinterpret_cast<const TARGET &>(*this);
 	}
 
+public:
+	//! Deduplicate column names for interop with external libraries
+	static void DeduplicateColumns(vector<string> &names);
+
 public:
 	//! Returns the name of the column for the given index
 	DUCKDB_API const string &ColumnName(idx_t index) const;
@@ -108,18 +112,15 @@ public:
 	//! Fetch() until both results are exhausted. The data in the results will be lost.
 	DUCKDB_API bool Equals(QueryResult &other);
 
-	bool TryFetch(unique_ptr<DataChunk> &result, PreservedError &error) {
+	bool TryFetch(unique_ptr<DataChunk> &result, ErrorData &error) {
 		try {
 			result = Fetch();
 			return success;
-		} catch (const Exception &ex) {
-			error = PreservedError(ex);
-			return false;
 		} catch (std::exception &ex) {
-			error = PreservedError(ex);
+			error = ErrorData(ex);
 			return false;
 		} catch (...) {
-			error = PreservedError("Unknown error in Fetch");
+			error = ErrorData("Unknown error in Fetch");
 			return false;
 		}
 	}

package/src/duckdb/src/include/duckdb/main/relation/read_csv_relation.hpp

@@ -8,7 +8,7 @@
 
 #pragma once
 
-#include "duckdb/execution/operator/scan/csv/csv_reader_options.hpp"
+#include "duckdb/execution/operator/csv_scanner/options/csv_reader_options.hpp"
 #include "duckdb/main/relation/table_function_relation.hpp"
 #include "duckdb/common/shared_ptr.hpp"
 #include "duckdb/common/case_insensitive_map.hpp"
@@ -17,13 +17,13 @@ namespace duckdb {
 
 class ReadCSVRelation : public TableFunctionRelation {
 public:
-	ReadCSVRelation(const shared_ptr<ClientContext> &context, const string &csv_file, vector<ColumnDefinition> columns,
-	                string alias = string());
-	ReadCSVRelation(const shared_ptr<ClientContext> &context, const string &csv_file, named_parameter_map_t &&options,
-	                string alias = string());
+	ReadCSVRelation(const shared_ptr<ClientContext> &context, const vector<string> &csv_files,
+	                named_parameter_map_t &&options, string alias = string());
 
 	string alias;
-	bool auto_detect;
+
+protected:
+	void InitializeAlias(const vector<string> &input);
 
 public:
 	string GetAlias() override;

package/src/duckdb/src/include/duckdb/main/relation/setop_relation.hpp

@@ -15,12 +15,14 @@ namespace duckdb {
 
 class SetOpRelation : public Relation {
 public:
-	SetOpRelation(shared_ptr<Relation> left, shared_ptr<Relation> right, SetOperationType setop_type);
+	SetOpRelation(shared_ptr<Relation> left, shared_ptr<Relation> right, SetOperationType setop_type,
+	              bool setop_all = false);
 
 	shared_ptr<Relation> left;
 	shared_ptr<Relation> right;
 	SetOperationType setop_type;
 	vector<ColumnDefinition> columns;
+	bool setop_all;
 
 public:
 	unique_ptr<QueryNode> GetQueryNode() override;

package/src/duckdb/src/include/duckdb/main/secret/secret.hpp

@@ -0,0 +1,206 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/main/secret/secret.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/named_parameter_map.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+
+namespace duckdb {
+class BaseSecret;
+
+//! Whether a secret is persistent or temporary
+enum class SecretPersistType : uint8_t { DEFAULT, TEMPORARY, PERSISTENT };
+
+//! Input passed to a CreateSecretFunction
+struct CreateSecretInput {
+	//! type
+	string type;
+	//! mode
+	string provider;
+	//! should the secret be persisted?
+	string storage_type;
+	//! (optional) alias provided by user
+	string name;
+	//! (optional) scope provided by user
+	vector<string> scope;
+	//! (optional) named parameter map, each create secret function has defined it's own set of these
+	case_insensitive_map_t<Value> options;
+};
+
+typedef unique_ptr<BaseSecret> (*secret_deserializer_t)(Deserializer &deserializer, BaseSecret base_secret);
+typedef unique_ptr<BaseSecret> (*create_secret_function_t)(ClientContext &context, CreateSecretInput &input);
+
+//! A CreateSecretFunction is a function adds a provider for a secret type.
+class CreateSecretFunction {
+public:
+	string secret_type;
+	string provider;
+	create_secret_function_t function;
+	named_parameter_type_map_t named_parameters;
+};
+
+//! CreateSecretFunctionsSet contains multiple functions of a single type, identified by the provider. The provider
+//! should be seen as the method of secret creation. (e.g. user-provided config, env variables, auto-detect)
+class CreateSecretFunctionSet {
+public:
+	CreateSecretFunctionSet(string &name) : name(name) {};
+	bool ProviderExists(const string &provider_name);
+	void AddFunction(CreateSecretFunction &function, OnCreateConflict on_conflict);
+	CreateSecretFunction &GetFunction(const string &provider);
+
+protected:
+	//! Create Secret Function type name
+	string name;
+	//! Maps of provider -> function
+	case_insensitive_map_t<CreateSecretFunction> functions;
+};
+
+//! Determines whether the secrets are allowed to be shown
+enum class SecretDisplayType : uint8_t { REDACTED, UNREDACTED };
+
+//! Secret types contain the base settings of a secret
+struct SecretType {
+	//! Unique name identifying the secret type
+	string name;
+	//! The deserialization function for the type
+	secret_deserializer_t deserializer;
+	//! Provider to use when non is specified
+	string default_provider;
+};
+
+//! Base class from which BaseSecret classes can be made.
+class BaseSecret {
+	friend class SecretManager;
+
+public:
+	BaseSecret(const vector<string> &prefix_paths, const string &type, const string &provider, const string &name)
+	    : prefix_paths(prefix_paths), type(type), provider(provider), name(name), serializable(false) {
+		D_ASSERT(!type.empty());
+	}
+	BaseSecret(const BaseSecret &other)
+	    : prefix_paths(other.prefix_paths), type(other.type), provider(other.provider), name(other.name),
+	      serializable(other.serializable) {
+		D_ASSERT(!type.empty());
+	}
+	virtual ~BaseSecret() = default;
+
+	//! The score of how well this secret's scope matches the path (by default: the length of the longest matching
+	//! prefix)
+	virtual int64_t MatchScore(const string &path) const;
+	//! Prints the secret as a string
+	virtual string ToString(SecretDisplayType mode = SecretDisplayType::REDACTED) const;
+	//! Serialize this secret
+	virtual void Serialize(Serializer &serializer) const;
+
+	virtual unique_ptr<const BaseSecret> Clone() const {
+		D_ASSERT(typeid(BaseSecret) == typeid(*this));
+		return make_uniq<BaseSecret>(*this);
+	}
+
+	//! Getters
+	const vector<string> &GetScope() const {
+		return prefix_paths;
+	}
+	const string &GetType() const {
+		return type;
+	}
+	const string &GetProvider() const {
+		return provider;
+	}
+	const string &GetName() const {
+		return name;
+	}
+	bool IsSerializable() const {
+		return serializable;
+	}
+
+protected:
+	//! Helper function to serialize the base BaseSecret class variables
+	virtual void SerializeBaseSecret(Serializer &serializer) const final;
+
+	//! prefixes to which the secret applies
+	vector<string> prefix_paths;
+
+	//! Type of secret
+	string type;
+	//! Provider of the secret
+	string provider;
+	//! Name of the secret
+	string name;
+	//! Whether the secret can be serialized/deserialized
+	bool serializable;
+};
+
+//! The KeyValueSecret is a class that implements a Secret as a set of key -> values. This class can be used
+//! for most use-cases of secrets as secrets generally tend to fit in a key value map.
+class KeyValueSecret : public BaseSecret {
+public:
+	KeyValueSecret(const vector<string> &prefix_paths, const string &type, const string &provider, const string &name)
+	    : BaseSecret(prefix_paths, type, provider, name) {
+		D_ASSERT(!type.empty());
+		serializable = true;
+	}
+	KeyValueSecret(BaseSecret &secret)
+	    : BaseSecret(secret.GetScope(), secret.GetType(), secret.GetProvider(), secret.GetName()) {
+		serializable = true;
+	};
+	KeyValueSecret(const KeyValueSecret &secret)
+	    : BaseSecret(secret.GetScope(), secret.GetType(), secret.GetProvider(), secret.GetName()) {
+		secret_map = secret.secret_map;
+		redact_keys = secret.redact_keys;
+		serializable = true;
+	};
+	KeyValueSecret(KeyValueSecret &&secret)
+	    : BaseSecret(secret.GetScope(), secret.GetType(), secret.GetProvider(), secret.GetName()) {
+		secret_map = std::move(secret.secret_map);
+		redact_keys = std::move(secret.redact_keys);
+		serializable = true;
+	};
+
+	//! Print the secret as a key value map in the format 'key1=value;key2=value2'
+	virtual string ToString(SecretDisplayType mode = SecretDisplayType::REDACTED) const override;
+	void Serialize(Serializer &serializer) const override;
+
+	//! Tries to get the value at key <key>, depending on error_on_missing will throw or return Value()
+	Value TryGetValue(const string &key, bool error_on_missing = false) const;
+
+	// FIXME: use serialization scripts
+	template <class TYPE>
+	static unique_ptr<BaseSecret> Deserialize(Deserializer &deserializer, BaseSecret base_secret) {
+		auto result = make_uniq<TYPE>(base_secret);
+		Value secret_map_value;
+		deserializer.ReadProperty(201, "secret_map", secret_map_value);
+
+		for (const auto &entry : ListValue::GetChildren(secret_map_value)) {
+			auto kv_struct = StructValue::GetChildren(entry);
+			result->secret_map[kv_struct[0].ToString()] = kv_struct[1].ToString();
+		}
+
+		Value redact_set_value;
+		deserializer.ReadProperty(202, "redact_keys", redact_set_value);
+		for (const auto &entry : ListValue::GetChildren(redact_set_value)) {
+			result->redact_keys.insert(entry.ToString());
+		}
+
+		return duckdb::unique_ptr_cast<TYPE, BaseSecret>(std::move(result));
+	}
+
+	unique_ptr<const BaseSecret> Clone() const override {
+		return make_uniq<KeyValueSecret>(*this);
+	}
+
+	//! the map of key -> values that make up the secret
+	case_insensitive_tree_t<Value> secret_map;
+	//! keys that are sensitive and should be redacted
+	case_insensitive_set_t redact_keys;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/main/secret/secret_manager.hpp

@@ -0,0 +1,211 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/main/secret/duck_secret_manager.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/catalog/default/default_generator.hpp"
+#include "duckdb/common/common.hpp"
+#include "duckdb/main/secret/secret.hpp"
+#include "duckdb/main/secret/secret_manager.hpp"
+#include "duckdb/main/secret/secret_storage.hpp"
+#include "duckdb/parser/parsed_data/create_secret_info.hpp"
+
+namespace duckdb {
+class SecretManager;
+struct DBConfig;
+class SchemaCatalogEntry;
+
+//! Return value of a Secret Lookup
+struct SecretMatch {
+public:
+	SecretMatch() : secret_entry(nullptr), score(NumericLimits<int64_t>::Minimum()) {
+	}
+
+	SecretMatch(const SecretMatch &other)
+	    : secret_entry((other.secret_entry != nullptr) ? make_uniq<SecretEntry>(*other.secret_entry) : nullptr),
+	      score(other.score) {
+	}
+
+	SecretMatch(SecretEntry &secret_entry, int64_t score)
+	    : secret_entry(make_uniq<SecretEntry>(secret_entry)), score(score) {
+	}
+
+	SecretMatch &operator=(const SecretMatch &other) {
+		this->secret_entry = (other.secret_entry != nullptr) ? make_uniq<SecretEntry>(*other.secret_entry) : nullptr;
+		this->score = other.score;
+		return *this;
+	};
+
+	//! Get the secret
+	const BaseSecret &GetSecret() const;
+
+	bool HasMatch() {
+		return secret_entry != nullptr;
+	}
+
+	unique_ptr<SecretEntry> secret_entry;
+	int64_t score;
+};
+
+//! A Secret Entry in the secret manager
+struct SecretEntry {
+public:
+	SecretEntry(unique_ptr<const BaseSecret> secret) : secret(secret != nullptr ? secret->Clone() : nullptr) {};
+
+	SecretEntry(const SecretEntry &other)
+	    : persist_type(other.persist_type), storage_mode(other.storage_mode),
+	      secret((other.secret != nullptr) ? other.secret->Clone() : nullptr) {
+	}
+
+	//! Whether the secret is persistent
+	SecretPersistType persist_type;
+	//! The storage backend of the secret
+	string storage_mode;
+	//! The secret pointer
+	unique_ptr<const BaseSecret> secret;
+};
+
+struct SecretManagerConfig {
+	static constexpr const bool DEFAULT_ALLOW_PERSISTENT_SECRETS = true;
+	//! The default persistence type for secrets
+	SecretPersistType default_persist_type = SecretPersistType::TEMPORARY;
+	//! Secret Path can be changed by until the secret manager is initialized, after that it will be set automatically
+	string secret_path = "";
+	//! The default secret path is determined on startup and can be used to reset the secret path
+	string default_secret_path = "";
+	//! The storage type for persistent secrets when none is specified;
+	string default_persistent_storage = "";
+	//! Persistent secrets are enabled by default
+	bool allow_persistent_secrets = DEFAULT_ALLOW_PERSISTENT_SECRETS;
+};
+
+//! The Secret Manager for DuckDB. Can handle both temporary and persistent secrets
+class SecretManager {
+	friend struct SecretEntry;
+
+public:
+	explicit SecretManager() = default;
+	virtual ~SecretManager() = default;
+
+	//! The default storage backends
+	static constexpr const char *TEMPORARY_STORAGE_NAME = "memory";
+	static constexpr const char *LOCAL_FILE_STORAGE_NAME = "local_file";
+
+	//! Static Helper Functions
+	DUCKDB_API static SecretManager &Get(ClientContext &context);
+	DUCKDB_API static SecretManager &Get(DatabaseInstance &db);
+
+	// Initialize the secret manager with the DB instance
+	DUCKDB_API void Initialize(DatabaseInstance &db);
+	//! Load a secret storage
+	DUCKDB_API void LoadSecretStorage(unique_ptr<SecretStorage> storage);
+
+	//! Deserialize a secret by automatically selecting the correct deserializer
+	DUCKDB_API unique_ptr<BaseSecret> DeserializeSecret(Deserializer &deserializer);
+	//! Register a new SecretType
+	DUCKDB_API void RegisterSecretType(SecretType &type);
+	//! Lookup a SecretType
+	DUCKDB_API SecretType LookupType(const string &type);
+	//! Register a Secret Function i.e. a secret provider for a secret type
+	DUCKDB_API void RegisterSecretFunction(CreateSecretFunction function, OnCreateConflict on_conflict);
+	//! Register a secret by providing a secret manually
+	DUCKDB_API unique_ptr<SecretEntry> RegisterSecret(CatalogTransaction transaction,
+	                                                  unique_ptr<const BaseSecret> secret, OnCreateConflict on_conflict,
+	                                                  SecretPersistType persist_type, const string &storage = "");
+	//! Create a secret from a CreateSecretInfo
+	DUCKDB_API unique_ptr<SecretEntry> CreateSecret(ClientContext &context, const CreateSecretInfo &info);
+	//! The Bind for create secret is done by the secret manager
+	DUCKDB_API BoundStatement BindCreateSecret(CatalogTransaction transaction, CreateSecretInfo &info);
+	//! Lookup the best matching secret by matching the secret scopes to the path
+	DUCKDB_API SecretMatch LookupSecret(CatalogTransaction transaction, const string &path, const string &type);
+	//! Get a secret by name, optionally from a specific storage
+	DUCKDB_API unique_ptr<SecretEntry> GetSecretByName(CatalogTransaction transaction, const string &name,
+	                                                   const string &storage = "");
+	//! Delete a secret by name, optionally by providing the storage to drop from
+	DUCKDB_API void DropSecretByName(CatalogTransaction transaction, const string &name,
+	                                 OnEntryNotFound on_entry_not_found,
+	                                 SecretPersistType persist_type = SecretPersistType::DEFAULT,
+	                                 const string &storage = "");
+	//! List all secrets from all secret storages
+	DUCKDB_API vector<SecretEntry> AllSecrets(CatalogTransaction transaction);
+
+	//! Secret Manager settings
+	DUCKDB_API virtual void SetEnablePersistentSecrets(bool enabled);
+	DUCKDB_API virtual void ResetEnablePersistentSecrets();
+	DUCKDB_API virtual bool PersistentSecretsEnabled();
+
+	DUCKDB_API virtual void SetDefaultStorage(const string &storage);
+	DUCKDB_API virtual void ResetDefaultStorage();
+	DUCKDB_API virtual string DefaultStorage();
+
+	DUCKDB_API virtual void SetPersistentSecretPath(const string &path);
+	DUCKDB_API virtual void ResetPersistentSecretPath();
+	DUCKDB_API virtual string PersistentSecretPath();
+
+	//! Utility functions
+	DUCKDB_API void DropSecretByName(ClientContext &context, const string &name, OnEntryNotFound on_entry_not_found,
+	                                 SecretPersistType persist_type = SecretPersistType::DEFAULT,
+	                                 const string &storage = "");
+
+private:
+	//! Lookup a SecretType
+	SecretType LookupTypeInternal(const string &type);
+	//! Lookup a CreateSecretFunction
+	optional_ptr<CreateSecretFunction> LookupFunctionInternal(const string &type, const string &provider);
+	//! Register a new Secret
+	unique_ptr<SecretEntry> RegisterSecretInternal(CatalogTransaction transaction, unique_ptr<const BaseSecret> secret,
+	                                               OnCreateConflict on_conflict, SecretPersistType persist_type,
+	                                               const string &storage = "");
+	//! Initialize the secret catalog_set and persistent secrets (lazily)
+	void InitializeSecrets(CatalogTransaction transaction);
+	//! Load a secret storage
+	void LoadSecretStorageInternal(unique_ptr<SecretStorage> storage);
+
+	//! Autoload extension for specific secret type
+	void AutoloadExtensionForType(const string &type);
+	//! Autoload extension for specific secret function
+	void AutoloadExtensionForFunction(const string &type, const string &provider);
+
+	//! Thread-safe accessors for secret_storages
+	vector<reference<SecretStorage>> GetSecretStorages();
+	optional_ptr<SecretStorage> GetSecretStorage(const string &name);
+
+	//! Throw an exception if the secret manager is initialized
+	void ThrowOnSettingChangeIfInitialized();
+
+	//! Lock for types, functions, settings and storages
+	mutex manager_lock;
+	//! Secret functions;
+	case_insensitive_map_t<CreateSecretFunctionSet> secret_functions;
+	//! Secret types;
+	case_insensitive_map_t<SecretType> secret_types;
+	//! Map of all registered SecretStorages
+	case_insensitive_map_t<unique_ptr<SecretStorage>> secret_storages;
+	//! While false, secret manager settings can still be changed
+	atomic<bool> initialized {false};
+	//! Configuration for secret manager
+	SecretManagerConfig config;
+	//! Pointer to current db instance
+	optional_ptr<DatabaseInstance> db;
+};
+
+//! The DefaultGenerator for persistent secrets. This is used to store lazy loaded secrets in the catalog
+class DefaultSecretGenerator : public DefaultGenerator {
+public:
+	DefaultSecretGenerator(Catalog &catalog, SecretManager &secret_manager, case_insensitive_set_t &persistent_secrets);
+
+public:
+	unique_ptr<CatalogEntry> CreateDefaultEntry(ClientContext &context, const string &entry_name) override;
+	vector<string> GetDefaultEntries() override;
+
+protected:
+	SecretManager &secret_manager;
+	case_insensitive_set_t persistent_secrets;
+};
+
+} // namespace duckdb