duckdb 0.9.3-dev2.0 → 0.10.0

package/src/duckdb/src/main/client_data.cpp

@@ -35,7 +35,6 @@ private:
 ClientData::ClientData(ClientContext &context) : catalog_search_path(make_uniq<CatalogSearchPath>(context)) {
 	auto &db = DatabaseInstance::GetDatabase(context);
 	profiler = make_shared<QueryProfiler>(context);
-	query_profiler_history = make_uniq<QueryProfilerHistory>();
 	temporary_objects = make_shared<AttachedDatabase>(db, AttachedDatabaseType::TEMP_DATABASE);
 	temporary_objects->oid = DatabaseManager::Get(db).ModifyCatalog();
 	random_engine = make_uniq<RandomEngine>();

package/src/duckdb/src/main/client_verify.cpp

@@ -1,4 +1,4 @@
-#include "duckdb/common/preserved_error.hpp"
+#include "duckdb/common/error_data.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/parser/statement/explain_statement.hpp"
 #include "duckdb/verification/statement_verifier.hpp"
@@ -21,11 +21,16 @@ static void ThrowIfExceptionIsInternal(StatementVerifier &verifier) {
 	}
 }
 
-PreservedError ClientContext::VerifyQuery(ClientContextLock &lock, const string &query,
-                                          unique_ptr<SQLStatement> statement) {
+ErrorData ClientContext::VerifyQuery(ClientContextLock &lock, const string &query, unique_ptr<SQLStatement> statement) {
 	D_ASSERT(statement->type == StatementType::SELECT_STATEMENT);
 	// Aggressive query verification
 
+#ifdef DUCKDB_RUN_SLOW_VERIFIERS
+	bool run_slow_verifiers = true;
+#else
+	bool run_slow_verifiers = false;
+#endif
+
 	// The purpose of this function is to test correctness of otherwise hard to test features:
 	// Copy() of statements and expressions
 	// Serialize()/Deserialize() of expressions
@@ -37,16 +42,28 @@ PreservedError ClientContext::VerifyQuery(ClientContextLock &lock, const string
 	const auto &stmt = *statement;
 	vector<unique_ptr<StatementVerifier>> statement_verifiers;
 	unique_ptr<StatementVerifier> prepared_statement_verifier;
+
+	// Base Statement verifiers: these are the verifiers we enable for regular builds
 	if (config.query_verification_enabled) {
 		statement_verifiers.emplace_back(StatementVerifier::Create(VerificationType::COPIED, stmt));
 		statement_verifiers.emplace_back(StatementVerifier::Create(VerificationType::DESERIALIZED, stmt));
 		statement_verifiers.emplace_back(StatementVerifier::Create(VerificationType::UNOPTIMIZED, stmt));
 		prepared_statement_verifier = StatementVerifier::Create(VerificationType::PREPARED, stmt);
+	}
+
+	// This verifier is enabled explicitly OR by enabling run_slow_verifiers
+	if (config.verify_fetch_row || (run_slow_verifiers && config.query_verification_enabled)) {
+		statement_verifiers.emplace_back(StatementVerifier::Create(VerificationType::FETCH_ROW_AS_SCAN, stmt));
+	}
+
+	// For the DEBUG_ASYNC build we enable this extra verifier
 #ifdef DUCKDB_DEBUG_ASYNC_SINK_SOURCE
-		// This verification is quite slow, so we only run it for the async sink/source debug mode
+	if (config.query_verification_enabled) {
 		statement_verifiers.emplace_back(StatementVerifier::Create(VerificationType::NO_OPERATOR_CACHING, stmt));
-#endif
 	}
+#endif
+
+	// Verify external always needs to be explicitly enabled and is never part of default verifier set
 	if (config.verify_external) {
 		statement_verifiers.emplace_back(StatementVerifier::Create(VerificationType::EXTERNAL, stmt));
 	}
@@ -115,8 +132,9 @@ PreservedError ClientContext::VerifyQuery(ClientContextLock &lock, const string
 		try {
 			RunStatementInternal(lock, explain_q, std::move(explain_stmt), false, false);
 		} catch (std::exception &ex) { // LCOV_EXCL_START
+			ErrorData error(ex);
 			interrupted = false;
-			return PreservedError("EXPLAIN failed but query did not (" + string(ex.what()) + ")");
+			return ErrorData("EXPLAIN failed but query did not (" + error.RawMessage() + ")");
 		} // LCOV_EXCL_STOP
 
 #ifdef DUCKDB_VERIFY_BOX_RENDERER
@@ -142,11 +160,11 @@ PreservedError ClientContext::VerifyQuery(ClientContextLock &lock, const string
 	for (auto &verifier : statement_verifiers) {
 		auto result = original->CompareResults(*verifier);
 		if (!result.empty()) {
-			return PreservedError(result);
+			return ErrorData(result);
 		}
 	}
 
-	return PreservedError();
+	return ErrorData();
 }
 
 } // namespace duckdb

package/src/duckdb/src/main/config.cpp

@@ -54,6 +54,7 @@ bool DBConfigOptions::debug_print_bindings = false;
 	{ nullptr, nullptr, LogicalTypeId::INVALID, nullptr, nullptr, nullptr, nullptr, nullptr }
 
 static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting),
+                                                 DUCKDB_GLOBAL(AllowPersistentSecrets),
                                                  DUCKDB_GLOBAL(CheckpointThresholdSetting),
                                                  DUCKDB_GLOBAL(DebugCheckpointAbort),
                                                  DUCKDB_LOCAL(DebugForceExternal),
@@ -69,8 +70,8 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_GLOBAL(EnableExternalAccessSetting),
                                                  DUCKDB_GLOBAL(EnableFSSTVectors),
                                                  DUCKDB_GLOBAL(AllowUnsignedExtensionsSetting),
-                                                 DUCKDB_LOCAL(CustomExtensionRepository),
-                                                 DUCKDB_LOCAL(AutoloadExtensionRepository),
+                                                 DUCKDB_GLOBAL(CustomExtensionRepository),
+                                                 DUCKDB_GLOBAL(AutoloadExtensionRepository),
                                                  DUCKDB_GLOBAL(AutoinstallKnownExtensions),
                                                  DUCKDB_GLOBAL(AutoloadKnownExtensions),
                                                  DUCKDB_GLOBAL(EnableObjectCacheSetting),
@@ -78,6 +79,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_LOCAL(EnableProfilingSetting),
                                                  DUCKDB_LOCAL(EnableProgressBarSetting),
                                                  DUCKDB_LOCAL(EnableProgressBarPrintSetting),
+                                                 DUCKDB_LOCAL(ErrorsAsJsonSetting),
                                                  DUCKDB_LOCAL(ExplainOutputSetting),
                                                  DUCKDB_GLOBAL(ExtensionDirectorySetting),
                                                  DUCKDB_GLOBAL(ExternalThreadsSetting),
@@ -91,6 +93,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_LOCAL(IntegerDivisionSetting),
                                                  DUCKDB_LOCAL(MaximumExpressionDepthSetting),
                                                  DUCKDB_GLOBAL(MaximumMemorySetting),
+                                                 DUCKDB_GLOBAL(OldImplicitCasting),
                                                  DUCKDB_GLOBAL_ALIAS("memory_limit", MaximumMemorySetting),
                                                  DUCKDB_GLOBAL_ALIAS("null_order", DefaultNullOrderSetting),
                                                  DUCKDB_LOCAL(OrderedAggregateThreshold),
@@ -100,13 +103,14 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_LOCAL(PivotLimitSetting),
                                                  DUCKDB_LOCAL(PreserveIdentifierCase),
                                                  DUCKDB_GLOBAL(PreserveInsertionOrder),
-                                                 DUCKDB_LOCAL(ProfilerHistorySize),
                                                  DUCKDB_LOCAL(ProfileOutputSetting),
                                                  DUCKDB_LOCAL(ProfilingModeSetting),
                                                  DUCKDB_LOCAL_ALIAS("profiling_output", ProfileOutputSetting),
                                                  DUCKDB_LOCAL(ProgressBarTimeSetting),
                                                  DUCKDB_LOCAL(SchemaSetting),
                                                  DUCKDB_LOCAL(SearchPathSetting),
+                                                 DUCKDB_GLOBAL(SecretDirectorySetting),
+                                                 DUCKDB_GLOBAL(DefaultSecretStorage),
                                                  DUCKDB_GLOBAL(TempDirectorySetting),
                                                  DUCKDB_GLOBAL(ThreadsSetting),
                                                  DUCKDB_GLOBAL(UsernameSetting),
@@ -183,6 +187,14 @@ void DBConfig::SetOptionByName(const string &name, const Value &value) {
 	}
 }
 
+void DBConfig::SetOptionsByName(const case_insensitive_map_t<Value> &values) {
+	for (auto &kv : values) {
+		auto &name = kv.first;
+		auto &value = kv.second;
+		SetOptionByName(name, value);
+	}
+}
+
 void DBConfig::SetOption(DatabaseInstance *db, const ConfigurationOption &option, const Value &value) {
 	lock_guard<mutex> l(config_lock);
 	if (!option.set_global) {
@@ -235,6 +247,10 @@ CastFunctionSet &DBConfig::GetCastFunctions() {
 	return *cast_functions;
 }
 
+IndexTypeSet &DBConfig::GetIndexTypes() {
+	return *index_types;
+}
+
 void DBConfig::SetDefaultMaxMemory() {
 	auto memory = FileSystem::GetAvailableMemory();
 	if (memory != DConstants::INVALID_INDEX) {
@@ -322,14 +338,6 @@ idx_t DBConfig::GetSystemMaxThreads(FileSystem &fs) {
 #endif
 }
 
-void DBConfig::SetDefaultMaxThreads() {
-#ifndef DUCKDB_NO_THREADS
-	options.maximum_threads = GetSystemMaxThreads(*file_system);
-#else
-	options.maximum_threads = 1;
-#endif
-}
-
 idx_t DBConfig::ParseMemoryLimit(const string &arg) {
 	if (arg[0] == '-' || arg == "null" || arg == "none") {
 		return DConstants::INVALID_INDEX;
@@ -376,8 +384,17 @@ idx_t DBConfig::ParseMemoryLimit(const string &arg) {
 		multiplier = 1000LL * 1000LL * 1000LL;
 	} else if (unit == "terabyte" || unit == "terabytes" || unit == "tb" || unit == "t") {
 		multiplier = 1000LL * 1000LL * 1000LL * 1000LL;
+	} else if (unit == "kib") {
+		multiplier = 1024LL;
+	} else if (unit == "mib") {
+		multiplier = 1024LL * 1024LL;
+	} else if (unit == "gib") {
+		multiplier = 1024LL * 1024LL * 1024LL;
+	} else if (unit == "tib") {
+		multiplier = 1024LL * 1024LL * 1024LL * 1024LL;
 	} else {
-		throw ParserException("Unknown unit for memory_limit: %s (expected: b, mb, gb or tb)", unit);
+		throw ParserException("Unknown unit for memory_limit: %s (expected: KB, MB, GB, TB for 1000^i units or KiB, "
+		                      "MiB, GiB, TiB for 1024^i unites)");
 	}
 	return (idx_t)multiplier * limit;
 }
@@ -421,7 +438,11 @@ OrderByNullType DBConfig::ResolveNullOrder(OrderType order_type, OrderByNullType
 }
 
 const std::string DBConfig::UserAgent() const {
-	auto user_agent = options.duckdb_api;
+	auto user_agent = GetDefaultUserAgent();
+
+	if (!options.duckdb_api.empty()) {
+		user_agent += " " + options.duckdb_api;
+	}
 
 	if (!options.custom_user_agent.empty()) {
 		user_agent += " " + options.custom_user_agent;

package/src/duckdb/src/main/connection.cpp

@@ -1,7 +1,6 @@
 #include "duckdb/main/connection.hpp"
 
 #include "duckdb/common/types/column/column_data_collection.hpp"
-#include "duckdb/execution/operator/scan/csv/parallel_csv_reader.hpp"
 #include "duckdb/function/table/read_csv.hpp"
 #include "duckdb/main/appender.hpp"
 #include "duckdb/main/client_context.hpp"
@@ -30,7 +29,21 @@ Connection::Connection(DatabaseInstance &database) : context(make_shared<ClientC
 Connection::Connection(DuckDB &database) : Connection(*database.instance) {
 }
 
+Connection::Connection(Connection &&other) noexcept {
+	std::swap(context, other.context);
+	std::swap(warning_cb, other.warning_cb);
+}
+
+Connection &Connection::operator=(Connection &&other) noexcept {
+	std::swap(context, other.context);
+	std::swap(warning_cb, other.warning_cb);
+	return *this;
+}
+
 Connection::~Connection() {
+	if (!context) {
+		return;
+	}
 	ConnectionManager::Get(*context->db).RemoveConnection(*context);
 }
 
@@ -223,21 +236,29 @@ shared_ptr<Relation> Connection::ReadCSV(const string &csv_file) {
 	return ReadCSV(csv_file, std::move(options));
 }
 
-shared_ptr<Relation> Connection::ReadCSV(const string &csv_file, named_parameter_map_t &&options) {
-	return make_shared<ReadCSVRelation>(context, csv_file, std::move(options));
+shared_ptr<Relation> Connection::ReadCSV(const vector<string> &csv_input, named_parameter_map_t &&options) {
+	return make_shared<ReadCSVRelation>(context, csv_input, std::move(options));
+}
+
+shared_ptr<Relation> Connection::ReadCSV(const string &csv_input, named_parameter_map_t &&options) {
+	vector<string> csv_files = {csv_input};
+	return ReadCSV(csv_files, std::move(options));
 }
 
 shared_ptr<Relation> Connection::ReadCSV(const string &csv_file, const vector<string> &columns) {
 	// parse columns
-	vector<ColumnDefinition> column_list;
+	named_parameter_map_t options;
+	child_list_t<Value> column_list;
 	for (auto &column : columns) {
 		auto col_list = Parser::ParseColumnList(column, context->GetParserOptions());
 		if (col_list.LogicalColumnCount() != 1) {
 			throw ParserException("Expected a single column definition");
 		}
-		column_list.push_back(std::move(col_list.GetColumnMutable(LogicalIndex(0))));
+		auto &col_def = col_list.GetColumnMutable(LogicalIndex(0));
+		column_list.push_back({col_def.GetName(), col_def.GetType().ToString()});
 	}
-	return make_shared<ReadCSVRelation>(context, csv_file, std::move(column_list));
+	vector<string> files {csv_file};
+	return make_shared<ReadCSVRelation>(context, files, std::move(options));
 }
 
 shared_ptr<Relation> Connection::ReadParquet(const string &parquet_file, bool binary_as_string) {

package/src/duckdb/src/main/connection_manager.cpp

@@ -0,0 +1,54 @@
+#include "duckdb/main/connection_manager.hpp"
+#include "duckdb/common/exception/transaction_exception.hpp"
+
+namespace duckdb {
+
+ConnectionManager::ConnectionManager() : is_locking(false) {
+}
+
+void ConnectionManager::AddConnection(ClientContext &context) {
+	lock_guard<mutex> lock(connections_lock);
+	connections.insert(make_pair(&context, weak_ptr<ClientContext>(context.shared_from_this())));
+}
+
+void ConnectionManager::RemoveConnection(ClientContext &context) {
+	lock_guard<mutex> lock(connections_lock);
+	connections.erase(&context);
+}
+
+vector<shared_ptr<ClientContext>> ConnectionManager::GetConnectionList() {
+	vector<shared_ptr<ClientContext>> result;
+	for (auto &it : connections) {
+		auto connection = it.second.lock();
+		if (!connection) {
+			connections.erase(it.first);
+			continue;
+		} else {
+			result.push_back(std::move(connection));
+		}
+	}
+
+	return result;
+}
+
+void ConnectionManager::LockClients(vector<ClientLockWrapper> &client_locks, ClientContext &context) {
+	{
+		lock_guard<mutex> l(lock_clients_lock);
+		if (is_locking) {
+			throw TransactionException("Failed to lock clients - another thread is running FORCE CHECKPOINT");
+		}
+		is_locking = true;
+	}
+	client_locks.emplace_back(connections_lock, nullptr);
+	auto connection_list = GetConnectionList();
+	for (auto &con : connection_list) {
+		if (con.get() == &context) {
+			continue;
+		}
+		auto &context_lock = con->context_lock;
+		client_locks.emplace_back(context_lock, std::move(con));
+	}
+	is_locking = false;
+}
+
+} // namespace duckdb

package/src/duckdb/src/main/database.cpp

@@ -9,17 +9,19 @@
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/connection_manager.hpp"
 #include "duckdb/main/database_manager.hpp"
+#include "duckdb/main/database_path_and_type.hpp"
 #include "duckdb/main/error_manager.hpp"
 #include "duckdb/main/extension_helper.hpp"
+#include "duckdb/main/secret/secret_manager.hpp"
 #include "duckdb/parallel/task_scheduler.hpp"
 #include "duckdb/parser/parsed_data/attach_info.hpp"
+#include "duckdb/planner/extension_callback.hpp"
 #include "duckdb/storage/object_cache.hpp"
 #include "duckdb/storage/standard_buffer_manager.hpp"
-#include "duckdb/main/database_path_and_type.hpp"
 #include "duckdb/storage/storage_extension.hpp"
 #include "duckdb/storage/storage_manager.hpp"
 #include "duckdb/transaction/transaction_manager.hpp"
-#include "duckdb/planner/extension_callback.hpp"
+#include "duckdb/execution/index/index_type_set.hpp"
 
 #ifndef DUCKDB_NO_THREADS
 #include "duckdb/common/thread.hpp"
@@ -29,21 +31,20 @@ namespace duckdb {
 
 DBConfig::DBConfig() {
 	compression_functions = make_uniq<CompressionFunctionSet>();
-	cast_functions = make_uniq<CastFunctionSet>();
+	cast_functions = make_uniq<CastFunctionSet>(*this);
+	index_types = make_uniq<IndexTypeSet>();
 	error_manager = make_uniq<ErrorManager>();
-	options.duckdb_api = StringUtil::Format("duckdb/%s(%s)", DuckDB::LibraryVersion(), DuckDB::Platform());
+	secret_manager = make_uniq<SecretManager>();
 }
 
-DBConfig::DBConfig(std::unordered_map<string, string> &config_dict, bool read_only) : DBConfig::DBConfig() {
+DBConfig::DBConfig(bool read_only) : DBConfig::DBConfig() {
 	if (read_only) {
 		options.access_mode = AccessMode::READ_ONLY;
 	}
-	for (auto &kv : config_dict) {
-		string key = kv.first;
-		string val = kv.second;
-		auto opt_val = Value(val);
-		DBConfig::SetOptionByName(key, opt_val);
-	}
+}
+
+DBConfig::DBConfig(const case_insensitive_map_t<Value> &config_dict, bool read_only) : DBConfig::DBConfig(read_only) {
+	SetOptionsByName(config_dict);
 }
 
 DBConfig::~DBConfig() {
@@ -53,6 +54,7 @@ DatabaseInstance::DatabaseInstance() {
 }
 
 DatabaseInstance::~DatabaseInstance() {
+	GetDatabaseManager().ResetDatabases();
 }
 
 BufferManager &BufferManager::GetBufferManager(DatabaseInstance &db) {
@@ -118,22 +120,13 @@ ConnectionManager &ConnectionManager::Get(DatabaseInstance &db) {
 	return db.GetConnectionManager();
 }
 
-ClientContext *ConnectionManager::GetConnection(DatabaseInstance *db) {
-	for (auto &conn : connections) {
-		if (conn.first->db.get() == db) {
-			return conn.first;
-		}
-	}
-	return nullptr;
-}
-
 ConnectionManager &ConnectionManager::Get(ClientContext &context) {
 	return ConnectionManager::Get(DatabaseInstance::GetDatabase(context));
 }
 
-duckdb::unique_ptr<AttachedDatabase> DatabaseInstance::CreateAttachedDatabase(AttachInfo &info, const string &type,
-                                                                              AccessMode access_mode) {
-	duckdb::unique_ptr<AttachedDatabase> attached_database;
+unique_ptr<AttachedDatabase> DatabaseInstance::CreateAttachedDatabase(ClientContext &context, const AttachInfo &info,
+                                                                      const string &type, AccessMode access_mode) {
+	unique_ptr<AttachedDatabase> attached_database;
 	if (!type.empty()) {
 		// find the storage extension
 		auto extension_name = ExtensionHelper::ApplyExtensionAlias(type);
@@ -145,7 +138,7 @@ duckdb::unique_ptr<AttachedDatabase> DatabaseInstance::CreateAttachedDatabase(At
 		if (entry->second->attach != nullptr && entry->second->create_transaction_manager != nullptr) {
 			// use storage extension to create the initial database
 			attached_database = make_uniq<AttachedDatabase>(*this, Catalog::GetSystemCatalog(*this), *entry->second,
-			                                                info.name, info, access_mode);
+			                                                context, info.name, info, access_mode);
 		} else {
 			attached_database =
 			    make_uniq<AttachedDatabase>(*this, Catalog::GetSystemCatalog(*this), info.name, info.path, access_mode);
@@ -163,16 +156,15 @@ void DatabaseInstance::CreateMainDatabase() {
 	info.name = AttachedDatabase::ExtractDatabaseName(config.options.database_path, GetFileSystem());
 	info.path = config.options.database_path;
 
-	auto attached_database = CreateAttachedDatabase(info, config.options.database_type, config.options.access_mode);
-	auto initial_database = attached_database.get();
+	optional_ptr<AttachedDatabase> initial_database;
 	{
 		Connection con(*this);
 		con.BeginTransaction();
-		db_manager->AddDatabase(*con.context, std::move(attached_database));
+		initial_database =
+		    db_manager->AttachDatabase(*con.context, info, config.options.database_type, config.options.access_mode);
 		con.Commit();
 	}
 
-	// initialize the database
 	initial_database->SetInitialDatabase();
 	initial_database->Initialize();
 }
@@ -193,12 +185,16 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
 		config_ptr = user_config;
 	}
 
+	if (config_ptr->options.duckdb_api.empty()) {
+		config_ptr->SetOptionByName("duckdb_api", "cpp");
+	}
+
 	if (config_ptr->options.temporary_directory.empty() && database_path) {
 		// no directory specified: use default temp path
 		config_ptr->options.temporary_directory = string(database_path) + ".tmp";
 
 		// special treatment for in-memory mode
-		if (strcmp(database_path, ":memory:") == 0) {
+		if (strcmp(database_path, IN_MEMORY_PATH) == 0) {
 			config_ptr->options.temporary_directory = ".tmp";
 		}
 	}
@@ -208,6 +204,7 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
 	} else {
 		config_ptr->options.database_path.clear();
 	}
+
 	Configure(*config_ptr);
 
 	if (user_config && !user_config->options.use_temporary_directory) {
@@ -221,12 +218,11 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
 	object_cache = make_uniq<ObjectCache>();
 	connection_manager = make_uniq<ConnectionManager>();
 
-	// check if we are opening a standard DuckDB database or an extension database
-	if (config.options.database_type.empty()) {
-		auto path_and_type = DBPathAndType::Parse(config.options.database_path, config);
-		config.options.database_type = path_and_type.type;
-		config.options.database_path = path_and_type.path;
-	}
+	// resolve the type of teh database we are opening
+	DBPathAndType::ResolveDatabaseType(config.options.database_path, config.options.database_type, config);
+
+	// initialize the secret manager
+	config.secret_manager->Initialize(*this);
 
 	// initialize the system catalog
 	db_manager->InitializeSystemCatalog();
@@ -236,7 +232,7 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
 		if (!config.file_system) {
 			throw InternalException("No file system!?");
 		}
-		ExtensionHelper::LoadExternalExtension(*this, *config.file_system, config.options.database_type, nullptr);
+		ExtensionHelper::LoadExternalExtension(*this, *config.file_system, config.options.database_type);
 	}
 
 	if (!config.options.unrecognized_options.empty()) {
@@ -248,7 +244,8 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
 	}
 
 	// only increase thread count after storage init because we get races on catalog otherwise
-	scheduler->SetThreads(config.options.maximum_threads);
+	scheduler->SetThreads(config.options.maximum_threads, config.options.external_threads);
+	scheduler->RelaunchThreads();
 }
 
 DuckDB::DuckDB(const char *path, DBConfig *new_config) : instance(make_shared<DatabaseInstance>()) {
@@ -267,11 +264,15 @@ DuckDB::DuckDB(DatabaseInstance &instance_p) : instance(instance_p.shared_from_t
 DuckDB::~DuckDB() {
 }
 
+SecretManager &DatabaseInstance::GetSecretManager() {
+	return *config.secret_manager;
+}
+
 BufferManager &DatabaseInstance::GetBufferManager() {
 	return *buffer_manager;
 }
 
-BufferPool &DatabaseInstance::GetBufferPool() {
+BufferPool &DatabaseInstance::GetBufferPool() const {
 	return *config.buffer_pool;
 }
 
@@ -320,16 +321,20 @@ void DatabaseInstance::Configure(DBConfig &new_config) {
 	if (config.options.access_mode == AccessMode::UNDEFINED) {
 		config.options.access_mode = AccessMode::READ_WRITE;
 	}
+	config.extension_parameters = new_config.extension_parameters;
 	if (new_config.file_system) {
 		config.file_system = std::move(new_config.file_system);
 	} else {
 		config.file_system = make_uniq<VirtualFileSystem>();
 	}
+	if (new_config.secret_manager) {
+		config.secret_manager = std::move(new_config.secret_manager);
+	}
 	if (config.options.maximum_memory == (idx_t)-1) {
 		config.SetDefaultMaxMemory();
 	}
 	if (new_config.options.maximum_threads == (idx_t)-1) {
-		config.SetDefaultMaxThreads();
+		config.options.maximum_threads = config.GetSystemMaxThreads(*config.file_system);
 	}
 	config.allocator = std::move(new_config.allocator);
 	if (!config.allocator) {