duckdb 1.3.4 → 1.4.0

package/src/duckdb/src/include/duckdb/main/config.hpp

@@ -16,11 +16,11 @@
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/encryption_state.hpp"
 #include "duckdb/common/enums/access_mode.hpp"
+#include "duckdb/common/enums/thread_pin_mode.hpp"
 #include "duckdb/common/enums/compression_type.hpp"
 #include "duckdb/common/enums/optimizer_type.hpp"
 #include "duckdb/common/enums/order_type.hpp"
 #include "duckdb/common/enums/set_scope.hpp"
-#include "duckdb/common/enums/window_aggregation_mode.hpp"
 #include "duckdb/common/file_system.hpp"
 #include "duckdb/common/set.hpp"
 #include "duckdb/common/types/value.hpp"
@@ -29,16 +29,14 @@
 #include "duckdb/execution/index/index_type_set.hpp"
 #include "duckdb/function/cast/default_casts.hpp"
 #include "duckdb/function/replacement_scan.hpp"
-#include "duckdb/main/client_properties.hpp"
 #include "duckdb/optimizer/optimizer_extension.hpp"
 #include "duckdb/parser/parsed_data/create_info.hpp"
 #include "duckdb/parser/parser_extension.hpp"
 #include "duckdb/planner/operator_extension.hpp"
 #include "duckdb/storage/compression/bitpacking.hpp"
 #include "duckdb/function/encoding_function.hpp"
+#include "duckdb/main/setting_info.hpp"
 #include "duckdb/logging/log_manager.hpp"
-#include "duckdb/common/enums/debug_vector_verification.hpp"
-#include "duckdb/logging/logging.hpp"
 
 namespace duckdb {
 
@@ -57,50 +55,12 @@ class SecretManager;
 class CompressionInfo;
 class EncryptionUtil;
 class HTTPUtil;
+class DatabaseFilePathManager;
 
 struct CompressionFunctionSet;
 struct DatabaseCacheEntry;
 struct DBConfig;
-
-enum class CheckpointAbort : uint8_t {
-	NO_ABORT = 0,
-	DEBUG_ABORT_BEFORE_TRUNCATE = 1,
-	DEBUG_ABORT_BEFORE_HEADER = 2,
-	DEBUG_ABORT_AFTER_FREE_LIST_WRITE = 3
-};
-
-typedef void (*set_global_function_t)(DatabaseInstance *db, DBConfig &config, const Value &parameter);
-typedef void (*set_local_function_t)(ClientContext &context, const Value &parameter);
-typedef void (*reset_global_function_t)(DatabaseInstance *db, DBConfig &config);
-typedef void (*reset_local_function_t)(ClientContext &context);
-typedef Value (*get_setting_function_t)(const ClientContext &context);
-
-struct ConfigurationOption {
-	const char *name;
-	const char *description;
-	const char *parameter_type;
-	set_global_function_t set_global;
-	set_local_function_t set_local;
-	reset_global_function_t reset_global;
-	reset_local_function_t reset_local;
-	get_setting_function_t get_setting;
-};
-
-typedef void (*set_option_callback_t)(ClientContext &context, SetScope scope, Value &parameter);
-
-struct ExtensionOption {
-	// NOLINTNEXTLINE: work around bug in clang-tidy
-	ExtensionOption(string description_p, LogicalType type_p, set_option_callback_t set_function_p,
-	                Value default_value_p)
-	    : description(std::move(description_p)), type(std::move(type_p)), set_function(set_function_p),
-	      default_value(std::move(default_value_p)) {
-	}
-
-	string description;
-	LogicalType type;
-	set_option_callback_t set_function;
-	Value default_value;
-};
+struct SettingLookupResult;
 
 class SerializationCompatibility {
 public:
@@ -174,12 +134,8 @@ struct DBConfigOptions {
 	bool buffer_manager_track_eviction_timestamps = false;
 	//! Whether or not to allow printing unredacted secrets
 	bool allow_unredacted_secrets = false;
-	//! The collation type of the database
-	string collation = string();
-	//! The order type used when none is specified (default: ASC)
-	OrderType default_order_type = OrderType::ASCENDING;
-	//! NULL ordering used when none is specified (default: NULLS LAST)
-	DefaultOrderByNullType default_null_order = DefaultOrderByNullType::NULLS_LAST;
+	//! Disables invalidating the database instance when encountering a fatal error.
+	bool disable_database_invalidation = false;
 	//! enable COPY and related commands
 	bool enable_external_access = true;
 	//! Whether or not the global http metadata cache is used
@@ -196,8 +152,6 @@ struct DBConfigOptions {
 	bool checkpoint_on_shutdown = true;
 	//! Serialize the metadata on checkpoint with compatibility for a given DuckDB version.
 	SerializationCompatibility serialization_compatibility = SerializationCompatibility::Default();
-	//! Debug flag that decides when a checkpoing should be aborted. Only used for testing purposes.
-	CheckpointAbort checkpoint_abort = CheckpointAbort::NO_ABORT;
 	//! Initialize the database with the standard set of DuckDB functions
 	//! You should probably not touch this unless you know what you are doing
 	bool initialize_default_database = true;
@@ -211,19 +165,6 @@ struct DBConfigOptions {
 	set<CompressionType> disabled_compression_methods;
 	//! Force a specific bitpacking mode to be used when using the bitpacking compression method
 	BitpackingMode force_bitpacking_mode = BitpackingMode::AUTO;
-	//! Debug setting for window aggregation mode: (window, combine, separate)
-	WindowAggregationMode window_mode = WindowAggregationMode::WINDOW;
-	//! Whether preserving insertion order should be preserved
-	bool preserve_insertion_order = true;
-	//! Whether Arrow Arrays use Large or Regular buffers
-	ArrowOffsetSize arrow_offset_size = ArrowOffsetSize::REGULAR;
-	//! Whether LISTs should produce Arrow ListViews
-	bool arrow_use_list_view = false;
-	//! Whenever a DuckDB type does not have a clear native or canonical extension match in Arrow, export the types
-	//! with a duckdb.type_name extension name
-	bool arrow_lossless_conversion = false;
-	//! Whether when producing arrow objects we produce string_views or regular strings
-	bool produce_arrow_string_views = false;
 	//! Database configuration variables as controlled by SET
 	case_insensitive_map_t<Value> set_variables;
 	//! Database configuration variable default values;
@@ -234,16 +175,6 @@ struct DBConfigOptions {
 	bool allow_unsigned_extensions = false;
 	//! Whether community extensions should be loaded
 	bool allow_community_extensions = true;
-	//! Whether extensions with missing metadata should be loaded
-	bool allow_extensions_metadata_mismatch = false;
-	//! Enable emitting FSST Vectors
-	bool enable_fsst_vectors = false;
-	//! Enable VIEWs to create dependencies
-	bool enable_view_dependencies = false;
-	//! Enable macros to create dependencies
-	bool enable_macro_dependencies = false;
-	//! Start transactions immediately in all attached databases - instead of lazily when a database is referenced
-	bool immediate_transaction_mode = false;
 	//! Debug setting - how to initialize  blocks in the storage layer when allocating
 	DebugInitialize debug_initialize = DebugInitialize::NO_INITIALIZE;
 	//! The set of user-provided options
@@ -264,30 +195,14 @@ struct DBConfigOptions {
 	string duckdb_api;
 	//! Metadata from DuckDB callers
 	string custom_user_agent;
-	//! Use old implicit casting style (i.e. allow everything to be implicitly casted to VARCHAR)
-	bool old_implicit_casting = false;
+	//! Encrypt the temp files
+	bool temp_file_encryption = false;
 	//! The default block allocation size for new duckdb database files (new as-in, they do not yet exist).
-	idx_t default_block_alloc_size = DUCKDB_BLOCK_ALLOC_SIZE;
+	idx_t default_block_alloc_size = DEFAULT_BLOCK_ALLOC_SIZE;
 	//! The default block header size for new duckdb database files.
 	idx_t default_block_header_size = DUCKDB_BLOCK_HEADER_STORAGE_SIZE;
 	//!  Whether or not to abort if a serialization exception is thrown during WAL playback (when reading truncated WAL)
 	bool abort_on_wal_failure = false;
-	//! The index_scan_percentage sets a threshold for index scans.
-	//! If fewer than MAX(index_scan_max_count, index_scan_percentage * total_row_count)
-	//! rows match, we perform an index scan instead of a table scan.
-	double index_scan_percentage = 0.001;
-	//! The index_scan_max_count sets a threshold for index scans.
-	//! If fewer than MAX(index_scan_max_count, index_scan_percentage * total_row_count)
-	//! rows match, we perform an index scan instead of a table scan.
-	idx_t index_scan_max_count = STANDARD_VECTOR_SIZE;
-	//! The maximum number of schemas we will look through for "did you mean..." style errors in the catalog
-	idx_t catalog_error_max_schemas = 100;
-	//!  Whether or not to always write to the WAL file, even if this is not required
-	bool debug_skip_checkpoint_on_commit = false;
-	//! Vector verification to enable (debug setting only)
-	DebugVectorVerification debug_verify_vector = DebugVectorVerification::NONE;
-	//! The maximum amount of vacuum tasks to schedule during a checkpoint
-	idx_t max_vacuum_tasks = 100;
 	//! Paths that are explicitly allowed, even if enable_external_access is false
 	unordered_set<string> allowed_paths;
 	//! Directories that are explicitly allowed, even if enable_external_access is false
@@ -296,14 +211,14 @@ struct DBConfigOptions {
 	LogConfig log_config = LogConfig();
 	//! Whether to enable external file caching using CachingFileSystem
 	bool enable_external_file_cache = true;
-	//! Output version of arrow depending on the format version
-	ArrowFormatVersion arrow_output_version = V1_0;
 	//! Partially process tasks before rescheduling - allows for more scheduler fairness between separate queries
 #ifdef DUCKDB_ALTERNATIVE_VERIFY
 	bool scheduler_process_partial = true;
 #else
 	bool scheduler_process_partial = false;
 #endif
+	//! Whether to pin threads to cores (linux only, default AUTOMATIC: on when there are more than 64 cores)
+	ThreadPinMode pin_threads = ThreadPinMode::AUTO;
 
 	bool operator==(const DBConfigOptions &other) const;
 };
@@ -318,7 +233,7 @@ public:
 	DUCKDB_API DBConfig(const case_insensitive_map_t<Value> &config_dict, bool read_only);
 	DUCKDB_API ~DBConfig();
 
-	mutex config_lock;
+	mutable mutex config_lock;
 	//! Replacement table scans are automatically attempted when a table name cannot be found in the schema
 	vector<ReplacementScan> replacement_scans;
 
@@ -357,6 +272,8 @@ public:
 	shared_ptr<HTTPUtil> http_util;
 	//! Reference to the database cache entry (if any)
 	shared_ptr<DatabaseCacheEntry> db_cache_entry;
+	//! Reference to the database file path manager
+	shared_ptr<DatabaseFilePathManager> path_manager;
 
 public:
 	DUCKDB_API static DBConfig &GetConfig(ClientContext &context);
@@ -365,26 +282,32 @@ public:
 	DUCKDB_API static const DBConfig &GetConfig(const ClientContext &context);
 	DUCKDB_API static const DBConfig &GetConfig(const DatabaseInstance &db);
 	DUCKDB_API static vector<ConfigurationOption> GetOptions();
+	DUCKDB_API static vector<ConfigurationAlias> GetAliases();
 	DUCKDB_API static idx_t GetOptionCount();
+	DUCKDB_API static idx_t GetAliasCount();
 	DUCKDB_API static vector<string> GetOptionNames();
 	DUCKDB_API static bool IsInMemoryDatabase(const char *database_path);
 
 	DUCKDB_API void AddExtensionOption(const string &name, string description, LogicalType parameter,
-	                                   const Value &default_value = Value(), set_option_callback_t function = nullptr);
+	                                   const Value &default_value = Value(), set_option_callback_t function = nullptr,
+	                                   SetScope default_scope = SetScope::SESSION);
 	//! Fetch an option by index. Returns a pointer to the option, or nullptr if out of range
 	DUCKDB_API static optional_ptr<const ConfigurationOption> GetOptionByIndex(idx_t index);
+	//! Fetcha n alias by index, or nullptr if out of range
+	DUCKDB_API static optional_ptr<const ConfigurationAlias> GetAliasByIndex(idx_t index);
 	//! Fetch an option by name. Returns a pointer to the option, or nullptr if none exists.
-	DUCKDB_API static optional_ptr<const ConfigurationOption> GetOptionByName(const string &name);
+	DUCKDB_API static optional_ptr<const ConfigurationOption> GetOptionByName(const String &name);
 	DUCKDB_API void SetOption(const ConfigurationOption &option, const Value &value);
-	DUCKDB_API void SetOption(DatabaseInstance *db, const ConfigurationOption &option, const Value &value);
+	DUCKDB_API void SetOption(optional_ptr<DatabaseInstance> db, const ConfigurationOption &option, const Value &value);
 	DUCKDB_API void SetOptionByName(const string &name, const Value &value);
 	DUCKDB_API void SetOptionsByName(const case_insensitive_map_t<Value> &values);
-	DUCKDB_API void ResetOption(DatabaseInstance *db, const ConfigurationOption &option);
+	DUCKDB_API void ResetOption(optional_ptr<DatabaseInstance> db, const ConfigurationOption &option);
 	DUCKDB_API void SetOption(const string &name, Value value);
-	DUCKDB_API void ResetOption(const string &name);
+	DUCKDB_API void ResetOption(const String &name);
+	DUCKDB_API void ResetGenericOption(const String &name);
 	static LogicalType ParseLogicalType(const string &type);
 
-	DUCKDB_API void CheckLock(const string &name);
+	DUCKDB_API void CheckLock(const String &name);
 
 	DUCKDB_API static idx_t ParseMemoryLimit(const string &arg);
 
@@ -418,19 +341,28 @@ public:
 	void SetDefaultMaxMemory();
 	void SetDefaultTempDirectory();
 
-	OrderType ResolveOrder(OrderType order_type) const;
-	OrderByNullType ResolveNullOrder(OrderType order_type, OrderByNullType null_type) const;
+	OrderType ResolveOrder(ClientContext &context, OrderType order_type) const;
+	OrderByNullType ResolveNullOrder(ClientContext &context, OrderType order_type, OrderByNullType null_type) const;
 	const string UserAgent() const;
 
-	template <class OP>
-	typename OP::RETURN_TYPE GetSetting(const ClientContext &context) {
-		std::lock_guard<mutex> lock(config_lock);
-		return OP::GetSetting(context).template GetValue<typename OP::RETURN_TYPE>();
+	SettingLookupResult TryGetCurrentSetting(const string &key, Value &result) const;
+
+	template <class OP, class SOURCE>
+	static typename std::enable_if<std::is_enum<typename OP::RETURN_TYPE>::value, typename OP::RETURN_TYPE>::type
+	GetSetting(const SOURCE &source) {
+		return EnumUtil::FromString<typename OP::RETURN_TYPE>(
+		    GetSettingInternal(source, OP::Name, OP::DefaultValue).ToString());
+	}
+
+	template <class OP, class SOURCE>
+	static typename std::enable_if<!std::is_enum<typename OP::RETURN_TYPE>::value, typename OP::RETURN_TYPE>::type
+	GetSetting(const SOURCE &source) {
+		return GetSettingInternal(source, OP::Name, OP::DefaultValue).template GetValue<typename OP::RETURN_TYPE>();
 	}
 
 	template <class OP>
-	Value GetSettingValue(const ClientContext &context) {
-		std::lock_guard<mutex> lock(config_lock);
+	Value GetSettingValue(const ClientContext &context) const {
+		lock_guard<mutex> lock(config_lock);
 		return OP::GetSetting(context);
 	}
 
@@ -439,6 +371,11 @@ public:
 	void AddAllowedPath(const string &path);
 	string SanitizeAllowedPath(const string &path) const;
 
+private:
+	static Value GetSettingInternal(const DatabaseInstance &db, const char *setting, const char *default_value);
+	static Value GetSettingInternal(const DBConfig &config, const char *setting, const char *default_value);
+	static Value GetSettingInternal(const ClientContext &context, const char *setting, const char *default_value);
+
 private:
 	unique_ptr<CompressionFunctionSet> compression_functions;
 	unique_ptr<EncodingFunctionSet> encoding_functions;

package/src/duckdb/src/include/duckdb/main/database.hpp

@@ -12,9 +12,9 @@
 #include "duckdb/main/capi/extension_api.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/main/extension.hpp"
-#include "duckdb/main/extension_install_info.hpp"
-#include "duckdb/main/settings.hpp"
 #include "duckdb/main/valid_checker.hpp"
+#include "duckdb/main/extension/extension_loader.hpp"
+#include "duckdb/main/extension_manager.hpp"
 
 namespace duckdb {
 class BufferManager;
@@ -23,6 +23,7 @@ class StorageManager;
 class Catalog;
 class TransactionManager;
 class ConnectionManager;
+class ExtensionManager;
 class FileSystem;
 class TaskScheduler;
 class ObjectCache;
@@ -33,12 +34,6 @@ struct DatabaseCacheEntry;
 class LogManager;
 class ExternalFileCache;
 
-struct ExtensionInfo {
-	bool is_loaded;
-	unique_ptr<ExtensionInstallInfo> install_info;
-	unique_ptr<ExtensionLoadedInfo> load_info;
-};
-
 class DatabaseInstance : public enable_shared_from_this<DatabaseInstance> {
 	friend class DuckDB;
 
@@ -59,9 +54,9 @@ public:
 	DUCKDB_API TaskScheduler &GetScheduler();
 	DUCKDB_API ObjectCache &GetObjectCache();
 	DUCKDB_API ConnectionManager &GetConnectionManager();
+	DUCKDB_API ExtensionManager &GetExtensionManager();
 	DUCKDB_API ValidChecker &GetValidChecker();
 	DUCKDB_API LogManager &GetLogManager() const;
-	DUCKDB_API void SetExtensionLoaded(const string &extension_name, ExtensionInstallInfo &install_info);
 
 	DUCKDB_API const duckdb_ext_api_v1 GetExtensionAPIV1();
 
@@ -70,15 +65,14 @@ public:
 	DUCKDB_API static DatabaseInstance &GetDatabase(ClientContext &context);
 	DUCKDB_API static const DatabaseInstance &GetDatabase(const ClientContext &context);
 
-	DUCKDB_API const unordered_map<string, ExtensionInfo> &GetExtensions();
 	DUCKDB_API bool ExtensionIsLoaded(const string &name);
 
 	DUCKDB_API SettingLookupResult TryGetCurrentSetting(const string &key, Value &result) const;
 
-	unique_ptr<AttachedDatabase> CreateAttachedDatabase(ClientContext &context, AttachInfo &info,
-	                                                    AttachOptions &options);
+	DUCKDB_API shared_ptr<EncryptionUtil> GetEncryptionUtil() const;
 
-	void AddExtensionInfo(const string &name, const ExtensionLoadedInfo &info);
+	shared_ptr<AttachedDatabase> CreateAttachedDatabase(ClientContext &context, AttachInfo &info,
+	                                                    AttachOptions &options);
 
 private:
 	void Initialize(const char *path, DBConfig *config);
@@ -93,7 +87,7 @@ private:
 	unique_ptr<TaskScheduler> scheduler;
 	unique_ptr<ObjectCache> object_cache;
 	unique_ptr<ConnectionManager> connection_manager;
-	unordered_map<string, ExtensionInfo> loaded_extensions_info;
+	unique_ptr<ExtensionManager> extension_manager;
 	ValidChecker db_validity;
 	unique_ptr<DatabaseFileSystem> db_file_system;
 	shared_ptr<LogManager> log_manager;
@@ -120,24 +114,26 @@ public:
 	template <class T>
 	void LoadStaticExtension() {
 		T extension;
-		if (ExtensionIsLoaded(extension.Name())) {
+		auto &manager = ExtensionManager::Get(*instance);
+		auto info = manager.BeginLoad(extension.Name());
+		if (!info) {
+			// already loaded - return
 			return;
 		}
-		extension.Load(*this);
+
+		// Instantiate a new loader
+		ExtensionLoader loader(*instance, extension.Name());
+
+		// Call the Load method of the extension
+		extension.Load(loader);
+
+		// Finalize the loading process
+		loader.FinalizeLoad();
+
 		ExtensionInstallInfo install_info;
 		install_info.mode = ExtensionInstallMode::STATICALLY_LINKED;
 		install_info.version = extension.Version();
-		instance->SetExtensionLoaded(extension.Name(), install_info);
-	}
-
-	// DEPRECATED function that some extensions may still use to call their own Load method from the
-	// _init function of their loadable extension. Don't use this. Instead opt for a static LoadInternal function called
-	// from both the _init function and the Extension::Load. (see autocomplete extension)
-	// TODO: when to remove this function?
-	template <class T>
-	void LoadExtension() {
-		T extension;
-		extension.Load(*this);
+		info->FinishLoad(install_info);
 	}
 
 	DUCKDB_API FileSystem &GetFileSystem();

package/src/duckdb/src/include/duckdb/main/database_file_opener.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/common/opener_file_system.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/main/database.hpp"
+#include "duckdb/logging/log_manager.hpp"
 
 namespace duckdb {
 class DatabaseInstance;

package/src/duckdb/src/include/duckdb/main/database_file_path_manager.hpp

@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/main/database_file_path_manager.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/mutex.hpp"
+#include "duckdb/common/case_insensitive_map.hpp"
+#include "duckdb/common/enums/on_create_conflict.hpp"
+
+namespace duckdb {
+struct AttachInfo;
+struct AttachOptions;
+
+enum class InsertDatabasePathResult { SUCCESS, ALREADY_EXISTS };
+
+//! The DatabaseFilePathManager is used to ensure we only ever open a single database file once
+class DatabaseFilePathManager {
+public:
+	idx_t ApproxDatabaseCount() const;
+	InsertDatabasePathResult InsertDatabasePath(const string &path, const string &name, OnCreateConflict on_conflict,
+	                                            AttachOptions &options);
+	void EraseDatabasePath(const string &path);
+
+private:
+	//! The lock to add entries to the database path map
+	mutable mutex db_paths_lock;
+	//! A set containing all attached database path
+	//! This allows to attach many databases efficiently, and to avoid attaching the
+	//! same file path twice
+	case_insensitive_map_t<string> db_paths_to_name;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/main/database_manager.hpp

@@ -17,6 +17,7 @@
 #include "duckdb/common/optional_ptr.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/parser/parsed_data/attach_info.hpp"
+#include "duckdb/main/database_file_path_manager.hpp"
 
 namespace duckdb {
 class AttachedDatabase;
@@ -48,9 +49,14 @@ public:
 	//! Get an attached database by its name
 	optional_ptr<AttachedDatabase> GetDatabase(ClientContext &context, const string &name);
 	//! Attach a new database
-	optional_ptr<AttachedDatabase> AttachDatabase(ClientContext &context, AttachInfo &info, AttachOptions &options);
+	shared_ptr<AttachedDatabase> AttachDatabase(ClientContext &context, AttachInfo &info, AttachOptions &options);
+
+	optional_ptr<AttachedDatabase> FinalizeAttach(ClientContext &context, AttachInfo &info,
+	                                              shared_ptr<AttachedDatabase> database);
 	//! Detach an existing database
 	void DetachDatabase(ClientContext &context, const string &name, OnEntryNotFound if_not_found);
+	//! Rollback the attach of a database
+	shared_ptr<AttachedDatabase> DetachInternal(const string &name);
 	//! Returns a reference to the system catalog
 	Catalog &GetSystemCatalog();
 
@@ -58,9 +64,7 @@ public:
 	void SetDefaultDatabase(ClientContext &context, const string &new_value);
 
 	//! Inserts a path to name mapping to the database paths map
-	void InsertDatabasePath(ClientContext &context, const string &path, const string &name);
-	//! Erases a path from the database paths map
-	void EraseDatabasePath(const string &path);
+	InsertDatabasePathResult InsertDatabasePath(const AttachInfo &info, AttachOptions &options);
 
 	//! Returns the database type. This might require checking the header of the file, in which case the file handle is
 	//! necessary. We can only grab the file handle, if it is not yet held, even for uncommitted changes. Thus, we have
@@ -68,10 +72,12 @@ public:
 	void GetDatabaseType(ClientContext &context, AttachInfo &info, const DBConfig &config, AttachOptions &options);
 	//! Scans the catalog set and adds each committed database entry, and each database entry of the current
 	//! transaction, to a vector holding AttachedDatabase references
-	vector<reference<AttachedDatabase>> GetDatabases(ClientContext &context,
-	                                                 const optional_idx max_db_count = optional_idx());
+	vector<shared_ptr<AttachedDatabase>> GetDatabases(ClientContext &context,
+	                                                  const optional_idx max_db_count = optional_idx());
 	//! Scans the catalog set and returns each committed database entry
-	vector<reference<AttachedDatabase>> GetDatabases();
+	vector<shared_ptr<AttachedDatabase>> GetDatabases();
+	//! Returns the approximate count of attached databases.
+	idx_t ApproxDatabaseCount();
 	//! Removes all databases from the catalog set. This is necessary for the database instance's destructor,
 	//! as the database manager has to be alive when destroying the catalog set objects.
 	void ResetDatabases(unique_ptr<TaskScheduler> &scheduler);
@@ -97,16 +103,13 @@ public:
 	//! Gets a list of all attached database paths
 	vector<string> GetAttachedDatabasePaths();
 
-private:
-	//! Returns a database with a specified path
-	optional_ptr<AttachedDatabase> GetDatabaseFromPath(ClientContext &context, const string &path);
-	void CheckPathConflict(ClientContext &context, const string &path);
-
 private:
 	//! The system database is a special database that holds system entries (e.g. functions)
-	unique_ptr<AttachedDatabase> system;
+	shared_ptr<AttachedDatabase> system;
+	//! Lock for databases
+	mutex databases_lock;
 	//! The set of attached databases
-	unique_ptr<CatalogSet> databases;
+	case_insensitive_map_t<shared_ptr<AttachedDatabase>> databases;
 	//! The next object id handed out by the NextOid method
 	atomic<idx_t> next_oid;
 	//! The current query number
@@ -115,13 +118,8 @@ private:
 	atomic<transaction_t> current_transaction_id;
 	//! The current default database
 	string default_database;
-
-	//! The lock to add entries to the database path map
-	mutex db_paths_lock;
-	//! A set containing all attached database path
-	//! This allows to attach many databases efficiently, and to avoid attaching the
-	//! same file path twice
-	case_insensitive_set_t db_paths;
+	//! Manager for ensuring we never open the same database file twice in the same program
+	shared_ptr<DatabaseFilePathManager> path_manager;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/database_path_and_type.hpp

@@ -13,11 +13,13 @@
 
 namespace duckdb {
 
+class QueryContext;
+
 struct DBPathAndType {
 	//! Parse database extension type and rest of path from combined form (type:path)
 	static void ExtractExtensionPrefix(string &path, string &db_type);
 	//! Check the magic bytes of a file and set the database type based on that
-	static void CheckMagicBytes(FileSystem &fs, string &path, string &db_type);
+	static void CheckMagicBytes(QueryContext context, FileSystem &fs, string &path, string &db_type);
 
 	//! Run ExtractExtensionPrefix followed by CheckMagicBytes
 	static void ResolveDatabaseType(FileSystem &fs, string &path, string &db_type);

package/src/duckdb/src/include/duckdb/main/db_instance_cache.hpp

@@ -11,11 +11,11 @@
 #include "duckdb/main/connection_manager.hpp"
 #include "duckdb/main/database.hpp"
 #include "duckdb/common/unordered_map.hpp"
-#include "duckdb/function/replacement_scan.hpp"
 #include <functional>
 
 namespace duckdb {
 class DBInstanceCache;
+class DatabaseFilePathManager;
 
 struct DatabaseCacheEntry {
 	DatabaseCacheEntry();
@@ -23,12 +23,13 @@ struct DatabaseCacheEntry {
 	~DatabaseCacheEntry();
 
 	weak_ptr<DuckDB> database;
+	mutex update_database_mutex;
 };
 
 class DBInstanceCache {
 public:
-	DBInstanceCache() {
-	}
+	DBInstanceCache();
+	~DBInstanceCache();
 
 	//! Gets a DB Instance from the cache if already exists (Fails if the configurations do not match)
 	shared_ptr<DuckDB> GetInstance(const string &database, const DBConfig &config_dict);
@@ -42,6 +43,7 @@ public:
 	                                       const std::function<void(DuckDB &)> &on_create = nullptr);
 
 private:
+	shared_ptr<DatabaseFilePathManager> path_manager;
 	//! A map with the cached instances <absolute_path/instance>
 	unordered_map<string, weak_ptr<DatabaseCacheEntry>> db_instances;
 
@@ -49,8 +51,10 @@ private:
 	mutex cache_lock;
 
 private:
-	shared_ptr<DuckDB> GetInstanceInternal(const string &database, const DBConfig &config_dict);
+	shared_ptr<DuckDB> GetInstanceInternal(const string &database, const DBConfig &config,
+	                                       std::unique_lock<std::mutex> &db_instances_lock);
 	shared_ptr<DuckDB> CreateInstanceInternal(const string &database, DBConfig &config_dict, bool cache_instance,
+	                                          std::unique_lock<std::mutex> db_instances_lock,
 	                                          const std::function<void(DuckDB &)> &on_create);
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/error_manager.hpp

@@ -13,6 +13,7 @@
 #include "duckdb/common/map.hpp"
 
 namespace duckdb {
+class String;
 class ClientContext;
 class DatabaseInstance;
 class TransactionException;
@@ -52,7 +53,8 @@ public:
 		return Get(context).FormatException(error_type, params...);
 	}
 
-	DUCKDB_API static InvalidInputException InvalidUnicodeError(const string &input, const string &context);
+	DUCKDB_API static InvalidInputException InvalidUnicodeError(const String &input, const string &context);
+
 	DUCKDB_API static FatalException InvalidatedDatabase(ClientContext &context, const string &invalidated_msg);
 	DUCKDB_API static TransactionException InvalidatedTransaction(ClientContext &context);