duckdb 1.3.3-dev4.0 → 1.3.3

package/src/duckdb/src/include/duckdb/main/config.hpp

@@ -16,11 +16,11 @@
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/encryption_state.hpp"
 #include "duckdb/common/enums/access_mode.hpp"
-#include "duckdb/common/enums/thread_pin_mode.hpp"
 #include "duckdb/common/enums/compression_type.hpp"
 #include "duckdb/common/enums/optimizer_type.hpp"
 #include "duckdb/common/enums/order_type.hpp"
 #include "duckdb/common/enums/set_scope.hpp"
+#include "duckdb/common/enums/window_aggregation_mode.hpp"
 #include "duckdb/common/file_system.hpp"
 #include "duckdb/common/set.hpp"
 #include "duckdb/common/types/value.hpp"
@@ -29,14 +29,16 @@
 #include "duckdb/execution/index/index_type_set.hpp"
 #include "duckdb/function/cast/default_casts.hpp"
 #include "duckdb/function/replacement_scan.hpp"
+#include "duckdb/main/client_properties.hpp"
 #include "duckdb/optimizer/optimizer_extension.hpp"
 #include "duckdb/parser/parsed_data/create_info.hpp"
 #include "duckdb/parser/parser_extension.hpp"
 #include "duckdb/planner/operator_extension.hpp"
 #include "duckdb/storage/compression/bitpacking.hpp"
 #include "duckdb/function/encoding_function.hpp"
-#include "duckdb/main/setting_info.hpp"
 #include "duckdb/logging/log_manager.hpp"
+#include "duckdb/common/enums/debug_vector_verification.hpp"
+#include "duckdb/logging/logging.hpp"
 
 namespace duckdb {
 
@@ -55,12 +57,50 @@ class SecretManager;
 class CompressionInfo;
 class EncryptionUtil;
 class HTTPUtil;
-class DatabaseFilePathManager;
 
 struct CompressionFunctionSet;
 struct DatabaseCacheEntry;
 struct DBConfig;
-struct SettingLookupResult;
+
+enum class CheckpointAbort : uint8_t {
+	NO_ABORT = 0,
+	DEBUG_ABORT_BEFORE_TRUNCATE = 1,
+	DEBUG_ABORT_BEFORE_HEADER = 2,
+	DEBUG_ABORT_AFTER_FREE_LIST_WRITE = 3
+};
+
+typedef void (*set_global_function_t)(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+typedef void (*set_local_function_t)(ClientContext &context, const Value &parameter);
+typedef void (*reset_global_function_t)(DatabaseInstance *db, DBConfig &config);
+typedef void (*reset_local_function_t)(ClientContext &context);
+typedef Value (*get_setting_function_t)(const ClientContext &context);
+
+struct ConfigurationOption {
+	const char *name;
+	const char *description;
+	const char *parameter_type;
+	set_global_function_t set_global;
+	set_local_function_t set_local;
+	reset_global_function_t reset_global;
+	reset_local_function_t reset_local;
+	get_setting_function_t get_setting;
+};
+
+typedef void (*set_option_callback_t)(ClientContext &context, SetScope scope, Value &parameter);
+
+struct ExtensionOption {
+	// NOLINTNEXTLINE: work around bug in clang-tidy
+	ExtensionOption(string description_p, LogicalType type_p, set_option_callback_t set_function_p,
+	                Value default_value_p)
+	    : description(std::move(description_p)), type(std::move(type_p)), set_function(set_function_p),
+	      default_value(std::move(default_value_p)) {
+	}
+
+	string description;
+	LogicalType type;
+	set_option_callback_t set_function;
+	Value default_value;
+};
 
 class SerializationCompatibility {
 public:
@@ -134,8 +174,12 @@ struct DBConfigOptions {
 	bool buffer_manager_track_eviction_timestamps = false;
 	//! Whether or not to allow printing unredacted secrets
 	bool allow_unredacted_secrets = false;
-	//! Disables invalidating the database instance when encountering a fatal error.
-	bool disable_database_invalidation = false;
+	//! The collation type of the database
+	string collation = string();
+	//! The order type used when none is specified (default: ASC)
+	OrderType default_order_type = OrderType::ASCENDING;
+	//! NULL ordering used when none is specified (default: NULLS LAST)
+	DefaultOrderByNullType default_null_order = DefaultOrderByNullType::NULLS_LAST;
 	//! enable COPY and related commands
 	bool enable_external_access = true;
 	//! Whether or not the global http metadata cache is used
@@ -152,6 +196,8 @@ struct DBConfigOptions {
 	bool checkpoint_on_shutdown = true;
 	//! Serialize the metadata on checkpoint with compatibility for a given DuckDB version.
 	SerializationCompatibility serialization_compatibility = SerializationCompatibility::Default();
+	//! Debug flag that decides when a checkpoing should be aborted. Only used for testing purposes.
+	CheckpointAbort checkpoint_abort = CheckpointAbort::NO_ABORT;
 	//! Initialize the database with the standard set of DuckDB functions
 	//! You should probably not touch this unless you know what you are doing
 	bool initialize_default_database = true;
@@ -165,6 +211,19 @@ struct DBConfigOptions {
 	set<CompressionType> disabled_compression_methods;
 	//! Force a specific bitpacking mode to be used when using the bitpacking compression method
 	BitpackingMode force_bitpacking_mode = BitpackingMode::AUTO;
+	//! Debug setting for window aggregation mode: (window, combine, separate)
+	WindowAggregationMode window_mode = WindowAggregationMode::WINDOW;
+	//! Whether preserving insertion order should be preserved
+	bool preserve_insertion_order = true;
+	//! Whether Arrow Arrays use Large or Regular buffers
+	ArrowOffsetSize arrow_offset_size = ArrowOffsetSize::REGULAR;
+	//! Whether LISTs should produce Arrow ListViews
+	bool arrow_use_list_view = false;
+	//! Whenever a DuckDB type does not have a clear native or canonical extension match in Arrow, export the types
+	//! with a duckdb.type_name extension name
+	bool arrow_lossless_conversion = false;
+	//! Whether when producing arrow objects we produce string_views or regular strings
+	bool produce_arrow_string_views = false;
 	//! Database configuration variables as controlled by SET
 	case_insensitive_map_t<Value> set_variables;
 	//! Database configuration variable default values;
@@ -175,6 +234,16 @@ struct DBConfigOptions {
 	bool allow_unsigned_extensions = false;
 	//! Whether community extensions should be loaded
 	bool allow_community_extensions = true;
+	//! Whether extensions with missing metadata should be loaded
+	bool allow_extensions_metadata_mismatch = false;
+	//! Enable emitting FSST Vectors
+	bool enable_fsst_vectors = false;
+	//! Enable VIEWs to create dependencies
+	bool enable_view_dependencies = false;
+	//! Enable macros to create dependencies
+	bool enable_macro_dependencies = false;
+	//! Start transactions immediately in all attached databases - instead of lazily when a database is referenced
+	bool immediate_transaction_mode = false;
 	//! Debug setting - how to initialize  blocks in the storage layer when allocating
 	DebugInitialize debug_initialize = DebugInitialize::NO_INITIALIZE;
 	//! The set of user-provided options
@@ -195,14 +264,30 @@ struct DBConfigOptions {
 	string duckdb_api;
 	//! Metadata from DuckDB callers
 	string custom_user_agent;
-	//! Encrypt the temp files
-	bool temp_file_encryption = false;
+	//! Use old implicit casting style (i.e. allow everything to be implicitly casted to VARCHAR)
+	bool old_implicit_casting = false;
 	//! The default block allocation size for new duckdb database files (new as-in, they do not yet exist).
-	idx_t default_block_alloc_size = DEFAULT_BLOCK_ALLOC_SIZE;
+	idx_t default_block_alloc_size = DUCKDB_BLOCK_ALLOC_SIZE;
 	//! The default block header size for new duckdb database files.
 	idx_t default_block_header_size = DUCKDB_BLOCK_HEADER_STORAGE_SIZE;
 	//!  Whether or not to abort if a serialization exception is thrown during WAL playback (when reading truncated WAL)
 	bool abort_on_wal_failure = false;
+	//! The index_scan_percentage sets a threshold for index scans.
+	//! If fewer than MAX(index_scan_max_count, index_scan_percentage * total_row_count)
+	//! rows match, we perform an index scan instead of a table scan.
+	double index_scan_percentage = 0.001;
+	//! The index_scan_max_count sets a threshold for index scans.
+	//! If fewer than MAX(index_scan_max_count, index_scan_percentage * total_row_count)
+	//! rows match, we perform an index scan instead of a table scan.
+	idx_t index_scan_max_count = STANDARD_VECTOR_SIZE;
+	//! The maximum number of schemas we will look through for "did you mean..." style errors in the catalog
+	idx_t catalog_error_max_schemas = 100;
+	//!  Whether or not to always write to the WAL file, even if this is not required
+	bool debug_skip_checkpoint_on_commit = false;
+	//! Vector verification to enable (debug setting only)
+	DebugVectorVerification debug_verify_vector = DebugVectorVerification::NONE;
+	//! The maximum amount of vacuum tasks to schedule during a checkpoint
+	idx_t max_vacuum_tasks = 100;
 	//! Paths that are explicitly allowed, even if enable_external_access is false
 	unordered_set<string> allowed_paths;
 	//! Directories that are explicitly allowed, even if enable_external_access is false
@@ -211,14 +296,14 @@ struct DBConfigOptions {
 	LogConfig log_config = LogConfig();
 	//! Whether to enable external file caching using CachingFileSystem
 	bool enable_external_file_cache = true;
+	//! Output version of arrow depending on the format version
+	ArrowFormatVersion arrow_output_version = V1_0;
 	//! Partially process tasks before rescheduling - allows for more scheduler fairness between separate queries
 #ifdef DUCKDB_ALTERNATIVE_VERIFY
 	bool scheduler_process_partial = true;
 #else
 	bool scheduler_process_partial = false;
 #endif
-	//! Whether to pin threads to cores (linux only, default AUTOMATIC: on when there are more than 64 cores)
-	ThreadPinMode pin_threads = ThreadPinMode::AUTO;
 
 	bool operator==(const DBConfigOptions &other) const;
 };
@@ -233,7 +318,7 @@ public:
 	DUCKDB_API DBConfig(const case_insensitive_map_t<Value> &config_dict, bool read_only);
 	DUCKDB_API ~DBConfig();
 
-	mutable mutex config_lock;
+	mutex config_lock;
 	//! Replacement table scans are automatically attempted when a table name cannot be found in the schema
 	vector<ReplacementScan> replacement_scans;
 
@@ -272,8 +357,6 @@ public:
 	shared_ptr<HTTPUtil> http_util;
 	//! Reference to the database cache entry (if any)
 	shared_ptr<DatabaseCacheEntry> db_cache_entry;
-	//! Reference to the database file path manager
-	shared_ptr<DatabaseFilePathManager> path_manager;
 
 public:
 	DUCKDB_API static DBConfig &GetConfig(ClientContext &context);
@@ -282,32 +365,26 @@ public:
 	DUCKDB_API static const DBConfig &GetConfig(const ClientContext &context);
 	DUCKDB_API static const DBConfig &GetConfig(const DatabaseInstance &db);
 	DUCKDB_API static vector<ConfigurationOption> GetOptions();
-	DUCKDB_API static vector<ConfigurationAlias> GetAliases();
 	DUCKDB_API static idx_t GetOptionCount();
-	DUCKDB_API static idx_t GetAliasCount();
 	DUCKDB_API static vector<string> GetOptionNames();
 	DUCKDB_API static bool IsInMemoryDatabase(const char *database_path);
 
 	DUCKDB_API void AddExtensionOption(const string &name, string description, LogicalType parameter,
-	                                   const Value &default_value = Value(), set_option_callback_t function = nullptr,
-	                                   SetScope default_scope = SetScope::SESSION);
+	                                   const Value &default_value = Value(), set_option_callback_t function = nullptr);
 	//! Fetch an option by index. Returns a pointer to the option, or nullptr if out of range
 	DUCKDB_API static optional_ptr<const ConfigurationOption> GetOptionByIndex(idx_t index);
-	//! Fetcha n alias by index, or nullptr if out of range
-	DUCKDB_API static optional_ptr<const ConfigurationAlias> GetAliasByIndex(idx_t index);
 	//! Fetch an option by name. Returns a pointer to the option, or nullptr if none exists.
-	DUCKDB_API static optional_ptr<const ConfigurationOption> GetOptionByName(const String &name);
+	DUCKDB_API static optional_ptr<const ConfigurationOption> GetOptionByName(const string &name);
 	DUCKDB_API void SetOption(const ConfigurationOption &option, const Value &value);
-	DUCKDB_API void SetOption(optional_ptr<DatabaseInstance> db, const ConfigurationOption &option, const Value &value);
+	DUCKDB_API void SetOption(DatabaseInstance *db, const ConfigurationOption &option, const Value &value);
 	DUCKDB_API void SetOptionByName(const string &name, const Value &value);
 	DUCKDB_API void SetOptionsByName(const case_insensitive_map_t<Value> &values);
-	DUCKDB_API void ResetOption(optional_ptr<DatabaseInstance> db, const ConfigurationOption &option);
+	DUCKDB_API void ResetOption(DatabaseInstance *db, const ConfigurationOption &option);
 	DUCKDB_API void SetOption(const string &name, Value value);
-	DUCKDB_API void ResetOption(const String &name);
-	DUCKDB_API void ResetGenericOption(const String &name);
+	DUCKDB_API void ResetOption(const string &name);
 	static LogicalType ParseLogicalType(const string &type);
 
-	DUCKDB_API void CheckLock(const String &name);
+	DUCKDB_API void CheckLock(const string &name);
 
 	DUCKDB_API static idx_t ParseMemoryLimit(const string &arg);
 
@@ -341,28 +418,19 @@ public:
 	void SetDefaultMaxMemory();
 	void SetDefaultTempDirectory();
 
-	OrderType ResolveOrder(ClientContext &context, OrderType order_type) const;
-	OrderByNullType ResolveNullOrder(ClientContext &context, OrderType order_type, OrderByNullType null_type) const;
+	OrderType ResolveOrder(OrderType order_type) const;
+	OrderByNullType ResolveNullOrder(OrderType order_type, OrderByNullType null_type) const;
 	const string UserAgent() const;
 
-	SettingLookupResult TryGetCurrentSetting(const string &key, Value &result) const;
-
-	template <class OP, class SOURCE>
-	static typename std::enable_if<std::is_enum<typename OP::RETURN_TYPE>::value, typename OP::RETURN_TYPE>::type
-	GetSetting(const SOURCE &source) {
-		return EnumUtil::FromString<typename OP::RETURN_TYPE>(
-		    GetSettingInternal(source, OP::Name, OP::DefaultValue).ToString());
-	}
-
-	template <class OP, class SOURCE>
-	static typename std::enable_if<!std::is_enum<typename OP::RETURN_TYPE>::value, typename OP::RETURN_TYPE>::type
-	GetSetting(const SOURCE &source) {
-		return GetSettingInternal(source, OP::Name, OP::DefaultValue).template GetValue<typename OP::RETURN_TYPE>();
+	template <class OP>
+	typename OP::RETURN_TYPE GetSetting(const ClientContext &context) {
+		std::lock_guard<mutex> lock(config_lock);
+		return OP::GetSetting(context).template GetValue<typename OP::RETURN_TYPE>();
 	}
 
 	template <class OP>
-	Value GetSettingValue(const ClientContext &context) const {
-		lock_guard<mutex> lock(config_lock);
+	Value GetSettingValue(const ClientContext &context) {
+		std::lock_guard<mutex> lock(config_lock);
 		return OP::GetSetting(context);
 	}
 
@@ -371,11 +439,6 @@ public:
 	void AddAllowedPath(const string &path);
 	string SanitizeAllowedPath(const string &path) const;
 
-private:
-	static Value GetSettingInternal(const DatabaseInstance &db, const char *setting, const char *default_value);
-	static Value GetSettingInternal(const DBConfig &config, const char *setting, const char *default_value);
-	static Value GetSettingInternal(const ClientContext &context, const char *setting, const char *default_value);
-
 private:
 	unique_ptr<CompressionFunctionSet> compression_functions;
 	unique_ptr<EncodingFunctionSet> encoding_functions;

package/src/duckdb/src/include/duckdb/main/database.hpp

@@ -12,9 +12,9 @@
 #include "duckdb/main/capi/extension_api.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/main/extension.hpp"
+#include "duckdb/main/extension_install_info.hpp"
+#include "duckdb/main/settings.hpp"
 #include "duckdb/main/valid_checker.hpp"
-#include "duckdb/main/extension/extension_loader.hpp"
-#include "duckdb/main/extension_manager.hpp"
 
 namespace duckdb {
 class BufferManager;
@@ -23,7 +23,6 @@ class StorageManager;
 class Catalog;
 class TransactionManager;
 class ConnectionManager;
-class ExtensionManager;
 class FileSystem;
 class TaskScheduler;
 class ObjectCache;
@@ -34,6 +33,12 @@ struct DatabaseCacheEntry;
 class LogManager;
 class ExternalFileCache;
 
+struct ExtensionInfo {
+	bool is_loaded;
+	unique_ptr<ExtensionInstallInfo> install_info;
+	unique_ptr<ExtensionLoadedInfo> load_info;
+};
+
 class DatabaseInstance : public enable_shared_from_this<DatabaseInstance> {
 	friend class DuckDB;
 
@@ -54,9 +59,9 @@ public:
 	DUCKDB_API TaskScheduler &GetScheduler();
 	DUCKDB_API ObjectCache &GetObjectCache();
 	DUCKDB_API ConnectionManager &GetConnectionManager();
-	DUCKDB_API ExtensionManager &GetExtensionManager();
 	DUCKDB_API ValidChecker &GetValidChecker();
 	DUCKDB_API LogManager &GetLogManager() const;
+	DUCKDB_API void SetExtensionLoaded(const string &extension_name, ExtensionInstallInfo &install_info);
 
 	DUCKDB_API const duckdb_ext_api_v1 GetExtensionAPIV1();
 
@@ -65,15 +70,16 @@ public:
 	DUCKDB_API static DatabaseInstance &GetDatabase(ClientContext &context);
 	DUCKDB_API static const DatabaseInstance &GetDatabase(const ClientContext &context);
 
+	DUCKDB_API const unordered_map<string, ExtensionInfo> &GetExtensions();
 	DUCKDB_API bool ExtensionIsLoaded(const string &name);
 
 	DUCKDB_API SettingLookupResult TryGetCurrentSetting(const string &key, Value &result) const;
 
-	DUCKDB_API shared_ptr<EncryptionUtil> GetEncryptionUtil() const;
-
-	shared_ptr<AttachedDatabase> CreateAttachedDatabase(ClientContext &context, AttachInfo &info,
+	unique_ptr<AttachedDatabase> CreateAttachedDatabase(ClientContext &context, AttachInfo &info,
 	                                                    AttachOptions &options);
 
+	void AddExtensionInfo(const string &name, const ExtensionLoadedInfo &info);
+
 private:
 	void Initialize(const char *path, DBConfig *config);
 	void LoadExtensionSettings();
@@ -87,7 +93,7 @@ private:
 	unique_ptr<TaskScheduler> scheduler;
 	unique_ptr<ObjectCache> object_cache;
 	unique_ptr<ConnectionManager> connection_manager;
-	unique_ptr<ExtensionManager> extension_manager;
+	unordered_map<string, ExtensionInfo> loaded_extensions_info;
 	ValidChecker db_validity;
 	unique_ptr<DatabaseFileSystem> db_file_system;
 	shared_ptr<LogManager> log_manager;
@@ -114,26 +120,24 @@ public:
 	template <class T>
 	void LoadStaticExtension() {
 		T extension;
-		auto &manager = ExtensionManager::Get(*instance);
-		auto info = manager.BeginLoad(extension.Name());
-		if (!info) {
-			// already loaded - return
+		if (ExtensionIsLoaded(extension.Name())) {
 			return;
 		}
-
-		// Instantiate a new loader
-		ExtensionLoader loader(*instance, extension.Name());
-
-		// Call the Load method of the extension
-		extension.Load(loader);
-
-		// Finalize the loading process
-		loader.FinalizeLoad();
-
+		extension.Load(*this);
 		ExtensionInstallInfo install_info;
 		install_info.mode = ExtensionInstallMode::STATICALLY_LINKED;
 		install_info.version = extension.Version();
-		info->FinishLoad(install_info);
+		instance->SetExtensionLoaded(extension.Name(), install_info);
+	}
+
+	// DEPRECATED function that some extensions may still use to call their own Load method from the
+	// _init function of their loadable extension. Don't use this. Instead opt for a static LoadInternal function called
+	// from both the _init function and the Extension::Load. (see autocomplete extension)
+	// TODO: when to remove this function?
+	template <class T>
+	void LoadExtension() {
+		T extension;
+		extension.Load(*this);
 	}
 
 	DUCKDB_API FileSystem &GetFileSystem();

package/src/duckdb/src/include/duckdb/main/database_file_opener.hpp

@@ -12,7 +12,6 @@
 #include "duckdb/common/opener_file_system.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/main/database.hpp"
-#include "duckdb/logging/log_manager.hpp"
 
 namespace duckdb {
 class DatabaseInstance;

package/src/duckdb/src/include/duckdb/main/database_manager.hpp

@@ -17,7 +17,6 @@
 #include "duckdb/common/optional_ptr.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/parser/parsed_data/attach_info.hpp"
-#include "duckdb/main/database_file_path_manager.hpp"
 
 namespace duckdb {
 class AttachedDatabase;
@@ -49,14 +48,9 @@ public:
 	//! Get an attached database by its name
 	optional_ptr<AttachedDatabase> GetDatabase(ClientContext &context, const string &name);
 	//! Attach a new database
-	shared_ptr<AttachedDatabase> AttachDatabase(ClientContext &context, AttachInfo &info, AttachOptions &options);
-
-	optional_ptr<AttachedDatabase> FinalizeAttach(ClientContext &context, AttachInfo &info,
-	                                              shared_ptr<AttachedDatabase> database);
+	optional_ptr<AttachedDatabase> AttachDatabase(ClientContext &context, AttachInfo &info, AttachOptions &options);
 	//! Detach an existing database
 	void DetachDatabase(ClientContext &context, const string &name, OnEntryNotFound if_not_found);
-	//! Rollback the attach of a database
-	shared_ptr<AttachedDatabase> DetachInternal(const string &name);
 	//! Returns a reference to the system catalog
 	Catalog &GetSystemCatalog();
 
@@ -64,7 +58,9 @@ public:
 	void SetDefaultDatabase(ClientContext &context, const string &new_value);
 
 	//! Inserts a path to name mapping to the database paths map
-	InsertDatabasePathResult InsertDatabasePath(const AttachInfo &info, AttachOptions &options);
+	void InsertDatabasePath(ClientContext &context, const string &path, const string &name);
+	//! Erases a path from the database paths map
+	void EraseDatabasePath(const string &path);
 
 	//! Returns the database type. This might require checking the header of the file, in which case the file handle is
 	//! necessary. We can only grab the file handle, if it is not yet held, even for uncommitted changes. Thus, we have
@@ -72,12 +68,10 @@ public:
 	void GetDatabaseType(ClientContext &context, AttachInfo &info, const DBConfig &config, AttachOptions &options);
 	//! Scans the catalog set and adds each committed database entry, and each database entry of the current
 	//! transaction, to a vector holding AttachedDatabase references
-	vector<shared_ptr<AttachedDatabase>> GetDatabases(ClientContext &context,
-	                                                  const optional_idx max_db_count = optional_idx());
+	vector<reference<AttachedDatabase>> GetDatabases(ClientContext &context,
+	                                                 const optional_idx max_db_count = optional_idx());
 	//! Scans the catalog set and returns each committed database entry
-	vector<shared_ptr<AttachedDatabase>> GetDatabases();
-	//! Returns the approximate count of attached databases.
-	idx_t ApproxDatabaseCount();
+	vector<reference<AttachedDatabase>> GetDatabases();
 	//! Removes all databases from the catalog set. This is necessary for the database instance's destructor,
 	//! as the database manager has to be alive when destroying the catalog set objects.
 	void ResetDatabases(unique_ptr<TaskScheduler> &scheduler);
@@ -103,13 +97,16 @@ public:
 	//! Gets a list of all attached database paths
 	vector<string> GetAttachedDatabasePaths();
 
+private:
+	//! Returns a database with a specified path
+	optional_ptr<AttachedDatabase> GetDatabaseFromPath(ClientContext &context, const string &path);
+	void CheckPathConflict(ClientContext &context, const string &path);
+
 private:
 	//! The system database is a special database that holds system entries (e.g. functions)
-	shared_ptr<AttachedDatabase> system;
-	//! Lock for databases
-	mutex databases_lock;
+	unique_ptr<AttachedDatabase> system;
 	//! The set of attached databases
-	case_insensitive_map_t<shared_ptr<AttachedDatabase>> databases;
+	unique_ptr<CatalogSet> databases;
 	//! The next object id handed out by the NextOid method
 	atomic<idx_t> next_oid;
 	//! The current query number
@@ -118,8 +115,13 @@ private:
 	atomic<transaction_t> current_transaction_id;
 	//! The current default database
 	string default_database;
-	//! Manager for ensuring we never open the same database file twice in the same program
-	shared_ptr<DatabaseFilePathManager> path_manager;
+
+	//! The lock to add entries to the database path map
+	mutex db_paths_lock;
+	//! A set containing all attached database path
+	//! This allows to attach many databases efficiently, and to avoid attaching the
+	//! same file path twice
+	case_insensitive_set_t db_paths;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/database_path_and_type.hpp

@@ -13,13 +13,11 @@
 
 namespace duckdb {
 
-class QueryContext;
-
 struct DBPathAndType {
 	//! Parse database extension type and rest of path from combined form (type:path)
 	static void ExtractExtensionPrefix(string &path, string &db_type);
 	//! Check the magic bytes of a file and set the database type based on that
-	static void CheckMagicBytes(QueryContext context, FileSystem &fs, string &path, string &db_type);
+	static void CheckMagicBytes(FileSystem &fs, string &path, string &db_type);
 
 	//! Run ExtractExtensionPrefix followed by CheckMagicBytes
 	static void ResolveDatabaseType(FileSystem &fs, string &path, string &db_type);

package/src/duckdb/src/include/duckdb/main/db_instance_cache.hpp

@@ -11,11 +11,11 @@
 #include "duckdb/main/connection_manager.hpp"
 #include "duckdb/main/database.hpp"
 #include "duckdb/common/unordered_map.hpp"
+#include "duckdb/function/replacement_scan.hpp"
 #include <functional>
 
 namespace duckdb {
 class DBInstanceCache;
-class DatabaseFilePathManager;
 
 struct DatabaseCacheEntry {
 	DatabaseCacheEntry();
@@ -23,13 +23,12 @@ struct DatabaseCacheEntry {
 	~DatabaseCacheEntry();
 
 	weak_ptr<DuckDB> database;
-	mutex update_database_mutex;
 };
 
 class DBInstanceCache {
 public:
-	DBInstanceCache();
-	~DBInstanceCache();
+	DBInstanceCache() {
+	}
 
 	//! Gets a DB Instance from the cache if already exists (Fails if the configurations do not match)
 	shared_ptr<DuckDB> GetInstance(const string &database, const DBConfig &config_dict);
@@ -43,7 +42,6 @@ public:
 	                                       const std::function<void(DuckDB &)> &on_create = nullptr);
 
 private:
-	shared_ptr<DatabaseFilePathManager> path_manager;
 	//! A map with the cached instances <absolute_path/instance>
 	unordered_map<string, weak_ptr<DatabaseCacheEntry>> db_instances;
 
@@ -51,10 +49,8 @@ private:
 	mutex cache_lock;
 
 private:
-	shared_ptr<DuckDB> GetInstanceInternal(const string &database, const DBConfig &config,
-	                                       std::unique_lock<std::mutex> &db_instances_lock);
+	shared_ptr<DuckDB> GetInstanceInternal(const string &database, const DBConfig &config_dict);
 	shared_ptr<DuckDB> CreateInstanceInternal(const string &database, DBConfig &config_dict, bool cache_instance,
-	                                          std::unique_lock<std::mutex> db_instances_lock,
 	                                          const std::function<void(DuckDB &)> &on_create);
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/error_manager.hpp

@@ -13,7 +13,6 @@
 #include "duckdb/common/map.hpp"
 
 namespace duckdb {
-class String;
 class ClientContext;
 class DatabaseInstance;
 class TransactionException;
@@ -53,8 +52,7 @@ public:
 		return Get(context).FormatException(error_type, params...);
 	}
 
-	DUCKDB_API static InvalidInputException InvalidUnicodeError(const String &input, const string &context);
-
+	DUCKDB_API static InvalidInputException InvalidUnicodeError(const string &input, const string &context);
 	DUCKDB_API static FatalException InvalidatedDatabase(ClientContext &context, const string &invalidated_msg);
 	DUCKDB_API static TransactionException InvalidatedTransaction(ClientContext &context);
 

package/src/duckdb/src/include/duckdb/main/extension.hpp

@@ -12,14 +12,14 @@
 #include "duckdb/common/winapi.hpp"
 
 namespace duckdb {
-class ExtensionLoader;
+class DuckDB;
 
 //! The Extension class is the base class used to define extensions
 class Extension {
 public:
 	DUCKDB_API virtual ~Extension();
 
-	DUCKDB_API virtual void Load(ExtensionLoader &db) = 0;
+	DUCKDB_API virtual void Load(DuckDB &db) = 0;
 	DUCKDB_API virtual std::string Name() = 0;
 	DUCKDB_API virtual std::string Version() const {
 		return "";