duckdb 0.9.3-dev2.0 → 0.10.0

package/src/duckdb/src/include/duckdb/main/client_config.hpp

@@ -59,6 +59,8 @@ struct ClientConfig {
 	bool query_verification_enabled = false;
 	//! Whether or not verification of external operators is enabled, used for testing
 	bool verify_external = false;
+	//! Whether or not verification of fetch row code is enabled, used for testing
+	bool verify_fetch_row = false;
 	//! Whether or not we should verify the serializer
 	bool verify_serializer = false;
 	//! Enable the running of optimizers
@@ -67,16 +69,14 @@ struct ClientConfig {
 	bool enable_caching_operators = true;
 	//! Force parallelism of small tables, used for testing
 	bool verify_parallelism = false;
-	//! Enable the optimizer to consider index joins, which are disabled on default
-	bool enable_index_join = false;
-	//! Force index join independent of table cardinality, used for testing
-	bool force_index_join = false;
 	//! Force out-of-core computation for operators that support it, used for testing
 	bool force_external = false;
 	//! Force disable cross product generation when hyper graph isn't connected, used for testing
 	bool force_no_cross_product = false;
 	//! Force use of IEJoin to implement AsOfJoin, used for testing
 	bool force_asof_iejoin = false;
+	//! Force use of fetch row instead of scan, used for testing
+	bool force_fetch_row = false;
 	//! Use range joins for inequalities, even if there are equality predicates
 	bool prefer_range_joins = false;
 	//! If this context should also try to use the available replacement scans
@@ -91,11 +91,6 @@ struct ClientConfig {
 	//! Callback to create a progress bar display
 	progress_bar_display_create_func_t display_create_func = nullptr;
 
-	//! Override for the default extension repository
-	string custom_extension_repo = "";
-	//! Override for the default autoload extensoin repository
-	string autoinstall_extension_repo = "";
-
 	//! The explain output type used when none is specified (default: PHYSICAL_ONLY)
 	ExplainOutputType explain_output_type = ExplainOutputType::PHYSICAL_ONLY;
 
@@ -108,6 +103,9 @@ struct ClientConfig {
 	//! Whether or not the "/" division operator defaults to integer division or floating point division
 	bool integer_division = false;
 
+	//! Output error messages as structured JSON instead of as a raw string
+	bool errors_as_json = false;
+
 	//! Generic options
 	case_insensitive_map_t<Value> set_variables;
 
@@ -120,7 +118,7 @@ public:
 	static const ClientConfig &GetConfig(const ClientContext &context);
 
 	bool AnyVerification() {
-		return query_verification_enabled || verify_external || verify_serializer;
+		return query_verification_enabled || verify_external || verify_serializer || verify_fetch_row;
 	}
 };
 

package/src/duckdb/src/include/duckdb/main/client_context.hpp

@@ -23,8 +23,9 @@
 #include "duckdb/common/atomic.hpp"
 #include "duckdb/main/client_config.hpp"
 #include "duckdb/main/external_dependencies.hpp"
-#include "duckdb/common/preserved_error.hpp"
+#include "duckdb/common/error_data.hpp"
 #include "duckdb/main/client_properties.hpp"
+#include "duckdb/main/client_context_state.hpp"
 
 namespace duckdb {
 class Appender;
@@ -43,7 +44,9 @@ struct CreateScalarFunctionInfo;
 class ScalarFunctionCatalogEntry;
 struct ActiveQueryContext;
 struct ParserOptions;
+class SimpleBufferedData;
 struct ClientData;
+class ClientContextState;
 
 struct PendingQueryParameters {
 	//! Prepared statement parameters (if any)
@@ -52,20 +55,13 @@ struct PendingQueryParameters {
 	bool allow_stream_result = false;
 };
 
-//! ClientContextState is virtual base class for ClientContext-local (or Query-Local, using QueryEnd callback) state
-//! e.g. caches that need to live as long as a ClientContext or Query.
-class ClientContextState {
-public:
-	virtual ~ClientContextState() {};
-	virtual void QueryEnd() = 0;
-};
-
 //! The ClientContext holds information relevant to the current client session
 //! during execution
 class ClientContext : public std::enable_shared_from_this<ClientContext> {
-	friend class PendingQueryResult;
-	friend class StreamQueryResult;
-	friend class DuckTransactionManager;
+	friend class PendingQueryResult; // LockContext
+	friend class SimpleBufferedData; // ExecuteTaskInternal
+	friend class StreamQueryResult;  // LockContext
+	friend class ConnectionManager;
 
 public:
 	DUCKDB_API explicit ClientContext(shared_ptr<DatabaseInstance> db);
@@ -148,7 +144,7 @@ public:
 	                                           const PendingQueryParameters &parameters);
 
 	//! Gets current percentage of the query's progress, returns 0 in case the progress bar is disabled.
-	DUCKDB_API double GetProgress();
+	DUCKDB_API QueryProgress GetQueryProgress();
 
 	//! Register function in the temporary schema
 	DUCKDB_API void RegisterFunction(CreateFunctionInfo &info);
@@ -174,10 +170,9 @@ public:
 	//! Returns the parser options for this client context
 	DUCKDB_API ParserOptions GetParserOptions() const;
 
-	DUCKDB_API unique_ptr<DataChunk> Fetch(ClientContextLock &lock, StreamQueryResult &result);
-
 	//! Whether or not the given result object (streaming query result or pending query result) is active
-	DUCKDB_API bool IsActiveResult(ClientContextLock &lock, BaseQueryResult *result);
+	DUCKDB_API bool IsActiveResult(ClientContextLock &lock, BaseQueryResult &result);
+	DUCKDB_API void SetActiveResult(ClientContextLock &lock, BaseQueryResult &result);
 
 	//! Returns the current executor
 	Executor &GetExecutor();
@@ -193,10 +188,13 @@ public:
 	//! Returns true if execution of the current query is finished
 	DUCKDB_API bool ExecutionIsFinished();
 
+	//! Process an error for display to the user
+	DUCKDB_API void ProcessError(ErrorData &error, const string &query) const;
+
 private:
 	//! Parse statements and resolve pragmas from a query
 	bool ParseStatements(ClientContextLock &lock, const string &query, vector<unique_ptr<SQLStatement>> &result,
-	                     PreservedError &error);
+	                     ErrorData &error);
 	//! Issues a query to the database and returns a Pending Query Result
 	unique_ptr<PendingQueryResult> PendingQueryInternal(ClientContextLock &lock, unique_ptr<SQLStatement> statement,
 	                                                    const PendingQueryParameters &parameters, bool verify = true);
@@ -206,7 +204,7 @@ private:
 	vector<unique_ptr<SQLStatement>> ParseStatementsInternal(ClientContextLock &lock, const string &query);
 	//! Perform aggressive query verification of a SELECT statement. Only called when query_verification_enabled is
 	//! true.
-	PreservedError VerifyQuery(ClientContextLock &lock, const string &query, unique_ptr<SQLStatement> statement);
+	ErrorData VerifyQuery(ClientContextLock &lock, const string &query, unique_ptr<SQLStatement> statement);
 
 	void InitialCleanup(ClientContextLock &lock);
 	//! Internal clean up, does not lock. Caller must hold the context_lock.
@@ -234,15 +232,13 @@ private:
 	void LogQueryInternal(ClientContextLock &lock, const string &query);
 
 	unique_ptr<QueryResult> FetchResultInternal(ClientContextLock &lock, PendingQueryResult &pending);
-	unique_ptr<DataChunk> FetchInternal(ClientContextLock &lock, Executor &executor, BaseQueryResult &result);
 
 	unique_ptr<ClientContextLock> LockContext();
 
-	void BeginTransactionInternal(ClientContextLock &lock, bool requires_valid_transaction);
 	void BeginQueryInternal(ClientContextLock &lock, const string &query);
-	PreservedError EndQueryInternal(ClientContextLock &lock, bool success, bool invalidate_transaction);
+	ErrorData EndQueryInternal(ClientContextLock &lock, bool success, bool invalidate_transaction);
 
-	PendingExecutionResult ExecuteTaskInternal(ClientContextLock &lock, PendingQueryResult &result);
+	PendingExecutionResult ExecuteTaskInternal(ClientContextLock &lock, BaseQueryResult &result, bool dry_run = false);
 
 	unique_ptr<PendingQueryResult> PendingStatementOrPreparedStatementInternal(
 	    ClientContextLock &lock, const string &query, unique_ptr<SQLStatement> statement,
@@ -255,13 +251,16 @@ private:
 	unique_ptr<PendingQueryResult> PendingQueryInternal(ClientContextLock &, const shared_ptr<Relation> &relation,
 	                                                    bool allow_stream_result);
 
+	template <class T>
+	unique_ptr<T> ErrorResult(ErrorData error, const string &query = string());
+
 private:
 	//! Lock on using the ClientContext in parallel
 	mutex context_lock;
 	//! The currently active query context
 	unique_ptr<ActiveQueryContext> active_query;
 	//! The current query progress
-	atomic<double> query_progress;
+	QueryProgress query_progress;
 };
 
 class ClientContextLock {

package/src/duckdb/src/include/duckdb/main/client_context_state.hpp

@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/main/client_context_state.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+
+namespace duckdb {
+class ClientContext;
+class MetaTransaction;
+
+//! ClientContextState is virtual base class for ClientContext-local (or Query-Local, using QueryEnd callback) state
+//! e.g. caches that need to live as long as a ClientContext or Query.
+class ClientContextState {
+public:
+	virtual ~ClientContextState() = default;
+	virtual void QueryBegin(ClientContext &context) {
+	}
+	virtual void QueryEnd() {
+	}
+	virtual void QueryEnd(ClientContext &context) {
+		QueryEnd();
+	}
+	virtual void TransactionBegin(MetaTransaction &transaction, ClientContext &context) {
+	}
+	virtual void TransactionCommit(MetaTransaction &transaction, ClientContext &context) {
+	}
+	virtual void TransactionRollback(MetaTransaction &transaction, ClientContext &context) {
+	}
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/main/client_data.hpp

@@ -13,7 +13,7 @@
 #include "duckdb/common/types/value.hpp"
 #include "duckdb/common/case_insensitive_map.hpp"
 #include "duckdb/common/atomic.hpp"
-#include "duckdb/execution/operator/scan/csv/csv_state_machine_cache.hpp"
+#include "duckdb/execution/operator/csv_scanner/state_machine/csv_state_machine_cache.hpp"
 
 namespace duckdb {
 class AttachedDatabase;
@@ -24,7 +24,6 @@ class FileOpener;
 class FileSystem;
 class HTTPState;
 class QueryProfiler;
-class QueryProfilerHistory;
 class PreparedStatementData;
 class SchemaCatalogEntry;
 struct RandomEngine;
@@ -35,8 +34,6 @@ struct ClientData {
 
 	//! Query profiler
 	shared_ptr<QueryProfiler> profiler;
-	//! QueryProfiler History
-	unique_ptr<QueryProfilerHistory> query_profiler_history;
 
 	//! The set of temporary objects that belong to this client
 	shared_ptr<AttachedDatabase> temporary_objects;
@@ -54,9 +51,6 @@ struct ClientData {
 	//! The file opener of the client context
 	unique_ptr<FileOpener> file_opener;
 
-	//! HTTP State in this query
-	shared_ptr<HTTPState> http_state;
-
 	//! The clients' file system wrapper
 	unique_ptr<FileSystem> client_file_system;
 

package/src/duckdb/src/include/duckdb/main/config.hpp

@@ -8,10 +8,10 @@
 
 #pragma once
 
-#include "duckdb/common/enums/access_mode.hpp"
 #include "duckdb/common/allocator.hpp"
 #include "duckdb/common/case_insensitive_map.hpp"
 #include "duckdb/common/common.hpp"
+#include "duckdb/common/enums/access_mode.hpp"
 #include "duckdb/common/enums/compression_type.hpp"
 #include "duckdb/common/enums/optimizer_type.hpp"
 #include "duckdb/common/enums/order_type.hpp"
@@ -22,15 +22,18 @@
 #include "duckdb/common/types/value.hpp"
 #include "duckdb/common/vector.hpp"
 #include "duckdb/common/winapi.hpp"
-#include "duckdb/storage/compression/bitpacking.hpp"
 #include "duckdb/function/cast/default_casts.hpp"
 #include "duckdb/function/replacement_scan.hpp"
 #include "duckdb/optimizer/optimizer_extension.hpp"
+#include "duckdb/parser/parsed_data/create_info.hpp"
 #include "duckdb/parser/parser_extension.hpp"
 #include "duckdb/planner/operator_extension.hpp"
+#include "duckdb/storage/compression/bitpacking.hpp"
 #include "duckdb/main/client_properties.hpp"
+#include "duckdb/execution/index/index_type_set.hpp"
 
 namespace duckdb {
+
 class BufferPool;
 class CastFunctionSet;
 class ClientContext;
@@ -40,6 +43,7 @@ class TableFunctionRef;
 class OperatorExtension;
 class StorageExtension;
 class ExtensionCallback;
+class SecretManager;
 
 struct CompressionFunctionSet;
 struct DBConfig;
@@ -108,16 +112,23 @@ struct DBConfigOptions {
 #else
 	bool autoinstall_known_extensions = false;
 #endif
+	//! Override for the default extension repository
+	string custom_extension_repo = "";
+	//! Override for the default autoload extensoin repository
+	string autoinstall_extension_repo = "";
 	//! The maximum memory used by the database system (in bytes). Default: 80% of System available memory
 	idx_t maximum_memory = (idx_t)-1;
 	//! The maximum amount of CPU threads used by the database system. Default: all available.
 	idx_t maximum_threads = (idx_t)-1;
-	//! The number of external threads that work on DuckDB tasks. Default: none.
-	idx_t external_threads = 0;
+	//! The number of external threads that work on DuckDB tasks. Default: 1.
+	//! Must be smaller or equal to maximum_threads.
+	idx_t external_threads = 1;
 	//! Whether or not to create and use a temporary directory to store intermediates that do not fit in memory
 	bool use_temporary_directory = true;
 	//! Directory to store temporary structures that do not fit in memory
 	string temporary_directory;
+	//! Whether or not to allow printing unredacted secrets
+	bool allow_unredacted_secrets = false;
 	//! The collation type of the database
 	string collation = string();
 	//! The order type used when none is specified (default: ASC)
@@ -177,6 +188,8 @@ struct DBConfigOptions {
 	string duckdb_api;
 	//! Metadata from DuckDB callers
 	string custom_user_agent;
+	//! Use old implicit casting style (i.e. allow everything to be implicitly casted to VARCHAR)
+	bool old_implicit_casting = false;
 
 	bool operator==(const DBConfigOptions &other) const;
 };
@@ -187,7 +200,8 @@ struct DBConfig {
 
 public:
 	DUCKDB_API DBConfig();
-	DUCKDB_API DBConfig(std::unordered_map<string, string> &config_dict, bool read_only);
+	DUCKDB_API DBConfig(bool read_only);
+	DUCKDB_API DBConfig(const case_insensitive_map_t<Value> &config_dict, bool read_only);
 	DUCKDB_API ~DBConfig();
 
 	mutex config_lock;
@@ -199,6 +213,8 @@ public:
 	//! The FileSystem to use, can be overwritten to allow for injecting custom file systems for testing purposes (e.g.
 	//! RamFS or something similar)
 	unique_ptr<FileSystem> file_system;
+	//! Secret manager
+	unique_ptr<SecretManager> secret_manager;
 	//! The allocator used by the system
 	unique_ptr<Allocator> allocator;
 	//! Database configuration options
@@ -236,10 +252,10 @@ public:
 	DUCKDB_API static ConfigurationOption *GetOptionByIndex(idx_t index);
 	//! Fetch an option by name. Returns a pointer to the option, or nullptr if none exists.
 	DUCKDB_API static ConfigurationOption *GetOptionByName(const string &name);
-
 	DUCKDB_API void SetOption(const ConfigurationOption &option, const Value &value);
 	DUCKDB_API void SetOption(DatabaseInstance *db, const ConfigurationOption &option, const Value &value);
 	DUCKDB_API void SetOptionByName(const string &name, const Value &value);
+	DUCKDB_API void SetOptionsByName(const case_insensitive_map_t<Value> &values);
 	DUCKDB_API void ResetOption(DatabaseInstance *db, const ConfigurationOption &option);
 	DUCKDB_API void SetOption(const string &name, Value value);
 	DUCKDB_API void ResetOption(const string &name);
@@ -257,8 +273,8 @@ public:
 	bool operator!=(const DBConfig &other);
 
 	DUCKDB_API CastFunctionSet &GetCastFunctions();
+	DUCKDB_API IndexTypeSet &GetIndexTypes();
 	static idx_t GetSystemMaxThreads(FileSystem &fs);
-	void SetDefaultMaxThreads();
 	void SetDefaultMaxMemory();
 
 	OrderType ResolveOrder(OrderType order_type) const;
@@ -268,6 +284,7 @@ public:
 private:
 	unique_ptr<CompressionFunctionSet> compression_functions;
 	unique_ptr<CastFunctionSet> cast_functions;
+	unique_ptr<IndexTypeSet> index_types;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/connection.hpp

@@ -40,6 +40,12 @@ class Connection {
 public:
 	DUCKDB_API explicit Connection(DuckDB &database);
 	DUCKDB_API explicit Connection(DatabaseInstance &database);
+	// disable copy constructors
+	Connection(const Connection &other) = delete;
+	Connection &operator=(const Connection &) = delete;
+	//! enable move constructors
+	DUCKDB_API Connection(Connection &&other) noexcept;
+	DUCKDB_API Connection &operator=(Connection &&) noexcept;
 	DUCKDB_API ~Connection();
 
 	shared_ptr<ClientContext> context;
@@ -131,7 +137,8 @@ public:
 
 	//! Reads CSV file
 	DUCKDB_API shared_ptr<Relation> ReadCSV(const string &csv_file);
-	DUCKDB_API shared_ptr<Relation> ReadCSV(const string &csv_file, named_parameter_map_t &&options);
+	DUCKDB_API shared_ptr<Relation> ReadCSV(const vector<string> &csv_input, named_parameter_map_t &&options);
+	DUCKDB_API shared_ptr<Relation> ReadCSV(const string &csv_input, named_parameter_map_t &&options);
 	DUCKDB_API shared_ptr<Relation> ReadCSV(const string &csv_file, const vector<string> &columns);
 
 	//! Reads Parquet file

package/src/duckdb/src/include/duckdb/main/connection_manager.hpp

@@ -17,37 +17,25 @@ namespace duckdb {
 class ClientContext;
 class DatabaseInstance;
 
-class ConnectionManager {
-public:
-	ConnectionManager() {
+struct ClientLockWrapper {
+	ClientLockWrapper(mutex &client_lock, shared_ptr<ClientContext> connection)
+	    : connection(std::move(connection)), connection_lock(make_uniq<lock_guard<mutex>>(client_lock)) {
 	}
 
-	void AddConnection(ClientContext &context) {
-		lock_guard<mutex> lock(connections_lock);
-		connections.insert(make_pair(&context, weak_ptr<ClientContext>(context.shared_from_this())));
-	}
+	shared_ptr<ClientContext> connection;
+	unique_ptr<lock_guard<mutex>> connection_lock;
+};
 
-	void RemoveConnection(ClientContext &context) {
-		lock_guard<mutex> lock(connections_lock);
-		connections.erase(&context);
-	}
+class ConnectionManager {
+public:
+	ConnectionManager();
 
-	vector<shared_ptr<ClientContext>> GetConnectionList() {
-		vector<shared_ptr<ClientContext>> result;
-		for (auto &it : connections) {
-			auto connection = it.second.lock();
-			if (!connection) {
-				connections.erase(it.first);
-				continue;
-			} else {
-				result.push_back(std::move(connection));
-			}
-		}
-
-		return result;
-	}
+	void AddConnection(ClientContext &context);
+	void RemoveConnection(ClientContext &context);
 
-	ClientContext *GetConnection(DatabaseInstance *db);
+	vector<shared_ptr<ClientContext>> GetConnectionList();
+
+	void LockClients(vector<ClientLockWrapper> &client_locks, ClientContext &context);
 
 	static ConnectionManager &Get(DatabaseInstance &db);
 	static ConnectionManager &Get(ClientContext &context);
@@ -55,6 +43,9 @@ public:
 public:
 	mutex connections_lock;
 	unordered_map<ClientContext *, weak_ptr<ClientContext>> connections;
+
+	mutex lock_clients_lock;
+	bool is_locking;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/database.hpp

@@ -35,7 +35,8 @@ public:
 	DBConfig config;
 
 public:
-	BufferPool &GetBufferPool();
+	BufferPool &GetBufferPool() const;
+	DUCKDB_API SecretManager &GetSecretManager();
 	DUCKDB_API BufferManager &GetBufferManager();
 	DUCKDB_API DatabaseManager &GetDatabaseManager();
 	DUCKDB_API FileSystem &GetFileSystem();
@@ -54,7 +55,8 @@ public:
 
 	DUCKDB_API bool TryGetCurrentSetting(const std::string &key, Value &result);
 
-	unique_ptr<AttachedDatabase> CreateAttachedDatabase(AttachInfo &info, const string &type, AccessMode access_mode);
+	unique_ptr<AttachedDatabase> CreateAttachedDatabase(ClientContext &context, const AttachInfo &info,
+	                                                    const string &type, AccessMode access_mode);
 
 private:
 	void Initialize(const char *path, DBConfig *config);

package/src/duckdb/src/include/duckdb/main/database_manager.hpp

@@ -8,12 +8,15 @@
 
 #pragma once
 
-#include "duckdb/common/common.hpp"
+#include "duckdb/common/atomic.hpp"
 #include "duckdb/common/case_insensitive_map.hpp"
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/enums/access_mode.hpp"
+#include "duckdb/common/enums/on_entry_not_found.hpp"
 #include "duckdb/common/mutex.hpp"
-#include "duckdb/common/atomic.hpp"
 #include "duckdb/common/optional_ptr.hpp"
-#include "duckdb/common/enums/on_entry_not_found.hpp"
+#include "duckdb/main/config.hpp"
+#include "duckdb/parser/parsed_data/attach_info.hpp"
 
 namespace duckdb {
 class AttachedDatabase;
@@ -36,18 +39,35 @@ public:
 	static DatabaseManager &Get(AttachedDatabase &db);
 
 	void InitializeSystemCatalog();
-	//! Get an attached database with the given name
+	//! Get an attached database by its name
 	optional_ptr<AttachedDatabase> GetDatabase(ClientContext &context, const string &name);
-	//! Add a new attached database to the database manager
-	void AddDatabase(ClientContext &context, unique_ptr<AttachedDatabase> db);
+	//! Attach a new database
+	optional_ptr<AttachedDatabase> AttachDatabase(ClientContext &context, const AttachInfo &info, const string &db_type,
+	                                              AccessMode access_mode);
+	//! Detach an existing database
 	void DetachDatabase(ClientContext &context, const string &name, OnEntryNotFound if_not_found);
 	//! Returns a reference to the system catalog
 	Catalog &GetSystemCatalog();
+
 	static const string &GetDefaultDatabase(ClientContext &context);
 	void SetDefaultDatabase(ClientContext &context, const string &new_value);
 
-	optional_ptr<AttachedDatabase> GetDatabaseFromPath(ClientContext &context, const string &path);
+	//! Inserts a path to name mapping to the database paths map
+	void InsertDatabasePath(ClientContext &context, const string &path, const string &name);
+	//! Erases a path from the database paths map
+	void EraseDatabasePath(const string &path);
+
+	//! Returns the database type. This might require checking the header of the file, in which case the file handle is
+	//! necessary. We can only grab the file handle, if it is not yet held, even for uncommitted changes. Thus, we have
+	//! to lock for this operation.
+	void GetDatabaseType(ClientContext &context, string &db_type, AttachInfo &info, const DBConfig &config,
+	                     const string &unrecognized_option);
+	//! Scans the catalog set and adds each committed database entry, and each database entry of the current
+	//! transaction, to a vector holding AttachedDatabase references
 	vector<reference<AttachedDatabase>> GetDatabases(ClientContext &context);
+	//! Removes all databases from the catalog set. This is necessary for the database instance's destructor,
+	//! as the database manager has to be alive when destroying the catalog set objects.
+	void ResetDatabases();
 
 	transaction_t GetNewQueryNumber() {
 		return current_query_number++;
@@ -62,6 +82,11 @@ public:
 		return !default_database.empty();
 	}
 
+private:
+	//! Returns a database with a specified path
+	optional_ptr<AttachedDatabase> GetDatabaseFromPath(ClientContext &context, const string &path);
+	void CheckPathConflict(ClientContext &context, const string &path);
+
 private:
 	//! The system database is a special database that holds system entries (e.g. functions)
 	unique_ptr<AttachedDatabase> system;
@@ -73,6 +98,13 @@ private:
 	atomic<transaction_t> current_query_number;
 	//! The current default database
 	string default_database;
+
+	//! The lock to add entries to the database path map
+	mutex db_paths_lock;
+	//! A set containing all attached database path
+	//! This allows to attach many databases efficiently, and to avoid attaching the
+	//! same file path twice
+	case_insensitive_set_t db_paths;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/database_path_and_type.hpp

@@ -14,11 +14,12 @@
 namespace duckdb {
 
 struct DBPathAndType {
-
 	//! Parse database extension type and rest of path from combined form (type:path)
-	static DBPathAndType Parse(const string &combined_path, const DBConfig &config);
+	static void ExtractExtensionPrefix(string &path, string &db_type);
+	//! Check the magic bytes of a file and set the database type based on that
+	static void CheckMagicBytes(string &path, string &db_type, const DBConfig &config);
 
-	const string path;
-	const string type;
+	//! Run ExtractExtensionPrefix followed by CheckMagicBytes
+	static void ResolveDatabaseType(string &path, string &db_type, const DBConfig &config);
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/error_manager.hpp

@@ -15,6 +15,7 @@
 namespace duckdb {
 class ClientContext;
 class DatabaseInstance;
+class TransactionException;
 
 enum class ErrorType : uint16_t {
 	// error message types
@@ -51,7 +52,9 @@ public:
 		return Get(context).FormatException(error_type, params...);
 	}
 
-	DUCKDB_API static string InvalidUnicodeError(const string &input, const string &context);
+	DUCKDB_API static InvalidInputException InvalidUnicodeError(const string &input, const string &context);
+	DUCKDB_API static FatalException InvalidatedDatabase(ClientContext &context, const string &invalidated_msg);
+	DUCKDB_API static TransactionException InvalidatedTransaction(ClientContext &context);
 
 	//! Adds a custom error for a specific error type
 	void AddCustomError(ErrorType type, string new_error);