duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/src/main/extension/extension_load.cpp

@@ -1,9 +1,12 @@
+#include "duckdb.h"
 #include "duckdb/common/dl.hpp"
+#include "duckdb/common/operator/cast_operators.hpp"
+#include "duckdb/common/string_util.hpp"
 #include "duckdb/common/virtual_file_system.hpp"
+#include "duckdb/main/capi/capi_internal.hpp"
+#include "duckdb/main/capi/extension_api.hpp"
 #include "duckdb/main/error_manager.hpp"
 #include "duckdb/main/extension_helper.hpp"
-#include "duckdb/common/serializer/buffered_file_reader.hpp"
-#include "duckdb/common/serializer/binary_deserializer.hpp"
 #include "mbedtls_wrapper.hpp"
 
 #ifndef DUCKDB_NO_THREADS
@@ -16,11 +19,106 @@
 
 namespace duckdb {
 
+//===--------------------------------------------------------------------===//
+// Extension C API
+//===--------------------------------------------------------------------===//
+
+//! State that is kept during the load phase of a C API extension
+struct DuckDBExtensionLoadState {
+	explicit DuckDBExtensionLoadState(DatabaseInstance &db_p) : db(db_p), database_data(nullptr) {
+	}
+
+	//! Create a DuckDBExtensionLoadState reference from a C API opaque pointer
+	static DuckDBExtensionLoadState &Get(duckdb_extension_info info) {
+		D_ASSERT(info);
+		return *reinterpret_cast<duckdb::DuckDBExtensionLoadState *>(info);
+	}
+
+	//! Convert to an opaque C API pointer
+	duckdb_extension_info ToCStruct() {
+		return reinterpret_cast<duckdb_extension_info>(this);
+	}
+
+	//! Ref to the database being loaded
+	DatabaseInstance &db;
+
+	//! This is the duckdb_database struct that will be passed to the extension during initialization. Note that the
+	//! extension does not need to free it.
+	unique_ptr<DatabaseData> database_data;
+
+	//! The function pointer struct passed to the extension. The extension is expected to copy this struct during
+	//! initialization
+	duckdb_ext_api_v0 api_struct;
+
+	//! Error handling
+	bool has_error = false;
+	//! The stored error from the loading process
+	ErrorData error_data;
+};
+
+//! Contains the callbacks that are passed to CAPI extensions to allow initialization
+struct ExtensionAccess {
+	//! Create the struct of function pointers to pass to the extension for initialization
+	static duckdb_extension_access CreateAccessStruct() {
+		return {SetError, GetDatabase, GetAPI};
+	}
+
+	//! Called by the extension to indicate failure to initialize the extension
+	static void SetError(duckdb_extension_info info, const char *error) {
+		auto &load_state = DuckDBExtensionLoadState::Get(info);
+
+		load_state.has_error = true;
+		load_state.error_data = ErrorData(ExceptionType::UNKNOWN_TYPE, error);
+	}
+
+	//! Called by the extension get a pointer to the database that is loading it
+	static duckdb_database *GetDatabase(duckdb_extension_info info) {
+		auto &load_state = DuckDBExtensionLoadState::Get(info);
+
+		try {
+			// Create the duckdb_database
+			load_state.database_data = make_uniq<DatabaseData>();
+			load_state.database_data->database = make_uniq<DuckDB>(load_state.db);
+			return reinterpret_cast<duckdb_database *>(load_state.database_data.get());
+		} catch (std::exception &ex) {
+			load_state.error_data = ErrorData(ex);
+			return nullptr;
+		} catch (...) {
+			load_state.error_data =
+			    ErrorData(ExceptionType::UNKNOWN_TYPE, "Unknown error in GetDatabase when trying to load extension!");
+			return nullptr;
+		}
+	}
+
+	//! Called by the extension get a pointer the correctly versioned extension C API struct.
+	static void *GetAPI(duckdb_extension_info info, const char *version) {
+
+		string version_string = version;
+		idx_t major, minor, patch;
+		auto parsed = VersioningUtils::ParseSemver(version_string, major, minor, patch);
+
+		auto &load_state = DuckDBExtensionLoadState::Get(info);
+
+		if (!parsed || !VersioningUtils::IsSupportedCAPIVersion(major, minor, patch)) {
+			load_state.has_error = true;
+			load_state.error_data =
+			    ErrorData(ExceptionType::UNKNOWN_TYPE,
+			              "Unsupported C CAPI version detected during extension initialization: " + string(version));
+			return nullptr;
+		}
+		load_state.api_struct = CreateAPIv0();
+		return &load_state.api_struct;
+	}
+};
+
 //===--------------------------------------------------------------------===//
 // Load External Extension
 //===--------------------------------------------------------------------===//
 #ifndef DUCKDB_DISABLE_EXTENSION_LOAD
+// The C++ init function
 typedef void (*ext_init_fun_t)(DatabaseInstance &);
+// The C init function
+typedef void (*ext_init_c_api_fun_t)(duckdb_extension_info info, duckdb_extension_access *access);
 typedef const char *(*ext_version_fun_t)(void);
 typedef bool (*ext_is_storage_t)(void);
 
@@ -32,6 +130,16 @@ static T LoadFunctionFromDLL(void *dll, const string &function_name, const strin
 	}
 	return (T)function;
 }
+#endif
+
+template <class T>
+static T TryLoadFunctionFromDLL(void *dll, const string &function_name, const string &filename) {
+	auto function = dlsym(dll, function_name.c_str());
+	if (!function) {
+		return nullptr;
+	}
+	return (T)function;
+}
 
 static void ComputeSHA256String(const string &to_hash, string *res) {
 	// Invoke MbedTls function to actually compute sha256
@@ -57,7 +165,6 @@ static void ComputeSHA256FileSegment(FileHandle *handle, const idx_t start, cons
 
 	*res = state.Finalize();
 }
-#endif
 
 static string FilterZeroAtEnd(string s) {
 	while (!s.empty() && s.back() == '\0') {
@@ -77,11 +184,24 @@ ParsedExtensionMetaData ExtensionHelper::ParseExtensionMetaData(const char *meta
 
 	std::reverse(metadata_field.begin(), metadata_field.end());
 
+	// Fetch the magic value and early out if this is invalid: the rest will just be bogus
 	result.magic_value = FilterZeroAtEnd(metadata_field[0]);
+	if (!result.AppearsValid()) {
+		return result;
+	}
+
 	result.platform = FilterZeroAtEnd(metadata_field[1]);
-	result.duckdb_version = FilterZeroAtEnd(metadata_field[2]);
+
 	result.extension_version = FilterZeroAtEnd(metadata_field[3]);
 
+	result.abi_type = EnumUtil::FromString<ExtensionABIType>(FilterZeroAtEnd(metadata_field[4]));
+
+	if (result.abi_type == ExtensionABIType::C_STRUCT) {
+		result.duckdb_capi_version = FilterZeroAtEnd(metadata_field[2]);
+	} else if (result.abi_type == ExtensionABIType::CPP) {
+		result.duckdb_version = FilterZeroAtEnd(metadata_field[2]);
+	}
+
 	result.signature = string(metadata, ParsedExtensionMetaData::FOOTER_SIZE - ParsedExtensionMetaData::SIGNATURE_SIZE);
 	return result;
 }
@@ -158,12 +278,12 @@ bool ExtensionHelper::CheckExtensionSignature(FileHandle &handle, ParsedExtensio
 	return false;
 }
 
-bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const string &extension,
+bool ExtensionHelper::TryInitialLoad(DatabaseInstance &db, FileSystem &fs, const string &extension,
                                      ExtensionInitResult &result, string &error) {
 #ifdef DUCKDB_DISABLE_EXTENSION_LOAD
 	throw PermissionException("Loading external extensions is disabled through a compile time flag");
 #else
-	if (!config.options.enable_external_access) {
+	if (!db.config.options.enable_external_access) {
 		throw PermissionException("Loading external extensions is disabled through configuration");
 	}
 	auto filename = fs.ConvertSeparators(extension);
@@ -198,8 +318,9 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str
 		filename = address;
 #else
 
-		string local_path = !config.options.extension_directory.empty() ? config.options.extension_directory
-		                                                                : ExtensionHelper::DefaultExtensionFolder(fs);
+		string local_path = !db.config.options.extension_directory.empty()
+		                        ? db.config.options.extension_directory
+		                        : ExtensionHelper::DefaultExtensionFolder(fs);
 
 		// convert random separators to platform-canonic
 		local_path = fs.ConvertSeparators(local_path);
@@ -236,12 +357,17 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str
 		metadata_mismatch_error = StringUtil::Format("Failed to load '%s', %s", extension, metadata_mismatch_error);
 	}
 
-	if (!config.options.allow_unsigned_extensions) {
-		bool signature_valid =
-		    CheckExtensionSignature(*handle, parsed_metadata, config.options.allow_community_extensions);
+	if (!db.config.options.allow_unsigned_extensions) {
+		bool signature_valid;
+		if (parsed_metadata.AppearsValid()) {
+			signature_valid =
+			    CheckExtensionSignature(*handle, parsed_metadata, db.config.options.allow_community_extensions);
+		} else {
+			signature_valid = false;
+		}
 
 		if (!signature_valid) {
-			throw IOException(config.error_manager->FormatException(ErrorType::UNSIGNED_EXTENSION, filename) +
+			throw IOException(db.config.error_manager->FormatException(ErrorType::UNSIGNED_EXTENSION, filename) +
 			                  metadata_mismatch_error);
 		}
 
@@ -249,7 +375,7 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str
 			// Signed extensions perform the full check
 			throw InvalidInputException(metadata_mismatch_error);
 		}
-	} else if (!config.options.allow_extensions_metadata_mismatch) {
+	} else if (!db.config.options.allow_extensions_metadata_mismatch) {
 		if (!metadata_mismatch_error.empty()) {
 			// Unsigned extensions AND configuration allowing n, loading allowed, mainly for
 			// debugging purposes
@@ -319,17 +445,18 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str
 #endif
 }
 
-ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileSystem &fs, const string &extension) {
+ExtensionInitResult ExtensionHelper::InitialLoad(DatabaseInstance &db, FileSystem &fs, const string &extension) {
 	string error;
 	ExtensionInitResult result;
-	if (!TryInitialLoad(config, fs, extension, result, error)) {
+	if (!TryInitialLoad(db, fs, extension, result, error)) {
 		if (!ExtensionHelper::AllowAutoInstall(extension)) {
 			throw IOException(error);
 		}
 		// the extension load failed - try installing the extension
-		ExtensionHelper::InstallExtension(config, fs, extension, false);
+		ExtensionInstallOptions options;
+		ExtensionHelper::InstallExtension(db, fs, extension, options);
 		// try loading again
-		if (!TryInitialLoad(config, fs, extension, result, error)) {
+		if (!TryInitialLoad(db, fs, extension, result, error)) {
 			throw IOException(error);
 		}
 	}
@@ -364,18 +491,45 @@ void ExtensionHelper::LoadExternalExtension(DatabaseInstance &db, FileSystem &fs
 #ifdef DUCKDB_DISABLE_EXTENSION_LOAD
 	throw PermissionException("Loading external extensions is disabled through a compile time flag");
 #else
-	auto res = InitialLoad(DBConfig::GetConfig(db), fs, extension);
+	auto res = InitialLoad(db, fs, extension);
 	auto init_fun_name = res.filebase + "_init";
 
-	ext_init_fun_t init_fun;
-	init_fun = LoadFunctionFromDLL<ext_init_fun_t>(res.lib_hdl, init_fun_name, res.filename);
+	// "OLD WAY" of loading extensions. If the <ext_name>_init exists, we choose that
+	ext_init_fun_t init_fun = TryLoadFunctionFromDLL<ext_init_fun_t>(res.lib_hdl, init_fun_name, res.filename);
+	if (init_fun) {
+		try {
+			(*init_fun)(db);
+		} catch (std::exception &e) {
+			ErrorData error(e);
+			throw InvalidInputException("Initialization function \"%s\" from file \"%s\" threw an exception: \"%s\"",
+			                            init_fun_name, res.filename, error.RawMessage());
+		}
+
+		D_ASSERT(res.install_info);
+
+		db.SetExtensionLoaded(extension, *res.install_info);
+		return;
+	}
+
+	// TODO: make this the only way of calling extensions?
+	// "NEW WAY" of loading extensions enabling C API only
+	init_fun_name = res.filebase + "_init_c_api";
+	ext_init_c_api_fun_t init_fun_capi =
+	    TryLoadFunctionFromDLL<ext_init_c_api_fun_t>(res.lib_hdl, init_fun_name, res.filename);
+
+	if (!init_fun_capi) {
+		throw IOException("File \"%s\" did not contain function \"%s\": %s", res.filename, init_fun_name, GetDLError());
+	}
+
+	// Create the load state
+	DuckDBExtensionLoadState load_state(db);
+
+	auto access = ExtensionAccess::CreateAccessStruct();
+	(*init_fun_capi)(load_state.ToCStruct(), &access);
 
-	try {
-		(*init_fun)(db);
-	} catch (std::exception &e) {
-		ErrorData error(e);
-		throw InvalidInputException("Initialization function \"%s\" from file \"%s\" threw an exception: \"%s\"",
-		                            init_fun_name, res.filename, error.RawMessage());
+	// Throw any error that the extension might have encountered
+	if (load_state.has_error) {
+		load_state.error_data.Throw("An error was thrown during initialization of the extension '" + extension + "': ");
 	}
 
 	D_ASSERT(res.install_info);

package/src/duckdb/src/main/extension/extension_util.cpp

@@ -10,12 +10,20 @@
 #include "duckdb/catalog/catalog_entry/scalar_function_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp"
 #include "duckdb/parser/parsed_data/create_collation_info.hpp"
+#include "duckdb/main/extension_install_info.hpp"
 #include "duckdb/catalog/catalog.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/main/secret/secret_manager.hpp"
+#include "duckdb/main/database.hpp"
 
 namespace duckdb {
 
+void ExtensionUtil::RegisterExtension(DatabaseInstance &db, const string &name,
+                                      const ExtensionLoadedInfo &description) {
+
+	db.AddExtensionInfo(name, description);
+}
+
 void ExtensionUtil::RegisterFunction(DatabaseInstance &db, ScalarFunctionSet set) {
 	D_ASSERT(!set.name.empty());
 	CreateScalarFunctionInfo info(std::move(set));

package/src/duckdb/src/main/extension.cpp

@@ -1,6 +1,8 @@
 #include "duckdb/main/extension.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/main/extension_helper.hpp"
+#include "duckdb/main/capi/extension_api.hpp"
+#include "duckdb/common/operator/cast_operators.hpp"
 
 namespace duckdb {
 
@@ -34,7 +36,6 @@ static string PrettyPrintString(const string &s) {
 }
 
 string ParsedExtensionMetaData::GetInvalidMetadataError() {
-	const string engine_version = string(ExtensionHelper::GetVersionDirectoryName());
 	const string engine_platform = string(DuckDB::Platform());
 
 	if (!AppearsValid()) {
@@ -42,11 +43,28 @@ string ParsedExtensionMetaData::GetInvalidMetadataError() {
 	}
 
 	string result;
-	if (engine_version != duckdb_version) {
-		result += StringUtil::Format("The file was built for DuckDB version '%s', but we can only load extensions "
-		                             "built for DuckDB version '%s'.",
-		                             PrettyPrintString(duckdb_version), engine_version);
+
+	if (abi_type == ExtensionABIType::CPP) {
+		const string engine_version = string(ExtensionHelper::GetVersionDirectoryName());
+
+		if (engine_version != duckdb_version) {
+			result += StringUtil::Format("The file was built for DuckDB version '%s', but we can only load extensions "
+			                             "built for DuckDB version '%s'.",
+			                             PrettyPrintString(duckdb_version), engine_version);
+		}
+	} else if (abi_type == ExtensionABIType::C_STRUCT) {
+
+		if (!VersioningUtils::IsSupportedCAPIVersion(duckdb_capi_version)) {
+			result +=
+			    StringUtil::Format("The file was built for DuckDB C API version '%s', but we can only load extensions "
+			                       "built for DuckDB C API 'v%lld.%lld.%lld' and lower.",
+			                       duckdb_capi_version, DUCKDB_EXTENSION_API_VERSION_MAJOR,
+			                       DUCKDB_EXTENSION_API_VERSION_MINOR, DUCKDB_EXTENSION_API_VERSION_PATCH);
+		}
+	} else {
+		throw InternalException("Unknown ABI type for extension: " + EnumUtil::ToString(abi_type));
 	}
+
 	if (engine_platform != platform) {
 		if (!result.empty()) {
 			result += " Also, t";
@@ -61,4 +79,53 @@ string ParsedExtensionMetaData::GetInvalidMetadataError() {
 	return result;
 }
 
+bool VersioningUtils::IsSupportedCAPIVersion(string &capi_version_string) {
+	idx_t major, minor, patch;
+	if (!ParseSemver(capi_version_string, major, minor, patch)) {
+		return false;
+	}
+
+	return IsSupportedCAPIVersion(major, minor, patch);
+}
+
+bool VersioningUtils::IsSupportedCAPIVersion(idx_t major, idx_t minor, idx_t patch) {
+	if (major > DUCKDB_EXTENSION_API_VERSION_MAJOR || minor > DUCKDB_EXTENSION_API_VERSION_MINOR ||
+	    patch > DUCKDB_EXTENSION_API_VERSION_PATCH) {
+		return false;
+	}
+
+	return true;
+}
+
+bool VersioningUtils::ParseSemver(string &semver, idx_t &major_out, idx_t &minor_out, idx_t &patch_out) {
+	if (!StringUtil::StartsWith(semver, "v")) {
+		return false;
+	}
+
+	auto without_v = semver.substr(1);
+
+	auto split = StringUtil::Split(without_v, '.');
+
+	if (split.size() != 3) {
+		return false;
+	}
+
+	idx_t major, minor, patch;
+	bool succeeded = true;
+
+	succeeded &= TryCast::Operation<string_t, idx_t>(split[0], major);
+	succeeded &= TryCast::Operation<string_t, idx_t>(split[1], minor);
+	succeeded &= TryCast::Operation<string_t, idx_t>(split[2], patch);
+
+	if (!succeeded) {
+		return false;
+	}
+
+	major_out = major;
+	minor_out = minor;
+	patch_out = patch;
+
+	return true;
+}
+
 } // namespace duckdb

package/src/duckdb/src/main/pending_query_result.cpp

@@ -43,6 +43,11 @@ void PendingQueryResult::CheckExecutableInternal(ClientContextLock &lock) {
 	}
 }
 
+void PendingQueryResult::WaitForTask() {
+	auto lock = LockContext();
+	context->WaitForTask(*lock, *this);
+}
+
 PendingExecutionResult PendingQueryResult::ExecuteTask() {
 	auto lock = LockContext();
 	return ExecuteTaskInternal(*lock);
@@ -65,16 +70,20 @@ PendingExecutionResult PendingQueryResult::ExecuteTaskInternal(ClientContextLock
 
 unique_ptr<QueryResult> PendingQueryResult::ExecuteInternal(ClientContextLock &lock) {
 	CheckExecutableInternal(lock);
-	// Busy wait while execution is not finished
-	if (allow_stream_result) {
-		while (!IsFinishedOrBlocked(ExecuteTaskInternal(lock))) {
-		}
-	} else {
-		while (!IsFinished(ExecuteTaskInternal(lock))) {
+
+	PendingExecutionResult execution_result;
+	while (!IsResultReady(execution_result = ExecuteTaskInternal(lock))) {
+		if (execution_result == PendingExecutionResult::BLOCKED) {
+			CheckExecutableInternal(lock);
+			context->WaitForTask(lock, *this);
 		}
 	}
 	if (HasError()) {
-		return make_uniq<MaterializedQueryResult>(error);
+		if (allow_stream_result) {
+			return make_uniq<StreamQueryResult>(error);
+		} else {
+			return make_uniq<MaterializedQueryResult>(error);
+		}
 	}
 	auto result = context->FetchResultInternal(lock, *this);
 	Close();
@@ -90,13 +99,12 @@ void PendingQueryResult::Close() {
 	context.reset();
 }
 
-bool PendingQueryResult::IsFinished(PendingExecutionResult result) {
-	return (result == PendingExecutionResult::RESULT_READY || result == PendingExecutionResult::EXECUTION_ERROR);
+bool PendingQueryResult::IsResultReady(PendingExecutionResult result) {
+	return (IsExecutionFinished(result) || result == PendingExecutionResult::RESULT_READY);
 }
 
-bool PendingQueryResult::IsFinishedOrBlocked(PendingExecutionResult result) {
-	return (result == PendingExecutionResult::RESULT_READY || result == PendingExecutionResult::EXECUTION_ERROR ||
-	        result == PendingExecutionResult::BLOCKED);
+bool PendingQueryResult::IsExecutionFinished(PendingExecutionResult result) {
+	return (result == PendingExecutionResult::EXECUTION_FINISHED || result == PendingExecutionResult::EXECUTION_ERROR);
 }
 
 } // namespace duckdb

package/src/duckdb/src/main/prepared_statement.cpp

@@ -6,8 +6,8 @@
 namespace duckdb {
 
 PreparedStatement::PreparedStatement(shared_ptr<ClientContext> context, shared_ptr<PreparedStatementData> data_p,
-                                     string query, idx_t n_param, case_insensitive_map_t<idx_t> named_param_map_p)
-    : context(std::move(context)), data(std::move(data_p)), query(std::move(query)), success(true), n_param(n_param),
+                                     string query, case_insensitive_map_t<idx_t> named_param_map_p)
+    : context(std::move(context)), data(std::move(data_p)), query(std::move(query)), success(true),
       named_param_map(std::move(named_param_map_p)) {
 	D_ASSERT(data || !success);
 }
@@ -68,7 +68,7 @@ case_insensitive_map_t<LogicalType> PreparedStatement::GetExpectedParameterTypes
 	return expected_types;
 }
 
-unique_ptr<QueryResult> PreparedStatement::Execute(case_insensitive_map_t<Value> &named_values,
+unique_ptr<QueryResult> PreparedStatement::Execute(case_insensitive_map_t<BoundParameterData> &named_values,
                                                    bool allow_stream_result) {
 	auto pending = PendingQuery(named_values, allow_stream_result);
 	if (pending->HasError()) {
@@ -86,15 +86,15 @@ unique_ptr<QueryResult> PreparedStatement::Execute(vector<Value> &values, bool a
 }
 
 unique_ptr<PendingQueryResult> PreparedStatement::PendingQuery(vector<Value> &values, bool allow_stream_result) {
-	case_insensitive_map_t<Value> named_values;
+	case_insensitive_map_t<BoundParameterData> named_values;
 	for (idx_t i = 0; i < values.size(); i++) {
 		auto &val = values[i];
-		named_values[std::to_string(i + 1)] = val;
+		named_values[std::to_string(i + 1)] = BoundParameterData(val);
 	}
 	return PendingQuery(named_values, allow_stream_result);
 }
 
-unique_ptr<PendingQueryResult> PreparedStatement::PendingQuery(case_insensitive_map_t<Value> &named_values,
+unique_ptr<PendingQueryResult> PreparedStatement::PendingQuery(case_insensitive_map_t<BoundParameterData> &named_values,
                                                                bool allow_stream_result) {
 	if (!success) {
 		auto exception = InvalidInputException("Attempting to execute an unsuccessfully prepared statement!");

package/src/duckdb/src/main/prepared_statement_data.cpp

@@ -1,7 +1,9 @@
 #include "duckdb/main/prepared_statement_data.hpp"
 #include "duckdb/execution/physical_operator.hpp"
 #include "duckdb/parser/sql_statement.hpp"
+#include "duckdb/catalog/catalog.hpp"
 #include "duckdb/common/exception/binder_exception.hpp"
+#include "duckdb/main/attached_database.hpp"
 #include "duckdb/main/database_manager.hpp"
 #include "duckdb/transaction/transaction.hpp"
 
@@ -21,15 +23,25 @@ void PreparedStatementData::CheckParameterCount(idx_t parameter_count) {
 	}
 }
 
-void StartTransactionInCatalog(ClientContext &context, const string &catalog_name) {
+bool CheckCatalogIdentity(ClientContext &context, const string &catalog_name,
+                          const StatementProperties::CatalogIdentity catalog_identity) {
+	// some catalogs don't support catalog version, we can't check identity in that case
+	if (!catalog_identity.catalog_version.IsValid()) {
+		return false;
+	}
 	auto database = DatabaseManager::Get(context).GetDatabase(context, catalog_name);
 	if (!database) {
 		throw BinderException("Prepared statement requires database %s but it was not attached", catalog_name);
 	}
 	Transaction::Get(context, *database);
+	auto current_catalog_oid = database->GetCatalog().GetOid();
+	auto current_catalog_version = database->GetCatalog().GetCatalogVersion(context);
+
+	return StatementProperties::CatalogIdentity {current_catalog_oid, current_catalog_version} == catalog_identity;
 }
 
-bool PreparedStatementData::RequireRebind(ClientContext &context, optional_ptr<case_insensitive_map_t<Value>> values) {
+bool PreparedStatementData::RequireRebind(ClientContext &context,
+                                          optional_ptr<case_insensitive_map_t<BoundParameterData>> values) {
 	idx_t count = values ? values->size() : 0;
 	CheckParameterCount(count);
 	if (!unbound_statement) {
@@ -49,28 +61,27 @@ bool PreparedStatementData::RequireRebind(ClientContext &context, optional_ptr<c
 		if (lookup == values->end()) {
 			break;
 		}
-		if (lookup->second.type() != it.second->return_type) {
+		if (lookup->second.GetValue().type() != it.second->return_type) {
 			return true;
 		}
 	}
-	// prior to checking the catalog version we need to explicitly start transactions in all affected databases
-	// this ensures all catalog entries we rely on are cached
-	for (auto &catalog_name : properties.read_databases) {
-		StartTransactionInCatalog(context, catalog_name);
-	}
-	for (auto &catalog_name : properties.modified_databases) {
-		StartTransactionInCatalog(context, catalog_name);
+	// Check the catalog versions to ensure all catalog entries we rely on are current
+	for (auto &it : properties.read_databases) {
+		if (!CheckCatalogIdentity(context, it.first, it.second)) {
+			return true;
+		}
 	}
-	if (Catalog::GetSystemCatalog(context).GetCatalogVersion() != catalog_version) {
-		//! context is out of bounds
-		return true;
+	for (auto &it : properties.modified_databases) {
+		if (!CheckCatalogIdentity(context, it.first, it.second)) {
+			return true;
+		}
 	}
 	return false;
 }
 
-void PreparedStatementData::Bind(case_insensitive_map_t<Value> values) {
+void PreparedStatementData::Bind(case_insensitive_map_t<BoundParameterData> values) {
 	// set parameters
-	D_ASSERT(!unbound_statement || unbound_statement->n_param == properties.parameter_count);
+	D_ASSERT(!unbound_statement || unbound_statement->named_param_map.size() == properties.parameter_count);
 	CheckParameterCount(values.size());
 
 	// bind the required values
@@ -81,13 +92,13 @@ void PreparedStatementData::Bind(case_insensitive_map_t<Value> values) {
 			throw BinderException("Could not find parameter with identifier %s", identifier);
 		}
 		D_ASSERT(it.second);
-		auto &value = lookup->second;
+		auto value = lookup->second.GetValue();
 		if (!value.DefaultTryCastAs(it.second->return_type)) {
 			throw BinderException(
 			    "Type mismatch for binding parameter with identifier %s, expected type %s but got type %s", identifier,
 			    it.second->return_type.ToString().c_str(), value.type().ToString().c_str());
 		}
-		it.second->SetValue(value);
+		it.second->SetValue(std::move(value));
 	}
 }