duckdb 0.10.3-dev0.0 → 0.10.3-dev6.0

package/src/duckdb/src/main/extension/extension_install.cpp

@@ -1,8 +1,13 @@
+#include "duckdb/common/exception/http_exception.hpp"
 #include "duckdb/common/gzip_file_system.hpp"
-#include "duckdb/common/types/uuid.hpp"
+#include "duckdb/common/local_file_system.hpp"
+#include "duckdb/common/serializer/binary_serializer.hpp"
 #include "duckdb/common/string_util.hpp"
-#include "duckdb/common/exception/http_exception.hpp"
+#include "duckdb/common/types/uuid.hpp"
+#include "duckdb/logging/http_logger.hpp"
+#include "duckdb/main/client_data.hpp"
 #include "duckdb/main/extension_helper.hpp"
+#include "duckdb/main/extension_install_info.hpp"
 
 #ifndef DISABLE_DUCKDB_REMOTE_INSTALL
 #ifndef DUCKDB_DISABLE_EXTENSION_LOAD
@@ -128,26 +133,33 @@ bool ExtensionHelper::CreateSuggestions(const string &extension_name, string &me
 	return false;
 }
 
-void ExtensionHelper::InstallExtension(DBConfig &config, FileSystem &fs, const string &extension, bool force_install,
-                                       const string &repository) {
+unique_ptr<ExtensionInstallInfo> ExtensionHelper::InstallExtension(DBConfig &config, FileSystem &fs,
+                                                                   const string &extension, bool force_install,
+                                                                   optional_ptr<ExtensionRepository> repository,
+                                                                   const string &version) {
 #ifdef WASM_LOADABLE_EXTENSIONS
 	// Install is currently a no-op
-	return;
+	return nullptr;
 #endif
 	string local_path = ExtensionDirectory(config, fs);
-	InstallExtensionInternal(config, fs, local_path, extension, force_install, repository);
+	return InstallExtensionInternal(config, fs, local_path, extension, force_install, version, repository);
 }
 
-void ExtensionHelper::InstallExtension(ClientContext &context, const string &extension, bool force_install,
-                                       const string &repository) {
+unique_ptr<ExtensionInstallInfo> ExtensionHelper::InstallExtension(ClientContext &context, const string &extension,
+                                                                   bool force_install,
+                                                                   optional_ptr<ExtensionRepository> repository,
+                                                                   const string &version) {
 #ifdef WASM_LOADABLE_EXTENSIONS
 	// Install is currently a no-op
-	return;
+	return nullptr;
 #endif
-	auto &config = DBConfig::GetConfig(context);
+	auto &db_config = DBConfig::GetConfig(context);
 	auto &fs = FileSystem::GetFileSystem(context);
 	string local_path = ExtensionDirectory(context);
-	InstallExtensionInternal(config, fs, local_path, extension, force_install, repository);
+	optional_ptr<HTTPLogger> http_logger =
+	    ClientConfig::GetConfig(context).enable_http_logging ? context.client_data->http_logger.get() : nullptr;
+	return InstallExtensionInternal(db_config, fs, local_path, extension, force_install, version, repository,
+	                                http_logger, context);
 }
 
 unsafe_unique_array<data_t> ReadExtensionFileFromDisk(FileSystem &fs, const string &path, idx_t &file_size) {
@@ -159,7 +171,7 @@ unsafe_unique_array<data_t> ReadExtensionFileFromDisk(FileSystem &fs, const stri
 	return in_buffer;
 }
 
-void WriteExtensionFileToDisk(FileSystem &fs, const string &path, void *data, idx_t data_size) {
+static void WriteExtensionFileToDisk(FileSystem &fs, const string &path, void *data, idx_t data_size) {
 	auto target_file = fs.OpenFile(path, FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_APPEND |
 	                                         FileFlags::FILE_FLAGS_FILE_CREATE_NEW);
 	target_file->Write(data, data_size);
@@ -167,25 +179,28 @@ void WriteExtensionFileToDisk(FileSystem &fs, const string &path, void *data, id
 	target_file.reset();
 }
 
-string ExtensionHelper::ExtensionUrlTemplate(optional_ptr<const DBConfig> db_config, const string &repository) {
-	string versioned_path = "/${REVISION}/${PLATFORM}/${NAME}.duckdb_extension";
+static void WriteExtensionMetadataFileToDisk(FileSystem &fs, const string &path, ExtensionInstallInfo &metadata) {
+	auto file_writer = BufferedFileWriter(fs, path);
+	BinarySerializer::Serialize(metadata, file_writer);
+	file_writer.Sync();
+}
+
+string ExtensionHelper::ExtensionUrlTemplate(optional_ptr<const DBConfig> db_config,
+                                             const ExtensionRepository &repository, const string &version) {
+	string versioned_path;
+	if (!version.empty()) {
+		versioned_path = "/${NAME}/" + version + "/${REVISION}/${PLATFORM}/${NAME}.duckdb_extension";
+	} else {
+		versioned_path = "/${REVISION}/${PLATFORM}/${NAME}.duckdb_extension";
+	}
 #ifdef WASM_LOADABLE_EXTENSIONS
-	string default_endpoint = "https://extensions.duckdb.org";
+	string default_endpoint = DEFAULT_REPOSITORY;
 	versioned_path = versioned_path + ".wasm";
 #else
-	string default_endpoint = "http://extensions.duckdb.org";
+	string default_endpoint = ExtensionRepository::DEFAULT_REPOSITORY_URL;
 	versioned_path = versioned_path + ".gz";
 #endif
-	string custom_endpoint = db_config ? db_config->options.custom_extension_repo : string();
-	string endpoint;
-	if (!repository.empty()) {
-		endpoint = repository;
-	} else if (!custom_endpoint.empty()) {
-		endpoint = custom_endpoint;
-	} else {
-		endpoint = default_endpoint;
-	}
-	string url_template = endpoint + versioned_path;
+	string url_template = repository.path + versioned_path;
 	return url_template;
 }
 
@@ -196,56 +211,128 @@ string ExtensionHelper::ExtensionFinalizeUrlTemplate(const string &url_template,
 	return url;
 }
 
-void ExtensionHelper::InstallExtensionInternal(DBConfig &config, FileSystem &fs, const string &local_path,
-                                               const string &extension, bool force_install, const string &repository) {
-#ifdef DUCKDB_DISABLE_EXTENSION_LOAD
-	throw PermissionException("Installing external extensions is disabled through a compile time flag");
-#else
-	if (!config.options.enable_external_access) {
-		throw PermissionException("Installing extensions is disabled through configuration");
+static void CheckExtensionMetadataOnInstall(DBConfig &config, void *in_buffer, idx_t file_size,
+                                            ExtensionInstallInfo &info, const string &extension_name) {
+	if (file_size < ParsedExtensionMetaData::FOOTER_SIZE) {
+		throw IOException("Failed to install '%s', file too small to be a valid DuckDB extension!", extension_name);
 	}
-	auto extension_name = ApplyExtensionAlias(fs.ExtractBaseName(extension));
 
-	string local_extension_path = fs.JoinPath(local_path, extension_name + ".duckdb_extension");
-	if (fs.FileExists(local_extension_path) && !force_install) {
-		return;
+	auto parsed_metadata = ExtensionHelper::ParseExtensionMetaData(static_cast<char *>(in_buffer) +
+	                                                               (file_size - ParsedExtensionMetaData::FOOTER_SIZE));
+
+	auto metadata_mismatch_error = parsed_metadata.GetInvalidMetadataError();
+
+	if (!metadata_mismatch_error.empty() && !config.options.allow_extensions_metadata_mismatch) {
+		throw IOException("Failed to install '%s'\n%s", extension_name, metadata_mismatch_error);
 	}
 
-	auto uuid = UUID::ToString(UUID::GenerateRandomUUID());
-	string temp_path = local_extension_path + ".tmp-" + uuid;
-	if (fs.FileExists(temp_path)) {
-		fs.RemoveFile(temp_path);
+	info.version = parsed_metadata.extension_version;
+}
+
+// Note: since this method is not atomic, this can fail in different ways, that should all be handled properly by
+// DuckDB:
+//   1. Crash after extension removal: extension is now uninstalled, metadata file still present
+//   2. Crash after metadata removal: extension is now uninstalled, extension dir is clean
+//   3. Crash after extension move: extension is now uninstalled, new metadata file present
+static void WriteExtensionFiles(FileSystem &fs, const string &temp_path, const string &local_extension_path,
+                                void *in_buffer, idx_t file_size, ExtensionInstallInfo &info) {
+	// Write extension to tmp file
+	WriteExtensionFileToDisk(fs, temp_path, in_buffer, file_size);
+
+	// Write metadata to tmp file
+	auto metadata_tmp_path = temp_path + ".info";
+	auto metadata_file_path = local_extension_path + ".info";
+	WriteExtensionMetadataFileToDisk(fs, metadata_tmp_path, info);
+
+	// First remove the local extension we are about to replace
+	if (fs.FileExists(local_extension_path)) {
+		fs.RemoveFile(local_extension_path);
 	}
-	auto is_http_url = StringUtil::Contains(extension, "http://");
-	if (ExtensionHelper::IsFullPath(extension)) {
-		if (fs.FileExists(extension)) {
-			idx_t file_size;
-			auto in_buffer = ReadExtensionFileFromDisk(fs, extension, file_size);
-			WriteExtensionFileToDisk(fs, temp_path, in_buffer.get(), file_size);
 
-			if (fs.FileExists(local_extension_path) && force_install) {
-				fs.RemoveFile(local_extension_path);
-			}
-			fs.MoveFile(temp_path, local_extension_path);
-			return;
-		} else if (!is_http_url) {
-			throw IOException("Failed to read extension from \"%s\": no such file", extension);
+	// Then remove the old metadata file
+	if (fs.FileExists(metadata_file_path)) {
+		fs.RemoveFile(metadata_file_path);
+	}
+
+	fs.MoveFile(metadata_tmp_path, metadata_file_path);
+	fs.MoveFile(temp_path, local_extension_path);
+}
+
+// Install an extension using a filesystem
+static unique_ptr<ExtensionInstallInfo> DirectInstallExtension(DBConfig &config, FileSystem &fs, const string &path,
+                                                               const string &temp_path, const string &extension_name,
+                                                               const string &local_extension_path, bool force_install,
+                                                               optional_ptr<ExtensionRepository> repository,
+                                                               optional_ptr<ClientContext> context) {
+	string file = fs.ConvertSeparators(path);
+
+	// Try autoloading httpfs for loading extensions over https
+	if (context) {
+		auto &db = DatabaseInstance::GetDatabase(*context);
+		if (StringUtil::StartsWith(path, "https://") && !db.ExtensionIsLoaded("httpfs") &&
+		    db.config.options.autoload_known_extensions) {
+			ExtensionHelper::AutoLoadExtension(*context, "httpfs");
 		}
 	}
 
-#ifdef DISABLE_DUCKDB_REMOTE_INSTALL
-	throw BinderException("Remote extension installation is disabled through configuration");
-#else
+	// Check if file exists
+	bool exists = fs.FileExists(file);
 
-	string url_template = ExtensionUrlTemplate(&config, repository);
+	// Recheck without .gz
+	if (!exists && StringUtil::EndsWith(file, ".gz")) {
+		file = file.substr(0, file.size() - 3);
+		exists = fs.FileExists(file);
+	}
 
-	if (is_http_url) {
-		url_template = extension;
-		extension_name = "";
+	// Throw error on failure
+	if (!exists) {
+		if (!fs.IsRemoteFile(file)) {
+			throw IOException("Failed to copy local extension \"%s\" at PATH \"%s\"\n", extension_name, file);
+		}
+		if (StringUtil::StartsWith(file, "https://")) {
+			throw IOException("Failed to install remote extension \"%s\" from url \"%s\"", extension_name, file);
+		}
 	}
 
-	string url = ExtensionFinalizeUrlTemplate(url_template, extension_name);
+	idx_t file_size;
+	auto in_buffer = ReadExtensionFileFromDisk(fs, file, file_size);
+
+	ExtensionInstallInfo info;
 
+	string decompressed_data;
+	void *extension_decompressed;
+	idx_t extension_decompressed_size;
+
+	if (GZipFileSystem::CheckIsZip(const_char_ptr_cast(in_buffer.get()), file_size)) {
+		decompressed_data = GZipFileSystem::UncompressGZIPString(const_char_ptr_cast(in_buffer.get()), file_size);
+		extension_decompressed = (void *)decompressed_data.data();
+		extension_decompressed_size = decompressed_data.size();
+	} else {
+		extension_decompressed = (void *)in_buffer.get();
+		extension_decompressed_size = file_size;
+	}
+
+	CheckExtensionMetadataOnInstall(config, extension_decompressed, extension_decompressed_size, info, extension_name);
+
+	if (!repository) {
+		info.mode = ExtensionInstallMode::CUSTOM_PATH;
+		info.full_path = file;
+	} else {
+		info.mode = ExtensionInstallMode::REPOSITORY;
+		info.full_path = file;
+		info.repository_url = repository->path;
+	}
+
+	WriteExtensionFiles(fs, temp_path, local_extension_path, extension_decompressed, extension_decompressed_size, info);
+
+	return make_uniq<ExtensionInstallInfo>(info);
+}
+
+static unique_ptr<ExtensionInstallInfo> InstallFromHttpUrl(DBConfig &config, const string &url,
+                                                           const string &extension_name, const string &temp_path,
+                                                           const string &local_extension_path, bool force_install,
+                                                           optional_ptr<ExtensionRepository> repository,
+                                                           optional_ptr<HTTPLogger> http_logger) {
 	string no_http = StringUtil::Replace(url, "http://", "");
 
 	idx_t next = no_http.find('/', 0);
@@ -253,34 +340,15 @@ void ExtensionHelper::InstallExtensionInternal(DBConfig &config, FileSystem &fs,
 		throw IOException("No slash in URL template");
 	}
 
-	// Special case to install extension from a local file, useful for testing
-	if (!StringUtil::Contains(url_template, "http://")) {
-		string file = fs.ConvertSeparators(url);
-		if (!fs.FileExists(file)) {
-			// check for non-gzipped variant
-			file = file.substr(0, file.size() - 3);
-			if (!fs.FileExists(file)) {
-				throw IOException("Failed to copy local extension \"%s\" at PATH \"%s\"\n", extension_name, file);
-			}
-		}
-		auto read_handle = fs.OpenFile(file, FileFlags::FILE_FLAGS_READ);
-		auto test_data = std::unique_ptr<unsigned char[]> {new unsigned char[read_handle->GetFileSize()]};
-		read_handle->Read(test_data.get(), read_handle->GetFileSize());
-		WriteExtensionFileToDisk(fs, temp_path, (void *)test_data.get(), read_handle->GetFileSize());
-
-		if (fs.FileExists(local_extension_path) && force_install) {
-			fs.RemoveFile(local_extension_path);
-		}
-		fs.MoveFile(temp_path, local_extension_path);
-		return;
-	}
-
 	// Push the substring [last, next) on to splits
 	auto hostname_without_http = no_http.substr(0, next);
 	auto url_local_part = no_http.substr(next);
 
 	auto url_base = "http://" + hostname_without_http;
 	duckdb_httplib::Client cli(url_base.c_str());
+	if (http_logger) {
+		cli.set_logger(http_logger->GetLogger<duckdb_httplib::Request, duckdb_httplib::Response>());
+	}
 
 	duckdb_httplib::Headers headers = {
 	    {"User-Agent", StringUtil::Format("%s %s", config.UserAgent(), DuckDB::SourceID())}};
@@ -291,7 +359,7 @@ void ExtensionHelper::InstallExtensionInternal(DBConfig &config, FileSystem &fs,
 		// create suggestions
 		string message;
 		auto exact_match = ExtensionHelper::CreateSuggestions(extension_name, message);
-		if (exact_match && !IsRelease(DuckDB::LibraryVersion())) {
+		if (exact_match && !ExtensionHelper::IsRelease(DuckDB::LibraryVersion())) {
 			message += "\nAre you using a development build? In this case, extensions might not (yet) be uploaded.";
 		}
 		if (res.error() == duckdb_httplib::Error::Success) {
@@ -304,12 +372,152 @@ void ExtensionHelper::InstallExtensionInternal(DBConfig &config, FileSystem &fs,
 	}
 	auto decompressed_body = GZipFileSystem::UncompressGZIPString(res->body);
 
-	WriteExtensionFileToDisk(fs, temp_path, (void *)decompressed_body.data(), decompressed_body.size());
+	ExtensionInstallInfo info;
+	CheckExtensionMetadataOnInstall(config, (void *)decompressed_body.data(), decompressed_body.size(), info,
+	                                extension_name);
 
-	if (fs.FileExists(local_extension_path) && force_install) {
-		fs.RemoveFile(local_extension_path);
+	if (repository) {
+		info.mode = ExtensionInstallMode::REPOSITORY;
+		info.full_path = url;
+		info.repository_url = repository->path;
+	} else {
+		info.mode = ExtensionInstallMode::CUSTOM_PATH;
+		info.full_path = url;
 	}
-	fs.MoveFile(temp_path, local_extension_path);
+
+	auto fs = FileSystem::CreateLocal();
+	WriteExtensionFiles(*fs, temp_path, local_extension_path, (void *)decompressed_body.data(),
+	                    decompressed_body.size(), info);
+
+	return make_uniq<ExtensionInstallInfo>(info);
+}
+
+// Install an extension using a hand-rolled http request
+static unique_ptr<ExtensionInstallInfo> InstallFromRepository(DBConfig &config, FileSystem &fs, const string &url,
+                                                              const string &extension_name,
+                                                              ExtensionRepository &repository, const string &temp_path,
+                                                              const string &local_extension_path, const string &version,
+                                                              bool force_install, optional_ptr<HTTPLogger> http_logger,
+                                                              optional_ptr<ClientContext> context) {
+	string url_template = ExtensionHelper::ExtensionUrlTemplate(&config, repository, version);
+	string generated_url = ExtensionHelper::ExtensionFinalizeUrlTemplate(url_template, extension_name);
+
+	// Special handling for http repository: avoid using regular filesystem (note: the filesystem is not used here)
+	if (StringUtil::StartsWith(repository.path, "http://")) {
+		return InstallFromHttpUrl(config, generated_url, extension_name, temp_path, local_extension_path, force_install,
+		                          repository, http_logger);
+	}
+
+	// Default case, let the FileSystem figure it out
+	return DirectInstallExtension(config, fs, generated_url, temp_path, extension_name, local_extension_path,
+	                              force_install, repository, context);
+}
+
+static bool IsHTTP(const string &path) {
+	return StringUtil::StartsWith(path, "http://") || !StringUtil::StartsWith(path, "https://");
+}
+
+static void ThrowErrorOnMismatchingExtensionOrigin(FileSystem &fs, const string &local_extension_path,
+                                                   const string &extension_name, const string &extension,
+                                                   optional_ptr<ExtensionRepository> repository) {
+	auto install_info = ExtensionInstallInfo::TryReadInfoFile(fs, local_extension_path + ".info", extension_name);
+
+	string format_string = "Installing extension '%s' failed. The extension is already installed "
+	                       "but the origin is different.\n"
+	                       "Currently installed extension is from %s '%s', while the extension to be "
+	                       "installed is from %s '%s'.\n"
+	                       "To solve this rerun this command with `FORCE INSTALL`";
+	string repo = "repository";
+	string custom_path = "custom_path";
+
+	if (install_info) {
+		if (install_info->mode == ExtensionInstallMode::REPOSITORY && repository &&
+		    install_info->repository_url != repository->path) {
+			throw InvalidInputException(format_string, extension_name, repo, install_info->repository_url, repo,
+			                            repository->path);
+		}
+		if (install_info->mode == ExtensionInstallMode::REPOSITORY && ExtensionHelper::IsFullPath(extension)) {
+			throw InvalidInputException(format_string, extension_name, repo, install_info->repository_url, custom_path,
+			                            extension);
+		}
+	}
+}
+
+unique_ptr<ExtensionInstallInfo>
+ExtensionHelper::InstallExtensionInternal(DBConfig &config, FileSystem &fs, const string &local_path,
+                                          const string &extension, bool force_install, const string &version,
+                                          optional_ptr<ExtensionRepository> repository,
+                                          optional_ptr<HTTPLogger> http_logger, optional_ptr<ClientContext> context) {
+#ifdef DUCKDB_DISABLE_EXTENSION_LOAD
+	throw PermissionException("Installing external extensions is disabled through a compile time flag");
+#else
+	if (!config.options.enable_external_access) {
+		throw PermissionException("Installing extensions is disabled through configuration");
+	}
+
+	auto extension_name = ApplyExtensionAlias(fs.ExtractBaseName(extension));
+	string local_extension_path = fs.JoinPath(local_path, extension_name + ".duckdb_extension");
+	string temp_path = local_extension_path + ".tmp-" + UUID::ToString(UUID::GenerateRandomUUID());
+
+	if (fs.FileExists(local_extension_path) && !force_install) {
+		// File exists: throw error if origin mismatches
+		if (!config.options.allow_extensions_metadata_mismatch && fs.FileExists(local_extension_path + ".info")) {
+			ThrowErrorOnMismatchingExtensionOrigin(fs, local_extension_path, extension_name, extension, repository);
+		}
+
+		// File exists, but that's okay, install is now a NOP
+		return nullptr;
+	}
+
+	if (fs.FileExists(temp_path)) {
+		fs.RemoveFile(temp_path);
+	}
+
+	if (ExtensionHelper::IsFullPath(extension) && repository) {
+		throw InvalidInputException("Cannot pass both a repository and a full path url");
+	}
+
+	// Resolve default repository if there is none set
+	ExtensionRepository resolved_repository;
+	if (!ExtensionHelper::IsFullPath(extension) && !repository) {
+		resolved_repository = ExtensionRepository::GetDefaultRepository(config);
+		repository = resolved_repository;
+	}
+
+	// Install extension from local, direct url
+	if (ExtensionHelper::IsFullPath(extension) && !IsHTTP(extension)) {
+		LocalFileSystem local_fs;
+		return DirectInstallExtension(config, local_fs, extension, temp_path, extension, local_extension_path,
+		                              force_install, nullptr, context);
+	}
+
+	// Install extension from local url based on a repository (Note that this will install it as a local file)
+	if (repository && !IsHTTP(repository->path)) {
+		LocalFileSystem local_fs;
+		return InstallFromRepository(config, fs, extension, extension_name, *repository, temp_path,
+		                             local_extension_path, version, force_install, http_logger, context);
+	}
+
+#ifdef DISABLE_DUCKDB_REMOTE_INSTALL
+	throw BinderException("Remote extension installation is disabled through configuration");
+#else
+
+	// Full path direct installation
+	if (IsFullPath(extension)) {
+		if (StringUtil::StartsWith(extension, "http://")) {
+			// HTTP takes separate path to avoid dependency on httpfs extension
+			return InstallFromHttpUrl(config, extension, extension_name, temp_path, local_extension_path, force_install,
+			                          nullptr, http_logger);
+		}
+
+		// Direct installation from local or remote path
+		return DirectInstallExtension(config, fs, extension, temp_path, extension, local_extension_path, force_install,
+		                              nullptr, context);
+	}
+
+	// Repository installation
+	return InstallFromRepository(config, fs, extension, extension_name, *repository, temp_path, local_extension_path,
+	                             version, force_install, http_logger, context);
 #endif
 #endif
 }