duckdb 0.8.2-dev3300.0 → 0.8.2-dev3456.0

package/src/duckdb/src/main/config.cpp

@@ -70,6 +70,9 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_GLOBAL(EnableFSSTVectors),
                                                  DUCKDB_GLOBAL(AllowUnsignedExtensionsSetting),
                                                  DUCKDB_LOCAL(CustomExtensionRepository),
+                                                 DUCKDB_LOCAL(AutoloadExtensionRepository),
+                                                 DUCKDB_GLOBAL(AutoinstallKnownExtensions),
+                                                 DUCKDB_GLOBAL(AutoloadKnownExtensions),
                                                  DUCKDB_GLOBAL(EnableObjectCacheSetting),
                                                  DUCKDB_GLOBAL(EnableHTTPMetadataCacheSetting),
                                                  DUCKDB_LOCAL(EnableProfilingSetting),

package/src/duckdb/src/main/extension/extension_helper.cpp

@@ -126,6 +126,7 @@ DefaultExtension ExtensionHelper::GetDefaultExtension(idx_t index) {
 //===--------------------------------------------------------------------===//
 static const char *auto_install[] = {"motherduck", "postgres_scanner", "sqlite_scanner", nullptr};
 
+// TODO: unify with new autoload mechanism
 bool ExtensionHelper::AllowAutoInstall(const string &extension) {
 	auto lcase = StringUtil::Lower(extension);
 	for (idx_t i = 0; auto_install[i]; i++) {
@@ -136,6 +137,62 @@ bool ExtensionHelper::AllowAutoInstall(const string &extension) {
 	return false;
 }
 
+bool ExtensionHelper::CanAutoloadExtension(const string &ext_name) {
+#ifdef DUCKDB_DISABLE_EXTENSION_LOAD
+	return false;
+#endif
+
+	if (ext_name.empty()) {
+		return false;
+	}
+	for (const auto &ext : AUTOLOADABLE_EXTENSIONS) {
+		if (ext_name == ext) {
+			return true;
+		}
+	}
+	return false;
+}
+
+string ExtensionHelper::AddExtensionInstallHintToErrorMsg(ClientContext &context, const string &base_error,
+                                                          const string &extension_name) {
+	auto &dbconfig = DBConfig::GetConfig(context);
+	string install_hint;
+
+	if (!ExtensionHelper::CanAutoloadExtension(extension_name)) {
+		install_hint = "Please try installing and loading the " + extension_name + " extension:\nINSTALL " +
+		               extension_name + ";\nLOAD " + extension_name + ";\n\n";
+	} else if (!dbconfig.options.autoload_known_extensions) {
+		install_hint =
+		    "Please try installing and loading the " + extension_name + " extension by running:\nINSTALL " +
+		    extension_name + ";\nLOAD " + extension_name +
+		    ";\n\nAlternatively, consider enabling auto-install "
+		    "and auto-load by running:\nSET autoinstall_known_extensions=1;\nSET autoload_known_extensions=1;";
+	} else if (!dbconfig.options.autoinstall_known_extensions) {
+		install_hint =
+		    "Please try installing the " + extension_name + " extension by running:\nINSTALL " + extension_name +
+		    ";\n\nAlternatively, consider enabling autoinstall by running:\nSET autoinstall_known_extensions=1;";
+	}
+
+	if (!install_hint.empty()) {
+		return base_error + "\n\n" + install_hint;
+	}
+
+	return base_error;
+}
+
+void ExtensionHelper::AutoLoadExtension(ClientContext &context, const string &extension_name) {
+	auto &dbconfig = DBConfig::GetConfig(context);
+	try {
+		if (dbconfig.options.autoinstall_known_extensions) {
+			ExtensionHelper::InstallExtension(context, extension_name, false,
+			                                  context.config.autoinstall_extension_repo);
+		}
+		ExtensionHelper::LoadExternalExtension(context, extension_name);
+	} catch (Exception &e) {
+		throw AutoloadException(extension_name, e);
+	}
+}
+
 //===--------------------------------------------------------------------===//
 // Load Statically Compiled Extension
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/main/extension/extension_install.cpp

@@ -117,16 +117,18 @@ bool ExtensionHelper::CreateSuggestions(const string &extension_name, string &me
 	return false;
 }
 
-void ExtensionHelper::InstallExtension(DBConfig &config, FileSystem &fs, const string &extension, bool force_install) {
+void ExtensionHelper::InstallExtension(DBConfig &config, FileSystem &fs, const string &extension, bool force_install,
+                                       const string &repository) {
 #ifdef WASM_LOADABLE_EXTENSIONS
 	// Install is currently a no-op
 	return;
 #endif
 	string local_path = ExtensionDirectory(config, fs);
-	InstallExtensionInternal(config, nullptr, fs, local_path, extension, force_install);
+	InstallExtensionInternal(config, nullptr, fs, local_path, extension, force_install, repository);
 }
 
-void ExtensionHelper::InstallExtension(ClientContext &context, const string &extension, bool force_install) {
+void ExtensionHelper::InstallExtension(ClientContext &context, const string &extension, bool force_install,
+                                       const string &repository) {
 #ifdef WASM_LOADABLE_EXTENSIONS
 	// Install is currently a no-op
 	return;
@@ -135,7 +137,7 @@ void ExtensionHelper::InstallExtension(ClientContext &context, const string &ext
 	auto &fs = FileSystem::GetFileSystem(context);
 	string local_path = ExtensionDirectory(context);
 	auto &client_config = ClientConfig::GetConfig(context);
-	InstallExtensionInternal(config, &client_config, fs, local_path, extension, force_install);
+	InstallExtensionInternal(config, &client_config, fs, local_path, extension, force_install, repository);
 }
 
 unsafe_unique_array<data_t> ReadExtensionFileFromDisk(FileSystem &fs, const string &path, idx_t &file_size) {
@@ -156,7 +158,8 @@ void WriteExtensionFileToDisk(FileSystem &fs, const string &path, void *data, id
 }
 
 void ExtensionHelper::InstallExtensionInternal(DBConfig &config, ClientConfig *client_config, FileSystem &fs,
-                                               const string &local_path, const string &extension, bool force_install) {
+                                               const string &local_path, const string &extension, bool force_install,
+                                               const string &repository) {
 #ifdef DUCKDB_DISABLE_EXTENSION_LOAD
 	throw PermissionException("Installing external extensions is disabled through a compile time flag");
 #else
@@ -181,6 +184,9 @@ void ExtensionHelper::InstallExtensionInternal(DBConfig &config, ClientConfig *c
 		auto in_buffer = ReadExtensionFileFromDisk(fs, extension, file_size);
 		WriteExtensionFileToDisk(fs, temp_path, in_buffer.get(), file_size);
 
+		if (fs.FileExists(local_extension_path) && force_install) {
+			fs.RemoveFile(local_extension_path);
+		}
 		fs.MoveFile(temp_path, local_extension_path);
 		return;
 	} else if (StringUtil::Contains(extension, "/") && !is_http_url) {
@@ -191,10 +197,17 @@ void ExtensionHelper::InstallExtensionInternal(DBConfig &config, ClientConfig *c
 	throw BinderException("Remote extension installation is disabled through configuration");
 #else
 
-	string default_endpoint = "http://extensions.duckdb.org";
 	string versioned_path = "/${REVISION}/${PLATFORM}/${NAME}.duckdb_extension.gz";
 	string custom_endpoint = client_config ? client_config->custom_extension_repo : string();
-	string &endpoint = !custom_endpoint.empty() ? custom_endpoint : default_endpoint;
+	string endpoint;
+	if (!repository.empty()) {
+		endpoint = repository;
+	} else if (!custom_endpoint.empty()) {
+		endpoint = custom_endpoint;
+	} else {
+		endpoint = "http://extensions.duckdb.org";
+	}
+
 	string url_template = endpoint + versioned_path;
 
 	if (is_http_url) {
@@ -213,6 +226,28 @@ void ExtensionHelper::InstallExtensionInternal(DBConfig &config, ClientConfig *c
 		throw IOException("No slash in URL template");
 	}
 
+	// Special case to install extension from a local file, useful for testing
+	if (!StringUtil::Contains(endpoint, "http://")) {
+		string file = fs.ConvertSeparators(url);
+		if (!fs.FileExists(file)) {
+			// check for non-gzipped variant
+			file = file.substr(0, file.size() - 3);
+			if (!fs.FileExists(file)) {
+				throw IOException("Failed to copy local extension \"%s\" at PATH \"%s\"\n", extension_name, file);
+			}
+		}
+		auto read_handle = fs.OpenFile(file, FileFlags::FILE_FLAGS_READ);
+		auto test_data = std::unique_ptr<unsigned char[]> {new unsigned char[read_handle->GetFileSize()]};
+		read_handle->Read(test_data.get(), read_handle->GetFileSize());
+		WriteExtensionFileToDisk(fs, temp_path, (void *)test_data.get(), read_handle->GetFileSize());
+
+		if (fs.FileExists(local_extension_path) && force_install) {
+			fs.RemoveFile(local_extension_path);
+		}
+		fs.MoveFile(temp_path, local_extension_path);
+		return;
+	}
+
 	// Push the substring [last, next) on to splits
 	auto hostname_without_http = no_http.substr(0, next);
 	auto url_local_part = no_http.substr(next);
@@ -243,6 +278,10 @@ void ExtensionHelper::InstallExtensionInternal(DBConfig &config, ClientConfig *c
 	auto decompressed_body = GZipFileSystem::UncompressGZIPString(res->body);
 
 	WriteExtensionFileToDisk(fs, temp_path, (void *)decompressed_body.data(), decompressed_body.size());
+
+	if (fs.FileExists(local_extension_path) && force_install) {
+		fs.RemoveFile(local_extension_path);
+	}
 	fs.MoveFile(temp_path, local_extension_path);
 #endif
 #endif

package/src/duckdb/src/main/settings/settings.cpp

@@ -526,6 +526,52 @@ Value CustomExtensionRepository::GetSetting(ClientContext &context) {
 	return Value(ClientConfig::GetConfig(context).custom_extension_repo);
 }
 
+//===--------------------------------------------------------------------===//
+// Autoload Extension Repository
+//===--------------------------------------------------------------------===//
+void AutoloadExtensionRepository::ResetLocal(ClientContext &context) {
+	ClientConfig::GetConfig(context).autoinstall_extension_repo = ClientConfig().autoinstall_extension_repo;
+}
+
+void AutoloadExtensionRepository::SetLocal(ClientContext &context, const Value &input) {
+	ClientConfig::GetConfig(context).autoinstall_extension_repo = StringUtil::Lower(input.ToString());
+}
+
+Value AutoloadExtensionRepository::GetSetting(ClientContext &context) {
+	return Value(ClientConfig::GetConfig(context).autoinstall_extension_repo);
+}
+
+//===--------------------------------------------------------------------===//
+// Autoinstall Known Extensions
+//===--------------------------------------------------------------------===//
+void AutoinstallKnownExtensions::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
+	config.options.autoinstall_known_extensions = input.GetValue<bool>();
+}
+
+void AutoinstallKnownExtensions::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
+	config.options.autoinstall_known_extensions = DBConfig().options.autoinstall_known_extensions;
+}
+
+Value AutoinstallKnownExtensions::GetSetting(ClientContext &context) {
+	auto &config = DBConfig::GetConfig(context);
+	return Value::BOOLEAN(config.options.autoinstall_known_extensions);
+}
+//===--------------------------------------------------------------------===//
+// Autoload Known Extensions
+//===--------------------------------------------------------------------===//
+void AutoloadKnownExtensions::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
+	config.options.autoload_known_extensions = input.GetValue<bool>();
+}
+
+void AutoloadKnownExtensions::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
+	config.options.autoload_known_extensions = DBConfig().options.autoload_known_extensions;
+}
+
+Value AutoloadKnownExtensions::GetSetting(ClientContext &context) {
+	auto &config = DBConfig::GetConfig(context);
+	return Value::BOOLEAN(config.options.autoload_known_extensions);
+}
+
 //===--------------------------------------------------------------------===//
 // Enable Progress Bar
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/parser/transform/statement/transform_load.cpp

@@ -9,6 +9,7 @@ unique_ptr<LoadStatement> Transformer::TransformLoad(duckdb_libpgquery::PGLoadSt
 	auto load_stmt = make_uniq<LoadStatement>();
 	auto load_info = make_uniq<LoadInfo>();
 	load_info->filename = std::string(stmt.filename);
+	load_info->repository = std::string(stmt.repository);
 	switch (stmt.load_type) {
 	case duckdb_libpgquery::PG_LOAD_TYPE_LOAD:
 		load_info->load_type = LoadType::LOAD;

package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp

@@ -10,6 +10,7 @@
 #include "duckdb/planner/operator/logical_get.hpp"
 #include "duckdb/parser/statement/select_statement.hpp"
 #include "duckdb/common/string_util.hpp"
+#include "duckdb/main/extension_helper.hpp"
 #include "duckdb/parser/tableref/table_function_ref.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/planner/tableref/bound_dummytableref.hpp"
@@ -17,6 +18,62 @@
 
 namespace duckdb {
 
+static bool TryLoadExtensionForReplacementScan(ClientContext &context, const string &table_name) {
+	auto lower_name = StringUtil::Lower(table_name);
+	auto &dbconfig = DBConfig::GetConfig(context);
+
+	if (!dbconfig.options.autoload_known_extensions) {
+		return false;
+	}
+
+	for (const auto &entry : EXTENSION_FILE_POSTFIXES) {
+		if (StringUtil::EndsWith(lower_name, entry.name)) {
+			ExtensionHelper::AutoLoadExtension(context, entry.extension);
+			return true;
+		}
+	}
+
+	for (const auto &entry : EXTENSION_FILE_CONTAINS) {
+		if (StringUtil::Contains(lower_name, entry.name)) {
+			ExtensionHelper::AutoLoadExtension(context, entry.extension);
+			return true;
+		}
+	}
+
+	return false;
+}
+
+unique_ptr<BoundTableRef> Binder::BindWithReplacementScan(ClientContext &context, const string &table_name,
+                                                          BaseTableRef &ref) {
+	auto &config = DBConfig::GetConfig(context);
+	if (context.config.use_replacement_scans) {
+		for (auto &scan : config.replacement_scans) {
+			auto replacement_function = scan.function(context, table_name, scan.data.get());
+			if (replacement_function) {
+				if (!ref.alias.empty()) {
+					// user-provided alias overrides the default alias
+					replacement_function->alias = ref.alias;
+				} else if (replacement_function->alias.empty()) {
+					// if the replacement scan itself did not provide an alias we use the table name
+					replacement_function->alias = ref.table_name;
+				}
+				if (replacement_function->type == TableReferenceType::TABLE_FUNCTION) {
+					auto &table_function = replacement_function->Cast<TableFunctionRef>();
+					table_function.column_name_alias = ref.column_name_alias;
+				} else if (replacement_function->type == TableReferenceType::SUBQUERY) {
+					auto &subquery = replacement_function->Cast<SubqueryRef>();
+					subquery.column_name_alias = ref.column_name_alias;
+				} else {
+					throw InternalException("Replacement scan should return either a table function or a subquery");
+				}
+				return Bind(*replacement_function);
+			}
+		}
+	}
+
+	return nullptr;
+}
+
 unique_ptr<BoundTableRef> Binder::Bind(BaseTableRef &ref) {
 	QueryErrorContext error_context(root_statement, ref.query_location);
 	// CTEs and views are also referred to using BaseTableRefs, hence need to distinguish here
@@ -108,29 +165,18 @@ unique_ptr<BoundTableRef> Binder::Bind(BaseTableRef &ref) {
 		}
 		table_name += (!table_name.empty() ? "." : "") + ref.table_name;
 		// table could not be found: try to bind a replacement scan
-		auto &config = DBConfig::GetConfig(context);
-		if (context.config.use_replacement_scans) {
-			for (auto &scan : config.replacement_scans) {
-				auto replacement_function = scan.function(context, table_name, scan.data.get());
-				if (replacement_function) {
-					if (!ref.alias.empty()) {
-						// user-provided alias overrides the default alias
-						replacement_function->alias = ref.alias;
-					} else if (replacement_function->alias.empty()) {
-						// if the replacement scan itself did not provide an alias we use the table name
-						replacement_function->alias = ref.table_name;
-					}
-					if (replacement_function->type == TableReferenceType::TABLE_FUNCTION) {
-						auto &table_function = replacement_function->Cast<TableFunctionRef>();
-						table_function.column_name_alias = ref.column_name_alias;
-					} else if (replacement_function->type == TableReferenceType::SUBQUERY) {
-						auto &subquery = replacement_function->Cast<SubqueryRef>();
-						subquery.column_name_alias = ref.column_name_alias;
-					} else {
-						throw InternalException("Replacement scan should return either a table function or a subquery");
-					}
-					return Bind(*replacement_function);
-				}
+		// Try replacement scan bind
+		auto replacement_scan_bind_result = BindWithReplacementScan(context, table_name, ref);
+		if (replacement_scan_bind_result) {
+			return replacement_scan_bind_result;
+		}
+
+		// Try autoloading an extension, then retry the replacement scan bind
+		auto extension_loaded = TryLoadExtensionForReplacementScan(context, table_name);
+		if (extension_loaded) {
+			replacement_scan_bind_result = BindWithReplacementScan(context, table_name, ref);
+			if (replacement_scan_bind_result) {
+				return replacement_scan_bind_result;
 			}
 		}
 

package/src/duckdb/src/planner/expression_binder/order_binder.cpp

@@ -1,14 +1,14 @@
 #include "duckdb/planner/expression_binder/order_binder.hpp"
 
 #include "duckdb/parser/expression/columnref_expression.hpp"
-#include "duckdb/parser/expression/positional_reference_expression.hpp"
 #include "duckdb/parser/expression/constant_expression.hpp"
+#include "duckdb/parser/expression/parameter_expression.hpp"
+#include "duckdb/parser/expression/positional_reference_expression.hpp"
 #include "duckdb/parser/expression/star_expression.hpp"
 #include "duckdb/parser/query_node/select_node.hpp"
-#include "duckdb/planner/expression_binder.hpp"
-#include "duckdb/parser/expression/parameter_expression.hpp"
-#include "duckdb/planner/expression/bound_parameter_expression.hpp"
 #include "duckdb/planner/binder.hpp"
+#include "duckdb/planner/expression/bound_parameter_expression.hpp"
+#include "duckdb/planner/expression_binder.hpp"
 
 namespace duckdb {
 
@@ -42,6 +42,7 @@ unique_ptr<Expression> OrderBinder::CreateExtraReference(unique_ptr<ParsedExpres
 	if (!extra_list) {
 		throw InternalException("CreateExtraReference called without extra_list");
 	}
+	projection_map[*expr] = extra_list->size();
 	auto result = CreateProjectionReference(*expr, extra_list->size());
 	extra_list->push_back(std::move(expr));
 	return result;

package/src/duckdb/src/storage/serialization/serialize_parse_info.cpp

@@ -296,12 +296,14 @@ void LoadInfo::FormatSerialize(FormatSerializer &serializer) const {
 	ParseInfo::FormatSerialize(serializer);
 	serializer.WriteProperty(200, "filename", filename);
 	serializer.WriteProperty(201, "load_type", load_type);
+	serializer.WriteProperty(202, "repository", repository);
 }
 
 unique_ptr<ParseInfo> LoadInfo::FormatDeserialize(FormatDeserializer &deserializer) {
 	auto result = duckdb::unique_ptr<LoadInfo>(new LoadInfo());
 	deserializer.ReadProperty(200, "filename", result->filename);
 	deserializer.ReadProperty(201, "load_type", result->load_type);
+	deserializer.ReadProperty(202, "repository", result->repository);
 	return std::move(result);
 }
 

package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp

@@ -1872,6 +1872,7 @@ typedef enum PGLoadInstallType { PG_LOAD_TYPE_LOAD,  PG_LOAD_TYPE_INSTALL, PG_LO
 typedef struct PGLoadStmt {
 	PGNodeTag type;
 	const char *filename; /* file to load */
+	const char *repository; /* optionally, the repository to load from */
 	PGLoadInstallType load_type;
 } PGLoadStmt;