duckdb 0.7.1-dev7.0 → 0.7.1

package/src/duckdb/src/include/duckdb/planner/expression_binder/index_binder.hpp

@@ -10,20 +10,27 @@
 
 #include "duckdb/planner/expression_binder.hpp"
 #include "duckdb/common/unordered_map.hpp"
+#include "duckdb/parser/parsed_data/create_index_info.hpp"
+#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 
 namespace duckdb {
 class BoundColumnRefExpression;
 
-//! The INDEX binder is responsible for binding an expression within an Index statement
+//! The IndexBinder is responsible for binding an expression within an index statement
 class IndexBinder : public ExpressionBinder {
 public:
-	IndexBinder(Binder &binder, ClientContext &context);
+	IndexBinder(Binder &binder, ClientContext &context, TableCatalogEntry *table = nullptr,
+	            CreateIndexInfo *info = nullptr);
 
 protected:
 	BindResult BindExpression(unique_ptr<ParsedExpression> *expr_ptr, idx_t depth,
 	                          bool root_expression = false) override;
-
 	string UnsupportedAggregateMessage() override;
+
+private:
+	// only for WAL replay
+	TableCatalogEntry *table;
+	CreateIndexInfo *info;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/operator/logical_execute.hpp

@@ -32,11 +32,7 @@ protected:
 		// already resolved
 	}
 	vector<ColumnBinding> GetColumnBindings() override {
-		vector<ColumnBinding> bindings;
-		for (idx_t i = 0; i < types.size(); i++) {
-			bindings.push_back(ColumnBinding(0, i));
-		}
-		return bindings;
+		return GenerateColumnBindings(0, types.size());
 	}
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/operator/logical_show.hpp

@@ -33,8 +33,7 @@ protected:
 		         LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR};
 	}
 	vector<ColumnBinding> GetColumnBindings() override {
-		return {ColumnBinding(0, 0), ColumnBinding(0, 1), ColumnBinding(0, 2),
-		        ColumnBinding(0, 3), ColumnBinding(0, 4), ColumnBinding(0, 5)};
+		return GenerateColumnBindings(0, types.size());
 	}
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/buffer_manager.hpp

@@ -23,6 +23,11 @@ class DatabaseInstance;
 class TemporaryDirectoryHandle;
 struct EvictionQueue;
 
+struct TemporaryFileInformation {
+	string path;
+	idx_t size;
+};
+
 //! The buffer manager is in charge of handling memory management for the database. It hands out memory buffers that can
 //! be used by the database internally.
 //
@@ -98,6 +103,9 @@ public:
 	DUCKDB_API void ReserveMemory(idx_t size);
 	DUCKDB_API void FreeReservedMemory(idx_t size);
 
+	//! Returns a list of all temporary files
+	vector<TemporaryFileInformation> GetTemporaryFiles();
+
 private:
 	//! Register an in-memory buffer of arbitrary size, as long as it is >= BLOCK_SIZE. can_destroy signifies whether or
 	//! not the buffer can be destroyed when unpinned, or whether or not it needs to be written to a temporary file so

package/src/duckdb/src/include/duckdb/storage/data_table.hpp

@@ -122,6 +122,12 @@ public:
 	void UpdateColumn(TableCatalogEntry &table, ClientContext &context, Vector &row_ids,
 	                  const vector<column_t> &column_path, DataChunk &updates);
 
+	//! Add an index to the DataTable. NOTE: for CREATE (UNIQUE) INDEX statements, we use the PhysicalCreateIndex
+	//! operator. This function is only used during the WAL replay, and is a much less performant index creation
+	//! approach.
+	void WALAddIndex(ClientContext &context, unique_ptr<Index> index,
+	                 const vector<unique_ptr<Expression>> &expressions);
+
 	//! Fetches an append lock
 	void AppendLock(TableAppendState &state);
 	//! Begin appending structs to this table, obtaining necessary locks, etc
@@ -176,7 +182,7 @@ public:
 	static bool IsForeignKeyIndex(const vector<PhysicalIndex> &fk_keys, Index &index, ForeignKeyType fk_type);
 
 	//! Initializes a special scan that is used to create an index on the table, it keeps locks on the table
-	void InitializeCreateIndexScan(CreateIndexScanState &state, const vector<column_t> &column_ids);
+	void InitializeWALCreateIndexScan(CreateIndexScanState &state, const vector<column_t> &column_ids);
 	//! Scans the next chunk for the CREATE INDEX operator
 	bool CreateIndexScan(TableScanState &state, DataChunk &result, TableScanType type);
 

package/src/duckdb/src/include/duckdb/storage/index.hpp

@@ -40,11 +40,11 @@ public:
 	IndexType type;
 	//! Associated table io manager
 	TableIOManager &table_io_manager;
-	//! Column identifiers to extract from the base table
+	//! Column identifiers to extract key columns from the base table
 	vector<column_t> column_ids;
-	//! Unordered_set of column_ids used by the index
+	//! Unordered set of column_ids used by the index
 	unordered_set<column_t> column_id_set;
-	//! Unbound expressions used by the index
+	//! Unbound expressions used by the index during optimizations
 	vector<unique_ptr<Expression>> unbound_expressions;
 	//! The physical types stored in the index
 	vector<PhysicalType> types;
@@ -64,94 +64,103 @@ public:
 	bool track_memory;
 
 public:
-	//! Initialize a scan on the index with the given expression and column ids
-	//! to fetch from the base table when we only have one query predicate
+	//! Initialize a single predicate scan on the index with the given expression and column IDs
 	virtual unique_ptr<IndexScanState> InitializeScanSinglePredicate(const Transaction &transaction, const Value &value,
 	                                                                 ExpressionType expressionType) = 0;
-	//! Initialize a scan on the index with the given expression and column ids
-	//! to fetch from the base table for two query predicates
+	//! Initialize a two predicate scan on the index with the given expression and column IDs
 	virtual unique_ptr<IndexScanState> InitializeScanTwoPredicates(Transaction &transaction, const Value &low_value,
 	                                                               ExpressionType low_expression_type,
 	                                                               const Value &high_value,
 	                                                               ExpressionType high_expression_type) = 0;
-	//! Perform a lookup on the index, fetching up to max_count result ids. Returns true if all row ids were fetched,
-	//! and false otherwise.
+	//! Performs a lookup on the index, fetching up to max_count result IDs. Returns true if all row IDs were fetched,
+	//! and false otherwise
 	virtual bool Scan(Transaction &transaction, DataTable &table, IndexScanState &state, idx_t max_count,
 	                  vector<row_t> &result_ids) = 0;
 
 	//! Obtain a lock on the index
 	virtual void InitializeLock(IndexLock &state);
-	//! Called when data is appended to the index. The lock obtained from InitializeAppend must be held
+	//! Called when data is appended to the index. The lock obtained from InitializeLock must be held
 	virtual bool Append(IndexLock &state, DataChunk &entries, Vector &row_identifiers) = 0;
+	//! Obtains a lock and calls Append while holding that lock
 	bool Append(DataChunk &entries, Vector &row_identifiers);
-	//! Verify that data can be appended to the index
+	//! Verify that data can be appended to the index without a constraint violation
 	virtual void VerifyAppend(DataChunk &chunk) = 0;
-	//! Verify that data can be appended to the index
+	//! Verify that data can be appended to the index without a constraint violation using the conflict manager
 	virtual void VerifyAppend(DataChunk &chunk, ConflictManager &conflict_manager) = 0;
-	//! Verify that data can be appended to the index for foreign key constraint
-	virtual void VerifyAppendForeignKey(DataChunk &chunk) = 0;
-	//! Verify that data can be delete from the index for foreign key constraint
-	virtual void VerifyDeleteForeignKey(DataChunk &chunk) = 0;
+	//! Performs constraint checking for a chunk of input data
+	virtual void CheckConstraintsForChunk(DataChunk &input, ConflictManager &conflict_manager) = 0;
 
-	//! Called when data inside the index is Deleted
+	//! Delete a chunk of entries from the index. The lock obtained from InitializeLock must be held
 	virtual void Delete(IndexLock &state, DataChunk &entries, Vector &row_identifiers) = 0;
+	//! Obtains a lock and calls Delete while holding that lock
 	void Delete(DataChunk &entries, Vector &row_identifiers);
 
-	//! Insert data into the index. Does not lock the index.
+	//! Insert a chunk of entries into the index
 	virtual bool Insert(IndexLock &lock, DataChunk &input, Vector &row_identifiers) = 0;
 
-	//! Merge other_index into this index.
+	//! Merge another index into this index. The lock obtained from InitializeLock must be held, and the other
+	//! index must also be locked during the merge
 	virtual bool MergeIndexes(IndexLock &state, Index *other_index) = 0;
+	//! Obtains a lock and calls MergeIndexes while holding that lock
 	bool MergeIndexes(Index *other_index);
 
 	//! Returns the string representation of an index
 	virtual string ToString() = 0;
-	//! Verifies that the memory_size value of the index matches its actual size
+	//! Verifies that the in-memory size value of the index matches its actual size
 	virtual void Verify() = 0;
-
-	//! Returns true if the index is affected by updates on the specified column ids, and false otherwise
+	//! Increases the memory size by the difference between the old size and the current size
+	//! and performs verifications
+	virtual void IncreaseAndVerifyMemorySize(idx_t old_memory_size) = 0;
+
+	//! Increases the in-memory size value
+	inline void IncreaseMemorySize(idx_t size) {
+		memory_size += size;
+	};
+	//! Decreases the in-memory size value
+	inline void DecreaseMemorySize(idx_t size) {
+		D_ASSERT(memory_size >= size);
+		memory_size -= size;
+	};
+
+	//! Returns true if the index is affected by updates on the specified column IDs, and false otherwise
 	bool IndexIsUpdated(const vector<PhysicalIndex> &column_ids) const;
 
-	//! Returns how many of the input values were found in the 'input' chunk, with the option to also record what those
-	//! matches were. For this purpose, nulls count as a match, and are returned in 'null_count'
-	virtual void LookupValues(DataChunk &input, ConflictManager &conflict_manager) = 0;
-
 	//! Returns unique flag
 	bool IsUnique() {
 		return (constraint_type == IndexConstraintType::UNIQUE || constraint_type == IndexConstraintType::PRIMARY);
 	}
-	//! Returns primary flag
+	//! Returns primary key flag
 	bool IsPrimary() {
 		return (constraint_type == IndexConstraintType::PRIMARY);
 	}
-	//! Returns foreign flag
+	//! Returns foreign key flag
 	bool IsForeign() {
 		return (constraint_type == IndexConstraintType::FOREIGN);
 	}
-	//! Serializes the index and returns the pair of block_id offset positions
-	virtual BlockPointer Serialize(duckdb::MetaBlockWriter &writer);
-	BlockPointer GetBlockPointer();
 
-	//! Returns block/offset of where index was most recently serialized.
+	//! Serializes the index and returns the pair of block_id offset positions
+	virtual BlockPointer Serialize(MetaBlockWriter &writer);
+	//! Returns the serialized data pointer to the block and offset of the serialized index
 	BlockPointer GetSerializedDataPointer() const {
 		return serialized_data_pointer;
 	}
 
-protected:
+	//! Execute the index expressions on an input chunk
 	void ExecuteExpressions(DataChunk &input, DataChunk &result);
 
-	//! Lock used for updating the index
+protected:
+	//! Lock used for any changes to the index
 	mutex lock;
-
-	//! Pointer to most recently checkpointed index data.
+	//! Pointer to serialized index data
 	BlockPointer serialized_data_pointer;
 
 private:
-	//! Bound expressions used by the index
+	//! Bound expressions used during expression execution
 	vector<unique_ptr<Expression>> bound_expressions;
-	//! Expression executor for the index expressions
+	//! Expression executor to execute the index expressions
 	ExpressionExecutor executor;
 
+	//! Bind the unbound expressions of the index
 	unique_ptr<Expression> BindExpression(unique_ptr<Expression> expr);
 };
 

package/src/duckdb/src/include/duckdb/storage/storage_extension.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/enums/access_mode.hpp"
+#include "duckdb/parser/tableref/table_function_ref.hpp"
 
 namespace duckdb {
 class AttachedDatabase;
@@ -27,11 +28,17 @@ typedef unique_ptr<Catalog> (*attach_function_t)(StorageExtensionInfo *storage_i
                                                  const string &name, AttachInfo &info, AccessMode access_mode);
 typedef unique_ptr<TransactionManager> (*create_transaction_manager_t)(StorageExtensionInfo *storage_info,
                                                                        AttachedDatabase &db, Catalog &catalog);
+typedef unique_ptr<TableFunctionRef> (*create_database_t)(StorageExtensionInfo *info, ClientContext &context,
+                                                          const string &database_name, const string &source_path);
+typedef unique_ptr<TableFunctionRef> (*drop_database_t)(StorageExtensionInfo *storage_info, ClientContext &context,
+                                                        const string &database_name);
 
 class StorageExtension {
 public:
 	attach_function_t attach;
 	create_transaction_manager_t create_transaction_manager;
+	create_database_t create_database;
+	drop_database_t drop_database;
 
 	//! Additional info passed to the various storage functions
 	shared_ptr<StorageExtensionInfo> storage_info;

package/src/duckdb/src/include/duckdb/storage/table/update_segment.hpp

@@ -23,6 +23,8 @@ struct UpdateNode;
 class UpdateSegment {
 public:
 	UpdateSegment(ColumnData &column_data);
+	// Construct a duplicate of 'other' with 'new_owner' as it's column data
+	UpdateSegment(UpdateSegment &other, ColumnData &new_owner);
 	~UpdateSegment();
 
 	ColumnData &column_data;

package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp

@@ -15,6 +15,7 @@
 #include "duckdb/catalog/catalog_entry/scalar_macro_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/sequence_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/table_macro_catalog_entry.hpp"
+#include "duckdb/catalog/catalog_entry/index_catalog_entry.hpp"
 #include "duckdb/main/attached_database.hpp"
 #include "duckdb/storage/storage_info.hpp"
 
@@ -77,6 +78,9 @@ protected:
 	void ReplayCreateTableMacro();
 	void ReplayDropTableMacro();
 
+	void ReplayCreateIndex();
+	void ReplayDropIndex();
+
 	void ReplayUseTable();
 	void ReplayInsert();
 	void ReplayDelete();
@@ -125,6 +129,9 @@ public:
 	void WriteCreateTableMacro(TableMacroCatalogEntry *entry);
 	void WriteDropTableMacro(TableMacroCatalogEntry *entry);
 
+	void WriteCreateIndex(IndexCatalogEntry *entry);
+	void WriteDropIndex(IndexCatalogEntry *entry);
+
 	void WriteCreateType(TypeCatalogEntry *entry);
 	void WriteDropType(TypeCatalogEntry *entry);
 	//! Sets the table used for subsequent insert/delete/update commands

package/src/duckdb/src/main/client_context.cpp

@@ -665,6 +665,7 @@ unique_ptr<PendingQueryResult> ClientContext::PendingStatementOrPreparedStatemen
 			statement = std::move(copied_statement);
 			break;
 		}
+#ifndef DUCKDB_ALTERNATIVE_VERIFY
 		case StatementType::COPY_STATEMENT:
 		case StatementType::INSERT_STATEMENT:
 		case StatementType::DELETE_STATEMENT:
@@ -685,6 +686,7 @@ unique_ptr<PendingQueryResult> ClientContext::PendingStatementOrPreparedStatemen
 			statement = std::move(parser.statements[0]);
 			break;
 		}
+#endif
 		default:
 			statement = std::move(copied_statement);
 			break;

package/src/duckdb/src/main/config.cpp

@@ -67,6 +67,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_LOCAL(EnableProgressBarPrintSetting),
                                                  DUCKDB_GLOBAL(ExperimentalParallelCSVSetting),
                                                  DUCKDB_LOCAL(ExplainOutputSetting),
+                                                 DUCKDB_GLOBAL(ExtensionDirectorySetting),
                                                  DUCKDB_GLOBAL(ExternalThreadsSetting),
                                                  DUCKDB_LOCAL(FileSearchPathSetting),
                                                  DUCKDB_GLOBAL(ForceCompressionSetting),

package/src/duckdb/src/main/database.cpp

@@ -144,9 +144,15 @@ unique_ptr<AttachedDatabase> DatabaseInstance::CreateAttachedDatabase(AttachInfo
 		if (entry == config.storage_extensions.end()) {
 			throw BinderException("Unrecognized storage type \"%s\"", type);
 		}
-		// use storage extension to create the initial database
-		attached_database = make_unique<AttachedDatabase>(*this, Catalog::GetSystemCatalog(*this), *entry->second,
-		                                                  info.name, info, access_mode);
+
+		if (entry->second->attach != nullptr && entry->second->create_transaction_manager != nullptr) {
+			// use storage extension to create the initial database
+			attached_database = make_unique<AttachedDatabase>(*this, Catalog::GetSystemCatalog(*this), *entry->second,
+			                                                  info.name, info, access_mode);
+		} else {
+			attached_database = make_unique<AttachedDatabase>(*this, Catalog::GetSystemCatalog(*this), info.name,
+			                                                  info.path, access_mode);
+		}
 	} else {
 		// check if this is an in-memory database or not
 		attached_database =
@@ -200,6 +206,7 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
 	AttachInfo info;
 	info.name = AttachedDatabase::ExtractDatabaseName(config.options.database_path);
 	info.path = config.options.database_path;
+
 	auto attached_database = CreateAttachedDatabase(info, database_type, config.options.access_mode);
 	auto initial_database = attached_database.get();
 	{
@@ -356,7 +363,8 @@ idx_t DuckDB::NumberOfThreads() {
 }
 
 bool DatabaseInstance::ExtensionIsLoaded(const std::string &name) {
-	return loaded_extensions.find(name) != loaded_extensions.end();
+	auto extension_name = ExtensionHelper::GetExtensionName(name);
+	return loaded_extensions.find(extension_name) != loaded_extensions.end();
 }
 
 bool DuckDB::ExtensionIsLoaded(const std::string &name) {
@@ -364,7 +372,8 @@ bool DuckDB::ExtensionIsLoaded(const std::string &name) {
 }
 
 void DatabaseInstance::SetExtensionLoaded(const std::string &name) {
-	loaded_extensions.insert(name);
+	auto extension_name = ExtensionHelper::GetExtensionName(name);
+	loaded_extensions.insert(extension_name);
 }
 
 bool DatabaseInstance::TryGetCurrentSetting(const std::string &key, Value &result) {

package/src/duckdb/src/main/extension/extension_alias.cpp

@@ -24,8 +24,9 @@ ExtensionAlias ExtensionHelper::GetExtensionAlias(idx_t index) {
 }
 
 string ExtensionHelper::ApplyExtensionAlias(string extension_name) {
+	auto lname = StringUtil::Lower(extension_name);
 	for (idx_t index = 0; internal_aliases[index].alias; index++) {
-		if (extension_name == internal_aliases[index].alias) {
+		if (lname == internal_aliases[index].alias) {
 			return internal_aliases[index].extension;
 		}
 	}

package/src/duckdb/src/main/extension/extension_helper.cpp

@@ -115,6 +115,21 @@ DefaultExtension ExtensionHelper::GetDefaultExtension(idx_t index) {
 	return internal_extensions[index];
 }
 
+//===--------------------------------------------------------------------===//
+// Allow Auto-Install Extensions
+//===--------------------------------------------------------------------===//
+static const char *auto_install[] = {"motherduck", "postgres_scanner", "sqlite_scanner", nullptr};
+
+bool ExtensionHelper::AllowAutoInstall(const string &extension) {
+	auto lcase = StringUtil::Lower(extension);
+	for (idx_t i = 0; auto_install[i]; i++) {
+		if (lcase == auto_install[i]) {
+			return true;
+		}
+	}
+	return false;
+}
+
 //===--------------------------------------------------------------------===//
 // Load Statically Compiled Extension
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/main/extension/extension_install.cpp

@@ -38,22 +38,57 @@ const vector<string> ExtensionHelper::PathComponents() {
 	return vector<string> {".duckdb", "extensions", GetVersionDirectoryName(), DuckDB::Platform()};
 }
 
-string ExtensionHelper::ExtensionDirectory(ClientContext &context) {
-	auto &fs = FileSystem::GetFileSystem(context);
-	string local_path = fs.GetHomeDirectory(FileSystem::GetFileOpener(context));
-	if (!fs.DirectoryExists(local_path)) {
-		throw IOException("Can't find the home directory at '%s'\nSpecify a home directory using the SET "
-		                  "home_directory='/path/to/dir' option.",
-		                  local_path);
+string ExtensionHelper::ExtensionDirectory(DBConfig &config, FileSystem &fs, FileOpener *opener) {
+	string extension_directory;
+	if (!config.options.extension_directory.empty()) { // create the extension directory if not present
+		extension_directory = config.options.extension_directory;
+		// TODO this should probably live in the FileSystem
+		// convert random separators to platform-canonic
+		extension_directory = fs.ConvertSeparators(extension_directory);
+		// expand ~ in extension directory
+		extension_directory = fs.ExpandPath(extension_directory, opener);
+		if (!fs.DirectoryExists(extension_directory)) {
+			auto sep = fs.PathSeparator();
+			auto splits = StringUtil::Split(extension_directory, sep);
+			D_ASSERT(!splits.empty());
+			string extension_directory_prefix;
+			if (StringUtil::StartsWith(extension_directory, sep)) {
+				extension_directory_prefix = sep; // this is swallowed by Split otherwise
+			}
+			for (auto &split : splits) {
+				extension_directory_prefix = extension_directory_prefix + split + sep;
+				if (!fs.DirectoryExists(extension_directory_prefix)) {
+					fs.CreateDirectory(extension_directory_prefix);
+				}
+			}
+		}
+	} else { // otherwise default to home
+		string home_directory = fs.GetHomeDirectory(opener);
+		// exception if the home directory does not exist, don't create whatever we think is home
+		if (!fs.DirectoryExists(home_directory)) {
+			throw IOException("Can't find the home directory at '%s'\nSpecify a home directory using the SET "
+			                  "home_directory='/path/to/dir' option.",
+			                  home_directory);
+		}
+		extension_directory = home_directory;
 	}
+	D_ASSERT(fs.DirectoryExists(extension_directory));
+
 	auto path_components = PathComponents();
 	for (auto &path_ele : path_components) {
-		local_path = fs.JoinPath(local_path, path_ele);
-		if (!fs.DirectoryExists(local_path)) {
-			fs.CreateDirectory(local_path);
+		extension_directory = fs.JoinPath(extension_directory, path_ele);
+		if (!fs.DirectoryExists(extension_directory)) {
+			fs.CreateDirectory(extension_directory);
 		}
 	}
-	return local_path;
+	return extension_directory;
+}
+
+string ExtensionHelper::ExtensionDirectory(ClientContext &context) {
+	auto &config = DBConfig::GetConfig(context);
+	auto &fs = FileSystem::GetFileSystem(context);
+	auto opener = FileSystem::GetFileOpener(context);
+	return ExtensionDirectory(config, fs, opener);
 }
 
 bool ExtensionHelper::CreateSuggestions(const string &extension_name, string &message) {
@@ -75,15 +110,24 @@ bool ExtensionHelper::CreateSuggestions(const string &extension_name, string &me
 	return false;
 }
 
+void ExtensionHelper::InstallExtension(DBConfig &config, FileSystem &fs, const string &extension, bool force_install) {
+	string local_path = ExtensionDirectory(config, fs, nullptr);
+	InstallExtensionInternal(config, nullptr, fs, local_path, extension, force_install);
+}
+
 void ExtensionHelper::InstallExtension(ClientContext &context, const string &extension, bool force_install) {
 	auto &config = DBConfig::GetConfig(context);
-	if (!config.options.enable_external_access) {
-		throw PermissionException("Installing extensions is disabled through configuration");
-	}
 	auto &fs = FileSystem::GetFileSystem(context);
-
 	string local_path = ExtensionDirectory(context);
+	auto &client_config = ClientConfig::GetConfig(context);
+	InstallExtensionInternal(config, &client_config, fs, local_path, extension, force_install);
+}
 
+void ExtensionHelper::InstallExtensionInternal(DBConfig &config, ClientConfig *client_config, FileSystem &fs,
+                                               const string &local_path, const string &extension, bool force_install) {
+	if (!config.options.enable_external_access) {
+		throw PermissionException("Installing extensions is disabled through configuration");
+	}
 	auto extension_name = ApplyExtensionAlias(fs.ExtractBaseName(extension));
 
 	string local_extension_path = fs.JoinPath(local_path, extension_name + ".duckdb_extension");
@@ -123,7 +167,7 @@ void ExtensionHelper::InstallExtension(ClientContext &context, const string &ext
 
 	string default_endpoint = "http://extensions.duckdb.org";
 	string versioned_path = "/${REVISION}/${PLATFORM}/${NAME}.duckdb_extension.gz";
-	string &custom_endpoint = ClientConfig::GetConfig(context).custom_extension_repo;
+	string custom_endpoint = client_config ? client_config->custom_extension_repo : string();
 	string &endpoint = !custom_endpoint.empty() ? custom_endpoint : default_endpoint;
 	string url_template = endpoint + versioned_path;
 

package/src/duckdb/src/main/extension/extension_load.cpp

@@ -22,7 +22,8 @@ static T LoadFunctionFromDLL(void *dll, const string &function_name, const strin
 	return (T)function;
 }
 
-ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *opener, const string &extension) {
+bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileOpener *opener, const string &extension,
+                                     ExtensionInitResult &result, string &error) {
 	if (!config.options.enable_external_access) {
 		throw PermissionException("Loading external extensions is disabled through configuration");
 	}
@@ -31,8 +32,14 @@ ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *o
 	auto filename = fs.ConvertSeparators(extension);
 
 	// shorthand case
-	if (!StringUtil::Contains(extension, ".") && !StringUtil::Contains(extension, fs.PathSeparator())) {
-		string local_path = fs.GetHomeDirectory(opener);
+	if (!ExtensionHelper::IsFullPath(extension)) {
+		string local_path = !config.options.extension_directory.empty() ? config.options.extension_directory
+		                                                                : fs.GetHomeDirectory(opener);
+
+		// convert random separators to platform-canonic
+		local_path = fs.ConvertSeparators(local_path);
+		// expand ~ in extension directory
+		local_path = fs.ExpandPath(local_path, opener);
 		auto path_components = PathComponents();
 		for (auto &path_ele : path_components) {
 			local_path = fs.JoinPath(local_path, path_ele);
@@ -40,14 +47,14 @@ ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *o
 		string extension_name = ApplyExtensionAlias(extension);
 		filename = fs.JoinPath(local_path, extension_name + ".duckdb_extension");
 	}
-
 	if (!fs.FileExists(filename)) {
 		string message;
 		bool exact_match = ExtensionHelper::CreateSuggestions(extension, message);
 		if (exact_match) {
 			message += "\nInstall it first using \"INSTALL " + extension + "\".";
 		}
-		throw IOException("Extension \"%s\" not found.\n%s", filename, message);
+		error = StringUtil::Format("Extension \"%s\" not found.\n%s", filename, message);
+		return false;
 	}
 	if (!config.options.allow_unsigned_extensions) {
 		auto handle = fs.OpenFile(filename, FileFlags::FILE_FLAGS_READ);
@@ -114,16 +121,43 @@ ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *o
 		                            extension_version, engine_version);
 	}
 
-	ExtensionInitResult res;
-	res.basename = basename;
-	res.filename = filename;
-	res.lib_hdl = lib_hdl;
-	return res;
+	result.basename = basename;
+	result.filename = filename;
+	result.lib_hdl = lib_hdl;
+	return true;
+}
+
+ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *opener, const string &extension) {
+	string error;
+	ExtensionInitResult result;
+	if (!TryInitialLoad(config, opener, extension, result, error)) {
+		throw IOException(error);
+	}
+	return result;
+}
+
+bool ExtensionHelper::IsFullPath(const string &extension) {
+	return StringUtil::Contains(extension, ".") || StringUtil::Contains(extension, "/") ||
+	       StringUtil::Contains(extension, "\\");
+}
+
+string ExtensionHelper::GetExtensionName(const string &extension) {
+	if (!IsFullPath(extension)) {
+		return extension;
+	}
+	auto splits = StringUtil::Split(StringUtil::Replace(extension, "\\", "/"), '/');
+	if (splits.empty()) {
+		return extension;
+	}
+	splits = StringUtil::Split(splits.back(), '.');
+	if (splits.empty()) {
+		return extension;
+	}
+	return StringUtil::Lower(splits.front());
 }
 
 void ExtensionHelper::LoadExternalExtension(DatabaseInstance &db, FileOpener *opener, const string &extension) {
-	auto &loaded_extensions = db.LoadedExtensions();
-	if (loaded_extensions.find(extension) != loaded_extensions.end()) {
+	if (db.ExtensionIsLoaded(extension)) {
 		return;
 	}
 
@@ -149,7 +183,22 @@ void ExtensionHelper::LoadExternalExtension(ClientContext &context, const string
 
 void ExtensionHelper::StorageInit(string &extension, DBConfig &config) {
 	extension = ExtensionHelper::ApplyExtensionAlias(extension);
-	auto res = InitialLoad(config, nullptr, extension); // TODO opener
+	ExtensionInitResult res;
+	string error;
+	if (!TryInitialLoad(config, nullptr, extension, res, error)) {
+		if (!ExtensionHelper::AllowAutoInstall(extension)) {
+			throw IOException(error);
+		}
+		// the extension load failed - try installing the extension
+		if (!config.file_system) {
+			throw InternalException("Attempting to install an extension without a file system");
+		}
+		ExtensionHelper::InstallExtension(config, *config.file_system, extension, false);
+		// try loading again
+		if (!TryInitialLoad(config, nullptr, extension, res, error)) {
+			throw IOException(error);
+		}
+	}
 	auto storage_fun_name = res.basename + "_storage_init";
 
 	ext_storage_init_t storage_init_fun;