duckdb 0.6.2-dev2115.0 → 0.6.2-dev2226.0

package/src/duckdb/src/include/duckdb/common/limits.hpp

@@ -74,6 +74,7 @@ struct NumericLimits<int32_t> {
 		return 10;
 	}
 };
+
 template <>
 struct NumericLimits<int64_t> {
 	DUCKDB_API static constexpr int64_t Minimum() {
@@ -104,6 +105,7 @@ struct NumericLimits<hugeint_t> {
 		return 39;
 	}
 };
+
 template <>
 struct NumericLimits<uint8_t> {
 	DUCKDB_API static constexpr uint8_t Minimum() {
@@ -119,6 +121,7 @@ struct NumericLimits<uint8_t> {
 		return 3;
 	}
 };
+
 template <>
 struct NumericLimits<uint16_t> {
 	DUCKDB_API static constexpr uint16_t Minimum() {

package/src/duckdb/src/include/duckdb/common/types/validity_mask.hpp

@@ -83,15 +83,7 @@ public:
 		return !validity_mask;
 	}
 	inline bool CheckAllValid(idx_t count) const {
-		if (AllValid()) {
-			return true;
-		}
-		idx_t entry_count = ValidityBuffer::EntryCount(count);
-		idx_t valid_count = 0;
-		for (idx_t i = 0; i < entry_count; i++) {
-			valid_count += validity_mask[i] == ValidityBuffer::MAX_ENTRY;
-		}
-		return valid_count == entry_count;
+		return CountValid(count) == count;
 	}
 
 	inline bool CheckAllValid(idx_t to, idx_t from) const {

package/src/duckdb/src/include/duckdb/common/types/vector.hpp

@@ -116,10 +116,10 @@ public:
 
 	//! Converts this Vector to a printable string representation
 	DUCKDB_API string ToString(idx_t count) const;
-	DUCKDB_API void Print(idx_t count);
+	DUCKDB_API void Print(idx_t count) const;
 
 	DUCKDB_API string ToString() const;
-	DUCKDB_API void Print();
+	DUCKDB_API void Print() const;
 
 	//! Flatten the vector, removing any compression and turning it into a FLAT_VECTOR
 	DUCKDB_API void Flatten(idx_t count);

package/src/duckdb/src/include/duckdb/execution/executor.hpp

@@ -89,6 +89,9 @@ public:
 	//! Returns the query result - can only be used if `HasResultCollector` returns true
 	unique_ptr<QueryResult> GetResult();
 
+	//! Returns true if all pipelines have been completed
+	bool ExecutionIsFinished();
+
 private:
 	void InitializeInternal(PhysicalOperator *physical_plan);
 

package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp

@@ -64,13 +64,13 @@ public:
 public:
 	//! Initialize a scan on the index with the given expression and column ids
 	//! to fetch from the base table for a single predicate
-	unique_ptr<IndexScanState> InitializeScanSinglePredicate(Transaction &transaction, Value value,
+	unique_ptr<IndexScanState> InitializeScanSinglePredicate(const Transaction &transaction, const Value &value,
 	                                                         ExpressionType expression_type) override;
 
 	//! Initialize a scan on the index with the given expression and column ids
 	//! to fetch from the base table for two predicates
-	unique_ptr<IndexScanState> InitializeScanTwoPredicates(Transaction &transaction, Value low_value,
-	                                                       ExpressionType low_expression_type, Value high_value,
+	unique_ptr<IndexScanState> InitializeScanTwoPredicates(Transaction &transaction, const Value &low_value,
+	                                                       ExpressionType low_expression_type, const Value &high_value,
 	                                                       ExpressionType high_expression_type) override;
 
 	//! Perform a lookup on the index

package/src/duckdb/src/include/duckdb/execution/operator/projection/physical_unnest.hpp

@@ -14,13 +14,14 @@
 
 namespace duckdb {
 
-//! PhysicalWindow implements window functions
+//! PhysicalUnnest implements the physical UNNEST operation
 class PhysicalUnnest : public PhysicalOperator {
 public:
 	PhysicalUnnest(vector<LogicalType> types, vector<unique_ptr<Expression>> select_list, idx_t estimated_cardinality,
 	               PhysicalOperatorType type = PhysicalOperatorType::UNNEST);
 
 	//! The projection list of the UNNEST
+	//! E.g. SELECT 1, UNNEST([1]), UNNEST([2, 3]); has two UNNESTs in its select_list
 	vector<unique_ptr<Expression>> select_list;
 
 public:
@@ -35,6 +36,9 @@ public:
 public:
 	static unique_ptr<OperatorState> GetState(ExecutionContext &context,
 	                                          const vector<unique_ptr<Expression>> &select_list);
+	//! Executes the UNNEST operator internally and emits a chunk of unnested data. If include_input is set, then
+	//! the resulting chunk also contains vectors for all non-UNNEST columns in the projection. If include_input is
+	//! not set, then the UNNEST behaves as a table function and only emits the unnested data.
 	static OperatorResultType ExecuteInternal(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
 	                                          OperatorState &state, const vector<unique_ptr<Expression>> &select_list,
 	                                          bool include_input = true);

package/src/duckdb/src/include/duckdb/main/client_context.hpp

@@ -188,6 +188,9 @@ public:
 
 	DUCKDB_API ClientProperties GetClientProperties() const;
 
+	//! Returns true if execution of the current query is finished
+	DUCKDB_API bool ExecutionIsFinished();
+
 private:
 	//! Parse statements and resolve pragmas from a query
 	bool ParseStatements(ClientContextLock &lock, const string &query, vector<unique_ptr<SQLStatement>> &result,

package/src/duckdb/src/include/duckdb/main/config.hpp

@@ -24,7 +24,6 @@
 #include "duckdb/common/winapi.hpp"
 #include "duckdb/storage/compression/bitpacking.hpp"
 #include "duckdb/function/cast/default_casts.hpp"
-#include "duckdb/function/replacement_open.hpp"
 #include "duckdb/function/replacement_scan.hpp"
 #include "duckdb/function/create_database_extension.hpp"
 #include "duckdb/optimizer/optimizer_extension.hpp"
@@ -163,9 +162,6 @@ public:
 	//! Replacement table scans are automatically attempted when a table name cannot be found in the schema
 	vector<ReplacementScan> replacement_scans;
 
-	//! Replacement open handlers are callbacks that run pre and post database initialization
-	vector<ReplacementOpen> replacement_opens;
-
 	//! Extra parameters that can be SET for loaded extensions
 	case_insensitive_map_t<ExtensionOption> extension_parameters;
 	//! The FileSystem to use, can be overwritten to allow for injecting custom file systems for testing purposes (e.g.

package/src/duckdb/src/include/duckdb/main/database.hpp

@@ -23,6 +23,7 @@ class ConnectionManager;
 class FileSystem;
 class TaskScheduler;
 class ObjectCache;
+struct AttachInfo;
 
 class DatabaseInstance : public std::enable_shared_from_this<DatabaseInstance> {
 	friend class DuckDB;
@@ -48,9 +49,14 @@ public:
 	DUCKDB_API static DatabaseInstance &GetDatabase(ClientContext &context);
 
 	DUCKDB_API const unordered_set<std::string> &LoadedExtensions();
+	DUCKDB_API bool ExtensionIsLoaded(const std::string &name);
 
 	DUCKDB_API bool TryGetCurrentSetting(const std::string &key, Value &result);
 
+	//! Get the database extension type from a given path
+	string ExtractDatabaseType(string &path);
+	unique_ptr<AttachedDatabase> CreateAttachedDatabase(AttachInfo &info, const string &type, AccessMode access_mode);
+
 private:
 	void Initialize(const char *path, DBConfig *config);
 

package/src/duckdb/src/include/duckdb/main/extension_helper.hpp

@@ -42,6 +42,7 @@ public:
 
 	static void InstallExtension(ClientContext &context, const string &extension, bool force_install);
 	static void LoadExternalExtension(ClientContext &context, const string &extension);
+	static void LoadExternalExtension(DatabaseInstance &db, FileOpener *opener, const string &extension);
 
 	static string ExtensionDirectory(ClientContext &context);
 
@@ -53,13 +54,14 @@ public:
 
 	static const vector<string> GetPublicKeys();
 
-	static unique_ptr<ReplacementOpenData> ReplacementOpenPre(const string &extension, DBConfig &config);
-	static void ReplacementOpenPost(ClientContext &context, const string &extension, DatabaseInstance &instance,
-	                                ReplacementOpenData *open_data);
+	static void StorageInit(string &extension, DBConfig &config);
 
 	// Returns extension name, or empty string if not a replacement open path
 	static string ExtractExtensionPrefixFromPath(const string &path);
 
+	//! Apply any known extension aliases
+	static string ApplyExtensionAlias(string extension_name);
+
 private:
 	static const vector<string> PathComponents();
 	static ExtensionInitResult InitialLoad(DBConfig &context, FileOpener *opener, const string &extension);
@@ -68,8 +70,6 @@ private:
 	//! Version tags occur with and without 'v', tag in extension path is always with 'v'
 	static const string NormalizeVersionTag(const string &version_tag);
 	static bool IsRelease(const string &version_tag);
-	//! Apply any known extension aliases
-	static string ApplyExtensionAlias(string extension_name);
 	static bool CreateSuggestions(const string &extension_name, string &message);
 
 private:

package/src/duckdb/src/include/duckdb/main/relation/write_csv_relation.hpp

@@ -14,11 +14,12 @@ namespace duckdb {
 
 class WriteCSVRelation : public Relation {
 public:
-	WriteCSVRelation(shared_ptr<Relation> child, string csv_file);
+	WriteCSVRelation(shared_ptr<Relation> child, string csv_file, case_insensitive_map_t<vector<Value>> options);
 
 	shared_ptr<Relation> child;
 	string csv_file;
 	vector<ColumnDefinition> columns;
+	case_insensitive_map_t<vector<Value>> options;
 
 public:
 	BoundStatement Bind(Binder &binder) override;

package/src/duckdb/src/include/duckdb/main/relation/write_parquet_relation.hpp

@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/main/relation/write_csv_relation.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/main/relation.hpp"
+
+namespace duckdb {
+
+class WriteParquetRelation : public Relation {
+public:
+	WriteParquetRelation(shared_ptr<Relation> child, string parquet_file,
+	                     case_insensitive_map_t<vector<Value>> options);
+
+	shared_ptr<Relation> child;
+	string parquet_file;
+	vector<ColumnDefinition> columns;
+	case_insensitive_map_t<vector<Value>> options;
+
+public:
+	BoundStatement Bind(Binder &binder) override;
+	const vector<ColumnDefinition> &Columns() override;
+	string ToString(idx_t depth) override;
+	bool IsReadOnly() override {
+		return false;
+	}
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/main/relation.hpp

@@ -126,7 +126,12 @@ public:
 	DUCKDB_API void Create(const string &schema_name, const string &table_name);
 
 	//! Write a relation to a CSV file
-	DUCKDB_API void WriteCSV(const string &csv_file);
+	DUCKDB_API void WriteCSV(const string &csv_file,
+	                         case_insensitive_map_t<vector<Value>> options = case_insensitive_map_t<vector<Value>>());
+	//! Write a relation to a Parquet file
+	DUCKDB_API void
+	WriteParquet(const string &parquet_file,
+	             case_insensitive_map_t<vector<Value>> options = case_insensitive_map_t<vector<Value>>());
 
 	//! Update a table, can only be used on a TableRelation
 	DUCKDB_API virtual void Update(const string &update, const string &condition = string());

package/src/duckdb/src/include/duckdb/parser/parsed_data/copy_info.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/common/vector.hpp"
 #include "duckdb/common/unordered_map.hpp"
 #include "duckdb/common/types/value.hpp"
+#include "duckdb/common/case_insensitive_map.hpp"
 
 namespace duckdb {
 
@@ -34,7 +35,7 @@ struct CopyInfo : public ParseInfo {
 	//! The file path to copy to/from
 	string file_path;
 	//! Set of (key, value) options
-	unordered_map<string, vector<Value>> options;
+	case_insensitive_map_t<vector<Value>> options;
 
 public:
 	unique_ptr<CopyInfo> Copy() const {

package/src/duckdb/src/include/duckdb/parser/statement/copy_statement.hpp

@@ -22,7 +22,7 @@ public:
 	// The SQL statement used instead of a table when copying data out to a file
 	unique_ptr<QueryNode> select_statement;
 	string ToString() const override;
-	string CopyOptionsToString(const string &format, const unordered_map<string, vector<Value>> &options) const;
+	string CopyOptionsToString(const string &format, const case_insensitive_map_t<vector<Value>> &options) const;
 
 protected:
 	CopyStatement(const CopyStatement &other);

package/src/duckdb/src/include/duckdb/planner/binder.hpp

@@ -168,7 +168,7 @@ public:
 	void BindOnConflictClause(LogicalInsert &insert, TableCatalogEntry &table, InsertStatement &stmt);
 
 	static void BindSchemaOrCatalog(ClientContext &context, string &catalog, string &schema);
-	static void BindLogicalType(ClientContext &context, LogicalType &type, const string &catalog = INVALID_CATALOG,
+	static void BindLogicalType(ClientContext &context, LogicalType &type, Catalog *catalog = nullptr,
 	                            const string &schema = INVALID_SCHEMA);
 
 	bool HasMatchingBinding(const string &table_name, const string &column_name, string &error_message);

package/src/duckdb/src/include/duckdb/storage/index.hpp

@@ -66,12 +66,13 @@ public:
 public:
 	//! Initialize a scan on the index with the given expression and column ids
 	//! to fetch from the base table when we only have one query predicate
-	virtual unique_ptr<IndexScanState> InitializeScanSinglePredicate(Transaction &transaction, Value value,
+	virtual unique_ptr<IndexScanState> InitializeScanSinglePredicate(const Transaction &transaction, const Value &value,
 	                                                                 ExpressionType expressionType) = 0;
 	//! Initialize a scan on the index with the given expression and column ids
 	//! to fetch from the base table for two query predicates
-	virtual unique_ptr<IndexScanState> InitializeScanTwoPredicates(Transaction &transaction, Value low_value,
-	                                                               ExpressionType low_expression_type, Value high_value,
+	virtual unique_ptr<IndexScanState> InitializeScanTwoPredicates(Transaction &transaction, const Value &low_value,
+	                                                               ExpressionType low_expression_type,
+	                                                               const Value &high_value,
 	                                                               ExpressionType high_expression_type) = 0;
 	//! Perform a lookup on the index, fetching up to max_count result ids. Returns true if all row ids were fetched,
 	//! and false otherwise.

package/src/duckdb/src/include/duckdb.h

@@ -2287,6 +2287,13 @@ on the task state.
 */
 DUCKDB_API void duckdb_destroy_task_state(duckdb_task_state state);
 
+/*!
+Returns true if execution of the current query is finished.
+
+* con: The connection on which to check
+*/
+DUCKDB_API bool duckdb_execution_is_finished(duckdb_connection con);
+
 #ifdef __cplusplus
 }
 #endif

package/src/duckdb/src/main/capi/threading-c.cpp

@@ -78,3 +78,11 @@ void duckdb_destroy_task_state(duckdb_task_state state_p) {
 	auto state = (CAPITaskState *)state_p;
 	delete state;
 }
+
+bool duckdb_execution_is_finished(duckdb_connection con) {
+	if (!con) {
+		return false;
+	}
+	duckdb::Connection *conn = (duckdb::Connection *)con;
+	return conn->context->ExecutionIsFinished();
+}

package/src/duckdb/src/main/client_context.cpp

@@ -1151,4 +1151,11 @@ ClientProperties ClientContext::GetClientProperties() const {
 	return properties;
 }
 
+bool ClientContext::ExecutionIsFinished() {
+	if (!active_query || !active_query->executor) {
+		return false;
+	}
+	return active_query->executor->ExecutionIsFinished();
+}
+
 } // namespace duckdb

package/src/duckdb/src/main/client_context_file_opener.cpp

@@ -8,4 +8,18 @@ bool ClientContextFileOpener::TryGetCurrentSetting(const string &key, Value &res
 	return context.TryGetCurrentSetting(key, result);
 }
 
+ClientContext *FileOpener::TryGetClientContext(FileOpener *opener) {
+	if (!opener) {
+		return nullptr;
+	}
+	return opener->TryGetClientContext();
+}
+
+bool FileOpener::TryGetCurrentSetting(FileOpener *opener, const string &key, Value &result) {
+	if (!opener) {
+		return false;
+	}
+	return opener->TryGetCurrentSetting(key, result);
+}
+
 } // namespace duckdb

package/src/duckdb/src/main/database.cpp

@@ -11,12 +11,12 @@
 #include "duckdb/main/connection_manager.hpp"
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/main/extension_helper.hpp"
-#include "duckdb/main/replacement_opens.hpp"
 #include "duckdb/function/cast/cast_function_set.hpp"
 #include "duckdb/main/error_manager.hpp"
 #include "duckdb/main/attached_database.hpp"
 #include "duckdb/parser/parsed_data/attach_info.hpp"
 #include "duckdb/storage/magic_bytes.hpp"
+#include "duckdb/storage/storage_extension.hpp"
 
 #ifndef DUCKDB_NO_THREADS
 #include "duckdb/common/thread.hpp"
@@ -26,7 +26,6 @@ namespace duckdb {
 
 DBConfig::DBConfig() {
 	compression_functions = make_unique<CompressionFunctionSet>();
-	replacement_opens.push_back(ExtensionPrefixReplacementOpen());
 	cast_functions = make_unique<CastFunctionSet>();
 	error_manager = make_unique<ErrorManager>();
 }
@@ -123,6 +122,42 @@ ConnectionManager &ConnectionManager::Get(ClientContext &context) {
 	return ConnectionManager::Get(DatabaseInstance::GetDatabase(context));
 }
 
+string DatabaseInstance::ExtractDatabaseType(string &path) {
+	// first check if there is an existing prefix
+	auto extension = ExtensionHelper::ExtractExtensionPrefixFromPath(path);
+	if (!extension.empty()) {
+		// path is prefixed with an extension - remove it
+		path = StringUtil::Replace(path, extension + ":", "");
+		return extension;
+	}
+	// if there isn't - check the magic bytes of the file (if any)
+	auto file_type = MagicBytes::CheckMagicBytes(config.file_system.get(), path);
+	if (file_type == DataFileType::SQLITE_FILE) {
+		return "sqlite";
+	}
+	return string();
+}
+
+unique_ptr<AttachedDatabase> DatabaseInstance::CreateAttachedDatabase(AttachInfo &info, const string &type,
+                                                                      AccessMode access_mode) {
+	unique_ptr<AttachedDatabase> attached_database;
+	if (!type.empty()) {
+		// find the storage extensionon database
+		auto entry = config.storage_extensions.find(type);
+		if (entry == config.storage_extensions.end()) {
+			throw BinderException("Unrecognized storage type \"%s\"", type);
+		}
+		// use storage extension to create the initial database
+		attached_database = make_unique<AttachedDatabase>(*this, Catalog::GetSystemCatalog(*this), *entry->second,
+		                                                  info.name, info, access_mode);
+	} else {
+		// check if this is an in-memory database or not
+		attached_database =
+		    make_unique<AttachedDatabase>(*this, Catalog::GetSystemCatalog(*this), info.name, info.path, access_mode);
+	}
+	return attached_database;
+}
+
 void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_config) {
 	DBConfig default_config;
 	DBConfig *config_ptr = &default_config;
@@ -145,22 +180,6 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
 	} else {
 		config_ptr->options.database_path.clear();
 	}
-
-	auto file_type = MagicBytes::CheckMagicBytes(config_ptr->file_system.get(), config_ptr->options.database_path);
-	if (file_type == DataFileType::SQLITE_FILE) {
-		config_ptr->options.database_path = "sqlite:" + config_ptr->options.database_path;
-	}
-
-	ReplacementOpen *replacement_open = nullptr;
-	for (auto &open : config_ptr->replacement_opens) {
-		if (open.pre_func) {
-			open.data = open.pre_func(*config_ptr, open.static_data.get());
-			if (open.data) {
-				replacement_open = &open;
-				break;
-			}
-		}
-	}
 	Configure(*config_ptr);
 
 	if (user_config && !user_config->options.use_temporary_directory) {
@@ -168,7 +187,6 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
 		config.options.temporary_directory = string();
 	}
 
-	auto &config = DBConfig::GetConfig(*this);
 	db_manager = make_unique<DatabaseManager>(*this);
 	buffer_manager =
 	    make_unique<BufferManager>(*this, config.options.temporary_directory, config.options.maximum_memory);
@@ -176,22 +194,16 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
 	object_cache = make_unique<ObjectCache>();
 	connection_manager = make_unique<ConnectionManager>();
 
-	auto database_name = AttachedDatabase::ExtractDatabaseName(config.options.database_path);
-	unique_ptr<AttachedDatabase> attached_database;
-	if (replacement_open && replacement_open->data && replacement_open->data->HasStorageExtension()) {
-		// use storage extension from replacement open to create initial database
-		AttachInfo info;
-		info.name = database_name;
-		info.path = config.options.database_path;
-		auto storage_extension = replacement_open->data->GetStorageExtension(info);
-		attached_database = make_unique<AttachedDatabase>(*this, Catalog::GetSystemCatalog(*this), *storage_extension,
-		                                                  database_name, info, config.options.access_mode);
-		replacement_open = nullptr;
-	} else {
-		// check if this is an in-memory database or not
-		attached_database = make_unique<AttachedDatabase>(*this, Catalog::GetSystemCatalog(*this), database_name,
-		                                                  config.options.database_path, config.options.access_mode);
+	// check if we are opening a standard DuckDB database or an extension database
+	auto database_type = ExtractDatabaseType(config.options.database_path);
+	if (!database_type.empty()) {
+		// we are opening an extension database, run storage_init
+		ExtensionHelper::StorageInit(database_type, config);
 	}
+	AttachInfo info;
+	info.name = AttachedDatabase::ExtractDatabaseName(config.options.database_path);
+	info.path = config.options.database_path;
+	auto attached_database = CreateAttachedDatabase(info, database_type, config.options.access_mode);
 	auto initial_database = attached_database.get();
 	{
 		Connection con(*this);
@@ -205,12 +217,13 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
 	// initialize the database
 	initial_database->Initialize();
 
+	if (!database_type.empty()) {
+		// if we are opening an extension database - load the extension
+		ExtensionHelper::LoadExternalExtension(*this, nullptr, database_type);
+	}
+
 	// only increase thread count after storage init because we get races on catalog otherwise
 	scheduler->SetThreads(config.options.maximum_threads);
-
-	if (replacement_open) {
-		replacement_open->post_func(*this, replacement_open->data.get());
-	}
 }
 
 DuckDB::DuckDB(const char *path, DBConfig *new_config) : instance(make_shared<DatabaseInstance>()) {
@@ -294,7 +307,6 @@ void DatabaseInstance::Configure(DBConfig &new_config) {
 		config.allocator = make_unique<Allocator>();
 	}
 	config.replacement_scans = std::move(new_config.replacement_scans);
-	config.replacement_opens = std::move(new_config.replacement_opens);
 	config.parser_extensions = std::move(new_config.parser_extensions);
 	config.error_manager = std::move(new_config.error_manager);
 	if (!config.error_manager) {
@@ -325,9 +337,14 @@ idx_t DuckDB::NumberOfThreads() {
 	return instance->NumberOfThreads();
 }
 
+bool DatabaseInstance::ExtensionIsLoaded(const std::string &name) {
+	return loaded_extensions.find(name) != loaded_extensions.end();
+}
+
 bool DuckDB::ExtensionIsLoaded(const std::string &name) {
-	return instance->loaded_extensions.find(name) != instance->loaded_extensions.end();
+	return instance->ExtensionIsLoaded(name);
 }
+
 void DatabaseInstance::SetExtensionLoaded(const std::string &name) {
 	loaded_extensions.insert(name);
 }

package/src/duckdb/src/main/extension/extension_load.cpp

@@ -1,6 +1,5 @@
 #include "duckdb/common/dl.hpp"
 #include "duckdb/common/virtual_file_system.hpp"
-#include "duckdb/function/replacement_open.hpp"
 #include "duckdb/main/extension_helper.hpp"
 #include "duckdb/main/error_manager.hpp"
 #include "mbedtls_wrapper.hpp"
@@ -12,6 +11,7 @@ namespace duckdb {
 //===--------------------------------------------------------------------===//
 typedef void (*ext_init_fun_t)(DatabaseInstance &);
 typedef const char *(*ext_version_fun_t)(void);
+typedef void (*ext_storage_init_t)(DBConfig &);
 
 template <class T>
 static T LoadFunctionFromDLL(void *dll, const string &function_name, const string &filename) {
@@ -121,14 +121,13 @@ ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *o
 	return res;
 }
 
-void ExtensionHelper::LoadExternalExtension(ClientContext &context, const string &extension) {
-	auto &db = DatabaseInstance::GetDatabase(context);
+void ExtensionHelper::LoadExternalExtension(DatabaseInstance &db, FileOpener *opener, const string &extension) {
 	auto &loaded_extensions = db.LoadedExtensions();
 	if (loaded_extensions.find(extension) != loaded_extensions.end()) {
 		return;
 	}
 
-	auto res = InitialLoad(DBConfig::GetConfig(context), FileSystem::GetFileOpener(context), extension);
+	auto res = InitialLoad(DBConfig::GetConfig(db), opener, extension);
 	auto init_fun_name = res.basename + "_init";
 
 	ext_init_fun_t init_fun;
@@ -141,38 +140,27 @@ void ExtensionHelper::LoadExternalExtension(ClientContext &context, const string
 		                            init_fun_name, res.filename, e.what());
 	}
 
-	DatabaseInstance::GetDatabase(context).SetExtensionLoaded(extension);
+	db.SetExtensionLoaded(extension);
 }
 
-unique_ptr<ReplacementOpenData> ExtensionHelper::ReplacementOpenPre(const string &extension, DBConfig &config) {
-
-	auto res = InitialLoad(config, nullptr, extension); // TODO opener
-	auto init_fun_name = res.basename + "_replacement_open_pre";
-
-	replacement_open_pre_t open_pre_fun;
-	open_pre_fun = LoadFunctionFromDLL<replacement_open_pre_t>(res.lib_hdl, init_fun_name, res.filename);
-
-	try {
-		return (*open_pre_fun)(config, nullptr);
-	} catch (std::exception &e) {
-		throw InvalidInputException("Initialization function \"%s\" from file \"%s\" threw an exception: \"%s\"",
-		                            init_fun_name, res.filename, e.what());
-	}
+void ExtensionHelper::LoadExternalExtension(ClientContext &context, const string &extension) {
+	LoadExternalExtension(DatabaseInstance::GetDatabase(context), FileSystem::GetFileOpener(context), extension);
 }
 
-void ExtensionHelper::ReplacementOpenPost(ClientContext &context, const string &extension, DatabaseInstance &instance,
-                                          ReplacementOpenData *open_data) {
-	auto res = InitialLoad(DBConfig::GetConfig(context), FileSystem::GetFileOpener(context), extension);
-	auto init_fun_name = res.basename + "_replacement_open_post";
+void ExtensionHelper::StorageInit(string &extension, DBConfig &config) {
+	extension = ExtensionHelper::ApplyExtensionAlias(extension);
+	auto res = InitialLoad(config, nullptr, extension); // TODO opener
+	auto storage_fun_name = res.basename + "_storage_init";
 
-	replacement_open_post_t open_post_fun;
-	open_post_fun = LoadFunctionFromDLL<replacement_open_post_t>(res.lib_hdl, init_fun_name, res.filename);
+	ext_storage_init_t storage_init_fun;
+	storage_init_fun = LoadFunctionFromDLL<ext_storage_init_t>(res.lib_hdl, storage_fun_name, res.filename);
 
 	try {
-		(*open_post_fun)(instance, open_data);
+		(*storage_init_fun)(config);
 	} catch (std::exception &e) {
-		throw InvalidInputException("Initialization function \"%s\" from file \"%s\" threw an exception: \"%s\"",
-		                            init_fun_name, res.filename, e.what());
+		throw InvalidInputException(
+		    "Storage initialization function \"%s\" from file \"%s\" threw an exception: \"%s\"", storage_fun_name,
+		    res.filename, e.what());
 	}
 }
 
@@ -189,6 +177,10 @@ string ExtensionHelper::ExtractExtensionPrefixFromPath(const string &path) {
 			return "";
 		}
 	}
+	if (extension == "http" || extension == "https" || extension == "s3") {
+		// these are not extensions
+		return "";
+	}
 	return extension;
 }