duckdb 0.5.2-dev250.0 → 0.5.2-dev269.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.5.2-dev250.0",
+  "version": "0.5.2-dev269.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -66611,6 +66611,13 @@ struct DefaultExtension {
 	bool statically_loaded;
 };
 
+struct ExtensionInitResult {
+	string filename;
+	string basename;
+
+	void *lib_hdl;
+};
+
 class ExtensionHelper {
 public:
 	static void LoadAllExtensions(DuckDB &db);
@@ -66627,8 +66634,13 @@ public:
 
 	static const vector<string> GetPublicKeys();
 
+	static unique_ptr<ReplacementOpenData> ReplacementOpenPre(const string &extension, DBConfig &config);
+	static void ReplacementOpenPost(ClientContext &context, const string &extension, DatabaseInstance &instance,
+	                                ReplacementOpenData *open_data);
+
 private:
 	static const vector<string> PathComponents();
+	static ExtensionInitResult InitialLoad(DBConfig &context, FileOpener *opener, const string &extension);
 	//! For tagged releases we use the tag, else we use the git commit hash
 	static const string GetVersionDirectoryName();
 	//! Version tags occur with and without 'v', tag in extension path is always with 'v'
@@ -129389,6 +129401,27 @@ bool Connection::HasActiveTransaction() {
 
 
 
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/main/extension_helper.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+namespace duckdb {
+
+class ExtensionPrefixReplacementOpen : public ReplacementOpen {
+public:
+	ExtensionPrefixReplacementOpen();
+};
+
+} // namespace duckdb
+
 
 
 #ifndef DUCKDB_NO_THREADS
@@ -129399,6 +129432,7 @@ namespace duckdb {
 
 DBConfig::DBConfig() {
 	compression_functions = make_unique<CompressionFunctionSet>();
+	replacement_opens.push_back(ExtensionPrefixReplacementOpen());
 	cast_functions = make_unique<CastFunctionSet>();
 }
 
@@ -129488,30 +129522,47 @@ ConnectionManager &ConnectionManager::Get(ClientContext &context) {
 	return ConnectionManager::Get(DatabaseInstance::GetDatabase(context));
 }
 
-void DatabaseInstance::Initialize(const char *path, DBConfig *new_config) {
-	if (new_config) {
-		// user-supplied configuration
-		Configure(*new_config);
-	} else {
-		// default configuration
-		DBConfig config;
-		Configure(config);
+void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_config) {
+	DBConfig default_config;
+	DBConfig *config_ptr = &default_config;
+	if (user_config) {
+		config_ptr = user_config;
 	}
-	if (config.options.temporary_directory.empty() && path) {
+
+	if (config_ptr->options.temporary_directory.empty() && database_path) {
 		// no directory specified: use default temp path
-		config.options.temporary_directory = string(path) + ".tmp";
+		config_ptr->options.temporary_directory = string(database_path) + ".tmp";
 
 		// special treatment for in-memory mode
-		if (strcmp(path, ":memory:") == 0) {
-			config.options.temporary_directory = ".tmp";
+		if (strcmp(database_path, ":memory:") == 0) {
+			config_ptr->options.temporary_directory = ".tmp";
 		}
 	}
-	if (new_config && !new_config->options.use_temporary_directory) {
+
+	if (database_path) {
+		config_ptr->options.database_path = database_path;
+	} else {
+		config_ptr->options.database_path.clear();
+	}
+
+	for (auto &open : config_ptr->replacement_opens) {
+		if (open.pre_func) {
+			open.data = open.pre_func(*config_ptr, open.static_data.get());
+			if (open.data) {
+				break;
+			}
+		}
+	}
+	Configure(*config_ptr);
+
+	if (user_config && !user_config->options.use_temporary_directory) {
 		// temporary directories explicitly disabled
 		config.options.temporary_directory = string();
 	}
 
-	storage = make_unique<StorageManager>(*this, path ? string(path) : string(),
+	//	config.create_storage_manager(config.options.database_path,
+	//	                              config.options.access_mode == AccessMode::READ_ONLY);
+	storage = make_unique<StorageManager>(*this, config.options.database_path,
 	                                      config.options.access_mode == AccessMode::READ_ONLY);
 	catalog = make_unique<Catalog>(*this);
 	transaction_manager = make_unique<TransactionManager>(*this);
@@ -129524,6 +129575,13 @@ void DatabaseInstance::Initialize(const char *path, DBConfig *new_config) {
 
 	// only increase thread count after storage init because we get races on catalog otherwise
 	scheduler->SetThreads(config.options.maximum_threads);
+
+	for (auto &open : config.replacement_opens) {
+		if (open.post_func && open.data) {
+			open.post_func(*this, open.data.get());
+			break;
+		}
+	}
 }
 
 DuckDB::DuckDB(const char *path, DBConfig *new_config) : instance(make_shared<DatabaseInstance>()) {
@@ -129583,6 +129641,7 @@ Allocator &Allocator::Get(DatabaseInstance &db) {
 }
 
 void DatabaseInstance::Configure(DBConfig &new_config) {
+	config.options.database_path = new_config.options.database_path;
 	config.options.access_mode = AccessMode::READ_WRITE;
 	if (new_config.options.access_mode != AccessMode::UNDEFINED) {
 		config.options.access_mode = new_config.options.access_mode;
@@ -129624,6 +129683,7 @@ void DatabaseInstance::Configure(DBConfig &new_config) {
 	config.options.enable_external_access = new_config.options.enable_external_access;
 	config.options.allow_unsigned_extensions = new_config.options.allow_unsigned_extensions;
 	config.replacement_scans = move(new_config.replacement_scans);
+	config.replacement_opens = move(new_config.replacement_opens); // TODO is this okay?
 	config.options.initialize_default_database = new_config.options.initialize_default_database;
 	config.options.disabled_optimizers = move(new_config.options.disabled_optimizers);
 	config.parser_extensions = move(new_config.parser_extensions);
@@ -138575,6 +138635,8 @@ public:
 // LICENSE_CHANGE_END
 
 
+
+
 namespace duckdb {
 
 //===--------------------------------------------------------------------===//
@@ -138592,18 +138654,17 @@ static T LoadFunctionFromDLL(void *dll, const string &function_name, const strin
 	return (T)function;
 }
 
-void ExtensionHelper::LoadExternalExtension(ClientContext &context, const string &extension) {
-	auto &db = DatabaseInstance::GetDatabase(context);
-	auto &config = DBConfig::GetConfig(context);
+ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileOpener *opener, const string &extension) {
 	if (!config.options.enable_external_access) {
 		throw PermissionException("Loading external extensions is disabled through configuration");
 	}
-	auto &fs = FileSystem::GetFileSystem(context);
+	VirtualFileSystem fallback_file_system; // config may not contain one yet
+	auto &fs = config.file_system ? *config.file_system : fallback_file_system;
 	auto filename = fs.ConvertSeparators(extension);
 
 	// shorthand case
 	if (!StringUtil::Contains(extension, ".") && !StringUtil::Contains(extension, fs.PathSeparator())) {
-		string local_path = fs.GetHomeDirectory(FileSystem::GetFileOpener(context));
+		string local_path = fs.GetHomeDirectory(opener);
 		auto path_components = PathComponents();
 		for (auto &path_ele : path_components) {
 			local_path = fs.JoinPath(local_path, path_ele);
@@ -138653,13 +138714,10 @@ void ExtensionHelper::LoadExternalExtension(ClientContext &context, const string
 	}
 
 	auto basename = fs.ExtractBaseName(filename);
-	auto init_fun_name = basename + "_init";
-	auto version_fun_name = basename + "_version";
 
-	ext_init_fun_t init_fun;
 	ext_version_fun_t version_fun;
+	auto version_fun_name = basename + "_version";
 
-	init_fun = LoadFunctionFromDLL<ext_init_fun_t>(lib_hdl, init_fun_name, filename);
 	version_fun = LoadFunctionFromDLL<ext_version_fun_t>(lib_hdl, version_fun_name, filename);
 
 	std::string engine_version = std::string(DuckDB::LibraryVersion());
@@ -138685,11 +138743,59 @@ void ExtensionHelper::LoadExternalExtension(ClientContext &context, const string
 		                            extension_version, engine_version);
 	}
 
+	ExtensionInitResult res;
+	res.basename = basename;
+	res.filename = filename;
+	res.lib_hdl = lib_hdl;
+	return res;
+}
+
+void ExtensionHelper::LoadExternalExtension(ClientContext &context, const string &extension) {
+	auto &db = DatabaseInstance::GetDatabase(context);
+
+	auto res = InitialLoad(DBConfig::GetConfig(context), FileSystem::GetFileOpener(context), extension);
+	auto init_fun_name = res.basename + "_init";
+
+	ext_init_fun_t init_fun;
+	init_fun = LoadFunctionFromDLL<ext_init_fun_t>(res.lib_hdl, init_fun_name, res.filename);
+
 	try {
 		(*init_fun)(db);
 	} catch (std::exception &e) {
 		throw InvalidInputException("Initialization function \"%s\" from file \"%s\" threw an exception: \"%s\"",
-		                            init_fun_name, filename, e.what());
+		                            init_fun_name, res.filename, e.what());
+	}
+}
+
+unique_ptr<ReplacementOpenData> ExtensionHelper::ReplacementOpenPre(const string &extension, DBConfig &config) {
+
+	auto res = InitialLoad(config, nullptr, extension); // TODO opener
+	auto init_fun_name = res.basename + "_replacement_open_pre";
+
+	replacement_open_pre_t open_pre_fun;
+	open_pre_fun = LoadFunctionFromDLL<replacement_open_pre_t>(res.lib_hdl, init_fun_name, res.filename);
+
+	try {
+		return (*open_pre_fun)(config, nullptr);
+	} catch (std::exception &e) {
+		throw InvalidInputException("Initialization function \"%s\" from file \"%s\" threw an exception: \"%s\"",
+		                            init_fun_name, res.filename, e.what());
+	}
+}
+
+void ExtensionHelper::ReplacementOpenPost(ClientContext &context, const string &extension, DatabaseInstance &instance,
+                                          ReplacementOpenData *open_data) {
+	auto res = InitialLoad(DBConfig::GetConfig(context), FileSystem::GetFileOpener(context), extension);
+	auto init_fun_name = res.basename + "_replacement_open_post";
+
+	replacement_open_post_t open_post_fun;
+	open_post_fun = LoadFunctionFromDLL<replacement_open_post_t>(res.lib_hdl, init_fun_name, res.filename);
+
+	try {
+		(*open_post_fun)(instance, open_data);
+	} catch (std::exception &e) {
+		throw InvalidInputException("Initialization function \"%s\" from file \"%s\" threw an exception: \"%s\"",
+		                            init_fun_name, res.filename, e.what());
 	}
 }
 
@@ -138706,6 +138812,61 @@ Extension::~Extension() {
 
 
 
+
+namespace duckdb {
+
+struct ExtensionPrefixOpenData : public ReplacementOpenData {
+	ExtensionPrefixOpenData(string extension, string path, unique_ptr<ReplacementOpenData> data)
+	    : extension(move(extension)), path(move(path)), data(move(data)) {
+	}
+	string extension;
+	string path;
+	unique_ptr<ReplacementOpenData> data;
+};
+
+static unique_ptr<ReplacementOpenData> ExtensionPrefixPreOpen(DBConfig &config, ReplacementOpenStaticData *) {
+	auto path = config.options.database_path;
+	auto first_colon = path.find(':');
+	if (first_colon == string::npos || first_colon < 2) { // needs to be at least two characters because windows c: ...
+		return nullptr;
+	}
+	auto extension = path.substr(0, first_colon);
+	D_ASSERT(extension.size() > 1);
+	// needs to be alphanumeric
+	for (auto &ch : extension) {
+		if (!isalnum(ch) && ch != '_') {
+			return nullptr;
+		}
+	}
+	auto extension_data = ExtensionHelper::ReplacementOpenPre(extension, config);
+	if (extension_data) {
+		return make_unique<ExtensionPrefixOpenData>(extension, path, move(extension_data));
+	}
+	return nullptr;
+}
+
+static void ExtensionPrefixPostOpen(DatabaseInstance &instance, ReplacementOpenData *open_data) {
+	if (!open_data) {
+		return;
+	}
+	D_ASSERT(open_data);
+	auto prefix_open_data = (ExtensionPrefixOpenData *)open_data;
+
+	Connection con(instance);
+	ExtensionHelper::LoadExternalExtension(*con.context, prefix_open_data->extension);
+	ExtensionHelper::ReplacementOpenPost(*con.context, prefix_open_data->extension, instance,
+	                                     prefix_open_data->data.get());
+}
+
+ExtensionPrefixReplacementOpen::ExtensionPrefixReplacementOpen()
+    : ReplacementOpen(ExtensionPrefixPreOpen, ExtensionPrefixPostOpen) {
+}
+
+} // namespace duckdb
+
+
+
+
 namespace duckdb {
 
 MaterializedQueryResult::MaterializedQueryResult(StatementType statement_type, StatementProperties properties,

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "7389dcf91"
-#define DUCKDB_VERSION "v0.5.2-dev250"
+#define DUCKDB_SOURCE_ID "4ad9877f0"
+#define DUCKDB_VERSION "v0.5.2-dev269"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -23916,6 +23916,52 @@ struct ReplacementScan {
 
 } // namespace duckdb
 
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/function/replacement_scan.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+namespace duckdb {
+
+class DatabaseInstance;
+struct DBConfig;
+
+struct ReplacementOpenData {
+	virtual ~ReplacementOpenData() {
+	}
+};
+
+struct ReplacementOpenStaticData {
+	virtual ~ReplacementOpenStaticData() {
+	}
+};
+
+typedef unique_ptr<ReplacementOpenData> (*replacement_open_pre_t)(DBConfig &config,
+                                                                  ReplacementOpenStaticData *static_data);
+typedef void (*replacement_open_post_t)(DatabaseInstance &instance, ReplacementOpenData *open_data);
+
+struct ReplacementOpen {
+	explicit ReplacementOpen(replacement_open_pre_t pre_func, replacement_open_post_t post_func)
+	    : pre_func(pre_func), post_func(post_func), data(nullptr) {
+	}
+
+	replacement_open_pre_t pre_func;
+	replacement_open_post_t post_func;
+
+	unique_ptr<ReplacementOpenData> data;
+
+	shared_ptr<ReplacementOpenStaticData> static_data;
+};
+
+} // namespace duckdb
+
 
 
 //===----------------------------------------------------------------------===//
@@ -24180,6 +24226,8 @@ struct ExtensionOption {
 };
 
 struct DBConfigOptions {
+	//! Database file path. May be empty for in-memory mode
+	string database_path;
 	//! Access mode of the database (AUTOMATIC, READ_ONLY or READ_WRITE)
 	AccessMode access_mode = AccessMode::AUTOMATIC;
 	//! Checkpoint when WAL reaches this size (default: 16MB)
@@ -24243,6 +24291,10 @@ public:
 
 	//! Replacement table scans are automatically attempted when a table name cannot be found in the schema
 	vector<ReplacementScan> replacement_scans;
+
+	//! Replacement open handlers are callbacks that run pre and post database initialization
+	vector<ReplacementOpen> replacement_opens;
+
 	//! Extra parameters that can be SET for loaded extensions
 	case_insensitive_map_t<ExtensionOption> extension_parameters;
 	//! The FileSystem to use, can be overwritten to allow for injecting custom file systems for testing purposes (e.g.