duckdb 0.8.2-dev3949.0 → 0.8.2-dev3989.0

package/duckdb_extension_config.cmake

@@ -4,7 +4,7 @@
 #
 # This is the default extension configuration for NodeJS builds. Basically it means that all these extensions are
 # "baked in" to the NodeJS binaries Note that the configuration here is only when building Node using the main
-# CMakeLists.txt file with the `BUILD_R` variable.
+# CMakeLists.txt file with the `BUILD_NODE` variable.
 # TODO: unify this by making setup.py also use this configuration, making this the config for all Node builds
 duckdb_extension_load(json)
 duckdb_extension_load(icu)

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.8.2-dev3949.0",
+  "version": "0.8.2-dev3989.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/common/file_buffer.cpp

@@ -68,7 +68,7 @@ FileBuffer::MemoryRequirement FileBuffer::CalculateMemory(uint64_t user_size) {
 		result.alloc_size = user_size;
 	} else {
 		result.header_size = Storage::BLOCK_HEADER_SIZE;
-		result.alloc_size = AlignValue<uint32_t, Storage::SECTOR_SIZE>(result.header_size + user_size);
+		result.alloc_size = AlignValue<idx_t, Storage::SECTOR_SIZE>(result.header_size + user_size);
 	}
 	return result;
 }

package/src/duckdb/src/function/cast/vector_cast_helpers.cpp

@@ -98,6 +98,9 @@ struct SplitStringListOperation {
 			child_start++;
 			return;
 		}
+		if (start_pos > pos) {
+			pos = start_pos;
+		}
 		child_data[child_start] = StringVector::AddString(child, buf + start_pos, pos - start_pos);
 		child_start++;
 	}
@@ -109,6 +112,7 @@ static bool SplitStringListInternal(const string_t &input, OP &state) {
 	idx_t len = input.GetSize();
 	idx_t lvl = 1;
 	idx_t pos = 0;
+	bool seen_value = false;
 
 	SkipWhitespace(buf, pos, len);
 	if (pos == len || buf[pos] != '[') {
@@ -132,9 +136,10 @@ static bool SplitStringListInternal(const string_t &input, OP &state) {
 			while (StringUtil::CharacterIsSpace(buf[pos - trailing_whitespace - 1])) {
 				trailing_whitespace++;
 			}
-			if (!(buf[pos] == ']' && start_pos == pos)) {
+			if (buf[pos] != ']' || start_pos != pos || seen_value) {
 				state.HandleValue(buf, start_pos, pos - trailing_whitespace);
-			} // else the list is empty
+				seen_value = true;
+			}
 			if (buf[pos] == ']') {
 				lvl--;
 				break;

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.8.2-dev3949"
+#define DUCKDB_VERSION "0.8.2-dev3989"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "c21a9cb87c"
+#define DUCKDB_SOURCE_ID "a7a88cb691"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/main/extension_helper.hpp

@@ -46,13 +46,16 @@ public:
 	static void InstallExtension(DBConfig &config, FileSystem &fs, const string &extension, bool force_install,
 	                             const string &respository = "");
 	static void LoadExternalExtension(ClientContext &context, const string &extension);
-	static void LoadExternalExtension(DatabaseInstance &db, FileSystem &fs, const string &extension);
+	static void LoadExternalExtension(DatabaseInstance &db, FileSystem &fs, const string &extension,
+	                                  optional_ptr<const ClientConfig> client_config);
 
 	//! Autoload an extension by name. Depending on the current settings, this will either load or install+load
 	static void AutoLoadExtension(ClientContext &context, const string &extension_name);
 
 	static string ExtensionDirectory(ClientContext &context);
 	static string ExtensionDirectory(DBConfig &config, FileSystem &fs);
+	static string ExtensionUrlTemplate(optional_ptr<const ClientConfig> config, const string &repository);
+	static string ExtensionFinalizeUrlTemplate(const string &url, const string &name);
 
 	static idx_t DefaultExtensionCount();
 	static DefaultExtension GetDefaultExtension(idx_t index);
@@ -101,9 +104,10 @@ private:
 	                                     const string &repository);
 	static const vector<string> PathComponents();
 	static bool AllowAutoInstall(const string &extension);
-	static ExtensionInitResult InitialLoad(DBConfig &config, FileSystem &fs, const string &extension);
+	static ExtensionInitResult InitialLoad(DBConfig &config, FileSystem &fs, const string &extension,
+	                                       optional_ptr<const ClientConfig> client_config);
 	static bool TryInitialLoad(DBConfig &config, FileSystem &fs, const string &extension, ExtensionInitResult &result,
-	                           string &error);
+	                           string &error, optional_ptr<const ClientConfig> client_config);
 	//! For tagged releases we use the tag, else we use the git commit hash
 	static const string GetVersionDirectoryName();
 	//! Version tags occur with and without 'v', tag in extension path is always with 'v'

package/src/duckdb/src/main/database.cpp

@@ -234,7 +234,7 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
 		if (!config.file_system) {
 			throw InternalException("No file system!?");
 		}
-		ExtensionHelper::LoadExternalExtension(*this, *config.file_system, config.options.database_type);
+		ExtensionHelper::LoadExternalExtension(*this, *config.file_system, config.options.database_type, nullptr);
 	}
 
 	if (!config.options.unrecognized_options.empty()) {

package/src/duckdb/src/main/extension/extension_install.cpp

@@ -45,7 +45,7 @@ const vector<string> ExtensionHelper::PathComponents() {
 
 string ExtensionHelper::ExtensionDirectory(DBConfig &config, FileSystem &fs) {
 #ifdef WASM_LOADABLE_EXTENSIONS
-	static_assertion(0, "ExtensionDirectory functionality is not supported in duckdb-wasm");
+	static_assert(0, "ExtensionDirectory functionality is not supported in duckdb-wasm");
 #endif
 	string extension_directory;
 	if (!config.options.extension_directory.empty()) { // create the extension directory if not present
@@ -157,6 +157,32 @@ void WriteExtensionFileToDisk(FileSystem &fs, const string &path, void *data, id
 	target_file.reset();
 }
 
+string ExtensionHelper::ExtensionUrlTemplate(optional_ptr<const ClientConfig> client_config, const string &repository) {
+	string default_endpoint = "http://extensions.duckdb.org";
+	string versioned_path = "/${REVISION}/${PLATFORM}/${NAME}.duckdb_extension.gz";
+#ifdef WASM_LOADABLE_EXTENSIONS
+	versioned_path = "/duckdb-wasm" + versioned_path;
+#endif
+	string custom_endpoint = client_config ? client_config->custom_extension_repo : string();
+	string endpoint;
+	if (!repository.empty()) {
+		endpoint = repository;
+	} else if (!custom_endpoint.empty()) {
+		endpoint = custom_endpoint;
+	} else {
+		endpoint = default_endpoint;
+	}
+	string url_template = endpoint + versioned_path;
+	return url_template;
+}
+
+string ExtensionHelper::ExtensionFinalizeUrlTemplate(const string &url_template, const string &extension_name) {
+	auto url = StringUtil::Replace(url_template, "${REVISION}", GetVersionDirectoryName());
+	url = StringUtil::Replace(url, "${PLATFORM}", DuckDB::Platform());
+	url = StringUtil::Replace(url, "${NAME}", extension_name);
+	return url;
+}
+
 void ExtensionHelper::InstallExtensionInternal(DBConfig &config, ClientConfig *client_config, FileSystem &fs,
                                                const string &local_path, const string &extension, bool force_install,
                                                const string &repository) {
@@ -197,27 +223,14 @@ void ExtensionHelper::InstallExtensionInternal(DBConfig &config, ClientConfig *c
 	throw BinderException("Remote extension installation is disabled through configuration");
 #else
 
-	string versioned_path = "/${REVISION}/${PLATFORM}/${NAME}.duckdb_extension.gz";
-	string custom_endpoint = client_config ? client_config->custom_extension_repo : string();
-	string endpoint;
-	if (!repository.empty()) {
-		endpoint = repository;
-	} else if (!custom_endpoint.empty()) {
-		endpoint = custom_endpoint;
-	} else {
-		endpoint = "http://extensions.duckdb.org";
-	}
-
-	string url_template = endpoint + versioned_path;
+	string url_template = ExtensionUrlTemplate(client_config, repository);
 
 	if (is_http_url) {
 		url_template = extension;
 		extension_name = "";
 	}
 
-	auto url = StringUtil::Replace(url_template, "${REVISION}", GetVersionDirectoryName());
-	url = StringUtil::Replace(url, "${PLATFORM}", DuckDB::Platform());
-	url = StringUtil::Replace(url, "${NAME}", extension_name);
+	string url = ExtensionFinalizeUrlTemplate(url_template, extension_name);
 
 	string no_http = StringUtil::Replace(url, "http://", "");
 
@@ -227,7 +240,7 @@ void ExtensionHelper::InstallExtensionInternal(DBConfig &config, ClientConfig *c
 	}
 
 	// Special case to install extension from a local file, useful for testing
-	if (!StringUtil::Contains(endpoint, "http://")) {
+	if (!StringUtil::Contains(url_template, "http://")) {
 		string file = fs.ConvertSeparators(url);
 		if (!fs.FileExists(file)) {
 			// check for non-gzipped variant

package/src/duckdb/src/main/extension/extension_load.cpp

@@ -58,7 +58,8 @@ static void ComputeSHA256FileSegment(FileHandle *handle, const idx_t start, cons
 #endif
 
 bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const string &extension,
-                                     ExtensionInitResult &result, string &error) {
+                                     ExtensionInitResult &result, string &error,
+                                     optional_ptr<const ClientConfig> client_config) {
 #ifdef DUCKDB_DISABLE_EXTENSION_LOAD
 	throw PermissionException("Loading external extensions is disabled through a compile time flag");
 #else
@@ -69,6 +70,30 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str
 
 	// shorthand case
 	if (!ExtensionHelper::IsFullPath(extension)) {
+#ifdef WASM_LOADABLE_EXTENSIONS
+		string url_template = ExtensionUrlTemplate(client_config, "");
+		string url = ExtensionFinalizeUrlTemplate(url_template, extension_name);
+
+		char *str = (char *)EM_ASM_PTR(
+		    {
+			    var jsString = ((typeof runtime == = 'object') && runtime &&
+			                    (typeof runtime.whereToLoad == = 'function') && runtime.whereToLoad)
+			                       ? runtime.whereToLoad(UTF8ToString($0))
+			                       : (UTF8ToString($1));
+			    var lengthBytes = lengthBytesUTF8(jsString) + 1;
+			    // 'jsString.length' would return the length of the string as UTF-16
+			    // units, but Emscripten C strings operate as UTF-8.
+			    var stringOnWasmHeap = _malloc(lengthBytes);
+			    stringToUTF8(jsString, stringOnWasmHeap, lengthBytes);
+			    return stringOnWasmHeap;
+		    },
+		    filename.c_str(), url.c_str());
+		std::string address(str);
+		free(str);
+
+		filename = address;
+#else
+
 		string local_path =
 		    !config.options.extension_directory.empty() ? config.options.extension_directory : fs.GetHomeDirectory();
 
@@ -82,6 +107,7 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str
 		}
 		string extension_name = ApplyExtensionAlias(extension);
 		filename = fs.JoinPath(local_path, extension_name + ".duckdb_extension");
+#endif
 	}
 	if (!fs.FileExists(filename)) {
 		string message;
@@ -216,17 +242,18 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str
 #endif
 }
 
-ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileSystem &fs, const string &extension) {
+ExtensionInitResult ExtensionHelper::InitialLoad(DBConfig &config, FileSystem &fs, const string &extension,
+                                                 optional_ptr<const ClientConfig> client_config) {
 	string error;
 	ExtensionInitResult result;
-	if (!TryInitialLoad(config, fs, extension, result, error)) {
+	if (!TryInitialLoad(config, fs, extension, result, error, client_config)) {
 		if (!ExtensionHelper::AllowAutoInstall(extension)) {
 			throw IOException(error);
 		}
 		// the extension load failed - try installing the extension
 		ExtensionHelper::InstallExtension(config, fs, extension, false);
 		// try loading again
-		if (!TryInitialLoad(config, fs, extension, result, error)) {
+		if (!TryInitialLoad(config, fs, extension, result, error, client_config)) {
 			throw IOException(error);
 		}
 	}
@@ -254,14 +281,15 @@ string ExtensionHelper::GetExtensionName(const string &original_name) {
 	return ExtensionHelper::ApplyExtensionAlias(splits.front());
 }
 
-void ExtensionHelper::LoadExternalExtension(DatabaseInstance &db, FileSystem &fs, const string &extension) {
+void ExtensionHelper::LoadExternalExtension(DatabaseInstance &db, FileSystem &fs, const string &extension,
+                                            optional_ptr<const ClientConfig> client_config) {
 	if (db.ExtensionIsLoaded(extension)) {
 		return;
 	}
 #ifdef DUCKDB_DISABLE_EXTENSION_LOAD
 	throw PermissionException("Loading external extensions is disabled through a compile time flag");
 #else
-	auto res = InitialLoad(DBConfig::GetConfig(db), fs, extension);
+	auto res = InitialLoad(DBConfig::GetConfig(db), fs, extension, client_config);
 	auto init_fun_name = res.basename + "_init";
 
 	ext_init_fun_t init_fun;
@@ -279,7 +307,8 @@ void ExtensionHelper::LoadExternalExtension(DatabaseInstance &db, FileSystem &fs
 }
 
 void ExtensionHelper::LoadExternalExtension(ClientContext &context, const string &extension) {
-	LoadExternalExtension(DatabaseInstance::GetDatabase(context), FileSystem::GetFileSystem(context), extension);
+	LoadExternalExtension(DatabaseInstance::GetDatabase(context), FileSystem::GetFileSystem(context), extension,
+	                      &ClientConfig::GetConfig(context));
 }
 
 string ExtensionHelper::ExtractExtensionPrefixFromPath(const string &path) {