@twin.org/node-core 0.0.2-next.25 → 0.0.2-next.26

package/README.md

@@ -1,6 +1,6 @@
 # TWIN Node Core
 
-TWIN Node Core for serving APIs using the specified configuration
+Core components for running TWIN nodes with dynamic extension loading and protocol-based module resolution.
 
 ## Installation
 
@@ -8,14 +8,28 @@ TWIN Node Core for serving APIs using the specified configuration
 npm install @twin.org/node-core
 ```
 
-## Examples
+## Quick Start
 
-Usage of the APIs is shown in the examples [docs/examples.md](docs/examples.md)
+```javascript
+import { start } from '@twin.org/node-core';
 
-## Reference
+// Start a TWIN node
+await start({
+  extensions: ['./my-extension.mjs']
+});
+```
+
+## Key Features
+
+- **Protocol-based extension loading** - Load from local files, npm packages, or HTTPS URLs
+- **Dynamic module resolution** - Automatic installation and caching
+- **Extension lifecycle hooks** - Customize node behavior at different stages
+- **Security controls** - Size limits, HTTPS-only, and cache TTL
+
+## Documentation
 
-Detailed reference documentation for the API can be found in [docs/reference/index.md](docs/reference/index.md)
+For detailed documentation, configuration options, and examples, see [docs/detailed-guide.md](docs/detailed-guide.md).
 
-## Changelog
+## License
 
-The changes between each version can be found in [docs/changelog.md](docs/changelog.md)
+Apache-2.0

package/dist/cjs/index.cjs

@@ -9,13 +9,14 @@ var entityStorageModels = require('@twin.org/entity-storage-models');
 var identityModels = require('@twin.org/identity-models');
 var vaultModels = require('@twin.org/vault-models');
 var walletModels = require('@twin.org/wallet-models');
+var node_child_process = require('node:child_process');
 var promises = require('node:fs/promises');
+var node_https = require('node:https');
 var path = require('node:path');
 var cliCore = require('@twin.org/cli-core');
+var modules = require('@twin.org/modules');
 var rightsManagementRestClient = require('@twin.org/rights-management-rest-client');
 var engineServer = require('@twin.org/engine-server');
-var modules = require('@twin.org/modules');
-var node_child_process = require('node:child_process');
 var dotenv = require('dotenv');
 var engine = require('@twin.org/engine');
 var engineCore = require('@twin.org/engine-core');
@@ -70,6 +71,35 @@ const NodeFeatures = {
     NodeWallet: "node-wallet"
 };
 
+// Copyright 2024 IOTA Stiftung.
+// SPDX-License-Identifier: Apache-2.0.
+/**
+ * The protocol types for modules.
+ */
+// eslint-disable-next-line @typescript-eslint/naming-convention
+const ModuleProtocol = {
+    /**
+     * Local module (starts with . or / or file://).
+     */
+    Local: "local",
+    /**
+     * NPM package (starts with npm:).
+     */
+    Npm: "npm",
+    /**
+     * HTTPS URL (starts with https://).
+     */
+    Https: "https",
+    /**
+     * HTTP URL (starts with http://).
+     */
+    Http: "http",
+    /**
+     * Default/standard module resolution.
+     */
+    Default: "default"
+};
+
 // Copyright 2024 IOTA Stiftung.
 // SPDX-License-Identifier: Apache-2.0.
 /**
@@ -203,6 +233,280 @@ function getFeatures(env) {
     }
     return features;
 }
+/**
+ * Parse the protocol from a module name.
+ * @param moduleName The module name to parse.
+ * @returns The parsed protocol information.
+ */
+function parseModuleProtocol(moduleName) {
+    const trimmed = moduleName.trim();
+    if (trimmed.startsWith("npm:")) {
+        return {
+            protocol: ModuleProtocol.Npm,
+            identifier: trimmed.slice(4),
+            original: trimmed
+        };
+    }
+    if (trimmed.startsWith("https://")) {
+        return {
+            protocol: ModuleProtocol.Https,
+            identifier: trimmed,
+            original: trimmed
+        };
+    }
+    if (trimmed.startsWith("http://")) {
+        return {
+            protocol: ModuleProtocol.Http,
+            identifier: trimmed,
+            original: trimmed
+        };
+    }
+    if (trimmed.startsWith("file://")) {
+        return {
+            protocol: ModuleProtocol.Local,
+            identifier: trimmed,
+            original: trimmed
+        };
+    }
+    if (modules.ModuleHelper.isLocalModule(trimmed)) {
+        return {
+            protocol: ModuleProtocol.Local,
+            identifier: trimmed,
+            original: trimmed
+        };
+    }
+    return {
+        protocol: ModuleProtocol.Default,
+        identifier: trimmed,
+        original: trimmed
+    };
+}
+/**
+ * Hash a URL to create a safe filename.
+ * @param url The URL to hash.
+ * @returns A hashed filename safe for the filesystem.
+ */
+function hashUrl(url) {
+    const urlBytes = core.Converter.utf8ToBytes(url);
+    const hashBytes = crypto.Sha256.sum256(urlBytes);
+    const hash = core.Converter.bytesToHex(hashBytes);
+    const ext = path.extname(new URL(url).pathname);
+    return `${hash}${ext}`;
+}
+/**
+ * Get the extensions cache directory.
+ * @param executionDirectory The execution directory.
+ * @param protocol The protocol type for subdirectory organization.
+ * @param cacheDirectory The cache directory base path.
+ * @returns The cache directory path.
+ */
+function getExtensionsCacheDir(executionDirectory, protocol, cacheDirectory) {
+    // Resolve to absolute path to ensure consistent behavior
+    const absoluteDir = path.resolve(executionDirectory);
+    const baseDir = cacheDirectory ?? ".tmp";
+    return path.join(absoluteDir, baseDir, "extensions", protocol);
+}
+/**
+ * Handle the npm: protocol by installing the package if needed.
+ * @param packageName The npm package name (without npm: prefix).
+ * @param executionDirectory The execution directory.
+ * @param cacheDirectory The cache directory base path.
+ * @returns The resolved path to the installed module.
+ */
+async function handleNpmProtocol(packageName, executionDirectory, cacheDirectory) {
+    const cacheDir = getExtensionsCacheDir(executionDirectory, ModuleProtocol.Npm, cacheDirectory);
+    // Extract just the package name (without version) for the directory
+    // e.g. "picocolors@1.0.0" becomes "picocolors"
+    // e.g. "@scope/package@1.0.0" becomes "@scope/package"
+    const lastAtIndex = packageName.lastIndexOf("@");
+    const packageNameOnly = lastAtIndex > 0 ? packageName.slice(0, lastAtIndex) : packageName;
+    const packageDir = path.join(cacheDir, "node_modules", packageNameOnly);
+    const packageJsonPath = path.join(packageDir, "package.json");
+    const exists = await fileExists(packageJsonPath);
+    if (exists) {
+        const mainFile = await resolvePackageEntryPoint(packageDir, packageNameOnly);
+        const modulePath = path.join(packageDir, mainFile);
+        return {
+            resolvedPath: modulePath,
+            cached: true
+        };
+    }
+    await promises.mkdir(cacheDir, { recursive: true });
+    cliCore.CLIDisplay.task(core.I18n.formatMessage("node.extensionNpmInstalling"), packageName);
+    try {
+        // Always pipe stdio to comply with env access restrictions in tests/lint
+        const stdio = "pipe";
+        node_child_process.execSync(`npm install ${packageName} --prefix "${cacheDir}" --no-save --no-package-lock`, {
+            cwd: cacheDir,
+            stdio
+        });
+    }
+    catch (err) {
+        throw new core.GeneralError("node", "extensionNpmInstallFailed", {
+            package: packageName
+        }, core.BaseError.fromError(err));
+    }
+    const mainFile = await resolvePackageEntryPoint(packageDir, packageNameOnly);
+    const modulePath = path.join(packageDir, mainFile);
+    return {
+        resolvedPath: modulePath,
+        cached: false
+    };
+}
+/**
+ * Check if a cached file has expired based on TTL and force refresh settings.
+ * @param metadataPath Path to the cache metadata file.
+ * @param ttlHours Time to live in hours.
+ * @param forceRefresh Whether to force refresh regardless of TTL.
+ * @returns True if the cache is expired or should be refreshed.
+ */
+async function isCacheExpired(metadataPath, ttlHours, forceRefresh) {
+    if (forceRefresh) {
+        return true;
+    }
+    try {
+        const metadata = await loadJsonFile(metadataPath);
+        const ttlMillis = ttlHours * 60 * 60 * 1000;
+        const expireTime = metadata.downloadedAt + ttlMillis;
+        return Date.now() > expireTime;
+    }
+    catch {
+        // If metadata doesn't exist or is corrupted, consider expired
+        return true;
+    }
+}
+/**
+ * Handle the https: protocol by downloading the module if needed.
+ * @param url The HTTPS URL to download from.
+ * @param executionDirectory The execution directory.
+ * @param maxSizeMb The maximum size in MB for the download.
+ * @param cacheDirectory The cache directory base path.
+ * @param ttlHours TTL in hours for cache expiration.
+ * @param forceRefresh Whether to force refresh the cache.
+ * @returns The resolved path to the downloaded module.
+ */
+async function handleHttpsProtocol(url, executionDirectory, maxSizeMb, cacheDirectory, ttlHours, forceRefresh) {
+    const effectiveTtlHours = ttlHours ?? 24;
+    const effectiveForceRefresh = forceRefresh ?? false;
+    const cacheDir = getExtensionsCacheDir(executionDirectory, ModuleProtocol.Https, cacheDirectory);
+    const filename = hashUrl(url);
+    const cachedPath = path.join(cacheDir, filename);
+    const metadataPath = `${cachedPath}.meta`;
+    const exists = await fileExists(cachedPath);
+    if (exists) {
+        const expired = await isCacheExpired(metadataPath, effectiveTtlHours, effectiveForceRefresh);
+        if (!expired) {
+            return {
+                resolvedPath: cachedPath,
+                cached: true
+            };
+        }
+        if (effectiveForceRefresh) {
+            cliCore.CLIDisplay.warning(core.I18n.formatMessage("node.extensionForceRefresh", { url }));
+        }
+        else {
+            cliCore.CLIDisplay.task(core.I18n.formatMessage("node.extensionCacheExpired", { url }));
+        }
+    }
+    cliCore.CLIDisplay.warning(core.I18n.formatMessage("node.extensionSecurityWarning", { url }));
+    cliCore.CLIDisplay.task(core.I18n.formatMessage("node.extensionHttpsDownloading"), url);
+    await promises.mkdir(cacheDir, { recursive: true });
+    const maxSizeBytes = maxSizeMb * 1024 * 1024;
+    let downloadedSize = 0;
+    const chunks = [];
+    try {
+        await new Promise((resolve, reject) => {
+            node_https.get(url, response => {
+                if (response.statusCode !== 200) {
+                    reject(new core.GeneralError("node", "extensionDownloadFailed", {
+                        url,
+                        status: response.statusCode ?? 0
+                    }));
+                    return;
+                }
+                response.on("data", (chunk) => {
+                    downloadedSize += chunk.length;
+                    if (downloadedSize > maxSizeBytes) {
+                        response.destroy();
+                        reject(new core.GeneralError("node", "extensionSizeLimitExceeded", {
+                            size: downloadedSize,
+                            limit: maxSizeBytes
+                        }));
+                        return;
+                    }
+                    chunks.push(chunk);
+                });
+                response.on("end", () => {
+                    resolve();
+                });
+                response.on("error", err => {
+                    reject(core.BaseError.fromError(err));
+                });
+            }).on("error", err => {
+                reject(core.BaseError.fromError(err));
+            });
+        });
+    }
+    catch (err) {
+        throw new core.GeneralError("node", "extensionDownloadFailed", {
+            url
+        }, core.BaseError.fromError(err));
+    }
+    const tempPath = `${cachedPath}.tmp`;
+    await promises.writeFile(tempPath, Buffer.concat(chunks));
+    // Atomic move from temp to final location
+    await promises.rename(tempPath, cachedPath);
+    // Save metadata for TTL tracking
+    const metadata = {
+        downloadedAt: Date.now(),
+        url,
+        size: Buffer.concat(chunks).length
+    };
+    await promises.writeFile(metadataPath, JSON.stringify(metadata, null, 2));
+    return {
+        resolvedPath: cachedPath,
+        cached: false
+    };
+}
+/**
+ * Resolve the main entry point from a package directory using Node.js resolution with fallback.
+ * Uses require.resolve() when possible for standard Node.js behavior, with manual fallback.
+ * @param packagePath The absolute path to the package directory.
+ * @param packageName The package name for require.resolve().
+ * @param fallback The fallback file name if no entry point is found.
+ * @returns The resolved entry point file name (relative to package directory).
+ */
+async function resolvePackageEntryPoint(packagePath, packageName, fallback = "index.js") {
+    try {
+        // Try require.resolve() first - handles exports, main, module automatically
+        const resolvedPath = require.resolve(packageName, { paths: [path.dirname(packagePath)] });
+        // Convert absolute path back to relative filename within package
+        const relativePath = path.relative(packagePath, resolvedPath);
+        return relativePath ?? fallback;
+    }
+    catch {
+        // Fallback to manual package.json parsing if require.resolve fails
+        try {
+            const packageJsonPath = path.join(packagePath, "package.json");
+            const packageJsonContent = await loadJsonFile(packageJsonPath);
+            // Future: Could expand exports field support here
+            return packageJsonContent.module ?? packageJsonContent.main ?? fallback;
+        }
+        catch {
+            return fallback;
+        }
+    }
+}
+/**
+ * Convert a file path to an import-compatible URL for cross-platform module loading.
+ * On Windows, adds the 'file://' protocol prefix required for dynamic imports.
+ * On other platforms, returns the path unchanged.
+ * @param filePath The absolute file path to convert.
+ * @returns A URL string compatible with dynamic import().
+ */
+function createModuleImportUrl(filePath) {
+    return process.platform === "win32" ? `file://${filePath}` : filePath;
+}
 
 // Copyright 2024 IOTA Stiftung.
 // SPDX-License-Identifier: Apache-2.0.
@@ -1994,7 +2298,7 @@ async function run(nodeOptions) {
         nodeOptions ??= {};
         const serverInfo = {
             name: nodeOptions?.serverName ?? "TWIN Node Server",
-            version: nodeOptions?.serverVersion ?? "0.0.2-next.25" // x-release-please-version
+            version: nodeOptions?.serverVersion ?? "0.0.2-next.26" // x-release-please-version
         };
         cliCore.CLIDisplay.header(serverInfo.name, serverInfo.version, "🌩️ ");
         if (!core.Is.stringValue(nodeOptions?.executionDirectory)) {
@@ -2132,10 +2436,13 @@ async function buildConfiguration(processEnv, options, serverInfo) {
     return { nodeEngineConfig, nodeEnvVars: envVars };
 }
 /**
- * Override module imports to use local files where possible.
+ * Override module imports to support protocol-based loading (npm:, https:) and local files.
  * @param executionDirectory The execution directory for resolving local module paths.
+ * @param envVars The environment variables containing extension configuration (optional, uses defaults if not provided).
  */
-function overrideModuleImport(executionDirectory) {
+function overrideModuleImport(executionDirectory, envVars) {
+    const maxSizeMb = core.Coerce.number(envVars?.extensionsMaxSizeMb) ?? 10;
+    const cacheDirectory = envVars?.extensionsCacheDirectory;
     modules.ModuleHelper.overrideImport(async (moduleName) => {
         if (moduleCache[moduleName]) {
             return {
@@ -2143,55 +2450,77 @@ function overrideModuleImport(executionDirectory) {
                 useDefault: false
             };
         }
-        // If the module path for example when dynamically loading
-        // modules looks like a local file then we try to resolve
-        // using the local file system
-        const isLocal = modules.ModuleHelper.isLocalModule(moduleName);
-        if (isLocal) {
-            // See if we can just resolve the filename locally
-            let localFilename = path.resolve(moduleName);
-            let exists = await fileExists(localFilename);
-            if (!exists) {
-                // Doesn't exist in the current directory, try the execution directory
-                localFilename = path.resolve(executionDirectory, moduleName);
-                exists = await fileExists(localFilename);
-            }
-            if (exists) {
-                // If the module exists then we can load it, otherwise
-                // we fallback to regular handling to see if that can import it
-                const module = await import(process.platform === "win32" ? `file://${localFilename}` : localFilename);
-                moduleCache[moduleName] = module;
-                return {
-                    module,
-                    useDefault: false
-                };
+        const parsed = parseModuleProtocol(moduleName);
+        let resolvedPath;
+        switch (parsed.protocol) {
+            case ModuleProtocol.Npm: {
+                const result = await handleNpmProtocol(parsed.identifier, executionDirectory, cacheDirectory);
+                resolvedPath = result.resolvedPath;
+                break;
             }
-        }
-        try {
-            // Try and load from node_modules manually
-            // This is needed for some environments where
-            // the module resolution doesn't work as expected
-            const npmRoot = node_child_process.execSync("npm root").toString().trim().replace(/\\/g, "/");
-            const packageJson = await loadJsonFile(path.resolve(npmRoot, moduleName, "package.json"));
-            const mainFile = packageJson?.module ?? packageJson?.main ?? "index.js";
-            const modulePath = path.resolve(npmRoot, moduleName, mainFile);
-            const exists = await fileExists(modulePath);
-            if (exists) {
-                const module = await import(process.platform === "win32" ? `file://${modulePath}` : modulePath);
-                moduleCache[moduleName] = module;
-                return {
-                    module,
-                    useDefault: false
-                };
+            case ModuleProtocol.Https: {
+                const result = await handleHttpsProtocol(parsed.identifier, executionDirectory, maxSizeMb, cacheDirectory, envVars?.extensionsCacheTtlHours, envVars?.extensionsForceRefresh);
+                resolvedPath = result.resolvedPath;
+                break;
+            }
+            case ModuleProtocol.Http: {
+                throw new core.GeneralError("node", "insecureProtocol", { protocol: ModuleProtocol.Http });
+            }
+            case ModuleProtocol.Local: {
+                let localFilename = path.resolve(moduleName);
+                let exists = await fileExists(localFilename);
+                if (!exists) {
+                    localFilename = path.resolve(executionDirectory, moduleName);
+                    exists = await fileExists(localFilename);
+                }
+                if (exists) {
+                    resolvedPath = localFilename;
+                }
+                break;
+            }
+            case ModuleProtocol.Default: {
+                try {
+                    const npmRoot = node_child_process.execSync("npm root").toString().trim().replace(/\\/g, "/");
+                    const packagePath = path.resolve(npmRoot, moduleName);
+                    const mainFile = await resolvePackageEntryPoint(packagePath, moduleName);
+                    const modulePath = path.resolve(packagePath, mainFile);
+                    const exists = await fileExists(modulePath);
+                    if (exists) {
+                        resolvedPath = modulePath;
+                        break;
+                    }
+                }
+                catch {
+                    // Continue to fallback resolution
+                }
+                // Fallback: resolve from npm protocol cache directory (installed via handleNpmProtocol)
+                try {
+                    const cacheNpmRoot = path.resolve(getExtensionsCacheDir(executionDirectory, ModuleProtocol.Npm, cacheDirectory), "node_modules");
+                    const packagePath = path.resolve(cacheNpmRoot, moduleName);
+                    const mainFile = await resolvePackageEntryPoint(packagePath, moduleName);
+                    const modulePath = path.resolve(packagePath, mainFile);
+                    const exists = await fileExists(modulePath);
+                    if (exists) {
+                        resolvedPath = modulePath;
+                    }
+                }
+                catch {
+                    // No cached resolution either; fall through
+                }
+                break;
             }
         }
-        catch {
-            // We just fallback to default handling if not possible
+        // Common module loading and caching logic
+        if (resolvedPath) {
+            const module = await import(createModuleImportUrl(resolvedPath));
+            moduleCache[moduleName] = module;
+            return {
+                module,
+                useDefault: false
+            };
         }
-        // We don't appear to be able to manually resolve this module
-        // So we let the default handling take care of it
-        // This will allow built-in modules and regular node_modules to load as normal
         return {
+            module: undefined,
             useDefault: true
         };
     });
@@ -2201,6 +2530,7 @@ exports.ATTESTATION_VERIFICATION_METHOD_ID = ATTESTATION_VERIFICATION_METHOD_ID;
 exports.AUTH_SIGNING_KEY_ID = AUTH_SIGNING_KEY_ID;
 exports.BLOB_STORAGE_ENCRYPTION_KEY_ID = BLOB_STORAGE_ENCRYPTION_KEY_ID;
 exports.IMMUTABLE_PROOF_VERIFICATION_METHOD_ID = IMMUTABLE_PROOF_VERIFICATION_METHOD_ID;
+exports.ModuleProtocol = ModuleProtocol;
 exports.NodeFeatures = NodeFeatures;
 exports.SYNCHRONISED_STORAGE_BLOB_STORAGE_ENCRYPTION_KEY_ID = SYNCHRONISED_STORAGE_BLOB_STORAGE_ENCRYPTION_KEY_ID;
 exports.VC_AUTHENTICATION_VERIFICATION_METHOD_ID = VC_AUTHENTICATION_VERIFICATION_METHOD_ID;
@@ -2214,19 +2544,27 @@ exports.bootstrapSynchronisedStorage = bootstrapSynchronisedStorage;
 exports.buildConfiguration = buildConfiguration;
 exports.buildEngineConfiguration = buildEngineConfiguration;
 exports.buildEngineServerConfiguration = buildEngineServerConfiguration;
+exports.createModuleImportUrl = createModuleImportUrl;
 exports.directoryExists = directoryExists;
 exports.extensionsConfiguration = extensionsConfiguration;
 exports.extensionsInitialiseEngine = extensionsInitialiseEngine;
 exports.extensionsInitialiseEngineServer = extensionsInitialiseEngineServer;
 exports.fileExists = fileExists;
 exports.getExecutionDirectory = getExecutionDirectory;
+exports.getExtensionsCacheDir = getExtensionsCacheDir;
 exports.getFeatures = getFeatures;
 exports.getFiles = getFiles;
 exports.getSubFolders = getSubFolders;
+exports.handleHttpsProtocol = handleHttpsProtocol;
+exports.handleNpmProtocol = handleNpmProtocol;
+exports.hashUrl = hashUrl;
 exports.initialiseLocales = initialiseLocales;
+exports.isCacheExpired = isCacheExpired;
 exports.loadJsonFile = loadJsonFile;
 exports.loadTextFile = loadTextFile;
 exports.overrideModuleImport = overrideModuleImport;
+exports.parseModuleProtocol = parseModuleProtocol;
+exports.resolvePackageEntryPoint = resolvePackageEntryPoint;
 exports.run = run;
 exports.shutdownExtensions = shutdownExtensions;
 exports.start = start;