@twin.org/node-core 0.0.2-next.24 → 0.0.2-next.26

package/dist/types/utils.d.ts

@@ -1,4 +1,7 @@
+import type { IModuleProtocol } from "./models/IModuleProtocol";
 import type { INodeEnvironmentVariables } from "./models/INodeEnvironmentVariables";
+import type { IProtocolHandlerResult } from "./models/IProtocolHandlerResult";
+import { ModuleProtocol } from "./models/moduleProtocol";
 import { NodeFeatures } from "./models/nodeFeatures";
 /**
  * Initialise the locales for the application.
@@ -52,3 +55,67 @@ export declare function loadJsonFile<T>(filename: string): Promise<T>;
  * @returns The features that are enabled on the node.
  */
 export declare function getFeatures(env: INodeEnvironmentVariables): NodeFeatures[];
+/**
+ * Parse the protocol from a module name.
+ * @param moduleName The module name to parse.
+ * @returns The parsed protocol information.
+ */
+export declare function parseModuleProtocol(moduleName: string): IModuleProtocol;
+/**
+ * Hash a URL to create a safe filename.
+ * @param url The URL to hash.
+ * @returns A hashed filename safe for the filesystem.
+ */
+export declare function hashUrl(url: string): string;
+/**
+ * Get the extensions cache directory.
+ * @param executionDirectory The execution directory.
+ * @param protocol The protocol type for subdirectory organization.
+ * @param cacheDirectory The cache directory base path.
+ * @returns The cache directory path.
+ */
+export declare function getExtensionsCacheDir(executionDirectory: string, protocol: ModuleProtocol, cacheDirectory?: string): string;
+/**
+ * Handle the npm: protocol by installing the package if needed.
+ * @param packageName The npm package name (without npm: prefix).
+ * @param executionDirectory The execution directory.
+ * @param cacheDirectory The cache directory base path.
+ * @returns The resolved path to the installed module.
+ */
+export declare function handleNpmProtocol(packageName: string, executionDirectory: string, cacheDirectory?: string): Promise<IProtocolHandlerResult>;
+/**
+ * Check if a cached file has expired based on TTL and force refresh settings.
+ * @param metadataPath Path to the cache metadata file.
+ * @param ttlHours Time to live in hours.
+ * @param forceRefresh Whether to force refresh regardless of TTL.
+ * @returns True if the cache is expired or should be refreshed.
+ */
+export declare function isCacheExpired(metadataPath: string, ttlHours: number, forceRefresh: boolean): Promise<boolean>;
+/**
+ * Handle the https: protocol by downloading the module if needed.
+ * @param url The HTTPS URL to download from.
+ * @param executionDirectory The execution directory.
+ * @param maxSizeMb The maximum size in MB for the download.
+ * @param cacheDirectory The cache directory base path.
+ * @param ttlHours TTL in hours for cache expiration.
+ * @param forceRefresh Whether to force refresh the cache.
+ * @returns The resolved path to the downloaded module.
+ */
+export declare function handleHttpsProtocol(url: string, executionDirectory: string, maxSizeMb: number, cacheDirectory?: string, ttlHours?: number, forceRefresh?: boolean): Promise<IProtocolHandlerResult>;
+/**
+ * Resolve the main entry point from a package directory using Node.js resolution with fallback.
+ * Uses require.resolve() when possible for standard Node.js behavior, with manual fallback.
+ * @param packagePath The absolute path to the package directory.
+ * @param packageName The package name for require.resolve().
+ * @param fallback The fallback file name if no entry point is found.
+ * @returns The resolved entry point file name (relative to package directory).
+ */
+export declare function resolvePackageEntryPoint(packagePath: string, packageName: string, fallback?: string): Promise<string>;
+/**
+ * Convert a file path to an import-compatible URL for cross-platform module loading.
+ * On Windows, adds the 'file://' protocol prefix required for dynamic imports.
+ * On other platforms, returns the path unchanged.
+ * @param filePath The absolute file path to convert.
+ * @returns A URL string compatible with dynamic import().
+ */
+export declare function createModuleImportUrl(filePath: string): string;

package/docs/changelog.md

@@ -1,5 +1,19 @@
 # @twin.org/node-core - Changelog
 
+## [0.0.2-next.26](https://github.com/twinfoundation/node/compare/node-core-v0.0.2-next.25...node-core-v0.0.2-next.26) (2025-10-10)
+
+
+### Features
+
+* adding npm and https protocols to load extensions ([#45](https://github.com/twinfoundation/node/issues/45)) ([33940b7](https://github.com/twinfoundation/node/commit/33940b7e771a0c5af32c18d442deb26a8631fd02))
+
+## [0.0.2-next.25](https://github.com/twinfoundation/node/compare/node-core-v0.0.2-next.24...node-core-v0.0.2-next.25) (2025-10-09)
+
+
+### Features
+
+* add validate-locales ([1a19dcb](https://github.com/twinfoundation/node/commit/1a19dcb005c2f0e3103e290db28c48a3464094cb))
+
 ## [0.0.2-next.24](https://github.com/twinfoundation/node/compare/node-core-v0.0.2-next.23...node-core-v0.0.2-next.24) (2025-10-08)
 
 

package/docs/detailed-guide.md

@@ -0,0 +1,129 @@
+# TWIN Node Core - Detailed Documentation
+
+## Overview
+
+TWIN Node Core provides the foundational components for running TWIN nodes, including dynamic extension loading, protocol-based module resolution, and lifecycle management.
+
+## Protocol-Based Extension Loading
+
+The TWIN Node Core supports dynamic extension loading from multiple sources:
+
+### Local Files
+
+Load extensions from your filesystem:
+
+```bash
+TWIN_NODE_EXTENSIONS="./my-extension.mjs"
+```
+
+### NPM Packages
+
+Automatically install and load npm packages:
+
+```bash
+TWIN_NODE_EXTENSIONS="npm:@twin.org/package"
+```
+
+### HTTPS URLs
+
+Download and cache remote extensions:
+
+```bash
+TWIN_NODE_EXTENSIONS="https://example.com/extension.mjs"
+```
+
+## Extension Lifecycle Hooks
+
+Extensions can implement lifecycle hooks to customize node behavior at different stages:
+
+- **`extensionInitialise`** - Called before engine initialization
+- **`extensionInitialiseEngine`** - Called during engine initialization
+- **`extensionInitialiseEngineServer`** - Called during server initialization
+- **`extensionShutdown`** - Called during graceful shutdown
+
+### Example Extension
+
+```javascript
+// my-extension.mjs
+export function extensionInitialise() {
+  console.log('Extension initializing...');
+}
+
+export function extensionInitialiseEngine(engine) {
+  console.log('Configuring engine...');
+  // Add custom components, routes, etc.
+}
+
+export function extensionInitialiseEngineServer(server) {
+  console.log('Configuring server...');
+  // Add middleware, routes, etc.
+}
+
+export function extensionShutdown() {
+  console.log('Extension shutting down...');
+  // Cleanup resources
+}
+```
+
+## Configuration Options
+
+### Cache Management
+
+- `TWIN_NODE_EXTENSIONS_CACHE_TTL_HOURS` - TTL for HTTPS extensions cache (default: 24)
+- `TWIN_NODE_EXTENSIONS_FORCE_REFRESH` - Force refresh all cached extensions (default: false)
+- `TWIN_NODE_EXTENSIONS_CACHE_DIRECTORY` - Custom cache directory (default: ".tmp")
+
+### Security
+
+- `TWIN_NODE_EXTENSIONS_MAX_SIZE_MB` - Maximum size for HTTPS downloads (default: 10)
+- `TWIN_NODE_EXTENSIONS_CLEAR_CACHE` - Clear cache on startup (default: false)
+
+## API Reference
+
+### Core Functions
+
+#### `overrideModuleImport(executionDirectory, envVars)`
+
+Sets up protocol-based module resolution for extensions.
+
+#### `buildConfiguration(options)`
+
+Builds node configuration from environment variables and options.
+
+#### `start(options)`
+
+Starts the TWIN node with the specified configuration.
+
+### Utility Functions
+
+#### `parseModuleProtocol(moduleName)`
+
+Parses module name to determine protocol type (npm, https, local, etc.).
+
+#### `handleNpmProtocol(packageName, executionDirectory, cacheDirectory)`
+
+Handles installation and resolution of npm packages.
+
+#### `handleHttpsProtocol(url, executionDirectory, maxSizeMb, cacheDirectory, ttlHours, forceRefresh)`
+
+Handles download and caching of HTTPS extensions.
+
+## Security Considerations
+
+- **HTTPS Only**: Remote extensions must use HTTPS protocol
+- **Size Limits**: Downloads are limited by `TWIN_NODE_EXTENSIONS_MAX_SIZE_MB`
+- **Cache TTL**: Extensions are automatically refreshed based on TTL
+- **Security Warnings**: Warnings are displayed when loading remote extensions
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Extension not found**: Check that the path/URL is correct and accessible
+2. **Cache issues**: Use `TWIN_NODE_EXTENSIONS_FORCE_REFRESH=true` to force refresh
+3. **Size limit exceeded**: Increase `TWIN_NODE_EXTENSIONS_MAX_SIZE_MB` if needed
+4. **Network issues**: Ensure HTTPS URLs are accessible and not blocked by firewall
+
+### Debug Mode
+
+Set `NODE_ENV=development` for additional debug output during extension loading.

package/docs/reference/functions/createModuleImportUrl.md

@@ -0,0 +1,21 @@
+# Function: createModuleImportUrl()
+
+> **createModuleImportUrl**(`filePath`): `string`
+
+Convert a file path to an import-compatible URL for cross-platform module loading.
+On Windows, adds the 'file://' protocol prefix required for dynamic imports.
+On other platforms, returns the path unchanged.
+
+## Parameters
+
+### filePath
+
+`string`
+
+The absolute file path to convert.
+
+## Returns
+
+`string`
+
+A URL string compatible with dynamic import().

package/docs/reference/functions/getExtensionsCacheDir.md

@@ -0,0 +1,31 @@
+# Function: getExtensionsCacheDir()
+
+> **getExtensionsCacheDir**(`executionDirectory`, `protocol`, `cacheDirectory?`): `string`
+
+Get the extensions cache directory.
+
+## Parameters
+
+### executionDirectory
+
+`string`
+
+The execution directory.
+
+### protocol
+
+[`ModuleProtocol`](../type-aliases/ModuleProtocol.md)
+
+The protocol type for subdirectory organization.
+
+### cacheDirectory?
+
+`string`
+
+The cache directory base path.
+
+## Returns
+
+`string`
+
+The cache directory path.

package/docs/reference/functions/handleHttpsProtocol.md

@@ -0,0 +1,49 @@
+# Function: handleHttpsProtocol()
+
+> **handleHttpsProtocol**(`url`, `executionDirectory`, `maxSizeMb`, `cacheDirectory?`, `ttlHours?`, `forceRefresh?`): `Promise`\<[`IProtocolHandlerResult`](../interfaces/IProtocolHandlerResult.md)\>
+
+Handle the https: protocol by downloading the module if needed.
+
+## Parameters
+
+### url
+
+`string`
+
+The HTTPS URL to download from.
+
+### executionDirectory
+
+`string`
+
+The execution directory.
+
+### maxSizeMb
+
+`number`
+
+The maximum size in MB for the download.
+
+### cacheDirectory?
+
+`string`
+
+The cache directory base path.
+
+### ttlHours?
+
+`number`
+
+TTL in hours for cache expiration.
+
+### forceRefresh?
+
+`boolean`
+
+Whether to force refresh the cache.
+
+## Returns
+
+`Promise`\<[`IProtocolHandlerResult`](../interfaces/IProtocolHandlerResult.md)\>
+
+The resolved path to the downloaded module.

package/docs/reference/functions/handleNpmProtocol.md

@@ -0,0 +1,31 @@
+# Function: handleNpmProtocol()
+
+> **handleNpmProtocol**(`packageName`, `executionDirectory`, `cacheDirectory?`): `Promise`\<[`IProtocolHandlerResult`](../interfaces/IProtocolHandlerResult.md)\>
+
+Handle the npm: protocol by installing the package if needed.
+
+## Parameters
+
+### packageName
+
+`string`
+
+The npm package name (without npm: prefix).
+
+### executionDirectory
+
+`string`
+
+The execution directory.
+
+### cacheDirectory?
+
+`string`
+
+The cache directory base path.
+
+## Returns
+
+`Promise`\<[`IProtocolHandlerResult`](../interfaces/IProtocolHandlerResult.md)\>
+
+The resolved path to the installed module.

package/docs/reference/functions/hashUrl.md

@@ -0,0 +1,19 @@
+# Function: hashUrl()
+
+> **hashUrl**(`url`): `string`
+
+Hash a URL to create a safe filename.
+
+## Parameters
+
+### url
+
+`string`
+
+The URL to hash.
+
+## Returns
+
+`string`
+
+A hashed filename safe for the filesystem.

package/docs/reference/functions/isCacheExpired.md

@@ -0,0 +1,31 @@
+# Function: isCacheExpired()
+
+> **isCacheExpired**(`metadataPath`, `ttlHours`, `forceRefresh`): `Promise`\<`boolean`\>
+
+Check if a cached file has expired based on TTL and force refresh settings.
+
+## Parameters
+
+### metadataPath
+
+`string`
+
+Path to the cache metadata file.
+
+### ttlHours
+
+`number`
+
+Time to live in hours.
+
+### forceRefresh
+
+`boolean`
+
+Whether to force refresh regardless of TTL.
+
+## Returns
+
+`Promise`\<`boolean`\>
+
+True if the cache is expired or should be refreshed.

package/docs/reference/functions/overrideModuleImport.md

@@ -1,8 +1,8 @@
 # Function: overrideModuleImport()
 
-> **overrideModuleImport**(`executionDirectory`): `void`
+> **overrideModuleImport**(`executionDirectory`, `envVars?`): `void`
 
-Override module imports to use local files where possible.
+Override module imports to support protocol-based loading (npm:, https:) and local files.
 
 ## Parameters
 
@@ -12,6 +12,12 @@ Override module imports to use local files where possible.
 
 The execution directory for resolving local module paths.
 
+### envVars?
+
+[`INodeEnvironmentVariables`](../interfaces/INodeEnvironmentVariables.md)
+
+The environment variables containing extension configuration (optional, uses defaults if not provided).
+
 ## Returns
 
 `void`

package/docs/reference/functions/parseModuleProtocol.md

@@ -0,0 +1,19 @@
+# Function: parseModuleProtocol()
+
+> **parseModuleProtocol**(`moduleName`): [`IModuleProtocol`](../interfaces/IModuleProtocol.md)
+
+Parse the protocol from a module name.
+
+## Parameters
+
+### moduleName
+
+`string`
+
+The module name to parse.
+
+## Returns
+
+[`IModuleProtocol`](../interfaces/IModuleProtocol.md)
+
+The parsed protocol information.

package/docs/reference/functions/resolvePackageEntryPoint.md

@@ -0,0 +1,32 @@
+# Function: resolvePackageEntryPoint()
+
+> **resolvePackageEntryPoint**(`packagePath`, `packageName`, `fallback`): `Promise`\<`string`\>
+
+Resolve the main entry point from a package directory using Node.js resolution with fallback.
+Uses require.resolve() when possible for standard Node.js behavior, with manual fallback.
+
+## Parameters
+
+### packagePath
+
+`string`
+
+The absolute path to the package directory.
+
+### packageName
+
+`string`
+
+The package name for require.resolve().
+
+### fallback
+
+`string` = `"index.js"`
+
+The fallback file name if no entry point is found.
+
+## Returns
+
+`Promise`\<`string`\>
+
+The resolved entry point file name (relative to package directory).

package/docs/reference/index.md

@@ -2,14 +2,18 @@
 
 ## Interfaces
 
+- [ICacheMetadata](interfaces/ICacheMetadata.md)
 - [IEngineEnvironmentVariables](interfaces/IEngineEnvironmentVariables.md)
 - [IEngineServerEnvironmentVariables](interfaces/IEngineServerEnvironmentVariables.md)
+- [IModuleProtocol](interfaces/IModuleProtocol.md)
 - [INodeEngineConfig](interfaces/INodeEngineConfig.md)
 - [INodeEnvironmentVariables](interfaces/INodeEnvironmentVariables.md)
 - [INodeOptions](interfaces/INodeOptions.md)
+- [IProtocolHandlerResult](interfaces/IProtocolHandlerResult.md)
 
 ## Type Aliases
 
+- [ModuleProtocol](type-aliases/ModuleProtocol.md)
 - [NodeExtensionInitialiseMethod](type-aliases/NodeExtensionInitialiseMethod.md)
 - [NodeExtensionInitialiseEngineMethod](type-aliases/NodeExtensionInitialiseEngineMethod.md)
 - [NodeExtensionInitialiseEngineServerMethod](type-aliases/NodeExtensionInitialiseEngineServerMethod.md)
@@ -24,6 +28,7 @@
 - [SYNCHRONISED\_STORAGE\_BLOB\_STORAGE\_ENCRYPTION\_KEY\_ID](variables/SYNCHRONISED_STORAGE_BLOB_STORAGE_ENCRYPTION_KEY_ID.md)
 - [VC\_AUTHENTICATION\_VERIFICATION\_METHOD\_ID](variables/VC_AUTHENTICATION_VERIFICATION_METHOD_ID.md)
 - [AUTH\_SIGNING\_KEY\_ID](variables/AUTH_SIGNING_KEY_ID.md)
+- [ModuleProtocol](variables/ModuleProtocol.md)
 - [NodeFeatures](variables/NodeFeatures.md)
 
 ## Functions
@@ -54,3 +59,11 @@
 - [loadTextFile](functions/loadTextFile.md)
 - [loadJsonFile](functions/loadJsonFile.md)
 - [getFeatures](functions/getFeatures.md)
+- [parseModuleProtocol](functions/parseModuleProtocol.md)
+- [hashUrl](functions/hashUrl.md)
+- [getExtensionsCacheDir](functions/getExtensionsCacheDir.md)
+- [handleNpmProtocol](functions/handleNpmProtocol.md)
+- [isCacheExpired](functions/isCacheExpired.md)
+- [handleHttpsProtocol](functions/handleHttpsProtocol.md)
+- [resolvePackageEntryPoint](functions/resolvePackageEntryPoint.md)
+- [createModuleImportUrl](functions/createModuleImportUrl.md)

package/docs/reference/interfaces/ICacheMetadata.md

@@ -0,0 +1,27 @@
+# Interface: ICacheMetadata
+
+Metadata for cached HTTPS extensions.
+
+## Properties
+
+### downloadedAt
+
+> **downloadedAt**: `number`
+
+Timestamp when the file was downloaded.
+
+***
+
+### url
+
+> **url**: `string`
+
+Original URL of the cached file.
+
+***
+
+### size
+
+> **size**: `number`
+
+Size of the cached file in bytes.

package/docs/reference/interfaces/IModuleProtocol.md

@@ -0,0 +1,27 @@
+# Interface: IModuleProtocol
+
+The parsed module protocol information.
+
+## Properties
+
+### protocol
+
+> **protocol**: [`ModuleProtocol`](../type-aliases/ModuleProtocol.md)
+
+The protocol type.
+
+***
+
+### identifier
+
+> **identifier**: `string`
+
+The identifier after the protocol (or the original if no protocol).
+
+***
+
+### original
+
+> **original**: `string`
+
+The original module string.

package/docs/reference/interfaces/INodeEnvironmentVariables.md

@@ -1861,3 +1861,73 @@ admin@node
 > `optional` **password**: `string`
 
 If the node-user feature is enabled, this will be the password of the user, if empty it will be randomly generated.
+
+***
+
+### extensionsMaxSizeMb?
+
+> `optional` **extensionsMaxSizeMb**: `number`
+
+Maximum size in MB for HTTPS extensions downloads.
+
+#### Default
+
+```ts
+10
+```
+
+***
+
+### extensionsClearCache?
+
+> `optional` **extensionsClearCache**: `boolean`
+
+Whether to clear the extensions cache on startup.
+
+#### Default
+
+```ts
+false
+```
+
+***
+
+### extensionsCacheDirectory?
+
+> `optional` **extensionsCacheDirectory**: `string`
+
+Custom directory for extensions cache storage.
+
+#### Default
+
+```ts
+".tmp"
+```
+
+***
+
+### extensionsCacheTtlHours?
+
+> `optional` **extensionsCacheTtlHours**: `number`
+
+TTL in hours for HTTPS extensions cache.
+
+#### Default
+
+```ts
+24
+```
+
+***
+
+### extensionsForceRefresh?
+
+> `optional` **extensionsForceRefresh**: `boolean`
+
+Force refresh of all cached extensions.
+
+#### Default
+
+```ts
+false
+```

package/docs/reference/interfaces/IProtocolHandlerResult.md

@@ -0,0 +1,19 @@
+# Interface: IProtocolHandlerResult
+
+The result from a protocol handler.
+
+## Properties
+
+### resolvedPath
+
+> **resolvedPath**: `string`
+
+The resolved path to the module file.
+
+***
+
+### cached
+
+> **cached**: `boolean`
+
+Whether the module was cached.

package/docs/reference/type-aliases/ModuleProtocol.md

@@ -0,0 +1,5 @@
+# Type Alias: ModuleProtocol
+
+> **ModuleProtocol** = *typeof* [`ModuleProtocol`](../variables/ModuleProtocol.md)\[keyof *typeof* [`ModuleProtocol`](../variables/ModuleProtocol.md)\]
+
+The protocol type for a module.