@rosepetal/node-red-contrib-async-function 1.0.0 → 1.0.2

package/nodes/lib/child-process-script.js

@@ -0,0 +1,264 @@
+/**
+ * Child Process Script
+ *
+ * Executes user-provided JavaScript code in a child process.
+ * Handles message communication with the parent process.
+ * Restores buffers from shared memory before execution.
+ */
+
+const { AsyncLocalStorage } = require('async_hooks');
+const { createRequire, builtinModules } = require('module');
+const path = require('path');
+const url = require('url');
+const { SharedMemoryManager } = require('./shared-memory-manager');
+const { AsyncMessageSerializer } = require('./message-serializer');
+
+// Track worker state
+let isTerminating = false;
+let isInitialized = false;
+
+// AsyncLocalStorage for tracking task context across async boundaries
+const taskContext = new AsyncLocalStorage();
+
+function hrtimeDiffToMs(start) {
+    if (typeof start !== 'bigint') {
+        return 0;
+    }
+    const diff = process.hrtime.bigint() - start;
+    return Number(diff) / 1e6;
+}
+
+// Shared memory manager and serializer are initialized after init message
+let shmManager = null;
+let serializer = null;
+
+// Cache compiled user code per worker for hot-path performance
+const AsyncFunction = (async function() {}).constructor;
+const MAX_CACHE_SIZE = 100;
+const compiledCodeCache = new Map();
+
+function getCachedFunction(cacheKey) {
+    const fn = compiledCodeCache.get(cacheKey);
+    if (fn) {
+        compiledCodeCache.delete(cacheKey);
+        compiledCodeCache.set(cacheKey, fn);
+    }
+    return fn;
+}
+
+function setCachedFunction(cacheKey, fn) {
+    if (compiledCodeCache.size >= MAX_CACHE_SIZE) {
+        const oldestKey = compiledCodeCache.keys().next().value;
+        compiledCodeCache.delete(oldestKey);
+    }
+    compiledCodeCache.set(cacheKey, fn);
+}
+
+// Module loading setup
+let baseRequire = require;
+const loadedModules = {};
+const moduleVars = [];
+const moduleValues = [];
+const failedModules = [];
+
+function configureBaseRequire(nodeRedUserDir) {
+    if (nodeRedUserDir) {
+        baseRequire = createRequire(path.join(nodeRedUserDir, 'package.json'));
+    } else {
+        baseRequire = require;
+    }
+    global.require = baseRequire;
+}
+
+function parseModuleSpec(spec) {
+    const match = /((?:@[^/]+\/)?[^/@]+)(\/[^/@]+)?(?:@([\s\S]+))?/.exec(spec);
+    if (!match) {
+        return { spec, module: spec, subpath: '', builtin: false };
+    }
+    const moduleName = match[1];
+    const subpath = match[2] || '';
+    let builtinName = moduleName;
+    if (builtinName.startsWith('node:')) {
+        builtinName = builtinName.slice(5);
+    }
+    const builtin = builtinModules.includes(builtinName);
+    return { spec, module: moduleName, subpath, builtin };
+}
+
+async function loadModule(spec) {
+    const parsed = parseModuleSpec(spec);
+    if (parsed.builtin) {
+        return baseRequire(parsed.module + parsed.subpath);
+    }
+    const resolvedPath = baseRequire.resolve(spec);
+    const moduleUrl = url.pathToFileURL(resolvedPath);
+    const imported = await import(moduleUrl);
+    return imported.default || imported;
+}
+
+async function loadConfiguredModules(libs) {
+    if (!Array.isArray(libs)) {
+        return;
+    }
+    for (const lib of libs) {
+        if (!lib || !lib.module || !lib.var) {
+            continue;
+        }
+        try {
+            loadedModules[lib.var] = await loadModule(lib.module);
+            moduleVars.push(lib.var);
+            moduleValues.push(loadedModules[lib.var]);
+        } catch (err) {
+            console.error(`[async-function] Failed to load module ${lib.module}: ${err.message}`);
+            failedModules.push({ module: lib.module, var: lib.var, error: err.message });
+        }
+    }
+}
+
+function sendMessage(payload) {
+    if (typeof process.send === 'function') {
+        process.send(payload);
+    }
+}
+
+function sendError(taskId, err) {
+    const error = err instanceof Error ? err : new Error(String(err));
+    sendMessage({
+        type: 'error',
+        taskId,
+        error: {
+            message: error.message,
+            stack: error.stack || '',
+            name: error.name || 'Error'
+        }
+    });
+}
+
+async function initializeWorker(initData) {
+    const nodeRedUserDir = initData && initData.nodeRedUserDir ? initData.nodeRedUserDir : null;
+    configureBaseRequire(nodeRedUserDir);
+
+    const threshold = initData && typeof initData.shmThreshold === 'number' ? initData.shmThreshold : undefined;
+    shmManager = new SharedMemoryManager({
+        threshold,
+        trackAttachments: false,
+        cleanupOrphanedFiles: false
+    });
+    serializer = new AsyncMessageSerializer(shmManager);
+
+    await loadConfiguredModules(initData ? initData.libs : []);
+    isInitialized = true;
+
+    sendMessage({
+        type: 'ready',
+        failedModules: failedModules.length > 0 ? failedModules : undefined
+    });
+}
+
+process.on('message', async (data) => {
+    if (!data || typeof data !== 'object') {
+        return;
+    }
+
+    if (data.type === 'init') {
+        try {
+            await initializeWorker(data);
+        } catch (err) {
+            failedModules.push({ module: '(init)', var: null, error: err.message });
+            sendMessage({
+                type: 'ready',
+                failedModules
+            });
+        }
+        return;
+    }
+
+    if (isTerminating) {
+        return;
+    }
+
+    if (!isInitialized) {
+        if (data.type === 'execute') {
+            sendError(data.taskId, new Error('Worker not initialized'));
+        }
+        return;
+    }
+
+    const { type, taskId, code, msg } = data;
+
+    if (type === 'execute') {
+        taskContext.run({ taskId }, async () => {
+            try {
+                const restoreStart = process.hrtime.bigint();
+                const restoredMsg = await serializer.restoreBuffers(msg);
+                const transferToWorkerMs = hrtimeDiffToMs(restoreStart);
+
+                const cacheKey = code + '|' + moduleVars.join(',');
+                let userFunction = getCachedFunction(cacheKey);
+                if (!userFunction) {
+                    userFunction = new AsyncFunction('msg', ...moduleVars, code);
+                    setCachedFunction(cacheKey, userFunction);
+                }
+
+                const execStart = process.hrtime.bigint();
+                const rawResult = await userFunction(restoredMsg, ...moduleValues);
+                const executionMs = hrtimeDiffToMs(execStart);
+
+                const encodeStart = process.hrtime.bigint();
+                const encodedResult = await serializer.sanitizeMessage(rawResult, null, taskId);
+                const transferToMainMs = hrtimeDiffToMs(encodeStart);
+
+                sendMessage({
+                    type: 'result',
+                    taskId,
+                    result: encodedResult,
+                    performance: {
+                        transferToWorkerMs,
+                        executionMs,
+                        transferToMainMs
+                    }
+                });
+            } catch (err) {
+                sendError(taskId, err);
+            }
+        });
+    } else if (type === 'terminate') {
+        isTerminating = true;
+        sendMessage({
+            type: 'terminated',
+            taskId
+        });
+        process.exit(0);
+    }
+});
+
+process.on('uncaughtException', (err) => {
+    if (!isTerminating) {
+        const store = taskContext.getStore();
+        sendMessage({
+            type: 'error',
+            taskId: store?.taskId ?? null,
+            error: {
+                message: `Uncaught exception: ${err.message}`,
+                stack: err.stack || '',
+                name: err.name || 'Error'
+            }
+        });
+    }
+});
+
+process.on('unhandledRejection', (reason) => {
+    if (!isTerminating) {
+        const store = taskContext.getStore();
+        const error = reason instanceof Error ? reason : new Error(String(reason));
+        sendMessage({
+            type: 'error',
+            taskId: store?.taskId ?? null,
+            error: {
+                message: `Unhandled rejection: ${error.message}`,
+                stack: error.stack || '',
+                name: 'UnhandledRejection'
+            }
+        });
+    }
+});

package/nodes/lib/message-serializer.js

@@ -2,7 +2,7 @@
  * Async Message Serializer
  *
  * Fast message cloning for worker thread communication.
- * Matches the shared-memory + msg-copy semantics used by the python executor "hot mode":
+ * Matches the shared-memory + msg-copy semantics used by the legacy executor "hot mode":
  * - Buffers (and typed arrays) can be offloaded to shared memory with descriptors
  * - Base64 fallback for shared-memory failures
  * - Circular references preserved via WeakMap

package/nodes/lib/worker-pool.js

@@ -105,6 +105,18 @@ class WorkerPool {
                 worker.on('message', (msg) => {
                     if (msg.type === 'ready') {
                         clearTimeout(readyTimeout);
+                        if (msg.failedModules && msg.failedModules.length > 0) {
+                            const detail = msg.failedModules
+                                .map((failed) => `${failed.module} (${failed.var}): ${failed.error}`)
+                                .join('; ');
+                            const err = new Error(`Worker failed to load module(s): ${detail}`);
+                            err.failedModules = msg.failedModules;
+                            worker.terminate().finally(() => {
+                                reject(err);
+                            });
+                            return;
+                        }
+
                         workerState.state = WorkerState.IDLE;
                         this.workers.push(workerState);
                         resolve(workerState);

package/nodes/lib/worker-script.js

@@ -7,12 +7,20 @@
  */
 
 const { parentPort, workerData } = require('worker_threads');
+const { AsyncLocalStorage } = require('async_hooks');
+const { createRequire, builtinModules } = require('module');
+const path = require('path');
+const url = require('url');
 const { SharedMemoryManager } = require('./shared-memory-manager');
 const { AsyncMessageSerializer } = require('./message-serializer');
 
 // Track worker state
 let isTerminating = false;
 
+// AsyncLocalStorage for tracking task context across async boundaries
+// This ensures unhandled rejections can be attributed to the correct task
+const taskContext = new AsyncLocalStorage();
+
 function hrtimeDiffToMs(start) {
     if (typeof start !== 'bigint') {
         return 0;
@@ -58,26 +66,58 @@ function setCachedFunction(cacheKey, fn) {
 const loadedModules = {};
 const moduleVars = [];
 const moduleValues = [];
+const failedModules = [];  // Track modules that failed to load
+
+const baseRequire = (() => {
+    if (workerData && workerData.nodeRedUserDir) {
+        return createRequire(path.join(workerData.nodeRedUserDir, 'package.json'));
+    }
+    return require;
+})();
 
-// Add Node-RED user directory to module search path for external modules
-if (workerData && workerData.nodeRedUserDir) {
-    const path = require('path');
-    const nodeModulesPath = path.join(workerData.nodeRedUserDir, 'node_modules');
-    if (!module.paths.includes(nodeModulesPath)) {
-        module.paths.unshift(nodeModulesPath);
+global.require = baseRequire;
+
+function parseModuleSpec(spec) {
+    const match = /((?:@[^/]+\/)?[^/@]+)(\/[^/@]+)?(?:@([\s\S]+))?/.exec(spec);
+    if (!match) {
+        return { spec, module: spec, subpath: '', builtin: false };
+    }
+    const moduleName = match[1];
+    const subpath = match[2] || '';
+    let builtinName = moduleName;
+    if (builtinName.startsWith('node:')) {
+        builtinName = builtinName.slice(5);
     }
+    const builtin = builtinModules.includes(builtinName);
+    return { spec, module: moduleName, subpath, builtin };
 }
 
-if (workerData && workerData.libs && Array.isArray(workerData.libs)) {
+async function loadModule(spec) {
+    const parsed = parseModuleSpec(spec);
+    if (parsed.builtin) {
+        return baseRequire(parsed.module + parsed.subpath);
+    }
+    const resolvedPath = baseRequire.resolve(spec);
+    const moduleUrl = url.pathToFileURL(resolvedPath);
+    const imported = await import(moduleUrl);
+    return imported.default || imported;
+}
+
+async function loadConfiguredModules() {
+    if (!workerData || !Array.isArray(workerData.libs)) {
+        return;
+    }
     for (const lib of workerData.libs) {
-        if (lib.module && lib.var) {
-            try {
-                loadedModules[lib.var] = require(lib.module);
-                moduleVars.push(lib.var);
-                moduleValues.push(loadedModules[lib.var]);
-            } catch (err) {
-                console.error(`[async-function] Failed to load module ${lib.module}: ${err.message}`);
-            }
+        if (!lib || !lib.module || !lib.var) {
+            continue;
+        }
+        try {
+            loadedModules[lib.var] = await loadModule(lib.module);
+            moduleVars.push(lib.var);
+            moduleValues.push(loadedModules[lib.var]);
+        } catch (err) {
+            console.error(`[async-function] Failed to load module ${lib.module}: ${err.message}`);
+            failedModules.push({ module: lib.module, var: lib.var, error: err.message });
         }
     }
 }
@@ -85,7 +125,13 @@ if (workerData && workerData.libs && Array.isArray(workerData.libs)) {
 /**
  * Handle incoming messages from main thread
  */
-if (parentPort) {
+async function initializeWorker() {
+    await loadConfiguredModules();
+
+    if (!parentPort) {
+        return;
+    }
+
     parentPort.on('message', async (data) => {
         // Ignore messages if terminating
         if (isTerminating) {
@@ -96,55 +142,60 @@ if (parentPort) {
 
         // Handle different message types
         if (type === 'execute') {
-            try {
-                // Restore + execute user code
-                const restoreStart = process.hrtime.bigint();
-                const restoredMsg = await serializer.restoreBuffers(msg);
-                const transferToPythonMs = hrtimeDiffToMs(restoreStart);
-
-                // Cache key includes code + module vars to handle different module configs
-                const cacheKey = code + '|' + moduleVars.join(',');
-                let userFunction = getCachedFunction(cacheKey);
-                if (!userFunction) {
-                    // Create function with msg + all module variables as parameters
-                    userFunction = new AsyncFunction('msg', ...moduleVars, code);
-                    setCachedFunction(cacheKey, userFunction);
-                }
-
-                const execStart = process.hrtime.bigint();
-                // Execute with msg and all loaded module values
-                const rawResult = await userFunction(restoredMsg, ...moduleValues);
-                const executionMs = hrtimeDiffToMs(execStart);
-
-                // Offload buffers in the result (large Buffers -> shared memory descriptors)
-                const encodeStart = process.hrtime.bigint();
-                const encodedResult = await serializer.sanitizeMessage(rawResult, null, taskId);
-                const transferToJsMs = hrtimeDiffToMs(encodeStart);
-
-                // Send result back to main thread
-                parentPort.postMessage({
-                    type: 'result',
-                    taskId,
-                    result: encodedResult,
-                    performance: {
-                        transfer_to_python_ms: transferToPythonMs,
-                        execution_ms: executionMs,
-                        transfer_to_js_ms: transferToJsMs
-                    }
-                });
-
-            } catch (err) {
-                // Send error back to main thread
-                parentPort.postMessage({
-                    type: 'error',
-                    taskId,
-                    error: {
-                        message: err.message,
-                        stack: err.stack,
-                        name: err.name
+            // Run task within AsyncLocalStorage context for proper error attribution
+            // This ensures unhandled rejections from fire-and-forget promises
+            // can be traced back to the correct task
+            taskContext.run({ taskId }, async () => {
+                try {
+                    // Restore + execute user code
+                    const restoreStart = process.hrtime.bigint();
+                    const restoredMsg = await serializer.restoreBuffers(msg);
+                    const transferToWorkerMs = hrtimeDiffToMs(restoreStart);
+
+                    // Cache key includes code + module vars to handle different module configs
+                    const cacheKey = code + '|' + moduleVars.join(',');
+                    let userFunction = getCachedFunction(cacheKey);
+                    if (!userFunction) {
+                        // Create function with msg + all module variables as parameters
+                        userFunction = new AsyncFunction('msg', ...moduleVars, code);
+                        setCachedFunction(cacheKey, userFunction);
                     }
-                });
-            }
+
+                    const execStart = process.hrtime.bigint();
+                    // Execute with msg and all loaded module values
+                    const rawResult = await userFunction(restoredMsg, ...moduleValues);
+                    const executionMs = hrtimeDiffToMs(execStart);
+
+                    // Offload buffers in the result (large Buffers -> shared memory descriptors)
+                    const encodeStart = process.hrtime.bigint();
+                    const encodedResult = await serializer.sanitizeMessage(rawResult, null, taskId);
+                    const transferToMainMs = hrtimeDiffToMs(encodeStart);
+
+                    // Send result back to main thread
+                    parentPort.postMessage({
+                        type: 'result',
+                        taskId,
+                        result: encodedResult,
+                        performance: {
+                            transferToWorkerMs,
+                            executionMs,
+                            transferToMainMs
+                        }
+                    });
+
+                } catch (err) {
+                    // Send error back to main thread
+                    parentPort.postMessage({
+                        type: 'error',
+                        taskId,
+                        error: {
+                            message: err.message,
+                            stack: err.stack,
+                            name: err.name
+                        }
+                    });
+                }
+            });
         } else if (type === 'terminate') {
             // Graceful termination requested
             isTerminating = true;
@@ -156,12 +207,13 @@ if (parentPort) {
         }
     });
 
-    // Handle errors
+    // Handle errors - use AsyncLocalStorage to get taskId for proper error attribution
     process.on('uncaughtException', (err) => {
         if (!isTerminating) {
+            const store = taskContext.getStore();
             parentPort.postMessage({
                 type: 'error',
-                taskId: null,
+                taskId: store?.taskId ?? null,
                 error: {
                     message: `Uncaught exception: ${err.message}`,
                     stack: err.stack,
@@ -173,9 +225,10 @@ if (parentPort) {
 
     process.on('unhandledRejection', (reason, _promise) => {
         if (!isTerminating) {
+            const store = taskContext.getStore();
             parentPort.postMessage({
                 type: 'error',
-                taskId: null,
+                taskId: store?.taskId ?? null,
                 error: {
                     message: `Unhandled rejection: ${reason}`,
                     stack: reason?.stack || '',
@@ -185,8 +238,18 @@ if (parentPort) {
         }
     });
 
-    // Signal ready
+    // Signal ready (include any module loading failures)
     parentPort.postMessage({
-        type: 'ready'
+        type: 'ready',
+        failedModules: failedModules.length > 0 ? failedModules : undefined
     });
 }
+
+initializeWorker().catch((err) => {
+    if (parentPort) {
+        parentPort.postMessage({
+            type: 'ready',
+            failedModules: [{ module: '(init)', var: null, error: err.message }]
+        });
+    }
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rosepetal/node-red-contrib-async-function",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "description": "A Node-RED function node that runs code in worker threads to keep your flows responsive",
   "repository": {
     "type": "git",