@rosepetal/node-red-contrib-async-function 1.0.2 → 1.0.3

package/nodes/lib/shared-memory-manager.js

@@ -7,7 +7,7 @@
  */
 
 const fs = require('fs').promises;
-const fsSync = require('fs');
+const { constants: fsConstants } = require('fs');
 const path = require('path');
 const os = require('os');
 const crypto = require('crypto');
@@ -22,7 +22,7 @@ class SharedMemoryManager {
      */
     constructor(options = {}) {
         this.threshold = options.threshold ?? 100 * 1024; // 100KB default
-        this.shmPath = this.detectShmPath();
+        this.shmPath = options.shmPath || null;
         this.trackAttachments = options.trackAttachments !== false;
         this.taskAttachments = new Map(); // taskId → Set<filePath>
         this.globalAttachments = new Set(); // All active files
@@ -32,24 +32,26 @@ class SharedMemoryManager {
             filesCreated: 0,
             filesDeleted: 0
         };
+        this.initialized = false;
 
-        // Cleanup orphaned files from previous crashes
-        if (options.cleanupOrphanedFiles !== false) {
-            this.cleanupOrphanedFiles();
-        }
+        this.ready = this.initialize(options).catch((err) => {
+            this.shmPath = os.tmpdir();
+            this.initialized = true;
+            console.warn(`Failed to initialize shared memory manager: ${err.message}`);
+        });
     }
 
     /**
      * Detect platform-specific shared memory path
      * @returns {string} Path to shared memory directory
      */
-    detectShmPath() {
+    async detectShmPath() {
         // Linux: /dev/shm (RAM-backed tmpfs)
         const shmPath = '/dev/shm';
 
         try {
             // Check if /dev/shm exists and is writable
-            fsSync.accessSync(shmPath, fsSync.constants.W_OK);
+            await fs.access(shmPath, fsConstants.W_OK);
             return shmPath;
         } catch (err) {
             // Fall back to os.tmpdir() (macOS, Windows, or Linux without /dev/shm)
@@ -57,6 +59,35 @@ class SharedMemoryManager {
         }
     }
 
+    /**
+     * Initialize shared memory manager
+     * @param {object} options - Configuration options
+     * @returns {Promise<void>}
+     */
+    async initialize(options = {}) {
+        if (this.initialized) {
+            return;
+        }
+
+        if (this.shmPath) {
+            try {
+                await fs.access(this.shmPath, fsConstants.W_OK);
+            } catch (_err) {
+                this.shmPath = await this.detectShmPath();
+            }
+        } else {
+            this.shmPath = await this.detectShmPath();
+        }
+        this.initialized = true;
+
+        // Cleanup orphaned files from previous crashes (async, non-blocking)
+        if (options.cleanupOrphanedFiles !== false) {
+            this.cleanupOrphanedFiles().catch((err) => {
+                console.warn(`Failed to cleanup orphaned files: ${err.message}`);
+            });
+        }
+    }
+
     /**
      * Generate unique collision-resistant filename
      * @param {number|string} taskId - Task identifier
@@ -80,6 +111,7 @@ class SharedMemoryManager {
      * @returns {Promise<object>} Descriptor object
      */
     async writeBuffer(buffer, taskId, bufferIndex) {
+        await this.ready;
         // Check if buffer exceeds threshold
         if (this.threshold > 0 && buffer.length <= this.threshold) {
             // Return inline buffer (no shared memory needed)
@@ -127,6 +159,7 @@ class SharedMemoryManager {
      * @returns {Promise<Buffer>} Buffer contents
      */
     async readBuffer(descriptor, options = {}) {
+        await this.ready;
         // Validate descriptor
         if (!descriptor || typeof descriptor !== 'object') {
             throw new Error('Invalid descriptor: must be an object');
@@ -234,6 +267,7 @@ class SharedMemoryManager {
      * @returns {Promise<void>}
      */
     async cleanupAll() {
+        await this.ready;
         // Delete all tracked files
         const deletePromises = [];
         for (const filePath of this.globalAttachments) {
@@ -259,11 +293,11 @@ class SharedMemoryManager {
     /**
      * Cleanup orphaned files from previous crashes
      */
-    cleanupOrphanedFiles() {
+    async cleanupOrphanedFiles() {
         const oneHourAgo = Date.now() - (60 * 60 * 1000);
 
         try {
-            const files = fsSync.readdirSync(this.shmPath);
+            const files = await fs.readdir(this.shmPath);
 
             for (const file of files) {
                 // Skip files that don't match our pattern
@@ -274,11 +308,15 @@ class SharedMemoryManager {
                 const filePath = path.join(this.shmPath, file);
 
                 try {
-                    const stats = fsSync.statSync(filePath);
+                    const stats = await fs.stat(filePath);
 
                     // Only cleanup files older than 1 hour
                     if (stats.mtimeMs < oneHourAgo) {
-                        fsSync.unlinkSync(filePath);
+                        await fs.unlink(filePath).catch((err) => {
+                            if (err.code !== 'ENOENT') {
+                                throw err;
+                            }
+                        });
                         console.log(`Cleaned up orphaned file: ${file}`);
                     }
                 } catch (err) {

package/nodes/lib/worker-pool.js

@@ -17,6 +17,7 @@ const DEFAULT_CONFIG = {
     taskTimeout: 30000,         // Default task timeout: 30s
     maxQueueSize: 100,          // Max queued messages
     shmThreshold: 0,            // Always use shared memory for Buffers
+    transferMode: 'transfer',   // transfer | shared | copy
     libs: [],                   // External modules to load in workers
     nodeRedUserDir: null,       // Node-RED user directory for module resolution
     workerScript: path.join(__dirname, 'worker-script.js')
@@ -79,6 +80,7 @@ class WorkerPool {
                 const worker = new Worker(this.config.workerScript, {
                     workerData: {
                         shmThreshold: this.config.shmThreshold,
+                        transferMode: this.config.transferMode,
                         libs: this.config.libs || [],
                         nodeRedUserDir: this.config.nodeRedUserDir
                     }
@@ -262,19 +264,31 @@ class WorkerPool {
         workerState.state = WorkerState.BUSY;
         workerState.taskId = taskId;
 
-        this.serializer.sanitizeMessage(msg, null, taskId).then(sanitizedMsg => {
+        const transferList = this.config.transferMode === 'transfer' ? [] : null;
+        const transferSet = transferList ? new Set() : null;
+
+        this.serializer.sanitizeMessage(msg, null, taskId, {
+            transferMode: this.config.transferMode,
+            transferList,
+            transferSet
+        }).then(sanitizedMsg => {
             // Start timeout after message preparation (matches hot-mode behavior)
             this.timeoutManager.startTimeout(taskId, timeout, () => {
                 this.handleTimeout(workerState, taskId);
             });
 
             // Send task to worker
-            workerState.worker.postMessage({
+            const payload = {
                 type: 'execute',
                 taskId,
                 code,
                 msg: sanitizedMsg
-            });
+            };
+            if (transferList && transferList.length > 0) {
+                workerState.worker.postMessage(payload, transferList);
+            } else {
+                workerState.worker.postMessage(payload);
+            }
         }).catch(err => {
             // Fail task if message prep fails
             const callback = this.callbacks.get(taskId);
@@ -292,7 +306,7 @@ class WorkerPool {
      * @param {object} message - Message from worker
      */
     handleWorkerMessage(workerState, message) {
-        const { type, taskId, result, error, performance } = message;
+        const { type, taskId, result, error, performance, contextUpdates, logs } = message || {};
 
         if (type === 'result') {
             // Task completed successfully
@@ -304,8 +318,18 @@ class WorkerPool {
                 // Recycle worker immediately; result restoration happens asynchronously
                 this.recycleWorker(workerState);
 
-                this.serializer.restoreBuffers(result).then(restoredResult => {
-                    callback(null, { result: restoredResult, performance: performance || null });
+                const restoreResult = this.serializer.restoreBuffers(result);
+                const restoreContext = contextUpdates
+                    ? this.serializer.restoreBuffers(contextUpdates)
+                    : Promise.resolve(contextUpdates);
+
+                Promise.all([restoreResult, restoreContext]).then(([restoredResult, restoredContext]) => {
+                    callback(null, {
+                        result: restoredResult,
+                        performance: performance || null,
+                        contextUpdates: restoredContext || null,
+                        logs: Array.isArray(logs) ? logs : null
+                    });
                 }).catch(restoreErr => {
                     callback(restoreErr instanceof Error ? restoreErr : new Error(String(restoreErr)), null);
                 });
@@ -322,10 +346,23 @@ class WorkerPool {
             const callback = this.callbacks.get(taskId);
             if (callback) {
                 this.callbacks.delete(taskId);
-                const err = new Error(error.message);
-                err.stack = error.stack;
-                err.name = error.name;
-                callback(err, null);
+                const err = new Error(error && error.message ? error.message : 'Worker error');
+                err.stack = error && error.stack ? error.stack : err.stack;
+                err.name = error && error.name ? error.name : err.name;
+
+                if (contextUpdates) {
+                    this.serializer.restoreBuffers(contextUpdates).then(restoredContext => {
+                        err.contextUpdates = restoredContext;
+                        err.logs = Array.isArray(logs) ? logs : null;
+                        callback(err, null);
+                    }).catch(() => {
+                        err.logs = Array.isArray(logs) ? logs : null;
+                        callback(err, null);
+                    });
+                } else {
+                    err.logs = Array.isArray(logs) ? logs : null;
+                    callback(err, null);
+                }
             }
 
             // Return worker to idle state

package/nodes/lib/worker-script.js

@@ -16,6 +16,9 @@ const { AsyncMessageSerializer } = require('./message-serializer');
 
 // Track worker state
 let isTerminating = false;
+const transferMode = workerData && typeof workerData.transferMode === 'string' ? workerData.transferMode : 'transfer';
+const MSG_WRAPPER_KEY = '__rosepetal_msg';
+const CONTEXT_WRAPPER_KEY = '__rosepetal_context';
 
 // AsyncLocalStorage for tracking task context across async boundaries
 // This ensures unhandled rejections can be attributed to the correct task
@@ -77,6 +80,120 @@ const baseRequire = (() => {
 
 global.require = baseRequire;
 
+function createContextProxy(initialData, updates) {
+    const data = (initialData && typeof initialData === 'object') ? initialData : {};
+    const updateMap = updates || {};
+
+    const getValue = (key, fallback) => {
+        if (Object.prototype.hasOwnProperty.call(updateMap, key)) {
+            return updateMap[key];
+        }
+        if (Object.prototype.hasOwnProperty.call(data, key)) {
+            return data[key];
+        }
+        return fallback;
+    };
+
+    const setValue = (key, value) => {
+        data[key] = value;
+        updateMap[key] = value;
+    };
+
+    const api = {
+        get: (key, storeOrCb, cbMaybe) => {
+            let callback = null;
+            if (typeof storeOrCb === 'function') {
+                callback = storeOrCb;
+            } else if (typeof cbMaybe === 'function') {
+                callback = cbMaybe;
+            }
+
+            const value = Array.isArray(key)
+                ? key.map((entry) => getValue(entry, undefined))
+                : getValue(key, undefined);
+
+            if (typeof callback === 'function') {
+                callback(null, value);
+                return undefined;
+            }
+            return value;
+        },
+        set: (key, value, cbMaybe) => {
+            let callback = typeof cbMaybe === 'function' ? cbMaybe : null;
+            let entries = [];
+
+            if (Array.isArray(key)) {
+                if (Array.isArray(value)) {
+                    entries = key.map((entry, index) => [entry, value[index]]);
+                } else if (value && typeof value === 'object') {
+                    entries = key.map((entry) => [entry, value[entry]]);
+                }
+            } else if (key && typeof key === 'object' && value !== null) {
+                entries = Object.entries(key);
+                if (typeof value === 'function') {
+                    callback = value;
+                }
+            } else {
+                entries = [[key, value]];
+            }
+
+            entries.forEach(([entryKey, entryValue]) => {
+                if (entryKey !== undefined) {
+                    setValue(entryKey, entryValue);
+                }
+            });
+
+            if (typeof callback === 'function') {
+                callback(null);
+            }
+            return undefined;
+        }
+    };
+
+    return new Proxy(api, {
+        get(target, prop) {
+            if (typeof prop === 'symbol' || prop in target) {
+                return target[prop];
+            }
+            return getValue(prop, undefined);
+        },
+        set(target, prop, value) {
+            if (typeof prop === 'symbol' || prop in target) {
+                target[prop] = value;
+                return true;
+            }
+            setValue(prop, value);
+            return true;
+        }
+    });
+}
+
+function createNodeProxy(logs) {
+    const push = (level, args) => {
+        if (!Array.isArray(logs)) {
+            return;
+        }
+        if (!args || args.length === 0) {
+            logs.push({ level, message: '' });
+            return;
+        }
+        const message = args.length === 1 ? args[0] : args.map((arg) => arg);
+        logs.push({ level, message });
+    };
+
+    return {
+        warn: (...args) => {
+            push('warn', args);
+        },
+        error: (...args) => {
+            push('error', args);
+        },
+        log: (...args) => {
+            push('log', args);
+        }
+    };
+}
+
 function parseModuleSpec(spec) {
     const match = /((?:@[^/]+\/)?[^/@]+)(\/[^/@]+)?(?:@([\s\S]+))?/.exec(spec);
     if (!match) {
@@ -146,54 +263,133 @@ async function initializeWorker() {
             // This ensures unhandled rejections from fire-and-forget promises
             // can be traced back to the correct task
             taskContext.run({ taskId }, async () => {
+                const contextUpdates = { flow: {}, global: {}, context: {} };
+                const logs = [];
+
                 try {
                     // Restore + execute user code
                     const restoreStart = process.hrtime.bigint();
-                    const restoredMsg = await serializer.restoreBuffers(msg);
+                    const restoredPayload = await serializer.restoreBuffers(msg);
                     const transferToWorkerMs = hrtimeDiffToMs(restoreStart);
 
+                    let contextPayload = {};
+                    let restoredMsg = restoredPayload;
+                    if (restoredPayload && typeof restoredPayload === 'object' && Object.prototype.hasOwnProperty.call(restoredPayload, MSG_WRAPPER_KEY)) {
+                        contextPayload = restoredPayload[CONTEXT_WRAPPER_KEY] || {};
+                        restoredMsg = restoredPayload[MSG_WRAPPER_KEY];
+                    }
+
+                    const nodeProxy = createNodeProxy(logs);
+                    const flowProxy = createContextProxy(contextPayload.flow, contextUpdates.flow);
+                    const globalProxy = createContextProxy(contextPayload.global, contextUpdates.global);
+                    const contextProxy = createContextProxy(contextPayload.context, contextUpdates.context);
+
                     // Cache key includes code + module vars to handle different module configs
                     const cacheKey = code + '|' + moduleVars.join(',');
                     let userFunction = getCachedFunction(cacheKey);
                     if (!userFunction) {
                         // Create function with msg + all module variables as parameters
-                        userFunction = new AsyncFunction('msg', ...moduleVars, code);
+                        userFunction = new AsyncFunction('msg', 'node', 'flow', 'global', 'context', ...moduleVars, code);
                         setCachedFunction(cacheKey, userFunction);
                     }
 
                     const execStart = process.hrtime.bigint();
                     // Execute with msg and all loaded module values
-                    const rawResult = await userFunction(restoredMsg, ...moduleValues);
+                    const rawResult = await userFunction(restoredMsg, nodeProxy, flowProxy, globalProxy, contextProxy, ...moduleValues);
                     const executionMs = hrtimeDiffToMs(execStart);
 
-                    // Offload buffers in the result (large Buffers -> shared memory descriptors)
+                    // Offload buffers in result + context updates
                     const encodeStart = process.hrtime.bigint();
-                    const encodedResult = await serializer.sanitizeMessage(rawResult, null, taskId);
+                    const transferList = transferMode === 'transfer' ? [] : null;
+                    const transferSet = transferList ? new Set() : null;
+                    const bufferCache = new WeakMap();
+
+                    const encodedResult = await serializer.sanitizeMessage(rawResult, null, taskId, {
+                        transferMode,
+                        transferList,
+                        transferSet,
+                        bufferCache
+                    });
+
+                    const hasUpdates = (
+                        Object.keys(contextUpdates.flow).length > 0 ||
+                        Object.keys(contextUpdates.global).length > 0 ||
+                        Object.keys(contextUpdates.context).length > 0
+                    );
+
+                    const encodedContextUpdates = hasUpdates
+                        ? await serializer.sanitizeMessage(contextUpdates, null, taskId, {
+                            transferMode,
+                            transferList,
+                            transferSet,
+                            bufferCache
+                        })
+                        : null;
+
                     const transferToMainMs = hrtimeDiffToMs(encodeStart);
 
                     // Send result back to main thread
-                    parentPort.postMessage({
+                    const payload = {
                         type: 'result',
                         taskId,
                         result: encodedResult,
+                        contextUpdates: encodedContextUpdates,
+                        logs: logs.length > 0 ? logs : null,
                         performance: {
                             transferToWorkerMs,
                             executionMs,
                             transferToMainMs
                         }
-                    });
+                    };
+
+                    if (transferList && transferList.length > 0) {
+                        parentPort.postMessage(payload, transferList);
+                    } else {
+                        parentPort.postMessage(payload);
+                    }
 
                 } catch (err) {
                     // Send error back to main thread
-                    parentPort.postMessage({
+                    const transferList = transferMode === 'transfer' ? [] : null;
+                    const transferSet = transferList ? new Set() : null;
+                    const bufferCache = new WeakMap();
+                    const hasUpdates = (
+                        Object.keys(contextUpdates.flow).length > 0 ||
+                        Object.keys(contextUpdates.global).length > 0 ||
+                        Object.keys(contextUpdates.context).length > 0
+                    );
+
+                    let encodedContextUpdates = null;
+                    if (hasUpdates) {
+                        try {
+                            encodedContextUpdates = await serializer.sanitizeMessage(contextUpdates, null, taskId, {
+                                transferMode,
+                                transferList,
+                                transferSet,
+                                bufferCache
+                            });
+                        } catch (_encodeErr) {
+                            encodedContextUpdates = null;
+                        }
+                    }
+
+                    const payload = {
                         type: 'error',
                         taskId,
                         error: {
                             message: err.message,
                             stack: err.stack,
                             name: err.name
-                        }
-                    });
+                        },
+                        contextUpdates: encodedContextUpdates,
+                        logs: logs.length > 0 ? logs : null
+                    };
+
+                    if (transferList && transferList.length > 0) {
+                        parentPort.postMessage(payload, transferList);
+                    } else {
+                        parentPort.postMessage(payload);
+                    }
                 }
             });
         } else if (type === 'terminate') {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rosepetal/node-red-contrib-async-function",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "description": "A Node-RED function node that runs code in worker threads to keep your flows responsive",
   "repository": {
     "type": "git",