@rosepetal/node-red-contrib-async-function 1.0.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -9
- package/nodes/lib/child-process-pool.js +33 -6
- package/nodes/lib/child-process-script.js +180 -6
- package/nodes/lib/message-serializer.js +171 -16
- package/nodes/lib/module-installer.js +55 -32
- package/nodes/lib/shared-memory-manager.js +50 -12
- package/nodes/lib/worker-pool.js +47 -10
- package/nodes/lib/worker-script.js +207 -11
- package/nodes/{async-function.html → worker-function.html} +15 -13
- package/nodes/{async-function.js → worker-function.js} +326 -33
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -20,7 +20,7 @@ Run heavy computations in Node-RED without slowing down your flows. This node wo
|
|
|
20
20
|
|
|
21
21
|
## How It Works
|
|
22
22
|
|
|
23
|
-
Drop
|
|
23
|
+
Drop a **worker function** node into your flow. Write your code just like you would in a regular function node. The difference? Your code runs in a separate worker thread by default (or a child process if configured), so heavy operations won't freeze Node-RED.
|
|
24
24
|
|
|
25
25
|
## When to Use This
|
|
26
26
|
|
|
@@ -31,7 +31,7 @@ Drop an **async function** node into your flow. Write your code just like you wo
|
|
|
31
31
|
|
|
32
32
|
**Skip It For:**
|
|
33
33
|
- Simple math or quick transformations (the regular function node is faster).
|
|
34
|
-
-
|
|
34
|
+
- Flows that require live context reads/writes during execution (context is snapshot-based).
|
|
35
35
|
|
|
36
36
|
## Node Options
|
|
37
37
|
|
|
@@ -62,11 +62,11 @@ return msg;
|
|
|
62
62
|
```
|
|
63
63
|
|
|
64
64
|
### Buffer Handling
|
|
65
|
-
- **Buffers** –
|
|
65
|
+
- **Buffers** – Worker Threads use zero-copy transfer when possible. Child Process mode and non-transferable buffers fall back to shared memory (`/dev/shm` on Linux, otherwise `os.tmpdir()`), with base64 fallback if needed.
|
|
66
66
|
|
|
67
67
|
## Typical Flow
|
|
68
68
|
|
|
69
|
-
1. Add
|
|
69
|
+
1. Add a **worker function** node to your workspace.
|
|
70
70
|
2. Connect an Inject node (input) and a Debug node (output).
|
|
71
71
|
3. Write a simple script:
|
|
72
72
|
```javascript
|
|
@@ -86,9 +86,11 @@ return msg;
|
|
|
86
86
|
- `console` – Logging functions
|
|
87
87
|
- `setTimeout`, `setInterval` – Timers
|
|
88
88
|
|
|
89
|
-
**
|
|
90
|
-
- `context`, `flow`, `global`
|
|
91
|
-
- `
|
|
89
|
+
**Notes:**
|
|
90
|
+
- `context`, `flow`, `global` are snapshot-based: reads are from the snapshot, writes are applied after the function completes
|
|
91
|
+
- Snapshot includes only literal keys found in `flow.get("key")` / `global.get("key")` / `context.get("key")`
|
|
92
|
+
- Context store selection is not supported (default store only)
|
|
93
|
+
- `node.warn/error/log` are collected and forwarded to the main thread
|
|
92
94
|
- Non-serializable objects (functions, symbols, etc.)
|
|
93
95
|
|
|
94
96
|
## Code Examples
|
|
@@ -170,7 +172,7 @@ The node shows you what's happening in real time:
|
|
|
170
172
|
- Best for operations taking more than 10ms to run.
|
|
171
173
|
- Each node maintains a fixed pool of workers—no startup delay or dynamic scaling overhead.
|
|
172
174
|
- Workers are dedicated per-node, ensuring predictable performance.
|
|
173
|
-
- **Binary Fast Path**:
|
|
175
|
+
- **Binary Fast Path**: Worker threads use zero-copy transfer when possible; shared memory is the fallback.
|
|
174
176
|
- Event loop never blocks, even when processing multi-MB binary data (images, files, etc.).
|
|
175
177
|
|
|
176
178
|
## Error Handling
|
|
@@ -192,7 +194,9 @@ npm install @rosepetal/node-red-contrib-async-function
|
|
|
192
194
|
|
|
193
195
|
Restart Node-RED and find the node in the **function** category.
|
|
194
196
|
|
|
195
|
-
## Migration
|
|
197
|
+
## Migration
|
|
198
|
+
|
|
199
|
+
### From minWorkers/maxWorkers
|
|
196
200
|
|
|
197
201
|
If you're upgrading from a version that used `minWorkers` and `maxWorkers`:
|
|
198
202
|
- Your existing flows will automatically migrate to use the new `numWorkers` parameter
|
|
@@ -17,6 +17,7 @@ const DEFAULT_CONFIG = {
|
|
|
17
17
|
taskTimeout: 30000, // Default task timeout: 30s
|
|
18
18
|
maxQueueSize: 100, // Max queued messages
|
|
19
19
|
shmThreshold: 0, // Always use shared memory for Buffers
|
|
20
|
+
transferMode: 'shared', // shared | copy (transfer not supported for child processes)
|
|
20
21
|
libs: [], // External modules to load in workers
|
|
21
22
|
nodeRedUserDir: null, // Node-RED user directory for module resolution
|
|
22
23
|
workerScript: path.join(__dirname, 'child-process-script.js')
|
|
@@ -166,7 +167,8 @@ class ChildProcessPool {
|
|
|
166
167
|
type: 'init',
|
|
167
168
|
libs: this.config.libs || [],
|
|
168
169
|
nodeRedUserDir: this.config.nodeRedUserDir,
|
|
169
|
-
shmThreshold: this.config.shmThreshold
|
|
170
|
+
shmThreshold: this.config.shmThreshold,
|
|
171
|
+
transferMode: this.config.transferMode
|
|
170
172
|
});
|
|
171
173
|
} catch (err) {
|
|
172
174
|
clearTimeout(readyTimeout);
|
|
@@ -298,7 +300,9 @@ class ChildProcessPool {
|
|
|
298
300
|
workerState.state = WorkerState.BUSY;
|
|
299
301
|
workerState.taskId = taskId;
|
|
300
302
|
|
|
301
|
-
this.serializer.sanitizeMessage(msg, null, taskId
|
|
303
|
+
this.serializer.sanitizeMessage(msg, null, taskId, {
|
|
304
|
+
transferMode: this.config.transferMode
|
|
305
|
+
}).then(sanitizedMsg => {
|
|
302
306
|
this.timeoutManager.startTimeout(taskId, timeout, () => {
|
|
303
307
|
this.handleTimeout(workerState, taskId);
|
|
304
308
|
});
|
|
@@ -329,7 +333,7 @@ class ChildProcessPool {
|
|
|
329
333
|
* @param {object} message - Message from worker
|
|
330
334
|
*/
|
|
331
335
|
handleWorkerMessage(workerState, message) {
|
|
332
|
-
const { type, taskId, result, error, performance } = message || {};
|
|
336
|
+
const { type, taskId, result, error, performance, contextUpdates, logs } = message || {};
|
|
333
337
|
|
|
334
338
|
if (type === 'result') {
|
|
335
339
|
this.timeoutManager.cancelTimeout(taskId);
|
|
@@ -339,8 +343,18 @@ class ChildProcessPool {
|
|
|
339
343
|
this.callbacks.delete(taskId);
|
|
340
344
|
this.recycleWorker(workerState);
|
|
341
345
|
|
|
342
|
-
this.serializer.restoreBuffers(result)
|
|
343
|
-
|
|
346
|
+
const restoreResult = this.serializer.restoreBuffers(result);
|
|
347
|
+
const restoreContext = contextUpdates
|
|
348
|
+
? this.serializer.restoreBuffers(contextUpdates)
|
|
349
|
+
: Promise.resolve(contextUpdates);
|
|
350
|
+
|
|
351
|
+
Promise.all([restoreResult, restoreContext]).then(([restoredResult, restoredContext]) => {
|
|
352
|
+
callback(null, {
|
|
353
|
+
result: restoredResult,
|
|
354
|
+
performance: performance || null,
|
|
355
|
+
contextUpdates: restoredContext || null,
|
|
356
|
+
logs: Array.isArray(logs) ? logs : null
|
|
357
|
+
});
|
|
344
358
|
}).catch(restoreErr => {
|
|
345
359
|
callback(restoreErr instanceof Error ? restoreErr : new Error(String(restoreErr)), null);
|
|
346
360
|
});
|
|
@@ -362,7 +376,20 @@ class ChildProcessPool {
|
|
|
362
376
|
if (error && error.name) {
|
|
363
377
|
err.name = error.name;
|
|
364
378
|
}
|
|
365
|
-
|
|
379
|
+
|
|
380
|
+
if (contextUpdates) {
|
|
381
|
+
this.serializer.restoreBuffers(contextUpdates).then(restoredContext => {
|
|
382
|
+
err.contextUpdates = restoredContext;
|
|
383
|
+
err.logs = Array.isArray(logs) ? logs : null;
|
|
384
|
+
callback(err, null);
|
|
385
|
+
}).catch(() => {
|
|
386
|
+
err.logs = Array.isArray(logs) ? logs : null;
|
|
387
|
+
callback(err, null);
|
|
388
|
+
});
|
|
389
|
+
} else {
|
|
390
|
+
err.logs = Array.isArray(logs) ? logs : null;
|
|
391
|
+
callback(err, null);
|
|
392
|
+
}
|
|
366
393
|
}
|
|
367
394
|
|
|
368
395
|
this.recycleWorker(workerState);
|
|
@@ -16,6 +16,9 @@ const { AsyncMessageSerializer } = require('./message-serializer');
|
|
|
16
16
|
// Track worker state
|
|
17
17
|
let isTerminating = false;
|
|
18
18
|
let isInitialized = false;
|
|
19
|
+
let transferMode = 'shared';
|
|
20
|
+
const MSG_WRAPPER_KEY = '__rosepetal_msg';
|
|
21
|
+
const CONTEXT_WRAPPER_KEY = '__rosepetal_context';
|
|
19
22
|
|
|
20
23
|
// AsyncLocalStorage for tracking task context across async boundaries
|
|
21
24
|
const taskContext = new AsyncLocalStorage();
|
|
@@ -70,6 +73,120 @@ function configureBaseRequire(nodeRedUserDir) {
|
|
|
70
73
|
global.require = baseRequire;
|
|
71
74
|
}
|
|
72
75
|
|
|
76
|
+
function createContextProxy(initialData, updates) {
|
|
77
|
+
const data = (initialData && typeof initialData === 'object') ? initialData : {};
|
|
78
|
+
const updateMap = updates || {};
|
|
79
|
+
|
|
80
|
+
const getValue = (key, fallback) => {
|
|
81
|
+
if (Object.prototype.hasOwnProperty.call(updateMap, key)) {
|
|
82
|
+
return updateMap[key];
|
|
83
|
+
}
|
|
84
|
+
if (Object.prototype.hasOwnProperty.call(data, key)) {
|
|
85
|
+
return data[key];
|
|
86
|
+
}
|
|
87
|
+
return fallback;
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
const setValue = (key, value) => {
|
|
91
|
+
data[key] = value;
|
|
92
|
+
updateMap[key] = value;
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
const api = {
|
|
96
|
+
get: (key, storeOrCb, cbMaybe) => {
|
|
97
|
+
let callback = null;
|
|
98
|
+
if (typeof storeOrCb === 'function') {
|
|
99
|
+
callback = storeOrCb;
|
|
100
|
+
} else if (typeof cbMaybe === 'function') {
|
|
101
|
+
callback = cbMaybe;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const value = Array.isArray(key)
|
|
105
|
+
? key.map((entry) => getValue(entry, undefined))
|
|
106
|
+
: getValue(key, undefined);
|
|
107
|
+
|
|
108
|
+
if (typeof callback === 'function') {
|
|
109
|
+
callback(null, value);
|
|
110
|
+
return undefined;
|
|
111
|
+
}
|
|
112
|
+
return value;
|
|
113
|
+
},
|
|
114
|
+
set: (key, value, cbMaybe) => {
|
|
115
|
+
let callback = typeof cbMaybe === 'function' ? cbMaybe : null;
|
|
116
|
+
let entries = [];
|
|
117
|
+
|
|
118
|
+
if (Array.isArray(key)) {
|
|
119
|
+
if (Array.isArray(value)) {
|
|
120
|
+
entries = key.map((entry, index) => [entry, value[index]]);
|
|
121
|
+
} else if (value && typeof value === 'object') {
|
|
122
|
+
entries = key.map((entry) => [entry, value[entry]]);
|
|
123
|
+
}
|
|
124
|
+
} else if (key && typeof key === 'object' && value !== null) {
|
|
125
|
+
entries = Object.entries(key);
|
|
126
|
+
if (typeof value === 'function') {
|
|
127
|
+
callback = value;
|
|
128
|
+
}
|
|
129
|
+
} else {
|
|
130
|
+
entries = [[key, value]];
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
entries.forEach(([entryKey, entryValue]) => {
|
|
134
|
+
if (entryKey !== undefined) {
|
|
135
|
+
setValue(entryKey, entryValue);
|
|
136
|
+
}
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
if (typeof callback === 'function') {
|
|
140
|
+
callback(null);
|
|
141
|
+
}
|
|
142
|
+
return undefined;
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
return new Proxy(api, {
|
|
147
|
+
get(target, prop) {
|
|
148
|
+
if (typeof prop === 'symbol' || prop in target) {
|
|
149
|
+
return target[prop];
|
|
150
|
+
}
|
|
151
|
+
return getValue(prop, undefined);
|
|
152
|
+
},
|
|
153
|
+
set(target, prop, value) {
|
|
154
|
+
if (typeof prop === 'symbol' || prop in target) {
|
|
155
|
+
target[prop] = value;
|
|
156
|
+
return true;
|
|
157
|
+
}
|
|
158
|
+
setValue(prop, value);
|
|
159
|
+
return true;
|
|
160
|
+
}
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function createNodeProxy(logs) {
|
|
165
|
+
const push = (level, args) => {
|
|
166
|
+
if (!Array.isArray(logs)) {
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
if (!args || args.length === 0) {
|
|
170
|
+
logs.push({ level, message: '' });
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
const message = args.length === 1 ? args[0] : args.map((arg) => arg);
|
|
174
|
+
logs.push({ level, message });
|
|
175
|
+
};
|
|
176
|
+
|
|
177
|
+
return {
|
|
178
|
+
warn: (...args) => {
|
|
179
|
+
push('warn', args);
|
|
180
|
+
},
|
|
181
|
+
error: (...args) => {
|
|
182
|
+
push('error', args);
|
|
183
|
+
},
|
|
184
|
+
log: (...args) => {
|
|
185
|
+
push('log', args);
|
|
186
|
+
}
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
|
|
73
190
|
function parseModuleSpec(spec) {
|
|
74
191
|
const match = /((?:@[^/]+\/)?[^/@]+)(\/[^/@]+)?(?:@([\s\S]+))?/.exec(spec);
|
|
75
192
|
if (!match) {
|
|
@@ -109,7 +226,7 @@ async function loadConfiguredModules(libs) {
|
|
|
109
226
|
moduleVars.push(lib.var);
|
|
110
227
|
moduleValues.push(loadedModules[lib.var]);
|
|
111
228
|
} catch (err) {
|
|
112
|
-
console.error(`[
|
|
229
|
+
console.error(`[worker-function] Failed to load module ${lib.module}: ${err.message}`);
|
|
113
230
|
failedModules.push({ module: lib.module, var: lib.var, error: err.message });
|
|
114
231
|
}
|
|
115
232
|
}
|
|
@@ -139,6 +256,9 @@ async function initializeWorker(initData) {
|
|
|
139
256
|
configureBaseRequire(nodeRedUserDir);
|
|
140
257
|
|
|
141
258
|
const threshold = initData && typeof initData.shmThreshold === 'number' ? initData.shmThreshold : undefined;
|
|
259
|
+
if (initData && typeof initData.transferMode === 'string') {
|
|
260
|
+
transferMode = initData.transferMode;
|
|
261
|
+
}
|
|
142
262
|
shmManager = new SharedMemoryManager({
|
|
143
263
|
threshold,
|
|
144
264
|
trackAttachments: false,
|
|
@@ -188,30 +308,59 @@ process.on('message', async (data) => {
|
|
|
188
308
|
|
|
189
309
|
if (type === 'execute') {
|
|
190
310
|
taskContext.run({ taskId }, async () => {
|
|
311
|
+
const contextUpdates = { flow: {}, global: {}, context: {} };
|
|
312
|
+
const logs = [];
|
|
313
|
+
|
|
191
314
|
try {
|
|
192
315
|
const restoreStart = process.hrtime.bigint();
|
|
193
|
-
const
|
|
316
|
+
const restoredPayload = await serializer.restoreBuffers(msg);
|
|
194
317
|
const transferToWorkerMs = hrtimeDiffToMs(restoreStart);
|
|
195
318
|
|
|
319
|
+
let contextPayload = {};
|
|
320
|
+
let restoredMsg = restoredPayload;
|
|
321
|
+
if (restoredPayload && typeof restoredPayload === 'object' && Object.prototype.hasOwnProperty.call(restoredPayload, MSG_WRAPPER_KEY)) {
|
|
322
|
+
contextPayload = restoredPayload[CONTEXT_WRAPPER_KEY] || {};
|
|
323
|
+
restoredMsg = restoredPayload[MSG_WRAPPER_KEY];
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
const nodeProxy = createNodeProxy(logs);
|
|
327
|
+
const flowProxy = createContextProxy(contextPayload.flow, contextUpdates.flow);
|
|
328
|
+
const globalProxy = createContextProxy(contextPayload.global, contextUpdates.global);
|
|
329
|
+
const contextProxy = createContextProxy(contextPayload.context, contextUpdates.context);
|
|
330
|
+
|
|
196
331
|
const cacheKey = code + '|' + moduleVars.join(',');
|
|
197
332
|
let userFunction = getCachedFunction(cacheKey);
|
|
198
333
|
if (!userFunction) {
|
|
199
|
-
userFunction = new AsyncFunction('msg', ...moduleVars, code);
|
|
334
|
+
userFunction = new AsyncFunction('msg', 'node', 'flow', 'global', 'context', ...moduleVars, code);
|
|
200
335
|
setCachedFunction(cacheKey, userFunction);
|
|
201
336
|
}
|
|
202
337
|
|
|
203
338
|
const execStart = process.hrtime.bigint();
|
|
204
|
-
const rawResult = await userFunction(restoredMsg, ...moduleValues);
|
|
339
|
+
const rawResult = await userFunction(restoredMsg, nodeProxy, flowProxy, globalProxy, contextProxy, ...moduleValues);
|
|
205
340
|
const executionMs = hrtimeDiffToMs(execStart);
|
|
206
341
|
|
|
207
342
|
const encodeStart = process.hrtime.bigint();
|
|
208
|
-
const
|
|
343
|
+
const bufferCache = new WeakMap();
|
|
344
|
+
const encodedResult = await serializer.sanitizeMessage(rawResult, null, taskId, {
|
|
345
|
+
transferMode,
|
|
346
|
+
bufferCache
|
|
347
|
+
});
|
|
348
|
+
const hasUpdates = (
|
|
349
|
+
Object.keys(contextUpdates.flow).length > 0 ||
|
|
350
|
+
Object.keys(contextUpdates.global).length > 0 ||
|
|
351
|
+
Object.keys(contextUpdates.context).length > 0
|
|
352
|
+
);
|
|
353
|
+
const encodedContextUpdates = hasUpdates
|
|
354
|
+
? await serializer.sanitizeMessage(contextUpdates, null, taskId, { transferMode, bufferCache })
|
|
355
|
+
: null;
|
|
209
356
|
const transferToMainMs = hrtimeDiffToMs(encodeStart);
|
|
210
357
|
|
|
211
358
|
sendMessage({
|
|
212
359
|
type: 'result',
|
|
213
360
|
taskId,
|
|
214
361
|
result: encodedResult,
|
|
362
|
+
contextUpdates: encodedContextUpdates,
|
|
363
|
+
logs: logs.length > 0 ? logs : null,
|
|
215
364
|
performance: {
|
|
216
365
|
transferToWorkerMs,
|
|
217
366
|
executionMs,
|
|
@@ -219,7 +368,32 @@ process.on('message', async (data) => {
|
|
|
219
368
|
}
|
|
220
369
|
});
|
|
221
370
|
} catch (err) {
|
|
222
|
-
|
|
371
|
+
let encodedContextUpdates = null;
|
|
372
|
+
const hasUpdates = (
|
|
373
|
+
Object.keys(contextUpdates.flow).length > 0 ||
|
|
374
|
+
Object.keys(contextUpdates.global).length > 0 ||
|
|
375
|
+
Object.keys(contextUpdates.context).length > 0
|
|
376
|
+
);
|
|
377
|
+
if (hasUpdates) {
|
|
378
|
+
try {
|
|
379
|
+
encodedContextUpdates = await serializer.sanitizeMessage(contextUpdates, null, taskId, { transferMode });
|
|
380
|
+
} catch (_encodeErr) {
|
|
381
|
+
encodedContextUpdates = null;
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
386
|
+
sendMessage({
|
|
387
|
+
type: 'error',
|
|
388
|
+
taskId,
|
|
389
|
+
error: {
|
|
390
|
+
message: error.message,
|
|
391
|
+
stack: error.stack || '',
|
|
392
|
+
name: error.name || 'Error'
|
|
393
|
+
},
|
|
394
|
+
contextUpdates: encodedContextUpdates,
|
|
395
|
+
logs: logs.length > 0 ? logs : null
|
|
396
|
+
});
|
|
223
397
|
}
|
|
224
398
|
});
|
|
225
399
|
} else if (type === 'terminate') {
|
|
@@ -10,6 +10,50 @@
|
|
|
10
10
|
|
|
11
11
|
const SHARED_SENTINEL_KEY = '__rosepetal_shm_path__';
|
|
12
12
|
const SHARED_BASE64_KEY = '__rosepetal_base64__';
|
|
13
|
+
const TRANSFER_SENTINEL_KEY = '__rosepetal_transfer_ab__';
|
|
14
|
+
|
|
15
|
+
function canTransferBuffer(value) {
|
|
16
|
+
if (!Buffer.isBuffer(value)) {
|
|
17
|
+
return false;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
if (!value.buffer || !(value.buffer instanceof ArrayBuffer)) {
|
|
21
|
+
return false;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// Avoid detaching pooled/sliced buffers that share backing stores
|
|
25
|
+
if (value.byteOffset !== 0) {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
if (value.byteLength !== value.buffer.byteLength) {
|
|
30
|
+
return false;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
return true;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function createTransferDescriptor(value, transferList, transferSet) {
|
|
37
|
+
if (!transferList) {
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
if (!canTransferBuffer(value)) {
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const arrayBuffer = value.buffer;
|
|
46
|
+
if (transferSet && !transferSet.has(arrayBuffer)) {
|
|
47
|
+
transferSet.add(arrayBuffer);
|
|
48
|
+
transferList.push(arrayBuffer);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
return {
|
|
52
|
+
[TRANSFER_SENTINEL_KEY]: arrayBuffer,
|
|
53
|
+
byteOffset: value.byteOffset,
|
|
54
|
+
byteLength: value.byteLength
|
|
55
|
+
};
|
|
56
|
+
}
|
|
13
57
|
|
|
14
58
|
/**
|
|
15
59
|
* Async Message Serializer Class
|
|
@@ -31,18 +75,29 @@ class AsyncMessageSerializer {
|
|
|
31
75
|
* @param {object} msg - Message object to sanitize
|
|
32
76
|
* @param {object} node - Node-RED node instance (for warnings)
|
|
33
77
|
* @param {number|string} taskId - Task identifier for shared memory tracking
|
|
78
|
+
* @param {object} options - Optional serialization options
|
|
34
79
|
* @returns {Promise<object>} Sanitized message object
|
|
35
80
|
*/
|
|
36
|
-
async sanitizeMessage(msg, node, taskId) {
|
|
81
|
+
async sanitizeMessage(msg, node, taskId, options = {}) {
|
|
37
82
|
if (!msg || typeof msg !== 'object') {
|
|
38
83
|
return msg;
|
|
39
84
|
}
|
|
40
85
|
|
|
41
86
|
const seen = new WeakMap();
|
|
42
87
|
const bufferIndex = { value: 0 }; // Mutable counter for buffer indexing
|
|
88
|
+
const transferMode = typeof options.transferMode === 'string' ? options.transferMode : 'shared';
|
|
89
|
+
const transferList = Array.isArray(options.transferList) ? options.transferList : null;
|
|
90
|
+
const transferSet = options.transferSet || (transferList ? new Set() : null);
|
|
91
|
+
const bufferCache = options.bufferCache || new WeakMap();
|
|
92
|
+
const cloneOptions = {
|
|
93
|
+
transferMode,
|
|
94
|
+
transferList,
|
|
95
|
+
transferSet,
|
|
96
|
+
bufferCache
|
|
97
|
+
};
|
|
43
98
|
|
|
44
99
|
try {
|
|
45
|
-
return await this.cloneValue(msg, seen, bufferIndex, taskId, 0);
|
|
100
|
+
return await this.cloneValue(msg, seen, bufferIndex, taskId, 0, cloneOptions);
|
|
46
101
|
} catch (err) {
|
|
47
102
|
if (node) {
|
|
48
103
|
node.warn(`Message cloning failed: ${err.message}, creating minimal message`);
|
|
@@ -58,11 +113,10 @@ class AsyncMessageSerializer {
|
|
|
58
113
|
* @param {number} depth - Current recursion depth
|
|
59
114
|
* @param {object} bufferIndex - Mutable buffer index counter
|
|
60
115
|
* @param {number|string} taskId - Task identifier
|
|
61
|
-
* @param {object}
|
|
62
|
-
* @param {string} path - Current property path (for warnings)
|
|
116
|
+
* @param {object} options - Serialization options
|
|
63
117
|
* @returns {Promise<*>} Cloned value
|
|
64
118
|
*/
|
|
65
|
-
async cloneValue(value, seen, bufferIndex, taskId, depth) {
|
|
119
|
+
async cloneValue(value, seen, bufferIndex, taskId, depth, options) {
|
|
66
120
|
// Optional depth guard (disabled by default for speed)
|
|
67
121
|
if (this.maxDepth > 0 && depth > this.maxDepth) {
|
|
68
122
|
return null;
|
|
@@ -83,14 +137,74 @@ class AsyncMessageSerializer {
|
|
|
83
137
|
return undefined;
|
|
84
138
|
}
|
|
85
139
|
|
|
86
|
-
// Buffers + typed arrays:
|
|
140
|
+
// Buffers + typed arrays: transfer, inline copy, or shared memory
|
|
87
141
|
if (Buffer.isBuffer(value)) {
|
|
88
|
-
|
|
142
|
+
if (options && options.bufferCache && options.bufferCache.has(value)) {
|
|
143
|
+
return options.bufferCache.get(value);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
if (options && options.transferMode === 'copy') {
|
|
147
|
+
if (options.bufferCache) {
|
|
148
|
+
options.bufferCache.set(value, value);
|
|
149
|
+
}
|
|
150
|
+
return value;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
if (options && options.transferMode === 'transfer' && options.transferList) {
|
|
154
|
+
const transferDescriptor = createTransferDescriptor(value, options.transferList, options.transferSet);
|
|
155
|
+
if (transferDescriptor) {
|
|
156
|
+
if (options.bufferCache) {
|
|
157
|
+
options.bufferCache.set(value, transferDescriptor);
|
|
158
|
+
}
|
|
159
|
+
return transferDescriptor;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
if (options.bufferCache) {
|
|
163
|
+
options.bufferCache.set(value, value);
|
|
164
|
+
}
|
|
165
|
+
return value;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const sharedDescriptor = await this.shmManager.writeBuffer(value, taskId, bufferIndex.value++);
|
|
169
|
+
if (options && options.bufferCache) {
|
|
170
|
+
options.bufferCache.set(value, sharedDescriptor);
|
|
171
|
+
}
|
|
172
|
+
return sharedDescriptor;
|
|
89
173
|
}
|
|
90
174
|
|
|
91
175
|
if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
|
|
92
176
|
const asBuffer = Buffer.from(value.buffer, value.byteOffset, value.byteLength);
|
|
93
|
-
|
|
177
|
+
if (options && options.bufferCache && options.bufferCache.has(asBuffer)) {
|
|
178
|
+
return options.bufferCache.get(asBuffer);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if (options && options.transferMode === 'copy') {
|
|
182
|
+
if (options.bufferCache) {
|
|
183
|
+
options.bufferCache.set(asBuffer, asBuffer);
|
|
184
|
+
}
|
|
185
|
+
return asBuffer;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
if (options && options.transferMode === 'transfer' && options.transferList) {
|
|
189
|
+
const transferDescriptor = createTransferDescriptor(asBuffer, options.transferList, options.transferSet);
|
|
190
|
+
if (transferDescriptor) {
|
|
191
|
+
if (options.bufferCache) {
|
|
192
|
+
options.bufferCache.set(asBuffer, transferDescriptor);
|
|
193
|
+
}
|
|
194
|
+
return transferDescriptor;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
if (options.bufferCache) {
|
|
198
|
+
options.bufferCache.set(asBuffer, asBuffer);
|
|
199
|
+
}
|
|
200
|
+
return asBuffer;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
const sharedDescriptor = await this.shmManager.writeBuffer(asBuffer, taskId, bufferIndex.value++);
|
|
204
|
+
if (options && options.bufferCache) {
|
|
205
|
+
options.bufferCache.set(asBuffer, sharedDescriptor);
|
|
206
|
+
}
|
|
207
|
+
return sharedDescriptor;
|
|
94
208
|
}
|
|
95
209
|
|
|
96
210
|
if (type === 'object') {
|
|
@@ -104,7 +218,7 @@ class AsyncMessageSerializer {
|
|
|
104
218
|
|
|
105
219
|
if (Array.isArray(value)) {
|
|
106
220
|
for (let i = 0; i < value.length; i++) {
|
|
107
|
-
const clonedItem = await this.cloneValue(value[i], seen, bufferIndex, taskId, depth + 1);
|
|
221
|
+
const clonedItem = await this.cloneValue(value[i], seen, bufferIndex, taskId, depth + 1, options);
|
|
108
222
|
clone[i] = clonedItem === undefined ? null : clonedItem;
|
|
109
223
|
}
|
|
110
224
|
return clone;
|
|
@@ -113,7 +227,7 @@ class AsyncMessageSerializer {
|
|
|
113
227
|
const keys = Object.keys(value);
|
|
114
228
|
for (let i = 0; i < keys.length; i++) {
|
|
115
229
|
const key = keys[i];
|
|
116
|
-
const clonedVal = await this.cloneValue(value[key], seen, bufferIndex, taskId, depth + 1);
|
|
230
|
+
const clonedVal = await this.cloneValue(value[key], seen, bufferIndex, taskId, depth + 1, options);
|
|
117
231
|
if (clonedVal !== undefined) {
|
|
118
232
|
clone[key] = clonedVal;
|
|
119
233
|
}
|
|
@@ -132,10 +246,11 @@ class AsyncMessageSerializer {
|
|
|
132
246
|
*/
|
|
133
247
|
async restoreBuffers(value) {
|
|
134
248
|
const seen = new WeakMap();
|
|
135
|
-
|
|
249
|
+
const descriptorCache = new WeakMap();
|
|
250
|
+
return this.restoreValue(value, seen, descriptorCache);
|
|
136
251
|
}
|
|
137
252
|
|
|
138
|
-
async restoreValue(value, seen) {
|
|
253
|
+
async restoreValue(value, seen, descriptorCache) {
|
|
139
254
|
// Handle null/undefined
|
|
140
255
|
if (value === null || value === undefined) {
|
|
141
256
|
return value;
|
|
@@ -162,7 +277,7 @@ class AsyncMessageSerializer {
|
|
|
162
277
|
seen.set(value, result);
|
|
163
278
|
|
|
164
279
|
for (let i = 0; i < value.length; i++) {
|
|
165
|
-
result[i] = await this.restoreValue(value[i], seen);
|
|
280
|
+
result[i] = await this.restoreValue(value[i], seen, descriptorCache);
|
|
166
281
|
}
|
|
167
282
|
|
|
168
283
|
return result;
|
|
@@ -170,10 +285,42 @@ class AsyncMessageSerializer {
|
|
|
170
285
|
|
|
171
286
|
// Handle objects
|
|
172
287
|
if (type === 'object') {
|
|
288
|
+
// Transfer-list descriptor
|
|
289
|
+
if (Object.prototype.hasOwnProperty.call(value, TRANSFER_SENTINEL_KEY)) {
|
|
290
|
+
if (descriptorCache && descriptorCache.has(value)) {
|
|
291
|
+
return descriptorCache.get(value);
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
try {
|
|
295
|
+
const arrayBuffer = value[TRANSFER_SENTINEL_KEY];
|
|
296
|
+
if (!(arrayBuffer instanceof ArrayBuffer)) {
|
|
297
|
+
return Buffer.alloc(0);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
const byteOffset = Number(value.byteOffset) || 0;
|
|
301
|
+
const byteLength = Number(value.byteLength) || arrayBuffer.byteLength;
|
|
302
|
+
const buffer = Buffer.from(arrayBuffer, byteOffset, byteLength);
|
|
303
|
+
if (descriptorCache) {
|
|
304
|
+
descriptorCache.set(value, buffer);
|
|
305
|
+
}
|
|
306
|
+
return buffer;
|
|
307
|
+
} catch (_err) {
|
|
308
|
+
return Buffer.alloc(0);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
173
312
|
// Shared memory descriptor
|
|
174
313
|
if (Object.prototype.hasOwnProperty.call(value, SHARED_SENTINEL_KEY)) {
|
|
314
|
+
if (descriptorCache && descriptorCache.has(value)) {
|
|
315
|
+
return descriptorCache.get(value);
|
|
316
|
+
}
|
|
317
|
+
|
|
175
318
|
try {
|
|
176
|
-
|
|
319
|
+
const buffer = await this.shmManager.readBuffer(value, { deleteAfterRead: true });
|
|
320
|
+
if (descriptorCache) {
|
|
321
|
+
descriptorCache.set(value, buffer);
|
|
322
|
+
}
|
|
323
|
+
return buffer;
|
|
177
324
|
} catch (_err) {
|
|
178
325
|
return Buffer.alloc(0);
|
|
179
326
|
}
|
|
@@ -181,8 +328,16 @@ class AsyncMessageSerializer {
|
|
|
181
328
|
|
|
182
329
|
// Base64 fallback
|
|
183
330
|
if (Object.prototype.hasOwnProperty.call(value, SHARED_BASE64_KEY)) {
|
|
331
|
+
if (descriptorCache && descriptorCache.has(value)) {
|
|
332
|
+
return descriptorCache.get(value);
|
|
333
|
+
}
|
|
334
|
+
|
|
184
335
|
try {
|
|
185
|
-
|
|
336
|
+
const buffer = Buffer.from(value[SHARED_BASE64_KEY] || '', 'base64');
|
|
337
|
+
if (descriptorCache) {
|
|
338
|
+
descriptorCache.set(value, buffer);
|
|
339
|
+
}
|
|
340
|
+
return buffer;
|
|
186
341
|
} catch (_err) {
|
|
187
342
|
return Buffer.alloc(0);
|
|
188
343
|
}
|
|
@@ -198,7 +353,7 @@ class AsyncMessageSerializer {
|
|
|
198
353
|
const entries = Object.entries(value);
|
|
199
354
|
for (let i = 0; i < entries.length; i++) {
|
|
200
355
|
const [key, val] = entries[i];
|
|
201
|
-
result[key] = await this.restoreValue(val, seen);
|
|
356
|
+
result[key] = await this.restoreValue(val, seen, descriptorCache);
|
|
202
357
|
}
|
|
203
358
|
|
|
204
359
|
return result;
|