@inteli.city/node-red-contrib-exec-collection 1.0.3 → 1.0.5

package/python.queue.js

@@ -0,0 +1,555 @@
+'use strict';
+
+// ── requires ──────────────────────────────────────────────────────────────────
+
+const nunjucks  = require('nunjucks');
+const { spawn } = require('child_process');
+const yaml       = require('js-yaml');
+const convertXML = require('xml-js');
+
+// ── LOCAL TEMPLATE CONTEXT ────────────────────────────────────────────────────
+// Raw context: objects are NOT pre-stringified. Instead, each value is wrapped
+// in a Proxy that serialises itself only when Nunjucks coerces it to a string
+// (i.e. at output time). This means:
+//   {{ payload }}     → JSON.stringify(payload)    (calls toString / toPrimitive)
+//   {{ payload.a }}   → the value of a (string/number as-is, object → JSON)
+//   {{ payload.a.b }} → the value of b
+// No "[object Object]" is ever produced.
+
+const _SKIP_KEYS = ['req', 'res', '_req', '_res'];
+
+function _safeStringify(obj) {
+    const seen = new WeakSet();
+    try {
+        return JSON.stringify(obj, (key, value) => {
+            if (typeof value === 'object' && value !== null) {
+                if (seen.has(value)) return '[Circular]';
+                seen.add(value);
+            }
+            return value;
+        });
+    } catch (e) {
+        return '[Unserializable]';
+    }
+}
+
+function _wrapValue(value) {
+    if (typeof value !== 'object' || value === null) return value;
+    return new Proxy(value, {
+        get(target, prop) {
+            if (prop === Symbol.toPrimitive) return () => _safeStringify(target);
+            if (prop === 'toString')         return () => _safeStringify(target);
+            const result = Reflect.get(target, prop);
+            if (typeof result === 'object' && result !== null) return _wrapValue(result);
+            return result;
+        }
+    });
+}
+
+function buildRawContext(msg, node) {
+    const ctx = {};
+    for (const k of Object.keys(msg || {})) {
+        if (_SKIP_KEYS.includes(k)) continue;
+        ctx[k] = _wrapValue(msg[k]);
+    }
+    ctx.flow   = { get: (key) => node.context().flow.get(key) };
+    ctx.global = { get: (key) => node.context().global.get(key) };
+    ctx.env    = process.env;
+    return ctx;
+}
+
+// ── BOOTSTRAP ─────────────────────────────────────────────────────────────────
+// Injected into each Python worker via -c.
+// Reads one JSON line per execution: { "code": "...python..." }
+// During execution stdout is intercepted; each write() emits a frame:
+//   { "t": "o", "v": "...chunk..." }  — output chunk
+// After execution:
+//   { "t": "d" }                       — done (success)
+//   { "t": "e", "v": "...traceback..." } — error
+// Each worker has its own isolated namespace — no shared state between workers.
+
+const PYTHON_BOOTSTRAP = `\
+import sys as _sys, json as _json, traceback as _tb
+
+class _Out(object):
+    def __init__(self, r): self._r = r; self._buf = ''
+    def write(self, s):
+        if not s: return
+        self._buf += s
+        while '\\n' in self._buf:
+            nl = self._buf.index('\\n')
+            chunk = self._buf[:nl+1]
+            self._buf = self._buf[nl+1:]
+            self._r.write(_json.dumps({'t':'o','v':chunk}) + '\\n')
+            self._r.flush()
+    def flush(self):
+        if self._buf:
+            self._r.write(_json.dumps({'t':'o','v':self._buf}) + '\\n')
+            self._r.flush()
+            self._buf = ''
+
+_real = _sys.stdout
+_ns = {}
+while True:
+    _line = _sys.stdin.readline()
+    if not _line: break
+    _line = _line.rstrip('\\n')
+    if not _line: continue
+    _out = _Out(_real)
+    _sys.stdout = _out
+    try:
+        _d = _json.loads(_line)
+        exec(_d['code'], _ns)
+        _out.flush()
+        _real.write(_json.dumps({'t':'d'}) + '\\n')
+        _real.flush()
+    except Exception:
+        _out.flush()
+        _real.write(_json.dumps({'t':'e','v':_tb.format_exc()}) + '\\n')
+        _real.flush()
+    finally:
+        _sys.stdout = _real
+`;
+
+// ── SECTION 1: TEMPLATE RESOLUTION ───────────────────────────────────────────
+
+function selectTemplate(msg, node) {
+    if (msg.hasOwnProperty('template') && (node.template === '' || node.template === null)) {
+        return msg.template;
+    }
+    return node.template;
+}
+
+
+// ── SECTION 2: OUTPUT PARSING ─────────────────────────────────────────────────
+
+// Pure: parses rawValue per outputFormat.
+// Returns { value } on success, { error: string } on failure.
+function parseOutputValue(rawValue, outputFormat) {
+    if (outputFormat === 'parsedJSON') {
+        try {
+            const parsed = JSON.parse(rawValue);
+            if (typeof parsed === 'number') {
+                return { error: 'Error parsing JSON: result is a plain number' };
+            }
+            return { value: parsed };
+        } catch (err) {
+            return { error: 'Error parsing JSON:\n\n' + err };
+        }
+    }
+
+    if (outputFormat === 'parsedXML') {
+        try {
+            return { value: convertXML.xml2js(rawValue, { compact: true, spaces: 4 }) };
+        } catch (err) {
+            return { error: 'Error parsing XML:\n\n' + err };
+        }
+    }
+
+    if (outputFormat === 'parsedYAML') {
+        try {
+            const parsed = yaml.load(rawValue);
+            if (typeof parsed === 'number') {
+                return { error: 'Error parsing YAML: result is a plain number' };
+            }
+            return { value: parsed };
+        } catch (err) {
+            return { error: 'Error parsing YAML:\n\n' + err };
+        }
+    }
+
+    return { value: rawValue };
+}
+
+
+// ── SECTION 3: NODE DEFINITION ────────────────────────────────────────────────
+
+module.exports = function(RED) {
+
+    function PythonQueueNode(n) {
+        RED.nodes.createNode(this, n);
+
+        this.name         = n.name;
+        this.template     = n.template || '';
+        this.outputFormat = n.output   || 'str';
+        this.field        = n.field    || 'payload';
+        this.fieldType    = n.fieldType || 'msg';
+        this.queue        = Math.max(1, Number(n.queue) || 1);
+        this.pythonConfig = RED.nodes.getNode(n.pythonConfig);
+        this.streamMode   = n.streamMode || 'delimited';
+
+        // Interpret escape sequences in the stored delimiter string: \n → newline, \t → tab, \r → CR
+        const delimRaw  = (n.delimiter !== undefined && n.delimiter !== '') ? n.delimiter : '\\n';
+        this.delimiter  = delimRaw.replace(/\\n/g, '\n').replace(/\\t/g, '\t').replace(/\\r/g, '\r');
+
+        const node       = this;
+        const pythonPath = (node.pythonConfig && node.pythonConfig.pythonPath) || 'python3';
+
+        // ── worker pool ───────────────────────────────────────────────────────
+        // Each worker: { proc, busy, currentItem, lineBuffer }
+        let workers      = [];
+        let pendingQueue = [];
+        let closing      = false;
+
+        // ── status state ──────────────────────────────────────────────────────
+        let currentState     = { waiting: 0, executing: 0, workers: 0 };
+        let lastEmittedState = null;
+        let statusTimer      = null;
+        let lastEmitTime     = 0;
+
+        const STATUS_INTERVAL_MS = 1000;
+
+        // ── idle timeout ──────────────────────────────────────────────────────
+        const IDLE_TIMEOUT_MS = 20 * 60 * 1000; // 20 minutes
+        let idleTimer = null;
+
+        // ── output helpers ────────────────────────────────────────────────────
+
+        // Emit one segment: parse it and call send(). Never calls done().
+        function emitSegment(item, value) {
+            const result = parseOutputValue(value, node.outputFormat);
+            if (result.error) {
+                node.error(result.error);
+                return;
+            }
+            if (node.fieldType === 'msg') {
+                const outMsg = Object.assign({}, item.msg);
+                RED.util.setMessageProperty(outMsg, node.field, result.value);
+                item.send(outMsg);
+            } else {
+                const context = RED.util.parseContextStore(node.field);
+                const target  = node.context()[node.fieldType];
+                const outMsg  = Object.assign({}, item.msg);
+                target.set(context.key, result.value, context.store, (err) => {
+                    if (err) node.error(err);
+                    else item.send(outMsg);
+                });
+            }
+        }
+
+        // Route an output chunk: raw → accumulate for single emit at done;
+        // delimited → buffer and split on delimiter.
+        function handleOutputChunk(item, chunk) {
+            if (node.streamMode === 'raw') {
+                item.rawBuffer += chunk;
+                return;
+            }
+            item.segmentBuffer += chunk;
+            const parts = item.segmentBuffer.split(node.delimiter);
+            item.segmentBuffer = parts.pop(); // keep incomplete remainder
+            for (const part of parts) {
+                if (part) emitSegment(item, part);
+            }
+        }
+
+        // ── stream frame handler ──────────────────────────────────────────────
+
+        // Handles one JSON line from the worker's stdout.
+        // Frames: { t:'o', v:chunk } | { t:'d' } | { t:'e', v:traceback }
+        function handleStreamFrame(worker, line) {
+            let frame;
+            try {
+                frame = JSON.parse(line);
+            } catch (e) {
+                node.error(`JSON parse error from Python: ${e.message}\nRaw: ${line}`);
+                if (worker.currentItem) {
+                    const item = worker.currentItem;
+                    worker.currentItem = null;
+                    worker.busy = false;
+                    item.done(e);
+                    resetIdleTimer();
+                    processNext();
+                    computeAndStoreState();
+                }
+                return;
+            }
+
+            const item = worker.currentItem;
+            if (!item) return; // killed — ignore late frames silently
+
+            if (frame.t === 'o') {
+                handleOutputChunk(item, frame.v);
+                return;
+            }
+
+            // 'd' or 'e' — job is finished
+            worker.currentItem = null;
+            worker.busy = false;
+
+            if (frame.t === 'e') {
+                node.error(`Python error:\n${frame.v}`);
+                item.done(new Error(frame.v));
+            } else {
+                // Flush buffered content before signalling done
+                if (node.streamMode === 'delimited' && item.segmentBuffer) {
+                    emitSegment(item, item.segmentBuffer);
+                    item.segmentBuffer = '';
+                }
+                if (node.streamMode === 'raw' && item.rawBuffer) {
+                    emitSegment(item, item.rawBuffer);
+                    item.rawBuffer = '';
+                }
+                item.done();
+            }
+
+            resetIdleTimer();
+            processNext();
+            computeAndStoreState();
+        }
+
+        // ── worker management ─────────────────────────────────────────────────
+
+        function createWorker() {
+            const proc = spawn(pythonPath, ['-u', '-c', PYTHON_BOOTSTRAP], {
+                stdio: ['pipe', 'pipe', 'pipe']
+            });
+
+            const worker = { proc, busy: false, currentItem: null, lineBuffer: '' };
+
+            proc.stdout.on('data', (data) => {
+                worker.lineBuffer += data.toString();
+                const lines = worker.lineBuffer.split('\n');
+                worker.lineBuffer = lines.pop();
+                for (const line of lines) {
+                    if (line.trim()) handleStreamFrame(worker, line);
+                }
+            });
+
+            proc.stderr.on('data', (data) => {
+                const text = Buffer.isBuffer(data) ? data.toString() : String(data);
+                if (text && text.trim().length > 0) node.warn(text.trim());
+            });
+
+            proc.on('error', (err) => {
+                if (closing) return;
+                const errMsg = (err && err.message) ? err.message : String(err);
+                node.error(`Python worker error: ${errMsg}`);
+                workers = workers.filter(w => w !== worker);
+                if (worker.currentItem) {
+                    worker.currentItem.done(new Error(errMsg));
+                    worker.currentItem = null;
+                }
+                worker.busy = false;
+                processNext();
+                computeAndStoreState();
+            });
+
+            proc.on('close', (code) => {
+                if (closing) return;
+                workers = workers.filter(w => w !== worker);
+                if (worker.currentItem) {
+                    node.warn(`Python worker exited (code ${code}) — failing current job`);
+                    worker.currentItem.done(new Error(`Python worker exited with code ${code}`));
+                    worker.currentItem = null;
+                }
+                worker.busy = false;
+                processNext();
+                computeAndStoreState();
+            });
+
+            return worker;
+        }
+
+        function ensureWorkers() {
+            while (workers.length < node.queue) {
+                workers.push(createWorker());
+            }
+        }
+
+        function stopAllWorkers() {
+            for (const w of workers) {
+                try { w.proc.kill('SIGTERM'); } catch (_) {}
+            }
+            workers = [];
+        }
+
+        // ── idle timer ────────────────────────────────────────────────────────
+
+        function resetIdleTimer() {
+            if (idleTimer) clearTimeout(idleTimer);
+            idleTimer = setTimeout(() => {
+                idleTimer = null;
+                const anyBusy = workers.some(w => w.busy);
+                if (!anyBusy && pendingQueue.length === 0 && workers.length > 0) {
+                    node.log('Idle timeout — stopping all Python workers');
+                    stopAllWorkers();
+                    computeAndStoreState();
+                }
+            }, IDLE_TIMEOUT_MS);
+        }
+
+        // ── scheduler ─────────────────────────────────────────────────────────
+
+        function processNext() {
+            if (pendingQueue.length === 0) return;
+
+            ensureWorkers();
+
+            const worker = workers.find(w => !w.busy);
+            if (!worker) return;
+
+            const item = pendingQueue.shift();
+            worker.busy        = true;
+            worker.currentItem = item;
+
+            try {
+                worker.proc.stdin.write(JSON.stringify({ code: item.rendered }) + '\n');
+            } catch (e) {
+                node.error(`Failed to write to Python stdin: ${e.message}`);
+                worker.busy        = false;
+                worker.currentItem = null;
+                item.done(e);
+            }
+
+            processNext(); // fill next free worker immediately
+        }
+
+        // ── status ────────────────────────────────────────────────────────────
+
+        function computeState() {
+            return {
+                waiting:   pendingQueue.length,
+                executing: workers.filter(w => w.busy).length,
+                workers:   workers.length
+            };
+        }
+
+        function computeStatusFromState(state) {
+            return {
+                fill:  state.workers > 0 ? 'blue' : 'grey',
+                shape: state.executing > 0 ? 'ring' : 'dot',
+                text:  `${state.waiting} (${state.executing}/${node.queue})`
+            };
+        }
+
+        function emitStatus() {
+            lastEmitTime = Date.now();
+            if (
+                lastEmittedState &&
+                currentState.waiting   === lastEmittedState.waiting &&
+                currentState.executing === lastEmittedState.executing &&
+                currentState.workers   === lastEmittedState.workers
+            ) {
+                return;
+            }
+            lastEmittedState = { ...currentState };
+            node.status(computeStatusFromState(currentState));
+        }
+
+        function scheduleStatusEmit() {
+            const now = Date.now();
+            const idle = currentState.waiting === 0 && currentState.executing === 0;
+            if (idle) {
+                if (statusTimer) { clearTimeout(statusTimer); statusTimer = null; }
+                emitStatus();
+                return;
+            }
+            if (now - lastEmitTime >= STATUS_INTERVAL_MS) {
+                emitStatus();
+                return;
+            }
+            if (!statusTimer) {
+                const delay = STATUS_INTERVAL_MS - (now - lastEmitTime);
+                statusTimer = setTimeout(() => {
+                    statusTimer = null;
+                    emitStatus();
+                }, delay);
+            }
+        }
+
+        function computeAndStoreState() {
+            currentState = computeState();
+            node.currentState = currentState;
+            scheduleStatusEmit();
+        }
+
+        // ── killAll ───────────────────────────────────────────────────────────
+        // Used by: msg.stop, close handler, HTTP admin endpoint.
+
+        function killAll() {
+            for (const w of workers) {
+                if (w.currentItem) {
+                    w.currentItem.done();
+                    w.currentItem = null;
+                }
+                try { w.proc.kill('SIGTERM'); } catch (_) {}
+            }
+            workers = [];
+            for (const item of pendingQueue) {
+                item.done();
+            }
+            pendingQueue = [];
+            currentState = { waiting: 0, executing: 0, workers: 0 };
+            node.currentState = currentState;
+            node.status({ fill: 'grey', shape: 'dot', text: `0 (0/${node.queue})` });
+        }
+
+        node.killAll = killAll;
+
+        // ── input handler ─────────────────────────────────────────────────────
+
+        node.on('input', function(msg, send, done) {
+            if (msg.stop === true) {
+                killAll();
+                done();
+                return;
+            }
+
+            const template = selectTemplate(msg, node);
+            const context  = buildRawContext(msg, node);
+
+            let rendered;
+            try {
+                const env = new nunjucks.Environment(null, { autoescape: false });
+                rendered = env.renderString(template, context);
+            } catch (e) {
+                node.error('Template render error: ' + e.message + '\nTip: wrap variables in quotes — print("{{ payload }}")');
+                done(e);
+                return;
+            }
+
+            pendingQueue.push({ msg, send, done, rendered, segmentBuffer: '', rawBuffer: '' });
+            resetIdleTimer();
+            processNext();
+            computeAndStoreState();
+        });
+
+        // ── close handler ─────────────────────────────────────────────────────
+
+        node.on('close', function(done) {
+            closing = true;
+            if (idleTimer)   { clearTimeout(idleTimer);   idleTimer   = null; }
+            if (statusTimer) { clearTimeout(statusTimer); statusTimer = null; }
+            killAll();
+            done();
+        });
+
+        // ── start ─────────────────────────────────────────────────────────────
+        // Lazy: workers start on first message, not at deploy time.
+        computeAndStoreState();
+    }
+
+    RED.nodes.registerType('python.queue', PythonQueueNode);
+
+    RED.httpAdmin.post('/python-queue/:id/kill', function(req, res) {
+        const n = RED.nodes.getNode(req.params.id);
+        if (n && typeof n.killAll === 'function') {
+            n.killAll();
+            res.sendStatus(200);
+        } else {
+            res.sendStatus(404);
+        }
+    });
+
+    RED.httpAdmin.get('/python-queue/:id/status', function(req, res) {
+        const n = RED.nodes.getNode(req.params.id);
+        if (n) {
+            res.json({
+                executing: n.currentState ? n.currentState.executing : 0,
+                workers:   n.currentState ? n.currentState.workers   : 0
+            });
+        } else {
+            res.sendStatus(404);
+        }
+    });
+};

package/utils/context.js

@@ -0,0 +1,54 @@
+'use strict';
+
+const SKIP_KEYS = ['req', 'res', '_req', '_res'];
+
+function safeStringify(obj) {
+    const seen = new WeakSet();
+    try {
+        return JSON.stringify(obj, (key, value) => {
+            if (typeof value === 'object' && value !== null) {
+                if (seen.has(value)) return '[Circular]';
+                seen.add(value);
+            }
+            return value;
+        });
+    } catch (e) {
+        return '[Unserializable Object]';
+    }
+}
+
+// Builds the Nunjucks rendering context from msg and node.
+// All msg values are coerced to strings so that {{ payload }} is always safe
+// to embed anywhere in the template without quotes or filters:
+//   string         → kept as-is
+//   number         → String(v)
+//   object         → safeStringify(v) (circular-safe, capped at 10 000 chars)
+//   null/undefined → ""
+// Keys in SKIP_KEYS (req, res, etc.) are omitted to avoid serializing
+// non-serializable Node-RED runtime objects.
+function buildContext(msg, node) {
+    const ctx = {};
+
+    for (const k of Object.keys(msg || {})) {
+        if (SKIP_KEYS.includes(k)) continue;
+
+        const v = msg[k];
+
+        if (v === null || v === undefined) {
+            ctx[k] = '';
+        } else if (typeof v === 'object') {
+            const str = safeStringify(v);
+            ctx[k] = str.length > 10000 ? '[Object too large]' : str;
+        } else {
+            ctx[k] = String(v);
+        }
+    }
+
+    ctx.flow   = { get: (key) => node.context().flow.get(key) };
+    ctx.global = { get: (key) => node.context().global.get(key) };
+    ctx.env    = process.env;
+
+    return ctx;
+}
+
+module.exports = { buildContext };