@inteli.city/node-red-contrib-exec-collection 1.0.4 → 1.0.5

package/node.queue.js

@@ -0,0 +1,569 @@
+'use strict';
+
+// ── requires ──────────────────────────────────────────────────────────────────
+
+const nunjucks  = require('nunjucks');
+const { spawn } = require('child_process');
+const yaml       = require('js-yaml');
+const convertXML = require('xml-js');
+
+// ── LOCAL TEMPLATE CONTEXT ────────────────────────────────────────────────────
+// Raw context: objects are NOT pre-stringified. Instead, each value is wrapped
+// in a Proxy that serialises itself only when Nunjucks coerces it to a string
+// (i.e. at output time). This means:
+//   {{ payload }}     → JSON.stringify(payload)    (calls toString / toPrimitive)
+//   {{ payload.a }}   → the value of a (string/number as-is, object → JSON)
+//   {{ payload.a.b }} → the value of b
+// No "[object Object]" is ever produced.
+
+const _SKIP_KEYS = ['req', 'res', '_req', '_res'];
+
+function _safeStringify(obj) {
+    const seen = new WeakSet();
+    try {
+        return JSON.stringify(obj, (key, value) => {
+            if (typeof value === 'object' && value !== null) {
+                if (seen.has(value)) return '[Circular]';
+                seen.add(value);
+            }
+            return value;
+        });
+    } catch (e) {
+        return '[Unserializable]';
+    }
+}
+
+function _wrapValue(value) {
+    if (typeof value !== 'object' || value === null) return value;
+    return new Proxy(value, {
+        get(target, prop) {
+            if (prop === Symbol.toPrimitive) return () => _safeStringify(target);
+            if (prop === 'toString')         return () => _safeStringify(target);
+            const result = Reflect.get(target, prop);
+            if (typeof result === 'object' && result !== null) return _wrapValue(result);
+            return result;
+        }
+    });
+}
+
+function buildRawContext(msg, node) {
+    const ctx = {};
+    for (const k of Object.keys(msg || {})) {
+        if (_SKIP_KEYS.includes(k)) continue;
+        ctx[k] = _wrapValue(msg[k]);
+    }
+    ctx.flow   = { get: (key) => node.context().flow.get(key) };
+    ctx.global = { get: (key) => node.context().global.get(key) };
+    ctx.env    = process.env;
+    return ctx;
+}
+
+// ── BOOTSTRAP ─────────────────────────────────────────────────────────────────
+// Injected into each Node.js worker via node -e.
+// Reads one JSON line per execution: { "code": "...js..." }
+// Runs the code in a persistent vm context. Each console.log() immediately emits:
+//   { "t": "o", "v": "...line\n" }  — output chunk
+// After execution:
+//   { "t": "d" }                     — done (success)
+//   { "t": "e", "v": "...stack..." } — error
+// Each worker has its own isolated context — no shared state between workers.
+
+const NODE_BOOTSTRAP = `
+'use strict';
+const vm       = require('vm');
+const readline = require('readline');
+
+const ctx = vm.createContext({
+    require,
+    global:  {},
+    process,
+    console: {
+        log:   (...a) => {
+            const v = a.map(x => typeof x === 'string' ? x : JSON.stringify(x)).join(' ');
+            process.stdout.write(JSON.stringify({ t: 'o', v: v + '\\n' }) + '\\n');
+        },
+        info:  (...a) => {
+            const v = a.map(x => typeof x === 'string' ? x : JSON.stringify(x)).join(' ');
+            process.stdout.write(JSON.stringify({ t: 'o', v: v + '\\n' }) + '\\n');
+        },
+        warn:  (...a) => process.stderr.write(a.join(' ') + '\\n'),
+        error: (...a) => process.stderr.write(a.join(' ') + '\\n'),
+    },
+    setTimeout,
+    clearTimeout,
+    setInterval,
+    clearInterval,
+    Buffer,
+});
+
+const rl = readline.createInterface({ input: process.stdin, crlfDelay: Infinity });
+
+rl.on('line', (line) => {
+    line = line.trim();
+    if (!line) return;
+
+    let d;
+    try {
+        d = JSON.parse(line);
+    } catch (e) {
+        process.stdout.write(JSON.stringify({ t: 'e', v: 'JSON parse error: ' + e.message }) + '\\n');
+        return;
+    }
+
+    try {
+        vm.runInContext('(function(){"use strict";' + d.code + '\\n})()', ctx);
+        process.stdout.write(JSON.stringify({ t: 'd' }) + '\\n');
+    } catch (e) {
+        process.stdout.write(JSON.stringify({ t: 'e', v: e.stack || e.message }) + '\\n');
+    }
+});
+`;
+
+// ── SECTION 1: TEMPLATE RESOLUTION ───────────────────────────────────────────
+
+// Returns the effective template: msg.template overrides only when
+// node.template is empty or null.
+function selectTemplate(msg, node) {
+    if (msg.hasOwnProperty('template') && (node.template === '' || node.template === null)) {
+        return msg.template;
+    }
+    return node.template;
+}
+
+
+// ── SECTION 2: OUTPUT PARSING ─────────────────────────────────────────────────
+
+// Pure: parses rawValue per outputFormat.
+// Returns { value } on success, { error: string } on failure.
+function parseOutputValue(rawValue, outputFormat) {
+    if (outputFormat === 'parsedJSON') {
+        try {
+            const parsed = JSON.parse(rawValue);
+            if (typeof parsed === 'number') {
+                return { error: 'Error parsing JSON: result is a plain number' };
+            }
+            return { value: parsed };
+        } catch (err) {
+            return { error: 'Error parsing JSON:\n\n' + err };
+        }
+    }
+
+    if (outputFormat === 'parsedXML') {
+        try {
+            return { value: convertXML.xml2js(rawValue, { compact: true, spaces: 4 }) };
+        } catch (err) {
+            return { error: 'Error parsing XML:\n\n' + err };
+        }
+    }
+
+    if (outputFormat === 'parsedYAML') {
+        try {
+            const parsed = yaml.load(rawValue);
+            if (typeof parsed === 'number') {
+                return { error: 'Error parsing YAML: result is a plain number' };
+            }
+            return { value: parsed };
+        } catch (err) {
+            return { error: 'Error parsing YAML:\n\n' + err };
+        }
+    }
+
+    return { value: rawValue };
+}
+
+
+// ── SECTION 3: NODE DEFINITION ────────────────────────────────────────────────
+
+module.exports = function(RED) {
+
+    function NodeQueueNode(n) {
+        RED.nodes.createNode(this, n);
+
+        this.name         = n.name;
+        this.template     = n.template || '';
+        this.outputFormat = n.output   || 'str';
+        this.field        = n.field    || 'payload';
+        this.fieldType    = n.fieldType || 'msg';
+        this.queue        = Math.max(1, Number(n.queue) || 1);
+        this.streamMode   = n.streamMode || 'delimited';
+
+        // Interpret escape sequences in the stored delimiter string: \n → newline, \t → tab, \r → CR
+        const delimRaw  = (n.delimiter !== undefined && n.delimiter !== '') ? n.delimiter : '\\n';
+        this.delimiter  = delimRaw.replace(/\\n/g, '\n').replace(/\\t/g, '\t').replace(/\\r/g, '\r');
+
+        const node = this;
+
+        // ── worker pool ───────────────────────────────────────────────────────
+        // Each worker: { proc, busy, currentItem, lineBuffer }
+        let workers      = [];
+        let pendingQueue = [];
+        let closing      = false;
+
+        // ── status state ──────────────────────────────────────────────────────
+        let currentState     = { waiting: 0, executing: 0, workers: 0 };
+        let lastEmittedState = null;
+        let statusTimer      = null;
+        let lastEmitTime     = 0;
+
+        const STATUS_INTERVAL_MS = 1000;
+
+        // ── idle timeout ──────────────────────────────────────────────────────
+        const IDLE_TIMEOUT_MS = 20 * 60 * 1000; // 20 minutes
+        let idleTimer = null;
+
+        // ── output helpers ────────────────────────────────────────────────────
+
+        // Emit one segment: parse it and call send(). Never calls done().
+        function emitSegment(item, value) {
+            const result = parseOutputValue(value, node.outputFormat);
+            if (result.error) {
+                node.error(result.error);
+                return;
+            }
+            if (node.fieldType === 'msg') {
+                const outMsg = Object.assign({}, item.msg);
+                RED.util.setMessageProperty(outMsg, node.field, result.value);
+                item.send(outMsg);
+            } else {
+                const context = RED.util.parseContextStore(node.field);
+                const target  = node.context()[node.fieldType];
+                const outMsg  = Object.assign({}, item.msg);
+                target.set(context.key, result.value, context.store, (err) => {
+                    if (err) node.error(err);
+                    else item.send(outMsg);
+                });
+            }
+        }
+
+        // Route an output chunk: raw → accumulate for single emit at done;
+        // delimited → buffer and split on delimiter.
+        function handleOutputChunk(item, chunk) {
+            if (node.streamMode === 'raw') {
+                item.rawBuffer += chunk;
+                return;
+            }
+            item.segmentBuffer += chunk;
+            const parts = item.segmentBuffer.split(node.delimiter);
+            item.segmentBuffer = parts.pop(); // keep incomplete remainder
+            for (const part of parts) {
+                if (part) emitSegment(item, part);
+            }
+        }
+
+        // ── stream frame handler ──────────────────────────────────────────────
+
+        // Handles one JSON line from the worker's stdout.
+        // Frames: { t:'o', v:chunk } | { t:'d' } | { t:'e', v:stack }
+        function handleStreamFrame(worker, line) {
+            let frame;
+            try {
+                frame = JSON.parse(line);
+            } catch (e) {
+                node.error(`JSON parse error from Node.js worker: ${e.message}\nRaw: ${line}`);
+                if (worker.currentItem) {
+                    const item = worker.currentItem;
+                    worker.currentItem = null;
+                    worker.busy = false;
+                    item.done(e);
+                    resetIdleTimer();
+                    processNext();
+                    computeAndStoreState();
+                }
+                return;
+            }
+
+            const item = worker.currentItem;
+            if (!item) return; // killed — ignore late frames silently
+
+            if (frame.t === 'o') {
+                handleOutputChunk(item, frame.v);
+                return;
+            }
+
+            // 'd' or 'e' — job is finished
+            worker.currentItem = null;
+            worker.busy = false;
+
+            if (frame.t === 'e') {
+                node.error(`Node.js worker error:\n${frame.v}`);
+                item.done(new Error(frame.v));
+            } else {
+                // Flush buffered content before signalling done
+                if (node.streamMode === 'delimited' && item.segmentBuffer) {
+                    emitSegment(item, item.segmentBuffer);
+                    item.segmentBuffer = '';
+                }
+                if (node.streamMode === 'raw' && item.rawBuffer) {
+                    emitSegment(item, item.rawBuffer);
+                    item.rawBuffer = '';
+                }
+                item.done();
+            }
+
+            resetIdleTimer();
+            processNext();
+            computeAndStoreState();
+        }
+
+        // ── worker management ─────────────────────────────────────────────────
+
+        function createWorker() {
+            const proc = spawn('node', ['-e', NODE_BOOTSTRAP], {
+                stdio: ['pipe', 'pipe', 'pipe']
+            });
+
+            const worker = { proc, busy: false, currentItem: null, lineBuffer: '' };
+
+            proc.stdout.on('data', (data) => {
+                worker.lineBuffer += data.toString();
+                const lines = worker.lineBuffer.split('\n');
+                worker.lineBuffer = lines.pop();
+                for (const line of lines) {
+                    if (line.trim()) handleStreamFrame(worker, line);
+                }
+            });
+
+            proc.stderr.on('data', (data) => {
+                const text = Buffer.isBuffer(data) ? data.toString() : String(data);
+                if (text && text.trim().length > 0) node.warn(text.trim());
+            });
+
+            proc.on('error', (err) => {
+                if (closing) return;
+                const errMsg = (err && err.message) ? err.message : String(err);
+                node.error(`Node.js worker error: ${errMsg}`);
+                workers = workers.filter(w => w !== worker);
+                if (worker.currentItem) {
+                    worker.currentItem.done(new Error(errMsg));
+                    worker.currentItem = null;
+                }
+                worker.busy = false;
+                processNext();
+                computeAndStoreState();
+            });
+
+            proc.on('close', (code) => {
+                if (closing) return;
+                workers = workers.filter(w => w !== worker);
+                if (worker.currentItem) {
+                    node.warn(`Node.js worker exited (code ${code}) — failing current job`);
+                    worker.currentItem.done(new Error(`Node.js worker exited with code ${code}`));
+                    worker.currentItem = null;
+                }
+                worker.busy = false;
+                processNext();
+                computeAndStoreState();
+            });
+
+            return worker;
+        }
+
+        function ensureWorkers() {
+            while (workers.length < node.queue) {
+                workers.push(createWorker());
+            }
+        }
+
+        function stopAllWorkers() {
+            for (const w of workers) {
+                try { w.proc.kill('SIGTERM'); } catch (_) {}
+            }
+            workers = [];
+        }
+
+        // ── idle timer ────────────────────────────────────────────────────────
+
+        function resetIdleTimer() {
+            if (idleTimer) clearTimeout(idleTimer);
+            idleTimer = setTimeout(() => {
+                idleTimer = null;
+                const anyBusy = workers.some(w => w.busy);
+                if (!anyBusy && pendingQueue.length === 0 && workers.length > 0) {
+                    node.log('Idle timeout — stopping all Node.js workers');
+                    stopAllWorkers();
+                    computeAndStoreState();
+                }
+            }, IDLE_TIMEOUT_MS);
+        }
+
+        // ── scheduler ─────────────────────────────────────────────────────────
+
+        function processNext() {
+            if (pendingQueue.length === 0) return;
+
+            ensureWorkers();
+
+            const worker = workers.find(w => !w.busy);
+            if (!worker) return;
+
+            const item = pendingQueue.shift();
+            worker.busy        = true;
+            worker.currentItem = item;
+
+            try {
+                worker.proc.stdin.write(JSON.stringify({ code: item.rendered }) + '\n');
+            } catch (e) {
+                node.error(`Failed to write to Node.js worker stdin: ${e.message}`);
+                worker.busy        = false;
+                worker.currentItem = null;
+                item.done(e);
+            }
+
+            processNext(); // fill next free worker immediately
+        }
+
+        // ── status ────────────────────────────────────────────────────────────
+
+        function computeState() {
+            return {
+                waiting:   pendingQueue.length,
+                executing: workers.filter(w => w.busy).length,
+                workers:   workers.length
+            };
+        }
+
+        function computeStatusFromState(state) {
+            return {
+                fill:  state.workers > 0 ? 'blue' : 'grey',
+                shape: state.executing > 0 ? 'ring' : 'dot',
+                text:  `${state.waiting} (${state.executing}/${node.queue})`
+            };
+        }
+
+        function emitStatus() {
+            lastEmitTime = Date.now();
+            if (
+                lastEmittedState &&
+                currentState.waiting   === lastEmittedState.waiting &&
+                currentState.executing === lastEmittedState.executing &&
+                currentState.workers   === lastEmittedState.workers
+            ) {
+                return;
+            }
+            lastEmittedState = { ...currentState };
+            node.status(computeStatusFromState(currentState));
+        }
+
+        function scheduleStatusEmit() {
+            const now = Date.now();
+            const idle = currentState.waiting === 0 && currentState.executing === 0;
+            if (idle) {
+                if (statusTimer) { clearTimeout(statusTimer); statusTimer = null; }
+                emitStatus();
+                return;
+            }
+            if (now - lastEmitTime >= STATUS_INTERVAL_MS) {
+                emitStatus();
+                return;
+            }
+            if (!statusTimer) {
+                const delay = STATUS_INTERVAL_MS - (now - lastEmitTime);
+                statusTimer = setTimeout(() => {
+                    statusTimer = null;
+                    emitStatus();
+                }, delay);
+            }
+        }
+
+        function computeAndStoreState() {
+            currentState = computeState();
+            node.currentState = currentState;
+            scheduleStatusEmit();
+        }
+
+        // ── killAll ───────────────────────────────────────────────────────────
+        // Used by: msg.stop, close handler, HTTP admin endpoint.
+
+        function killAll() {
+            for (const w of workers) {
+                if (w.currentItem) {
+                    w.currentItem.done();
+                    w.currentItem = null;
+                }
+                try { w.proc.kill('SIGTERM'); } catch (_) {}
+            }
+            workers = [];
+            for (const item of pendingQueue) {
+                item.done();
+            }
+            pendingQueue = [];
+            currentState = { waiting: 0, executing: 0, workers: 0 };
+            node.currentState = currentState;
+            node.status({ fill: 'grey', shape: 'dot', text: `0 (0/${node.queue})` });
+        }
+
+        node.killAll = killAll;
+
+        // ── input handler ─────────────────────────────────────────────────────
+
+        node.on('input', function(msg, send, done) {
+            if (msg.stop === true) {
+                killAll();
+                done();
+                return;
+            }
+
+            const template = selectTemplate(msg, node);
+            const context  = buildRawContext(msg, node);
+
+            let rendered;
+            try {
+                const env = new nunjucks.Environment(null, { autoescape: false });
+                const stripped = template
+                    .replace(/\/\/.*$/gm, '')
+                    .replace(/\/\*[\s\S]*?\*\//g, '');
+                rendered = env.renderString(stripped, context);
+            } catch (e) {
+                node.error(
+                    'Template render error: ' + e.message +
+                    '\nTip: Do not use {{ }} inside comments. Nunjucks parses the entire template.'
+                );
+                done(e);
+                return;
+            }
+
+            pendingQueue.push({ msg, send, done, rendered, segmentBuffer: '', rawBuffer: '' });
+            resetIdleTimer();
+            processNext();
+            computeAndStoreState();
+        });
+
+        // ── close handler ─────────────────────────────────────────────────────
+
+        node.on('close', function(done) {
+            closing = true;
+            if (idleTimer)   { clearTimeout(idleTimer);   idleTimer   = null; }
+            if (statusTimer) { clearTimeout(statusTimer); statusTimer = null; }
+            killAll();
+            done();
+        });
+
+        // ── start ─────────────────────────────────────────────────────────────
+        // Lazy: workers start on first message, not at deploy time.
+        computeAndStoreState();
+    }
+
+    RED.nodes.registerType('node.queue', NodeQueueNode);
+
+    RED.httpAdmin.post('/node-queue/:id/kill', function(req, res) {
+        const n = RED.nodes.getNode(req.params.id);
+        if (n && typeof n.killAll === 'function') {
+            n.killAll();
+            res.sendStatus(200);
+        } else {
+            res.sendStatus(404);
+        }
+    });
+
+    RED.httpAdmin.get('/node-queue/:id/status', function(req, res) {
+        const n = RED.nodes.getNode(req.params.id);
+        if (n) {
+            res.json({
+                executing: n.currentState ? n.currentState.executing : 0,
+                workers:   n.currentState ? n.currentState.workers   : 0
+            });
+        } else {
+            res.sendStatus(404);
+        }
+    });
+};

package/package.json

@@ -1,29 +1,29 @@
 {
     "name" : "@inteli.city/node-red-contrib-exec-collection",
-	"version": "1.0.4",
-	"dependencies": {
-		"mustache": "*",
-		"fs": "*",
+    "version": "1.0.5",
+    "dependencies": {
+        "fs": "*",
         "tmp-promise": "*",
-		"js-yaml": "*",
-		"xml-js": "*",
-		"child_process": "*",
-		"queue": "^6.0.2",
+        "js-yaml": "*",
+        "xml-js": "*",
+        "child_process": "*",
+        "queue": "^6.0.2",
         "nunjucks": "^3.2.4",
-		"os": "*",
-		"terminate": "*",
-		"pm2": "*",
+        "os": "*",
+        "terminate": "*",
         "pg-promise": "*"
-	},
-	"keywords": [ "node-red" ],
-	"license": "Apache-2.0",
+    },
+    "keywords": [ "node-red" ],
+    "license": "Apache-2.0",
     "node-red" : {
-		"version": ">=2.0.0",
+        "version": ">=2.0.0",
         "nodes": {
-            "exec queue":   "exec.queue.js",
-            "async pg":  "async.pg.js",
-            "template njk":  "template.njk.js",
-            "module njk":  "module.njk.js"
+            "exec queue":    "exec.queue.js",
+            "exec service":  "exec.service.js",
+            "python queue":  "python.queue.js",
+            "python config": "python.config.js",
+            "node queue":    "node.queue.js",
+            "async pg":      "async.pg.js"
         }
     }
 }

package/python.config.html

@@ -0,0 +1,55 @@
+<script type="text/markdown" data-help-name="python.config">
+## python.config
+
+Defines the Python executable used by python.queue.
+
+The path must point to a valid Python binary. This can be:
+- a system Python (e.g. `/usr/bin/python3`)
+- a virtual environment Python (e.g. `/path/to/venv/bin/python`)
+
+This node does not create or manage environments.
+The environment must already exist and be fully configured.
+
+---
+
+[Full documentation](https://www.npmjs.com/package/@inteli.city/node-red-contrib-exec-collection)
+</script>
+
+<script type="text/html" data-template-name="python.config">
+    <div class="form-row">
+        <label for="node-config-input-name">
+            <i class="fa fa-tag"></i>
+            <span data-i18n="node-red:common.label.name"></span>
+        </label>
+        <input type="text" id="node-config-input-name" data-i18n="[placeholder]node-red:common.label.name">
+    </div>
+
+    <div class="form-row">
+        <label for="node-config-input-pythonPath">
+            <i class="fa fa-terminal"></i>
+            <span>Python Path</span>
+        </label>
+        <input type="text" id="node-config-input-pythonPath" placeholder="/usr/bin/python3">
+    </div>
+
+    <div class="form-row" style="margin-top:-4px;">
+        <div style="margin-left:105px; font-size:11px; color:#888; line-height:1.5em;">
+            This can be a system Python or a virtual environment (venv) executable.<br>
+            <strong>Examples:</strong> <span style="color:#c0392b;">/usr/bin/python3</span> &nbsp;&middot;&nbsp; <span style="color:#c0392b;">/home/user/myenv/bin/python</span><br>
+            <em>Note: The venv must already exist and have all dependencies installed.</em>
+        </div>
+    </div>
+</script>
+
+<script type="text/javascript">
+    RED.nodes.registerType('python.config', {
+        category: 'config',
+        defaults: {
+            name:       { value: '' },
+            pythonPath: { value: '', required: true }
+        },
+        label: function() {
+            return this.name || this.pythonPath || 'python.config';
+        }
+    });
+</script>