@bldgblocks/node-red-contrib-control 0.1.33 → 0.1.36

package/nodes/history-buffer.js

@@ -0,0 +1,461 @@
+module.exports = function(RED) {
+    const fs = require('fs');
+    const path = require('path');
+    const readline = require('readline');
+    const utils = require('./utils')(RED);
+
+    function HistoryBufferNode(config) {
+        RED.nodes.createNode(this, config);
+        const node = this;
+
+        // Configuration
+        node.name = config.name;
+        node.bufferHours = parseFloat(config.bufferHours) || 3;
+
+        // Validate configuration
+        if (isNaN(node.bufferHours) || node.bufferHours < 0.01 || node.bufferHours > 24) {
+            node.bufferHours = 3;
+        }
+
+        // Constants
+        const TRENDS_DIR = getTrendsDirPath();
+        // Use node ID in filename to prevent collisions between multiple history nodes
+        const BUFFER_FILE = path.join(TRENDS_DIR, `buffer_${node.id}.json`);
+        // Legacy file path for migration
+        const LEGACY_BUFFER_FILE = path.join(TRENDS_DIR, 'buffer_current.json');
+        
+        const COMMIT_INTERVAL_MS = 30 * 1000;  // 30 seconds
+        const PRUNE_INTERVAL_MS = 60 * 1000;   // 60 seconds
+        const STARTUP_DELAY_MS = 5000;         // 5 seconds before loading history
+        const MAX_BUFFER_AGE_MS = node.bufferHours * 3600 * 1000;
+
+        // State
+        let liveBuffer = [];                    // Accumulate points since last commit to BUFFER_FILE
+        let commitTimer = null;
+        let pruneTimer = null;
+        let messageCount = 0;
+        let cachedChunkCount = 0;               // Cached count of historical files
+        let isInitializing = true;              // Flag: initialization in progress
+        let queuedMessages = [];                // Queue messages during initialization
+        
+        // Status throttling
+        let lastStatusUpdate = 0;
+
+        utils.setStatusOK(node, `ready, buffer: ${node.bufferHours}h`);
+
+        // =====================================================================
+        // Directory Helper
+        // =====================================================================
+        function getTrendsDirPath() {
+            const userDir = RED.settings.userDir || 
+                           process.env.NODE_RED_HOME ||
+                           path.join(require('os').homedir(), '.node-red');
+            return path.join(userDir, '.bldgblocks', 'trends');
+        }
+
+        // =====================================================================
+        // Initialize from files on startup (Async/Streaming)
+        // =====================================================================
+        async function initializeFromFiles() {
+            // 1. Ensure directory exists
+            try {
+                await fs.promises.mkdir(TRENDS_DIR, { recursive: true });
+            } catch (err) {
+                node.error(`Failed to create trends directory: ${err.message}`);
+                return;
+            }
+
+            // 2. Rotate/Migrate buffers
+            // Only rotate if they are "stale" (from a previous hour), otherwise append to them.
+            await rotateStartupBuffers();
+
+            // 3. Find and filter valid trend files
+            const historicalFiles = await getHistoricalFiles();
+
+            const validFiles = historicalFiles.filter(fileName => {
+                // Filename format: trend_TIMESTAMP_NODEID.json
+                // We split by '_' and take index 1.
+                const parts = fileName.split('_');
+                const timestamp = parseInt(parts[1]);
+                if (isNaN(timestamp)) return false;
+
+                // Simple check: is the file from within our retention window?
+                // Timestamp in filename is the time of rotation (end of that file's period)
+                const ageMs = Date.now() - (timestamp * 1000);
+                return ageMs <= MAX_BUFFER_AGE_MS + (3600 * 1000); // Add 1h grace period
+            });
+
+            updateStatus(`loading ${validFiles.length} files...`, true);
+
+            // 4. Stream load all data into memory for the chart
+            // We load into a single array because the Chart node needs a 'replace' action with full dataset.
+            // Streaming happens file-by-line to avoid loading full file string content into RAM.
+            let allHistory = [];
+
+            // 4a. Load Trend Files
+            for (let i = 0; i < validFiles.length; i++) {
+                const filePath = path.join(TRENDS_DIR, validFiles[i]);
+                try {
+                    await streamLoadFile(filePath, allHistory);
+                } catch (err) {
+                    node.warn(`Failed to process ${validFiles[i]}: ${err.message}`);
+                }
+                
+                if (i % 2 === 0) updateStatus(`loading ${i + 1}/${validFiles.length}...`);
+            }
+
+    
+            // 4b. Load Active Buffer (if it wasn't rotated)
+            try {
+                await fs.promises.access(BUFFER_FILE);
+                await streamLoadFile(BUFFER_FILE, allHistory);
+            } catch (err) {
+                // No active buffer, that's fine
+            }
+
+            // 5. Finalize setup
+            finalizeAndSend(allHistory, validFiles.length);
+        }
+
+        async function rotateStartupBuffers() {
+            const now = new Date();
+            const currentTimestamp = Math.floor(now.getTime() / 1000);
+
+            // Check for this node's specific buffer
+            try {
+                const stats = await fs.promises.stat(BUFFER_FILE);
+                const fileModifiedTime = new Date(stats.mtime);
+                
+                // Rotation Logic:
+                // If the file is from a different hour than NOW, it is "stale" and should be rotated.
+                // If it is from current hour, keep it active (resume appending). 
+                // This prevents creating many small files on repeated reboots.
+                const isStale = (fileModifiedTime.getHours() !== now.getHours()) || 
+                                (now.getTime() - stats.mtimeMs > 3600 * 1000 * 1.5); // Safety: > 1.5h old
+
+                if (isStale) {
+                    const fileTs = Math.floor(stats.mtimeMs / 1000);
+                    const newName = path.join(TRENDS_DIR, `trend_${fileTs}_${node.id}.json`);
+                    await fs.promises.rename(BUFFER_FILE, newName);
+                }
+
+            } catch (err) {
+                // File likely doesn't exist, ignore
+            }
+
+            // Check for legacy buffer (migration)
+            try {
+                await fs.promises.access(LEGACY_BUFFER_FILE);
+                const legacyName = path.join(TRENDS_DIR, `trend_${currentTimestamp}_legacy.json`);
+                await fs.promises.rename(LEGACY_BUFFER_FILE, legacyName);
+            } catch (err) {
+                // Ignore
+            }
+        }
+
+        async function getHistoricalFiles() {
+            try {
+                const files = await fs.promises.readdir(TRENDS_DIR);
+                // Filter for our files: trend_TIMESTAMP_*.json
+                return files
+                    .filter(f => f.startsWith('trend_') && f.endsWith('.json'))
+                    .sort(); // String sort works for fixed-length timestamps usually, but numeric would be safer
+            } catch (err) {
+                return [];
+            }
+        }
+
+        function streamLoadFile(filePath, accumulatorArray) {
+            return new Promise((resolve, reject) => {
+                const fileStream = fs.createReadStream(filePath);
+                
+                fileStream.on('error', (err) => {
+                    // Log the specific error before resolving
+                    node.warn(`Stream read error for ${path.basename(filePath)}: ${err.message}`);
+                    resolve(); 
+                });
+
+                const rl = readline.createInterface({
+                    input: fileStream,
+                    crlfDelay: Infinity
+                });
+
+                rl.on('line', (line) => {
+                    if (!line.trim()) return;
+                    try {
+                        const pt = JSON.parse(line);
+                        // Optional: we could validate timestamp here if needed
+                        accumulatorArray.push(pt);
+                    } catch (err) {
+                        // Skip malformed lines
+                    }
+                });
+
+                rl.on('close', () => {
+                    resolve();
+                });
+            });
+        }
+
+        function finalizeAndSend(allHistory, fileCount) {
+            if (allHistory.length > 0) {
+                // Send history replace message
+                node.send({
+                    topic: config.topic || node.name || 'history',
+                    payload: allHistory,
+                    action: 'replace'
+                });
+            }
+
+            isInitializing = false;
+            cachedChunkCount = fileCount;
+            
+            startCommitTimer();
+            startPruneTimer();
+
+            // Dump queue
+            if (queuedMessages.length > 0) {
+                const toProcess = queuedMessages.splice(0, queuedMessages.length);
+                
+                toProcess.forEach(qMsg => {
+                    liveBuffer.push(qMsg);
+                    node.send({
+                        topic: qMsg.topic,
+                        payload: {
+                            topic: qMsg.topic,
+                            payload: qMsg.payload,
+                            ts: qMsg.ts
+                        },
+                        action: 'append'
+                    });
+                });
+            }
+            
+            updateStatus(`${messageCount} msgs, ${cachedChunkCount} chunks, buf: ${liveBuffer.length}`, true);
+        }
+
+        // =====================================================================
+        // Timers & File Management
+        // =====================================================================
+        function startCommitTimer() {
+            if (commitTimer) clearInterval(commitTimer);
+
+            commitTimer = setInterval(async () => {
+                // Yield to rotation or empty buffer
+                if (liveBuffer.length === 0 || node.isRotating) return;
+
+                // Take control of current points
+                const pointsToCommit = liveBuffer;
+                // Reset live buffer immediately so new messages go into next batch
+                liveBuffer = [];
+                
+                const lines = pointsToCommit.map(p => JSON.stringify(p)).join('\n');
+                
+                try {
+                    // Only add prefix newline if file exists and has content (simplified check)
+                    // We'll let the next append handle its own newline or just ensure we join efficiently.
+                    // Actually, safest is to append WITH a preceding newline if file exists.
+                    
+                    let prefix = '';
+                    try {
+                        const stats = await fs.promises.stat(BUFFER_FILE);
+                        if (stats.size > 0) prefix = '\n';
+                    } catch(e) { /* File doesn't exist, no prefix needed */ }
+
+                    await fs.promises.appendFile(BUFFER_FILE, prefix + lines);
+                } catch (err) {
+                    node.warn(`Buffer commit failed: ${err.message}`);
+                    // Put points back at the start of buffer if write failed
+                    // Use concat to avoid stack overflow with large arrays
+                    liveBuffer = pointsToCommit.concat(liveBuffer);
+                }
+            }, COMMIT_INTERVAL_MS);
+        }
+
+        function startPruneTimer() {
+            if (pruneTimer) clearInterval(pruneTimer);
+            pruneTimer = setInterval(() => pruneOldChunks(), PRUNE_INTERVAL_MS);
+        }
+
+        async function pruneOldChunks() {
+            const files = await getHistoricalFiles();
+            const now = Date.now();
+
+            for (const file of files) {
+                const parts = file.split('_');
+                const timestamp = parseInt(parts[1]);
+                if (isNaN(timestamp)) continue;
+
+                // Age check
+                const ageMs = now - (timestamp * 1000);
+                if (ageMs > MAX_BUFFER_AGE_MS) {
+                    try {
+                        await fs.promises.unlink(path.join(TRENDS_DIR, file));
+                        cachedChunkCount = Math.max(0, cachedChunkCount - 1);
+                    } catch (err) {
+                        node.warn(`Prune failed for ${file}: ${err.message}`);
+                    }
+                }
+            }
+        }
+
+        async function checkHourBoundary() {
+            const currentHour = new Date().getHours();
+            
+            if (node.lastRotationHour === undefined) {
+                node.lastRotationHour = currentHour;
+                return;
+            }
+
+            if (currentHour !== node.lastRotationHour) {
+                // Prevent race condition:
+                // If multiple msgs arrive while rotating, ignore them.
+                if (node.isRotating) return;
+                
+                node.isRotating = true; // Lock
+                
+                try {
+                    await rotateBuffer();
+                    // Update hour only on success to allow retry if it fails
+                    node.lastRotationHour = currentHour;
+                } catch (err) {
+                    node.warn(`Rotation failed: ${err.message}`);
+                } finally {
+                    node.isRotating = false; // Unlock
+                }
+            }
+        }
+
+        async function rotateBuffer() {
+            // Force commit of anything pending in memory before rotation
+            // Note: In this architecture, liveBuffer is usually empty due to commit timer,
+            // but might have recent points. 
+            // We just append to file, THEN rename.
+            
+            if (liveBuffer.length > 0) {
+                const pointsToCommit = liveBuffer;
+                liveBuffer = []; // Clear memory
+                const lines = pointsToCommit.map(p => JSON.stringify(p)).join('\n');
+                try {
+                    let prefix = '';
+                    try {
+                        const stats = await fs.promises.stat(BUFFER_FILE);
+                        if (stats.size > 0) prefix = '\n';
+                    } catch(e) { /* File doesn't exist */ }
+
+                    await fs.promises.appendFile(BUFFER_FILE, prefix + lines);
+                } catch (err) {
+                    // If append fails, we might lose these points during rotation, 
+                    // put them back and abort rotation. Use concat for safety.
+                    liveBuffer = pointsToCommit.concat(liveBuffer);
+                    node.warn(`Rotation aborted, append failed: ${err.message}`);
+                    return;
+                }
+            }
+
+            // Perform Rotation (Rename)
+            const timestamp = Math.floor(Date.now() / 1000);
+            const newName = path.join(TRENDS_DIR, `trend_${timestamp}_${node.id}.json`);
+
+            try {
+                await fs.promises.access(BUFFER_FILE);
+                await fs.promises.rename(BUFFER_FILE, newName);
+                cachedChunkCount++;
+            } catch (err) {
+                // BUFFER_FILE doesn't exist - no data to rotate
+            }
+        }
+
+        function updateStatus(text, force) {
+            const now = Date.now();
+            if (force || (now - lastStatusUpdate > 1000)) {
+                utils.setStatusChanged(node, text);
+                lastStatusUpdate = now;
+            }
+        }
+
+        // =====================================================================
+        // Message Handler
+        // =====================================================================
+        node.on('input', function(msg, send, done) {
+            send = send || function() { node.send.apply(node, arguments); };
+
+            if (!msg || !msg.hasOwnProperty('payload')) {
+                if (done) done();
+                return;
+            }
+
+            // Initialization Queue
+            if (isInitializing) {
+                queuedMessages.push({
+                    topic: msg.topic,
+                    payload: msg.payload,
+                    ts: msg.ts || Date.now()
+                });
+                if (done) done();
+                return;
+            }
+
+            // Check if we need to rotate files
+            checkHourBoundary(); 
+
+            // Process Message
+            const ts = msg.ts || Date.now();
+            
+            // Add to in-memory buffer for the next commit cycle
+            liveBuffer.push({
+                topic: msg.topic,
+                payload: msg.payload,
+                ts: ts
+            });
+
+            // Pass through to chart immediately
+            send({
+                topic: msg.topic,
+                payload: {
+                    topic: msg.topic,
+                    payload: msg.payload,
+                    ts: ts
+                },
+                action: 'append'
+            });
+
+            messageCount++;
+            
+            // Status update (throttled internally to 1s)
+            updateStatus(`${messageCount} msgs, ${cachedChunkCount} chunks, buf: ${liveBuffer.length}`, messageCount === 1);
+
+            if (done) done();
+        });
+
+        // =====================================================================
+        // Shutdown
+        // =====================================================================
+        node.on('close', function(done) {
+            if (commitTimer) clearInterval(commitTimer);
+            if (pruneTimer) clearInterval(pruneTimer);
+
+            // Attempt one final sync save if data exists
+            if (liveBuffer.length > 0) {
+                const lines = liveBuffer.map(p => JSON.stringify(p)).join('\n') + '\n';
+                try {
+                    // We must use Sync here because Node-RED close can lead to process exit
+                    // before async callbacks fire.
+                    fs.appendFileSync(BUFFER_FILE, lines);
+                } catch (e) {
+                    // ignore
+                }
+            }
+
+            utils.setStatusOK(node, 'closed');
+            done();
+        });
+
+        // Start
+        setTimeout(() => {
+            initializeFromFiles();
+        }, STARTUP_DELAY_MS);
+    }
+
+    RED.nodes.registerType("history-buffer", HistoryBufferNode);
+};

package/nodes/history-collector.html

@@ -11,6 +11,11 @@
         <label for="node-input-seriesName" title="Specify the measurement series name"><i class="fa fa-list"></i> Series</label>
         <input type="text" id="node-input-seriesName">
     </div>
+    <div class="form-row">
+        <label for="node-input-inputProperty" title="Source for value extraction (msg property or jsonata expression)"><i class="fa fa-folder-open"></i> Input Value</label>
+        <input type="text" id="node-input-inputProperty" placeholder="payload">
+        <input type="hidden" id="node-input-inputPropertyType">
+    </div>
     <div class="form-row">
         <label for="node-input-tags"><i class="fa fa-tags"></i> Tags</label>
         <input type="hidden" id="node-input-tags">
@@ -49,12 +54,14 @@
         defaults: {
             historyConfig: { value: "", type: "history-config", required: true },
             seriesName: { value: "", required: true },
+            inputProperty: { value: "payload" },
+            inputPropertyType: { value: "msg" },
             tags: { value: "" },
             storageType: { value: "batchObject" },
             name: { value: "" }
         },
         inputs: 1,
-        outputs: 1,
+        outputs: 0,
         icon: "file.png",
         paletteLabel: "history collector",
         label: function() {
@@ -67,6 +74,17 @@
             const tagContainer = $("#node-input-tag-container");
             const hiddenTagInput = $("#node-input-tags");
 
+            // Initialize typed input for input property (msg or jsonata)
+            try {
+                $("#node-input-inputProperty").typedInput({
+                    default: "msg",
+                    types: ["msg", "jsonata"],
+                    typeField: "#node-input-inputPropertyType"
+                }).typedInput("type", node.inputPropertyType || "msg").typedInput("value", node.inputProperty);
+            } catch (err) {
+                console.error("Error initializing inputProperty typedInput:", err);
+            }
+
             // Initialize empty typedInputs first
             seriesInput.typedInput({
                 types: [{ value: "series", options: [] }]
@@ -147,7 +165,10 @@
         },
         oneditsave: function() {
             const seriesInput = $("#node-input-seriesName");
+            const inputProperty = $("#node-input-inputProperty");
             this.seriesName = seriesInput.typedInput('value');
+            this.inputProperty = inputProperty.typedInput('value');
+            this.inputPropertyType = inputProperty.typedInput('type');
             this.storageType = $("#node-input-storageType").val();
             // tags are already in the hidden input thanks to the checkbox change handler
         }
@@ -275,6 +296,13 @@ The node supports five `storageType` options, each producing a unique output for
 - Timestamp: Generated as nanoseconds (ms * 1e6) for InfluxDB v2 precision.
 - Error Handling: Invalid configurations or payloads trigger warnings and status updates (red ring).
 
+### Status
+- Green (dot): Configuration update
+- Blue (dot): Data collected successfully
+- Blue (ring): No state change
+- Red (ring): Configuration error or invalid payload
+- Yellow (ring): Warning (validation issue)
+
 ### References
 - [Node-RED Documentation](https://nodered.org/docs/)
 - [GitHub Repository](https://github.com/BldgBlocks/node-red-contrib-buildingblocks-control)

package/nodes/history-collector.js

@@ -1,10 +1,14 @@
 module.exports = function(RED) {
+    const utils = require("./utils")(RED);
+
     function HistoryCollectorNode(config) {
         RED.nodes.createNode(this, config);
         this.historyConfig = RED.nodes.getNode(config.historyConfig);
         this.seriesName = config.seriesName;
         this.storageType = config.storageType || 'memory';
         this.tags = config.tags || '';
+        this.inputProperty = config.inputProperty || "payload";
+        this.inputPropertyType = config.inputPropertyType || "msg";
         const node = this;
 
         // Parse tags into key-value object
@@ -24,33 +28,48 @@ module.exports = function(RED) {
             return tags;
         }
 
-        node.on('input', function(msg) {
+        node.on('input', async function(msg, send, done) {
+            send = send || function() { node.send.apply(node, arguments); };
+
             // Guard against invalid message
             if (!msg) {
-                node.status({ fill: "red", shape: "ring", text: "invalid message" });
+                utils.setStatusError(node, "invalid message");
                 node.error('Invalid message received');
+                if (done) done();
                 return;
             }
 
             // Validate configuration
             if (!node.historyConfig) {
-                node.status({ fill: "red", shape: "ring", text: "missing history config" });
+                utils.setStatusError(node, "missing history config");
                 node.error('Missing history configuration', msg);
+                if (done) done();
                 return;
             }
             if (!node.seriesName) {
-                node.status({ fill: "red", shape: "ring", text: "missing series name" });
+                utils.setStatusError(node, "missing series name");
                 node.error('Missing series name', msg);
+                if (done) done();
                 return;
             }
             if (!node.historyConfig.name) {
-                node.status({ fill: "red", shape: "ring", text: "missing bucket name" });
+                utils.setStatusError(node, "missing bucket name");
                 node.error('Missing bucket name in history configuration', msg);
+                if (done) done();
+                return;
+            }
+
+            // Evaluate input property (msg or jsonata)
+            let payloadValue;
+            try {
+                payloadValue = await utils.evaluateNodeProperty(node.inputProperty, node.inputPropertyType, node, msg);
+            } catch (err) {
+                utils.setStatusError(node, "input evaluation error");
+                if (done) done();
                 return;
             }
 
             // Validate payload
-            let payloadValue = msg.payload;
             let formattedValue;
             if (typeof payloadValue === 'number') {
                 formattedValue = isNaN(payloadValue) ? null : payloadValue;
@@ -62,14 +81,14 @@ module.exports = function(RED) {
                     formattedValue = parseInt(payloadValue); // Handle InfluxDB integer format
                 }
             } else {
-                node.status({ fill: "red", shape: "ring", text: "invalid payload" });
-                node.warn(`Invalid payload type: ${typeof payloadValue}`);
+                utils.setStatusError(node, "invalid payload");
+                if (done) done();
                 return;
             }
 
             if (formattedValue === null) {
-                node.status({ fill: "red", shape: "ring", text: "invalid payload" });
-                node.warn(`Invalid payload value: ${msg.payload}`);
+                utils.setStatusError(node, "invalid payload");
+                if (done) done();
                 return;
             }
 
@@ -85,7 +104,7 @@ module.exports = function(RED) {
             const line = `${escapedMeasurementName}${tagsString ? ',' + tagsString : ''} value=${valueString} ${timestamp}`;
 
             // Set initial status
-            node.status({ fill: "green", shape: "dot", text: "configuration received" });
+            utils.setStatusOK(node, "configuration received");
 
             // Handle storage type
             if (node.storageType === 'memory') {
@@ -101,12 +120,12 @@ module.exports = function(RED) {
                 }
 
                 node.context().global.set(contextKey, bucketData);
-                node.status({ fill: "blue", shape: "dot", text: `stored: ${valueString}` });
+                utils.setStatusChanged(node, `stored: ${valueString}`);
             } else if (node.storageType === 'lineProtocol') {
                 msg.measurement = escapedMeasurementName;
                 msg.payload = line;
                 node.send(msg);
-                node.status({ fill: "blue", shape: "dot", text: `sent: ${valueString}` });
+                utils.setStatusChanged(node, `sent: ${valueString}`);
             } else if (node.storageType === 'object') {
                 msg.measurement = escapedMeasurementName;
                 msg.payload = {
@@ -116,7 +135,7 @@ module.exports = function(RED) {
                     timestamp: timestamp
                 };
                 node.send(msg);
-                node.status({ fill: "blue", shape: "dot", text: `sent: ${valueString}` });
+                utils.setStatusChanged(node, `sent: ${valueString}`);
             } else if (node.storageType === 'objectArray') {
                 msg.measurement = escapedMeasurementName;
                 msg.timestamp = timestamp;
@@ -127,7 +146,7 @@ module.exports = function(RED) {
                     tagsObj
                 ]
                 node.send(msg);
-                node.status({ fill: "blue", shape: "dot", text: `sent: ${valueString}` });
+                utils.setStatusChanged(node, `sent: ${valueString}`);
             } else if (node.storageType === 'batchObject') {
                 msg.payload = {
                     measurement: escapedMeasurementName,
@@ -138,8 +157,10 @@ module.exports = function(RED) {
                     tags: tagsObj
                 }
                 node.send(msg);
-                node.status({ fill: "blue", shape: "dot", text: `sent: ${valueString}` });
+                utils.setStatusChanged(node, `sent: ${valueString}`);
             }
+
+            if (done) done();
         });
 
         node.on("close", function(done) {