@bldgblocks/node-red-contrib-control 0.1.34 → 0.1.36

package/nodes/global-setter.js

@@ -41,7 +41,7 @@ module.exports = function(RED) {
         // This runs in background immediately after deployment
         (async function initialize() {
             if (!node.varName) {
-                node.status({ fill: "red", shape: "ring", text: "no variable defined" });
+                utils.setStatusError(node, "no variable defined");
                 return;
             }
             try {
@@ -49,14 +49,29 @@ module.exports = function(RED) {
                 let state = await utils.getGlobalState(node, node.varName, node.storeName);
                 if (!state || typeof state !== 'object' || !state.priority) {
                     // If not, set default
-                    const newState = buildDefaultState();
-                    await utils.setGlobalState(node, node.varName, node.storeName, newState);
-                    node.status({ fill: "grey", shape: "dot", text: `initialized: default:${node.defaultValue}` });
+                    state = buildDefaultState();
+                    await utils.setGlobalState(node, node.varName, node.storeName, state);
+                    utils.setStatusOK(node, `initialized: default:${node.defaultValue}`);
+                } else {
+                    utils.setStatusOK(node, `loaded: ${state.value}`);
                 }
+                
+                // Send properly formed state object downstream after full initialization
+                // Allows network-register and other downstream nodes to register on startup
+                // Use setTimeout with delay to allow getter nodes time to establish their event listeners
+                setTimeout(() => {
+                    // Emit event so getter nodes with 'always' update mode receive initial value
+                    RED.events.emit("bldgblocks:global:value-changed", {
+                        key: node.varName,
+                        store: node.storeName,
+                        data: state
+                    });
+                    node.send(state);
+                }, 500);
             } catch (err) {
                 // Silently fail or log if init fails (DB down on boot?)
                 node.error(`Init Error: ${err.message}`);
-                node.status({ fill: "red", shape: "dot", text: "Init Error" });
+                utils.setStatusError(node, "Init Error");
             }
         })();
 
@@ -70,7 +85,7 @@ module.exports = function(RED) {
                 if (!msg) return utils.sendError(node, msg, done, "invalid message");
                 
                 if (node.isBusy) {
-                    node.status({ fill: "yellow", shape: "ring", text: "busy - dropped msg" });
+                    utils.setStatusBusy(node, "busy - dropped msg");
                     if (done) done(); 
                     return;
                 }
@@ -101,7 +116,7 @@ module.exports = function(RED) {
 
                 // Handle Reload
                 if (msg.context === "reload") {
-                    RED.events.emit("bldgblocks-global-update", { key: node.varName, store: node.storeName, data: state });
+                    RED.events.emit("bldgblocks:global:value-changed", { key: node.varName, store: node.storeName, data: state });
                     await utils.setGlobalState(node, node.varName, node.storeName, state);
                     
                     prefix = state.activePriority === 'default' ? '' : 'P';
@@ -111,9 +126,14 @@ module.exports = function(RED) {
                 }
 
                 // Get Input Value
-                const inputValue = RED.util.getMessageProperty(msg, node.inputProperty);
+                let inputValue;
+                try {
+                    inputValue = RED.util.getMessageProperty(msg, node.inputProperty);
+                } catch (err) {
+                    inputValue = undefined;
+                }
                 if (inputValue === undefined) {
-                    return utils.sendError(node, msg, done, `msg.${node.inputProperty} not found`);
+                    return utils.sendError(node, msg, done, `msg.${node.inputProperty} not found or invalid property path`);
                 }
 
                 // Update State
@@ -140,7 +160,9 @@ module.exports = function(RED) {
                 if (value === state.value && priority === state.activePriority) {
                     prefix = `${node.writePriority === 'default' ? '' : 'P'}`;
                     const noChangeText = `no change: ${prefix}${node.writePriority}:${state.value}${state.units}`;
-                    node.status({ fill: "green", shape: "dot", text: noChangeText });
+                    utils.setStatusUnchanged(node, noChangeText);
+                    // Pass message through even if no context change
+                    send({ ...state });
                     if (done) done();
                     return;
                 }
@@ -168,12 +190,18 @@ module.exports = function(RED) {
 
                 // Save (Async) and Emit
                 await utils.setGlobalState(node, node.varName, node.storeName, state);
+                // *** REQUIRE DEFAULT STORE ***
+                // Require default store to keep values in memory for polled getter nodes so they are not constantly re-reading from DB
+                // to avoid hammering edge devices with repeated reads. Writes are only on change. On event (reactive) sends the data in the event. 
+                if (node.storeName !== 'default') {
+                    await utils.setGlobalState(node, node.varName, 'default', state);
+                }
 
                 prefix = `${node.writePriority === 'default' ? '' : 'P'}`;
                 const statePrefix = `${state.activePriority === 'default' ? '' : 'P'}`;
                 const statusText = `write: ${prefix}${node.writePriority}:${inputValue}${state.units} > active: ${statePrefix}${state.activePriority}:${state.value}${state.units}`;
 
-                RED.events.emit("bldgblocks-global-update", {
+                RED.events.emit("bldgblocks:global:value-changed", {
                     key: node.varName,
                     store: node.storeName,
                     data: state
@@ -189,7 +217,7 @@ module.exports = function(RED) {
 
         node.on('close', function(removed, done) {
             if (removed && node.varName) {
-                RED.events.removeAllListeners("bldgblocks-global-update");
+                RED.events.removeAllListeners("bldgblocks:global:value-changed");
                 // Callback style safe for close
                 node.context().global.set(node.varName, undefined, node.storeName, function() {
                     done();

package/nodes/history-buffer.html

@@ -0,0 +1,96 @@
+<script type="text/html" data-template-name="history-buffer">
+    <div class="form-row">
+        <label for="node-input-name" title="Display name shown on the canvas"><i class="fa fa-tag"></i> Name</label>
+        <input type="text" id="node-input-name" placeholder="Trend History">
+    </div>
+    <div class="form-row">
+        <label for="node-input-bufferHours" title="How many hours of history to retain (default 3)"><i class="fa fa-clock-o"></i> Buffer Hours</label>
+        <input type="number" id="node-input-bufferHours" placeholder="3" min="0.01" max="24" step="0.01">
+    </div>
+</script>
+
+<script type="text/javascript">
+    RED.nodes.registerType("history-buffer", {
+        category: "bldgblocks history",
+        color: "#b9f2ff",
+        defaults: {
+            name: { value: "" },
+            bufferHours: {
+                value: 3,
+                required: true,
+                validate: function(v) {
+                    const val = parseFloat(v);
+                    return !isNaN(val) && val >= 0.01 && val <= 24;
+                }
+            }
+        },
+        inputs: 1,
+        outputs: 1,
+        inputLabels: ["data"],
+        outputLabels: ["chart"],
+        icon: "font-awesome/fa-history",
+        paletteLabel: "history buffer",
+        label: function() {
+            return this.name ? `${this.name} (${this.bufferHours}h)` : `history buffer (${this.bufferHours}h)`;
+        }
+    });
+</script>
+
+<script type="text/markdown" data-help-name="history-buffer">
+Persistent trend history for FlowFuse Dashboard 2.0 `ui-chart` nodes.
+
+Stores time-series data as hourly files on disk, surviving Node-RED restarts without blocking startup or using global context.
+
+### Inputs
+: topic (string) : Series identifier (e.g., "Return Temp").
+: payload (number) : Data value for this series.
+: ts (number) : Timestamp in milliseconds (optional, auto-filled).
+
+### Outputs
+: topic (string) : Series identifier.
+: payload (number|array) : Single value (append) or array of points (replace).
+: ts (number) : Timestamp in milliseconds.
+: action (string) : `"replace"` on startup, `"append"` for live data.
+
+### ui-chart Configuration (IMPORTANT)
+
+The `ui-chart` node **must** be configured to read data from the correct keys.
+
+In the chart's **Properties** section, set:
+
+| Property | Type | Value |
+|----------|------|-------|
+| **Series** | `key` | `topic` |
+| **X** | `key` | `ts` |
+| **Y** | `key` | `payload` |
+
+**Critical:** The type must be `key` (not `msg`). This tells the chart to look *inside* each array element for these fields.
+
+Also set:
+- **X-Axis Type**: `Timescale`
+- **X-Axis Limit**: Match your buffer hours or leave default
+
+### Storage
+
+Data is stored as Line-Delimited JSON in `~/.node-red/.bldgblocks/trends/`:
+- `buffer_<nodeid>.json` : Active buffer, committed every 30 seconds
+- `trend_<timestamp>_<nodeid>.json` : Hourly rotation files
+
+On startup:
+1. Waits 5 seconds for Node-RED to stabilize
+2. Loads all files within retention window
+3. Sends historical data with `action: "replace"`
+4. Begins accepting live data
+
+Supports 0.01 to 24 hours retention. Old files pruned automatically.
+
+### Status
+- **Green** : Ready
+- **Blue** : Active, receiving data
+- **Yellow** : Warning (file issue)
+- **Red** : Error
+
+### References
+- [ui-chart Documentation](https://dashboard.flowfuse.com/nodes/widgets/ui-chart.html)
+- [GitHub](https://github.com/BldgBlocks/node-red-contrib-control.git)
+</script>

package/nodes/history-buffer.js

@@ -0,0 +1,461 @@
+module.exports = function(RED) {
+    const fs = require('fs');
+    const path = require('path');
+    const readline = require('readline');
+    const utils = require('./utils')(RED);
+
+    function HistoryBufferNode(config) {
+        RED.nodes.createNode(this, config);
+        const node = this;
+
+        // Configuration
+        node.name = config.name;
+        node.bufferHours = parseFloat(config.bufferHours) || 3;
+
+        // Validate configuration
+        if (isNaN(node.bufferHours) || node.bufferHours < 0.01 || node.bufferHours > 24) {
+            node.bufferHours = 3;
+        }
+
+        // Constants
+        const TRENDS_DIR = getTrendsDirPath();
+        // Use node ID in filename to prevent collisions between multiple history nodes
+        const BUFFER_FILE = path.join(TRENDS_DIR, `buffer_${node.id}.json`);
+        // Legacy file path for migration
+        const LEGACY_BUFFER_FILE = path.join(TRENDS_DIR, 'buffer_current.json');
+        
+        const COMMIT_INTERVAL_MS = 30 * 1000;  // 30 seconds
+        const PRUNE_INTERVAL_MS = 60 * 1000;   // 60 seconds
+        const STARTUP_DELAY_MS = 5000;         // 5 seconds before loading history
+        const MAX_BUFFER_AGE_MS = node.bufferHours * 3600 * 1000;
+
+        // State
+        let liveBuffer = [];                    // Accumulate points since last commit to BUFFER_FILE
+        let commitTimer = null;
+        let pruneTimer = null;
+        let messageCount = 0;
+        let cachedChunkCount = 0;               // Cached count of historical files
+        let isInitializing = true;              // Flag: initialization in progress
+        let queuedMessages = [];                // Queue messages during initialization
+        
+        // Status throttling
+        let lastStatusUpdate = 0;
+
+        utils.setStatusOK(node, `ready, buffer: ${node.bufferHours}h`);
+
+        // =====================================================================
+        // Directory Helper
+        // =====================================================================
+        function getTrendsDirPath() {
+            const userDir = RED.settings.userDir || 
+                           process.env.NODE_RED_HOME ||
+                           path.join(require('os').homedir(), '.node-red');
+            return path.join(userDir, '.bldgblocks', 'trends');
+        }
+
+        // =====================================================================
+        // Initialize from files on startup (Async/Streaming)
+        // =====================================================================
+        async function initializeFromFiles() {
+            // 1. Ensure directory exists
+            try {
+                await fs.promises.mkdir(TRENDS_DIR, { recursive: true });
+            } catch (err) {
+                node.error(`Failed to create trends directory: ${err.message}`);
+                return;
+            }
+
+            // 2. Rotate/Migrate buffers
+            // Only rotate if they are "stale" (from a previous hour), otherwise append to them.
+            await rotateStartupBuffers();
+
+            // 3. Find and filter valid trend files
+            const historicalFiles = await getHistoricalFiles();
+
+            const validFiles = historicalFiles.filter(fileName => {
+                // Filename format: trend_TIMESTAMP_NODEID.json
+                // We split by '_' and take index 1.
+                const parts = fileName.split('_');
+                const timestamp = parseInt(parts[1]);
+                if (isNaN(timestamp)) return false;
+
+                // Simple check: is the file from within our retention window?
+                // Timestamp in filename is the time of rotation (end of that file's period)
+                const ageMs = Date.now() - (timestamp * 1000);
+                return ageMs <= MAX_BUFFER_AGE_MS + (3600 * 1000); // Add 1h grace period
+            });
+
+            updateStatus(`loading ${validFiles.length} files...`, true);
+
+            // 4. Stream load all data into memory for the chart
+            // We load into a single array because the Chart node needs a 'replace' action with full dataset.
+            // Streaming happens file-by-line to avoid loading full file string content into RAM.
+            let allHistory = [];
+
+            // 4a. Load Trend Files
+            for (let i = 0; i < validFiles.length; i++) {
+                const filePath = path.join(TRENDS_DIR, validFiles[i]);
+                try {
+                    await streamLoadFile(filePath, allHistory);
+                } catch (err) {
+                    node.warn(`Failed to process ${validFiles[i]}: ${err.message}`);
+                }
+                
+                if (i % 2 === 0) updateStatus(`loading ${i + 1}/${validFiles.length}...`);
+            }
+
+    
+            // 4b. Load Active Buffer (if it wasn't rotated)
+            try {
+                await fs.promises.access(BUFFER_FILE);
+                await streamLoadFile(BUFFER_FILE, allHistory);
+            } catch (err) {
+                // No active buffer, that's fine
+            }
+
+            // 5. Finalize setup
+            finalizeAndSend(allHistory, validFiles.length);
+        }
+
+        async function rotateStartupBuffers() {
+            const now = new Date();
+            const currentTimestamp = Math.floor(now.getTime() / 1000);
+
+            // Check for this node's specific buffer
+            try {
+                const stats = await fs.promises.stat(BUFFER_FILE);
+                const fileModifiedTime = new Date(stats.mtime);
+                
+                // Rotation Logic:
+                // If the file is from a different hour than NOW, it is "stale" and should be rotated.
+                // If it is from current hour, keep it active (resume appending). 
+                // This prevents creating many small files on repeated reboots.
+                const isStale = (fileModifiedTime.getHours() !== now.getHours()) || 
+                                (now.getTime() - stats.mtimeMs > 3600 * 1000 * 1.5); // Safety: > 1.5h old
+
+                if (isStale) {
+                    const fileTs = Math.floor(stats.mtimeMs / 1000);
+                    const newName = path.join(TRENDS_DIR, `trend_${fileTs}_${node.id}.json`);
+                    await fs.promises.rename(BUFFER_FILE, newName);
+                }
+
+            } catch (err) {
+                // File likely doesn't exist, ignore
+            }
+
+            // Check for legacy buffer (migration)
+            try {
+                await fs.promises.access(LEGACY_BUFFER_FILE);
+                const legacyName = path.join(TRENDS_DIR, `trend_${currentTimestamp}_legacy.json`);
+                await fs.promises.rename(LEGACY_BUFFER_FILE, legacyName);
+            } catch (err) {
+                // Ignore
+            }
+        }
+
+        async function getHistoricalFiles() {
+            try {
+                const files = await fs.promises.readdir(TRENDS_DIR);
+                // Filter for our files: trend_TIMESTAMP_*.json
+                return files
+                    .filter(f => f.startsWith('trend_') && f.endsWith('.json'))
+                    .sort(); // String sort works for fixed-length timestamps usually, but numeric would be safer
+            } catch (err) {
+                return [];
+            }
+        }
+
+        function streamLoadFile(filePath, accumulatorArray) {
+            return new Promise((resolve, reject) => {
+                const fileStream = fs.createReadStream(filePath);
+                
+                fileStream.on('error', (err) => {
+                    // Log the specific error before resolving
+                    node.warn(`Stream read error for ${path.basename(filePath)}: ${err.message}`);
+                    resolve(); 
+                });
+
+                const rl = readline.createInterface({
+                    input: fileStream,
+                    crlfDelay: Infinity
+                });
+
+                rl.on('line', (line) => {
+                    if (!line.trim()) return;
+                    try {
+                        const pt = JSON.parse(line);
+                        // Optional: we could validate timestamp here if needed
+                        accumulatorArray.push(pt);
+                    } catch (err) {
+                        // Skip malformed lines
+                    }
+                });
+
+                rl.on('close', () => {
+                    resolve();
+                });
+            });
+        }
+
+        function finalizeAndSend(allHistory, fileCount) {
+            if (allHistory.length > 0) {
+                // Send history replace message
+                node.send({
+                    topic: config.topic || node.name || 'history',
+                    payload: allHistory,
+                    action: 'replace'
+                });
+            }
+
+            isInitializing = false;
+            cachedChunkCount = fileCount;
+            
+            startCommitTimer();
+            startPruneTimer();
+
+            // Dump queue
+            if (queuedMessages.length > 0) {
+                const toProcess = queuedMessages.splice(0, queuedMessages.length);
+                
+                toProcess.forEach(qMsg => {
+                    liveBuffer.push(qMsg);
+                    node.send({
+                        topic: qMsg.topic,
+                        payload: {
+                            topic: qMsg.topic,
+                            payload: qMsg.payload,
+                            ts: qMsg.ts
+                        },
+                        action: 'append'
+                    });
+                });
+            }
+            
+            updateStatus(`${messageCount} msgs, ${cachedChunkCount} chunks, buf: ${liveBuffer.length}`, true);
+        }
+
+        // =====================================================================
+        // Timers & File Management
+        // =====================================================================
+        function startCommitTimer() {
+            if (commitTimer) clearInterval(commitTimer);
+
+            commitTimer = setInterval(async () => {
+                // Yield to rotation or empty buffer
+                if (liveBuffer.length === 0 || node.isRotating) return;
+
+                // Take control of current points
+                const pointsToCommit = liveBuffer;
+                // Reset live buffer immediately so new messages go into next batch
+                liveBuffer = [];
+                
+                const lines = pointsToCommit.map(p => JSON.stringify(p)).join('\n');
+                
+                try {
+                    // Only add prefix newline if file exists and has content (simplified check)
+                    // We'll let the next append handle its own newline or just ensure we join efficiently.
+                    // Actually, safest is to append WITH a preceding newline if file exists.
+                    
+                    let prefix = '';
+                    try {
+                        const stats = await fs.promises.stat(BUFFER_FILE);
+                        if (stats.size > 0) prefix = '\n';
+                    } catch(e) { /* File doesn't exist, no prefix needed */ }
+
+                    await fs.promises.appendFile(BUFFER_FILE, prefix + lines);
+                } catch (err) {
+                    node.warn(`Buffer commit failed: ${err.message}`);
+                    // Put points back at the start of buffer if write failed
+                    // Use concat to avoid stack overflow with large arrays
+                    liveBuffer = pointsToCommit.concat(liveBuffer);
+                }
+            }, COMMIT_INTERVAL_MS);
+        }
+
+        function startPruneTimer() {
+            if (pruneTimer) clearInterval(pruneTimer);
+            pruneTimer = setInterval(() => pruneOldChunks(), PRUNE_INTERVAL_MS);
+        }
+
+        async function pruneOldChunks() {
+            const files = await getHistoricalFiles();
+            const now = Date.now();
+
+            for (const file of files) {
+                const parts = file.split('_');
+                const timestamp = parseInt(parts[1]);
+                if (isNaN(timestamp)) continue;
+
+                // Age check
+                const ageMs = now - (timestamp * 1000);
+                if (ageMs > MAX_BUFFER_AGE_MS) {
+                    try {
+                        await fs.promises.unlink(path.join(TRENDS_DIR, file));
+                        cachedChunkCount = Math.max(0, cachedChunkCount - 1);
+                    } catch (err) {
+                        node.warn(`Prune failed for ${file}: ${err.message}`);
+                    }
+                }
+            }
+        }
+
+        async function checkHourBoundary() {
+            const currentHour = new Date().getHours();
+            
+            if (node.lastRotationHour === undefined) {
+                node.lastRotationHour = currentHour;
+                return;
+            }
+
+            if (currentHour !== node.lastRotationHour) {
+                // Prevent race condition:
+                // If multiple msgs arrive while rotating, ignore them.
+                if (node.isRotating) return;
+                
+                node.isRotating = true; // Lock
+                
+                try {
+                    await rotateBuffer();
+                    // Update hour only on success to allow retry if it fails
+                    node.lastRotationHour = currentHour;
+                } catch (err) {
+                    node.warn(`Rotation failed: ${err.message}`);
+                } finally {
+                    node.isRotating = false; // Unlock
+                }
+            }
+        }
+
+        async function rotateBuffer() {
+            // Force commit of anything pending in memory before rotation
+            // Note: In this architecture, liveBuffer is usually empty due to commit timer,
+            // but might have recent points. 
+            // We just append to file, THEN rename.
+            
+            if (liveBuffer.length > 0) {
+                const pointsToCommit = liveBuffer;
+                liveBuffer = []; // Clear memory
+                const lines = pointsToCommit.map(p => JSON.stringify(p)).join('\n');
+                try {
+                    let prefix = '';
+                    try {
+                        const stats = await fs.promises.stat(BUFFER_FILE);
+                        if (stats.size > 0) prefix = '\n';
+                    } catch(e) { /* File doesn't exist */ }
+
+                    await fs.promises.appendFile(BUFFER_FILE, prefix + lines);
+                } catch (err) {
+                    // If append fails, we might lose these points during rotation, 
+                    // put them back and abort rotation. Use concat for safety.
+                    liveBuffer = pointsToCommit.concat(liveBuffer);
+                    node.warn(`Rotation aborted, append failed: ${err.message}`);
+                    return;
+                }
+            }
+
+            // Perform Rotation (Rename)
+            const timestamp = Math.floor(Date.now() / 1000);
+            const newName = path.join(TRENDS_DIR, `trend_${timestamp}_${node.id}.json`);
+
+            try {
+                await fs.promises.access(BUFFER_FILE);
+                await fs.promises.rename(BUFFER_FILE, newName);
+                cachedChunkCount++;
+            } catch (err) {
+                // BUFFER_FILE doesn't exist - no data to rotate
+            }
+        }
+
+        function updateStatus(text, force) {
+            const now = Date.now();
+            if (force || (now - lastStatusUpdate > 1000)) {
+                utils.setStatusChanged(node, text);
+                lastStatusUpdate = now;
+            }
+        }
+
+        // =====================================================================
+        // Message Handler
+        // =====================================================================
+        node.on('input', function(msg, send, done) {
+            send = send || function() { node.send.apply(node, arguments); };
+
+            if (!msg || !msg.hasOwnProperty('payload')) {
+                if (done) done();
+                return;
+            }
+
+            // Initialization Queue
+            if (isInitializing) {
+                queuedMessages.push({
+                    topic: msg.topic,
+                    payload: msg.payload,
+                    ts: msg.ts || Date.now()
+                });
+                if (done) done();
+                return;
+            }
+
+            // Check if we need to rotate files
+            checkHourBoundary(); 
+
+            // Process Message
+            const ts = msg.ts || Date.now();
+            
+            // Add to in-memory buffer for the next commit cycle
+            liveBuffer.push({
+                topic: msg.topic,
+                payload: msg.payload,
+                ts: ts
+            });
+
+            // Pass through to chart immediately
+            send({
+                topic: msg.topic,
+                payload: {
+                    topic: msg.topic,
+                    payload: msg.payload,
+                    ts: ts
+                },
+                action: 'append'
+            });
+
+            messageCount++;
+            
+            // Status update (throttled internally to 1s)
+            updateStatus(`${messageCount} msgs, ${cachedChunkCount} chunks, buf: ${liveBuffer.length}`, messageCount === 1);
+
+            if (done) done();
+        });
+
+        // =====================================================================
+        // Shutdown
+        // =====================================================================
+        node.on('close', function(done) {
+            if (commitTimer) clearInterval(commitTimer);
+            if (pruneTimer) clearInterval(pruneTimer);
+
+            // Attempt one final sync save if data exists
+            if (liveBuffer.length > 0) {
+                const lines = liveBuffer.map(p => JSON.stringify(p)).join('\n') + '\n';
+                try {
+                    // We must use Sync here because Node-RED close can lead to process exit
+                    // before async callbacks fire.
+                    fs.appendFileSync(BUFFER_FILE, lines);
+                } catch (e) {
+                    // ignore
+                }
+            }
+
+            utils.setStatusOK(node, 'closed');
+            done();
+        });
+
+        // Start
+        setTimeout(() => {
+            initializeFromFiles();
+        }, STARTUP_DELAY_MS);
+    }
+
+    RED.nodes.registerType("history-buffer", HistoryBufferNode);
+};