node-red-contrib-3dm-space 2.3.0 → 2.4.0

package/3dm-cloud-in.html

@@ -6,7 +6,9 @@
             name: { value: "" },
             server: { value: "", type: "3dm-cloud-config", required: true },
             keyFilter: { value: "" },
-            outputMode: { value: "value" }
+            outputMode: { value: "value" },
+            subscribeLastValues: { value: false },
+            onlyChanges: { value: false }
         },
         inputs: 0,
         outputs: 1,
@@ -49,6 +51,14 @@
             <option value="object">Object with key</option>
         </select>
     </div>
+    <div class="form-row">
+        <input type="checkbox" id="node-input-subscribeLastValues" style="display:inline-block; width:auto; margin-left:105px; vertical-align:top;">
+        <label for="node-input-subscribeLastValues" style="width:70%;">Subscribe to last values</label>
+    </div>
+    <div class="form-row">
+        <input type="checkbox" id="node-input-onlyChanges" style="display:inline-block; width:auto; margin-left:105px; vertical-align:top;">
+        <label for="node-input-onlyChanges" style="width:70%;">Only pass changed values</label>
+    </div>
 </script>
 
 <script type="text/html" data-help-name="3dm-cloud-in">
@@ -68,4 +78,8 @@
         <li><b>Value only</b>: outputs only the selected value.</li>
         <li><b>Object with key</b>: outputs a JSON object containing the selected key and value.</li>
     </ul>
+    <p><b>Subscribe to last values</b> (off by default): requests saved shared attributes once on connection and on each reconnect, then continues receiving live updates. With an Attribute Key, requests that key; otherwise requests all shared attributes. This restores saved settings, not missed command history or telemetry.</p>
+    <p><b>Only pass changed values</b> (off by default): suppresses repeats of the selected key, comparing after five-decimal rounding. With no key, remembers values per attribute and suppresses a message only when every included key is unchanged. When any key changes, the complete received attribute object is passed through. Nested object key order is ignored; array order and value types matter.</p>
+    <p>The first value passes. Memory survives reconnects, but resets when this node is redeployed or Node-RED restarts. Do not enable change filtering for repeated one-shot commands that must run each time.</p>
+    <p>No polling or disk writes are added. Change memory is capped at 1,024 keys and 1 MiB of serialized key/value bytes per node (actual JavaScript heap usage is higher). Evicted or oversized values may pass again. A saved-value request times out after 15 seconds without repeated application retries; live updates remain active.</p>
 </script>

package/3dm-cloud-in.js

@@ -8,6 +8,21 @@ module.exports = function (RED) {
      */
     var ATTRIBUTES_TOPIC = "v1/devices/me/attributes";
     var MQTT_QOS = 1;
+    var nextRequestId = 0;
+    var CACHE_MAX_KEYS = 1024;
+    var CACHE_MAX_BYTES = 1024 * 1024;
+
+    function stableJson(value) {
+        if (Array.isArray(value)) {
+            return "[" + value.map(stableJson).join(",") + "]";
+        }
+        if (value && typeof value === "object") {
+            return "{" + Object.keys(value).sort().map(function (key) {
+                return JSON.stringify(key) + ":" + stableJson(value[key]);
+            }).join(",") + "}";
+        }
+        return JSON.stringify(value);
+    }
 
     function ThreeDMCloudInNode(config) {
         RED.nodes.createNode(this, config);
@@ -18,6 +33,46 @@ module.exports = function (RED) {
         node.server = RED.nodes.getNode(config.server);
         node.keyFilter = config.keyFilter || "";
         node.outputMode = config.outputMode || "value";
+        node.subscribeLastValues = config.subscribeLastValues === true || config.subscribeLastValues === "true";
+        node.onlyChanges = config.onlyChanges === true || config.onlyChanges === "true";
+
+        var closed = false;
+        var connected = false;
+        var generation = 0;
+        var pending = null;
+        var cache = new Map();
+        var cacheBytes = 0;
+
+        function clearPending() {
+            if (pending && pending.timer) {
+                clearTimeout(pending.timer);
+            }
+            pending = null;
+        }
+
+        function changed(data) {
+            if (!node.onlyChanges) { return true; }
+            var object = data && typeof data === "object" && !Array.isArray(data);
+            var keys = node.keyFilter ? [node.keyFilter] : (object ? Object.keys(data) : [""]);
+            var anyChanged = false;
+            keys.forEach(function (key) {
+                var value = node.keyFilter || object ? data[key] : data;
+                var serialized = stableJson(value);
+                var previous = cache.get(key);
+                if (!previous || previous.value !== serialized) { anyChanged = true; }
+                if (previous) { cacheBytes -= previous.bytes; cache.delete(key); }
+                var bytes = Buffer.byteLength(key) + Buffer.byteLength(serialized);
+                if (bytes > CACHE_MAX_BYTES) { return; }
+                while (cache.size >= CACHE_MAX_KEYS || cacheBytes + bytes > CACHE_MAX_BYTES) {
+                    var oldest = cache.keys().next().value;
+                    cacheBytes -= cache.get(oldest).bytes;
+                    cache.delete(oldest);
+                }
+                cache.set(key, { value: serialized, bytes: bytes });
+                cacheBytes += bytes;
+            });
+            return anyChanged;
+        }
 
         if (!node.server) {
             node.status({ fill: "red", shape: "ring", text: "missing config" });
@@ -46,17 +101,7 @@ module.exports = function (RED) {
             return Object.prototype.hasOwnProperty.call(obj, key);
         }
 
-        var stateHandler = function (connected) {
-            updateStatus(connected);
-        };
-
-        var messageHandler = function (topic, payload) {
-            if (topic !== ATTRIBUTES_TOPIC) {
-                return;
-            }
-
-            var data = roundPayload(parsePayload(payload));
-
+        function outputData(data) {
             if (node.keyFilter) {
                 if (typeof data !== "object" || data === null || Array.isArray(data)) {
                     return;
@@ -66,6 +111,8 @@ module.exports = function (RED) {
                     return;
                 }
 
+                if (!changed(data)) { return; }
+
                 var msg = {
                     topic: node.keyFilter,
                     key: node.keyFilter,
@@ -85,31 +132,114 @@ module.exports = function (RED) {
                 return;
             }
 
+            if (!changed(data)) { return; }
             node.send({
                 topic: "3dm.attributes",
                 payload: data
             });
         };
 
-        node.server.on("state", stateHandler);
-        node.server.on("cloud-message", messageHandler);
+        function requestLastValues(epoch) {
+            if (!node.subscribeLastValues || closed || !connected || epoch !== generation) { return; }
+            node.server.subscribe("v1/devices/me/attributes/response/+", { qos: MQTT_QOS }, function (err) {
+                if (closed || !connected || epoch !== generation) { return; }
+                if (err) {
+                    node.warn("3DM Cloud saved attribute response subscribe failed: " + (err.message || err));
+                    return;
+                }
+                var id = String(++nextRequestId);
+                pending = { id: id, touched: new Set(), touchedBytes: 0, discard: false };
+                pending.timer = setTimeout(function () {
+                    clearPending();
+                    if (!closed) { node.warn("3DM Cloud saved attribute request timed out; live updates remain active."); }
+                }, 15000);
+                var request = { sharedKeys: node.keyFilter };
+                node.server.publish("v1/devices/me/attributes/request/" + id, request, { qos: MQTT_QOS }, function (error) {
+                    if (error && pending && pending.id === id) {
+                        clearPending();
+                        node.warn("3DM Cloud saved attribute request failed: " + (error.message || error));
+                    }
+                });
+            });
+        }
 
-        node.server.subscribe(ATTRIBUTES_TOPIC, { qos: MQTT_QOS }, function (err) {
-            if (err) {
-                node.status({ fill: "red", shape: "ring", text: "subscribe failed" });
-                node.warn("3DM Cloud attribute subscribe failed: " + (err.message || err));
-            } else {
-                updateStatus(node.server.connected);
+        var stateHandler = function (online) {
+            if (closed) { return; }
+            online = Boolean(online);
+            updateStatus(online);
+            if (online === connected) { return; }
+            connected = online;
+            generation += 1;
+            clearPending();
+            if (!online) { return; }
+            var epoch = generation;
+            // Subscribe to live changes first, then await the response subscription ACK.
+            node.server.subscribe(ATTRIBUTES_TOPIC, { qos: MQTT_QOS }, function (err) {
+                if (closed || !connected || epoch !== generation) { return; }
+                if (err) {
+                    node.status({ fill: "red", shape: "ring", text: "subscribe failed" });
+                    node.warn("3DM Cloud attribute subscribe failed: " + (err.message || err));
+                    return;
+                }
+                requestLastValues(epoch);
+            });
+        };
+
+        var messageHandler = function (topic, payload) {
+            if (closed) { return; }
+            if (topic === ATTRIBUTES_TOPIC) {
+                var live = roundPayload(parsePayload(payload));
+                if (pending && live && typeof live === "object") {
+                    Object.keys(live).forEach(function (key) {
+                        if (pending.touched.has(key) || pending.discard) { return; }
+                        var bytes = Buffer.byteLength(key);
+                        if (pending.touched.size >= CACHE_MAX_KEYS || pending.touchedBytes + bytes > CACHE_MAX_BYTES) {
+                            pending.discard = true;
+                        } else {
+                            pending.touched.add(key);
+                            pending.touchedBytes += bytes;
+                        }
+                    });
+                }
+                outputData(live);
+                return;
             }
-        });
+            if (!pending || topic !== "v1/devices/me/attributes/response/" + pending.id) { return; }
+            var response = parsePayload(payload);
+            var active = pending;
+            clearPending();
+            if (response && response.error) {
+                node.warn("3DM Cloud saved attribute response error: " + String(response.error));
+                return;
+            }
+            if (!response || typeof response !== "object" || !response.shared ||
+                    typeof response.shared !== "object" || Array.isArray(response.shared) || active.discard) { return; }
+            var data = roundPayload(response.shared);
+            active.touched.forEach(function (key) { delete data[key]; });
+            if (Object.keys(data).length > 0) { outputData(data); }
+        };
+
+        node.server.on("state", stateHandler);
+        node.server.on("cloud-message", messageHandler);
 
-        updateStatus(node.server.connected);
+        // Register the live subscription even when initially offline.
+        if (node.server.connected) {
+            stateHandler(true);
+        } else {
+            node.server.subscribe(ATTRIBUTES_TOPIC, { qos: MQTT_QOS }, function (err) {
+                if (!closed && err) { node.warn("3DM Cloud attribute subscribe failed: " + (err.message || err)); }
+            });
+            updateStatus(false);
+        }
 
         node.on("close", function () {
-            if (node.server) {
-                node.server.removeListener("state", stateHandler);
-                node.server.removeListener("cloud-message", messageHandler);
-            }
+            closed = true;
+            generation += 1;
+            clearPending();
+            cache.clear();
+            cacheBytes = 0;
+            node.server.removeListener("state", stateHandler);
+            node.server.removeListener("cloud-message", messageHandler);
         });
     }
 

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 2.4.0 — 2026-10-20
+
+- Add optional Subscribe to last values to 3DM In for initial connect and reconnect recovery of saved shared attributes.
+- Add optional Only pass changed values, retaining a bounded per-node comparison cache across communications disconnects.
+- Keep both options off by default and preserve existing output formats, five-decimal precision, cloud-out status pin, and Smithtek proprietary licence.
+- Correlate response IDs, wait for subscription acknowledgement, ignore obsolete responses, protect newer live values, and clear request timers on disconnect/close.
+- Document memory limits, local cache measurements, server/network impact, and command-filtering behaviour.
+
+
 ## 2.3.0 — 2026-10-18
 
 - Add a second 3DM Out output for boolean cloud connection status: true online, false offline, on deployment and state changes.

package/README.md

@@ -577,6 +577,41 @@ Use this for:
 
 ---
 
+## 3DM In Saved Values and Change Filtering
+
+Version 2.4.0 adds two independent checkboxes. Both default to **off**, preserving existing flows.
+
+### Subscribe to last values
+
+When enabled, the node requests the current saved **shared attributes** on initial connection and on every reconnect. With an Attribute Key selected, it requests that key only. With no key, it requests all shared attributes. Restored values use the existing Output Type and five-decimal precision settings, then normal live updates continue.
+
+This reads saved state, not historical telemetry or a replay of every missed command. The response subscription is acknowledged before the request is published. Requests have unique IDs so nodes sharing a login only handle their own replies. A 15-second response timeout clears the pending request and logs a warning; live updates continue. There is no periodic polling or repeated application retry loop. The next reconnect requests again. A live value received while awaiting a snapshot takes precedence over that snapshot for the same key. Temporary live-key tracking is bounded to 1,024 keys and 1 MiB of key-name bytes. If that limit is exceeded before the response, the snapshot is discarded to avoid overwriting newer live values.
+
+### Only pass changed values
+
+When enabled, the first observed value passes. Later messages pass only when the selected key changes, including values restored after reconnect. Values are compared after rounding to five decimal places. Types matter (for example, false and 0 differ). Nested object property order is ignored; array order matters.
+
+With no Attribute Key, the node remembers each received attribute separately. A message is suppressed only if every included key is unchanged. When any key is new or changed, the complete received attribute object passes, including any unchanged keys in that message. Missing keys in a partial update are not interpreted as deleted attributes.
+
+The cache survives a communications disconnect. It resets when this node is recreated by a deployment or Node-RED restarts, so the first value then passes again. Do not enable this option for repeated one-shot commands that must execute each time even if the value is identical. Comparing values does not provide command execution acknowledgement.
+
+### Node-RED memory and CPU impact
+
+- No new MQTT connection, disk storage, persistent history, or polling loop is added.
+- With change filtering off, no previous attribute values are retained. Saved-value fetching holds one pending request ID, one timeout, and temporary tracking of live keys while awaiting its reply.
+- With change filtering on, each node retains only its latest comparison value per relevant attribute. The cache is limited to **1,024 keys** and **1 MiB of serialized UTF-8 key/value bytes**, whichever is reached first. Actual JavaScript heap use is higher because of strings, maps, and object overhead; the 1 MiB limit is not a process-memory limit. Parsing, rounding, and comparison also allocate temporary memory proportional to incoming payload size.
+- Oversized values are passed without being cached. Least-recently-used keys are evicted to stay within the limits. These values may therefore pass again even if unchanged.
+- Synthetic Node.js v22.17.0 tests measured roughly **0.1 KiB** additional retained heap for populating one boolean cache entry, **10 KiB** for one 10 KiB string, and **9.4 KiB** for 100 numeric attributes. These are approximate cache-population measurements after node creation, excluding the base node, empty cache, Node-RED runtime, MQTT connection, and transient request/payload allocations. Gateway hardware and Node.js versions will differ.
+- Comparison work grows with the size of the value. Nested object keys are sorted for consistent comparison. For ordinary booleans and numeric setpoints, little data needs comparing; large configuration objects cost more. No production CPU or latency benchmark has been performed.
+
+### 3DM.space server and network impact
+
+Each enabled 3DM In node sends **one attribute read request per successful connection/reconnect**, receives one response, and subscribes to the response topic on the existing MQTT connection. MQTT acknowledgements and subscription traffic add protocol overhead. If ten enabled In nodes share a login, one reconnect causes ten read requests and normally ten responses; requests are not combined between nodes. Repeated connection failures can increase this traffic, and large all-attribute responses use more bandwidth than a selected-key request.
+
+The feature reads attributes; it does not write telemetry or configuration. Server processing includes request handling, attribute retrieval, and response serialization. Exact server RAM/CPU use depends on the deployed ThingsBoard version, attribute sizes, device count, and cache/database configuration; it has not been measured on 3DM.space. Change filtering runs locally **after receipt**, so it reduces downstream Node-RED work but does not reduce the server's live attribute notifications or MQTT bandwidth. Prefer a selected Attribute Key when only one value is needed.
+
+Implementation follows the [ThingsBoard MQTT attribute request API](https://thingsboard.io/docs/pe/reference/mqtt-api/attributes/). Local simulated MQTT tests verify the request/response flow; a live reconnect test against your 3DM.space instance is still required to confirm deployment-specific behaviour.
+
 ## 3DM In Status Messages
 
 ### connected

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-red-contrib-3dm-space",
-  "version": "2.3.0",
+  "version": "2.4.0",
   "description": "Node-RED nodes for 3DM.space SCADA Cloud telemetry, attributes, and local config storage",
   "main": "3dm-cloud-config.js",
   "license": "SEE LICENSE IN LICENSE",