node-red-contrib-uos-nats 1.3.68 → 1.3.72

package/README.md

@@ -109,6 +109,17 @@ Publishes your own data to the Data Hub.
 
 ---
 
+## Performance & Reliability
+- **High-Speed Decoding (Filter-on-Decode):** The node now intelligently filters incoming data at the byte-level. Even if a provider sends thousands of variables, Node-RED only decodes the ones you have selected. This massively reduces CPU usage.
+- **Large Buffers:** The internal NATS buffer has been increased (10MB) to handle event bursts (e.g. rapid switching).
+- **Slow Consumer Warning:** If Node-RED cannot keep up, a "SLOW CONSUMER" warning will appear in the debug log to alert you of dropped messages.
+
+## UI Features
+- **Grouped Variables:** Variables are automatically grouped by folder (prefix) in the selection list (e.g. `ur20._4com...`).
+- **Smart Filtering:** 
+    - **Read Node** shows all variables.
+    - **Write Node** automatically hides Read-Only variables to prevent errors.
+
 ## Troubleshooting
 
 - **Provider not visible?** Ensure **Provider ID** matches your **Client ID**. Easiest way: Leave Provider ID empty in the node.

package/lib/payloads.js

@@ -218,38 +218,48 @@ export function decodeWriteVariablesCommand(buffer) {
     return decodeVariableList(cmd.variables());
 }
 
-export function decodeVariableList(list) {
+export function decodeVariableList(list, whitelistIds = null) {
     if (!list)
         return [];
     const result = [];
+
+    // Optimization: Reuse holders to reduce GC pressure
+    const holderInt64 = new VariableValueInt64();
+    const holderFloat64 = new VariableValueFloat64();
+    const holderBoolean = new VariableValueBoolean();
+    const holderString = new VariableValueString();
+
     for (let i = 0; i < list.itemsLength(); i += 1) {
         const item = list.items(i);
         if (!item)
             continue;
+
+        // Optimization: Filter-on-Decode
+        // If whitelist is provided and this ID is NOT in it, skip decoding
+        if (whitelistIds && !whitelistIds.has(item.id())) {
+            continue;
+        }
+
         let decoded;
         switch (item.valueType()) {
             case VariableValue.Int64: {
-                const holder = new VariableValueInt64();
-                item.value(holder);
-                decoded = Number(holder.value());
+                item.value(holderInt64);
+                decoded = Number(holderInt64.value());
                 break;
             }
             case VariableValue.Float64: {
-                const holder = new VariableValueFloat64();
-                item.value(holder);
-                decoded = holder.value();
+                item.value(holderFloat64);
+                decoded = holderFloat64.value();
                 break;
             }
             case VariableValue.Boolean: {
-                const holder = new VariableValueBoolean();
-                item.value(holder);
-                decoded = holder.value();
+                item.value(holderBoolean);
+                decoded = holderBoolean.value();
                 break;
             }
             case VariableValue.String: {
-                const holder = new VariableValueString();
-                item.value(holder);
-                decoded = holder.value();
+                item.value(holderString);
+                decoded = holderString.value();
                 break;
             }
             default:

package/nodes/datahub-input.html

@@ -155,34 +155,123 @@
 
         const selectedMap = getSelectedMap();
 
+        // Grouping Logic
+        const groups = {};
+        const noGroup = [];
+
         vars.forEach(v => {
+          const key = v.key || '';
+          const parts = key.split('.');
+          if (parts.length > 1) {
+            const groupName = parts[0];
+            if (!groups[groupName]) groups[groupName] = [];
+            groups[groupName].push(v);
+          } else {
+            noGroup.push(v);
+          }
+        });
+
+        // Helper to render a single row
+        const createRow = (v) => {
           let rawId = (v.id !== undefined && v.id !== null) ? v.id : v.Id;
           const safeId = (rawId !== undefined && rawId !== null) ? rawId : 'ERR';
           const isSelected = selectedMap.has(v.key);
 
           const row = $('<div>', { class: 'var-row', style: rowStyle });
-
           const cbContainer = $('<div>', { style: 'text-align:center;' });
           const cb = $('<input type="checkbox" class="var-checkbox">')
             .prop('checked', isSelected)
             .data('key', v.key)
             .data('id', safeId);
           cbContainer.append(cb);
-
           const label = $('<div>', { style: keyStyle, title: v.key }).text(v.key);
-
           row.append(cbContainer).append(label);
-          $listContainer.append(row);
+          return row;
+        };
+
+        // Render Groups
+        Object.keys(groups).sort().forEach(groupName => {
+          const groupVars = groups[groupName];
+
+          // Header
+          const header = $('<div>', {
+            style: 'background:#eee; padding:5px 10px; font-weight:bold; cursor:pointer; border-bottom:1px solid #ddd; display:flex; align-items:center;'
+          });
+          const icon = $('<i class="fa fa-caret-right" style="margin-right:5px; width:10px;"></i>');
+          header.append(icon).append($('<span>').text(groupName + ` (${groupVars.length})`));
+
+          // Container for items
+          const itemContainer = $('<div>', { style: 'display:none; padding-left:0;' });
+
+          groupVars.forEach(v => {
+            itemContainer.append(createRow(v));
+          });
+
+          // Toggle Logic
+          header.click(function () {
+            const isVisible = itemContainer.is(':visible');
+            itemContainer.toggle(!isVisible);
+            icon.removeClass(isVisible ? 'fa-caret-down' : 'fa-caret-right').addClass(isVisible ? 'fa-caret-right' : 'fa-caret-down');
+          });
+
+          // Auto-expand if any child is selected OR if filtering
+          const hasSelection = groupVars.some(v => selectedMap.has(v.key));
+          if (hasSelection) {
+            header.click(); // Expand
+          }
+
+          $listContainer.append(header).append(itemContainer);
         });
+
+        // Render Ungrouped
+        if (noGroup.length > 0) {
+          const header = $('<div>', {
+            style: 'background:#eee; padding:5px 10px; font-weight:bold; border-bottom:1px solid #ddd; color:#666;'
+          }).text('Other Variables');
+          $listContainer.append(header);
+
+          noGroup.forEach(v => {
+            $listContainer.append(createRow(v));
+          });
+        }
       };
 
+      // Adjusted filter to open groups when searching
       const filterList = (term) => {
-        const rows = $listContainer.find('.var-row');
-        term = term.toLowerCase();
-        rows.each(function () {
-          const text = $(this).find('div:nth-child(2)').text().toLowerCase(); // 2nd child is name (grid)
-          $(this).toggle(text.indexOf(term) > -1);
+        const termLower = term.toLowerCase();
+
+        // Show all containers first to filter their rows
+        $listContainer.children().show();
+
+        // Iterate groups (header + div container pairs)
+        // Group Headers are usually direct children?
+        // Actually, logic needs to be robust.
+
+        // Simplest: Re-render? No, state/checkboxes lost.
+        // Hiding logic:
+        $listContainer.find('.var-row').each(function () {
+          const text = $(this).text().toLowerCase();
+          const match = text.indexOf(termLower) > -1;
+          $(this).toggle(match);
+
+          // If matching, ensure parent group container is visible
+          if (match) {
+            $(this).parent().show();
+            // Ensure header icon is correct (caret down)
+            const container = $(this).parent();
+            const header = container.prev();
+            if (header.find('.fa-caret-right').length) {
+              header.find('.fa').removeClass('fa-caret-right').addClass('fa-caret-down');
+            }
+          }
         });
+
+        // Hide empty group headers/containers?
+        // Let's keep it simple for now: Open all if search term exists
+        if (termLower.length > 0) {
+          $listContainer.find('div[style*="display:none"]').show();
+          $listContainer.find('.fa-caret-right').removeClass('fa-caret-right').addClass('fa-caret-down');
+        }
       };
 
       $searchInput.on('keyup', function () {
@@ -218,7 +307,7 @@
         // We generally don't. But fetch providers works if deployed.
 
         $.ajax({
-          url: 'uos/providers/' + configNodeId + '/' + providerId + '/variables',
+          url: 'uos/providers/' + configNodeId + '/' + providerId + '/variables?mode=read',
           success: function (data) {
             $fetchBtn.prop('disabled', false);
             $statusMsg.text('');
@@ -342,6 +431,12 @@
 
 <script type="text/html" data-help-name="datahub-input">
     <p><b>DataHub - Read</b> reads variables from u-OS Data Hub providers.</p>
+    <h3>Features</h3>
+    <ul>
+        <li><b>Smart Filtering:</b> Only selected variables are decoded (Filter-on-Decode), significantly reducing CPU usage.</li>
+        <li><b>Grouped Selection:</b> Variables are organized by prefix (e.g., Device Name) for easier navigation.</li>
+        <li><b>High Performance:</b> Enhanced buffering handles rapid data bursts reliably.</li>
+    </ul>
     <h3>Configuration</h3>
     <ol>
         <li><b>Config:</b> Select your u-OS connection.</li>

package/nodes/datahub-input.js

@@ -74,12 +74,14 @@ module.exports = function (RED) {
     // Pre-populate raw map with manual definitions
     this.manualDefs.forEach(d => defMap.set(d.id, { ...d, type: 'MANUAL', dataType: 'UNKNOWN', access: 'READ_ONLY' }));
 
+    // Optimization: Use Set for O(1) lookups
+    const variableSet = new Set(this.variables.map(v => normalizeKey(v)));
+
     const shouldInclude = (key) => {
-      if (!this.variables.length) {
+      if (variableSet.size === 0) {
         return true;
       }
-      const needle = normalizeKey(key);
-      return this.variables.includes(needle);
+      return variableSet.has(normalizeKey(key));
     };
 
     const processStates = (states) => {
@@ -101,6 +103,12 @@ module.exports = function (RED) {
         this.warnOnce('Filtering active but Variable Definitions failed to load (API Error). Names cannot be resolved. Try using "Name:ID" format to manually map variables.');
       }
 
+      // Optimization: redundant check removed if whitelist is active, but keeping as safeguard 
+      // If we used a whitelist, we know we only have relevant IDs.
+      // However, keeping strict check is safer for "Name-based" consistency.
+      // But we can optimize: if mapped.length is same as whitelist size, we are good?
+      // Actually, let's keep it simple: Filter-on-Decode handles the heavy lifting (byte level).
+      // This JS filter is now cheap (O(1) lookup). We'll keep it for correctness in case of aliasing/manual IDs.
       return mapped.filter((state) => shouldInclude(state.key));
     };
 
@@ -219,7 +227,9 @@ module.exports = function (RED) {
 
                 const bb = new flatbuffers.ByteBuffer(snapshotMsg.data);
                 const snapshotObj = ReadVariablesQueryResponse.getRootAsReadVariablesQueryResponse(bb);
-                const states = payloads.decodeVariableList(snapshotObj.variables());
+                // Optimization: Pass activeWhitelist to decoder
+                // Although snapshot usually contains only requested IDs, this adds safety/speed for full snapshots
+                const states = payloads.decodeVariableList(snapshotObj.variables(), activeWhitelist);
 
                 const filteredSnapshot = processStates(states);
                 if (filteredSnapshot.length > 0) {
@@ -267,6 +277,20 @@ module.exports = function (RED) {
 
 
 
+        // Create Whitelist Set for Optimized Decoding
+        const whitelistIds = new Set();
+        // Resolve keys to IDs and populate whitelist
+        if (this.variables.length > 0) {
+          const requestedKeys = new Set(this.variables);
+          for (const def of defMap.values()) {
+            if (requestedKeys.has(def.key)) {
+              whitelistIds.add(Number(def.id));
+            }
+          }
+        }
+        // If whitelist is empty (Wildcard mode), pass null to decoder to read ALL.
+        const activeWhitelist = whitelistIds.size > 0 ? whitelistIds : null;
+
         await performSnapshot();
 
         // RECONNECT LOGIC
@@ -284,12 +308,21 @@ module.exports = function (RED) {
           }
 
           this.log(`Subscribing to changes for ${this.providerId}...`);
-          sub = nc.subscribe(subjects.varsChangedEvent(this.providerId));
+
+          // Optimization: Increase maxMessages/maxBytes pending limits
+          // Default is often small (e.g. 65k bytes or related msg count). Raising to prevent SlowConsumer on bursts.
+          sub = nc.subscribe(subjects.varsChangedEvent(this.providerId), {
+            maxMessages: 10000,
+            maxBytes: 10 * 1024 * 1024 // 10MB Puffer
+          });
+
           (async () => {
             for await (const msg of sub) {
               const eventBB = new flatbuffers.ByteBuffer(msg.data);
               const event = VariablesChangedEvent.getRootAsVariablesChangedEvent(eventBB);
-              const changeStates = payloads.decodeVariableList(event.changedVariables());
+              // Optimization: Pass activeWhitelist to decoder
+              const changeStates = payloads.decodeVariableList(event.changedVariables(), activeWhitelist);
+
               const filtered = processStates(changeStates);
               if (filtered.length === 0) {
                 continue;
@@ -309,6 +342,18 @@ module.exports = function (RED) {
               this.warn(`subscription error: ${err.message}`);
             }
           });
+
+          // Optimization: Monitor for Slow Consumer / Dropped Messages via status iterator
+          (async () => {
+            if (sub && sub.status) {
+              for await (const s of sub.status()) {
+                if (s.type === 'slow_consumer') {
+                  this.warn(`SLOW CONSUMER DETECTED! Use less variables or faster CPU. Dropped: ${s.data}`);
+                  this.status({ fill: 'red', shape: 'dot', text: 'slow consumer (drops)' });
+                }
+              }
+            }
+          })().catch(() => { }); // Ignore status loop errors
         };
 
         connection.on('reconnected', () => {

package/nodes/datahub-write.html

@@ -85,7 +85,24 @@
                     return;
                 }
 
+                // Grouping Logic
+                const groups = {};
+                const noGroup = [];
+
                 vars.forEach(v => {
+                    const key = v.key || '';
+                    const parts = key.split('.');
+                    if (parts.length > 1) {
+                        const groupName = parts[0];
+                        if (!groups[groupName]) groups[groupName] = [];
+                        groups[groupName].push(v);
+                    } else {
+                        noGroup.push(v);
+                    }
+                });
+
+                // Helper to render a single row
+                const createRow = (v) => {
                     const safeId = (v.id !== undefined) ? v.id : (v.Id !== undefined ? v.Id : 'ERR');
                     const isSelected = String(v.key) === String($inputKey.val()); // Match by Key primarily now
 
@@ -97,6 +114,7 @@
                     row.append(cbContainer).append(keyCol);
 
                     const selectRow = () => {
+                        // Radio Behavior for Write Node (Single Select)
                         $('.var-checkbox').prop('checked', false);
                         cb.prop('checked', true);
                         $inputId.val(safeId !== 'ERR' ? safeId : '');
@@ -109,20 +127,94 @@
                     row.on('mouseenter', function () { $(this).css('background', '#f7f7f7'); });
                     row.on('mouseleave', function () { $(this).css('background', 'transparent'); });
 
-                    $listContainer.append(row);
+                    return row;
+                };
+
+                // Render Groups
+                Object.keys(groups).sort().forEach(groupName => {
+                    const groupVars = groups[groupName];
+
+                    // Header
+                    const header = $('<div>', {
+                        style: 'background:#eee; padding:5px 10px; font-weight:bold; cursor:pointer; border-bottom:1px solid #ddd; display:flex; align-items:center;'
+                    });
+                    const icon = $('<i class="fa fa-caret-right" style="margin-right:5px; width:10px;"></i>');
+                    header.append(icon).append($('<span>').text(groupName + ` (${groupVars.length})`));
+
+                    // Container for items
+                    const itemContainer = $('<div>', { style: 'display:none; padding-left:0;' });
+
+                    groupVars.forEach(v => {
+                        itemContainer.append(createRow(v));
+                    });
+
+                    // Toggle Logic
+                    header.click(function () {
+                        const isVisible = itemContainer.is(':visible');
+                        itemContainer.toggle(!isVisible);
+                        icon.removeClass(isVisible ? 'fa-caret-down' : 'fa-caret-right').addClass(isVisible ? 'fa-caret-right' : 'fa-caret-down');
+                    });
+
+                    // Auto-expand if child is selected
+                    const hasSelection = groupVars.some(v => String(v.key) === String($inputKey.val()));
+                    if (hasSelection) {
+                        header.click(); // Expand
+                    }
+
+                    $listContainer.append(header).append(itemContainer);
                 });
 
-                // Scroll to selection
+                // Render Ungrouped
+                if (noGroup.length > 0) {
+                    const header = $('<div>', {
+                        style: 'background:#eee; padding:5px 10px; font-weight:bold; border-bottom:1px solid #ddd; color:#666;'
+                    }).text('Other Variables');
+                    $listContainer.append(header);
+
+                    noGroup.forEach(v => {
+                        $listContainer.append(createRow(v));
+                    });
+                }
+
+                // Scroll to selection (simple attempt)
                 const selectedCb = $listContainer.find('.var-checkbox:checked');
-                if (selectedCb.length) $listContainer.scrollTop(selectedCb.closest('.var-row').offset().top - $listContainer.offset().top + $listContainer.scrollTop() - 40);
+                if (selectedCb.length) {
+                    // Try to scroll parent container
+                    // This is harder with hidden groups, but auto-expand handles visibility
+                    // Scroll logic might need delay or calculating offsets of visible items.
+                    // Skipping precise scroll for now.
+                }
             };
 
-            $searchInput.on('keyup', function () {
-                const term = $(this).val().toLowerCase();
+            // Adjusted filter to open groups when searching
+            const filterList = (term) => {
+                const termLower = term.toLowerCase();
+                $listContainer.children().show();
+
                 $listContainer.find('.var-row').each(function () {
-                    const text = $(this).find('div:nth-child(2)').text().toLowerCase();
-                    $(this).toggle(text.indexOf(term) > -1);
+                    const text = $(this).text().toLowerCase();
+                    const match = text.indexOf(termLower) > -1;
+                    $(this).toggle(match);
+
+                    if (match) {
+                        const container = $(this).parent();
+                        container.show();
+                        // Ensure header icon matches state
+                        const header = container.prev();
+                        if (header.find('.fa-caret-right').length) {
+                            header.find('.fa').removeClass('fa-caret-right').addClass('fa-caret-down');
+                        }
+                    }
                 });
+
+                if (termLower.length > 0) {
+                    $listContainer.find('div[style*="display:none"]').show();
+                    $listContainer.find('.fa-caret-right').removeClass('fa-caret-right').addClass('fa-caret-down');
+                }
+            };
+
+            $searchInput.on('keyup', function () {
+                filterList($(this).val());
             });
 
             $fetchBtn.on('click', function () {
@@ -136,7 +228,10 @@
                 $statusMsg.text('Fetching...').css('color', 'blue');
                 $listContainer.html('<div style="padding:20px; text-align:center;"><i class="fa fa-spinner fa-spin"></i> Loading...</div>');
 
-                $.getJSON('uos/providers/' + configNodeId + '/' + providerId + '/variables', function (data) {
+                $listContainer.html('<div style="padding:20px; text-align:center;"><i class="fa fa-spinner fa-spin"></i> Loading...</div>');
+
+                // Request filtering from server
+                $.getJSON('uos/providers/' + configNodeId + '/' + providerId + '/variables?mode=write', function (data) {
                     $fetchBtn.prop('disabled', false);
                     $statusMsg.text('');
 
@@ -242,11 +337,12 @@
 </script>
 
 <script type="text/html" data-help-name="datahub-write">
-    <p><b>DataHub - Write</b> sends write commands to external Data Hub providers.</p>
+    <p><b>DataHub - Write</b> sends write commands to external Data Hub providers.
+    It now features <b>Smart Filtering</b> to show only writable variables and <b>Grouped Lists</b> for better organization.</p>
     <h3>Configuration</h3>
     <ol>
-        <li><b>Provider ID:</b> Target provider (e.g. <code>u_os_adm</code>). Use the search button to find it.</li>
-        <li><b>Variable:</b> Select a target variable. <b>Only writable variables are shown.</b></li>
+        <li><b>Provider ID:</b> Target provider (e.g., <code>u_os_adm</code>). Use the search button to find it.</li>
+        <li><b>Variable:</b> Select a target variable. <b>Only variables with Write access are displayed.</b></li>
     </ol>
     <p>If no variable is selected, the node works in <b>Batch Mode</b> or <b>Dynamic Mode</b>.</p>
     <ul>

package/nodes/uos-config.js

@@ -708,6 +708,28 @@ module.exports = function (RED) {
           });
         }
 
+        // Filter by Mode
+        const mode = req.query.mode; // 'read', 'write', or defaults to all
+
+        if (mode && Array.isArray(variables)) {
+          variables = variables.filter(v => {
+            // Safe Access Type check (defaults to READ_ONLY if missing)
+            // access is usually: 'READ_ONLY' or 'READ_WRITE'
+            const access = String(v.access || v.accessType || 'READ_ONLY').toUpperCase();
+
+            if (mode === 'write') {
+              // WRITE requires READ_WRITE (or WRITE_ONLY if that existed)
+              return access.includes('WRITE');
+            } else if (mode === 'read') {
+              // READ allows everything (READ_ONLY and READ_WRITE)
+              // So theoretically we return everything.
+              // But maybe the user wants to filter OUT Write-Only? (Not common in u-OS)
+              return true;
+            }
+            return true;
+          });
+        }
+
         res.json(variables);
       }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-red-contrib-uos-nats",
-  "version": "1.3.68",
+  "version": "1.3.72",
   "description": "Node-RED nodes for Weidmüller u-OS Data Hub. Read, write, and provide variables via NATS protocol with OAuth2 authentication. Features: Variable Key resolution, custom icons, example flows, and provider definition caching.",
   "author": {
     "name": "IoTUeli",
@@ -59,6 +59,11 @@
       "datahub-write": "nodes/datahub-write.js"
     }
   },
+  "scripts": {
+    "build:offline": "node scripts/build-offline.js",
+    "generate:fbs": "bash scripts/generate-fbs.sh",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
   "devDependencies": {
     "dotenv": "^17.2.3"
   }