@leverege/build-tools 2.119.0-pedro.1 → 2.119.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.119.0-pedro.1",
+  "version": "2.119.0",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {

package/src/dashboard/json/pitcrew/pubsub-repeater.json

@@ -1,7 +1,7 @@
 {
   "uid": "pubsub-repeater",
   "title": "PubSub Repeater",
-  "description": "PubSub Repeater. Metric names come from repeaterMetricNames() in src/processors/RepeaterMetrics.js, the single declared list - cross-check panels against it rather than against grep. Four things to know. (1) @leverege/metrics has NO label support, so the mapping name is baked into the metric NAME; a mapping's hyphens appear as underscores. Because rate() strips __name__, per-mapping series cannot be produced by one query, so the fleet row is a panel repeated over $mappings instead. (2) Names are DOUBLY prefixed: PROMETHEUS_STATUS_PREFIX (pubsub_repeater) plus the counter's own repeater_ prefix, giving pubsub_repeater_repeater_<mapping>_<suffix>. Verified against the live stg-pitcrew endpoint. (3) A counter that has never been incremented does not exist in Prometheus, so panels that add counters together use `or vector(0)` per term: PromQL addition with a missing operand yields no result at all, which would blank a healthy mapping's ratio panels entirely. A single-series panel for a reason that never fired still reads as No data rather than 0. (4) Retired mappings keep reporting until their series age out of retention, so $mappings can offer names no longer in REPEATER_MAPPINGS.",
+  "description": "PubSub Repeater. Metric names come from repeaterMetricNames() in src/processors/RepeaterMetrics.js, the single declared list - cross-check panels against it rather than against grep. Four things to know. (1) @leverege/metrics has NO label support, so the mapping name is baked into the metric NAME; a mapping's hyphens appear as underscores. Because rate() strips __name__, per-mapping series cannot be produced by one query, so the fleet row is a panel repeated over $mappings instead. (2) Names are DOUBLY prefixed: PROMETHEUS_STATUS_PREFIX (pubsub_repeater) plus the counter's own repeater_ prefix, giving pubsub_repeater_repeater_<mapping>_<suffix>. Verified against the live stg-pitcrew endpoint. (3) A counter that has never been incremented does not exist in Prometheus, so panels that add counters together use `or vector(0)` per term: PromQL addition with a missing operand yields no result at all, which would blank a healthy mapping's ratio panels entirely. A single-series panel for a reason that never fired still reads as No data rather than 0. (4) Retired mappings keep reporting until their series age out of retention, so $mappings can offer names no longer in REPEATER_MAPPINGS. (5) The imagine_api_* counters are not this service's own: @leverege/comms NodeComms emits <prefix>_<method> per request and <prefix>_<method>_error on any non-ok response, with the prefix defaulted to imagine_api by @leverege/api. No metricsMethod is passed, so there is no per-operation breakdown.",
   "tags": [
     "pubsub-repeater",
     "platform",
@@ -861,8 +861,8 @@
     },
     {
       "type": "timeseries",
-      "title": "Imagine API calls / sec",
-      "description": "GET is device resolution, so it is the device-header cache miss rate: it should sit far below the inbound message rate, and a GET rate tracking inbound means the cache is not helping (check REPEATER_DEVICE_CACHE_EXPIRES). POST is resource header creation on the target.",
+      "title": "Event loop lag",
+      "description": "The ordered queues are in-memory and drained on this event loop, so lag here is directly messages sitting behind a correlation id. Watch it alongside the awaited resource copy: requireRsrcCopy holds a message for up to rsrcCopyAwaitTimeout, and with ensureOrdering everything behind that correlation id waits with it.",
       "id": 15,
       "datasource": {
         "type": "prometheus",
@@ -876,7 +876,7 @@
       },
       "fieldConfig": {
         "defaults": {
-          "unit": "reqps",
+          "unit": "s",
           "min": 0,
           "custom": {
             "drawStyle": "line",
@@ -910,8 +910,8 @@
             "type": "prometheus",
             "uid": "${datasource}"
           },
-          "expr": "sum(rate(pubsub_repeater_imagine_api_get{job=~\"$job\"}[5m]))",
-          "legendFormat": "get/s (device lookups)",
+          "expr": "max(nodejs_eventloop_lag_p99_seconds{job=~\"$job\"})",
+          "legendFormat": "p99",
           "refId": "A"
         },
         {
@@ -919,16 +919,16 @@
             "type": "prometheus",
             "uid": "${datasource}"
           },
-          "expr": "sum(rate(pubsub_repeater_imagine_api_post{job=~\"$job\"}[5m]))",
-          "legendFormat": "post/s (rsrc creates)",
+          "expr": "max(nodejs_eventloop_lag_mean_seconds{job=~\"$job\"})",
+          "legendFormat": "mean",
           "refId": "B"
         }
       ]
     },
     {
       "type": "timeseries",
-      "title": "Event loop lag p99",
-      "description": "The ordered queues are in-memory and drained on this event loop, so lag here is directly messages sitting behind a correlation id. Watch it alongside the awaited resource copy: requireRsrcCopy holds a message for up to rsrcCopyAwaitTimeout, and with ensureOrdering everything behind that correlation id waits with it.",
+      "title": "Heap used",
+      "description": "Per pod. The in-memory ordered queues have no bound, so a target topic that stops accepting publishes shows up here as heap climbing before it shows up anywhere else.",
       "id": 16,
       "datasource": {
         "type": "prometheus",
@@ -936,13 +936,13 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 8,
+        "w": 12,
         "x": 0,
         "y": 48
       },
       "fieldConfig": {
         "defaults": {
-          "unit": "s",
+          "unit": "bytes",
           "min": 0,
           "custom": {
             "drawStyle": "line",
@@ -976,26 +976,96 @@
             "type": "prometheus",
             "uid": "${datasource}"
           },
-          "expr": "max(nodejs_eventloop_lag_p99_seconds{job=~\"$job\"})",
-          "legendFormat": "p99",
+          "expr": "nodejs_heap_size_used_bytes{job=~\"$job\"}",
+          "legendFormat": "{{pod}}",
           "refId": "A"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Pods reporting and CPU",
+      "description": "A pod restart loses whatever the in-memory queues were holding. Those messages were acknowledged on receipt, so nothing redelivers them: a restart is silent data loss, visible only as a step here.",
+      "id": 17,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 48
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0,
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 10,
+            "showPoints": "never",
+            "stacking": {
+              "mode": "none"
+            }
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ]
         },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "${datasource}"
           },
-          "expr": "max(nodejs_eventloop_lag_mean_seconds{job=~\"$job\"})",
-          "legendFormat": "mean",
+          "expr": "count(count by (pod) (nodejs_heap_size_used_bytes{job=~\"$job\"}))",
+          "legendFormat": "pods reporting",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job\"}[5m]))",
+          "legendFormat": "cpu cores",
           "refId": "B"
         }
       ]
     },
+    {
+      "type": "row",
+      "title": "Imagine API: call volume and outcomes",
+      "id": 18,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 56
+      },
+      "collapsed": false,
+      "panels": []
+    },
     {
       "type": "timeseries",
-      "title": "Heap used",
-      "description": "Per pod. The in-memory ordered queues have no bound, so a target topic that stops accepting publishes shows up here as heap climbing before it shows up anywhere else.",
-      "id": 17,
+      "title": "Imagine API calls / sec",
+      "description": "Emitted by @leverege/comms NodeComms.preRequest, one counter per HTTP method, so this is every request the Imagine client made. GET is device resolution and it is counted only on a cache MISS, so it should sit far below the inbound message rate; a GET rate tracking inbound means the device-header cache is not helping (check REPEATER_DEVICE_CACHE_EXPIRES). POST is resource header creation on the target, so it tracks rsrc_copied.",
+      "id": 19,
       "datasource": {
         "type": "prometheus",
         "uid": "${datasource}"
@@ -1003,12 +1073,12 @@
       "gridPos": {
         "h": 8,
         "w": 8,
-        "x": 8,
-        "y": 48
+        "x": 0,
+        "y": 57
       },
       "fieldConfig": {
         "defaults": {
-          "unit": "bytes",
+          "unit": "reqps",
           "min": 0,
           "custom": {
             "drawStyle": "line",
@@ -1042,17 +1112,26 @@
             "type": "prometheus",
             "uid": "${datasource}"
           },
-          "expr": "nodejs_heap_size_used_bytes{job=~\"$job\"}",
-          "legendFormat": "{{pod}}",
+          "expr": "sum(rate(pubsub_repeater_imagine_api_get{job=~\"$job\"}[5m])) or vector(0)",
+          "legendFormat": "get/s (device lookups)",
           "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "sum(rate(pubsub_repeater_imagine_api_post{job=~\"$job\"}[5m])) or vector(0)",
+          "legendFormat": "post/s (rsrc creates)",
+          "refId": "B"
         }
       ]
     },
     {
       "type": "timeseries",
-      "title": "Restarts and replica count",
-      "description": "A pod restart loses whatever the in-memory queues were holding. Those messages were acknowledged on receipt, so nothing redelivers them: a restart is silent data loss, visible only here.",
-      "id": 18,
+      "title": "Imagine API errors / sec",
+      "description": "NodeComms.postRequest increments <method>_error whenever the response is missing or not ok, so this is EVERY non-2xx, not only transport failures. Two consequences. A 404 from a device lookup that legitimately found nothing lands here, so a healthy get_error floor is normal and should track dropped_deviceNotFound rather than alarm you. And 401 or 403 here is the Imagine API options secret being wrong or expired, which otherwise surfaces only as every message dropping as deviceNotFound. The base method counter is incremented for the same request too, so errors are a subset of the calls panel, never additional to it. Expect a nonzero post_error floor by design: copyRsrcOnce creates the target resource header first, and resource-server answers an existing header with a 400 whose message says already exists. RepeaterProcessor treats that as the success it describes, which is what makes the copy retry idempotent, but the comms lib has already counted the 400 here. So post_error tracking rsrc_copied at a low ratio is the retry path working, not a fault.",
+      "id": 20,
       "datasource": {
         "type": "prometheus",
         "uid": "${datasource}"
@@ -1060,12 +1139,12 @@
       "gridPos": {
         "h": 8,
         "w": 8,
-        "x": 16,
-        "y": 48
+        "x": 8,
+        "y": 57
       },
       "fieldConfig": {
         "defaults": {
-          "unit": "short",
+          "unit": "reqps",
           "min": 0,
           "custom": {
             "drawStyle": "line",
@@ -1099,8 +1178,8 @@
             "type": "prometheus",
             "uid": "${datasource}"
           },
-          "expr": "count(count by (pod) (nodejs_heap_size_used_bytes{job=~\"$job\"}))",
-          "legendFormat": "pods reporting",
+          "expr": "sum(rate(pubsub_repeater_imagine_api_get_error{job=~\"$job\"}[5m])) or vector(0)",
+          "legendFormat": "get errors/s",
           "refId": "A"
         },
         {
@@ -1108,11 +1187,57 @@
             "type": "prometheus",
             "uid": "${datasource}"
           },
-          "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job\"}[5m]))",
-          "legendFormat": "cpu cores",
+          "expr": "sum(rate(pubsub_repeater_imagine_api_post_error{job=~\"$job\"}[5m])) or vector(0)",
+          "legendFormat": "post errors/s",
           "refId": "B"
         }
       ]
+    },
+    {
+      "type": "stat",
+      "title": "Imagine API error share",
+      "description": "Errors over all calls, across the selected range. Reads 0 while no error counter exists at all, which is the healthy state: the _error counters are created lazily on the first failure, so they are absent rather than zero until something fails. A step change here is worth more than the absolute value, since the floor is whatever the legitimate 404 rate happens to be.",
+      "id": 21,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 57
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "colorMode": "value",
+        "graphMode": "area"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "((sum(increase(pubsub_repeater_imagine_api_get_error{job=~\"$job\"}[$__range])) or vector(0)) + (sum(increase(pubsub_repeater_imagine_api_post_error{job=~\"$job\"}[$__range])) or vector(0))) / clamp_min((sum(increase(pubsub_repeater_imagine_api_get{job=~\"$job\"}[$__range])) or vector(0)) + (sum(increase(pubsub_repeater_imagine_api_post{job=~\"$job\"}[$__range])) or vector(0)), 1e-9)",
+          "legendFormat": "error share",
+          "refId": "A"
+        }
+      ]
     }
   ]
 }