@leverege/build-tools 2.118.0 → 2.119.0-pedro.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -0,0 +1,1243 @@
|
|
|
1
|
+
{
|
|
2
|
+
"uid": "pubsub-repeater",
|
|
3
|
+
"title": "PubSub Repeater",
|
|
4
|
+
"description": "PubSub Repeater. Metric names come from repeaterMetricNames() in src/processors/RepeaterMetrics.js, the single declared list - cross-check panels against it rather than against grep. Four things to know. (1) @leverege/metrics has NO label support, so the mapping name is baked into the metric NAME; a mapping's hyphens appear as underscores. Because rate() strips __name__, per-mapping series cannot be produced by one query, so the fleet row is a panel repeated over $mappings instead. (2) Names are DOUBLY prefixed: PROMETHEUS_STATUS_PREFIX (pubsub_repeater) plus the counter's own repeater_ prefix, giving pubsub_repeater_repeater_<mapping>_<suffix>. Verified against the live stg-pitcrew endpoint. (3) A counter that has never been incremented does not exist in Prometheus, so panels that add counters together use `or vector(0)` per term: PromQL addition with a missing operand yields no result at all, which would blank a healthy mapping's ratio panels entirely. A single-series panel for a reason that never fired still reads as No data rather than 0. (4) Retired mappings keep reporting until their series age out of retention, so $mappings can offer names no longer in REPEATER_MAPPINGS. (5) The imagine_api_* counters are not this service's own: @leverege/comms NodeComms emits <prefix>_<method> per request and <prefix>_<method>_error on any non-ok response, with the prefix defaulted to imagine_api by @leverege/api. No metricsMethod is passed, so there is no per-operation breakdown.",
|
|
5
|
+
"tags": [
|
|
6
|
+
"pubsub-repeater",
|
|
7
|
+
"platform",
|
|
8
|
+
"repeater"
|
|
9
|
+
],
|
|
10
|
+
"timezone": "",
|
|
11
|
+
"editable": true,
|
|
12
|
+
"graphTooltip": 1,
|
|
13
|
+
"refresh": "1m",
|
|
14
|
+
"schemaVersion": 39,
|
|
15
|
+
"version": 1,
|
|
16
|
+
"time": {
|
|
17
|
+
"from": "now-6h",
|
|
18
|
+
"to": "now"
|
|
19
|
+
},
|
|
20
|
+
"timepicker": {},
|
|
21
|
+
"annotations": {
|
|
22
|
+
"list": []
|
|
23
|
+
},
|
|
24
|
+
"links": [],
|
|
25
|
+
"templating": {
|
|
26
|
+
"list": [
|
|
27
|
+
{
|
|
28
|
+
"type": "datasource",
|
|
29
|
+
"name": "datasource",
|
|
30
|
+
"label": "Data source",
|
|
31
|
+
"query": "prometheus",
|
|
32
|
+
"current": {},
|
|
33
|
+
"hide": 0
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
"type": "query",
|
|
37
|
+
"name": "job",
|
|
38
|
+
"label": "Job",
|
|
39
|
+
"datasource": {
|
|
40
|
+
"type": "prometheus",
|
|
41
|
+
"uid": "${datasource}"
|
|
42
|
+
},
|
|
43
|
+
"description": "Scrape job, so one dashboard can serve several deployments of this service.",
|
|
44
|
+
"query": "label_values({__name__=~\"pubsub_repeater_repeater_.*_forwarded\"}, job)",
|
|
45
|
+
"refresh": 1,
|
|
46
|
+
"includeAll": true,
|
|
47
|
+
"multi": true,
|
|
48
|
+
"current": {},
|
|
49
|
+
"hide": 0
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
"type": "query",
|
|
53
|
+
"name": "repeater",
|
|
54
|
+
"label": "Mapping (detail rows)",
|
|
55
|
+
"datasource": {
|
|
56
|
+
"type": "prometheus",
|
|
57
|
+
"uid": "${datasource}"
|
|
58
|
+
},
|
|
59
|
+
"description": "Single mapping for the detail rows. Extracted from the metric NAME, because this library emits no labels. Hyphens in the configured name appear here as underscores.",
|
|
60
|
+
"query": "metrics(pubsub_repeater_repeater_.*_forwarded)",
|
|
61
|
+
"regex": "/pubsub_repeater_repeater_(?<value>.*)_forwarded/",
|
|
62
|
+
"refresh": 1,
|
|
63
|
+
"includeAll": false,
|
|
64
|
+
"multi": false,
|
|
65
|
+
"current": {},
|
|
66
|
+
"hide": 0
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
"type": "query",
|
|
70
|
+
"name": "mappings",
|
|
71
|
+
"label": "Mappings (fleet row)",
|
|
72
|
+
"datasource": {
|
|
73
|
+
"type": "prometheus",
|
|
74
|
+
"uid": "${datasource}"
|
|
75
|
+
},
|
|
76
|
+
"description": "Mappings the fleet row repeats a panel over. Multi-select, because one query cannot produce a series per mapping.",
|
|
77
|
+
"query": "metrics(pubsub_repeater_repeater_.*_forwarded)",
|
|
78
|
+
"regex": "/pubsub_repeater_repeater_(?<value>.*)_forwarded/",
|
|
79
|
+
"refresh": 1,
|
|
80
|
+
"includeAll": true,
|
|
81
|
+
"multi": true,
|
|
82
|
+
"current": {},
|
|
83
|
+
"hide": 0
|
|
84
|
+
}
|
|
85
|
+
]
|
|
86
|
+
},
|
|
87
|
+
"panels": [
|
|
88
|
+
{
|
|
89
|
+
"type": "row",
|
|
90
|
+
"title": "Fleet: one panel per mapping",
|
|
91
|
+
"id": 1,
|
|
92
|
+
"gridPos": {
|
|
93
|
+
"h": 1,
|
|
94
|
+
"w": 24,
|
|
95
|
+
"x": 0,
|
|
96
|
+
"y": 0
|
|
97
|
+
},
|
|
98
|
+
"collapsed": false,
|
|
99
|
+
"panels": []
|
|
100
|
+
},
|
|
101
|
+
{
|
|
102
|
+
"type": "timeseries",
|
|
103
|
+
"title": "$mappings",
|
|
104
|
+
"description": "One panel per selected mapping, because @leverege/metrics emits no labels: the mapping name is part of the metric NAME, so a single multi-series query is not possible (rate() strips __name__, which is the only place the mapping is recorded). forwarded flat at zero with drops climbing is the signature failure of this service.",
|
|
105
|
+
"id": 2,
|
|
106
|
+
"datasource": {
|
|
107
|
+
"type": "prometheus",
|
|
108
|
+
"uid": "${datasource}"
|
|
109
|
+
},
|
|
110
|
+
"gridPos": {
|
|
111
|
+
"h": 7,
|
|
112
|
+
"w": 8,
|
|
113
|
+
"x": 0,
|
|
114
|
+
"y": 1
|
|
115
|
+
},
|
|
116
|
+
"fieldConfig": {
|
|
117
|
+
"defaults": {
|
|
118
|
+
"unit": "reqps",
|
|
119
|
+
"min": 0,
|
|
120
|
+
"custom": {
|
|
121
|
+
"drawStyle": "line",
|
|
122
|
+
"lineWidth": 1,
|
|
123
|
+
"fillOpacity": 10,
|
|
124
|
+
"showPoints": "never",
|
|
125
|
+
"stacking": {
|
|
126
|
+
"mode": "none"
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
},
|
|
130
|
+
"overrides": []
|
|
131
|
+
},
|
|
132
|
+
"options": {
|
|
133
|
+
"legend": {
|
|
134
|
+
"displayMode": "table",
|
|
135
|
+
"placement": "bottom",
|
|
136
|
+
"calcs": [
|
|
137
|
+
"lastNotNull",
|
|
138
|
+
"max"
|
|
139
|
+
]
|
|
140
|
+
},
|
|
141
|
+
"tooltip": {
|
|
142
|
+
"mode": "multi",
|
|
143
|
+
"sort": "desc"
|
|
144
|
+
}
|
|
145
|
+
},
|
|
146
|
+
"targets": [
|
|
147
|
+
{
|
|
148
|
+
"datasource": {
|
|
149
|
+
"type": "prometheus",
|
|
150
|
+
"uid": "${datasource}"
|
|
151
|
+
},
|
|
152
|
+
"expr": "(sum(rate(pubsub_repeater_repeater_${mappings}_forwarded{job=~\"$job\"}[5m])) or vector(0))",
|
|
153
|
+
"legendFormat": "forwarded/s",
|
|
154
|
+
"refId": "A"
|
|
155
|
+
},
|
|
156
|
+
{
|
|
157
|
+
"datasource": {
|
|
158
|
+
"type": "prometheus",
|
|
159
|
+
"uid": "${datasource}"
|
|
160
|
+
},
|
|
161
|
+
"expr": "(sum(rate(pubsub_repeater_repeater_${mappings}_dropped_notMember{job=~\"$job\"}[5m])) or vector(0)) + (sum(rate(pubsub_repeater_repeater_${mappings}_dropped_noDevicePath{job=~\"$job\"}[5m])) or vector(0)) + (sum(rate(pubsub_repeater_repeater_${mappings}_dropped_deviceNotFound{job=~\"$job\"}[5m])) or vector(0)) + (sum(rate(pubsub_repeater_repeater_${mappings}_dropped_filtered{job=~\"$job\"}[5m])) or vector(0)) + (sum(rate(pubsub_repeater_repeater_${mappings}_dropped_rsrcCopyFailed{job=~\"$job\"}[5m])) or vector(0))",
|
|
162
|
+
"legendFormat": "dropped/s",
|
|
163
|
+
"refId": "B"
|
|
164
|
+
}
|
|
165
|
+
],
|
|
166
|
+
"repeat": "mappings",
|
|
167
|
+
"repeatDirection": "h"
|
|
168
|
+
},
|
|
169
|
+
{
|
|
170
|
+
"type": "row",
|
|
171
|
+
"title": "Selected mapping: forwarding",
|
|
172
|
+
"id": 3,
|
|
173
|
+
"gridPos": {
|
|
174
|
+
"h": 1,
|
|
175
|
+
"w": 24,
|
|
176
|
+
"x": 0,
|
|
177
|
+
"y": 8
|
|
178
|
+
},
|
|
179
|
+
"collapsed": false,
|
|
180
|
+
"panels": []
|
|
181
|
+
},
|
|
182
|
+
{
|
|
183
|
+
"type": "stat",
|
|
184
|
+
"title": "Totals over the selected range",
|
|
185
|
+
"description": "The headline for one mapping. forwarded should dominate. notMember dominating means nothing is enrolled, which is the expected state for a new mapping. noDevicePath dominating means the subscription filter is not narrowing the topic to the message shape the paths were written for. A counter that has never been incremented does not exist in Prometheus at all, so it reads as No data rather than 0.",
|
|
186
|
+
"id": 4,
|
|
187
|
+
"datasource": {
|
|
188
|
+
"type": "prometheus",
|
|
189
|
+
"uid": "${datasource}"
|
|
190
|
+
},
|
|
191
|
+
"gridPos": {
|
|
192
|
+
"h": 5,
|
|
193
|
+
"w": 24,
|
|
194
|
+
"x": 0,
|
|
195
|
+
"y": 9
|
|
196
|
+
},
|
|
197
|
+
"fieldConfig": {
|
|
198
|
+
"defaults": {
|
|
199
|
+
"unit": "short",
|
|
200
|
+
"min": 0
|
|
201
|
+
},
|
|
202
|
+
"overrides": []
|
|
203
|
+
},
|
|
204
|
+
"options": {
|
|
205
|
+
"reduceOptions": {
|
|
206
|
+
"calcs": [
|
|
207
|
+
"lastNotNull"
|
|
208
|
+
],
|
|
209
|
+
"fields": "",
|
|
210
|
+
"values": false
|
|
211
|
+
},
|
|
212
|
+
"textMode": "auto",
|
|
213
|
+
"colorMode": "value",
|
|
214
|
+
"graphMode": "area"
|
|
215
|
+
},
|
|
216
|
+
"targets": [
|
|
217
|
+
{
|
|
218
|
+
"datasource": {
|
|
219
|
+
"type": "prometheus",
|
|
220
|
+
"uid": "${datasource}"
|
|
221
|
+
},
|
|
222
|
+
"expr": "sum(increase(pubsub_repeater_repeater_${repeater}_forwarded{job=~\"$job\"}[$__range]))",
|
|
223
|
+
"legendFormat": "forwarded",
|
|
224
|
+
"refId": "A"
|
|
225
|
+
},
|
|
226
|
+
{
|
|
227
|
+
"datasource": {
|
|
228
|
+
"type": "prometheus",
|
|
229
|
+
"uid": "${datasource}"
|
|
230
|
+
},
|
|
231
|
+
"expr": "sum(increase(pubsub_repeater_repeater_${repeater}_dropped_notMember{job=~\"$job\"}[$__range]))",
|
|
232
|
+
"legendFormat": "notMember",
|
|
233
|
+
"refId": "B"
|
|
234
|
+
},
|
|
235
|
+
{
|
|
236
|
+
"datasource": {
|
|
237
|
+
"type": "prometheus",
|
|
238
|
+
"uid": "${datasource}"
|
|
239
|
+
},
|
|
240
|
+
"expr": "sum(increase(pubsub_repeater_repeater_${repeater}_dropped_noDevicePath{job=~\"$job\"}[$__range]))",
|
|
241
|
+
"legendFormat": "noDevicePath",
|
|
242
|
+
"refId": "C"
|
|
243
|
+
},
|
|
244
|
+
{
|
|
245
|
+
"datasource": {
|
|
246
|
+
"type": "prometheus",
|
|
247
|
+
"uid": "${datasource}"
|
|
248
|
+
},
|
|
249
|
+
"expr": "sum(increase(pubsub_repeater_repeater_${repeater}_dropped_deviceNotFound{job=~\"$job\"}[$__range]))",
|
|
250
|
+
"legendFormat": "deviceNotFound",
|
|
251
|
+
"refId": "D"
|
|
252
|
+
},
|
|
253
|
+
{
|
|
254
|
+
"datasource": {
|
|
255
|
+
"type": "prometheus",
|
|
256
|
+
"uid": "${datasource}"
|
|
257
|
+
},
|
|
258
|
+
"expr": "sum(increase(pubsub_repeater_repeater_${repeater}_dropped_filtered{job=~\"$job\"}[$__range]))",
|
|
259
|
+
"legendFormat": "filtered",
|
|
260
|
+
"refId": "E"
|
|
261
|
+
},
|
|
262
|
+
{
|
|
263
|
+
"datasource": {
|
|
264
|
+
"type": "prometheus",
|
|
265
|
+
"uid": "${datasource}"
|
|
266
|
+
},
|
|
267
|
+
"expr": "sum(increase(pubsub_repeater_repeater_${repeater}_dropped_rsrcCopyFailed{job=~\"$job\"}[$__range]))",
|
|
268
|
+
"legendFormat": "rsrcCopyFailed",
|
|
269
|
+
"refId": "F"
|
|
270
|
+
},
|
|
271
|
+
{
|
|
272
|
+
"datasource": {
|
|
273
|
+
"type": "prometheus",
|
|
274
|
+
"uid": "${datasource}"
|
|
275
|
+
},
|
|
276
|
+
"expr": "sum(increase(pubsub_repeater_repeater_${repeater}_rsrc_copied{job=~\"$job\"}[$__range]))",
|
|
277
|
+
"legendFormat": "rsrc copied",
|
|
278
|
+
"refId": "G"
|
|
279
|
+
}
|
|
280
|
+
]
|
|
281
|
+
},
|
|
282
|
+
{
|
|
283
|
+
"type": "timeseries",
|
|
284
|
+
"title": "Forwarded / sec",
|
|
285
|
+
"description": "Messages published to the target topic, summed across replicas. Counted after the publish returns, so a message that threw on the way out is not counted here.",
|
|
286
|
+
"id": 5,
|
|
287
|
+
"datasource": {
|
|
288
|
+
"type": "prometheus",
|
|
289
|
+
"uid": "${datasource}"
|
|
290
|
+
},
|
|
291
|
+
"gridPos": {
|
|
292
|
+
"h": 8,
|
|
293
|
+
"w": 12,
|
|
294
|
+
"x": 0,
|
|
295
|
+
"y": 14
|
|
296
|
+
},
|
|
297
|
+
"fieldConfig": {
|
|
298
|
+
"defaults": {
|
|
299
|
+
"unit": "reqps",
|
|
300
|
+
"min": 0,
|
|
301
|
+
"custom": {
|
|
302
|
+
"drawStyle": "line",
|
|
303
|
+
"lineWidth": 1,
|
|
304
|
+
"fillOpacity": 10,
|
|
305
|
+
"showPoints": "never",
|
|
306
|
+
"stacking": {
|
|
307
|
+
"mode": "none"
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
},
|
|
311
|
+
"overrides": []
|
|
312
|
+
},
|
|
313
|
+
"options": {
|
|
314
|
+
"legend": {
|
|
315
|
+
"displayMode": "table",
|
|
316
|
+
"placement": "bottom",
|
|
317
|
+
"calcs": [
|
|
318
|
+
"lastNotNull",
|
|
319
|
+
"max"
|
|
320
|
+
]
|
|
321
|
+
},
|
|
322
|
+
"tooltip": {
|
|
323
|
+
"mode": "multi",
|
|
324
|
+
"sort": "desc"
|
|
325
|
+
}
|
|
326
|
+
},
|
|
327
|
+
"targets": [
|
|
328
|
+
{
|
|
329
|
+
"datasource": {
|
|
330
|
+
"type": "prometheus",
|
|
331
|
+
"uid": "${datasource}"
|
|
332
|
+
},
|
|
333
|
+
"expr": "sum(rate(pubsub_repeater_repeater_${repeater}_forwarded{job=~\"$job\"}[5m]))",
|
|
334
|
+
"legendFormat": "forwarded/s",
|
|
335
|
+
"refId": "A"
|
|
336
|
+
}
|
|
337
|
+
]
|
|
338
|
+
},
|
|
339
|
+
{
|
|
340
|
+
"type": "timeseries",
|
|
341
|
+
"title": "Drops / sec by reason",
|
|
342
|
+
"description": "Stacked, because the shape that matters is one band dominating. notMember is nothing enrolled. noDevicePath is a wrong membership path or an unfiltered topic carrying several message shapes. deviceNotFound is a mapping pointed at the wrong source project. filtered is msgFilters, before any device lookup. rsrcCopyFailed is a resource that never landed on the target.",
|
|
343
|
+
"id": 6,
|
|
344
|
+
"datasource": {
|
|
345
|
+
"type": "prometheus",
|
|
346
|
+
"uid": "${datasource}"
|
|
347
|
+
},
|
|
348
|
+
"gridPos": {
|
|
349
|
+
"h": 8,
|
|
350
|
+
"w": 12,
|
|
351
|
+
"x": 12,
|
|
352
|
+
"y": 14
|
|
353
|
+
},
|
|
354
|
+
"fieldConfig": {
|
|
355
|
+
"defaults": {
|
|
356
|
+
"unit": "reqps",
|
|
357
|
+
"min": 0,
|
|
358
|
+
"custom": {
|
|
359
|
+
"drawStyle": "line",
|
|
360
|
+
"lineWidth": 1,
|
|
361
|
+
"fillOpacity": 10,
|
|
362
|
+
"showPoints": "never",
|
|
363
|
+
"stacking": {
|
|
364
|
+
"mode": "normal"
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
},
|
|
368
|
+
"overrides": []
|
|
369
|
+
},
|
|
370
|
+
"options": {
|
|
371
|
+
"legend": {
|
|
372
|
+
"displayMode": "table",
|
|
373
|
+
"placement": "bottom",
|
|
374
|
+
"calcs": [
|
|
375
|
+
"lastNotNull",
|
|
376
|
+
"max"
|
|
377
|
+
]
|
|
378
|
+
},
|
|
379
|
+
"tooltip": {
|
|
380
|
+
"mode": "multi",
|
|
381
|
+
"sort": "desc"
|
|
382
|
+
}
|
|
383
|
+
},
|
|
384
|
+
"targets": [
|
|
385
|
+
{
|
|
386
|
+
"datasource": {
|
|
387
|
+
"type": "prometheus",
|
|
388
|
+
"uid": "${datasource}"
|
|
389
|
+
},
|
|
390
|
+
"expr": "sum(rate(pubsub_repeater_repeater_${repeater}_dropped_notMember{job=~\"$job\"}[5m]))",
|
|
391
|
+
"legendFormat": "notMember",
|
|
392
|
+
"refId": "A"
|
|
393
|
+
},
|
|
394
|
+
{
|
|
395
|
+
"datasource": {
|
|
396
|
+
"type": "prometheus",
|
|
397
|
+
"uid": "${datasource}"
|
|
398
|
+
},
|
|
399
|
+
"expr": "sum(rate(pubsub_repeater_repeater_${repeater}_dropped_noDevicePath{job=~\"$job\"}[5m]))",
|
|
400
|
+
"legendFormat": "noDevicePath",
|
|
401
|
+
"refId": "B"
|
|
402
|
+
},
|
|
403
|
+
{
|
|
404
|
+
"datasource": {
|
|
405
|
+
"type": "prometheus",
|
|
406
|
+
"uid": "${datasource}"
|
|
407
|
+
},
|
|
408
|
+
"expr": "sum(rate(pubsub_repeater_repeater_${repeater}_dropped_deviceNotFound{job=~\"$job\"}[5m]))",
|
|
409
|
+
"legendFormat": "deviceNotFound",
|
|
410
|
+
"refId": "C"
|
|
411
|
+
},
|
|
412
|
+
{
|
|
413
|
+
"datasource": {
|
|
414
|
+
"type": "prometheus",
|
|
415
|
+
"uid": "${datasource}"
|
|
416
|
+
},
|
|
417
|
+
"expr": "sum(rate(pubsub_repeater_repeater_${repeater}_dropped_filtered{job=~\"$job\"}[5m]))",
|
|
418
|
+
"legendFormat": "filtered",
|
|
419
|
+
"refId": "D"
|
|
420
|
+
},
|
|
421
|
+
{
|
|
422
|
+
"datasource": {
|
|
423
|
+
"type": "prometheus",
|
|
424
|
+
"uid": "${datasource}"
|
|
425
|
+
},
|
|
426
|
+
"expr": "sum(rate(pubsub_repeater_repeater_${repeater}_dropped_rsrcCopyFailed{job=~\"$job\"}[5m]))",
|
|
427
|
+
"legendFormat": "rsrcCopyFailed",
|
|
428
|
+
"refId": "E"
|
|
429
|
+
}
|
|
430
|
+
]
|
|
431
|
+
},
|
|
432
|
+
{
|
|
433
|
+
"type": "timeseries",
|
|
434
|
+
"title": "Outcome mix",
|
|
435
|
+
"description": "The same data as 100 percent of the messages this mapping looked at. Useful when absolute rates are low: the mix says whether a mapping is healthy-but-quiet or busy-but-dropping.",
|
|
436
|
+
"id": 7,
|
|
437
|
+
"datasource": {
|
|
438
|
+
"type": "prometheus",
|
|
439
|
+
"uid": "${datasource}"
|
|
440
|
+
},
|
|
441
|
+
"gridPos": {
|
|
442
|
+
"h": 8,
|
|
443
|
+
"w": 12,
|
|
444
|
+
"x": 0,
|
|
445
|
+
"y": 22
|
|
446
|
+
},
|
|
447
|
+
"fieldConfig": {
|
|
448
|
+
"defaults": {
|
|
449
|
+
"unit": "percentunit",
|
|
450
|
+
"min": 0,
|
|
451
|
+
"custom": {
|
|
452
|
+
"drawStyle": "line",
|
|
453
|
+
"lineWidth": 1,
|
|
454
|
+
"fillOpacity": 10,
|
|
455
|
+
"showPoints": "never",
|
|
456
|
+
"stacking": {
|
|
457
|
+
"mode": "percent"
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
},
|
|
461
|
+
"overrides": []
|
|
462
|
+
},
|
|
463
|
+
"options": {
|
|
464
|
+
"legend": {
|
|
465
|
+
"displayMode": "table",
|
|
466
|
+
"placement": "bottom",
|
|
467
|
+
"calcs": [
|
|
468
|
+
"lastNotNull",
|
|
469
|
+
"max"
|
|
470
|
+
]
|
|
471
|
+
},
|
|
472
|
+
"tooltip": {
|
|
473
|
+
"mode": "multi",
|
|
474
|
+
"sort": "desc"
|
|
475
|
+
}
|
|
476
|
+
},
|
|
477
|
+
"targets": [
|
|
478
|
+
{
|
|
479
|
+
"datasource": {
|
|
480
|
+
"type": "prometheus",
|
|
481
|
+
"uid": "${datasource}"
|
|
482
|
+
},
|
|
483
|
+
"expr": "sum(rate(pubsub_repeater_repeater_${repeater}_forwarded{job=~\"$job\"}[5m]))",
|
|
484
|
+
"legendFormat": "forwarded",
|
|
485
|
+
"refId": "A"
|
|
486
|
+
},
|
|
487
|
+
{
|
|
488
|
+
"datasource": {
|
|
489
|
+
"type": "prometheus",
|
|
490
|
+
"uid": "${datasource}"
|
|
491
|
+
},
|
|
492
|
+
"expr": "sum(rate(pubsub_repeater_repeater_${repeater}_dropped_notMember{job=~\"$job\"}[5m]))",
|
|
493
|
+
"legendFormat": "notMember",
|
|
494
|
+
"refId": "B"
|
|
495
|
+
},
|
|
496
|
+
{
|
|
497
|
+
"datasource": {
|
|
498
|
+
"type": "prometheus",
|
|
499
|
+
"uid": "${datasource}"
|
|
500
|
+
},
|
|
501
|
+
"expr": "sum(rate(pubsub_repeater_repeater_${repeater}_dropped_noDevicePath{job=~\"$job\"}[5m]))",
|
|
502
|
+
"legendFormat": "noDevicePath",
|
|
503
|
+
"refId": "C"
|
|
504
|
+
},
|
|
505
|
+
{
|
|
506
|
+
"datasource": {
|
|
507
|
+
"type": "prometheus",
|
|
508
|
+
"uid": "${datasource}"
|
|
509
|
+
},
|
|
510
|
+
"expr": "sum(rate(pubsub_repeater_repeater_${repeater}_dropped_deviceNotFound{job=~\"$job\"}[5m]))",
|
|
511
|
+
"legendFormat": "deviceNotFound",
|
|
512
|
+
"refId": "D"
|
|
513
|
+
},
|
|
514
|
+
{
|
|
515
|
+
"datasource": {
|
|
516
|
+
"type": "prometheus",
|
|
517
|
+
"uid": "${datasource}"
|
|
518
|
+
},
|
|
519
|
+
"expr": "sum(rate(pubsub_repeater_repeater_${repeater}_dropped_filtered{job=~\"$job\"}[5m]))",
|
|
520
|
+
"legendFormat": "filtered",
|
|
521
|
+
"refId": "E"
|
|
522
|
+
},
|
|
523
|
+
{
|
|
524
|
+
"datasource": {
|
|
525
|
+
"type": "prometheus",
|
|
526
|
+
"uid": "${datasource}"
|
|
527
|
+
},
|
|
528
|
+
"expr": "sum(rate(pubsub_repeater_repeater_${repeater}_dropped_rsrcCopyFailed{job=~\"$job\"}[5m]))",
|
|
529
|
+
"legendFormat": "rsrcCopyFailed",
|
|
530
|
+
"refId": "F"
|
|
531
|
+
}
|
|
532
|
+
]
|
|
533
|
+
},
|
|
534
|
+
{
|
|
535
|
+
"type": "timeseries",
|
|
536
|
+
"title": "Forward ratio",
|
|
537
|
+
"description": "forwarded divided by everything the mapping saw. A stable ratio well under 1 is normal: it is every message from every store not enrolled. What matters is a sudden change with no enrollment behind it.",
|
|
538
|
+
"id": 8,
|
|
539
|
+
"datasource": {
|
|
540
|
+
"type": "prometheus",
|
|
541
|
+
"uid": "${datasource}"
|
|
542
|
+
},
|
|
543
|
+
"gridPos": {
|
|
544
|
+
"h": 8,
|
|
545
|
+
"w": 12,
|
|
546
|
+
"x": 12,
|
|
547
|
+
"y": 22
|
|
548
|
+
},
|
|
549
|
+
"fieldConfig": {
|
|
550
|
+
"defaults": {
|
|
551
|
+
"unit": "percentunit",
|
|
552
|
+
"min": 0,
|
|
553
|
+
"custom": {
|
|
554
|
+
"drawStyle": "line",
|
|
555
|
+
"lineWidth": 1,
|
|
556
|
+
"fillOpacity": 10,
|
|
557
|
+
"showPoints": "never",
|
|
558
|
+
"stacking": {
|
|
559
|
+
"mode": "none"
|
|
560
|
+
}
|
|
561
|
+
},
|
|
562
|
+
"max": 1
|
|
563
|
+
},
|
|
564
|
+
"overrides": []
|
|
565
|
+
},
|
|
566
|
+
"options": {
|
|
567
|
+
"legend": {
|
|
568
|
+
"displayMode": "table",
|
|
569
|
+
"placement": "bottom",
|
|
570
|
+
"calcs": [
|
|
571
|
+
"lastNotNull",
|
|
572
|
+
"max"
|
|
573
|
+
]
|
|
574
|
+
},
|
|
575
|
+
"tooltip": {
|
|
576
|
+
"mode": "multi",
|
|
577
|
+
"sort": "desc"
|
|
578
|
+
}
|
|
579
|
+
},
|
|
580
|
+
"targets": [
|
|
581
|
+
{
|
|
582
|
+
"datasource": {
|
|
583
|
+
"type": "prometheus",
|
|
584
|
+
"uid": "${datasource}"
|
|
585
|
+
},
|
|
586
|
+
"expr": "(sum(rate(pubsub_repeater_repeater_${repeater}_forwarded{job=~\"$job\"}[5m])) or vector(0)) / clamp_min((sum(rate(pubsub_repeater_repeater_${repeater}_forwarded{job=~\"$job\"}[5m])) or vector(0)) + (sum(rate(pubsub_repeater_repeater_${repeater}_dropped_notMember{job=~\"$job\"}[5m])) or vector(0)) + (sum(rate(pubsub_repeater_repeater_${repeater}_dropped_noDevicePath{job=~\"$job\"}[5m])) or vector(0)) + (sum(rate(pubsub_repeater_repeater_${repeater}_dropped_deviceNotFound{job=~\"$job\"}[5m])) or vector(0)) + (sum(rate(pubsub_repeater_repeater_${repeater}_dropped_filtered{job=~\"$job\"}[5m])) or vector(0)) + (sum(rate(pubsub_repeater_repeater_${repeater}_dropped_rsrcCopyFailed{job=~\"$job\"}[5m])) or vector(0)), 1e-9)",
|
|
587
|
+
"legendFormat": "forwarded share",
|
|
588
|
+
"refId": "A"
|
|
589
|
+
}
|
|
590
|
+
]
|
|
591
|
+
},
|
|
592
|
+
{
|
|
593
|
+
"type": "row",
|
|
594
|
+
"title": "Selected mapping: resource copying",
|
|
595
|
+
"id": 9,
|
|
596
|
+
"gridPos": {
|
|
597
|
+
"h": 1,
|
|
598
|
+
"w": 24,
|
|
599
|
+
"x": 0,
|
|
600
|
+
"y": 30
|
|
601
|
+
},
|
|
602
|
+
"collapsed": false,
|
|
603
|
+
"panels": []
|
|
604
|
+
},
|
|
605
|
+
{
|
|
606
|
+
"type": "timeseries",
|
|
607
|
+
"title": "Resource outcomes / sec",
|
|
608
|
+
"description": "Counted per resource, not per message. skipped is a resource this service decided not to copy: no network path, or no device on one side. failed is one it tried and could not, classically the missing Leverege Storage Object Admin grant in the source project. skipped climbing while forwarded stays healthy means a wrong rsrcDeviceNetworkPathPath on a mapping that opted out of requireRsrcCopy, which is invisible everywhere else.",
|
|
609
|
+
"id": 10,
|
|
610
|
+
"datasource": {
|
|
611
|
+
"type": "prometheus",
|
|
612
|
+
"uid": "${datasource}"
|
|
613
|
+
},
|
|
614
|
+
"gridPos": {
|
|
615
|
+
"h": 8,
|
|
616
|
+
"w": 12,
|
|
617
|
+
"x": 0,
|
|
618
|
+
"y": 31
|
|
619
|
+
},
|
|
620
|
+
"fieldConfig": {
|
|
621
|
+
"defaults": {
|
|
622
|
+
"unit": "reqps",
|
|
623
|
+
"min": 0,
|
|
624
|
+
"custom": {
|
|
625
|
+
"drawStyle": "line",
|
|
626
|
+
"lineWidth": 1,
|
|
627
|
+
"fillOpacity": 10,
|
|
628
|
+
"showPoints": "never",
|
|
629
|
+
"stacking": {
|
|
630
|
+
"mode": "normal"
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
},
|
|
634
|
+
"overrides": []
|
|
635
|
+
},
|
|
636
|
+
"options": {
|
|
637
|
+
"legend": {
|
|
638
|
+
"displayMode": "table",
|
|
639
|
+
"placement": "bottom",
|
|
640
|
+
"calcs": [
|
|
641
|
+
"lastNotNull",
|
|
642
|
+
"max"
|
|
643
|
+
]
|
|
644
|
+
},
|
|
645
|
+
"tooltip": {
|
|
646
|
+
"mode": "multi",
|
|
647
|
+
"sort": "desc"
|
|
648
|
+
}
|
|
649
|
+
},
|
|
650
|
+
"targets": [
|
|
651
|
+
{
|
|
652
|
+
"datasource": {
|
|
653
|
+
"type": "prometheus",
|
|
654
|
+
"uid": "${datasource}"
|
|
655
|
+
},
|
|
656
|
+
"expr": "sum(rate(pubsub_repeater_repeater_${repeater}_rsrc_copied{job=~\"$job\"}[5m]))",
|
|
657
|
+
"legendFormat": "copied",
|
|
658
|
+
"refId": "A"
|
|
659
|
+
},
|
|
660
|
+
{
|
|
661
|
+
"datasource": {
|
|
662
|
+
"type": "prometheus",
|
|
663
|
+
"uid": "${datasource}"
|
|
664
|
+
},
|
|
665
|
+
"expr": "sum(rate(pubsub_repeater_repeater_${repeater}_rsrc_failed{job=~\"$job\"}[5m]))",
|
|
666
|
+
"legendFormat": "failed",
|
|
667
|
+
"refId": "B"
|
|
668
|
+
},
|
|
669
|
+
{
|
|
670
|
+
"datasource": {
|
|
671
|
+
"type": "prometheus",
|
|
672
|
+
"uid": "${datasource}"
|
|
673
|
+
},
|
|
674
|
+
"expr": "sum(rate(pubsub_repeater_repeater_${repeater}_rsrc_skipped{job=~\"$job\"}[5m]))",
|
|
675
|
+
"legendFormat": "skipped",
|
|
676
|
+
"refId": "C"
|
|
677
|
+
}
|
|
678
|
+
]
|
|
679
|
+
},
|
|
680
|
+
{
|
|
681
|
+
"type": "timeseries",
|
|
682
|
+
"title": "Messages dropped for resources / sec",
|
|
683
|
+
"description": "Messages dropped because requireRsrcCopy was set and at least one resource was not on the target by the deadline. Counted once per message, unlike the per-resource counters beside it. Absent on a mapping without ensureRsrcs, which copies nothing.",
|
|
684
|
+
"id": 11,
|
|
685
|
+
"datasource": {
|
|
686
|
+
"type": "prometheus",
|
|
687
|
+
"uid": "${datasource}"
|
|
688
|
+
},
|
|
689
|
+
"gridPos": {
|
|
690
|
+
"h": 8,
|
|
691
|
+
"w": 6,
|
|
692
|
+
"x": 12,
|
|
693
|
+
"y": 31
|
|
694
|
+
},
|
|
695
|
+
"fieldConfig": {
|
|
696
|
+
"defaults": {
|
|
697
|
+
"unit": "reqps",
|
|
698
|
+
"min": 0,
|
|
699
|
+
"custom": {
|
|
700
|
+
"drawStyle": "line",
|
|
701
|
+
"lineWidth": 1,
|
|
702
|
+
"fillOpacity": 10,
|
|
703
|
+
"showPoints": "never",
|
|
704
|
+
"stacking": {
|
|
705
|
+
"mode": "none"
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
},
|
|
709
|
+
"overrides": []
|
|
710
|
+
},
|
|
711
|
+
"options": {
|
|
712
|
+
"legend": {
|
|
713
|
+
"displayMode": "table",
|
|
714
|
+
"placement": "bottom",
|
|
715
|
+
"calcs": [
|
|
716
|
+
"lastNotNull",
|
|
717
|
+
"max"
|
|
718
|
+
]
|
|
719
|
+
},
|
|
720
|
+
"tooltip": {
|
|
721
|
+
"mode": "multi",
|
|
722
|
+
"sort": "desc"
|
|
723
|
+
}
|
|
724
|
+
},
|
|
725
|
+
"targets": [
|
|
726
|
+
{
|
|
727
|
+
"datasource": {
|
|
728
|
+
"type": "prometheus",
|
|
729
|
+
"uid": "${datasource}"
|
|
730
|
+
},
|
|
731
|
+
"expr": "sum(rate(pubsub_repeater_repeater_${repeater}_dropped_rsrcCopyFailed{job=~\"$job\"}[5m]))",
|
|
732
|
+
"legendFormat": "rsrcCopyFailed/s",
|
|
733
|
+
"refId": "A"
|
|
734
|
+
}
|
|
735
|
+
]
|
|
736
|
+
},
|
|
737
|
+
{
|
|
738
|
+
"type": "stat",
|
|
739
|
+
"title": "Resource copy success ratio",
|
|
740
|
+
"description": "copied over everything attempted or skipped, over the selected range. Under requireRsrcCopy this tracks the forward ratio closely, because a resource that does not land takes its message down with it.",
|
|
741
|
+
"id": 12,
|
|
742
|
+
"datasource": {
|
|
743
|
+
"type": "prometheus",
|
|
744
|
+
"uid": "${datasource}"
|
|
745
|
+
},
|
|
746
|
+
"gridPos": {
|
|
747
|
+
"h": 8,
|
|
748
|
+
"w": 6,
|
|
749
|
+
"x": 18,
|
|
750
|
+
"y": 31
|
|
751
|
+
},
|
|
752
|
+
"fieldConfig": {
|
|
753
|
+
"defaults": {
|
|
754
|
+
"unit": "percentunit",
|
|
755
|
+
"min": 0
|
|
756
|
+
},
|
|
757
|
+
"overrides": []
|
|
758
|
+
},
|
|
759
|
+
"options": {
|
|
760
|
+
"reduceOptions": {
|
|
761
|
+
"calcs": [
|
|
762
|
+
"lastNotNull"
|
|
763
|
+
],
|
|
764
|
+
"fields": "",
|
|
765
|
+
"values": false
|
|
766
|
+
},
|
|
767
|
+
"textMode": "auto",
|
|
768
|
+
"colorMode": "value",
|
|
769
|
+
"graphMode": "area"
|
|
770
|
+
},
|
|
771
|
+
"targets": [
|
|
772
|
+
{
|
|
773
|
+
"datasource": {
|
|
774
|
+
"type": "prometheus",
|
|
775
|
+
"uid": "${datasource}"
|
|
776
|
+
},
|
|
777
|
+
"expr": "(sum(increase(pubsub_repeater_repeater_${repeater}_rsrc_copied{job=~\"$job\"}[$__range])) or vector(0)) / clamp_min((sum(increase(pubsub_repeater_repeater_${repeater}_rsrc_copied{job=~\"$job\"}[$__range])) or vector(0)) + (sum(increase(pubsub_repeater_repeater_${repeater}_rsrc_failed{job=~\"$job\"}[$__range])) or vector(0)) + (sum(increase(pubsub_repeater_repeater_${repeater}_rsrc_skipped{job=~\"$job\"}[$__range])) or vector(0)), 1e-9)",
|
|
778
|
+
"legendFormat": "copied share",
|
|
779
|
+
"refId": "A"
|
|
780
|
+
}
|
|
781
|
+
]
|
|
782
|
+
},
|
|
783
|
+
{
|
|
784
|
+
"type": "row",
|
|
785
|
+
"title": "Service health: all mappings together",
|
|
786
|
+
"id": 13,
|
|
787
|
+
"gridPos": {
|
|
788
|
+
"h": 1,
|
|
789
|
+
"w": 24,
|
|
790
|
+
"x": 0,
|
|
791
|
+
"y": 39
|
|
792
|
+
},
|
|
793
|
+
"collapsed": false,
|
|
794
|
+
"panels": []
|
|
795
|
+
},
|
|
796
|
+
{
|
|
797
|
+
"type": "timeseries",
|
|
798
|
+
"title": "Inbound vs accounted / sec",
|
|
799
|
+
"description": "msg_q_processed_count is every message the readers took off their subscriptions, service-wide with no per-mapping breakdown. Compared against forwarded plus drops summed over ALL mappings, it is the only way to see the one outcome this service does not count: a message with nothing at correlationIdPath is dropped with a warn and incremented nowhere. A persistent gap between the two lines is that drop, or a publish that threw. Check the pod logs. The accounted line is a hand-rolled offset difference rather than rate(), because rate() strips __name__ and a multi-metric-name selector then collapses into duplicate labelsets, which Prometheus rejects outright. It is clamped at 0 so a pod restart reads as a flat line rather than a negative spike.",
|
|
800
|
+
"id": 14,
|
|
801
|
+
"datasource": {
|
|
802
|
+
"type": "prometheus",
|
|
803
|
+
"uid": "${datasource}"
|
|
804
|
+
},
|
|
805
|
+
"gridPos": {
|
|
806
|
+
"h": 8,
|
|
807
|
+
"w": 12,
|
|
808
|
+
"x": 0,
|
|
809
|
+
"y": 40
|
|
810
|
+
},
|
|
811
|
+
"fieldConfig": {
|
|
812
|
+
"defaults": {
|
|
813
|
+
"unit": "reqps",
|
|
814
|
+
"min": 0,
|
|
815
|
+
"custom": {
|
|
816
|
+
"drawStyle": "line",
|
|
817
|
+
"lineWidth": 1,
|
|
818
|
+
"fillOpacity": 10,
|
|
819
|
+
"showPoints": "never",
|
|
820
|
+
"stacking": {
|
|
821
|
+
"mode": "none"
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
},
|
|
825
|
+
"overrides": []
|
|
826
|
+
},
|
|
827
|
+
"options": {
|
|
828
|
+
"legend": {
|
|
829
|
+
"displayMode": "table",
|
|
830
|
+
"placement": "bottom",
|
|
831
|
+
"calcs": [
|
|
832
|
+
"lastNotNull",
|
|
833
|
+
"max"
|
|
834
|
+
]
|
|
835
|
+
},
|
|
836
|
+
"tooltip": {
|
|
837
|
+
"mode": "multi",
|
|
838
|
+
"sort": "desc"
|
|
839
|
+
}
|
|
840
|
+
},
|
|
841
|
+
"targets": [
|
|
842
|
+
{
|
|
843
|
+
"datasource": {
|
|
844
|
+
"type": "prometheus",
|
|
845
|
+
"uid": "${datasource}"
|
|
846
|
+
},
|
|
847
|
+
"expr": "sum(rate(pubsub_repeater_msg_q_processed_count{job=~\"$job\"}[5m]))",
|
|
848
|
+
"legendFormat": "processed/s (inbound)",
|
|
849
|
+
"refId": "A"
|
|
850
|
+
},
|
|
851
|
+
{
|
|
852
|
+
"datasource": {
|
|
853
|
+
"type": "prometheus",
|
|
854
|
+
"uid": "${datasource}"
|
|
855
|
+
},
|
|
856
|
+
"expr": "clamp_min((sum({__name__=~\"pubsub_repeater_repeater_.*_forwarded\", job=~\"$job\"}) - sum({__name__=~\"pubsub_repeater_repeater_.*_forwarded\", job=~\"$job\"} offset 5m)) + (sum({__name__=~\"pubsub_repeater_repeater_.*_dropped_.*\", job=~\"$job\"}) - sum({__name__=~\"pubsub_repeater_repeater_.*_dropped_.*\", job=~\"$job\"} offset 5m)), 0) / 300",
|
|
857
|
+
"legendFormat": "forwarded + dropped/s (accounted)",
|
|
858
|
+
"refId": "B"
|
|
859
|
+
}
|
|
860
|
+
]
|
|
861
|
+
},
|
|
862
|
+
{
|
|
863
|
+
"type": "timeseries",
|
|
864
|
+
"title": "Event loop lag",
|
|
865
|
+
"description": "The ordered queues are in-memory and drained on this event loop, so lag here is directly messages sitting behind a correlation id. Watch it alongside the awaited resource copy: requireRsrcCopy holds a message for up to rsrcCopyAwaitTimeout, and with ensureOrdering everything behind that correlation id waits with it.",
|
|
866
|
+
"id": 15,
|
|
867
|
+
"datasource": {
|
|
868
|
+
"type": "prometheus",
|
|
869
|
+
"uid": "${datasource}"
|
|
870
|
+
},
|
|
871
|
+
"gridPos": {
|
|
872
|
+
"h": 8,
|
|
873
|
+
"w": 12,
|
|
874
|
+
"x": 12,
|
|
875
|
+
"y": 40
|
|
876
|
+
},
|
|
877
|
+
"fieldConfig": {
|
|
878
|
+
"defaults": {
|
|
879
|
+
"unit": "s",
|
|
880
|
+
"min": 0,
|
|
881
|
+
"custom": {
|
|
882
|
+
"drawStyle": "line",
|
|
883
|
+
"lineWidth": 1,
|
|
884
|
+
"fillOpacity": 10,
|
|
885
|
+
"showPoints": "never",
|
|
886
|
+
"stacking": {
|
|
887
|
+
"mode": "none"
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
},
|
|
891
|
+
"overrides": []
|
|
892
|
+
},
|
|
893
|
+
"options": {
|
|
894
|
+
"legend": {
|
|
895
|
+
"displayMode": "table",
|
|
896
|
+
"placement": "bottom",
|
|
897
|
+
"calcs": [
|
|
898
|
+
"lastNotNull",
|
|
899
|
+
"max"
|
|
900
|
+
]
|
|
901
|
+
},
|
|
902
|
+
"tooltip": {
|
|
903
|
+
"mode": "multi",
|
|
904
|
+
"sort": "desc"
|
|
905
|
+
}
|
|
906
|
+
},
|
|
907
|
+
"targets": [
|
|
908
|
+
{
|
|
909
|
+
"datasource": {
|
|
910
|
+
"type": "prometheus",
|
|
911
|
+
"uid": "${datasource}"
|
|
912
|
+
},
|
|
913
|
+
"expr": "max(nodejs_eventloop_lag_p99_seconds{job=~\"$job\"})",
|
|
914
|
+
"legendFormat": "p99",
|
|
915
|
+
"refId": "A"
|
|
916
|
+
},
|
|
917
|
+
{
|
|
918
|
+
"datasource": {
|
|
919
|
+
"type": "prometheus",
|
|
920
|
+
"uid": "${datasource}"
|
|
921
|
+
},
|
|
922
|
+
"expr": "max(nodejs_eventloop_lag_mean_seconds{job=~\"$job\"})",
|
|
923
|
+
"legendFormat": "mean",
|
|
924
|
+
"refId": "B"
|
|
925
|
+
}
|
|
926
|
+
]
|
|
927
|
+
},
|
|
928
|
+
{
|
|
929
|
+
"type": "timeseries",
|
|
930
|
+
"title": "Heap used",
|
|
931
|
+
"description": "Per pod. The in-memory ordered queues have no bound, so a target topic that stops accepting publishes shows up here as heap climbing before it shows up anywhere else.",
|
|
932
|
+
"id": 16,
|
|
933
|
+
"datasource": {
|
|
934
|
+
"type": "prometheus",
|
|
935
|
+
"uid": "${datasource}"
|
|
936
|
+
},
|
|
937
|
+
"gridPos": {
|
|
938
|
+
"h": 8,
|
|
939
|
+
"w": 12,
|
|
940
|
+
"x": 0,
|
|
941
|
+
"y": 48
|
|
942
|
+
},
|
|
943
|
+
"fieldConfig": {
|
|
944
|
+
"defaults": {
|
|
945
|
+
"unit": "bytes",
|
|
946
|
+
"min": 0,
|
|
947
|
+
"custom": {
|
|
948
|
+
"drawStyle": "line",
|
|
949
|
+
"lineWidth": 1,
|
|
950
|
+
"fillOpacity": 10,
|
|
951
|
+
"showPoints": "never",
|
|
952
|
+
"stacking": {
|
|
953
|
+
"mode": "none"
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
},
|
|
957
|
+
"overrides": []
|
|
958
|
+
},
|
|
959
|
+
"options": {
|
|
960
|
+
"legend": {
|
|
961
|
+
"displayMode": "table",
|
|
962
|
+
"placement": "bottom",
|
|
963
|
+
"calcs": [
|
|
964
|
+
"lastNotNull",
|
|
965
|
+
"max"
|
|
966
|
+
]
|
|
967
|
+
},
|
|
968
|
+
"tooltip": {
|
|
969
|
+
"mode": "multi",
|
|
970
|
+
"sort": "desc"
|
|
971
|
+
}
|
|
972
|
+
},
|
|
973
|
+
"targets": [
|
|
974
|
+
{
|
|
975
|
+
"datasource": {
|
|
976
|
+
"type": "prometheus",
|
|
977
|
+
"uid": "${datasource}"
|
|
978
|
+
},
|
|
979
|
+
"expr": "nodejs_heap_size_used_bytes{job=~\"$job\"}",
|
|
980
|
+
"legendFormat": "{{pod}}",
|
|
981
|
+
"refId": "A"
|
|
982
|
+
}
|
|
983
|
+
]
|
|
984
|
+
},
|
|
985
|
+
{
|
|
986
|
+
"type": "timeseries",
|
|
987
|
+
"title": "Pods reporting and CPU",
|
|
988
|
+
"description": "A pod restart loses whatever the in-memory queues were holding. Those messages were acknowledged on receipt, so nothing redelivers them: a restart is silent data loss, visible only as a step here.",
|
|
989
|
+
"id": 17,
|
|
990
|
+
"datasource": {
|
|
991
|
+
"type": "prometheus",
|
|
992
|
+
"uid": "${datasource}"
|
|
993
|
+
},
|
|
994
|
+
"gridPos": {
|
|
995
|
+
"h": 8,
|
|
996
|
+
"w": 12,
|
|
997
|
+
"x": 12,
|
|
998
|
+
"y": 48
|
|
999
|
+
},
|
|
1000
|
+
"fieldConfig": {
|
|
1001
|
+
"defaults": {
|
|
1002
|
+
"unit": "short",
|
|
1003
|
+
"min": 0,
|
|
1004
|
+
"custom": {
|
|
1005
|
+
"drawStyle": "line",
|
|
1006
|
+
"lineWidth": 1,
|
|
1007
|
+
"fillOpacity": 10,
|
|
1008
|
+
"showPoints": "never",
|
|
1009
|
+
"stacking": {
|
|
1010
|
+
"mode": "none"
|
|
1011
|
+
}
|
|
1012
|
+
}
|
|
1013
|
+
},
|
|
1014
|
+
"overrides": []
|
|
1015
|
+
},
|
|
1016
|
+
"options": {
|
|
1017
|
+
"legend": {
|
|
1018
|
+
"displayMode": "table",
|
|
1019
|
+
"placement": "bottom",
|
|
1020
|
+
"calcs": [
|
|
1021
|
+
"lastNotNull",
|
|
1022
|
+
"max"
|
|
1023
|
+
]
|
|
1024
|
+
},
|
|
1025
|
+
"tooltip": {
|
|
1026
|
+
"mode": "multi",
|
|
1027
|
+
"sort": "desc"
|
|
1028
|
+
}
|
|
1029
|
+
},
|
|
1030
|
+
"targets": [
|
|
1031
|
+
{
|
|
1032
|
+
"datasource": {
|
|
1033
|
+
"type": "prometheus",
|
|
1034
|
+
"uid": "${datasource}"
|
|
1035
|
+
},
|
|
1036
|
+
"expr": "count(count by (pod) (nodejs_heap_size_used_bytes{job=~\"$job\"}))",
|
|
1037
|
+
"legendFormat": "pods reporting",
|
|
1038
|
+
"refId": "A"
|
|
1039
|
+
},
|
|
1040
|
+
{
|
|
1041
|
+
"datasource": {
|
|
1042
|
+
"type": "prometheus",
|
|
1043
|
+
"uid": "${datasource}"
|
|
1044
|
+
},
|
|
1045
|
+
"expr": "sum(rate(process_cpu_seconds_total{job=~\"$job\"}[5m]))",
|
|
1046
|
+
"legendFormat": "cpu cores",
|
|
1047
|
+
"refId": "B"
|
|
1048
|
+
}
|
|
1049
|
+
]
|
|
1050
|
+
},
|
|
1051
|
+
{
|
|
1052
|
+
"type": "row",
|
|
1053
|
+
"title": "Imagine API: call volume and outcomes",
|
|
1054
|
+
"id": 18,
|
|
1055
|
+
"gridPos": {
|
|
1056
|
+
"h": 1,
|
|
1057
|
+
"w": 24,
|
|
1058
|
+
"x": 0,
|
|
1059
|
+
"y": 56
|
|
1060
|
+
},
|
|
1061
|
+
"collapsed": false,
|
|
1062
|
+
"panels": []
|
|
1063
|
+
},
|
|
1064
|
+
{
|
|
1065
|
+
"type": "timeseries",
|
|
1066
|
+
"title": "Imagine API calls / sec",
|
|
1067
|
+
"description": "Emitted by @leverege/comms NodeComms.preRequest, one counter per HTTP method, so this is every request the Imagine client made. GET is device resolution and it is counted only on a cache MISS, so it should sit far below the inbound message rate; a GET rate tracking inbound means the device-header cache is not helping (check REPEATER_DEVICE_CACHE_EXPIRES). POST is resource header creation on the target, so it tracks rsrc_copied.",
|
|
1068
|
+
"id": 19,
|
|
1069
|
+
"datasource": {
|
|
1070
|
+
"type": "prometheus",
|
|
1071
|
+
"uid": "${datasource}"
|
|
1072
|
+
},
|
|
1073
|
+
"gridPos": {
|
|
1074
|
+
"h": 8,
|
|
1075
|
+
"w": 8,
|
|
1076
|
+
"x": 0,
|
|
1077
|
+
"y": 57
|
|
1078
|
+
},
|
|
1079
|
+
"fieldConfig": {
|
|
1080
|
+
"defaults": {
|
|
1081
|
+
"unit": "reqps",
|
|
1082
|
+
"min": 0,
|
|
1083
|
+
"custom": {
|
|
1084
|
+
"drawStyle": "line",
|
|
1085
|
+
"lineWidth": 1,
|
|
1086
|
+
"fillOpacity": 10,
|
|
1087
|
+
"showPoints": "never",
|
|
1088
|
+
"stacking": {
|
|
1089
|
+
"mode": "none"
|
|
1090
|
+
}
|
|
1091
|
+
}
|
|
1092
|
+
},
|
|
1093
|
+
"overrides": []
|
|
1094
|
+
},
|
|
1095
|
+
"options": {
|
|
1096
|
+
"legend": {
|
|
1097
|
+
"displayMode": "table",
|
|
1098
|
+
"placement": "bottom",
|
|
1099
|
+
"calcs": [
|
|
1100
|
+
"lastNotNull",
|
|
1101
|
+
"max"
|
|
1102
|
+
]
|
|
1103
|
+
},
|
|
1104
|
+
"tooltip": {
|
|
1105
|
+
"mode": "multi",
|
|
1106
|
+
"sort": "desc"
|
|
1107
|
+
}
|
|
1108
|
+
},
|
|
1109
|
+
"targets": [
|
|
1110
|
+
{
|
|
1111
|
+
"datasource": {
|
|
1112
|
+
"type": "prometheus",
|
|
1113
|
+
"uid": "${datasource}"
|
|
1114
|
+
},
|
|
1115
|
+
"expr": "sum(rate(pubsub_repeater_imagine_api_get{job=~\"$job\"}[5m])) or vector(0)",
|
|
1116
|
+
"legendFormat": "get/s (device lookups)",
|
|
1117
|
+
"refId": "A"
|
|
1118
|
+
},
|
|
1119
|
+
{
|
|
1120
|
+
"datasource": {
|
|
1121
|
+
"type": "prometheus",
|
|
1122
|
+
"uid": "${datasource}"
|
|
1123
|
+
},
|
|
1124
|
+
"expr": "sum(rate(pubsub_repeater_imagine_api_post{job=~\"$job\"}[5m])) or vector(0)",
|
|
1125
|
+
"legendFormat": "post/s (rsrc creates)",
|
|
1126
|
+
"refId": "B"
|
|
1127
|
+
}
|
|
1128
|
+
]
|
|
1129
|
+
},
|
|
1130
|
+
{
|
|
1131
|
+
"type": "timeseries",
|
|
1132
|
+
"title": "Imagine API errors / sec",
|
|
1133
|
+
"description": "NodeComms.postRequest increments <method>_error whenever the response is missing or not ok, so this is EVERY non-2xx, not only transport failures. Two consequences. A 404 from a device lookup that legitimately found nothing lands here, so a healthy get_error floor is normal and should track dropped_deviceNotFound rather than alarm you. And 401 or 403 here is the Imagine API options secret being wrong or expired, which otherwise surfaces only as every message dropping as deviceNotFound. The base method counter is incremented for the same request too, so errors are a subset of the calls panel, never additional to it. Expect a nonzero post_error floor by design: copyRsrcOnce creates the target resource header first, and resource-server answers an existing header with a 400 whose message says already exists. RepeaterProcessor treats that as the success it describes, which is what makes the copy retry idempotent, but the comms lib has already counted the 400 here. So post_error tracking rsrc_copied at a low ratio is the retry path working, not a fault.",
|
|
1134
|
+
"id": 20,
|
|
1135
|
+
"datasource": {
|
|
1136
|
+
"type": "prometheus",
|
|
1137
|
+
"uid": "${datasource}"
|
|
1138
|
+
},
|
|
1139
|
+
"gridPos": {
|
|
1140
|
+
"h": 8,
|
|
1141
|
+
"w": 8,
|
|
1142
|
+
"x": 8,
|
|
1143
|
+
"y": 57
|
|
1144
|
+
},
|
|
1145
|
+
"fieldConfig": {
|
|
1146
|
+
"defaults": {
|
|
1147
|
+
"unit": "reqps",
|
|
1148
|
+
"min": 0,
|
|
1149
|
+
"custom": {
|
|
1150
|
+
"drawStyle": "line",
|
|
1151
|
+
"lineWidth": 1,
|
|
1152
|
+
"fillOpacity": 10,
|
|
1153
|
+
"showPoints": "never",
|
|
1154
|
+
"stacking": {
|
|
1155
|
+
"mode": "none"
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
},
|
|
1159
|
+
"overrides": []
|
|
1160
|
+
},
|
|
1161
|
+
"options": {
|
|
1162
|
+
"legend": {
|
|
1163
|
+
"displayMode": "table",
|
|
1164
|
+
"placement": "bottom",
|
|
1165
|
+
"calcs": [
|
|
1166
|
+
"lastNotNull",
|
|
1167
|
+
"max"
|
|
1168
|
+
]
|
|
1169
|
+
},
|
|
1170
|
+
"tooltip": {
|
|
1171
|
+
"mode": "multi",
|
|
1172
|
+
"sort": "desc"
|
|
1173
|
+
}
|
|
1174
|
+
},
|
|
1175
|
+
"targets": [
|
|
1176
|
+
{
|
|
1177
|
+
"datasource": {
|
|
1178
|
+
"type": "prometheus",
|
|
1179
|
+
"uid": "${datasource}"
|
|
1180
|
+
},
|
|
1181
|
+
"expr": "sum(rate(pubsub_repeater_imagine_api_get_error{job=~\"$job\"}[5m])) or vector(0)",
|
|
1182
|
+
"legendFormat": "get errors/s",
|
|
1183
|
+
"refId": "A"
|
|
1184
|
+
},
|
|
1185
|
+
{
|
|
1186
|
+
"datasource": {
|
|
1187
|
+
"type": "prometheus",
|
|
1188
|
+
"uid": "${datasource}"
|
|
1189
|
+
},
|
|
1190
|
+
"expr": "sum(rate(pubsub_repeater_imagine_api_post_error{job=~\"$job\"}[5m])) or vector(0)",
|
|
1191
|
+
"legendFormat": "post errors/s",
|
|
1192
|
+
"refId": "B"
|
|
1193
|
+
}
|
|
1194
|
+
]
|
|
1195
|
+
},
|
|
1196
|
+
{
|
|
1197
|
+
"type": "stat",
|
|
1198
|
+
"title": "Imagine API error share",
|
|
1199
|
+
"description": "Errors over all calls, across the selected range. Reads 0 while no error counter exists at all, which is the healthy state: the _error counters are created lazily on the first failure, so they are absent rather than zero until something fails. A step change here is worth more than the absolute value, since the floor is whatever the legitimate 404 rate happens to be.",
|
|
1200
|
+
"id": 21,
|
|
1201
|
+
"datasource": {
|
|
1202
|
+
"type": "prometheus",
|
|
1203
|
+
"uid": "${datasource}"
|
|
1204
|
+
},
|
|
1205
|
+
"gridPos": {
|
|
1206
|
+
"h": 8,
|
|
1207
|
+
"w": 8,
|
|
1208
|
+
"x": 16,
|
|
1209
|
+
"y": 57
|
|
1210
|
+
},
|
|
1211
|
+
"fieldConfig": {
|
|
1212
|
+
"defaults": {
|
|
1213
|
+
"unit": "percentunit",
|
|
1214
|
+
"min": 0
|
|
1215
|
+
},
|
|
1216
|
+
"overrides": []
|
|
1217
|
+
},
|
|
1218
|
+
"options": {
|
|
1219
|
+
"reduceOptions": {
|
|
1220
|
+
"calcs": [
|
|
1221
|
+
"lastNotNull"
|
|
1222
|
+
],
|
|
1223
|
+
"fields": "",
|
|
1224
|
+
"values": false
|
|
1225
|
+
},
|
|
1226
|
+
"textMode": "auto",
|
|
1227
|
+
"colorMode": "value",
|
|
1228
|
+
"graphMode": "area"
|
|
1229
|
+
},
|
|
1230
|
+
"targets": [
|
|
1231
|
+
{
|
|
1232
|
+
"datasource": {
|
|
1233
|
+
"type": "prometheus",
|
|
1234
|
+
"uid": "${datasource}"
|
|
1235
|
+
},
|
|
1236
|
+
"expr": "((sum(increase(pubsub_repeater_imagine_api_get_error{job=~\"$job\"}[$__range])) or vector(0)) + (sum(increase(pubsub_repeater_imagine_api_post_error{job=~\"$job\"}[$__range])) or vector(0))) / clamp_min((sum(increase(pubsub_repeater_imagine_api_get{job=~\"$job\"}[$__range])) or vector(0)) + (sum(increase(pubsub_repeater_imagine_api_post{job=~\"$job\"}[$__range])) or vector(0)), 1e-9)",
|
|
1237
|
+
"legendFormat": "error share",
|
|
1238
|
+
"refId": "A"
|
|
1239
|
+
}
|
|
1240
|
+
]
|
|
1241
|
+
}
|
|
1242
|
+
]
|
|
1243
|
+
}
|