@leverege/build-tools 2.119.0 → 2.121.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,1613 @@
|
|
|
1
|
+
{
|
|
2
|
+
"uid": "pitcrew-ingestor",
|
|
3
|
+
"title": "PitCrew Ingestor",
|
|
4
|
+
"description": "Pitcrew Ingestor. Metric names are not declared in one list the way the pubsub-repeater's are - they are M.inc() call sites spread across src/processors, src/schedulers and src/utils - so cross-check panels against a live scrape of :5111/metrics as well as against grep. Four things to know. (1) @leverege/metrics has NO label support, so every dimension is a separate metric NAME; there is no type or method label to group by. (2) Counters are three-deep and nested: imagine_request_total, then a verb counter, then a leaf counter, all incremented on the same call - a totals line is the sum of the lines under it, never additional to them. The same holds for pubsub_message_total over the per-type counters. (3) A counter that has never been incremented does not exist in Prometheus at all, so error panels are empty when healthy, and any panel that ADDS counters must guard every term with or vector(0) or a missing operand blanks the whole panel. (4) Branch-dependent counters: pubsub_image_collection_message_total exists only on the PIT-2434 line and pubsub_state_message_total / imagine_set_value_* only on development, so one of each pair is expected to read No data. The row that earns this dashboard is Device cache: getCachedDevice hardcodes a null interface, so every cache miss is a blueprint-less api-server read fanned out across all Elasticsearch shards.",
|
|
5
|
+
"tags": [
|
|
6
|
+
"pitcrew-ingestor",
|
|
7
|
+
"pitcrew",
|
|
8
|
+
"ingestor"
|
|
9
|
+
],
|
|
10
|
+
"timezone": "",
|
|
11
|
+
"editable": true,
|
|
12
|
+
"graphTooltip": 1,
|
|
13
|
+
"refresh": "1m",
|
|
14
|
+
"schemaVersion": 39,
|
|
15
|
+
"version": 1,
|
|
16
|
+
"time": {
|
|
17
|
+
"from": "now-6h",
|
|
18
|
+
"to": "now"
|
|
19
|
+
},
|
|
20
|
+
"timepicker": {},
|
|
21
|
+
"annotations": {
|
|
22
|
+
"list": []
|
|
23
|
+
},
|
|
24
|
+
"links": [],
|
|
25
|
+
"templating": {
|
|
26
|
+
"list": [
|
|
27
|
+
{
|
|
28
|
+
"type": "datasource",
|
|
29
|
+
"name": "datasource",
|
|
30
|
+
"label": "Data source",
|
|
31
|
+
"query": "prometheus",
|
|
32
|
+
"current": {},
|
|
33
|
+
"hide": 0
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
"type": "query",
|
|
37
|
+
"name": "job",
|
|
38
|
+
"label": "Job",
|
|
39
|
+
"datasource": {
|
|
40
|
+
"type": "prometheus",
|
|
41
|
+
"uid": "${datasource}"
|
|
42
|
+
},
|
|
43
|
+
"description": "Scrape job, so one dashboard can serve several deployments of this service.",
|
|
44
|
+
"query": "label_values(pitcrew_ingestor_pubsub_message_total, job)",
|
|
45
|
+
"refresh": 1,
|
|
46
|
+
"includeAll": true,
|
|
47
|
+
"multi": true,
|
|
48
|
+
"current": {},
|
|
49
|
+
"hide": 0
|
|
50
|
+
}
|
|
51
|
+
]
|
|
52
|
+
},
|
|
53
|
+
"panels": [
|
|
54
|
+
{
|
|
55
|
+
"type": "row",
|
|
56
|
+
"title": "Intake: messages off the subscriptions",
|
|
57
|
+
"id": 1,
|
|
58
|
+
"gridPos": {
|
|
59
|
+
"h": 1,
|
|
60
|
+
"w": 24,
|
|
61
|
+
"x": 0,
|
|
62
|
+
"y": 0
|
|
63
|
+
},
|
|
64
|
+
"collapsed": false,
|
|
65
|
+
"panels": []
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
"type": "timeseries",
|
|
69
|
+
"title": "Messages / sec by type",
|
|
70
|
+
"description": "One counter per processor, incremented at the top of its handler before any work. Stacked, because the shape that matters is the mix shifting rather than any one line's absolute value. A type that reads No data has simply never fired: @leverege/metrics creates a counter lazily on first increment, so an unused processor has no series at all. Two of these track the branch, not the cluster - image collection exists only on the PIT-2434 line (VisionImageCollectionProcessor) and state write only on development (StateWriteProcessor, PIT-2419), so whichever image is deployed, one of them is expected to be absent.",
|
|
71
|
+
"id": 2,
|
|
72
|
+
"datasource": {
|
|
73
|
+
"type": "prometheus",
|
|
74
|
+
"uid": "${datasource}"
|
|
75
|
+
},
|
|
76
|
+
"gridPos": {
|
|
77
|
+
"h": 8,
|
|
78
|
+
"w": 12,
|
|
79
|
+
"x": 0,
|
|
80
|
+
"y": 1
|
|
81
|
+
},
|
|
82
|
+
"fieldConfig": {
|
|
83
|
+
"defaults": {
|
|
84
|
+
"unit": "reqps",
|
|
85
|
+
"min": 0,
|
|
86
|
+
"custom": {
|
|
87
|
+
"drawStyle": "line",
|
|
88
|
+
"lineWidth": 1,
|
|
89
|
+
"fillOpacity": 10,
|
|
90
|
+
"showPoints": "never",
|
|
91
|
+
"stacking": {
|
|
92
|
+
"mode": "normal"
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
},
|
|
96
|
+
"overrides": []
|
|
97
|
+
},
|
|
98
|
+
"options": {
|
|
99
|
+
"legend": {
|
|
100
|
+
"displayMode": "table",
|
|
101
|
+
"placement": "bottom",
|
|
102
|
+
"calcs": [
|
|
103
|
+
"lastNotNull",
|
|
104
|
+
"max"
|
|
105
|
+
]
|
|
106
|
+
},
|
|
107
|
+
"tooltip": {
|
|
108
|
+
"mode": "multi",
|
|
109
|
+
"sort": "desc"
|
|
110
|
+
}
|
|
111
|
+
},
|
|
112
|
+
"targets": [
|
|
113
|
+
{
|
|
114
|
+
"datasource": {
|
|
115
|
+
"type": "prometheus",
|
|
116
|
+
"uid": "${datasource}"
|
|
117
|
+
},
|
|
118
|
+
"expr": "sum(rate(pitcrew_ingestor_pubsub_imc_message_total{job=~\"$job\"}[5m]))",
|
|
119
|
+
"legendFormat": "imc",
|
|
120
|
+
"refId": "A"
|
|
121
|
+
},
|
|
122
|
+
{
|
|
123
|
+
"datasource": {
|
|
124
|
+
"type": "prometheus",
|
|
125
|
+
"uid": "${datasource}"
|
|
126
|
+
},
|
|
127
|
+
"expr": "sum(rate(pitcrew_ingestor_pubsub_sampling_message_total{job=~\"$job\"}[5m]))",
|
|
128
|
+
"legendFormat": "sampling",
|
|
129
|
+
"refId": "B"
|
|
130
|
+
},
|
|
131
|
+
{
|
|
132
|
+
"datasource": {
|
|
133
|
+
"type": "prometheus",
|
|
134
|
+
"uid": "${datasource}"
|
|
135
|
+
},
|
|
136
|
+
"expr": "sum(rate(pitcrew_ingestor_pubsub_person_occupancy_message_total{job=~\"$job\"}[5m]))",
|
|
137
|
+
"legendFormat": "person occupancy",
|
|
138
|
+
"refId": "C"
|
|
139
|
+
},
|
|
140
|
+
{
|
|
141
|
+
"datasource": {
|
|
142
|
+
"type": "prometheus",
|
|
143
|
+
"uid": "${datasource}"
|
|
144
|
+
},
|
|
145
|
+
"expr": "sum(rate(pitcrew_ingestor_pubsub_service_device_person_occupancy_message_total{job=~\"$job\"}[5m]))",
|
|
146
|
+
"legendFormat": "service device person occupancy",
|
|
147
|
+
"refId": "D"
|
|
148
|
+
},
|
|
149
|
+
{
|
|
150
|
+
"datasource": {
|
|
151
|
+
"type": "prometheus",
|
|
152
|
+
"uid": "${datasource}"
|
|
153
|
+
},
|
|
154
|
+
"expr": "sum(rate(pitcrew_ingestor_pubsub_vehicle_occupancy_message_total{job=~\"$job\"}[5m]))",
|
|
155
|
+
"legendFormat": "vehicle occupancy",
|
|
156
|
+
"refId": "E"
|
|
157
|
+
},
|
|
158
|
+
{
|
|
159
|
+
"datasource": {
|
|
160
|
+
"type": "prometheus",
|
|
161
|
+
"uid": "${datasource}"
|
|
162
|
+
},
|
|
163
|
+
"expr": "sum(rate(pitcrew_ingestor_pubsub_license_plate_message_total{job=~\"$job\"}[5m]))",
|
|
164
|
+
"legendFormat": "license plate",
|
|
165
|
+
"refId": "F"
|
|
166
|
+
},
|
|
167
|
+
{
|
|
168
|
+
"datasource": {
|
|
169
|
+
"type": "prometheus",
|
|
170
|
+
"uid": "${datasource}"
|
|
171
|
+
},
|
|
172
|
+
"expr": "sum(rate(pitcrew_ingestor_pubsub_showroom_entry_exit_message_total{job=~\"$job\"}[5m]))",
|
|
173
|
+
"legendFormat": "showroom entry/exit",
|
|
174
|
+
"refId": "G"
|
|
175
|
+
},
|
|
176
|
+
{
|
|
177
|
+
"datasource": {
|
|
178
|
+
"type": "prometheus",
|
|
179
|
+
"uid": "${datasource}"
|
|
180
|
+
},
|
|
181
|
+
"expr": "sum(rate(pitcrew_ingestor_pubsub_mean_labor_scheduler_message_total{job=~\"$job\"}[5m]))",
|
|
182
|
+
"legendFormat": "mean labor scheduler",
|
|
183
|
+
"refId": "H"
|
|
184
|
+
},
|
|
185
|
+
{
|
|
186
|
+
"datasource": {
|
|
187
|
+
"type": "prometheus",
|
|
188
|
+
"uid": "${datasource}"
|
|
189
|
+
},
|
|
190
|
+
"expr": "sum(rate(pitcrew_ingestor_pubsub_image_collection_message_total{job=~\"$job\"}[5m]))",
|
|
191
|
+
"legendFormat": "image collection",
|
|
192
|
+
"refId": "I"
|
|
193
|
+
},
|
|
194
|
+
{
|
|
195
|
+
"datasource": {
|
|
196
|
+
"type": "prometheus",
|
|
197
|
+
"uid": "${datasource}"
|
|
198
|
+
},
|
|
199
|
+
"expr": "sum(rate(pitcrew_ingestor_pubsub_state_message_total{job=~\"$job\"}[5m]))",
|
|
200
|
+
"legendFormat": "state write",
|
|
201
|
+
"refId": "J"
|
|
202
|
+
}
|
|
203
|
+
]
|
|
204
|
+
},
|
|
205
|
+
{
|
|
206
|
+
"type": "stat",
|
|
207
|
+
"title": "Totals over the selected range",
|
|
208
|
+
"description": "The intake headline. all types is pubsub_message_total, which every processor increments alongside its own counter, so it is the sum of the tiles beside it and not additional to them. A counter that has never been incremented does not exist in Prometheus, so it reads No data rather than 0.",
|
|
209
|
+
"id": 3,
|
|
210
|
+
"datasource": {
|
|
211
|
+
"type": "prometheus",
|
|
212
|
+
"uid": "${datasource}"
|
|
213
|
+
},
|
|
214
|
+
"gridPos": {
|
|
215
|
+
"h": 8,
|
|
216
|
+
"w": 12,
|
|
217
|
+
"x": 12,
|
|
218
|
+
"y": 1
|
|
219
|
+
},
|
|
220
|
+
"fieldConfig": {
|
|
221
|
+
"defaults": {
|
|
222
|
+
"unit": "short",
|
|
223
|
+
"min": 0
|
|
224
|
+
},
|
|
225
|
+
"overrides": []
|
|
226
|
+
},
|
|
227
|
+
"options": {
|
|
228
|
+
"reduceOptions": {
|
|
229
|
+
"calcs": [
|
|
230
|
+
"lastNotNull"
|
|
231
|
+
],
|
|
232
|
+
"fields": "",
|
|
233
|
+
"values": false
|
|
234
|
+
},
|
|
235
|
+
"textMode": "auto",
|
|
236
|
+
"colorMode": "value",
|
|
237
|
+
"graphMode": "area"
|
|
238
|
+
},
|
|
239
|
+
"targets": [
|
|
240
|
+
{
|
|
241
|
+
"datasource": {
|
|
242
|
+
"type": "prometheus",
|
|
243
|
+
"uid": "${datasource}"
|
|
244
|
+
},
|
|
245
|
+
"expr": "sum(increase(pitcrew_ingestor_pubsub_message_total{job=~\"$job\"}[$__range]))",
|
|
246
|
+
"legendFormat": "all types",
|
|
247
|
+
"refId": "A"
|
|
248
|
+
},
|
|
249
|
+
{
|
|
250
|
+
"datasource": {
|
|
251
|
+
"type": "prometheus",
|
|
252
|
+
"uid": "${datasource}"
|
|
253
|
+
},
|
|
254
|
+
"expr": "sum(increase(pitcrew_ingestor_pubsub_imc_message_total{job=~\"$job\"}[$__range]))",
|
|
255
|
+
"legendFormat": "imc",
|
|
256
|
+
"refId": "B"
|
|
257
|
+
},
|
|
258
|
+
{
|
|
259
|
+
"datasource": {
|
|
260
|
+
"type": "prometheus",
|
|
261
|
+
"uid": "${datasource}"
|
|
262
|
+
},
|
|
263
|
+
"expr": "sum(increase(pitcrew_ingestor_pubsub_sampling_message_total{job=~\"$job\"}[$__range]))",
|
|
264
|
+
"legendFormat": "sampling",
|
|
265
|
+
"refId": "C"
|
|
266
|
+
},
|
|
267
|
+
{
|
|
268
|
+
"datasource": {
|
|
269
|
+
"type": "prometheus",
|
|
270
|
+
"uid": "${datasource}"
|
|
271
|
+
},
|
|
272
|
+
"expr": "sum(increase(pitcrew_ingestor_pubsub_person_occupancy_message_total{job=~\"$job\"}[$__range]))",
|
|
273
|
+
"legendFormat": "person occupancy",
|
|
274
|
+
"refId": "D"
|
|
275
|
+
},
|
|
276
|
+
{
|
|
277
|
+
"datasource": {
|
|
278
|
+
"type": "prometheus",
|
|
279
|
+
"uid": "${datasource}"
|
|
280
|
+
},
|
|
281
|
+
"expr": "sum(increase(pitcrew_ingestor_pubsub_service_device_person_occupancy_message_total{job=~\"$job\"}[$__range]))",
|
|
282
|
+
"legendFormat": "service device person occupancy",
|
|
283
|
+
"refId": "E"
|
|
284
|
+
},
|
|
285
|
+
{
|
|
286
|
+
"datasource": {
|
|
287
|
+
"type": "prometheus",
|
|
288
|
+
"uid": "${datasource}"
|
|
289
|
+
},
|
|
290
|
+
"expr": "sum(increase(pitcrew_ingestor_pubsub_vehicle_occupancy_message_total{job=~\"$job\"}[$__range]))",
|
|
291
|
+
"legendFormat": "vehicle occupancy",
|
|
292
|
+
"refId": "F"
|
|
293
|
+
},
|
|
294
|
+
{
|
|
295
|
+
"datasource": {
|
|
296
|
+
"type": "prometheus",
|
|
297
|
+
"uid": "${datasource}"
|
|
298
|
+
},
|
|
299
|
+
"expr": "sum(increase(pitcrew_ingestor_pubsub_license_plate_message_total{job=~\"$job\"}[$__range]))",
|
|
300
|
+
"legendFormat": "license plate",
|
|
301
|
+
"refId": "G"
|
|
302
|
+
},
|
|
303
|
+
{
|
|
304
|
+
"datasource": {
|
|
305
|
+
"type": "prometheus",
|
|
306
|
+
"uid": "${datasource}"
|
|
307
|
+
},
|
|
308
|
+
"expr": "sum(increase(pitcrew_ingestor_pubsub_showroom_entry_exit_message_total{job=~\"$job\"}[$__range]))",
|
|
309
|
+
"legendFormat": "showroom entry/exit",
|
|
310
|
+
"refId": "H"
|
|
311
|
+
},
|
|
312
|
+
{
|
|
313
|
+
"datasource": {
|
|
314
|
+
"type": "prometheus",
|
|
315
|
+
"uid": "${datasource}"
|
|
316
|
+
},
|
|
317
|
+
"expr": "sum(increase(pitcrew_ingestor_pubsub_mean_labor_scheduler_message_total{job=~\"$job\"}[$__range]))",
|
|
318
|
+
"legendFormat": "mean labor scheduler",
|
|
319
|
+
"refId": "I"
|
|
320
|
+
},
|
|
321
|
+
{
|
|
322
|
+
"datasource": {
|
|
323
|
+
"type": "prometheus",
|
|
324
|
+
"uid": "${datasource}"
|
|
325
|
+
},
|
|
326
|
+
"expr": "sum(increase(pitcrew_ingestor_pubsub_image_collection_message_total{job=~\"$job\"}[$__range]))",
|
|
327
|
+
"legendFormat": "image collection",
|
|
328
|
+
"refId": "J"
|
|
329
|
+
},
|
|
330
|
+
{
|
|
331
|
+
"datasource": {
|
|
332
|
+
"type": "prometheus",
|
|
333
|
+
"uid": "${datasource}"
|
|
334
|
+
},
|
|
335
|
+
"expr": "sum(increase(pitcrew_ingestor_pubsub_state_message_total{job=~\"$job\"}[$__range]))",
|
|
336
|
+
"legendFormat": "state write",
|
|
337
|
+
"refId": "K"
|
|
338
|
+
}
|
|
339
|
+
]
|
|
340
|
+
},
|
|
341
|
+
{
|
|
342
|
+
"type": "row",
|
|
343
|
+
"title": "Accounting and the ordered queues",
|
|
344
|
+
"id": 4,
|
|
345
|
+
"gridPos": {
|
|
346
|
+
"h": 1,
|
|
347
|
+
"w": 24,
|
|
348
|
+
"x": 0,
|
|
349
|
+
"y": 9
|
|
350
|
+
},
|
|
351
|
+
"collapsed": false,
|
|
352
|
+
"panels": []
|
|
353
|
+
},
|
|
354
|
+
{
|
|
355
|
+
"type": "timeseries",
|
|
356
|
+
"title": "Inbound vs typed / sec",
|
|
357
|
+
"description": "msg_q_processed_count is emitted by the message-queue layer for every message taken off a subscription. pubsub_message_total is incremented by the processors that claim one. The gap between the two lines is the outcome nothing else counts: a message on a subscribed topic that no processor routes, or one that threw before its first increment. A steady few percent is normal (dev read 348994 vs 335534, about 4%); a widening gap means a message shape arriving that this build does not handle - check the pod logs. Unlike the pubsub-repeater panel this mirrors, both series are single metric names, so plain rate() is safe here and no offset arithmetic is needed.",
|
|
358
|
+
"id": 5,
|
|
359
|
+
"datasource": {
|
|
360
|
+
"type": "prometheus",
|
|
361
|
+
"uid": "${datasource}"
|
|
362
|
+
},
|
|
363
|
+
"gridPos": {
|
|
364
|
+
"h": 8,
|
|
365
|
+
"w": 12,
|
|
366
|
+
"x": 0,
|
|
367
|
+
"y": 10
|
|
368
|
+
},
|
|
369
|
+
"fieldConfig": {
|
|
370
|
+
"defaults": {
|
|
371
|
+
"unit": "reqps",
|
|
372
|
+
"min": 0,
|
|
373
|
+
"custom": {
|
|
374
|
+
"drawStyle": "line",
|
|
375
|
+
"lineWidth": 1,
|
|
376
|
+
"fillOpacity": 10,
|
|
377
|
+
"showPoints": "never",
|
|
378
|
+
"stacking": {
|
|
379
|
+
"mode": "none"
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
},
|
|
383
|
+
"overrides": []
|
|
384
|
+
},
|
|
385
|
+
"options": {
|
|
386
|
+
"legend": {
|
|
387
|
+
"displayMode": "table",
|
|
388
|
+
"placement": "bottom",
|
|
389
|
+
"calcs": [
|
|
390
|
+
"lastNotNull",
|
|
391
|
+
"max"
|
|
392
|
+
]
|
|
393
|
+
},
|
|
394
|
+
"tooltip": {
|
|
395
|
+
"mode": "multi",
|
|
396
|
+
"sort": "desc"
|
|
397
|
+
}
|
|
398
|
+
},
|
|
399
|
+
"targets": [
|
|
400
|
+
{
|
|
401
|
+
"datasource": {
|
|
402
|
+
"type": "prometheus",
|
|
403
|
+
"uid": "${datasource}"
|
|
404
|
+
},
|
|
405
|
+
"expr": "sum(rate(pitcrew_ingestor_msg_q_processed_count{job=~\"$job\"}[5m]))",
|
|
406
|
+
"legendFormat": "processed/s (inbound)",
|
|
407
|
+
"refId": "A"
|
|
408
|
+
},
|
|
409
|
+
{
|
|
410
|
+
"datasource": {
|
|
411
|
+
"type": "prometheus",
|
|
412
|
+
"uid": "${datasource}"
|
|
413
|
+
},
|
|
414
|
+
"expr": "sum(rate(pitcrew_ingestor_pubsub_message_total{job=~\"$job\"}[5m]))",
|
|
415
|
+
"legendFormat": "typed/s (accounted)",
|
|
416
|
+
"refId": "B"
|
|
417
|
+
}
|
|
418
|
+
]
|
|
419
|
+
},
|
|
420
|
+
{
|
|
421
|
+
"type": "timeseries",
|
|
422
|
+
"title": "Ordered queue throughput / sec",
|
|
423
|
+
"description": "enqueued is counted by both InMemoryOrderedQueue and RedisOrderedQueue as a message is handed to the queue for its correlation id; processed and errors are counted by OrderedQueue as it drains one. processed should track enqueued with only the drain delay between them. errors is created lazily on the first failure, so an absent line means nothing has ever thrown in a drain, not that the counter is broken.",
|
|
424
|
+
"id": 6,
|
|
425
|
+
"datasource": {
|
|
426
|
+
"type": "prometheus",
|
|
427
|
+
"uid": "${datasource}"
|
|
428
|
+
},
|
|
429
|
+
"gridPos": {
|
|
430
|
+
"h": 8,
|
|
431
|
+
"w": 12,
|
|
432
|
+
"x": 12,
|
|
433
|
+
"y": 10
|
|
434
|
+
},
|
|
435
|
+
"fieldConfig": {
|
|
436
|
+
"defaults": {
|
|
437
|
+
"unit": "reqps",
|
|
438
|
+
"min": 0,
|
|
439
|
+
"custom": {
|
|
440
|
+
"drawStyle": "line",
|
|
441
|
+
"lineWidth": 1,
|
|
442
|
+
"fillOpacity": 10,
|
|
443
|
+
"showPoints": "never",
|
|
444
|
+
"stacking": {
|
|
445
|
+
"mode": "none"
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
},
|
|
449
|
+
"overrides": []
|
|
450
|
+
},
|
|
451
|
+
"options": {
|
|
452
|
+
"legend": {
|
|
453
|
+
"displayMode": "table",
|
|
454
|
+
"placement": "bottom",
|
|
455
|
+
"calcs": [
|
|
456
|
+
"lastNotNull",
|
|
457
|
+
"max"
|
|
458
|
+
]
|
|
459
|
+
},
|
|
460
|
+
"tooltip": {
|
|
461
|
+
"mode": "multi",
|
|
462
|
+
"sort": "desc"
|
|
463
|
+
}
|
|
464
|
+
},
|
|
465
|
+
"targets": [
|
|
466
|
+
{
|
|
467
|
+
"datasource": {
|
|
468
|
+
"type": "prometheus",
|
|
469
|
+
"uid": "${datasource}"
|
|
470
|
+
},
|
|
471
|
+
"expr": "sum(rate(pitcrew_ingestor_ordered_q_enqueued_count{job=~\"$job\"}[5m]))",
|
|
472
|
+
"legendFormat": "enqueued/s",
|
|
473
|
+
"refId": "A"
|
|
474
|
+
},
|
|
475
|
+
{
|
|
476
|
+
"datasource": {
|
|
477
|
+
"type": "prometheus",
|
|
478
|
+
"uid": "${datasource}"
|
|
479
|
+
},
|
|
480
|
+
"expr": "sum(rate(pitcrew_ingestor_ordered_q_processed_count{job=~\"$job\"}[5m]))",
|
|
481
|
+
"legendFormat": "processed/s",
|
|
482
|
+
"refId": "B"
|
|
483
|
+
},
|
|
484
|
+
{
|
|
485
|
+
"datasource": {
|
|
486
|
+
"type": "prometheus",
|
|
487
|
+
"uid": "${datasource}"
|
|
488
|
+
},
|
|
489
|
+
"expr": "sum(rate(pitcrew_ingestor_ordered_q_process_error_count{job=~\"$job\"}[5m]))",
|
|
490
|
+
"legendFormat": "errors/s",
|
|
491
|
+
"refId": "C"
|
|
492
|
+
}
|
|
493
|
+
]
|
|
494
|
+
},
|
|
495
|
+
{
|
|
496
|
+
"type": "timeseries",
|
|
497
|
+
"title": "Ordered queue backlog",
|
|
498
|
+
"description": "enqueued minus processed, summed across pods. The ordered queues are in memory and unbounded, and messages are acknowledged on receipt, so nothing redelivers what a pod was holding when it died: a backlog that climbs is a data-loss exposure, not only a latency one. Watch it against Heap used, which is where the same problem shows up second. Clamped at 0 so a pod entering or leaving between the two sums reads flat rather than negative.",
|
|
499
|
+
"id": 7,
|
|
500
|
+
"datasource": {
|
|
501
|
+
"type": "prometheus",
|
|
502
|
+
"uid": "${datasource}"
|
|
503
|
+
},
|
|
504
|
+
"gridPos": {
|
|
505
|
+
"h": 8,
|
|
506
|
+
"w": 12,
|
|
507
|
+
"x": 0,
|
|
508
|
+
"y": 18
|
|
509
|
+
},
|
|
510
|
+
"fieldConfig": {
|
|
511
|
+
"defaults": {
|
|
512
|
+
"unit": "short",
|
|
513
|
+
"min": 0,
|
|
514
|
+
"custom": {
|
|
515
|
+
"drawStyle": "line",
|
|
516
|
+
"lineWidth": 1,
|
|
517
|
+
"fillOpacity": 10,
|
|
518
|
+
"showPoints": "never",
|
|
519
|
+
"stacking": {
|
|
520
|
+
"mode": "none"
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
},
|
|
524
|
+
"overrides": []
|
|
525
|
+
},
|
|
526
|
+
"options": {
|
|
527
|
+
"legend": {
|
|
528
|
+
"displayMode": "table",
|
|
529
|
+
"placement": "bottom",
|
|
530
|
+
"calcs": [
|
|
531
|
+
"lastNotNull",
|
|
532
|
+
"max"
|
|
533
|
+
]
|
|
534
|
+
},
|
|
535
|
+
"tooltip": {
|
|
536
|
+
"mode": "multi",
|
|
537
|
+
"sort": "desc"
|
|
538
|
+
}
|
|
539
|
+
},
|
|
540
|
+
"targets": [
|
|
541
|
+
{
|
|
542
|
+
"datasource": {
|
|
543
|
+
"type": "prometheus",
|
|
544
|
+
"uid": "${datasource}"
|
|
545
|
+
},
|
|
546
|
+
"expr": "clamp_min(sum(pitcrew_ingestor_ordered_q_enqueued_count{job=~\"$job\"}) - sum(pitcrew_ingestor_ordered_q_processed_count{job=~\"$job\"}), 0)",
|
|
547
|
+
"legendFormat": "backlog",
|
|
548
|
+
"refId": "A"
|
|
549
|
+
}
|
|
550
|
+
]
|
|
551
|
+
},
|
|
552
|
+
{
|
|
553
|
+
"type": "timeseries",
|
|
554
|
+
"title": "Queues draining concurrently",
|
|
555
|
+
"description": "A gauge, maintained with M.change(+1) and M.change(-1) around each drain loop in OrderedQueue, so it counts correlation ids being drained right now - it is not queue depth, which is the panel to its left. A pod killed mid-drain never runs its decrement, so its series can sit stuck above 0 until the scrape target disappears; read a floor that never returns to 0 as a restart artefact rather than as load.",
|
|
556
|
+
"id": 8,
|
|
557
|
+
"datasource": {
|
|
558
|
+
"type": "prometheus",
|
|
559
|
+
"uid": "${datasource}"
|
|
560
|
+
},
|
|
561
|
+
"gridPos": {
|
|
562
|
+
"h": 8,
|
|
563
|
+
"w": 12,
|
|
564
|
+
"x": 12,
|
|
565
|
+
"y": 18
|
|
566
|
+
},
|
|
567
|
+
"fieldConfig": {
|
|
568
|
+
"defaults": {
|
|
569
|
+
"unit": "short",
|
|
570
|
+
"min": 0,
|
|
571
|
+
"custom": {
|
|
572
|
+
"drawStyle": "line",
|
|
573
|
+
"lineWidth": 1,
|
|
574
|
+
"fillOpacity": 10,
|
|
575
|
+
"showPoints": "never",
|
|
576
|
+
"stacking": {
|
|
577
|
+
"mode": "none"
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
},
|
|
581
|
+
"overrides": []
|
|
582
|
+
},
|
|
583
|
+
"options": {
|
|
584
|
+
"legend": {
|
|
585
|
+
"displayMode": "table",
|
|
586
|
+
"placement": "bottom",
|
|
587
|
+
"calcs": [
|
|
588
|
+
"lastNotNull",
|
|
589
|
+
"max"
|
|
590
|
+
]
|
|
591
|
+
},
|
|
592
|
+
"tooltip": {
|
|
593
|
+
"mode": "multi",
|
|
594
|
+
"sort": "desc"
|
|
595
|
+
}
|
|
596
|
+
},
|
|
597
|
+
"targets": [
|
|
598
|
+
{
|
|
599
|
+
"datasource": {
|
|
600
|
+
"type": "prometheus",
|
|
601
|
+
"uid": "${datasource}"
|
|
602
|
+
},
|
|
603
|
+
"expr": "sum(pitcrew_ingestor_ordered_q_processing_queues{job=~\"$job\"})",
|
|
604
|
+
"legendFormat": "queues draining",
|
|
605
|
+
"refId": "A"
|
|
606
|
+
}
|
|
607
|
+
]
|
|
608
|
+
},
|
|
609
|
+
{
|
|
610
|
+
"type": "row",
|
|
611
|
+
"title": "Device cache: the api-server fan-out driver",
|
|
612
|
+
"id": 9,
|
|
613
|
+
"gridPos": {
|
|
614
|
+
"h": 1,
|
|
615
|
+
"w": 24,
|
|
616
|
+
"x": 0,
|
|
617
|
+
"y": 26
|
|
618
|
+
},
|
|
619
|
+
"collapsed": false,
|
|
620
|
+
"panels": []
|
|
621
|
+
},
|
|
622
|
+
{
|
|
623
|
+
"type": "timeseries",
|
|
624
|
+
"title": "Cache calls vs backing gets / sec",
|
|
625
|
+
"description": "The panel this dashboard exists for. getCachedDevice increments cache_get_device_total on every call and delegates to a 5-minute-TTL cache; imagine_get_device_request_total is incremented inside getDevice, which runs only on a miss. So the gap between the lines is the cache working. It matters more than a usual hit rate because getCachedDevice hardcodes a null interface (computeArgs [ deviceId, null, options ] in src/utils/ImagineUtil.js), which makes every miss a blueprint-less device read that api-server fans out across all Elasticsearch shards. The lower line is therefore this service's contribution to api_server_search_controller_get_no_bp: measured on stg 2026-09-03 at about 1.42/s across 5 pods, roughly 45% of the cluster total. Two call sites in ServiceDevicePersonOccupancyProcessor call getDevice directly rather than through the cache, so the lower line slightly overstates pure misses.",
|
|
626
|
+
"id": 10,
|
|
627
|
+
"datasource": {
|
|
628
|
+
"type": "prometheus",
|
|
629
|
+
"uid": "${datasource}"
|
|
630
|
+
},
|
|
631
|
+
"gridPos": {
|
|
632
|
+
"h": 8,
|
|
633
|
+
"w": 12,
|
|
634
|
+
"x": 0,
|
|
635
|
+
"y": 27
|
|
636
|
+
},
|
|
637
|
+
"fieldConfig": {
|
|
638
|
+
"defaults": {
|
|
639
|
+
"unit": "reqps",
|
|
640
|
+
"min": 0,
|
|
641
|
+
"custom": {
|
|
642
|
+
"drawStyle": "line",
|
|
643
|
+
"lineWidth": 1,
|
|
644
|
+
"fillOpacity": 10,
|
|
645
|
+
"showPoints": "never",
|
|
646
|
+
"stacking": {
|
|
647
|
+
"mode": "none"
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
},
|
|
651
|
+
"overrides": []
|
|
652
|
+
},
|
|
653
|
+
"options": {
|
|
654
|
+
"legend": {
|
|
655
|
+
"displayMode": "table",
|
|
656
|
+
"placement": "bottom",
|
|
657
|
+
"calcs": [
|
|
658
|
+
"lastNotNull",
|
|
659
|
+
"max"
|
|
660
|
+
]
|
|
661
|
+
},
|
|
662
|
+
"tooltip": {
|
|
663
|
+
"mode": "multi",
|
|
664
|
+
"sort": "desc"
|
|
665
|
+
}
|
|
666
|
+
},
|
|
667
|
+
"targets": [
|
|
668
|
+
{
|
|
669
|
+
"datasource": {
|
|
670
|
+
"type": "prometheus",
|
|
671
|
+
"uid": "${datasource}"
|
|
672
|
+
},
|
|
673
|
+
"expr": "sum(rate(pitcrew_ingestor_cache_get_device_total{job=~\"$job\"}[5m]))",
|
|
674
|
+
"legendFormat": "getCachedDevice calls/s",
|
|
675
|
+
"refId": "A"
|
|
676
|
+
},
|
|
677
|
+
{
|
|
678
|
+
"datasource": {
|
|
679
|
+
"type": "prometheus",
|
|
680
|
+
"uid": "${datasource}"
|
|
681
|
+
},
|
|
682
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_get_device_request_total{job=~\"$job\"}[5m]))",
|
|
683
|
+
"legendFormat": "backing api-server gets/s (misses)",
|
|
684
|
+
"refId": "B"
|
|
685
|
+
}
|
|
686
|
+
]
|
|
687
|
+
},
|
|
688
|
+
{
|
|
689
|
+
"type": "stat",
|
|
690
|
+
"title": "Device cache hit ratio",
|
|
691
|
+
"description": "1 minus misses over calls, across the selected range. Every point this drops costs a blueprint-less Elasticsearch fan-out, so treat a step down as a cost event and not only a latency one - the usual causes are a restart emptying the cache or the working set of devices growing past what a 5-minute TTL can hold. Reads 100% when neither counter exists yet, because the clamped denominator makes an all-zero range indistinguishable from a perfect one; trust it only once the panel beside it is drawing.",
|
|
692
|
+
"id": 11,
|
|
693
|
+
"datasource": {
|
|
694
|
+
"type": "prometheus",
|
|
695
|
+
"uid": "${datasource}"
|
|
696
|
+
},
|
|
697
|
+
"gridPos": {
|
|
698
|
+
"h": 8,
|
|
699
|
+
"w": 6,
|
|
700
|
+
"x": 12,
|
|
701
|
+
"y": 27
|
|
702
|
+
},
|
|
703
|
+
"fieldConfig": {
|
|
704
|
+
"defaults": {
|
|
705
|
+
"unit": "percentunit",
|
|
706
|
+
"min": 0
|
|
707
|
+
},
|
|
708
|
+
"overrides": []
|
|
709
|
+
},
|
|
710
|
+
"options": {
|
|
711
|
+
"reduceOptions": {
|
|
712
|
+
"calcs": [
|
|
713
|
+
"lastNotNull"
|
|
714
|
+
],
|
|
715
|
+
"fields": "",
|
|
716
|
+
"values": false
|
|
717
|
+
},
|
|
718
|
+
"textMode": "auto",
|
|
719
|
+
"colorMode": "value",
|
|
720
|
+
"graphMode": "area"
|
|
721
|
+
},
|
|
722
|
+
"targets": [
|
|
723
|
+
{
|
|
724
|
+
"datasource": {
|
|
725
|
+
"type": "prometheus",
|
|
726
|
+
"uid": "${datasource}"
|
|
727
|
+
},
|
|
728
|
+
"expr": "1 - ((sum(increase(pitcrew_ingestor_imagine_get_device_request_total{job=~\"$job\"}[$__range])) or vector(0)) / clamp_min((sum(increase(pitcrew_ingestor_cache_get_device_total{job=~\"$job\"}[$__range])) or vector(0)), 1e-9))",
|
|
729
|
+
"legendFormat": "hit ratio",
|
|
730
|
+
"refId": "A"
|
|
731
|
+
}
|
|
732
|
+
]
|
|
733
|
+
},
|
|
734
|
+
{
|
|
735
|
+
"type": "timeseries",
|
|
736
|
+
"title": "Cache calls / sec by kind",
|
|
737
|
+
"description": "The three cached lookups, each counted in its own getCached* wrapper. cache_get_devices_total is declared in the source but its call site (getCachedDevices) is commented out, so it will never appear here - that is the code, not a scrape problem. cache_request_total and cache_get_request_total are deliberately left off this panel: both are incremented by all three wrappers, so plotting them would just draw the total twice.",
|
|
738
|
+
"id": 12,
|
|
739
|
+
"datasource": {
|
|
740
|
+
"type": "prometheus",
|
|
741
|
+
"uid": "${datasource}"
|
|
742
|
+
},
|
|
743
|
+
"gridPos": {
|
|
744
|
+
"h": 8,
|
|
745
|
+
"w": 6,
|
|
746
|
+
"x": 18,
|
|
747
|
+
"y": 27
|
|
748
|
+
},
|
|
749
|
+
"fieldConfig": {
|
|
750
|
+
"defaults": {
|
|
751
|
+
"unit": "reqps",
|
|
752
|
+
"min": 0,
|
|
753
|
+
"custom": {
|
|
754
|
+
"drawStyle": "line",
|
|
755
|
+
"lineWidth": 1,
|
|
756
|
+
"fillOpacity": 10,
|
|
757
|
+
"showPoints": "never",
|
|
758
|
+
"stacking": {
|
|
759
|
+
"mode": "none"
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
},
|
|
763
|
+
"overrides": []
|
|
764
|
+
},
|
|
765
|
+
"options": {
|
|
766
|
+
"legend": {
|
|
767
|
+
"displayMode": "table",
|
|
768
|
+
"placement": "bottom",
|
|
769
|
+
"calcs": [
|
|
770
|
+
"lastNotNull",
|
|
771
|
+
"max"
|
|
772
|
+
]
|
|
773
|
+
},
|
|
774
|
+
"tooltip": {
|
|
775
|
+
"mode": "multi",
|
|
776
|
+
"sort": "desc"
|
|
777
|
+
}
|
|
778
|
+
},
|
|
779
|
+
"targets": [
|
|
780
|
+
{
|
|
781
|
+
"datasource": {
|
|
782
|
+
"type": "prometheus",
|
|
783
|
+
"uid": "${datasource}"
|
|
784
|
+
},
|
|
785
|
+
"expr": "sum(rate(pitcrew_ingestor_cache_get_device_total{job=~\"$job\"}[5m]))",
|
|
786
|
+
"legendFormat": "device",
|
|
787
|
+
"refId": "A"
|
|
788
|
+
},
|
|
789
|
+
{
|
|
790
|
+
"datasource": {
|
|
791
|
+
"type": "prometheus",
|
|
792
|
+
"uid": "${datasource}"
|
|
793
|
+
},
|
|
794
|
+
"expr": "sum(rate(pitcrew_ingestor_cache_get_device_by_network_alias_total{job=~\"$job\"}[5m]))",
|
|
795
|
+
"legendFormat": "device by network alias",
|
|
796
|
+
"refId": "B"
|
|
797
|
+
},
|
|
798
|
+
{
|
|
799
|
+
"datasource": {
|
|
800
|
+
"type": "prometheus",
|
|
801
|
+
"uid": "${datasource}"
|
|
802
|
+
},
|
|
803
|
+
"expr": "sum(rate(pitcrew_ingestor_cache_get_many_to_many_parent_devices_total{job=~\"$job\"}[5m]))",
|
|
804
|
+
"legendFormat": "many-to-many parents",
|
|
805
|
+
"refId": "C"
|
|
806
|
+
}
|
|
807
|
+
]
|
|
808
|
+
},
|
|
809
|
+
{
|
|
810
|
+
"type": "row",
|
|
811
|
+
"title": "Imagine reads and writes",
|
|
812
|
+
"id": 13,
|
|
813
|
+
"gridPos": {
|
|
814
|
+
"h": 1,
|
|
815
|
+
"w": 24,
|
|
816
|
+
"x": 0,
|
|
817
|
+
"y": 35
|
|
818
|
+
},
|
|
819
|
+
"collapsed": false,
|
|
820
|
+
"panels": []
|
|
821
|
+
},
|
|
822
|
+
{
|
|
823
|
+
"type": "timeseries",
|
|
824
|
+
"title": "Imagine requests / sec by kind",
|
|
825
|
+
"description": "ImagineUtil increments three counters on every call: imagine_request_total, a verb counter, and a leaf counter for the specific method. This panel is the verb level, so all requests is the sum of the lines beneath it and never additional to them. set value exists only on development (StateWriteProcessor, PIT-2419); on the PIT-2434 line it is absent by design, the mirror of image collection on the intake panel.",
|
|
826
|
+
"id": 14,
|
|
827
|
+
"datasource": {
|
|
828
|
+
"type": "prometheus",
|
|
829
|
+
"uid": "${datasource}"
|
|
830
|
+
},
|
|
831
|
+
"gridPos": {
|
|
832
|
+
"h": 8,
|
|
833
|
+
"w": 12,
|
|
834
|
+
"x": 0,
|
|
835
|
+
"y": 36
|
|
836
|
+
},
|
|
837
|
+
"fieldConfig": {
|
|
838
|
+
"defaults": {
|
|
839
|
+
"unit": "reqps",
|
|
840
|
+
"min": 0,
|
|
841
|
+
"custom": {
|
|
842
|
+
"drawStyle": "line",
|
|
843
|
+
"lineWidth": 1,
|
|
844
|
+
"fillOpacity": 10,
|
|
845
|
+
"showPoints": "never",
|
|
846
|
+
"stacking": {
|
|
847
|
+
"mode": "none"
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
},
|
|
851
|
+
"overrides": []
|
|
852
|
+
},
|
|
853
|
+
"options": {
|
|
854
|
+
"legend": {
|
|
855
|
+
"displayMode": "table",
|
|
856
|
+
"placement": "bottom",
|
|
857
|
+
"calcs": [
|
|
858
|
+
"lastNotNull",
|
|
859
|
+
"max"
|
|
860
|
+
]
|
|
861
|
+
},
|
|
862
|
+
"tooltip": {
|
|
863
|
+
"mode": "multi",
|
|
864
|
+
"sort": "desc"
|
|
865
|
+
}
|
|
866
|
+
},
|
|
867
|
+
"targets": [
|
|
868
|
+
{
|
|
869
|
+
"datasource": {
|
|
870
|
+
"type": "prometheus",
|
|
871
|
+
"uid": "${datasource}"
|
|
872
|
+
},
|
|
873
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_request_total{job=~\"$job\"}[5m]))",
|
|
874
|
+
"legendFormat": "all requests",
|
|
875
|
+
"refId": "A"
|
|
876
|
+
},
|
|
877
|
+
{
|
|
878
|
+
"datasource": {
|
|
879
|
+
"type": "prometheus",
|
|
880
|
+
"uid": "${datasource}"
|
|
881
|
+
},
|
|
882
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_get_request_total{job=~\"$job\"}[5m]))",
|
|
883
|
+
"legendFormat": "get",
|
|
884
|
+
"refId": "B"
|
|
885
|
+
},
|
|
886
|
+
{
|
|
887
|
+
"datasource": {
|
|
888
|
+
"type": "prometheus",
|
|
889
|
+
"uid": "${datasource}"
|
|
890
|
+
},
|
|
891
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_update_request_total{job=~\"$job\"}[5m]))",
|
|
892
|
+
"legendFormat": "update",
|
|
893
|
+
"refId": "C"
|
|
894
|
+
},
|
|
895
|
+
{
|
|
896
|
+
"datasource": {
|
|
897
|
+
"type": "prometheus",
|
|
898
|
+
"uid": "${datasource}"
|
|
899
|
+
},
|
|
900
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_create_request_total{job=~\"$job\"}[5m]))",
|
|
901
|
+
"legendFormat": "create",
|
|
902
|
+
"refId": "D"
|
|
903
|
+
},
|
|
904
|
+
{
|
|
905
|
+
"datasource": {
|
|
906
|
+
"type": "prometheus",
|
|
907
|
+
"uid": "${datasource}"
|
|
908
|
+
},
|
|
909
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_create_relationship_request_total{job=~\"$job\"}[5m]))",
|
|
910
|
+
"legendFormat": "create relationship",
|
|
911
|
+
"refId": "E"
|
|
912
|
+
},
|
|
913
|
+
{
|
|
914
|
+
"datasource": {
|
|
915
|
+
"type": "prometheus",
|
|
916
|
+
"uid": "${datasource}"
|
|
917
|
+
},
|
|
918
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_delete_relationship_request_total{job=~\"$job\"}[5m]))",
|
|
919
|
+
"legendFormat": "delete relationship",
|
|
920
|
+
"refId": "F"
|
|
921
|
+
},
|
|
922
|
+
{
|
|
923
|
+
"datasource": {
|
|
924
|
+
"type": "prometheus",
|
|
925
|
+
"uid": "${datasource}"
|
|
926
|
+
},
|
|
927
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_set_value_request_total{job=~\"$job\"}[5m]))",
|
|
928
|
+
"legendFormat": "set value",
|
|
929
|
+
"refId": "G"
|
|
930
|
+
}
|
|
931
|
+
]
|
|
932
|
+
},
|
|
933
|
+
{
|
|
934
|
+
"type": "timeseries",
|
|
935
|
+
"title": "Imagine errors / sec by kind",
|
|
936
|
+
"description": "The same three-level nesting as the panel beside it, so all errors is the sum of the rest. Every counter here is created on its first failure, which means an empty panel is the healthy state and not a broken query. These count throws out of ImagineUtil after its own retry handling, so they are a stricter signal than the raw HTTP errors two rows down: a 404 that the calling processor tolerates never reaches this panel.",
|
|
937
|
+
"id": 15,
|
|
938
|
+
"datasource": {
|
|
939
|
+
"type": "prometheus",
|
|
940
|
+
"uid": "${datasource}"
|
|
941
|
+
},
|
|
942
|
+
"gridPos": {
|
|
943
|
+
"h": 8,
|
|
944
|
+
"w": 12,
|
|
945
|
+
"x": 12,
|
|
946
|
+
"y": 36
|
|
947
|
+
},
|
|
948
|
+
"fieldConfig": {
|
|
949
|
+
"defaults": {
|
|
950
|
+
"unit": "reqps",
|
|
951
|
+
"min": 0,
|
|
952
|
+
"custom": {
|
|
953
|
+
"drawStyle": "line",
|
|
954
|
+
"lineWidth": 1,
|
|
955
|
+
"fillOpacity": 10,
|
|
956
|
+
"showPoints": "never",
|
|
957
|
+
"stacking": {
|
|
958
|
+
"mode": "none"
|
|
959
|
+
}
|
|
960
|
+
}
|
|
961
|
+
},
|
|
962
|
+
"overrides": []
|
|
963
|
+
},
|
|
964
|
+
"options": {
|
|
965
|
+
"legend": {
|
|
966
|
+
"displayMode": "table",
|
|
967
|
+
"placement": "bottom",
|
|
968
|
+
"calcs": [
|
|
969
|
+
"lastNotNull",
|
|
970
|
+
"max"
|
|
971
|
+
]
|
|
972
|
+
},
|
|
973
|
+
"tooltip": {
|
|
974
|
+
"mode": "multi",
|
|
975
|
+
"sort": "desc"
|
|
976
|
+
}
|
|
977
|
+
},
|
|
978
|
+
"targets": [
|
|
979
|
+
{
|
|
980
|
+
"datasource": {
|
|
981
|
+
"type": "prometheus",
|
|
982
|
+
"uid": "${datasource}"
|
|
983
|
+
},
|
|
984
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_error_total{job=~\"$job\"}[5m]))",
|
|
985
|
+
"legendFormat": "all errors",
|
|
986
|
+
"refId": "A"
|
|
987
|
+
},
|
|
988
|
+
{
|
|
989
|
+
"datasource": {
|
|
990
|
+
"type": "prometheus",
|
|
991
|
+
"uid": "${datasource}"
|
|
992
|
+
},
|
|
993
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_get_error_total{job=~\"$job\"}[5m]))",
|
|
994
|
+
"legendFormat": "get",
|
|
995
|
+
"refId": "B"
|
|
996
|
+
},
|
|
997
|
+
{
|
|
998
|
+
"datasource": {
|
|
999
|
+
"type": "prometheus",
|
|
1000
|
+
"uid": "${datasource}"
|
|
1001
|
+
},
|
|
1002
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_update_error_total{job=~\"$job\"}[5m]))",
|
|
1003
|
+
"legendFormat": "update",
|
|
1004
|
+
"refId": "C"
|
|
1005
|
+
},
|
|
1006
|
+
{
|
|
1007
|
+
"datasource": {
|
|
1008
|
+
"type": "prometheus",
|
|
1009
|
+
"uid": "${datasource}"
|
|
1010
|
+
},
|
|
1011
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_create_error_total{job=~\"$job\"}[5m]))",
|
|
1012
|
+
"legendFormat": "create",
|
|
1013
|
+
"refId": "D"
|
|
1014
|
+
},
|
|
1015
|
+
{
|
|
1016
|
+
"datasource": {
|
|
1017
|
+
"type": "prometheus",
|
|
1018
|
+
"uid": "${datasource}"
|
|
1019
|
+
},
|
|
1020
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_create_relationship_error_total{job=~\"$job\"}[5m]))",
|
|
1021
|
+
"legendFormat": "create relationship",
|
|
1022
|
+
"refId": "E"
|
|
1023
|
+
},
|
|
1024
|
+
{
|
|
1025
|
+
"datasource": {
|
|
1026
|
+
"type": "prometheus",
|
|
1027
|
+
"uid": "${datasource}"
|
|
1028
|
+
},
|
|
1029
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_delete_relationship_error_total{job=~\"$job\"}[5m]))",
|
|
1030
|
+
"legendFormat": "delete relationship",
|
|
1031
|
+
"refId": "F"
|
|
1032
|
+
},
|
|
1033
|
+
{
|
|
1034
|
+
"datasource": {
|
|
1035
|
+
"type": "prometheus",
|
|
1036
|
+
"uid": "${datasource}"
|
|
1037
|
+
},
|
|
1038
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_set_value_error_total{job=~\"$job\"}[5m]))",
|
|
1039
|
+
"legendFormat": "set value",
|
|
1040
|
+
"refId": "G"
|
|
1041
|
+
}
|
|
1042
|
+
]
|
|
1043
|
+
},
|
|
1044
|
+
{
|
|
1045
|
+
"type": "timeseries",
|
|
1046
|
+
"title": "Device operations / sec",
|
|
1047
|
+
"description": "The leaf counters, one per ImagineUtil method, which is the level that says what the service is actually doing to Imagine. update device is the write path - device data patches, which merge rather than replace. get device is the read that the cache above is trying to suppress, so this line and the cache panel's lower line are the same series seen in two contexts.",
|
|
1048
|
+
"id": 16,
|
|
1049
|
+
"datasource": {
|
|
1050
|
+
"type": "prometheus",
|
|
1051
|
+
"uid": "${datasource}"
|
|
1052
|
+
},
|
|
1053
|
+
"gridPos": {
|
|
1054
|
+
"h": 8,
|
|
1055
|
+
"w": 12,
|
|
1056
|
+
"x": 0,
|
|
1057
|
+
"y": 44
|
|
1058
|
+
},
|
|
1059
|
+
"fieldConfig": {
|
|
1060
|
+
"defaults": {
|
|
1061
|
+
"unit": "reqps",
|
|
1062
|
+
"min": 0,
|
|
1063
|
+
"custom": {
|
|
1064
|
+
"drawStyle": "line",
|
|
1065
|
+
"lineWidth": 1,
|
|
1066
|
+
"fillOpacity": 10,
|
|
1067
|
+
"showPoints": "never",
|
|
1068
|
+
"stacking": {
|
|
1069
|
+
"mode": "none"
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
},
|
|
1073
|
+
"overrides": []
|
|
1074
|
+
},
|
|
1075
|
+
"options": {
|
|
1076
|
+
"legend": {
|
|
1077
|
+
"displayMode": "table",
|
|
1078
|
+
"placement": "bottom",
|
|
1079
|
+
"calcs": [
|
|
1080
|
+
"lastNotNull",
|
|
1081
|
+
"max"
|
|
1082
|
+
]
|
|
1083
|
+
},
|
|
1084
|
+
"tooltip": {
|
|
1085
|
+
"mode": "multi",
|
|
1086
|
+
"sort": "desc"
|
|
1087
|
+
}
|
|
1088
|
+
},
|
|
1089
|
+
"targets": [
|
|
1090
|
+
{
|
|
1091
|
+
"datasource": {
|
|
1092
|
+
"type": "prometheus",
|
|
1093
|
+
"uid": "${datasource}"
|
|
1094
|
+
},
|
|
1095
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_get_device_request_total{job=~\"$job\"}[5m]))",
|
|
1096
|
+
"legendFormat": "get device",
|
|
1097
|
+
"refId": "A"
|
|
1098
|
+
},
|
|
1099
|
+
{
|
|
1100
|
+
"datasource": {
|
|
1101
|
+
"type": "prometheus",
|
|
1102
|
+
"uid": "${datasource}"
|
|
1103
|
+
},
|
|
1104
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_get_device_by_network_request_total{job=~\"$job\"}[5m]))",
|
|
1105
|
+
"legendFormat": "get device by network alias",
|
|
1106
|
+
"refId": "B"
|
|
1107
|
+
},
|
|
1108
|
+
{
|
|
1109
|
+
"datasource": {
|
|
1110
|
+
"type": "prometheus",
|
|
1111
|
+
"uid": "${datasource}"
|
|
1112
|
+
},
|
|
1113
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_update_device_request_total{job=~\"$job\"}[5m]))",
|
|
1114
|
+
"legendFormat": "update device",
|
|
1115
|
+
"refId": "C"
|
|
1116
|
+
},
|
|
1117
|
+
{
|
|
1118
|
+
"datasource": {
|
|
1119
|
+
"type": "prometheus",
|
|
1120
|
+
"uid": "${datasource}"
|
|
1121
|
+
},
|
|
1122
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_create_device_request_total{job=~\"$job\"}[5m]))",
|
|
1123
|
+
"legendFormat": "create device",
|
|
1124
|
+
"refId": "D"
|
|
1125
|
+
},
|
|
1126
|
+
{
|
|
1127
|
+
"datasource": {
|
|
1128
|
+
"type": "prometheus",
|
|
1129
|
+
"uid": "${datasource}"
|
|
1130
|
+
},
|
|
1131
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_get_many_to_many_parent_devices_total{job=~\"$job\"}[5m]))",
|
|
1132
|
+
"legendFormat": "many-to-many parents",
|
|
1133
|
+
"refId": "E"
|
|
1134
|
+
}
|
|
1135
|
+
]
|
|
1136
|
+
},
|
|
1137
|
+
{
|
|
1138
|
+
"type": "stat",
|
|
1139
|
+
"title": "Imagine error share",
|
|
1140
|
+
"description": "imagine_error_total over imagine_request_total across the selected range. Both are incremented by every ImagineUtil path, so this is a true service-wide ratio rather than a sample. Reads 0 while no error counter exists at all, which is the healthy state. A step change is worth more than the absolute value; sustained above a few percent, check the HTTP layer below to separate a bad credential (401/403 on every method) from a single failing endpoint.",
|
|
1141
|
+
"id": 17,
|
|
1142
|
+
"datasource": {
|
|
1143
|
+
"type": "prometheus",
|
|
1144
|
+
"uid": "${datasource}"
|
|
1145
|
+
},
|
|
1146
|
+
"gridPos": {
|
|
1147
|
+
"h": 8,
|
|
1148
|
+
"w": 12,
|
|
1149
|
+
"x": 12,
|
|
1150
|
+
"y": 44
|
|
1151
|
+
},
|
|
1152
|
+
"fieldConfig": {
|
|
1153
|
+
"defaults": {
|
|
1154
|
+
"unit": "percentunit",
|
|
1155
|
+
"min": 0
|
|
1156
|
+
},
|
|
1157
|
+
"overrides": []
|
|
1158
|
+
},
|
|
1159
|
+
"options": {
|
|
1160
|
+
"reduceOptions": {
|
|
1161
|
+
"calcs": [
|
|
1162
|
+
"lastNotNull"
|
|
1163
|
+
],
|
|
1164
|
+
"fields": "",
|
|
1165
|
+
"values": false
|
|
1166
|
+
},
|
|
1167
|
+
"textMode": "auto",
|
|
1168
|
+
"colorMode": "value",
|
|
1169
|
+
"graphMode": "area"
|
|
1170
|
+
},
|
|
1171
|
+
"targets": [
|
|
1172
|
+
{
|
|
1173
|
+
"datasource": {
|
|
1174
|
+
"type": "prometheus",
|
|
1175
|
+
"uid": "${datasource}"
|
|
1176
|
+
},
|
|
1177
|
+
"expr": "(sum(increase(pitcrew_ingestor_imagine_error_total{job=~\"$job\"}[$__range])) or vector(0)) / clamp_min((sum(increase(pitcrew_ingestor_imagine_request_total{job=~\"$job\"}[$__range])) or vector(0)), 1e-9)",
|
|
1178
|
+
"legendFormat": "error share",
|
|
1179
|
+
"refId": "A"
|
|
1180
|
+
}
|
|
1181
|
+
]
|
|
1182
|
+
},
|
|
1183
|
+
{
|
|
1184
|
+
"type": "row",
|
|
1185
|
+
"title": "Imagine API: HTTP layer",
|
|
1186
|
+
"id": 18,
|
|
1187
|
+
"gridPos": {
|
|
1188
|
+
"h": 1,
|
|
1189
|
+
"w": 24,
|
|
1190
|
+
"x": 0,
|
|
1191
|
+
"y": 52
|
|
1192
|
+
},
|
|
1193
|
+
"collapsed": false,
|
|
1194
|
+
"panels": []
|
|
1195
|
+
},
|
|
1196
|
+
{
|
|
1197
|
+
"type": "timeseries",
|
|
1198
|
+
"title": "Imagine API calls / sec by method",
|
|
1199
|
+
"description": "Emitted by @leverege/comms NodeComms.preRequest, one counter per HTTP method, so this is every request the Imagine client made - including calls made through the api layer that ImagineUtil does not count itself. That makes it the ground truth for load placed on api-server. patch is the device-data write path; get should sit well below the inbound message rate, because device reads are served from a 5-minute cache.",
|
|
1200
|
+
"id": 19,
|
|
1201
|
+
"datasource": {
|
|
1202
|
+
"type": "prometheus",
|
|
1203
|
+
"uid": "${datasource}"
|
|
1204
|
+
},
|
|
1205
|
+
"gridPos": {
|
|
1206
|
+
"h": 8,
|
|
1207
|
+
"w": 8,
|
|
1208
|
+
"x": 0,
|
|
1209
|
+
"y": 53
|
|
1210
|
+
},
|
|
1211
|
+
"fieldConfig": {
|
|
1212
|
+
"defaults": {
|
|
1213
|
+
"unit": "reqps",
|
|
1214
|
+
"min": 0,
|
|
1215
|
+
"custom": {
|
|
1216
|
+
"drawStyle": "line",
|
|
1217
|
+
"lineWidth": 1,
|
|
1218
|
+
"fillOpacity": 10,
|
|
1219
|
+
"showPoints": "never",
|
|
1220
|
+
"stacking": {
|
|
1221
|
+
"mode": "none"
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
},
|
|
1225
|
+
"overrides": []
|
|
1226
|
+
},
|
|
1227
|
+
"options": {
|
|
1228
|
+
"legend": {
|
|
1229
|
+
"displayMode": "table",
|
|
1230
|
+
"placement": "bottom",
|
|
1231
|
+
"calcs": [
|
|
1232
|
+
"lastNotNull",
|
|
1233
|
+
"max"
|
|
1234
|
+
]
|
|
1235
|
+
},
|
|
1236
|
+
"tooltip": {
|
|
1237
|
+
"mode": "multi",
|
|
1238
|
+
"sort": "desc"
|
|
1239
|
+
}
|
|
1240
|
+
},
|
|
1241
|
+
"targets": [
|
|
1242
|
+
{
|
|
1243
|
+
"datasource": {
|
|
1244
|
+
"type": "prometheus",
|
|
1245
|
+
"uid": "${datasource}"
|
|
1246
|
+
},
|
|
1247
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_api_get{job=~\"$job\"}[5m]))",
|
|
1248
|
+
"legendFormat": "get/s",
|
|
1249
|
+
"refId": "A"
|
|
1250
|
+
},
|
|
1251
|
+
{
|
|
1252
|
+
"datasource": {
|
|
1253
|
+
"type": "prometheus",
|
|
1254
|
+
"uid": "${datasource}"
|
|
1255
|
+
},
|
|
1256
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_api_post{job=~\"$job\"}[5m]))",
|
|
1257
|
+
"legendFormat": "post/s",
|
|
1258
|
+
"refId": "B"
|
|
1259
|
+
},
|
|
1260
|
+
{
|
|
1261
|
+
"datasource": {
|
|
1262
|
+
"type": "prometheus",
|
|
1263
|
+
"uid": "${datasource}"
|
|
1264
|
+
},
|
|
1265
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_api_patch{job=~\"$job\"}[5m]))",
|
|
1266
|
+
"legendFormat": "patch/s",
|
|
1267
|
+
"refId": "C"
|
|
1268
|
+
},
|
|
1269
|
+
{
|
|
1270
|
+
"datasource": {
|
|
1271
|
+
"type": "prometheus",
|
|
1272
|
+
"uid": "${datasource}"
|
|
1273
|
+
},
|
|
1274
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_api_delete{job=~\"$job\"}[5m]))",
|
|
1275
|
+
"legendFormat": "delete/s",
|
|
1276
|
+
"refId": "D"
|
|
1277
|
+
}
|
|
1278
|
+
]
|
|
1279
|
+
},
|
|
1280
|
+
{
|
|
1281
|
+
"type": "timeseries",
|
|
1282
|
+
"title": "Imagine API errors / sec",
|
|
1283
|
+
"description": "NodeComms.postRequest increments <method>_error whenever the response is missing or not ok, so this is EVERY non-2xx and not only transport failures. A 404 from a lookup that legitimately found nothing lands here, so a healthy nonzero floor is normal - compare against the Imagine errors panel above, which counts only what actually threw. 401 or 403 on every method is the Imagine API key or system id being wrong, which otherwise surfaces only as processors quietly failing to resolve devices. On dev only patch_error exists today; the other three are created lazily on their first failure.",
|
|
1284
|
+
"id": 20,
|
|
1285
|
+
"datasource": {
|
|
1286
|
+
"type": "prometheus",
|
|
1287
|
+
"uid": "${datasource}"
|
|
1288
|
+
},
|
|
1289
|
+
"gridPos": {
|
|
1290
|
+
"h": 8,
|
|
1291
|
+
"w": 8,
|
|
1292
|
+
"x": 8,
|
|
1293
|
+
"y": 53
|
|
1294
|
+
},
|
|
1295
|
+
"fieldConfig": {
|
|
1296
|
+
"defaults": {
|
|
1297
|
+
"unit": "reqps",
|
|
1298
|
+
"min": 0,
|
|
1299
|
+
"custom": {
|
|
1300
|
+
"drawStyle": "line",
|
|
1301
|
+
"lineWidth": 1,
|
|
1302
|
+
"fillOpacity": 10,
|
|
1303
|
+
"showPoints": "never",
|
|
1304
|
+
"stacking": {
|
|
1305
|
+
"mode": "none"
|
|
1306
|
+
}
|
|
1307
|
+
}
|
|
1308
|
+
},
|
|
1309
|
+
"overrides": []
|
|
1310
|
+
},
|
|
1311
|
+
"options": {
|
|
1312
|
+
"legend": {
|
|
1313
|
+
"displayMode": "table",
|
|
1314
|
+
"placement": "bottom",
|
|
1315
|
+
"calcs": [
|
|
1316
|
+
"lastNotNull",
|
|
1317
|
+
"max"
|
|
1318
|
+
]
|
|
1319
|
+
},
|
|
1320
|
+
"tooltip": {
|
|
1321
|
+
"mode": "multi",
|
|
1322
|
+
"sort": "desc"
|
|
1323
|
+
}
|
|
1324
|
+
},
|
|
1325
|
+
"targets": [
|
|
1326
|
+
{
|
|
1327
|
+
"datasource": {
|
|
1328
|
+
"type": "prometheus",
|
|
1329
|
+
"uid": "${datasource}"
|
|
1330
|
+
},
|
|
1331
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_api_get_error{job=~\"$job\"}[5m]))",
|
|
1332
|
+
"legendFormat": "get errors/s",
|
|
1333
|
+
"refId": "A"
|
|
1334
|
+
},
|
|
1335
|
+
{
|
|
1336
|
+
"datasource": {
|
|
1337
|
+
"type": "prometheus",
|
|
1338
|
+
"uid": "${datasource}"
|
|
1339
|
+
},
|
|
1340
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_api_post_error{job=~\"$job\"}[5m]))",
|
|
1341
|
+
"legendFormat": "post errors/s",
|
|
1342
|
+
"refId": "B"
|
|
1343
|
+
},
|
|
1344
|
+
{
|
|
1345
|
+
"datasource": {
|
|
1346
|
+
"type": "prometheus",
|
|
1347
|
+
"uid": "${datasource}"
|
|
1348
|
+
},
|
|
1349
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_api_patch_error{job=~\"$job\"}[5m]))",
|
|
1350
|
+
"legendFormat": "patch errors/s",
|
|
1351
|
+
"refId": "C"
|
|
1352
|
+
},
|
|
1353
|
+
{
|
|
1354
|
+
"datasource": {
|
|
1355
|
+
"type": "prometheus",
|
|
1356
|
+
"uid": "${datasource}"
|
|
1357
|
+
},
|
|
1358
|
+
"expr": "sum(rate(pitcrew_ingestor_imagine_api_delete_error{job=~\"$job\"}[5m]))",
|
|
1359
|
+
"legendFormat": "delete errors/s",
|
|
1360
|
+
"refId": "D"
|
|
1361
|
+
}
|
|
1362
|
+
]
|
|
1363
|
+
},
|
|
1364
|
+
{
|
|
1365
|
+
"type": "stat",
|
|
1366
|
+
"title": "Imagine API error share",
|
|
1367
|
+
"description": "Errors over all calls at the HTTP layer, across the selected range. Every term is guarded with or vector(0), because PromQL addition with one missing operand yields no result at all and would blank this panel for a perfectly healthy service - the error counters do not exist until something fails. The floor is whatever the legitimate 404 rate happens to be, so read the step and not the number.",
|
|
1368
|
+
"id": 21,
|
|
1369
|
+
"datasource": {
|
|
1370
|
+
"type": "prometheus",
|
|
1371
|
+
"uid": "${datasource}"
|
|
1372
|
+
},
|
|
1373
|
+
"gridPos": {
|
|
1374
|
+
"h": 8,
|
|
1375
|
+
"w": 8,
|
|
1376
|
+
"x": 16,
|
|
1377
|
+
"y": 53
|
|
1378
|
+
},
|
|
1379
|
+
"fieldConfig": {
|
|
1380
|
+
"defaults": {
|
|
1381
|
+
"unit": "percentunit",
|
|
1382
|
+
"min": 0
|
|
1383
|
+
},
|
|
1384
|
+
"overrides": []
|
|
1385
|
+
},
|
|
1386
|
+
"options": {
|
|
1387
|
+
"reduceOptions": {
|
|
1388
|
+
"calcs": [
|
|
1389
|
+
"lastNotNull"
|
|
1390
|
+
],
|
|
1391
|
+
"fields": "",
|
|
1392
|
+
"values": false
|
|
1393
|
+
},
|
|
1394
|
+
"textMode": "auto",
|
|
1395
|
+
"colorMode": "value",
|
|
1396
|
+
"graphMode": "area"
|
|
1397
|
+
},
|
|
1398
|
+
"targets": [
|
|
1399
|
+
{
|
|
1400
|
+
"datasource": {
|
|
1401
|
+
"type": "prometheus",
|
|
1402
|
+
"uid": "${datasource}"
|
|
1403
|
+
},
|
|
1404
|
+
"expr": "((sum(increase(pitcrew_ingestor_imagine_api_get_error{job=~\"$job\"}[$__range])) or vector(0)) + (sum(increase(pitcrew_ingestor_imagine_api_post_error{job=~\"$job\"}[$__range])) or vector(0)) + (sum(increase(pitcrew_ingestor_imagine_api_patch_error{job=~\"$job\"}[$__range])) or vector(0)) + (sum(increase(pitcrew_ingestor_imagine_api_delete_error{job=~\"$job\"}[$__range])) or vector(0))) / clamp_min((sum(increase(pitcrew_ingestor_imagine_api_get{job=~\"$job\"}[$__range])) or vector(0)) + (sum(increase(pitcrew_ingestor_imagine_api_post{job=~\"$job\"}[$__range])) or vector(0)) + (sum(increase(pitcrew_ingestor_imagine_api_patch{job=~\"$job\"}[$__range])) or vector(0)) + (sum(increase(pitcrew_ingestor_imagine_api_delete{job=~\"$job\"}[$__range])) or vector(0)), 1e-9)",
|
|
1405
|
+
"legendFormat": "error share",
|
|
1406
|
+
"refId": "A"
|
|
1407
|
+
}
|
|
1408
|
+
]
|
|
1409
|
+
},
|
|
1410
|
+
{
|
|
1411
|
+
"type": "row",
|
|
1412
|
+
"title": "Service health",
|
|
1413
|
+
"id": 22,
|
|
1414
|
+
"gridPos": {
|
|
1415
|
+
"h": 1,
|
|
1416
|
+
"w": 24,
|
|
1417
|
+
"x": 0,
|
|
1418
|
+
"y": 61
|
|
1419
|
+
},
|
|
1420
|
+
"collapsed": false,
|
|
1421
|
+
"panels": []
|
|
1422
|
+
},
|
|
1423
|
+
{
|
|
1424
|
+
"type": "timeseries",
|
|
1425
|
+
"title": "Event loop lag",
|
|
1426
|
+
"description": "The ordered queues drain on this event loop, so lag here is directly messages sitting behind a correlation id. It is also the first place a slow api-server shows up: every drain awaits a device read, so latency on the Imagine side arrives here as lag before it arrives anywhere else.",
|
|
1427
|
+
"id": 23,
|
|
1428
|
+
"datasource": {
|
|
1429
|
+
"type": "prometheus",
|
|
1430
|
+
"uid": "${datasource}"
|
|
1431
|
+
},
|
|
1432
|
+
"gridPos": {
|
|
1433
|
+
"h": 8,
|
|
1434
|
+
"w": 8,
|
|
1435
|
+
"x": 0,
|
|
1436
|
+
"y": 62
|
|
1437
|
+
},
|
|
1438
|
+
"fieldConfig": {
|
|
1439
|
+
"defaults": {
|
|
1440
|
+
"unit": "s",
|
|
1441
|
+
"min": 0,
|
|
1442
|
+
"custom": {
|
|
1443
|
+
"drawStyle": "line",
|
|
1444
|
+
"lineWidth": 1,
|
|
1445
|
+
"fillOpacity": 10,
|
|
1446
|
+
"showPoints": "never",
|
|
1447
|
+
"stacking": {
|
|
1448
|
+
"mode": "none"
|
|
1449
|
+
}
|
|
1450
|
+
}
|
|
1451
|
+
},
|
|
1452
|
+
"overrides": []
|
|
1453
|
+
},
|
|
1454
|
+
"options": {
|
|
1455
|
+
"legend": {
|
|
1456
|
+
"displayMode": "table",
|
|
1457
|
+
"placement": "bottom",
|
|
1458
|
+
"calcs": [
|
|
1459
|
+
"lastNotNull",
|
|
1460
|
+
"max"
|
|
1461
|
+
]
|
|
1462
|
+
},
|
|
1463
|
+
"tooltip": {
|
|
1464
|
+
"mode": "multi",
|
|
1465
|
+
"sort": "desc"
|
|
1466
|
+
}
|
|
1467
|
+
},
|
|
1468
|
+
"targets": [
|
|
1469
|
+
{
|
|
1470
|
+
"datasource": {
|
|
1471
|
+
"type": "prometheus",
|
|
1472
|
+
"uid": "${datasource}"
|
|
1473
|
+
},
|
|
1474
|
+
"expr": "max(nodejs_eventloop_lag_p99_seconds{job=~\"$job\"})",
|
|
1475
|
+
"legendFormat": "p99",
|
|
1476
|
+
"refId": "A"
|
|
1477
|
+
},
|
|
1478
|
+
{
|
|
1479
|
+
"datasource": {
|
|
1480
|
+
"type": "prometheus",
|
|
1481
|
+
"uid": "${datasource}"
|
|
1482
|
+
},
|
|
1483
|
+
"expr": "max(nodejs_eventloop_lag_mean_seconds{job=~\"$job\"})",
|
|
1484
|
+
"legendFormat": "mean",
|
|
1485
|
+
"refId": "B"
|
|
1486
|
+
}
|
|
1487
|
+
]
|
|
1488
|
+
},
|
|
1489
|
+
{
|
|
1490
|
+
"type": "timeseries",
|
|
1491
|
+
"title": "Heap used",
|
|
1492
|
+
"description": "Per pod, deliberately unaggregated: an unbounded in-memory queue backing up on a single replica is exactly what you are looking for, and a sum would hide it. Read alongside the ordered queue backlog panel - heap climbing on one pod while the others sit flat is a single correlation id wedged behind a slow write.",
|
|
1493
|
+
"id": 24,
|
|
1494
|
+
"datasource": {
|
|
1495
|
+
"type": "prometheus",
|
|
1496
|
+
"uid": "${datasource}"
|
|
1497
|
+
},
|
|
1498
|
+
"gridPos": {
|
|
1499
|
+
"h": 8,
|
|
1500
|
+
"w": 8,
|
|
1501
|
+
"x": 8,
|
|
1502
|
+
"y": 62
|
|
1503
|
+
},
|
|
1504
|
+
"fieldConfig": {
|
|
1505
|
+
"defaults": {
|
|
1506
|
+
"unit": "bytes",
|
|
1507
|
+
"min": 0,
|
|
1508
|
+
"custom": {
|
|
1509
|
+
"drawStyle": "line",
|
|
1510
|
+
"lineWidth": 1,
|
|
1511
|
+
"fillOpacity": 10,
|
|
1512
|
+
"showPoints": "never",
|
|
1513
|
+
"stacking": {
|
|
1514
|
+
"mode": "none"
|
|
1515
|
+
}
|
|
1516
|
+
}
|
|
1517
|
+
},
|
|
1518
|
+
"overrides": []
|
|
1519
|
+
},
|
|
1520
|
+
"options": {
|
|
1521
|
+
"legend": {
|
|
1522
|
+
"displayMode": "table",
|
|
1523
|
+
"placement": "bottom",
|
|
1524
|
+
"calcs": [
|
|
1525
|
+
"lastNotNull",
|
|
1526
|
+
"max"
|
|
1527
|
+
]
|
|
1528
|
+
},
|
|
1529
|
+
"tooltip": {
|
|
1530
|
+
"mode": "multi",
|
|
1531
|
+
"sort": "desc"
|
|
1532
|
+
}
|
|
1533
|
+
},
|
|
1534
|
+
"targets": [
|
|
1535
|
+
{
|
|
1536
|
+
"datasource": {
|
|
1537
|
+
"type": "prometheus",
|
|
1538
|
+
"uid": "${datasource}"
|
|
1539
|
+
},
|
|
1540
|
+
"expr": "nodejs_heap_size_used_bytes{job=~\"$job\"}",
|
|
1541
|
+
"legendFormat": "{{pod}}",
|
|
1542
|
+
"refId": "A"
|
|
1543
|
+
}
|
|
1544
|
+
]
|
|
1545
|
+
},
|
|
1546
|
+
{
|
|
1547
|
+
"type": "timeseries",
|
|
1548
|
+
"title": "Pods reporting and CPU",
|
|
1549
|
+
"description": "A pod restart loses whatever the in-memory ordered queues were holding, and those messages were acknowledged on receipt, so nothing redelivers them: a restart is silent data loss, visible here as a step. The service autoscales on CPU (75% target, 1 to 4 replicas on dev), so the two lines are meant to move together; CPU climbing while the pod count sits at its ceiling is the case to escalate.",
|
|
1550
|
+
"id": 25,
|
|
1551
|
+
"datasource": {
|
|
1552
|
+
"type": "prometheus",
|
|
1553
|
+
"uid": "${datasource}"
|
|
1554
|
+
},
|
|
1555
|
+
"gridPos": {
|
|
1556
|
+
"h": 8,
|
|
1557
|
+
"w": 8,
|
|
1558
|
+
"x": 16,
|
|
1559
|
+
"y": 62
|
|
1560
|
+
},
|
|
1561
|
+
"fieldConfig": {
|
|
1562
|
+
"defaults": {
|
|
1563
|
+
"unit": "short",
|
|
1564
|
+
"min": 0,
|
|
1565
|
+
"custom": {
|
|
1566
|
+
"drawStyle": "line",
|
|
1567
|
+
"lineWidth": 1,
|
|
1568
|
+
"fillOpacity": 10,
|
|
1569
|
+
"showPoints": "never",
|
|
1570
|
+
"stacking": {
|
|
1571
|
+
"mode": "none"
|
|
1572
|
+
}
|
|
1573
|
+
}
|
|
1574
|
+
},
|
|
1575
|
+
"overrides": []
|
|
1576
|
+
},
|
|
1577
|
+
"options": {
|
|
1578
|
+
"legend": {
|
|
1579
|
+
"displayMode": "table",
|
|
1580
|
+
"placement": "bottom",
|
|
1581
|
+
"calcs": [
|
|
1582
|
+
"lastNotNull",
|
|
1583
|
+
"max"
|
|
1584
|
+
]
|
|
1585
|
+
},
|
|
1586
|
+
"tooltip": {
|
|
1587
|
+
"mode": "multi",
|
|
1588
|
+
"sort": "desc"
|
|
1589
|
+
}
|
|
1590
|
+
},
|
|
1591
|
+
"targets": [
|
|
1592
|
+
{
|
|
1593
|
+
"datasource": {
|
|
1594
|
+
"type": "prometheus",
|
|
1595
|
+
"uid": "${datasource}"
|
|
1596
|
+
},
|
|
1597
|
+
"expr": "count(count by (pod) (nodejs_heap_size_used_bytes{job=~\"$job\"}))",
|
|
1598
|
+
"legendFormat": "pods reporting",
|
|
1599
|
+
"refId": "A"
|
|
1600
|
+
},
|
|
1601
|
+
{
|
|
1602
|
+
"datasource": {
|
|
1603
|
+
"type": "prometheus",
|
|
1604
|
+
"uid": "${datasource}"
|
|
1605
|
+
},
|
|
1606
|
+
"expr": "sum(rate(process_cpu_seconds_total{job=~\"$job\"}[5m]))",
|
|
1607
|
+
"legendFormat": "cpu cores",
|
|
1608
|
+
"refId": "B"
|
|
1609
|
+
}
|
|
1610
|
+
]
|
|
1611
|
+
}
|
|
1612
|
+
]
|
|
1613
|
+
}
|