@leverege/build-tools 2.120.0 → 2.121.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -0,0 +1,1137 @@
|
|
|
1
|
+
{
|
|
2
|
+
"annotations": {
|
|
3
|
+
"list": []
|
|
4
|
+
},
|
|
5
|
+
"description": "Pub/Sub intake, ordered-queue processing, Imagine API load and resource uploads for the pitcrew-vision-logic-server. Metric names carry no service prefix, so every panel is scoped by job.",
|
|
6
|
+
"editable": true,
|
|
7
|
+
"graphTooltip": 1,
|
|
8
|
+
"links": [],
|
|
9
|
+
"panels": [
|
|
10
|
+
{
|
|
11
|
+
"type": "row",
|
|
12
|
+
"title": "Service health",
|
|
13
|
+
"id": 1,
|
|
14
|
+
"gridPos": {
|
|
15
|
+
"h": 1,
|
|
16
|
+
"w": 24,
|
|
17
|
+
"x": 0,
|
|
18
|
+
"y": 0
|
|
19
|
+
},
|
|
20
|
+
"collapsed": false,
|
|
21
|
+
"panels": []
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"type": "stat",
|
|
25
|
+
"title": "Pods scraped",
|
|
26
|
+
"description": "Targets currently answering /metrics for this job. This is the pod count Prometheus can see, which is what every rate on this dashboard is summed over. If it disagrees with the replica count in the autoscaling row, a pod is up but not being scraped, and every panel here is understating the service.",
|
|
27
|
+
"id": 2,
|
|
28
|
+
"datasource": {
|
|
29
|
+
"type": "prometheus",
|
|
30
|
+
"uid": "${datasource}"
|
|
31
|
+
},
|
|
32
|
+
"gridPos": {
|
|
33
|
+
"h": 8,
|
|
34
|
+
"w": 6,
|
|
35
|
+
"x": 0,
|
|
36
|
+
"y": 1
|
|
37
|
+
},
|
|
38
|
+
"fieldConfig": {
|
|
39
|
+
"defaults": {
|
|
40
|
+
"unit": "short",
|
|
41
|
+
"min": 0
|
|
42
|
+
},
|
|
43
|
+
"overrides": []
|
|
44
|
+
},
|
|
45
|
+
"options": {
|
|
46
|
+
"reduceOptions": {
|
|
47
|
+
"calcs": [
|
|
48
|
+
"lastNotNull"
|
|
49
|
+
],
|
|
50
|
+
"fields": "",
|
|
51
|
+
"values": false
|
|
52
|
+
},
|
|
53
|
+
"textMode": "auto",
|
|
54
|
+
"colorMode": "value",
|
|
55
|
+
"graphMode": "area"
|
|
56
|
+
},
|
|
57
|
+
"targets": [
|
|
58
|
+
{
|
|
59
|
+
"datasource": {
|
|
60
|
+
"type": "prometheus",
|
|
61
|
+
"uid": "${datasource}"
|
|
62
|
+
},
|
|
63
|
+
"expr": "sum(up{job=~\"$job\"})",
|
|
64
|
+
"legendFormat": "up",
|
|
65
|
+
"refId": "A"
|
|
66
|
+
}
|
|
67
|
+
]
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"type": "stat",
|
|
71
|
+
"title": "Deployed version",
|
|
72
|
+
"description": "The app version label the ServiceMonitor copies off the pod. Two values at once means a rollout is mid-flight, and counters from the two builds are being summed together in every other panel.",
|
|
73
|
+
"id": 3,
|
|
74
|
+
"datasource": {
|
|
75
|
+
"type": "prometheus",
|
|
76
|
+
"uid": "${datasource}"
|
|
77
|
+
},
|
|
78
|
+
"gridPos": {
|
|
79
|
+
"h": 8,
|
|
80
|
+
"w": 6,
|
|
81
|
+
"x": 6,
|
|
82
|
+
"y": 1
|
|
83
|
+
},
|
|
84
|
+
"fieldConfig": {
|
|
85
|
+
"defaults": {
|
|
86
|
+
"unit": "short"
|
|
87
|
+
},
|
|
88
|
+
"overrides": []
|
|
89
|
+
},
|
|
90
|
+
"options": {
|
|
91
|
+
"reduceOptions": {
|
|
92
|
+
"calcs": [
|
|
93
|
+
"lastNotNull"
|
|
94
|
+
],
|
|
95
|
+
"fields": "",
|
|
96
|
+
"values": false
|
|
97
|
+
},
|
|
98
|
+
"textMode": "name",
|
|
99
|
+
"colorMode": "value",
|
|
100
|
+
"graphMode": "none"
|
|
101
|
+
},
|
|
102
|
+
"targets": [
|
|
103
|
+
{
|
|
104
|
+
"datasource": {
|
|
105
|
+
"type": "prometheus",
|
|
106
|
+
"uid": "${datasource}"
|
|
107
|
+
},
|
|
108
|
+
"expr": "max by (app_kubernetes_io_version) (up{job=~\"$job\"})",
|
|
109
|
+
"legendFormat": "{{app_kubernetes_io_version}}",
|
|
110
|
+
"refId": "A"
|
|
111
|
+
}
|
|
112
|
+
]
|
|
113
|
+
},
|
|
114
|
+
{
|
|
115
|
+
"type": "timeseries",
|
|
116
|
+
"title": "Scrape health per pod",
|
|
117
|
+
"description": "1 while the pod answers /metrics, 0 when the scrape fails. A pod that drops to 0 keeps its last counter values in Prometheus, so rates go flat rather than falling. Gaps here explain sudden steps in the throughput panels.",
|
|
118
|
+
"id": 4,
|
|
119
|
+
"datasource": {
|
|
120
|
+
"type": "prometheus",
|
|
121
|
+
"uid": "${datasource}"
|
|
122
|
+
},
|
|
123
|
+
"gridPos": {
|
|
124
|
+
"h": 8,
|
|
125
|
+
"w": 12,
|
|
126
|
+
"x": 12,
|
|
127
|
+
"y": 1
|
|
128
|
+
},
|
|
129
|
+
"fieldConfig": {
|
|
130
|
+
"defaults": {
|
|
131
|
+
"unit": "short",
|
|
132
|
+
"min": 0
|
|
133
|
+
},
|
|
134
|
+
"overrides": []
|
|
135
|
+
},
|
|
136
|
+
"targets": [
|
|
137
|
+
{
|
|
138
|
+
"datasource": {
|
|
139
|
+
"type": "prometheus",
|
|
140
|
+
"uid": "${datasource}"
|
|
141
|
+
},
|
|
142
|
+
"expr": "up{job=~\"$job\"}",
|
|
143
|
+
"legendFormat": "{{pod}}",
|
|
144
|
+
"refId": "A"
|
|
145
|
+
}
|
|
146
|
+
]
|
|
147
|
+
},
|
|
148
|
+
{
|
|
149
|
+
"type": "row",
|
|
150
|
+
"title": "Intake and autoscaling",
|
|
151
|
+
"id": 5,
|
|
152
|
+
"gridPos": {
|
|
153
|
+
"h": 1,
|
|
154
|
+
"w": 24,
|
|
155
|
+
"x": 0,
|
|
156
|
+
"y": 9
|
|
157
|
+
},
|
|
158
|
+
"collapsed": false,
|
|
159
|
+
"panels": []
|
|
160
|
+
},
|
|
161
|
+
{
|
|
162
|
+
"type": "timeseries",
|
|
163
|
+
"title": "Subscription backlog",
|
|
164
|
+
"description": "Undelivered messages on the Pub/Sub subscription, exported from Cloud Monitoring by the stackdriver-exporter. This is the same series prometheus-adapter serves to the HPA, so this is literally what drives scale-up, not a proxy for it.",
|
|
165
|
+
"id": 6,
|
|
166
|
+
"datasource": {
|
|
167
|
+
"type": "prometheus",
|
|
168
|
+
"uid": "${datasource}"
|
|
169
|
+
},
|
|
170
|
+
"gridPos": {
|
|
171
|
+
"h": 8,
|
|
172
|
+
"w": 12,
|
|
173
|
+
"x": 0,
|
|
174
|
+
"y": 10
|
|
175
|
+
},
|
|
176
|
+
"fieldConfig": {
|
|
177
|
+
"defaults": {
|
|
178
|
+
"unit": "short",
|
|
179
|
+
"min": 0
|
|
180
|
+
},
|
|
181
|
+
"overrides": []
|
|
182
|
+
},
|
|
183
|
+
"targets": [
|
|
184
|
+
{
|
|
185
|
+
"datasource": {
|
|
186
|
+
"type": "prometheus",
|
|
187
|
+
"uid": "${datasource}"
|
|
188
|
+
},
|
|
189
|
+
"expr": "sum(pubsub_subscription_num_undelivered_messages{subscription_id=~\"$subscription\"})",
|
|
190
|
+
"legendFormat": "backlog",
|
|
191
|
+
"refId": "A"
|
|
192
|
+
}
|
|
193
|
+
]
|
|
194
|
+
},
|
|
195
|
+
{
|
|
196
|
+
"type": "timeseries",
|
|
197
|
+
"title": "Backlog per pod vs HPA target",
|
|
198
|
+
"description": "Backlog divided by scraped pods, which is the quantity the HPA compares against pubsubBacklog.targetPerPod (30 in dev and stg). Sustained time above the target line with replicas already at max is the signal that the ceiling is too low, not that the service is slow.",
|
|
199
|
+
"id": 7,
|
|
200
|
+
"datasource": {
|
|
201
|
+
"type": "prometheus",
|
|
202
|
+
"uid": "${datasource}"
|
|
203
|
+
},
|
|
204
|
+
"gridPos": {
|
|
205
|
+
"h": 8,
|
|
206
|
+
"w": 12,
|
|
207
|
+
"x": 12,
|
|
208
|
+
"y": 10
|
|
209
|
+
},
|
|
210
|
+
"fieldConfig": {
|
|
211
|
+
"defaults": {
|
|
212
|
+
"unit": "short",
|
|
213
|
+
"min": 0
|
|
214
|
+
},
|
|
215
|
+
"overrides": []
|
|
216
|
+
},
|
|
217
|
+
"targets": [
|
|
218
|
+
{
|
|
219
|
+
"datasource": {
|
|
220
|
+
"type": "prometheus",
|
|
221
|
+
"uid": "${datasource}"
|
|
222
|
+
},
|
|
223
|
+
"expr": "sum(pubsub_subscription_num_undelivered_messages{subscription_id=~\"$subscription\"}) / clamp_min(sum(up{job=~\"$job\"}), 1)",
|
|
224
|
+
"legendFormat": "backlog/pod",
|
|
225
|
+
"refId": "A"
|
|
226
|
+
}
|
|
227
|
+
]
|
|
228
|
+
},
|
|
229
|
+
{
|
|
230
|
+
"type": "timeseries",
|
|
231
|
+
"title": "Topic publish rate",
|
|
232
|
+
"description": "Messages/sec published to the feeding topic, the HPA's rate floor. It climbs before the backlog does, so a spike here that is not followed by a replica increase means the rate rule is not firing.",
|
|
233
|
+
"id": 8,
|
|
234
|
+
"datasource": {
|
|
235
|
+
"type": "prometheus",
|
|
236
|
+
"uid": "${datasource}"
|
|
237
|
+
},
|
|
238
|
+
"gridPos": {
|
|
239
|
+
"h": 8,
|
|
240
|
+
"w": 12,
|
|
241
|
+
"x": 0,
|
|
242
|
+
"y": 18
|
|
243
|
+
},
|
|
244
|
+
"fieldConfig": {
|
|
245
|
+
"defaults": {
|
|
246
|
+
"unit": "reqps",
|
|
247
|
+
"min": 0
|
|
248
|
+
},
|
|
249
|
+
"overrides": []
|
|
250
|
+
},
|
|
251
|
+
"targets": [
|
|
252
|
+
{
|
|
253
|
+
"datasource": {
|
|
254
|
+
"type": "prometheus",
|
|
255
|
+
"uid": "${datasource}"
|
|
256
|
+
},
|
|
257
|
+
"expr": "sum(pubsub_topic_publish_rate{topic_id=~\"$topic\"})",
|
|
258
|
+
"legendFormat": "publish/s",
|
|
259
|
+
"refId": "A"
|
|
260
|
+
}
|
|
261
|
+
]
|
|
262
|
+
},
|
|
263
|
+
{
|
|
264
|
+
"type": "timeseries",
|
|
265
|
+
"title": "Replicas",
|
|
266
|
+
"description": "Current, HPA-desired and the configured ceiling. desired pinned to max while the backlog grows is the classic under-provisioned shape. current below desired for more than a minute or two means pods are not getting scheduled, which on the preemptible pool usually means no spot capacity.",
|
|
267
|
+
"id": 9,
|
|
268
|
+
"datasource": {
|
|
269
|
+
"type": "prometheus",
|
|
270
|
+
"uid": "${datasource}"
|
|
271
|
+
},
|
|
272
|
+
"gridPos": {
|
|
273
|
+
"h": 8,
|
|
274
|
+
"w": 12,
|
|
275
|
+
"x": 12,
|
|
276
|
+
"y": 18
|
|
277
|
+
},
|
|
278
|
+
"fieldConfig": {
|
|
279
|
+
"defaults": {
|
|
280
|
+
"unit": "short",
|
|
281
|
+
"min": 0
|
|
282
|
+
},
|
|
283
|
+
"overrides": []
|
|
284
|
+
},
|
|
285
|
+
"targets": [
|
|
286
|
+
{
|
|
287
|
+
"datasource": {
|
|
288
|
+
"type": "prometheus",
|
|
289
|
+
"uid": "${datasource}"
|
|
290
|
+
},
|
|
291
|
+
"expr": "sum(kube_deployment_status_replicas{deployment=\"pitcrew-vision-logic-server\"})",
|
|
292
|
+
"legendFormat": "current",
|
|
293
|
+
"refId": "A"
|
|
294
|
+
},
|
|
295
|
+
{
|
|
296
|
+
"datasource": {
|
|
297
|
+
"type": "prometheus",
|
|
298
|
+
"uid": "${datasource}"
|
|
299
|
+
},
|
|
300
|
+
"expr": "sum(kube_horizontalpodautoscaler_status_desired_replicas{horizontalpodautoscaler=\"pitcrew-vision-logic-server\"})",
|
|
301
|
+
"legendFormat": "hpa desired",
|
|
302
|
+
"refId": "B"
|
|
303
|
+
},
|
|
304
|
+
{
|
|
305
|
+
"datasource": {
|
|
306
|
+
"type": "prometheus",
|
|
307
|
+
"uid": "${datasource}"
|
|
308
|
+
},
|
|
309
|
+
"expr": "sum(kube_horizontalpodautoscaler_spec_max_replicas{horizontalpodautoscaler=\"pitcrew-vision-logic-server\"})",
|
|
310
|
+
"legendFormat": "max",
|
|
311
|
+
"refId": "C"
|
|
312
|
+
}
|
|
313
|
+
]
|
|
314
|
+
},
|
|
315
|
+
{
|
|
316
|
+
"type": "row",
|
|
317
|
+
"title": "Ordered queue: the processing pipeline",
|
|
318
|
+
"id": 10,
|
|
319
|
+
"gridPos": {
|
|
320
|
+
"h": 1,
|
|
321
|
+
"w": 24,
|
|
322
|
+
"x": 0,
|
|
323
|
+
"y": 26
|
|
324
|
+
},
|
|
325
|
+
"collapsed": false,
|
|
326
|
+
"panels": []
|
|
327
|
+
},
|
|
328
|
+
{
|
|
329
|
+
"type": "timeseries",
|
|
330
|
+
"title": "Queue throughput / sec",
|
|
331
|
+
"description": "enqueue is work accepted off the subscription, dequeue is work pulled for execution, processed is work finished. In steady state all three sit on top of each other. enqueue running above processed is the only shape that accumulates, and it is what the in-flight panel beside this one measures.",
|
|
332
|
+
"id": 11,
|
|
333
|
+
"datasource": {
|
|
334
|
+
"type": "prometheus",
|
|
335
|
+
"uid": "${datasource}"
|
|
336
|
+
},
|
|
337
|
+
"gridPos": {
|
|
338
|
+
"h": 8,
|
|
339
|
+
"w": 12,
|
|
340
|
+
"x": 0,
|
|
341
|
+
"y": 27
|
|
342
|
+
},
|
|
343
|
+
"fieldConfig": {
|
|
344
|
+
"defaults": {
|
|
345
|
+
"unit": "reqps",
|
|
346
|
+
"min": 0
|
|
347
|
+
},
|
|
348
|
+
"overrides": []
|
|
349
|
+
},
|
|
350
|
+
"targets": [
|
|
351
|
+
{
|
|
352
|
+
"datasource": {
|
|
353
|
+
"type": "prometheus",
|
|
354
|
+
"uid": "${datasource}"
|
|
355
|
+
},
|
|
356
|
+
"expr": "sum(rate(ordered_q_enqueue_count_total{job=~\"$job\"}[5m]))",
|
|
357
|
+
"legendFormat": "enqueue/s",
|
|
358
|
+
"refId": "A"
|
|
359
|
+
},
|
|
360
|
+
{
|
|
361
|
+
"datasource": {
|
|
362
|
+
"type": "prometheus",
|
|
363
|
+
"uid": "${datasource}"
|
|
364
|
+
},
|
|
365
|
+
"expr": "sum(rate(ordered_q_dequeue_count_total{job=~\"$job\"}[5m]))",
|
|
366
|
+
"legendFormat": "dequeue/s",
|
|
367
|
+
"refId": "B"
|
|
368
|
+
},
|
|
369
|
+
{
|
|
370
|
+
"datasource": {
|
|
371
|
+
"type": "prometheus",
|
|
372
|
+
"uid": "${datasource}"
|
|
373
|
+
},
|
|
374
|
+
"expr": "sum(rate(ordered_q_processed_count_total{job=~\"$job\"}[5m]))",
|
|
375
|
+
"legendFormat": "processed/s",
|
|
376
|
+
"refId": "C"
|
|
377
|
+
}
|
|
378
|
+
]
|
|
379
|
+
},
|
|
380
|
+
{
|
|
381
|
+
"type": "timeseries",
|
|
382
|
+
"title": "In-flight work",
|
|
383
|
+
"description": "enqueued minus processed, clamped at zero so a pod restart reads flat instead of deeply negative. Counters are per-pod and reset on restart, so treat a step change right after a rollout as an artifact. A steadily rising line is genuine backlog inside the process, upstream of the Pub/Sub backlog.",
|
|
384
|
+
"id": 12,
|
|
385
|
+
"datasource": {
|
|
386
|
+
"type": "prometheus",
|
|
387
|
+
"uid": "${datasource}"
|
|
388
|
+
},
|
|
389
|
+
"gridPos": {
|
|
390
|
+
"h": 8,
|
|
391
|
+
"w": 12,
|
|
392
|
+
"x": 12,
|
|
393
|
+
"y": 27
|
|
394
|
+
},
|
|
395
|
+
"fieldConfig": {
|
|
396
|
+
"defaults": {
|
|
397
|
+
"unit": "short",
|
|
398
|
+
"min": 0
|
|
399
|
+
},
|
|
400
|
+
"overrides": []
|
|
401
|
+
},
|
|
402
|
+
"targets": [
|
|
403
|
+
{
|
|
404
|
+
"datasource": {
|
|
405
|
+
"type": "prometheus",
|
|
406
|
+
"uid": "${datasource}"
|
|
407
|
+
},
|
|
408
|
+
"expr": "clamp_min(sum(ordered_q_enqueue_count_total{job=~\"$job\"}) - sum(ordered_q_processed_count_total{job=~\"$job\"}), 0)",
|
|
409
|
+
"legendFormat": "in flight",
|
|
410
|
+
"refId": "A"
|
|
411
|
+
}
|
|
412
|
+
]
|
|
413
|
+
},
|
|
414
|
+
{
|
|
415
|
+
"type": "timeseries",
|
|
416
|
+
"title": "Active processing queues",
|
|
417
|
+
"description": "The ordered-queue gauge: how many per-key queues have work in them right now. Ordering is per key, so this is also the effective parallelism. Pinned at 0 while messages are arriving means work is being dropped before it reaches the queue; a value that only climbs means keys are never draining.",
|
|
418
|
+
"id": 13,
|
|
419
|
+
"datasource": {
|
|
420
|
+
"type": "prometheus",
|
|
421
|
+
"uid": "${datasource}"
|
|
422
|
+
},
|
|
423
|
+
"gridPos": {
|
|
424
|
+
"h": 8,
|
|
425
|
+
"w": 12,
|
|
426
|
+
"x": 0,
|
|
427
|
+
"y": 35
|
|
428
|
+
},
|
|
429
|
+
"fieldConfig": {
|
|
430
|
+
"defaults": {
|
|
431
|
+
"unit": "short",
|
|
432
|
+
"min": 0
|
|
433
|
+
},
|
|
434
|
+
"overrides": []
|
|
435
|
+
},
|
|
436
|
+
"targets": [
|
|
437
|
+
{
|
|
438
|
+
"datasource": {
|
|
439
|
+
"type": "prometheus",
|
|
440
|
+
"uid": "${datasource}"
|
|
441
|
+
},
|
|
442
|
+
"expr": "sum(ordered_q_processing_queues{job=~\"$job\"})",
|
|
443
|
+
"legendFormat": "queues",
|
|
444
|
+
"refId": "A"
|
|
445
|
+
},
|
|
446
|
+
{
|
|
447
|
+
"datasource": {
|
|
448
|
+
"type": "prometheus",
|
|
449
|
+
"uid": "${datasource}"
|
|
450
|
+
},
|
|
451
|
+
"expr": "max(ordered_q_processing_queues{job=~\"$job\"})",
|
|
452
|
+
"legendFormat": "busiest pod",
|
|
453
|
+
"refId": "B"
|
|
454
|
+
}
|
|
455
|
+
]
|
|
456
|
+
},
|
|
457
|
+
{
|
|
458
|
+
"type": "stat",
|
|
459
|
+
"title": "Queue totals over the range",
|
|
460
|
+
"description": "Enqueued against processed for the selected window. A gap between them is work that entered and has not come out, and should match the in-flight panel. Both read No data if the service has not run in the window, since leverege-metrics creates counters lazily on first increment.",
|
|
461
|
+
"id": 14,
|
|
462
|
+
"datasource": {
|
|
463
|
+
"type": "prometheus",
|
|
464
|
+
"uid": "${datasource}"
|
|
465
|
+
},
|
|
466
|
+
"gridPos": {
|
|
467
|
+
"h": 8,
|
|
468
|
+
"w": 12,
|
|
469
|
+
"x": 12,
|
|
470
|
+
"y": 35
|
|
471
|
+
},
|
|
472
|
+
"fieldConfig": {
|
|
473
|
+
"defaults": {
|
|
474
|
+
"unit": "short",
|
|
475
|
+
"min": 0
|
|
476
|
+
},
|
|
477
|
+
"overrides": []
|
|
478
|
+
},
|
|
479
|
+
"options": {
|
|
480
|
+
"reduceOptions": {
|
|
481
|
+
"calcs": [
|
|
482
|
+
"lastNotNull"
|
|
483
|
+
],
|
|
484
|
+
"fields": "",
|
|
485
|
+
"values": false
|
|
486
|
+
},
|
|
487
|
+
"textMode": "auto",
|
|
488
|
+
"colorMode": "value",
|
|
489
|
+
"graphMode": "area"
|
|
490
|
+
},
|
|
491
|
+
"targets": [
|
|
492
|
+
{
|
|
493
|
+
"datasource": {
|
|
494
|
+
"type": "prometheus",
|
|
495
|
+
"uid": "${datasource}"
|
|
496
|
+
},
|
|
497
|
+
"expr": "sum(increase(ordered_q_enqueue_count_total{job=~\"$job\"}[$__range]))",
|
|
498
|
+
"legendFormat": "enqueued",
|
|
499
|
+
"refId": "A"
|
|
500
|
+
},
|
|
501
|
+
{
|
|
502
|
+
"datasource": {
|
|
503
|
+
"type": "prometheus",
|
|
504
|
+
"uid": "${datasource}"
|
|
505
|
+
},
|
|
506
|
+
"expr": "sum(increase(ordered_q_processed_count_total{job=~\"$job\"}[$__range]))",
|
|
507
|
+
"legendFormat": "processed",
|
|
508
|
+
"refId": "B"
|
|
509
|
+
}
|
|
510
|
+
]
|
|
511
|
+
},
|
|
512
|
+
{
|
|
513
|
+
"type": "row",
|
|
514
|
+
"title": "Imagine API: what this service asks of api-server",
|
|
515
|
+
"id": 15,
|
|
516
|
+
"gridPos": {
|
|
517
|
+
"h": 1,
|
|
518
|
+
"w": 24,
|
|
519
|
+
"x": 0,
|
|
520
|
+
"y": 43
|
|
521
|
+
},
|
|
522
|
+
"collapsed": false,
|
|
523
|
+
"panels": []
|
|
524
|
+
},
|
|
525
|
+
{
|
|
526
|
+
"type": "timeseries",
|
|
527
|
+
"title": "Requests / sec by verb",
|
|
528
|
+
"description": "One counter per HTTP verb from leverege-api's comms client. This is the load pvls puts on api-server. get and patch dominate: the pipeline reads device state and writes results back. A verb with no line has never been called, so its counter does not exist rather than reading zero.",
|
|
529
|
+
"id": 16,
|
|
530
|
+
"datasource": {
|
|
531
|
+
"type": "prometheus",
|
|
532
|
+
"uid": "${datasource}"
|
|
533
|
+
},
|
|
534
|
+
"gridPos": {
|
|
535
|
+
"h": 8,
|
|
536
|
+
"w": 12,
|
|
537
|
+
"x": 0,
|
|
538
|
+
"y": 44
|
|
539
|
+
},
|
|
540
|
+
"fieldConfig": {
|
|
541
|
+
"defaults": {
|
|
542
|
+
"unit": "reqps",
|
|
543
|
+
"min": 0,
|
|
544
|
+
"custom": {
|
|
545
|
+
"stacking": {
|
|
546
|
+
"mode": "normal",
|
|
547
|
+
"group": "A"
|
|
548
|
+
},
|
|
549
|
+
"fillOpacity": 20
|
|
550
|
+
}
|
|
551
|
+
},
|
|
552
|
+
"overrides": []
|
|
553
|
+
},
|
|
554
|
+
"targets": [
|
|
555
|
+
{
|
|
556
|
+
"datasource": {
|
|
557
|
+
"type": "prometheus",
|
|
558
|
+
"uid": "${datasource}"
|
|
559
|
+
},
|
|
560
|
+
"expr": "sum(rate(imagine_api_get_total{job=~\"$job\"}[5m]))",
|
|
561
|
+
"legendFormat": "get",
|
|
562
|
+
"refId": "A"
|
|
563
|
+
},
|
|
564
|
+
{
|
|
565
|
+
"datasource": {
|
|
566
|
+
"type": "prometheus",
|
|
567
|
+
"uid": "${datasource}"
|
|
568
|
+
},
|
|
569
|
+
"expr": "sum(rate(imagine_api_patch_total{job=~\"$job\"}[5m]))",
|
|
570
|
+
"legendFormat": "patch",
|
|
571
|
+
"refId": "B"
|
|
572
|
+
},
|
|
573
|
+
{
|
|
574
|
+
"datasource": {
|
|
575
|
+
"type": "prometheus",
|
|
576
|
+
"uid": "${datasource}"
|
|
577
|
+
},
|
|
578
|
+
"expr": "sum(rate(imagine_api_post_total{job=~\"$job\"}[5m]))",
|
|
579
|
+
"legendFormat": "post",
|
|
580
|
+
"refId": "C"
|
|
581
|
+
},
|
|
582
|
+
{
|
|
583
|
+
"datasource": {
|
|
584
|
+
"type": "prometheus",
|
|
585
|
+
"uid": "${datasource}"
|
|
586
|
+
},
|
|
587
|
+
"expr": "sum(rate(imagine_api_put_total{job=~\"$job\"}[5m]))",
|
|
588
|
+
"legendFormat": "put",
|
|
589
|
+
"refId": "D"
|
|
590
|
+
},
|
|
591
|
+
{
|
|
592
|
+
"datasource": {
|
|
593
|
+
"type": "prometheus",
|
|
594
|
+
"uid": "${datasource}"
|
|
595
|
+
},
|
|
596
|
+
"expr": "sum(rate(imagine_api_delete_total{job=~\"$job\"}[5m]))",
|
|
597
|
+
"legendFormat": "delete",
|
|
598
|
+
"refId": "E"
|
|
599
|
+
}
|
|
600
|
+
]
|
|
601
|
+
},
|
|
602
|
+
{
|
|
603
|
+
"type": "timeseries",
|
|
604
|
+
"title": "Errors / sec by verb",
|
|
605
|
+
"description": "The _error counters, incremented when the request raises or the response status falls outside 200-299. Note this mirrors the Node client's response.ok check, so a 204 is NOT an error here even though the Python client raises on it. A flat zero line is healthy; a missing line means that verb has never failed.",
|
|
606
|
+
"id": 17,
|
|
607
|
+
"datasource": {
|
|
608
|
+
"type": "prometheus",
|
|
609
|
+
"uid": "${datasource}"
|
|
610
|
+
},
|
|
611
|
+
"gridPos": {
|
|
612
|
+
"h": 8,
|
|
613
|
+
"w": 12,
|
|
614
|
+
"x": 12,
|
|
615
|
+
"y": 44
|
|
616
|
+
},
|
|
617
|
+
"fieldConfig": {
|
|
618
|
+
"defaults": {
|
|
619
|
+
"unit": "reqps",
|
|
620
|
+
"min": 0
|
|
621
|
+
},
|
|
622
|
+
"overrides": []
|
|
623
|
+
},
|
|
624
|
+
"targets": [
|
|
625
|
+
{
|
|
626
|
+
"datasource": {
|
|
627
|
+
"type": "prometheus",
|
|
628
|
+
"uid": "${datasource}"
|
|
629
|
+
},
|
|
630
|
+
"expr": "sum(rate(imagine_api_get_error_total{job=~\"$job\"}[5m]))",
|
|
631
|
+
"legendFormat": "get",
|
|
632
|
+
"refId": "A"
|
|
633
|
+
},
|
|
634
|
+
{
|
|
635
|
+
"datasource": {
|
|
636
|
+
"type": "prometheus",
|
|
637
|
+
"uid": "${datasource}"
|
|
638
|
+
},
|
|
639
|
+
"expr": "sum(rate(imagine_api_patch_error_total{job=~\"$job\"}[5m]))",
|
|
640
|
+
"legendFormat": "patch",
|
|
641
|
+
"refId": "B"
|
|
642
|
+
},
|
|
643
|
+
{
|
|
644
|
+
"datasource": {
|
|
645
|
+
"type": "prometheus",
|
|
646
|
+
"uid": "${datasource}"
|
|
647
|
+
},
|
|
648
|
+
"expr": "sum(rate(imagine_api_post_error_total{job=~\"$job\"}[5m]))",
|
|
649
|
+
"legendFormat": "post",
|
|
650
|
+
"refId": "C"
|
|
651
|
+
},
|
|
652
|
+
{
|
|
653
|
+
"datasource": {
|
|
654
|
+
"type": "prometheus",
|
|
655
|
+
"uid": "${datasource}"
|
|
656
|
+
},
|
|
657
|
+
"expr": "sum(rate(imagine_api_put_error_total{job=~\"$job\"}[5m]))",
|
|
658
|
+
"legendFormat": "put",
|
|
659
|
+
"refId": "D"
|
|
660
|
+
}
|
|
661
|
+
]
|
|
662
|
+
},
|
|
663
|
+
{
|
|
664
|
+
"type": "timeseries",
|
|
665
|
+
"title": "Error ratio",
|
|
666
|
+
"description": "All Imagine errors over all Imagine requests across the window. Every verb is named explicitly and guarded with or vector(0): a __name__ regex cannot be used here because rate() strips the metric name and the families then collide as duplicate labelsets. The denominator is clamped so an idle service reads 0 rather than dividing by zero.",
|
|
667
|
+
"id": 18,
|
|
668
|
+
"datasource": {
|
|
669
|
+
"type": "prometheus",
|
|
670
|
+
"uid": "${datasource}"
|
|
671
|
+
},
|
|
672
|
+
"gridPos": {
|
|
673
|
+
"h": 8,
|
|
674
|
+
"w": 12,
|
|
675
|
+
"x": 0,
|
|
676
|
+
"y": 52
|
|
677
|
+
},
|
|
678
|
+
"fieldConfig": {
|
|
679
|
+
"defaults": {
|
|
680
|
+
"unit": "percentunit",
|
|
681
|
+
"min": 0
|
|
682
|
+
},
|
|
683
|
+
"overrides": []
|
|
684
|
+
},
|
|
685
|
+
"targets": [
|
|
686
|
+
{
|
|
687
|
+
"datasource": {
|
|
688
|
+
"type": "prometheus",
|
|
689
|
+
"uid": "${datasource}"
|
|
690
|
+
},
|
|
691
|
+
"expr": "((sum(rate(imagine_api_get_error_total{job=~\"$job\"}[5m])) or vector(0)) + (sum(rate(imagine_api_patch_error_total{job=~\"$job\"}[5m])) or vector(0)) + (sum(rate(imagine_api_post_error_total{job=~\"$job\"}[5m])) or vector(0)) + (sum(rate(imagine_api_put_error_total{job=~\"$job\"}[5m])) or vector(0)) + (sum(rate(imagine_api_delete_error_total{job=~\"$job\"}[5m])) or vector(0))) / clamp_min((sum(rate(imagine_api_get_total{job=~\"$job\"}[5m])) or vector(0)) + (sum(rate(imagine_api_patch_total{job=~\"$job\"}[5m])) or vector(0)) + (sum(rate(imagine_api_post_total{job=~\"$job\"}[5m])) or vector(0)) + (sum(rate(imagine_api_put_total{job=~\"$job\"}[5m])) or vector(0)) + (sum(rate(imagine_api_delete_total{job=~\"$job\"}[5m])) or vector(0)), 1e-9)",
|
|
692
|
+
"legendFormat": "error ratio",
|
|
693
|
+
"refId": "A"
|
|
694
|
+
}
|
|
695
|
+
]
|
|
696
|
+
},
|
|
697
|
+
{
|
|
698
|
+
"type": "stat",
|
|
699
|
+
"title": "Imagine totals over the range",
|
|
700
|
+
"description": "Attempts and failures for the selected window. The attempt counter is incremented before the request is sent, so requests that time out are counted here; errors is a strict subset of it rather than an additional figure.",
|
|
701
|
+
"id": 19,
|
|
702
|
+
"datasource": {
|
|
703
|
+
"type": "prometheus",
|
|
704
|
+
"uid": "${datasource}"
|
|
705
|
+
},
|
|
706
|
+
"gridPos": {
|
|
707
|
+
"h": 8,
|
|
708
|
+
"w": 12,
|
|
709
|
+
"x": 12,
|
|
710
|
+
"y": 52
|
|
711
|
+
},
|
|
712
|
+
"fieldConfig": {
|
|
713
|
+
"defaults": {
|
|
714
|
+
"unit": "short",
|
|
715
|
+
"min": 0
|
|
716
|
+
},
|
|
717
|
+
"overrides": []
|
|
718
|
+
},
|
|
719
|
+
"options": {
|
|
720
|
+
"reduceOptions": {
|
|
721
|
+
"calcs": [
|
|
722
|
+
"lastNotNull"
|
|
723
|
+
],
|
|
724
|
+
"fields": "",
|
|
725
|
+
"values": false
|
|
726
|
+
},
|
|
727
|
+
"textMode": "auto",
|
|
728
|
+
"colorMode": "value",
|
|
729
|
+
"graphMode": "area"
|
|
730
|
+
},
|
|
731
|
+
"targets": [
|
|
732
|
+
{
|
|
733
|
+
"datasource": {
|
|
734
|
+
"type": "prometheus",
|
|
735
|
+
"uid": "${datasource}"
|
|
736
|
+
},
|
|
737
|
+
"expr": "(sum(increase(imagine_api_get_total{job=~\"$job\"}[$__range])) or vector(0)) + (sum(increase(imagine_api_patch_total{job=~\"$job\"}[$__range])) or vector(0)) + (sum(increase(imagine_api_post_total{job=~\"$job\"}[$__range])) or vector(0)) + (sum(increase(imagine_api_put_total{job=~\"$job\"}[$__range])) or vector(0)) + (sum(increase(imagine_api_delete_total{job=~\"$job\"}[$__range])) or vector(0))",
|
|
738
|
+
"legendFormat": "requests",
|
|
739
|
+
"refId": "A"
|
|
740
|
+
},
|
|
741
|
+
{
|
|
742
|
+
"datasource": {
|
|
743
|
+
"type": "prometheus",
|
|
744
|
+
"uid": "${datasource}"
|
|
745
|
+
},
|
|
746
|
+
"expr": "(sum(increase(imagine_api_get_error_total{job=~\"$job\"}[$__range])) or vector(0)) + (sum(increase(imagine_api_patch_error_total{job=~\"$job\"}[$__range])) or vector(0)) + (sum(increase(imagine_api_post_error_total{job=~\"$job\"}[$__range])) or vector(0)) + (sum(increase(imagine_api_put_error_total{job=~\"$job\"}[$__range])) or vector(0)) + (sum(increase(imagine_api_delete_error_total{job=~\"$job\"}[$__range])) or vector(0))",
|
|
747
|
+
"legendFormat": "errors",
|
|
748
|
+
"refId": "B"
|
|
749
|
+
}
|
|
750
|
+
]
|
|
751
|
+
},
|
|
752
|
+
{
|
|
753
|
+
"type": "row",
|
|
754
|
+
"title": "Resource uploads: annotated frames to Imagine",
|
|
755
|
+
"id": 20,
|
|
756
|
+
"gridPos": {
|
|
757
|
+
"h": 1,
|
|
758
|
+
"w": 24,
|
|
759
|
+
"x": 0,
|
|
760
|
+
"y": 60
|
|
761
|
+
},
|
|
762
|
+
"collapsed": false,
|
|
763
|
+
"panels": []
|
|
764
|
+
},
|
|
765
|
+
{
|
|
766
|
+
"type": "timeseries",
|
|
767
|
+
"title": "Uploads / sec",
|
|
768
|
+
"description": "The AsyncResourceUploadManager lifecycle. queued is accepted into the upload pool, success is confirmed stored, error is a failed attempt and drop is work discarded because the pool was saturated. In health, queued and success track each other and the other two stay flat at zero.",
|
|
769
|
+
"id": 21,
|
|
770
|
+
"datasource": {
|
|
771
|
+
"type": "prometheus",
|
|
772
|
+
"uid": "${datasource}"
|
|
773
|
+
},
|
|
774
|
+
"gridPos": {
|
|
775
|
+
"h": 8,
|
|
776
|
+
"w": 12,
|
|
777
|
+
"x": 0,
|
|
778
|
+
"y": 61
|
|
779
|
+
},
|
|
780
|
+
"fieldConfig": {
|
|
781
|
+
"defaults": {
|
|
782
|
+
"unit": "reqps",
|
|
783
|
+
"min": 0
|
|
784
|
+
},
|
|
785
|
+
"overrides": []
|
|
786
|
+
},
|
|
787
|
+
"targets": [
|
|
788
|
+
{
|
|
789
|
+
"datasource": {
|
|
790
|
+
"type": "prometheus",
|
|
791
|
+
"uid": "${datasource}"
|
|
792
|
+
},
|
|
793
|
+
"expr": "sum(rate(leverege_api_resource_upload_queued_total{job=~\"$job\"}[5m]))",
|
|
794
|
+
"legendFormat": "queued/s",
|
|
795
|
+
"refId": "A"
|
|
796
|
+
},
|
|
797
|
+
{
|
|
798
|
+
"datasource": {
|
|
799
|
+
"type": "prometheus",
|
|
800
|
+
"uid": "${datasource}"
|
|
801
|
+
},
|
|
802
|
+
"expr": "sum(rate(leverege_api_resource_upload_success_total{job=~\"$job\"}[5m]))",
|
|
803
|
+
"legendFormat": "success/s",
|
|
804
|
+
"refId": "B"
|
|
805
|
+
},
|
|
806
|
+
{
|
|
807
|
+
"datasource": {
|
|
808
|
+
"type": "prometheus",
|
|
809
|
+
"uid": "${datasource}"
|
|
810
|
+
},
|
|
811
|
+
"expr": "sum(rate(leverege_api_resource_upload_error_total{job=~\"$job\"}[5m]))",
|
|
812
|
+
"legendFormat": "error/s",
|
|
813
|
+
"refId": "C"
|
|
814
|
+
},
|
|
815
|
+
{
|
|
816
|
+
"datasource": {
|
|
817
|
+
"type": "prometheus",
|
|
818
|
+
"uid": "${datasource}"
|
|
819
|
+
},
|
|
820
|
+
"expr": "sum(rate(leverege_api_resource_upload_drop_total{job=~\"$job\"}[5m]))",
|
|
821
|
+
"legendFormat": "drop/s",
|
|
822
|
+
"refId": "D"
|
|
823
|
+
}
|
|
824
|
+
]
|
|
825
|
+
},
|
|
826
|
+
{
|
|
827
|
+
"type": "timeseries",
|
|
828
|
+
"title": "Uploads outstanding",
|
|
829
|
+
"description": "queued minus everything that has left the pool. Every term is guarded with or vector(0), because error and drop have never fired in dev and a missing operand would otherwise make the whole expression return nothing and blank the panel for a perfectly healthy service. A rising line means the pool is not keeping up with the pipeline.",
|
|
830
|
+
"id": 22,
|
|
831
|
+
"datasource": {
|
|
832
|
+
"type": "prometheus",
|
|
833
|
+
"uid": "${datasource}"
|
|
834
|
+
},
|
|
835
|
+
"gridPos": {
|
|
836
|
+
"h": 8,
|
|
837
|
+
"w": 12,
|
|
838
|
+
"x": 12,
|
|
839
|
+
"y": 61
|
|
840
|
+
},
|
|
841
|
+
"fieldConfig": {
|
|
842
|
+
"defaults": {
|
|
843
|
+
"unit": "short",
|
|
844
|
+
"min": 0
|
|
845
|
+
},
|
|
846
|
+
"overrides": []
|
|
847
|
+
},
|
|
848
|
+
"targets": [
|
|
849
|
+
{
|
|
850
|
+
"datasource": {
|
|
851
|
+
"type": "prometheus",
|
|
852
|
+
"uid": "${datasource}"
|
|
853
|
+
},
|
|
854
|
+
"expr": "clamp_min((sum(leverege_api_resource_upload_queued_total{job=~\"$job\"}) or vector(0)) - (sum(leverege_api_resource_upload_success_total{job=~\"$job\"}) or vector(0)) - (sum(leverege_api_resource_upload_error_total{job=~\"$job\"}) or vector(0)) - (sum(leverege_api_resource_upload_drop_total{job=~\"$job\"}) or vector(0)), 0)",
|
|
855
|
+
"legendFormat": "outstanding",
|
|
856
|
+
"refId": "A"
|
|
857
|
+
}
|
|
858
|
+
]
|
|
859
|
+
},
|
|
860
|
+
{
|
|
861
|
+
"type": "stat",
|
|
862
|
+
"title": "Upload totals over the range",
|
|
863
|
+
"description": "Counts for the window. drop is the one to watch: it means frames were thrown away rather than retried, so the timeline in Imagine has holes that no downstream retry will fill.",
|
|
864
|
+
"id": 23,
|
|
865
|
+
"datasource": {
|
|
866
|
+
"type": "prometheus",
|
|
867
|
+
"uid": "${datasource}"
|
|
868
|
+
},
|
|
869
|
+
"gridPos": {
|
|
870
|
+
"h": 8,
|
|
871
|
+
"w": 12,
|
|
872
|
+
"x": 0,
|
|
873
|
+
"y": 69
|
|
874
|
+
},
|
|
875
|
+
"fieldConfig": {
|
|
876
|
+
"defaults": {
|
|
877
|
+
"unit": "short",
|
|
878
|
+
"min": 0
|
|
879
|
+
},
|
|
880
|
+
"overrides": []
|
|
881
|
+
},
|
|
882
|
+
"options": {
|
|
883
|
+
"reduceOptions": {
|
|
884
|
+
"calcs": [
|
|
885
|
+
"lastNotNull"
|
|
886
|
+
],
|
|
887
|
+
"fields": "",
|
|
888
|
+
"values": false
|
|
889
|
+
},
|
|
890
|
+
"textMode": "auto",
|
|
891
|
+
"colorMode": "value",
|
|
892
|
+
"graphMode": "area"
|
|
893
|
+
},
|
|
894
|
+
"targets": [
|
|
895
|
+
{
|
|
896
|
+
"datasource": {
|
|
897
|
+
"type": "prometheus",
|
|
898
|
+
"uid": "${datasource}"
|
|
899
|
+
},
|
|
900
|
+
"expr": "sum(increase(leverege_api_resource_upload_queued_total{job=~\"$job\"}[$__range])) or vector(0)",
|
|
901
|
+
"legendFormat": "queued",
|
|
902
|
+
"refId": "A"
|
|
903
|
+
},
|
|
904
|
+
{
|
|
905
|
+
"datasource": {
|
|
906
|
+
"type": "prometheus",
|
|
907
|
+
"uid": "${datasource}"
|
|
908
|
+
},
|
|
909
|
+
"expr": "sum(increase(leverege_api_resource_upload_success_total{job=~\"$job\"}[$__range])) or vector(0)",
|
|
910
|
+
"legendFormat": "success",
|
|
911
|
+
"refId": "B"
|
|
912
|
+
},
|
|
913
|
+
{
|
|
914
|
+
"datasource": {
|
|
915
|
+
"type": "prometheus",
|
|
916
|
+
"uid": "${datasource}"
|
|
917
|
+
},
|
|
918
|
+
"expr": "sum(increase(leverege_api_resource_upload_error_total{job=~\"$job\"}[$__range])) or vector(0)",
|
|
919
|
+
"legendFormat": "error",
|
|
920
|
+
"refId": "C"
|
|
921
|
+
},
|
|
922
|
+
{
|
|
923
|
+
"datasource": {
|
|
924
|
+
"type": "prometheus",
|
|
925
|
+
"uid": "${datasource}"
|
|
926
|
+
},
|
|
927
|
+
"expr": "sum(increase(leverege_api_resource_upload_drop_total{job=~\"$job\"}[$__range])) or vector(0)",
|
|
928
|
+
"legendFormat": "drop",
|
|
929
|
+
"refId": "D"
|
|
930
|
+
}
|
|
931
|
+
]
|
|
932
|
+
},
|
|
933
|
+
{
|
|
934
|
+
"type": "timeseries",
|
|
935
|
+
"title": "Upload success ratio",
|
|
936
|
+
"description": "success over queued across the window. Anything below 1 means frames entered the pool and did not land. Denominator clamped so an idle window reads 0 instead of erroring.",
|
|
937
|
+
"id": 24,
|
|
938
|
+
"datasource": {
|
|
939
|
+
"type": "prometheus",
|
|
940
|
+
"uid": "${datasource}"
|
|
941
|
+
},
|
|
942
|
+
"gridPos": {
|
|
943
|
+
"h": 8,
|
|
944
|
+
"w": 12,
|
|
945
|
+
"x": 12,
|
|
946
|
+
"y": 69
|
|
947
|
+
},
|
|
948
|
+
"fieldConfig": {
|
|
949
|
+
"defaults": {
|
|
950
|
+
"unit": "percentunit",
|
|
951
|
+
"min": 0
|
|
952
|
+
},
|
|
953
|
+
"overrides": []
|
|
954
|
+
},
|
|
955
|
+
"targets": [
|
|
956
|
+
{
|
|
957
|
+
"datasource": {
|
|
958
|
+
"type": "prometheus",
|
|
959
|
+
"uid": "${datasource}"
|
|
960
|
+
},
|
|
961
|
+
"expr": "(sum(increase(leverege_api_resource_upload_success_total{job=~\"$job\"}[$__range])) or vector(0)) / clamp_min(sum(increase(leverege_api_resource_upload_queued_total{job=~\"$job\"}[$__range])) or vector(0), 1e-9)",
|
|
962
|
+
"legendFormat": "success ratio",
|
|
963
|
+
"refId": "A"
|
|
964
|
+
}
|
|
965
|
+
]
|
|
966
|
+
},
|
|
967
|
+
{
|
|
968
|
+
"type": "row",
|
|
969
|
+
"title": "Runtime",
|
|
970
|
+
"id": 25,
|
|
971
|
+
"gridPos": {
|
|
972
|
+
"h": 1,
|
|
973
|
+
"w": 24,
|
|
974
|
+
"x": 0,
|
|
975
|
+
"y": 77
|
|
976
|
+
},
|
|
977
|
+
"collapsed": false,
|
|
978
|
+
"panels": []
|
|
979
|
+
},
|
|
980
|
+
{
|
|
981
|
+
"type": "timeseries",
|
|
982
|
+
"title": "CPU per pod",
|
|
983
|
+
"description": "From cAdvisor, not from the app. py-server's shared-metrics bridge deliberately drops prometheus_client's own process_* and python_* series once USE_SHARED_METRICS is on, because they only ever described whichever worker answered the scrape. Container metrics are the honest source here. Compare against the 2000m limit.",
|
|
984
|
+
"id": 26,
|
|
985
|
+
"datasource": {
|
|
986
|
+
"type": "prometheus",
|
|
987
|
+
"uid": "${datasource}"
|
|
988
|
+
},
|
|
989
|
+
"gridPos": {
|
|
990
|
+
"h": 8,
|
|
991
|
+
"w": 12,
|
|
992
|
+
"x": 0,
|
|
993
|
+
"y": 78
|
|
994
|
+
},
|
|
995
|
+
"fieldConfig": {
|
|
996
|
+
"defaults": {
|
|
997
|
+
"unit": "short",
|
|
998
|
+
"min": 0
|
|
999
|
+
},
|
|
1000
|
+
"overrides": []
|
|
1001
|
+
},
|
|
1002
|
+
"targets": [
|
|
1003
|
+
{
|
|
1004
|
+
"datasource": {
|
|
1005
|
+
"type": "prometheus",
|
|
1006
|
+
"uid": "${datasource}"
|
|
1007
|
+
},
|
|
1008
|
+
"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{pod=~\"pitcrew-vision-logic-server.*\", container!=\"\"}[5m]))",
|
|
1009
|
+
"legendFormat": "{{pod}}",
|
|
1010
|
+
"refId": "A"
|
|
1011
|
+
}
|
|
1012
|
+
]
|
|
1013
|
+
},
|
|
1014
|
+
{
|
|
1015
|
+
"type": "timeseries",
|
|
1016
|
+
"title": "Memory per pod",
|
|
1017
|
+
"description": "Working set per pod against the 4000Mi limit. Deliberately per-pod rather than summed: this service holds frames in memory, and a single leaking replica is exactly what you are looking for. A pod that climbs steadily to the limit will be OOM-killed and show up as a restart in the health row.",
|
|
1018
|
+
"id": 27,
|
|
1019
|
+
"datasource": {
|
|
1020
|
+
"type": "prometheus",
|
|
1021
|
+
"uid": "${datasource}"
|
|
1022
|
+
},
|
|
1023
|
+
"gridPos": {
|
|
1024
|
+
"h": 8,
|
|
1025
|
+
"w": 12,
|
|
1026
|
+
"x": 12,
|
|
1027
|
+
"y": 78
|
|
1028
|
+
},
|
|
1029
|
+
"fieldConfig": {
|
|
1030
|
+
"defaults": {
|
|
1031
|
+
"unit": "bytes",
|
|
1032
|
+
"min": 0
|
|
1033
|
+
},
|
|
1034
|
+
"overrides": []
|
|
1035
|
+
},
|
|
1036
|
+
"targets": [
|
|
1037
|
+
{
|
|
1038
|
+
"datasource": {
|
|
1039
|
+
"type": "prometheus",
|
|
1040
|
+
"uid": "${datasource}"
|
|
1041
|
+
},
|
|
1042
|
+
"expr": "sum by (pod) (container_memory_working_set_bytes{pod=~\"pitcrew-vision-logic-server.*\", container!=\"\"})",
|
|
1043
|
+
"legendFormat": "{{pod}}",
|
|
1044
|
+
"refId": "A"
|
|
1045
|
+
}
|
|
1046
|
+
]
|
|
1047
|
+
}
|
|
1048
|
+
],
|
|
1049
|
+
"refresh": "1m",
|
|
1050
|
+
"schemaVersion": 39,
|
|
1051
|
+
"tags": [
|
|
1052
|
+
"pitcrew-vision-logic-server",
|
|
1053
|
+
"pitcrew",
|
|
1054
|
+
"vision"
|
|
1055
|
+
],
|
|
1056
|
+
"templating": {
|
|
1057
|
+
"list": [
|
|
1058
|
+
{
|
|
1059
|
+
"type": "datasource",
|
|
1060
|
+
"name": "datasource",
|
|
1061
|
+
"label": "Data source",
|
|
1062
|
+
"query": "prometheus",
|
|
1063
|
+
"current": {},
|
|
1064
|
+
"hide": 0
|
|
1065
|
+
},
|
|
1066
|
+
{
|
|
1067
|
+
"type": "query",
|
|
1068
|
+
"name": "job",
|
|
1069
|
+
"label": "Job",
|
|
1070
|
+
"datasource": {
|
|
1071
|
+
"type": "prometheus",
|
|
1072
|
+
"uid": "${datasource}"
|
|
1073
|
+
},
|
|
1074
|
+
"description": "Scrape job, so one dashboard can serve several deployments of this service. Derived from up{app_kubernetes_io_name=...} rather than from an app counter: the metric names here carry no service prefix, so label_values over a counter also returns every other py-server service that has emitted it inside the retention window.",
|
|
1075
|
+
"query": "label_values(up{app_kubernetes_io_name=\"pitcrew-vision-logic-server\"}, job)",
|
|
1076
|
+
"refresh": 1,
|
|
1077
|
+
"includeAll": true,
|
|
1078
|
+
"multi": true,
|
|
1079
|
+
"current": {
|
|
1080
|
+
"selected": false,
|
|
1081
|
+
"text": "pitcrew-vision-logic-server",
|
|
1082
|
+
"value": "pitcrew-vision-logic-server"
|
|
1083
|
+
},
|
|
1084
|
+
"hide": 0
|
|
1085
|
+
},
|
|
1086
|
+
{
|
|
1087
|
+
"type": "query",
|
|
1088
|
+
"name": "subscription",
|
|
1089
|
+
"label": "Subscription",
|
|
1090
|
+
"datasource": {
|
|
1091
|
+
"type": "prometheus",
|
|
1092
|
+
"uid": "${datasource}"
|
|
1093
|
+
},
|
|
1094
|
+
"description": "Pub/Sub subscription feeding this service, as exported by the stackdriver-exporter. Drives the backlog panels and is the series the HPA scales on.",
|
|
1095
|
+
"query": "label_values(pubsub_subscription_num_undelivered_messages, subscription_id)",
|
|
1096
|
+
"refresh": 1,
|
|
1097
|
+
"includeAll": false,
|
|
1098
|
+
"multi": false,
|
|
1099
|
+
"current": {
|
|
1100
|
+
"selected": false,
|
|
1101
|
+
"text": "pitcrew-vision-imc-logic",
|
|
1102
|
+
"value": "pitcrew-vision-imc-logic"
|
|
1103
|
+
},
|
|
1104
|
+
"hide": 0
|
|
1105
|
+
},
|
|
1106
|
+
{
|
|
1107
|
+
"type": "query",
|
|
1108
|
+
"name": "topic",
|
|
1109
|
+
"label": "Topic",
|
|
1110
|
+
"datasource": {
|
|
1111
|
+
"type": "prometheus",
|
|
1112
|
+
"uid": "${datasource}"
|
|
1113
|
+
},
|
|
1114
|
+
"description": "Topic feeding the subscription above. Drives the publish-rate panel, the HPA's rate floor.",
|
|
1115
|
+
"query": "label_values(pubsub_topic_publish_rate, topic_id)",
|
|
1116
|
+
"refresh": 1,
|
|
1117
|
+
"includeAll": false,
|
|
1118
|
+
"multi": false,
|
|
1119
|
+
"current": {
|
|
1120
|
+
"selected": false,
|
|
1121
|
+
"text": "pitcrew-vision",
|
|
1122
|
+
"value": "pitcrew-vision"
|
|
1123
|
+
},
|
|
1124
|
+
"hide": 0
|
|
1125
|
+
}
|
|
1126
|
+
]
|
|
1127
|
+
},
|
|
1128
|
+
"time": {
|
|
1129
|
+
"from": "now-6h",
|
|
1130
|
+
"to": "now"
|
|
1131
|
+
},
|
|
1132
|
+
"timepicker": {},
|
|
1133
|
+
"timezone": "",
|
|
1134
|
+
"title": "PitCrew Vision Logic Server",
|
|
1135
|
+
"uid": "pitcrew-vision-logic-server",
|
|
1136
|
+
"version": 1
|
|
1137
|
+
}
|