@senad-d/observme 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/.env.example +57 -0
  2. package/CHANGELOG.md +52 -0
  3. package/LICENSE +21 -0
  4. package/ObservMe-Production-Docs/00-README.md +79 -0
  5. package/ObservMe-Production-Docs/01-requirements-and-scope.md +207 -0
  6. package/ObservMe-Production-Docs/02-reference-architecture.md +306 -0
  7. package/ObservMe-Production-Docs/03-pi-event-and-session-model.md +266 -0
  8. package/ObservMe-Production-Docs/04-telemetry-semantic-conventions.md +722 -0
  9. package/ObservMe-Production-Docs/05-otel-pipeline-and-collector.md +355 -0
  10. package/ObservMe-Production-Docs/06-security-privacy-redaction.md +294 -0
  11. package/ObservMe-Production-Docs/07-extension-implementation-blueprint.md +447 -0
  12. package/ObservMe-Production-Docs/08-query-grafana-integration.md +276 -0
  13. package/ObservMe-Production-Docs/09-dashboards-alerts-slos.md +640 -0
  14. package/ObservMe-Production-Docs/10-testing-release-operations.md +319 -0
  15. package/ObservMe-Production-Docs/11-deployment-runbooks.md +203 -0
  16. package/ObservMe-Production-Docs/12-configuration-reference.md +337 -0
  17. package/ObservMe-Production-Docs/13-source-notes.md +33 -0
  18. package/ObservMe-Production-Docs/pi-session-format.md +427 -0
  19. package/README.md +356 -0
  20. package/SECURITY.md +45 -0
  21. package/dashboards/observme-agent-node-graphs.json +250 -0
  22. package/dashboards/observme-agents.json +1880 -0
  23. package/dashboards/observme-alerts.yaml +113 -0
  24. package/dashboards/observme-branches-compactions.json +1042 -0
  25. package/dashboards/observme-cost.json +1254 -0
  26. package/dashboards/observme-errors.json +1659 -0
  27. package/dashboards/observme-export-health.json +1802 -0
  28. package/dashboards/observme-latency.json +1494 -0
  29. package/dashboards/observme-llm-conversations.json +730 -0
  30. package/dashboards/observme-logs-llm.json +644 -0
  31. package/dashboards/observme-models.json +933 -0
  32. package/dashboards/observme-overview.json +2129 -0
  33. package/dashboards/observme-slo-health.json +737 -0
  34. package/dashboards/observme-slos.yaml +56 -0
  35. package/dashboards/observme-tools.json +902 -0
  36. package/dashboards/observme-trace-journey.json +1793 -0
  37. package/docs/STRUCTURE.md +49 -0
  38. package/docs/agent-subagent-observability-requirements.md +997 -0
  39. package/docs/compatibility-matrix.md +39 -0
  40. package/docs/configuration-tui-design-standard.md +767 -0
  41. package/docs/configuration.md +38 -0
  42. package/docs/review-validation.md +115 -0
  43. package/docs/validation-flow.md +117 -0
  44. package/examples/collector.yaml +123 -0
  45. package/examples/observme.yaml +131 -0
  46. package/img/demo.gif +0 -0
  47. package/img/icon.svg +47 -0
  48. package/package.json +103 -0
  49. package/src/commands/obs-agents-runtime.ts +150 -0
  50. package/src/commands/obs-agents.ts +491 -0
  51. package/src/commands/obs-args.ts +63 -0
  52. package/src/commands/obs-backfill.ts +1334 -0
  53. package/src/commands/obs-command-support.ts +43 -0
  54. package/src/commands/obs-cost.ts +228 -0
  55. package/src/commands/obs-diagnostics.ts +22 -0
  56. package/src/commands/obs-errors.ts +156 -0
  57. package/src/commands/obs-health.ts +301 -0
  58. package/src/commands/obs-link.ts +90 -0
  59. package/src/commands/obs-logs.ts +194 -0
  60. package/src/commands/obs-loki-summary.ts +184 -0
  61. package/src/commands/obs-session.ts +259 -0
  62. package/src/commands/obs-status.ts +359 -0
  63. package/src/commands/obs-tools.ts +274 -0
  64. package/src/commands/obs-trace.ts +411 -0
  65. package/src/commands/obs.ts +211 -0
  66. package/src/config/bootstrap-project-config.ts +300 -0
  67. package/src/config/defaults.ts +143 -0
  68. package/src/config/load-config.ts +631 -0
  69. package/src/config/project-paths.ts +61 -0
  70. package/src/config/schema.ts +405 -0
  71. package/src/config/validate.ts +456 -0
  72. package/src/constants.ts +4 -0
  73. package/src/diagnostics/sanitize.ts +6 -0
  74. package/src/extension.ts +38 -0
  75. package/src/otel/logs.ts +160 -0
  76. package/src/otel/metrics.ts +165 -0
  77. package/src/otel/otlp-endpoint.ts +10 -0
  78. package/src/otel/sdk.ts +114 -0
  79. package/src/otel/shutdown.ts +102 -0
  80. package/src/otel/traces.ts +166 -0
  81. package/src/pi/agent-lineage.ts +378 -0
  82. package/src/pi/agent-tree-tracker.ts +258 -0
  83. package/src/pi/event-handlers/agent-turn.ts +155 -0
  84. package/src/pi/event-handlers/lifecycle.ts +642 -0
  85. package/src/pi/event-handlers/llm.ts +115 -0
  86. package/src/pi/event-handlers/session-events.ts +159 -0
  87. package/src/pi/event-handlers/tool-bash.ts +275 -0
  88. package/src/pi/handler-internals.ts +2154 -0
  89. package/src/pi/handler-runtime.ts +633 -0
  90. package/src/pi/handler-types.ts +261 -0
  91. package/src/pi/handlers.ts +75 -0
  92. package/src/pi/subagent-spawn.ts +975 -0
  93. package/src/pi/subagent-types.ts +29 -0
  94. package/src/privacy/content-capture.ts +104 -0
  95. package/src/privacy/hash.ts +93 -0
  96. package/src/privacy/redact.ts +619 -0
  97. package/src/privacy/secret-patterns.ts +185 -0
  98. package/src/privacy/truncate.ts +69 -0
  99. package/src/query/grafana-readiness.ts +164 -0
  100. package/src/query/grafana-transport.ts +481 -0
  101. package/src/query/grafana.ts +371 -0
  102. package/src/query/loki.ts +332 -0
  103. package/src/query/prometheus.ts +388 -0
  104. package/src/query/tempo.ts +332 -0
  105. package/src/safety/sensitive-input.ts +208 -0
  106. package/src/semconv/attributes.ts +279 -0
  107. package/src/semconv/metrics.ts +146 -0
  108. package/src/semconv/spans.ts +19 -0
  109. package/src/semconv/values.ts +13 -0
  110. package/src/util/bounded-map.ts +97 -0
  111. package/tsconfig.json +15 -0
@@ -0,0 +1,1880 @@
1
+ {
2
+ "__requires": [
3
+ {
4
+ "id": "grafana",
5
+ "name": "Grafana",
6
+ "type": "grafana",
7
+ "version": "10.0.0"
8
+ },
9
+ {
10
+ "id": "prometheus",
11
+ "name": "Prometheus",
12
+ "type": "datasource",
13
+ "version": "1.0.0"
14
+ },
15
+ {
16
+ "id": "loki",
17
+ "name": "Loki",
18
+ "type": "datasource",
19
+ "version": "1.0.0"
20
+ },
21
+ {
22
+ "id": "row",
23
+ "name": "Row",
24
+ "type": "panel",
25
+ "version": ""
26
+ },
27
+ {
28
+ "id": "timeseries",
29
+ "name": "Time series",
30
+ "type": "panel",
31
+ "version": ""
32
+ },
33
+ {
34
+ "id": "stat",
35
+ "name": "Stat",
36
+ "type": "panel",
37
+ "version": ""
38
+ },
39
+ {
40
+ "id": "table",
41
+ "name": "Table",
42
+ "type": "panel",
43
+ "version": ""
44
+ }
45
+ ],
46
+ "annotations": {
47
+ "list": []
48
+ },
49
+ "editable": true,
50
+ "fiscalYearStartMonth": 0,
51
+ "graphTooltip": 0,
52
+ "id": null,
53
+ "links": [
54
+ {
55
+ "targetBlank": false,
56
+ "title": "Overview",
57
+ "url": "/d/observme-overview/observme-overview?${__url_time_range}"
58
+ },
59
+ {
60
+ "targetBlank": false,
61
+ "title": "Agent Node Graphs",
62
+ "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}"
63
+ },
64
+ {
65
+ "targetBlank": false,
66
+ "title": "Trace Journey",
67
+ "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}"
68
+ },
69
+ {
70
+ "targetBlank": false,
71
+ "title": "Errors",
72
+ "url": "/d/observme-errors/observme-errors?${__url_time_range}"
73
+ },
74
+ {
75
+ "targetBlank": false,
76
+ "title": "SLO Health",
77
+ "url": "/d/observme-slo-health/observme-slo-health?${__url_time_range}"
78
+ }
79
+ ],
80
+ "liveNow": false,
81
+ "panels": [
82
+ {
83
+ "collapsed": false,
84
+ "gridPos": {
85
+ "h": 1,
86
+ "w": 24,
87
+ "x": 0,
88
+ "y": 0
89
+ },
90
+ "id": 100,
91
+ "panels": [],
92
+ "title": "Health ratios",
93
+ "type": "row"
94
+ },
95
+ {
96
+ "datasource": {
97
+ "type": "prometheus",
98
+ "uid": "prometheus"
99
+ },
100
+ "description": "Selected-range spawn failures divided by spawn attempts; observme_subagents_spawned_total increments when a spawn starts. Zero means no failures were observed or no spawn attempts occurred. Any non-zero result aligns with the warning-level ObservMeSubagentSpawnFailures alert.",
101
+ "fieldConfig": {
102
+ "defaults": {
103
+ "color": {
104
+ "mode": "thresholds"
105
+ },
106
+ "mappings": [],
107
+ "thresholds": {
108
+ "mode": "absolute",
109
+ "steps": [
110
+ {
111
+ "color": "green",
112
+ "value": null
113
+ },
114
+ {
115
+ "color": "yellow",
116
+ "value": 1e-12
117
+ }
118
+ ]
119
+ },
120
+ "unit": "percentunit"
121
+ },
122
+ "overrides": []
123
+ },
124
+ "gridPos": {
125
+ "h": 5,
126
+ "w": 6,
127
+ "x": 0,
128
+ "y": 1
129
+ },
130
+ "id": 14,
131
+ "links": [
132
+ {
133
+ "targetBlank": false,
134
+ "title": "Open Trace Journey",
135
+ "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}"
136
+ }
137
+ ],
138
+ "options": {
139
+ "colorMode": "value",
140
+ "graphMode": "area",
141
+ "justifyMode": "auto",
142
+ "orientation": "auto",
143
+ "reduceOptions": {
144
+ "calcs": [
145
+ "lastNotNull"
146
+ ],
147
+ "fields": "",
148
+ "values": false
149
+ },
150
+ "showPercentChange": false,
151
+ "textMode": "auto",
152
+ "wideLayout": true
153
+ },
154
+ "pluginVersion": "11.1.0",
155
+ "targets": [
156
+ {
157
+ "datasource": {
158
+ "type": "prometheus",
159
+ "uid": "prometheus"
160
+ },
161
+ "editorMode": "code",
162
+ "expr": "(sum(increase(observme_subagent_spawn_failures_total[$__range])) or vector(0)) / clamp_min((sum(increase(observme_subagents_spawned_total[$__range])) or vector(0)), 1)",
163
+ "refId": "A",
164
+ "instant": true,
165
+ "range": false
166
+ }
167
+ ],
168
+ "title": "Spawn failure ratio",
169
+ "type": "stat"
170
+ },
171
+ {
172
+ "datasource": {
173
+ "type": "prometheus",
174
+ "uid": "prometheus"
175
+ },
176
+ "description": "Selected-range orphan events divided by observed orchestration opportunities (spawn starts plus agent runs). This is an aggregate pressure indicator, not a per-agent probability. Zero means no orphans were observed or no opportunities occurred; any non-zero orphan signal aligns with ObservMeOrphanAgentsDetected.",
177
+ "fieldConfig": {
178
+ "defaults": {
179
+ "color": {
180
+ "mode": "thresholds"
181
+ },
182
+ "mappings": [],
183
+ "thresholds": {
184
+ "mode": "absolute",
185
+ "steps": [
186
+ {
187
+ "color": "green",
188
+ "value": null
189
+ },
190
+ {
191
+ "color": "yellow",
192
+ "value": 1e-12
193
+ }
194
+ ]
195
+ },
196
+ "unit": "percentunit"
197
+ },
198
+ "overrides": []
199
+ },
200
+ "gridPos": {
201
+ "h": 5,
202
+ "w": 6,
203
+ "x": 6,
204
+ "y": 1
205
+ },
206
+ "id": 15,
207
+ "links": [
208
+ {
209
+ "targetBlank": false,
210
+ "title": "Open Trace Journey",
211
+ "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}"
212
+ }
213
+ ],
214
+ "options": {
215
+ "colorMode": "value",
216
+ "graphMode": "area",
217
+ "justifyMode": "auto",
218
+ "orientation": "auto",
219
+ "reduceOptions": {
220
+ "calcs": [
221
+ "lastNotNull"
222
+ ],
223
+ "fields": "",
224
+ "values": false
225
+ },
226
+ "showPercentChange": false,
227
+ "textMode": "auto",
228
+ "wideLayout": true
229
+ },
230
+ "pluginVersion": "11.1.0",
231
+ "targets": [
232
+ {
233
+ "datasource": {
234
+ "type": "prometheus",
235
+ "uid": "prometheus"
236
+ },
237
+ "editorMode": "code",
238
+ "expr": "(sum(increase(observme_orphan_agents_total[$__range])) or vector(0)) / clamp_min((sum(increase(observme_subagents_spawned_total[$__range])) or vector(0)) + (sum(increase(observme_agent_runs_total[$__range])) or vector(0)), 1)",
239
+ "refId": "A",
240
+ "instant": true,
241
+ "range": false
242
+ }
243
+ ],
244
+ "title": "Orphan ratio vs spawns/runs",
245
+ "type": "stat"
246
+ },
247
+ {
248
+ "datasource": {
249
+ "type": "prometheus",
250
+ "uid": "prometheus"
251
+ },
252
+ "description": "Selected-range subagent trace-context propagation failures divided by known spawn starts. Root-depth failures are excluded so the numerator matches the spawn denominator. Zero means no matching failures were observed or no subagent spawns occurred; any non-zero result aligns with ObservMeTraceContextPropagationFailures.",
253
+ "fieldConfig": {
254
+ "defaults": {
255
+ "color": {
256
+ "mode": "thresholds"
257
+ },
258
+ "mappings": [],
259
+ "thresholds": {
260
+ "mode": "absolute",
261
+ "steps": [
262
+ {
263
+ "color": "green",
264
+ "value": null
265
+ },
266
+ {
267
+ "color": "yellow",
268
+ "value": 1e-12
269
+ }
270
+ ]
271
+ },
272
+ "unit": "percentunit"
273
+ },
274
+ "overrides": []
275
+ },
276
+ "gridPos": {
277
+ "h": 5,
278
+ "w": 6,
279
+ "x": 12,
280
+ "y": 1
281
+ },
282
+ "id": 16,
283
+ "links": [
284
+ {
285
+ "targetBlank": false,
286
+ "title": "Open Trace Journey",
287
+ "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}"
288
+ }
289
+ ],
290
+ "options": {
291
+ "colorMode": "value",
292
+ "graphMode": "area",
293
+ "justifyMode": "auto",
294
+ "orientation": "auto",
295
+ "reduceOptions": {
296
+ "calcs": [
297
+ "lastNotNull"
298
+ ],
299
+ "fields": "",
300
+ "values": false
301
+ },
302
+ "showPercentChange": false,
303
+ "textMode": "auto",
304
+ "wideLayout": true
305
+ },
306
+ "pluginVersion": "11.1.0",
307
+ "targets": [
308
+ {
309
+ "datasource": {
310
+ "type": "prometheus",
311
+ "uid": "prometheus"
312
+ },
313
+ "editorMode": "code",
314
+ "expr": "(sum(increase(observme_trace_context_propagation_failures_total{subagent_depth=~\"[1-9][0-9]*\"}[$__range])) or vector(0)) / clamp_min((sum(increase(observme_subagents_spawned_total[$__range])) or vector(0)), 1)",
315
+ "refId": "A",
316
+ "instant": true,
317
+ "range": false
318
+ }
319
+ ],
320
+ "title": "Propagation failure ratio",
321
+ "type": "stat"
322
+ },
323
+ {
324
+ "datasource": {
325
+ "type": "prometheus",
326
+ "uid": "prometheus"
327
+ },
328
+ "description": "Selected-range parent recoveries divided by completed child-agent failures. A range with no child failures renders 100% because no recovery was required; use the failure/recovery trend to distinguish healthy idle from active recovery. The 50% and 90% color bands are tunable guidance, not alert thresholds.",
329
+ "fieldConfig": {
330
+ "defaults": {
331
+ "color": {
332
+ "mode": "thresholds"
333
+ },
334
+ "mappings": [],
335
+ "thresholds": {
336
+ "mode": "absolute",
337
+ "steps": [
338
+ {
339
+ "color": "red",
340
+ "value": null
341
+ },
342
+ {
343
+ "color": "yellow",
344
+ "value": 0.5
345
+ },
346
+ {
347
+ "color": "green",
348
+ "value": 0.9
349
+ }
350
+ ]
351
+ },
352
+ "unit": "percentunit"
353
+ },
354
+ "overrides": []
355
+ },
356
+ "gridPos": {
357
+ "h": 5,
358
+ "w": 6,
359
+ "x": 18,
360
+ "y": 1
361
+ },
362
+ "id": 17,
363
+ "links": [
364
+ {
365
+ "targetBlank": false,
366
+ "title": "Open Trace Journey",
367
+ "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}"
368
+ }
369
+ ],
370
+ "options": {
371
+ "colorMode": "value",
372
+ "graphMode": "area",
373
+ "justifyMode": "auto",
374
+ "orientation": "auto",
375
+ "reduceOptions": {
376
+ "calcs": [
377
+ "lastNotNull"
378
+ ],
379
+ "fields": "",
380
+ "values": false
381
+ },
382
+ "showPercentChange": false,
383
+ "textMode": "auto",
384
+ "wideLayout": true
385
+ },
386
+ "pluginVersion": "11.1.0",
387
+ "targets": [
388
+ {
389
+ "datasource": {
390
+ "type": "prometheus",
391
+ "uid": "prometheus"
392
+ },
393
+ "editorMode": "code",
394
+ "expr": "clamp_max(((sum(increase(observme_parent_recovered_from_child_failure_total[$__range])) or vector(0)) / clamp_min((sum(increase(observme_child_agent_failures_total[$__range])) or vector(0)), 1)) + ((sum(increase(observme_child_agent_failures_total[$__range])) or vector(0)) == bool 0), 1)",
395
+ "refId": "A",
396
+ "instant": true,
397
+ "range": false
398
+ }
399
+ ],
400
+ "title": "Child recovery ratio",
401
+ "type": "stat"
402
+ },
403
+ {
404
+ "datasource": {
405
+ "type": "prometheus",
406
+ "uid": "prometheus"
407
+ },
408
+ "description": "Aggregate orphan-agent rate by the emitted bounded status and reason labels. The metric does not carry role, depth, or execution IDs; use the lineage table for per-execution context. No series can mean no orphan events in the range.",
409
+ "fieldConfig": {
410
+ "defaults": {
411
+ "color": {
412
+ "mode": "palette-classic"
413
+ },
414
+ "mappings": [],
415
+ "thresholds": {
416
+ "mode": "absolute",
417
+ "steps": [
418
+ {
419
+ "color": "green",
420
+ "value": null
421
+ }
422
+ ]
423
+ },
424
+ "unit": "ops"
425
+ },
426
+ "overrides": []
427
+ },
428
+ "gridPos": {
429
+ "h": 8,
430
+ "w": 8,
431
+ "x": 0,
432
+ "y": 6
433
+ },
434
+ "id": 10,
435
+ "options": {
436
+ "legend": {
437
+ "calcs": [],
438
+ "displayMode": "list",
439
+ "placement": "bottom",
440
+ "showLegend": true
441
+ },
442
+ "tooltip": {
443
+ "mode": "single",
444
+ "sort": "none"
445
+ }
446
+ },
447
+ "targets": [
448
+ {
449
+ "datasource": {
450
+ "type": "prometheus",
451
+ "uid": "prometheus"
452
+ },
453
+ "editorMode": "code",
454
+ "expr": "sum(rate(observme_orphan_agents_total[$__rate_interval])) by (status, reason)",
455
+ "range": true,
456
+ "refId": "A",
457
+ "legendFormat": "{{status}} / {{reason}}"
458
+ }
459
+ ],
460
+ "title": "Orphan agents",
461
+ "type": "timeseries",
462
+ "links": [
463
+ {
464
+ "targetBlank": false,
465
+ "title": "Open Trace Journey",
466
+ "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}"
467
+ },
468
+ {
469
+ "targetBlank": false,
470
+ "title": "Open Agent Node Graphs",
471
+ "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}"
472
+ }
473
+ ]
474
+ },
475
+ {
476
+ "datasource": {
477
+ "type": "prometheus",
478
+ "uid": "prometheus"
479
+ },
480
+ "description": "Trace-context propagation-failure rate grouped by safe role and depth labels.",
481
+ "fieldConfig": {
482
+ "defaults": {
483
+ "color": {
484
+ "mode": "palette-classic"
485
+ },
486
+ "mappings": [],
487
+ "thresholds": {
488
+ "mode": "absolute",
489
+ "steps": [
490
+ {
491
+ "color": "green",
492
+ "value": null
493
+ }
494
+ ]
495
+ },
496
+ "unit": "ops"
497
+ },
498
+ "overrides": []
499
+ },
500
+ "gridPos": {
501
+ "h": 8,
502
+ "w": 8,
503
+ "x": 8,
504
+ "y": 6
505
+ },
506
+ "id": 11,
507
+ "options": {
508
+ "legend": {
509
+ "calcs": [],
510
+ "displayMode": "list",
511
+ "placement": "bottom",
512
+ "showLegend": true
513
+ },
514
+ "tooltip": {
515
+ "mode": "single",
516
+ "sort": "none"
517
+ }
518
+ },
519
+ "targets": [
520
+ {
521
+ "datasource": {
522
+ "type": "prometheus",
523
+ "uid": "prometheus"
524
+ },
525
+ "editorMode": "code",
526
+ "expr": "sum(rate(observme_trace_context_propagation_failures_total[$__rate_interval])) by (agent_role, subagent_depth)",
527
+ "range": true,
528
+ "refId": "A",
529
+ "legendFormat": "{{agent_role}} depth {{subagent_depth}}"
530
+ }
531
+ ],
532
+ "title": "Trace-context propagation failures",
533
+ "type": "timeseries",
534
+ "links": [
535
+ {
536
+ "targetBlank": false,
537
+ "title": "Open Trace Journey",
538
+ "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}"
539
+ },
540
+ {
541
+ "targetBlank": false,
542
+ "title": "Open Agent Node Graphs",
543
+ "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}"
544
+ }
545
+ ]
546
+ },
547
+ {
548
+ "datasource": {
549
+ "type": "prometheus",
550
+ "uid": "prometheus"
551
+ },
552
+ "description": "Child-agent failure and parent-recovery rates grouped by safe role and depth labels.",
553
+ "fieldConfig": {
554
+ "defaults": {
555
+ "color": {
556
+ "mode": "palette-classic"
557
+ },
558
+ "mappings": [],
559
+ "thresholds": {
560
+ "mode": "absolute",
561
+ "steps": [
562
+ {
563
+ "color": "green",
564
+ "value": null
565
+ }
566
+ ]
567
+ },
568
+ "unit": "ops"
569
+ },
570
+ "overrides": []
571
+ },
572
+ "gridPos": {
573
+ "h": 8,
574
+ "w": 8,
575
+ "x": 16,
576
+ "y": 6
577
+ },
578
+ "id": 13,
579
+ "options": {
580
+ "legend": {
581
+ "calcs": [],
582
+ "displayMode": "list",
583
+ "placement": "bottom",
584
+ "showLegend": true
585
+ },
586
+ "tooltip": {
587
+ "mode": "single",
588
+ "sort": "none"
589
+ }
590
+ },
591
+ "targets": [
592
+ {
593
+ "datasource": {
594
+ "type": "prometheus",
595
+ "uid": "prometheus"
596
+ },
597
+ "editorMode": "code",
598
+ "expr": "sum(rate(observme_child_agent_failures_total[$__rate_interval])) by (agent_role, subagent_depth)",
599
+ "range": true,
600
+ "refId": "A",
601
+ "legendFormat": "child failures {{agent_role}} depth {{subagent_depth}}"
602
+ },
603
+ {
604
+ "datasource": {
605
+ "type": "prometheus",
606
+ "uid": "prometheus"
607
+ },
608
+ "editorMode": "code",
609
+ "expr": "sum(rate(observme_parent_recovered_from_child_failure_total[$__rate_interval])) by (agent_role, subagent_depth)",
610
+ "range": true,
611
+ "refId": "B",
612
+ "legendFormat": "recovered {{agent_role}} depth {{subagent_depth}}"
613
+ }
614
+ ],
615
+ "title": "Child-agent failures and recovered child failures",
616
+ "type": "timeseries",
617
+ "links": [
618
+ {
619
+ "targetBlank": false,
620
+ "title": "Open Trace Journey",
621
+ "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}"
622
+ },
623
+ {
624
+ "targetBlank": false,
625
+ "title": "Open Agent Node Graphs",
626
+ "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}"
627
+ }
628
+ ]
629
+ },
630
+ {
631
+ "collapsed": false,
632
+ "gridPos": {
633
+ "h": 1,
634
+ "w": 24,
635
+ "x": 0,
636
+ "y": 14
637
+ },
638
+ "id": 101,
639
+ "panels": [],
640
+ "title": "Activity and latency",
641
+ "type": "row"
642
+ },
643
+ {
644
+ "datasource": {
645
+ "type": "prometheus",
646
+ "uid": "prometheus"
647
+ },
648
+ "description": "Agent-run rate grouped only by the low-cardinality agent_role label.",
649
+ "fieldConfig": {
650
+ "defaults": {
651
+ "color": {
652
+ "mode": "palette-classic"
653
+ },
654
+ "mappings": [],
655
+ "thresholds": {
656
+ "mode": "absolute",
657
+ "steps": [
658
+ {
659
+ "color": "green",
660
+ "value": null
661
+ }
662
+ ]
663
+ },
664
+ "unit": "ops"
665
+ },
666
+ "overrides": []
667
+ },
668
+ "gridPos": {
669
+ "h": 8,
670
+ "w": 8,
671
+ "x": 0,
672
+ "y": 15
673
+ },
674
+ "id": 1,
675
+ "options": {
676
+ "legend": {
677
+ "calcs": [],
678
+ "displayMode": "list",
679
+ "placement": "bottom",
680
+ "showLegend": true
681
+ },
682
+ "tooltip": {
683
+ "mode": "single",
684
+ "sort": "none"
685
+ }
686
+ },
687
+ "targets": [
688
+ {
689
+ "datasource": {
690
+ "type": "prometheus",
691
+ "uid": "prometheus"
692
+ },
693
+ "editorMode": "code",
694
+ "expr": "sum(rate(observme_agent_runs_total[$__rate_interval])) by (agent_role)",
695
+ "range": true,
696
+ "refId": "A",
697
+ "legendFormat": "{{agent_role}}"
698
+ }
699
+ ],
700
+ "title": "Agent runs by role",
701
+ "type": "timeseries",
702
+ "links": [
703
+ {
704
+ "targetBlank": false,
705
+ "title": "Open Trace Journey",
706
+ "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}"
707
+ },
708
+ {
709
+ "targetBlank": false,
710
+ "title": "Open Agent Node Graphs",
711
+ "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}"
712
+ }
713
+ ]
714
+ },
715
+ {
716
+ "datasource": {
717
+ "type": "prometheus",
718
+ "uid": "prometheus"
719
+ },
720
+ "description": "Subagent spawn rate grouped by low-cardinality depth, type, and reason labels.",
721
+ "fieldConfig": {
722
+ "defaults": {
723
+ "color": {
724
+ "mode": "palette-classic"
725
+ },
726
+ "mappings": [],
727
+ "thresholds": {
728
+ "mode": "absolute",
729
+ "steps": [
730
+ {
731
+ "color": "green",
732
+ "value": null
733
+ }
734
+ ]
735
+ },
736
+ "unit": "ops"
737
+ },
738
+ "overrides": []
739
+ },
740
+ "gridPos": {
741
+ "h": 8,
742
+ "w": 8,
743
+ "x": 8,
744
+ "y": 15
745
+ },
746
+ "id": 2,
747
+ "options": {
748
+ "legend": {
749
+ "calcs": [],
750
+ "displayMode": "list",
751
+ "placement": "bottom",
752
+ "showLegend": true
753
+ },
754
+ "tooltip": {
755
+ "mode": "single",
756
+ "sort": "none"
757
+ }
758
+ },
759
+ "targets": [
760
+ {
761
+ "datasource": {
762
+ "type": "prometheus",
763
+ "uid": "prometheus"
764
+ },
765
+ "editorMode": "code",
766
+ "expr": "sum(rate(observme_subagents_spawned_total[$__rate_interval])) by (subagent_depth, spawn_type, spawn_reason)",
767
+ "range": true,
768
+ "refId": "A",
769
+ "legendFormat": "depth {{subagent_depth}} / {{spawn_type}} / {{spawn_reason}}"
770
+ }
771
+ ],
772
+ "title": "Subagent spawns by depth and spawn type",
773
+ "type": "timeseries",
774
+ "links": [
775
+ {
776
+ "targetBlank": false,
777
+ "title": "Open Trace Journey",
778
+ "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}"
779
+ },
780
+ {
781
+ "targetBlank": false,
782
+ "title": "Open Agent Node Graphs",
783
+ "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}"
784
+ }
785
+ ]
786
+ },
787
+ {
788
+ "datasource": {
789
+ "type": "prometheus",
790
+ "uid": "prometheus"
791
+ },
792
+ "description": "Subagent spawn-failure rate grouped by the labels actually emitted on this counter: bounded spawn_type and error_class. Spawn reason and depth remain available on spawn-attempt metrics and in lineage logs.",
793
+ "fieldConfig": {
794
+ "defaults": {
795
+ "color": {
796
+ "mode": "palette-classic"
797
+ },
798
+ "mappings": [],
799
+ "thresholds": {
800
+ "mode": "absolute",
801
+ "steps": [
802
+ {
803
+ "color": "green",
804
+ "value": null
805
+ }
806
+ ]
807
+ },
808
+ "unit": "ops"
809
+ },
810
+ "overrides": []
811
+ },
812
+ "gridPos": {
813
+ "h": 8,
814
+ "w": 8,
815
+ "x": 16,
816
+ "y": 15
817
+ },
818
+ "id": 3,
819
+ "options": {
820
+ "legend": {
821
+ "calcs": [],
822
+ "displayMode": "list",
823
+ "placement": "bottom",
824
+ "showLegend": true
825
+ },
826
+ "tooltip": {
827
+ "mode": "single",
828
+ "sort": "none"
829
+ }
830
+ },
831
+ "targets": [
832
+ {
833
+ "datasource": {
834
+ "type": "prometheus",
835
+ "uid": "prometheus"
836
+ },
837
+ "editorMode": "code",
838
+ "expr": "sum(rate(observme_subagent_spawn_failures_total[$__rate_interval])) by (spawn_type, error_class)",
839
+ "range": true,
840
+ "refId": "A",
841
+ "legendFormat": "{{spawn_type}} / {{error_class}}"
842
+ }
843
+ ],
844
+ "title": "Subagent spawn failures",
845
+ "type": "timeseries",
846
+ "links": [
847
+ {
848
+ "targetBlank": false,
849
+ "title": "Open Trace Journey",
850
+ "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}"
851
+ },
852
+ {
853
+ "targetBlank": false,
854
+ "title": "Open Agent Node Graphs",
855
+ "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}"
856
+ }
857
+ ]
858
+ },
859
+ {
860
+ "datasource": {
861
+ "type": "prometheus",
862
+ "uid": "prometheus"
863
+ },
864
+ "description": "p95 agent-run duration grouped by agent_role only.",
865
+ "fieldConfig": {
866
+ "defaults": {
867
+ "color": {
868
+ "mode": "palette-classic"
869
+ },
870
+ "mappings": [],
871
+ "thresholds": {
872
+ "mode": "absolute",
873
+ "steps": [
874
+ {
875
+ "color": "green",
876
+ "value": null
877
+ }
878
+ ]
879
+ },
880
+ "unit": "ms"
881
+ },
882
+ "overrides": []
883
+ },
884
+ "gridPos": {
885
+ "h": 8,
886
+ "w": 8,
887
+ "x": 0,
888
+ "y": 23
889
+ },
890
+ "id": 4,
891
+ "options": {
892
+ "legend": {
893
+ "calcs": [],
894
+ "displayMode": "list",
895
+ "placement": "bottom",
896
+ "showLegend": true
897
+ },
898
+ "tooltip": {
899
+ "mode": "single",
900
+ "sort": "none"
901
+ }
902
+ },
903
+ "targets": [
904
+ {
905
+ "datasource": {
906
+ "type": "prometheus",
907
+ "uid": "prometheus"
908
+ },
909
+ "editorMode": "code",
910
+ "expr": "histogram_quantile(0.95, sum(rate(observme_agent_run_duration_ms_bucket[$__rate_interval])) by (agent_role, le))",
911
+ "range": true,
912
+ "refId": "A",
913
+ "legendFormat": "{{agent_role}}"
914
+ }
915
+ ],
916
+ "title": "p95 agent-run duration",
917
+ "type": "timeseries"
918
+ },
919
+ {
920
+ "datasource": {
921
+ "type": "prometheus",
922
+ "uid": "prometheus"
923
+ },
924
+ "description": "p95 subagent-spawn duration grouped by bounded spawn labels.",
925
+ "fieldConfig": {
926
+ "defaults": {
927
+ "color": {
928
+ "mode": "palette-classic"
929
+ },
930
+ "mappings": [],
931
+ "thresholds": {
932
+ "mode": "absolute",
933
+ "steps": [
934
+ {
935
+ "color": "green",
936
+ "value": null
937
+ }
938
+ ]
939
+ },
940
+ "unit": "ms"
941
+ },
942
+ "overrides": []
943
+ },
944
+ "gridPos": {
945
+ "h": 8,
946
+ "w": 8,
947
+ "x": 8,
948
+ "y": 23
949
+ },
950
+ "id": 5,
951
+ "options": {
952
+ "legend": {
953
+ "calcs": [],
954
+ "displayMode": "list",
955
+ "placement": "bottom",
956
+ "showLegend": true
957
+ },
958
+ "tooltip": {
959
+ "mode": "single",
960
+ "sort": "none"
961
+ }
962
+ },
963
+ "targets": [
964
+ {
965
+ "datasource": {
966
+ "type": "prometheus",
967
+ "uid": "prometheus"
968
+ },
969
+ "editorMode": "code",
970
+ "expr": "histogram_quantile(0.95, sum(rate(observme_subagent_spawn_duration_ms_bucket[$__rate_interval])) by (agent_role, spawn_type, spawn_reason, le))",
971
+ "range": true,
972
+ "refId": "A",
973
+ "legendFormat": "{{agent_role}} / {{spawn_type}} / {{spawn_reason}}"
974
+ }
975
+ ],
976
+ "title": "p95 subagent-spawn duration",
977
+ "type": "timeseries"
978
+ },
979
+ {
980
+ "datasource": {
981
+ "type": "prometheus",
982
+ "uid": "prometheus"
983
+ },
984
+ "description": "p95 parent wait and join latency grouped by agent_role only.",
985
+ "fieldConfig": {
986
+ "defaults": {
987
+ "color": {
988
+ "mode": "palette-classic"
989
+ },
990
+ "mappings": [],
991
+ "thresholds": {
992
+ "mode": "absolute",
993
+ "steps": [
994
+ {
995
+ "color": "green",
996
+ "value": null
997
+ }
998
+ ]
999
+ },
1000
+ "unit": "ms"
1001
+ },
1002
+ "overrides": []
1003
+ },
1004
+ "gridPos": {
1005
+ "h": 8,
1006
+ "w": 8,
1007
+ "x": 16,
1008
+ "y": 23
1009
+ },
1010
+ "id": 12,
1011
+ "options": {
1012
+ "legend": {
1013
+ "calcs": [],
1014
+ "displayMode": "list",
1015
+ "placement": "bottom",
1016
+ "showLegend": true
1017
+ },
1018
+ "tooltip": {
1019
+ "mode": "single",
1020
+ "sort": "none"
1021
+ }
1022
+ },
1023
+ "targets": [
1024
+ {
1025
+ "datasource": {
1026
+ "type": "prometheus",
1027
+ "uid": "prometheus"
1028
+ },
1029
+ "editorMode": "code",
1030
+ "expr": "histogram_quantile(0.95, sum(rate(observme_agent_wait_duration_ms_bucket[$__rate_interval])) by (agent_role, le))",
1031
+ "range": true,
1032
+ "refId": "A",
1033
+ "legendFormat": "wait p95 {{agent_role}}"
1034
+ },
1035
+ {
1036
+ "datasource": {
1037
+ "type": "prometheus",
1038
+ "uid": "prometheus"
1039
+ },
1040
+ "editorMode": "code",
1041
+ "expr": "histogram_quantile(0.95, sum(rate(observme_agent_join_duration_ms_bucket[$__rate_interval])) by (agent_role, le))",
1042
+ "range": true,
1043
+ "refId": "B",
1044
+ "legendFormat": "join p95 {{agent_role}}"
1045
+ }
1046
+ ],
1047
+ "title": "Parent wait/join latency",
1048
+ "type": "timeseries"
1049
+ },
1050
+ {
1051
+ "collapsed": false,
1052
+ "gridPos": {
1053
+ "h": 1,
1054
+ "w": 24,
1055
+ "x": 0,
1056
+ "y": 31
1057
+ },
1058
+ "id": 102,
1059
+ "panels": [],
1060
+ "title": "Tree shape",
1061
+ "type": "row"
1062
+ },
1063
+ {
1064
+ "datasource": {
1065
+ "type": "prometheus",
1066
+ "uid": "prometheus"
1067
+ },
1068
+ "description": "Current active-agent gauge grouped by the emitted agent_role label, plus the global total. The tunable total=100 reference matches ObservMeActiveAgentsStuckHigh; a missing gauge is unknown telemetry, not healthy zero.",
1069
+ "fieldConfig": {
1070
+ "defaults": {
1071
+ "color": {
1072
+ "mode": "thresholds"
1073
+ },
1074
+ "mappings": [],
1075
+ "thresholds": {
1076
+ "mode": "absolute",
1077
+ "steps": [
1078
+ {
1079
+ "color": "green",
1080
+ "value": null
1081
+ },
1082
+ {
1083
+ "color": "red",
1084
+ "value": 100
1085
+ }
1086
+ ]
1087
+ },
1088
+ "unit": "short"
1089
+ },
1090
+ "overrides": []
1091
+ },
1092
+ "gridPos": {
1093
+ "h": 8,
1094
+ "w": 8,
1095
+ "x": 0,
1096
+ "y": 32
1097
+ },
1098
+ "id": 7,
1099
+ "options": {
1100
+ "legend": {
1101
+ "calcs": [],
1102
+ "displayMode": "list",
1103
+ "placement": "bottom",
1104
+ "showLegend": true
1105
+ },
1106
+ "tooltip": {
1107
+ "mode": "single",
1108
+ "sort": "none"
1109
+ }
1110
+ },
1111
+ "targets": [
1112
+ {
1113
+ "datasource": {
1114
+ "type": "prometheus",
1115
+ "uid": "prometheus"
1116
+ },
1117
+ "editorMode": "code",
1118
+ "expr": "sum(observme_active_agents) by (agent_role)",
1119
+ "range": true,
1120
+ "refId": "A",
1121
+ "legendFormat": "{{agent_role}}"
1122
+ },
1123
+ {
1124
+ "datasource": {
1125
+ "type": "prometheus",
1126
+ "uid": "prometheus"
1127
+ },
1128
+ "editorMode": "code",
1129
+ "expr": "sum(observme_active_agents)",
1130
+ "range": true,
1131
+ "refId": "B",
1132
+ "legendFormat": "total active agents"
1133
+ },
1134
+ {
1135
+ "datasource": {
1136
+ "type": "prometheus",
1137
+ "uid": "prometheus"
1138
+ },
1139
+ "editorMode": "code",
1140
+ "expr": "vector(100)",
1141
+ "range": true,
1142
+ "refId": "C",
1143
+ "legendFormat": "tunable alert threshold 100"
1144
+ }
1145
+ ],
1146
+ "title": "Active agents by role",
1147
+ "type": "timeseries"
1148
+ },
1149
+ {
1150
+ "datasource": {
1151
+ "type": "prometheus",
1152
+ "uid": "prometheus"
1153
+ },
1154
+ "description": "p95 tree depth and width from bounded aggregate histogram labels; no execution IDs enter Prometheus. The depth=5 reference matches ObservMeExcessiveAgentTreeDepth and is tunable to the deployment delegation policy.",
1155
+ "fieldConfig": {
1156
+ "defaults": {
1157
+ "color": {
1158
+ "mode": "thresholds"
1159
+ },
1160
+ "mappings": [],
1161
+ "thresholds": {
1162
+ "mode": "absolute",
1163
+ "steps": [
1164
+ {
1165
+ "color": "green",
1166
+ "value": null
1167
+ },
1168
+ {
1169
+ "color": "red",
1170
+ "value": 5
1171
+ }
1172
+ ]
1173
+ },
1174
+ "unit": "short"
1175
+ },
1176
+ "overrides": []
1177
+ },
1178
+ "gridPos": {
1179
+ "h": 8,
1180
+ "w": 8,
1181
+ "x": 8,
1182
+ "y": 32
1183
+ },
1184
+ "id": 8,
1185
+ "options": {
1186
+ "legend": {
1187
+ "calcs": [],
1188
+ "displayMode": "list",
1189
+ "placement": "bottom",
1190
+ "showLegend": true
1191
+ },
1192
+ "tooltip": {
1193
+ "mode": "single",
1194
+ "sort": "none"
1195
+ }
1196
+ },
1197
+ "targets": [
1198
+ {
1199
+ "datasource": {
1200
+ "type": "prometheus",
1201
+ "uid": "prometheus"
1202
+ },
1203
+ "editorMode": "code",
1204
+ "expr": "histogram_quantile(0.95, sum(rate(observme_agent_tree_depth_bucket[$__rate_interval])) by (subagent_depth, le))",
1205
+ "range": true,
1206
+ "refId": "A",
1207
+ "legendFormat": "depth p95 {{subagent_depth}}"
1208
+ },
1209
+ {
1210
+ "datasource": {
1211
+ "type": "prometheus",
1212
+ "uid": "prometheus"
1213
+ },
1214
+ "editorMode": "code",
1215
+ "expr": "histogram_quantile(0.95, sum(rate(observme_agent_tree_width_bucket[$__rate_interval])) by (subagent_depth, le))",
1216
+ "range": true,
1217
+ "refId": "B",
1218
+ "legendFormat": "width p95 {{subagent_depth}}"
1219
+ },
1220
+ {
1221
+ "datasource": {
1222
+ "type": "prometheus",
1223
+ "uid": "prometheus"
1224
+ },
1225
+ "editorMode": "code",
1226
+ "expr": "vector(5)",
1227
+ "refId": "C",
1228
+ "legendFormat": "alert threshold depth 5",
1229
+ "range": true
1230
+ }
1231
+ ],
1232
+ "title": "Agent-tree depth and width",
1233
+ "type": "timeseries",
1234
+ "links": [
1235
+ {
1236
+ "targetBlank": false,
1237
+ "title": "Open Trace Journey",
1238
+ "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}"
1239
+ },
1240
+ {
1241
+ "targetBlank": false,
1242
+ "title": "Open Agent Node Graphs",
1243
+ "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}"
1244
+ }
1245
+ ]
1246
+ },
1247
+ {
1248
+ "datasource": {
1249
+ "type": "prometheus",
1250
+ "uid": "prometheus"
1251
+ },
1252
+ "description": "p95 child fan-out grouped by bounded subagent depth. The fan-out=20 reference matches ObservMeRunawayAgentFanOut and is tunable to normal orchestrator workload.",
1253
+ "fieldConfig": {
1254
+ "defaults": {
1255
+ "color": {
1256
+ "mode": "thresholds"
1257
+ },
1258
+ "mappings": [],
1259
+ "thresholds": {
1260
+ "mode": "absolute",
1261
+ "steps": [
1262
+ {
1263
+ "color": "green",
1264
+ "value": null
1265
+ },
1266
+ {
1267
+ "color": "red",
1268
+ "value": 20
1269
+ }
1270
+ ]
1271
+ },
1272
+ "unit": "short"
1273
+ },
1274
+ "overrides": []
1275
+ },
1276
+ "gridPos": {
1277
+ "h": 8,
1278
+ "w": 8,
1279
+ "x": 16,
1280
+ "y": 32
1281
+ },
1282
+ "id": 9,
1283
+ "options": {
1284
+ "legend": {
1285
+ "calcs": [],
1286
+ "displayMode": "list",
1287
+ "placement": "bottom",
1288
+ "showLegend": true
1289
+ },
1290
+ "tooltip": {
1291
+ "mode": "single",
1292
+ "sort": "none"
1293
+ }
1294
+ },
1295
+ "targets": [
1296
+ {
1297
+ "datasource": {
1298
+ "type": "prometheus",
1299
+ "uid": "prometheus"
1300
+ },
1301
+ "editorMode": "code",
1302
+ "expr": "histogram_quantile(0.95, sum(rate(observme_agent_fanout_count_bucket[$__rate_interval])) by (subagent_depth, le))",
1303
+ "range": true,
1304
+ "refId": "A",
1305
+ "legendFormat": "depth {{subagent_depth}}"
1306
+ },
1307
+ {
1308
+ "datasource": {
1309
+ "type": "prometheus",
1310
+ "uid": "prometheus"
1311
+ },
1312
+ "editorMode": "code",
1313
+ "expr": "vector(20)",
1314
+ "refId": "B",
1315
+ "legendFormat": "alert threshold fan-out 20",
1316
+ "range": true
1317
+ }
1318
+ ],
1319
+ "title": "Fan-out per parent operation",
1320
+ "type": "timeseries",
1321
+ "links": [
1322
+ {
1323
+ "targetBlank": false,
1324
+ "title": "Open Trace Journey",
1325
+ "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}"
1326
+ },
1327
+ {
1328
+ "targetBlank": false,
1329
+ "title": "Open Agent Node Graphs",
1330
+ "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}"
1331
+ }
1332
+ ]
1333
+ },
1334
+ {
1335
+ "collapsed": false,
1336
+ "gridPos": {
1337
+ "h": 1,
1338
+ "w": 24,
1339
+ "x": 0,
1340
+ "y": 40
1341
+ },
1342
+ "id": 103,
1343
+ "panels": [],
1344
+ "title": "Top offenders",
1345
+ "type": "row"
1346
+ },
1347
+ {
1348
+ "datasource": {
1349
+ "type": "prometheus",
1350
+ "uid": "prometheus"
1351
+ },
1352
+ "description": "Top agent roles by p95 run duration, paired with range run volume to avoid over-reading sparse percentile series.",
1353
+ "fieldConfig": {
1354
+ "defaults": {
1355
+ "color": {
1356
+ "mode": "palette-classic"
1357
+ },
1358
+ "mappings": [],
1359
+ "thresholds": {
1360
+ "mode": "absolute",
1361
+ "steps": [
1362
+ {
1363
+ "color": "green",
1364
+ "value": null
1365
+ }
1366
+ ]
1367
+ },
1368
+ "unit": "short"
1369
+ },
1370
+ "overrides": []
1371
+ },
1372
+ "gridPos": {
1373
+ "h": 8,
1374
+ "w": 12,
1375
+ "x": 0,
1376
+ "y": 41
1377
+ },
1378
+ "id": 18,
1379
+ "links": [
1380
+ {
1381
+ "targetBlank": false,
1382
+ "title": "Open Trace Journey",
1383
+ "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}"
1384
+ }
1385
+ ],
1386
+ "options": {
1387
+ "cellHeight": "sm",
1388
+ "footer": {
1389
+ "countRows": false,
1390
+ "fields": "",
1391
+ "reducer": [
1392
+ "sum"
1393
+ ],
1394
+ "show": false
1395
+ },
1396
+ "showHeader": true
1397
+ },
1398
+ "pluginVersion": "11.1.0",
1399
+ "targets": [
1400
+ {
1401
+ "datasource": {
1402
+ "type": "prometheus",
1403
+ "uid": "prometheus"
1404
+ },
1405
+ "editorMode": "code",
1406
+ "expr": "topk(10, histogram_quantile(0.95, sum(increase(observme_agent_run_duration_ms_bucket[$__range])) by (agent_role, le)))",
1407
+ "refId": "A",
1408
+ "legendFormat": "p95 {{agent_role}}",
1409
+ "format": "table",
1410
+ "instant": true,
1411
+ "range": false
1412
+ },
1413
+ {
1414
+ "datasource": {
1415
+ "type": "prometheus",
1416
+ "uid": "prometheus"
1417
+ },
1418
+ "editorMode": "code",
1419
+ "expr": "topk(10, sum(increase(observme_agent_runs_total[$__range])) by (agent_role))",
1420
+ "refId": "B",
1421
+ "legendFormat": "runs {{agent_role}}",
1422
+ "format": "table",
1423
+ "instant": true,
1424
+ "range": false
1425
+ }
1426
+ ],
1427
+ "title": "Top slow agent roles",
1428
+ "type": "table"
1429
+ },
1430
+ {
1431
+ "datasource": {
1432
+ "type": "prometheus",
1433
+ "uid": "prometheus"
1434
+ },
1435
+ "description": "Top selected-range spawn failures by emitted spawn_type and error_class, paired with spawn-attempt volume by spawn type/reason. The failure counter does not emit spawn_reason, so the volume frame is context rather than direct reason attribution; use lineage logs for an individual failure.",
1436
+ "fieldConfig": {
1437
+ "defaults": {
1438
+ "color": {
1439
+ "mode": "palette-classic"
1440
+ },
1441
+ "mappings": [],
1442
+ "thresholds": {
1443
+ "mode": "absolute",
1444
+ "steps": [
1445
+ {
1446
+ "color": "green",
1447
+ "value": null
1448
+ }
1449
+ ]
1450
+ },
1451
+ "unit": "short"
1452
+ },
1453
+ "overrides": []
1454
+ },
1455
+ "gridPos": {
1456
+ "h": 8,
1457
+ "w": 12,
1458
+ "x": 12,
1459
+ "y": 41
1460
+ },
1461
+ "id": 19,
1462
+ "links": [
1463
+ {
1464
+ "targetBlank": false,
1465
+ "title": "Open Trace Journey",
1466
+ "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}"
1467
+ }
1468
+ ],
1469
+ "options": {
1470
+ "cellHeight": "sm",
1471
+ "footer": {
1472
+ "countRows": false,
1473
+ "fields": "",
1474
+ "reducer": [
1475
+ "sum"
1476
+ ],
1477
+ "show": false
1478
+ },
1479
+ "showHeader": true
1480
+ },
1481
+ "pluginVersion": "11.1.0",
1482
+ "targets": [
1483
+ {
1484
+ "datasource": {
1485
+ "type": "prometheus",
1486
+ "uid": "prometheus"
1487
+ },
1488
+ "editorMode": "code",
1489
+ "expr": "topk(10, sum(increase(observme_subagent_spawn_failures_total[$__range])) by (spawn_type, error_class))",
1490
+ "refId": "A",
1491
+ "legendFormat": "failures {{spawn_type}} / {{error_class}}",
1492
+ "format": "table",
1493
+ "instant": true,
1494
+ "range": false
1495
+ },
1496
+ {
1497
+ "datasource": {
1498
+ "type": "prometheus",
1499
+ "uid": "prometheus"
1500
+ },
1501
+ "editorMode": "code",
1502
+ "expr": "topk(10, sum(increase(observme_subagents_spawned_total[$__range])) by (spawn_type, spawn_reason))",
1503
+ "refId": "B",
1504
+ "legendFormat": "attempts {{spawn_type}} / {{spawn_reason}}",
1505
+ "format": "table",
1506
+ "instant": true,
1507
+ "range": false
1508
+ }
1509
+ ],
1510
+ "title": "Failing spawn reasons",
1511
+ "type": "table"
1512
+ },
1513
+ {
1514
+ "datasource": {
1515
+ "type": "prometheus",
1516
+ "uid": "prometheus"
1517
+ },
1518
+ "description": "Selected-range aggregate orphan counts by emitted status/reason, paired with spawn-attempt volume by depth. The orphan metric intentionally has no depth label, so this table does not claim direct orphan-to-depth attribution; use the lineage log rows for execution context.",
1519
+ "fieldConfig": {
1520
+ "defaults": {
1521
+ "color": {
1522
+ "mode": "palette-classic"
1523
+ },
1524
+ "mappings": [],
1525
+ "thresholds": {
1526
+ "mode": "absolute",
1527
+ "steps": [
1528
+ {
1529
+ "color": "green",
1530
+ "value": null
1531
+ }
1532
+ ]
1533
+ },
1534
+ "unit": "short"
1535
+ },
1536
+ "overrides": []
1537
+ },
1538
+ "gridPos": {
1539
+ "h": 8,
1540
+ "w": 12,
1541
+ "x": 0,
1542
+ "y": 49
1543
+ },
1544
+ "id": 20,
1545
+ "links": [
1546
+ {
1547
+ "targetBlank": false,
1548
+ "title": "Open Trace Journey",
1549
+ "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}"
1550
+ }
1551
+ ],
1552
+ "options": {
1553
+ "cellHeight": "sm",
1554
+ "footer": {
1555
+ "countRows": false,
1556
+ "fields": "",
1557
+ "reducer": [
1558
+ "sum"
1559
+ ],
1560
+ "show": false
1561
+ },
1562
+ "showHeader": true
1563
+ },
1564
+ "pluginVersion": "11.1.0",
1565
+ "targets": [
1566
+ {
1567
+ "datasource": {
1568
+ "type": "prometheus",
1569
+ "uid": "prometheus"
1570
+ },
1571
+ "editorMode": "code",
1572
+ "expr": "topk(10, sum(increase(observme_orphan_agents_total[$__range])) by (status, reason))",
1573
+ "refId": "A",
1574
+ "legendFormat": "orphans {{status}} / {{reason}}",
1575
+ "format": "table",
1576
+ "instant": true,
1577
+ "range": false
1578
+ },
1579
+ {
1580
+ "datasource": {
1581
+ "type": "prometheus",
1582
+ "uid": "prometheus"
1583
+ },
1584
+ "editorMode": "code",
1585
+ "expr": "topk(10, sum(increase(observme_subagents_spawned_total[$__range])) by (subagent_depth))",
1586
+ "refId": "B",
1587
+ "legendFormat": "spawn attempts depth {{subagent_depth}}",
1588
+ "format": "table",
1589
+ "instant": true,
1590
+ "range": false
1591
+ }
1592
+ ],
1593
+ "title": "Orphan-prone depths",
1594
+ "type": "table"
1595
+ },
1596
+ {
1597
+ "datasource": {
1598
+ "type": "prometheus",
1599
+ "uid": "prometheus"
1600
+ },
1601
+ "description": "Top selected-range p95 fan-out by emitted role/depth labels, paired with spawn-attempt volume for sparse-series context. The tunable fan-out=20 reference matches ObservMeRunawayAgentFanOut.",
1602
+ "fieldConfig": {
1603
+ "defaults": {
1604
+ "color": {
1605
+ "mode": "palette-classic"
1606
+ },
1607
+ "mappings": [],
1608
+ "thresholds": {
1609
+ "mode": "absolute",
1610
+ "steps": [
1611
+ {
1612
+ "color": "green",
1613
+ "value": null
1614
+ }
1615
+ ]
1616
+ },
1617
+ "unit": "short"
1618
+ },
1619
+ "overrides": []
1620
+ },
1621
+ "gridPos": {
1622
+ "h": 8,
1623
+ "w": 12,
1624
+ "x": 12,
1625
+ "y": 49
1626
+ },
1627
+ "id": 21,
1628
+ "links": [
1629
+ {
1630
+ "targetBlank": false,
1631
+ "title": "Open Trace Journey",
1632
+ "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}"
1633
+ }
1634
+ ],
1635
+ "options": {
1636
+ "cellHeight": "sm",
1637
+ "footer": {
1638
+ "countRows": false,
1639
+ "fields": "",
1640
+ "reducer": [
1641
+ "sum"
1642
+ ],
1643
+ "show": false
1644
+ },
1645
+ "showHeader": true
1646
+ },
1647
+ "pluginVersion": "11.1.0",
1648
+ "targets": [
1649
+ {
1650
+ "datasource": {
1651
+ "type": "prometheus",
1652
+ "uid": "prometheus"
1653
+ },
1654
+ "editorMode": "code",
1655
+ "expr": "topk(10, histogram_quantile(0.95, sum(increase(observme_agent_fanout_count_bucket[$__range])) by (agent_role, subagent_depth, le)))",
1656
+ "refId": "A",
1657
+ "legendFormat": "{{agent_role}} depth {{subagent_depth}}",
1658
+ "format": "table",
1659
+ "instant": true,
1660
+ "range": false
1661
+ },
1662
+ {
1663
+ "datasource": {
1664
+ "type": "prometheus",
1665
+ "uid": "prometheus"
1666
+ },
1667
+ "editorMode": "code",
1668
+ "expr": "topk(10, sum(increase(observme_subagents_spawned_total[$__range])) by (agent_role, subagent_depth))",
1669
+ "refId": "B",
1670
+ "legendFormat": "spawn attempts {{agent_role}} depth {{subagent_depth}}",
1671
+ "format": "table",
1672
+ "instant": true,
1673
+ "range": false
1674
+ },
1675
+ {
1676
+ "datasource": {
1677
+ "type": "prometheus",
1678
+ "uid": "prometheus"
1679
+ },
1680
+ "editorMode": "code",
1681
+ "expr": "vector(20)",
1682
+ "refId": "C",
1683
+ "legendFormat": "tunable alert threshold fan-out 20",
1684
+ "format": "table",
1685
+ "instant": true,
1686
+ "range": false
1687
+ }
1688
+ ],
1689
+ "title": "High fan-out roles",
1690
+ "type": "table"
1691
+ },
1692
+ {
1693
+ "collapsed": false,
1694
+ "gridPos": {
1695
+ "h": 1,
1696
+ "w": 24,
1697
+ "x": 0,
1698
+ "y": 57
1699
+ },
1700
+ "id": 104,
1701
+ "panels": [],
1702
+ "title": "Lineage logs",
1703
+ "type": "row"
1704
+ },
1705
+ {
1706
+ "datasource": {
1707
+ "type": "loki",
1708
+ "uid": "loki"
1709
+ },
1710
+ "description": "Chronological agent-tree handoffs and lineage failures. A deterministic safe JSON line exposes event, session, workflow, parent, child/agent, run, trace, and span fields; raw/content fields are hidden. trace_id opens Tempo and agent_run opens a filtered Trace Journey when values exist. The default Collector does not promote pi.agent.child.id, so child can be empty while parent/agent context remains available. Empty means no matching handoff or lineage-failure logs in the selected range.",
1711
+ "fieldConfig": {
1712
+ "defaults": {
1713
+ "color": {
1714
+ "mode": "palette-classic"
1715
+ },
1716
+ "mappings": [],
1717
+ "thresholds": {
1718
+ "mode": "absolute",
1719
+ "steps": [
1720
+ {
1721
+ "color": "green",
1722
+ "value": null
1723
+ }
1724
+ ]
1725
+ },
1726
+ "unit": "short"
1727
+ },
1728
+ "overrides": [
1729
+ {
1730
+ "matcher": {
1731
+ "id": "byName",
1732
+ "options": "trace_id"
1733
+ },
1734
+ "properties": [
1735
+ {
1736
+ "id": "links",
1737
+ "value": [
1738
+ {
1739
+ "targetBlank": false,
1740
+ "title": "Open Tempo trace",
1741
+ "url": "/explore?left=%7B%22datasource%22:%22tempo%22,%22queries%22:%5B%7B%22query%22:%22${__value.raw}%22%7D%5D,%22range%22:%7B%22from%22:%22${__from}%22,%22to%22:%22${__to}%22%7D%7D"
1742
+ }
1743
+ ]
1744
+ }
1745
+ ]
1746
+ },
1747
+ {
1748
+ "matcher": {
1749
+ "id": "byName",
1750
+ "options": "agent_run"
1751
+ },
1752
+ "properties": [
1753
+ {
1754
+ "id": "links",
1755
+ "value": [
1756
+ {
1757
+ "targetBlank": false,
1758
+ "title": "Open filtered Trace Journey",
1759
+ "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}&var-session_id=${__data.fields[\"session\"]}&var-workflow_id=${__data.fields[\"workflow\"]}&var-agent_id=${__data.fields[\"agent\"]}&var-agent_run_id=${__value.raw}"
1760
+ }
1761
+ ]
1762
+ }
1763
+ ]
1764
+ }
1765
+ ]
1766
+ },
1767
+ "gridPos": {
1768
+ "h": 10,
1769
+ "w": 24,
1770
+ "x": 0,
1771
+ "y": 58
1772
+ },
1773
+ "id": 6,
1774
+ "options": {
1775
+ "cellHeight": "sm",
1776
+ "footer": {
1777
+ "countRows": false,
1778
+ "fields": "",
1779
+ "reducer": [
1780
+ "sum"
1781
+ ],
1782
+ "show": false
1783
+ },
1784
+ "showHeader": true,
1785
+ "sortBy": [
1786
+ {
1787
+ "desc": false,
1788
+ "displayName": "Time"
1789
+ }
1790
+ ]
1791
+ },
1792
+ "targets": [
1793
+ {
1794
+ "datasource": {
1795
+ "type": "loki",
1796
+ "uid": "loki"
1797
+ },
1798
+ "editorMode": "code",
1799
+ "expr": "{service_name=\"observme-pi-extension\", event_category=\"agent-tree\", event_name=~\"agent[.]spawn[.](started|completed|failed)|agent[.](wait|join)[.](started|completed)|agent[.]orphaned|trace_context[.]propagation_failed\"} | line_format `{\"event\":{{ printf \"%q\" .event_name }},\"session\":{{ printf \"%q\" .pi_session_id }},\"workflow\":{{ printf \"%q\" .pi_workflow_id }},\"parent\":{{ printf \"%q\" .pi_agent_parent_id }},\"child\":{{ printf \"%q\" .pi_agent_child_id }},\"agent\":{{ printf \"%q\" .pi_agent_id }},\"agent_run\":{{ printf \"%q\" .pi_agent_run_id }},\"trace_id\":{{ printf \"%q\" .trace_id }},\"span_id\":{{ printf \"%q\" .span_id }}}`",
1800
+ "range": true,
1801
+ "refId": "A",
1802
+ "legendFormat": "{{event_name}}",
1803
+ "maxLines": 500,
1804
+ "queryType": "range"
1805
+ }
1806
+ ],
1807
+ "title": "Parent/child handoffs with trace links",
1808
+ "type": "table",
1809
+ "links": [
1810
+ {
1811
+ "targetBlank": false,
1812
+ "title": "Open matching Trace Journey",
1813
+ "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}"
1814
+ }
1815
+ ],
1816
+ "transformations": [
1817
+ {
1818
+ "id": "labelsToFields",
1819
+ "options": {
1820
+ "mode": "columns"
1821
+ }
1822
+ },
1823
+ {
1824
+ "id": "extractFields",
1825
+ "options": {
1826
+ "format": "json",
1827
+ "keepTime": true,
1828
+ "replace": false,
1829
+ "source": "Line"
1830
+ }
1831
+ },
1832
+ {
1833
+ "id": "organize",
1834
+ "options": {
1835
+ "excludeByName": {
1836
+ "Line": true,
1837
+ "id": true,
1838
+ "labels": true,
1839
+ "tsNs": true,
1840
+ "service_name": true,
1841
+ "telemetry_source": true
1842
+ },
1843
+ "renameByName": {
1844
+ "event_name": "event",
1845
+ "pi_session_id": "session",
1846
+ "pi_workflow_id": "workflow",
1847
+ "pi_agent_parent_id": "parent",
1848
+ "pi_agent_child_id": "child",
1849
+ "pi_agent_id": "agent",
1850
+ "pi_agent_run_id": "agent_run",
1851
+ "trace_id": "trace_id",
1852
+ "span_id": "span_id"
1853
+ }
1854
+ }
1855
+ }
1856
+ ],
1857
+ "pluginVersion": "11.1.0"
1858
+ }
1859
+ ],
1860
+ "refresh": "30s",
1861
+ "schemaVersion": 39,
1862
+ "tags": [
1863
+ "observme",
1864
+ "agents",
1865
+ "subagents"
1866
+ ],
1867
+ "templating": {
1868
+ "list": []
1869
+ },
1870
+ "time": {
1871
+ "from": "now-6h",
1872
+ "to": "now"
1873
+ },
1874
+ "timepicker": {},
1875
+ "timezone": "browser",
1876
+ "title": "ObservMe Agents and Subagents",
1877
+ "uid": "observme-agents",
1878
+ "version": 2,
1879
+ "weekStart": ""
1880
+ }