@senad-d/observme 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/.env.example +57 -0
  2. package/CHANGELOG.md +52 -0
  3. package/LICENSE +21 -0
  4. package/ObservMe-Production-Docs/00-README.md +79 -0
  5. package/ObservMe-Production-Docs/01-requirements-and-scope.md +207 -0
  6. package/ObservMe-Production-Docs/02-reference-architecture.md +306 -0
  7. package/ObservMe-Production-Docs/03-pi-event-and-session-model.md +266 -0
  8. package/ObservMe-Production-Docs/04-telemetry-semantic-conventions.md +722 -0
  9. package/ObservMe-Production-Docs/05-otel-pipeline-and-collector.md +355 -0
  10. package/ObservMe-Production-Docs/06-security-privacy-redaction.md +294 -0
  11. package/ObservMe-Production-Docs/07-extension-implementation-blueprint.md +447 -0
  12. package/ObservMe-Production-Docs/08-query-grafana-integration.md +276 -0
  13. package/ObservMe-Production-Docs/09-dashboards-alerts-slos.md +640 -0
  14. package/ObservMe-Production-Docs/10-testing-release-operations.md +319 -0
  15. package/ObservMe-Production-Docs/11-deployment-runbooks.md +203 -0
  16. package/ObservMe-Production-Docs/12-configuration-reference.md +337 -0
  17. package/ObservMe-Production-Docs/13-source-notes.md +33 -0
  18. package/ObservMe-Production-Docs/pi-session-format.md +427 -0
  19. package/README.md +356 -0
  20. package/SECURITY.md +45 -0
  21. package/dashboards/observme-agent-node-graphs.json +250 -0
  22. package/dashboards/observme-agents.json +1880 -0
  23. package/dashboards/observme-alerts.yaml +113 -0
  24. package/dashboards/observme-branches-compactions.json +1042 -0
  25. package/dashboards/observme-cost.json +1254 -0
  26. package/dashboards/observme-errors.json +1659 -0
  27. package/dashboards/observme-export-health.json +1802 -0
  28. package/dashboards/observme-latency.json +1494 -0
  29. package/dashboards/observme-llm-conversations.json +730 -0
  30. package/dashboards/observme-logs-llm.json +644 -0
  31. package/dashboards/observme-models.json +933 -0
  32. package/dashboards/observme-overview.json +2129 -0
  33. package/dashboards/observme-slo-health.json +737 -0
  34. package/dashboards/observme-slos.yaml +56 -0
  35. package/dashboards/observme-tools.json +902 -0
  36. package/dashboards/observme-trace-journey.json +1793 -0
  37. package/docs/STRUCTURE.md +49 -0
  38. package/docs/agent-subagent-observability-requirements.md +997 -0
  39. package/docs/compatibility-matrix.md +39 -0
  40. package/docs/configuration-tui-design-standard.md +767 -0
  41. package/docs/configuration.md +38 -0
  42. package/docs/review-validation.md +115 -0
  43. package/docs/validation-flow.md +117 -0
  44. package/examples/collector.yaml +123 -0
  45. package/examples/observme.yaml +131 -0
  46. package/img/demo.gif +0 -0
  47. package/img/icon.svg +47 -0
  48. package/package.json +103 -0
  49. package/src/commands/obs-agents-runtime.ts +150 -0
  50. package/src/commands/obs-agents.ts +491 -0
  51. package/src/commands/obs-args.ts +63 -0
  52. package/src/commands/obs-backfill.ts +1334 -0
  53. package/src/commands/obs-command-support.ts +43 -0
  54. package/src/commands/obs-cost.ts +228 -0
  55. package/src/commands/obs-diagnostics.ts +22 -0
  56. package/src/commands/obs-errors.ts +156 -0
  57. package/src/commands/obs-health.ts +301 -0
  58. package/src/commands/obs-link.ts +90 -0
  59. package/src/commands/obs-logs.ts +194 -0
  60. package/src/commands/obs-loki-summary.ts +184 -0
  61. package/src/commands/obs-session.ts +259 -0
  62. package/src/commands/obs-status.ts +359 -0
  63. package/src/commands/obs-tools.ts +274 -0
  64. package/src/commands/obs-trace.ts +411 -0
  65. package/src/commands/obs.ts +211 -0
  66. package/src/config/bootstrap-project-config.ts +300 -0
  67. package/src/config/defaults.ts +143 -0
  68. package/src/config/load-config.ts +631 -0
  69. package/src/config/project-paths.ts +61 -0
  70. package/src/config/schema.ts +405 -0
  71. package/src/config/validate.ts +456 -0
  72. package/src/constants.ts +4 -0
  73. package/src/diagnostics/sanitize.ts +6 -0
  74. package/src/extension.ts +38 -0
  75. package/src/otel/logs.ts +160 -0
  76. package/src/otel/metrics.ts +165 -0
  77. package/src/otel/otlp-endpoint.ts +10 -0
  78. package/src/otel/sdk.ts +114 -0
  79. package/src/otel/shutdown.ts +102 -0
  80. package/src/otel/traces.ts +166 -0
  81. package/src/pi/agent-lineage.ts +378 -0
  82. package/src/pi/agent-tree-tracker.ts +258 -0
  83. package/src/pi/event-handlers/agent-turn.ts +155 -0
  84. package/src/pi/event-handlers/lifecycle.ts +642 -0
  85. package/src/pi/event-handlers/llm.ts +115 -0
  86. package/src/pi/event-handlers/session-events.ts +159 -0
  87. package/src/pi/event-handlers/tool-bash.ts +275 -0
  88. package/src/pi/handler-internals.ts +2154 -0
  89. package/src/pi/handler-runtime.ts +633 -0
  90. package/src/pi/handler-types.ts +261 -0
  91. package/src/pi/handlers.ts +75 -0
  92. package/src/pi/subagent-spawn.ts +975 -0
  93. package/src/pi/subagent-types.ts +29 -0
  94. package/src/privacy/content-capture.ts +104 -0
  95. package/src/privacy/hash.ts +93 -0
  96. package/src/privacy/redact.ts +619 -0
  97. package/src/privacy/secret-patterns.ts +185 -0
  98. package/src/privacy/truncate.ts +69 -0
  99. package/src/query/grafana-readiness.ts +164 -0
  100. package/src/query/grafana-transport.ts +481 -0
  101. package/src/query/grafana.ts +371 -0
  102. package/src/query/loki.ts +332 -0
  103. package/src/query/prometheus.ts +388 -0
  104. package/src/query/tempo.ts +332 -0
  105. package/src/safety/sensitive-input.ts +208 -0
  106. package/src/semconv/attributes.ts +279 -0
  107. package/src/semconv/metrics.ts +146 -0
  108. package/src/semconv/spans.ts +19 -0
  109. package/src/semconv/values.ts +13 -0
  110. package/src/util/bounded-map.ts +97 -0
  111. package/tsconfig.json +15 -0
@@ -0,0 +1,1254 @@
1
+ {
2
+ "__requires": [
3
+ {
4
+ "id": "grafana",
5
+ "name": "Grafana",
6
+ "type": "grafana",
7
+ "version": "10.0.0"
8
+ },
9
+ {
10
+ "id": "prometheus",
11
+ "name": "Prometheus",
12
+ "type": "datasource",
13
+ "version": "1.0.0"
14
+ },
15
+ {
16
+ "id": "row",
17
+ "name": "Row",
18
+ "type": "panel",
19
+ "version": ""
20
+ },
21
+ {
22
+ "id": "timeseries",
23
+ "name": "Time series",
24
+ "type": "panel",
25
+ "version": ""
26
+ },
27
+ {
28
+ "id": "stat",
29
+ "name": "Stat",
30
+ "type": "panel",
31
+ "version": ""
32
+ },
33
+ {
34
+ "id": "bargauge",
35
+ "name": "Bar gauge",
36
+ "type": "panel",
37
+ "version": ""
38
+ },
39
+ {
40
+ "id": "text",
41
+ "name": "Text",
42
+ "type": "panel",
43
+ "version": ""
44
+ }
45
+ ],
46
+ "annotations": {
47
+ "list": [
48
+ {
49
+ "datasource": {
50
+ "type": "prometheus",
51
+ "uid": "prometheus"
52
+ },
53
+ "enable": true,
54
+ "hide": false,
55
+ "iconColor": "rgba(31, 120, 193, 1)",
56
+ "name": "Model changes",
57
+ "expr": "sum(increase(observme_model_changes_total[$__interval])) by (provider, model) > 0",
58
+ "titleFormat": "Model change",
59
+ "textFormat": "{{provider}} / {{model}}"
60
+ },
61
+ {
62
+ "datasource": {
63
+ "type": "prometheus",
64
+ "uid": "prometheus"
65
+ },
66
+ "enable": true,
67
+ "hide": false,
68
+ "iconColor": "rgba(245, 183, 0, 1)",
69
+ "name": "Thinking level changes",
70
+ "expr": "sum(increase(observme_thinking_level_changes_total[$__interval])) by (environment) > 0",
71
+ "titleFormat": "Thinking level change",
72
+ "textFormat": "{{environment}}"
73
+ }
74
+ ]
75
+ },
76
+ "editable": true,
77
+ "fiscalYearStartMonth": 0,
78
+ "graphTooltip": 1,
79
+ "id": null,
80
+ "links": [
81
+ {
82
+ "icon": "external link",
83
+ "includeVars": true,
84
+ "keepTime": true,
85
+ "targetBlank": false,
86
+ "title": "Overview",
87
+ "tooltip": "Open the overview with the current time range.",
88
+ "type": "link",
89
+ "url": "/d/observme-overview/observme-overview?${__url_time_range}"
90
+ },
91
+ {
92
+ "icon": "external link",
93
+ "includeVars": true,
94
+ "keepTime": true,
95
+ "targetBlank": false,
96
+ "title": "Models",
97
+ "tooltip": "Open model traffic, latency, stop-reason, and cost-efficiency panels.",
98
+ "type": "link",
99
+ "url": "/d/observme-models/observme-models?${__url_time_range}"
100
+ },
101
+ {
102
+ "icon": "external link",
103
+ "includeVars": true,
104
+ "keepTime": true,
105
+ "targetBlank": false,
106
+ "title": "LLM lifecycle logs",
107
+ "tooltip": "Open LLM token and non-content lifecycle logs.",
108
+ "type": "link",
109
+ "url": "/d/observme-logs-llm/observme-logs-llm?${__url_time_range}"
110
+ },
111
+ {
112
+ "icon": "external link",
113
+ "includeVars": true,
114
+ "keepTime": true,
115
+ "targetBlank": false,
116
+ "title": "LLM Conversations",
117
+ "tooltip": "Open optional captured LLM content with privacy guidance.",
118
+ "type": "link",
119
+ "url": "/d/observme-llm-conversations/observme-llm-conversations?${__url_time_range}"
120
+ },
121
+ {
122
+ "icon": "external link",
123
+ "includeVars": true,
124
+ "keepTime": true,
125
+ "targetBlank": false,
126
+ "title": "Errors",
127
+ "tooltip": "Open LLM failure ratios, failure logs, and trace links.",
128
+ "type": "link",
129
+ "url": "/d/observme-errors/observme-errors?${__url_time_range}"
130
+ }
131
+ ],
132
+ "liveNow": false,
133
+ "panels": [
134
+ {
135
+ "collapsed": false,
136
+ "gridPos": {
137
+ "h": 1,
138
+ "w": 24,
139
+ "x": 0,
140
+ "y": 0
141
+ },
142
+ "id": 100,
143
+ "panels": [],
144
+ "title": "Spend overview",
145
+ "type": "row"
146
+ },
147
+ {
148
+ "datasource": {
149
+ "type": "prometheus",
150
+ "uid": "prometheus"
151
+ },
152
+ "description": "Total provider-reported LLM spend in the selected dashboard range. Missing cost series render as zero. The $40/$50 colors are review references; the $50 alert threshold applies specifically to a one-hour window.",
153
+ "fieldConfig": {
154
+ "defaults": {
155
+ "color": {
156
+ "mode": "thresholds"
157
+ },
158
+ "mappings": [],
159
+ "thresholds": {
160
+ "mode": "absolute",
161
+ "steps": [
162
+ {
163
+ "color": "green",
164
+ "value": null
165
+ },
166
+ {
167
+ "color": "yellow",
168
+ "value": 40
169
+ },
170
+ {
171
+ "color": "red",
172
+ "value": 50
173
+ }
174
+ ]
175
+ },
176
+ "unit": "currencyUSD"
177
+ },
178
+ "overrides": []
179
+ },
180
+ "gridPos": {
181
+ "h": 5,
182
+ "w": 6,
183
+ "x": 0,
184
+ "y": 1
185
+ },
186
+ "id": 1,
187
+ "options": {
188
+ "colorMode": "value",
189
+ "graphMode": "area",
190
+ "justifyMode": "auto",
191
+ "orientation": "auto",
192
+ "reduceOptions": {
193
+ "calcs": [
194
+ "lastNotNull"
195
+ ],
196
+ "fields": "",
197
+ "values": false
198
+ },
199
+ "showPercentChange": false,
200
+ "textMode": "auto",
201
+ "wideLayout": true
202
+ },
203
+ "pluginVersion": "11.1.0",
204
+ "targets": [
205
+ {
206
+ "datasource": {
207
+ "type": "prometheus",
208
+ "uid": "prometheus"
209
+ },
210
+ "editorMode": "code",
211
+ "expr": "sum(increase(observme_llm_cost_usd_total[$__range])) or vector(0)",
212
+ "instant": true,
213
+ "range": false,
214
+ "refId": "A"
215
+ }
216
+ ],
217
+ "title": "Total selected-range cost",
218
+ "type": "stat"
219
+ },
220
+ {
221
+ "datasource": {
222
+ "type": "prometheus",
223
+ "uid": "prometheus"
224
+ },
225
+ "description": "Average hourly LLM cost estimated from selected-range spend. Yellow/red align with 80%/100% of the default $50-in-1h cost-spike alert threshold; tune for your deployment.",
226
+ "fieldConfig": {
227
+ "defaults": {
228
+ "color": {
229
+ "mode": "thresholds"
230
+ },
231
+ "mappings": [],
232
+ "thresholds": {
233
+ "mode": "absolute",
234
+ "steps": [
235
+ {
236
+ "color": "green",
237
+ "value": null
238
+ },
239
+ {
240
+ "color": "yellow",
241
+ "value": 40
242
+ },
243
+ {
244
+ "color": "red",
245
+ "value": 50
246
+ }
247
+ ]
248
+ },
249
+ "unit": "currencyUSD"
250
+ },
251
+ "overrides": []
252
+ },
253
+ "gridPos": {
254
+ "h": 5,
255
+ "w": 6,
256
+ "x": 6,
257
+ "y": 1
258
+ },
259
+ "id": 2,
260
+ "options": {
261
+ "colorMode": "value",
262
+ "graphMode": "area",
263
+ "justifyMode": "auto",
264
+ "orientation": "auto",
265
+ "reduceOptions": {
266
+ "calcs": [
267
+ "lastNotNull"
268
+ ],
269
+ "fields": "",
270
+ "values": false
271
+ },
272
+ "showPercentChange": false,
273
+ "textMode": "auto",
274
+ "wideLayout": true
275
+ },
276
+ "pluginVersion": "11.1.0",
277
+ "targets": [
278
+ {
279
+ "datasource": {
280
+ "type": "prometheus",
281
+ "uid": "prometheus"
282
+ },
283
+ "editorMode": "code",
284
+ "expr": "(sum(increase(observme_llm_cost_usd_total[$__range])) or vector(0)) / ($__range_s / 3600)",
285
+ "instant": true,
286
+ "range": false,
287
+ "refId": "A"
288
+ }
289
+ ],
290
+ "title": "Cost burn rate per hour",
291
+ "type": "stat"
292
+ },
293
+ {
294
+ "datasource": {
295
+ "type": "prometheus",
296
+ "uid": "prometheus"
297
+ },
298
+ "description": "Selected-range spend divided by the default $50/hour budget scaled to the dashboard range length. This is an average budget-envelope comparison, not the state of the rolling one-hour cost-spike alert.",
299
+ "fieldConfig": {
300
+ "defaults": {
301
+ "color": {
302
+ "mode": "thresholds"
303
+ },
304
+ "mappings": [],
305
+ "thresholds": {
306
+ "mode": "absolute",
307
+ "steps": [
308
+ {
309
+ "color": "green",
310
+ "value": null
311
+ },
312
+ {
313
+ "color": "yellow",
314
+ "value": 0.8
315
+ },
316
+ {
317
+ "color": "red",
318
+ "value": 1
319
+ }
320
+ ]
321
+ },
322
+ "unit": "percentunit"
323
+ },
324
+ "overrides": []
325
+ },
326
+ "gridPos": {
327
+ "h": 5,
328
+ "w": 6,
329
+ "x": 12,
330
+ "y": 1
331
+ },
332
+ "id": 3,
333
+ "options": {
334
+ "colorMode": "value",
335
+ "graphMode": "area",
336
+ "justifyMode": "auto",
337
+ "orientation": "auto",
338
+ "reduceOptions": {
339
+ "calcs": [
340
+ "lastNotNull"
341
+ ],
342
+ "fields": "",
343
+ "values": false
344
+ },
345
+ "showPercentChange": false,
346
+ "textMode": "auto",
347
+ "wideLayout": true
348
+ },
349
+ "pluginVersion": "11.1.0",
350
+ "targets": [
351
+ {
352
+ "datasource": {
353
+ "type": "prometheus",
354
+ "uid": "prometheus"
355
+ },
356
+ "editorMode": "code",
357
+ "expr": "(sum(increase(observme_llm_cost_usd_total[$__range])) or vector(0)) / (50 * ($__range_s / 3600))",
358
+ "instant": true,
359
+ "range": false,
360
+ "refId": "A"
361
+ }
362
+ ],
363
+ "title": "Selected-range budget usage",
364
+ "type": "stat"
365
+ },
366
+ {
367
+ "datasource": {
368
+ "type": "prometheus",
369
+ "uid": "prometheus"
370
+ },
371
+ "description": "Selected-range average burn extrapolated to 24 hours. Yellow/red align with 80%/100% of the default $1,200 daily equivalent; short or unrepresentative ranges can be noisy.",
372
+ "fieldConfig": {
373
+ "defaults": {
374
+ "color": {
375
+ "mode": "thresholds"
376
+ },
377
+ "mappings": [],
378
+ "thresholds": {
379
+ "mode": "absolute",
380
+ "steps": [
381
+ {
382
+ "color": "green",
383
+ "value": null
384
+ },
385
+ {
386
+ "color": "yellow",
387
+ "value": 960
388
+ },
389
+ {
390
+ "color": "red",
391
+ "value": 1200
392
+ }
393
+ ]
394
+ },
395
+ "unit": "currencyUSD"
396
+ },
397
+ "overrides": []
398
+ },
399
+ "gridPos": {
400
+ "h": 5,
401
+ "w": 6,
402
+ "x": 18,
403
+ "y": 1
404
+ },
405
+ "id": 4,
406
+ "options": {
407
+ "colorMode": "value",
408
+ "graphMode": "area",
409
+ "justifyMode": "auto",
410
+ "orientation": "auto",
411
+ "reduceOptions": {
412
+ "calcs": [
413
+ "lastNotNull"
414
+ ],
415
+ "fields": "",
416
+ "values": false
417
+ },
418
+ "showPercentChange": false,
419
+ "textMode": "auto",
420
+ "wideLayout": true
421
+ },
422
+ "pluginVersion": "11.1.0",
423
+ "targets": [
424
+ {
425
+ "datasource": {
426
+ "type": "prometheus",
427
+ "uid": "prometheus"
428
+ },
429
+ "editorMode": "code",
430
+ "expr": "(sum(increase(observme_llm_cost_usd_total[$__range])) or vector(0)) * 86400 / $__range_s",
431
+ "instant": true,
432
+ "range": false,
433
+ "refId": "A"
434
+ }
435
+ ],
436
+ "title": "Projected 24h cost",
437
+ "type": "stat"
438
+ },
439
+ {
440
+ "collapsed": false,
441
+ "gridPos": {
442
+ "h": 1,
443
+ "w": 24,
444
+ "x": 0,
445
+ "y": 6
446
+ },
447
+ "id": 101,
448
+ "panels": [],
449
+ "title": "Cost attribution",
450
+ "type": "row"
451
+ },
452
+ {
453
+ "datasource": {
454
+ "type": "prometheus",
455
+ "uid": "prometheus"
456
+ },
457
+ "description": "Top provider cost totals over the selected range. Missing cost series render as zero; open Models for traffic and cost-efficiency context.",
458
+ "fieldConfig": {
459
+ "defaults": {
460
+ "color": {
461
+ "mode": "palette-classic"
462
+ },
463
+ "mappings": [],
464
+ "thresholds": {
465
+ "mode": "absolute",
466
+ "steps": [
467
+ {
468
+ "color": "green",
469
+ "value": null
470
+ }
471
+ ]
472
+ },
473
+ "unit": "currencyUSD"
474
+ },
475
+ "overrides": []
476
+ },
477
+ "gridPos": {
478
+ "h": 8,
479
+ "w": 8,
480
+ "x": 0,
481
+ "y": 7
482
+ },
483
+ "id": 5,
484
+ "options": {
485
+ "displayMode": "gradient",
486
+ "maxVizHeight": 300,
487
+ "minVizHeight": 16,
488
+ "minVizWidth": 8,
489
+ "namePlacement": "auto",
490
+ "orientation": "horizontal",
491
+ "reduceOptions": {
492
+ "calcs": [
493
+ "lastNotNull"
494
+ ],
495
+ "fields": "",
496
+ "values": false
497
+ },
498
+ "showUnfilled": true,
499
+ "sizing": "auto",
500
+ "valueMode": "color"
501
+ },
502
+ "targets": [
503
+ {
504
+ "datasource": {
505
+ "type": "prometheus",
506
+ "uid": "prometheus"
507
+ },
508
+ "editorMode": "code",
509
+ "expr": "(topk(10, sum by (provider) (increase(observme_llm_cost_usd_total[$__range]))) or on() vector(0))",
510
+ "instant": true,
511
+ "legendFormat": "{{provider}}",
512
+ "range": false,
513
+ "refId": "A"
514
+ }
515
+ ],
516
+ "title": "Cost by provider",
517
+ "type": "bargauge"
518
+ },
519
+ {
520
+ "datasource": {
521
+ "type": "prometheus",
522
+ "uid": "prometheus"
523
+ },
524
+ "description": "Top provider/model cost totals over the selected range. Keeping provider in the grouping prevents identically named models from different providers being merged.",
525
+ "fieldConfig": {
526
+ "defaults": {
527
+ "color": {
528
+ "mode": "palette-classic"
529
+ },
530
+ "mappings": [],
531
+ "thresholds": {
532
+ "mode": "absolute",
533
+ "steps": [
534
+ {
535
+ "color": "green",
536
+ "value": null
537
+ }
538
+ ]
539
+ },
540
+ "unit": "currencyUSD"
541
+ },
542
+ "overrides": []
543
+ },
544
+ "gridPos": {
545
+ "h": 8,
546
+ "w": 8,
547
+ "x": 8,
548
+ "y": 7
549
+ },
550
+ "id": 6,
551
+ "options": {
552
+ "displayMode": "gradient",
553
+ "maxVizHeight": 300,
554
+ "minVizHeight": 16,
555
+ "minVizWidth": 8,
556
+ "namePlacement": "auto",
557
+ "orientation": "horizontal",
558
+ "reduceOptions": {
559
+ "calcs": [
560
+ "lastNotNull"
561
+ ],
562
+ "fields": "",
563
+ "values": false
564
+ },
565
+ "showUnfilled": true,
566
+ "sizing": "auto",
567
+ "valueMode": "color"
568
+ },
569
+ "targets": [
570
+ {
571
+ "datasource": {
572
+ "type": "prometheus",
573
+ "uid": "prometheus"
574
+ },
575
+ "editorMode": "code",
576
+ "expr": "(topk(10, sum by (provider, model) (increase(observme_llm_cost_usd_total[$__range]))) or on() vector(0))",
577
+ "instant": true,
578
+ "legendFormat": "{{provider}} / {{model}}",
579
+ "range": false,
580
+ "refId": "A"
581
+ }
582
+ ],
583
+ "title": "Cost by provider/model",
584
+ "type": "bargauge"
585
+ },
586
+ {
587
+ "datasource": {
588
+ "type": "prometheus",
589
+ "uid": "prometheus"
590
+ },
591
+ "description": "Selected-range provider-reported LLM cost grouped by deployment environment. Missing cost series render as zero.",
592
+ "fieldConfig": {
593
+ "defaults": {
594
+ "color": {
595
+ "mode": "palette-classic"
596
+ },
597
+ "mappings": [],
598
+ "thresholds": {
599
+ "mode": "absolute",
600
+ "steps": [
601
+ {
602
+ "color": "green",
603
+ "value": null
604
+ }
605
+ ]
606
+ },
607
+ "unit": "currencyUSD"
608
+ },
609
+ "overrides": []
610
+ },
611
+ "gridPos": {
612
+ "h": 8,
613
+ "w": 8,
614
+ "x": 16,
615
+ "y": 7
616
+ },
617
+ "id": 7,
618
+ "options": {
619
+ "displayMode": "gradient",
620
+ "maxVizHeight": 300,
621
+ "minVizHeight": 16,
622
+ "minVizWidth": 8,
623
+ "namePlacement": "auto",
624
+ "orientation": "horizontal",
625
+ "reduceOptions": {
626
+ "calcs": [
627
+ "lastNotNull"
628
+ ],
629
+ "fields": "",
630
+ "values": false
631
+ },
632
+ "showUnfilled": true,
633
+ "sizing": "auto",
634
+ "valueMode": "color"
635
+ },
636
+ "targets": [
637
+ {
638
+ "datasource": {
639
+ "type": "prometheus",
640
+ "uid": "prometheus"
641
+ },
642
+ "editorMode": "code",
643
+ "expr": "(sum by (environment) (increase(observme_llm_cost_usd_total[$__range])) or on() vector(0))",
644
+ "instant": true,
645
+ "legendFormat": "{{environment}}",
646
+ "range": false,
647
+ "refId": "A"
648
+ }
649
+ ],
650
+ "title": "Cost by environment",
651
+ "type": "bargauge"
652
+ },
653
+ {
654
+ "datasource": {
655
+ "type": "prometheus",
656
+ "uid": "prometheus"
657
+ },
658
+ "description": "Top provider/model shares of total selected-range LLM spend. This adds explicit percent-of-total context beside the absolute attribution panels; missing cost series render as zero.",
659
+ "fieldConfig": {
660
+ "defaults": {
661
+ "color": {
662
+ "mode": "palette-classic"
663
+ },
664
+ "mappings": [],
665
+ "max": 1,
666
+ "min": 0,
667
+ "thresholds": {
668
+ "mode": "absolute",
669
+ "steps": [
670
+ {
671
+ "color": "green",
672
+ "value": null
673
+ }
674
+ ]
675
+ },
676
+ "unit": "percentunit"
677
+ },
678
+ "overrides": []
679
+ },
680
+ "gridPos": {
681
+ "h": 7,
682
+ "w": 24,
683
+ "x": 0,
684
+ "y": 15
685
+ },
686
+ "id": 14,
687
+ "links": [
688
+ {
689
+ "targetBlank": false,
690
+ "title": "Open Models",
691
+ "url": "/d/observme-models/observme-models?${__url_time_range}"
692
+ }
693
+ ],
694
+ "options": {
695
+ "displayMode": "gradient",
696
+ "maxVizHeight": 300,
697
+ "minVizHeight": 16,
698
+ "minVizWidth": 8,
699
+ "namePlacement": "auto",
700
+ "orientation": "horizontal",
701
+ "reduceOptions": {
702
+ "calcs": [
703
+ "lastNotNull"
704
+ ],
705
+ "fields": "",
706
+ "values": false
707
+ },
708
+ "showUnfilled": true,
709
+ "sizing": "auto",
710
+ "valueMode": "color"
711
+ },
712
+ "pluginVersion": "11.1.0",
713
+ "targets": [
714
+ {
715
+ "datasource": {
716
+ "type": "prometheus",
717
+ "uid": "prometheus"
718
+ },
719
+ "editorMode": "code",
720
+ "expr": "(topk(10, sum by (provider, model) (increase(observme_llm_cost_usd_total[$__range])) / scalar(clamp_min((sum(increase(observme_llm_cost_usd_total[$__range])) or vector(0)), 1e-9))) or on() vector(0))",
721
+ "instant": true,
722
+ "legendFormat": "{{provider}} / {{model}}",
723
+ "range": false,
724
+ "refId": "A"
725
+ }
726
+ ],
727
+ "title": "Cost share by provider/model",
728
+ "type": "bargauge"
729
+ },
730
+ {
731
+ "collapsed": false,
732
+ "gridPos": {
733
+ "h": 1,
734
+ "w": 24,
735
+ "x": 0,
736
+ "y": 22
737
+ },
738
+ "id": 102,
739
+ "panels": [],
740
+ "title": "Cost trend",
741
+ "type": "row"
742
+ },
743
+ {
744
+ "datasource": {
745
+ "type": "prometheus",
746
+ "uid": "prometheus"
747
+ },
748
+ "description": "LLM cost added across all providers and models for each Prometheus-safe rolling interval. $__rate_interval keeps the window large enough for counter samples; small costs display with six decimal places.",
749
+ "fieldConfig": {
750
+ "defaults": {
751
+ "color": {
752
+ "mode": "palette-classic"
753
+ },
754
+ "decimals": 6,
755
+ "mappings": [],
756
+ "min": 0,
757
+ "thresholds": {
758
+ "mode": "absolute",
759
+ "steps": [
760
+ {
761
+ "color": "green",
762
+ "value": null
763
+ }
764
+ ]
765
+ },
766
+ "unit": "currencyUSD"
767
+ },
768
+ "overrides": []
769
+ },
770
+ "gridPos": {
771
+ "h": 8,
772
+ "w": 24,
773
+ "x": 0,
774
+ "y": 23
775
+ },
776
+ "id": 8,
777
+ "links": [
778
+ {
779
+ "targetBlank": false,
780
+ "title": "Open Models",
781
+ "url": "/d/observme-models/observme-models?${__url_time_range}"
782
+ },
783
+ {
784
+ "targetBlank": false,
785
+ "title": "Open LLM lifecycle logs",
786
+ "url": "/d/observme-logs-llm/observme-logs-llm?${__url_time_range}"
787
+ }
788
+ ],
789
+ "options": {
790
+ "legend": {
791
+ "calcs": [
792
+ "sum",
793
+ "max"
794
+ ],
795
+ "displayMode": "table",
796
+ "placement": "bottom",
797
+ "showLegend": true
798
+ },
799
+ "tooltip": {
800
+ "mode": "multi",
801
+ "sort": "desc"
802
+ }
803
+ },
804
+ "pluginVersion": "11.1.0",
805
+ "targets": [
806
+ {
807
+ "datasource": {
808
+ "type": "prometheus",
809
+ "uid": "prometheus"
810
+ },
811
+ "editorMode": "code",
812
+ "expr": "sum(increase(observme_llm_cost_usd_total[$__rate_interval])) or vector(0)",
813
+ "legendFormat": "all providers / models",
814
+ "range": true,
815
+ "refId": "A"
816
+ }
817
+ ],
818
+ "title": "Cost over time",
819
+ "type": "timeseries"
820
+ },
821
+ {
822
+ "collapsed": false,
823
+ "gridPos": {
824
+ "h": 1,
825
+ "w": 24,
826
+ "x": 0,
827
+ "y": 31
828
+ },
829
+ "id": 103,
830
+ "panels": [],
831
+ "title": "Tokens and cache",
832
+ "type": "row"
833
+ },
834
+ {
835
+ "datasource": {
836
+ "type": "prometheus",
837
+ "uid": "prometheus"
838
+ },
839
+ "description": "Highest cumulative token value observed per exporter series in the selected range, summed globally by type. This preserves short-lived Pi session exports that counter increases can miss. Provider/model detail stays in Models and LLM lifecycle logs.",
840
+ "fieldConfig": {
841
+ "defaults": {
842
+ "color": {
843
+ "mode": "palette-classic"
844
+ },
845
+ "mappings": [],
846
+ "thresholds": {
847
+ "mode": "absolute",
848
+ "steps": [
849
+ {
850
+ "color": "green",
851
+ "value": null
852
+ }
853
+ ]
854
+ },
855
+ "unit": "short"
856
+ },
857
+ "overrides": []
858
+ },
859
+ "gridPos": {
860
+ "h": 8,
861
+ "w": 12,
862
+ "x": 0,
863
+ "y": 32
864
+ },
865
+ "id": 9,
866
+ "options": {
867
+ "displayMode": "gradient",
868
+ "maxVizHeight": 300,
869
+ "minVizHeight": 16,
870
+ "minVizWidth": 8,
871
+ "namePlacement": "auto",
872
+ "orientation": "horizontal",
873
+ "reduceOptions": {
874
+ "calcs": [
875
+ "lastNotNull"
876
+ ],
877
+ "fields": "",
878
+ "values": false
879
+ },
880
+ "showUnfilled": true,
881
+ "sizing": "auto",
882
+ "valueMode": "color"
883
+ },
884
+ "targets": [
885
+ {
886
+ "datasource": {
887
+ "type": "prometheus",
888
+ "uid": "prometheus"
889
+ },
890
+ "editorMode": "code",
891
+ "expr": "sum(max_over_time(observme_llm_input_tokens_total[$__range])) or vector(0)",
892
+ "instant": true,
893
+ "legendFormat": "input",
894
+ "range": false,
895
+ "refId": "A"
896
+ },
897
+ {
898
+ "datasource": {
899
+ "type": "prometheus",
900
+ "uid": "prometheus"
901
+ },
902
+ "editorMode": "code",
903
+ "expr": "sum(max_over_time(observme_llm_output_tokens_total[$__range])) or vector(0)",
904
+ "instant": true,
905
+ "legendFormat": "output",
906
+ "range": false,
907
+ "refId": "B"
908
+ },
909
+ {
910
+ "datasource": {
911
+ "type": "prometheus",
912
+ "uid": "prometheus"
913
+ },
914
+ "editorMode": "code",
915
+ "expr": "sum(max_over_time(observme_llm_total_tokens_total[$__range])) or vector(0)",
916
+ "instant": true,
917
+ "legendFormat": "total",
918
+ "range": false,
919
+ "refId": "C"
920
+ },
921
+ {
922
+ "datasource": {
923
+ "type": "prometheus",
924
+ "uid": "prometheus"
925
+ },
926
+ "editorMode": "code",
927
+ "expr": "sum(max_over_time(observme_llm_reasoning_tokens_total[$__range])) or vector(0)",
928
+ "instant": true,
929
+ "legendFormat": "reasoning",
930
+ "range": false,
931
+ "refId": "D"
932
+ },
933
+ {
934
+ "datasource": {
935
+ "type": "prometheus",
936
+ "uid": "prometheus"
937
+ },
938
+ "editorMode": "code",
939
+ "expr": "sum(max_over_time(observme_llm_cache_read_tokens_total[$__range])) or vector(0)",
940
+ "instant": true,
941
+ "legendFormat": "cache read",
942
+ "range": false,
943
+ "refId": "E"
944
+ },
945
+ {
946
+ "datasource": {
947
+ "type": "prometheus",
948
+ "uid": "prometheus"
949
+ },
950
+ "editorMode": "code",
951
+ "expr": "sum(max_over_time(observme_llm_cache_write_tokens_total[$__range])) or vector(0)",
952
+ "instant": true,
953
+ "legendFormat": "cache write",
954
+ "range": false,
955
+ "refId": "F"
956
+ },
957
+ {
958
+ "datasource": {
959
+ "type": "prometheus",
960
+ "uid": "prometheus"
961
+ },
962
+ "editorMode": "code",
963
+ "expr": "sum(max_over_time(observme_llm_cache_write_1h_tokens_total[$__range])) or vector(0)",
964
+ "instant": true,
965
+ "legendFormat": "cache write 1h",
966
+ "range": false,
967
+ "refId": "G"
968
+ }
969
+ ],
970
+ "title": "Token totals by type",
971
+ "type": "bargauge"
972
+ },
973
+ {
974
+ "datasource": {
975
+ "type": "prometheus",
976
+ "uid": "prometheus"
977
+ },
978
+ "description": "Top provider/model cache-read, cache-write, and one-hour cache-write token totals over the selected range. Missing cache metrics render as zero.",
979
+ "fieldConfig": {
980
+ "defaults": {
981
+ "color": {
982
+ "mode": "palette-classic"
983
+ },
984
+ "mappings": [],
985
+ "thresholds": {
986
+ "mode": "absolute",
987
+ "steps": [
988
+ {
989
+ "color": "green",
990
+ "value": null
991
+ }
992
+ ]
993
+ },
994
+ "unit": "short"
995
+ },
996
+ "overrides": []
997
+ },
998
+ "gridPos": {
999
+ "h": 8,
1000
+ "w": 12,
1001
+ "x": 12,
1002
+ "y": 32
1003
+ },
1004
+ "id": 10,
1005
+ "options": {
1006
+ "displayMode": "gradient",
1007
+ "maxVizHeight": 300,
1008
+ "minVizHeight": 16,
1009
+ "minVizWidth": 8,
1010
+ "namePlacement": "auto",
1011
+ "orientation": "horizontal",
1012
+ "reduceOptions": {
1013
+ "calcs": [
1014
+ "lastNotNull"
1015
+ ],
1016
+ "fields": "",
1017
+ "values": false
1018
+ },
1019
+ "showUnfilled": true,
1020
+ "sizing": "auto",
1021
+ "valueMode": "color"
1022
+ },
1023
+ "targets": [
1024
+ {
1025
+ "datasource": {
1026
+ "type": "prometheus",
1027
+ "uid": "prometheus"
1028
+ },
1029
+ "editorMode": "code",
1030
+ "expr": "(topk(10, sum by (provider, model) (increase(observme_llm_cache_read_tokens_total[$__range]))) or on() vector(0))",
1031
+ "instant": true,
1032
+ "legendFormat": "cache read {{provider}} / {{model}}",
1033
+ "range": false,
1034
+ "refId": "A"
1035
+ },
1036
+ {
1037
+ "datasource": {
1038
+ "type": "prometheus",
1039
+ "uid": "prometheus"
1040
+ },
1041
+ "editorMode": "code",
1042
+ "expr": "(topk(10, sum by (provider, model) (increase(observme_llm_cache_write_tokens_total[$__range]))) or on() vector(0))",
1043
+ "instant": true,
1044
+ "legendFormat": "cache write {{provider}} / {{model}}",
1045
+ "range": false,
1046
+ "refId": "B"
1047
+ },
1048
+ {
1049
+ "datasource": {
1050
+ "type": "prometheus",
1051
+ "uid": "prometheus"
1052
+ },
1053
+ "editorMode": "code",
1054
+ "expr": "(topk(10, sum by (provider, model) (increase(observme_llm_cache_write_1h_tokens_total[$__range]))) or on() vector(0))",
1055
+ "instant": true,
1056
+ "legendFormat": "cache write 1h {{provider}} / {{model}}",
1057
+ "range": false,
1058
+ "refId": "C"
1059
+ }
1060
+ ],
1061
+ "title": "Cache read/write token usage",
1062
+ "type": "bargauge"
1063
+ },
1064
+ {
1065
+ "datasource": {
1066
+ "type": "prometheus",
1067
+ "uid": "prometheus"
1068
+ },
1069
+ "description": "Estimated cache-read share of reusable prompt tokens: cache reads divided by input plus cache-read tokens. This is a proxy, not a provider hit-count or health metric; zero can mean idle workload or unsupported cache accounting.",
1070
+ "fieldConfig": {
1071
+ "defaults": {
1072
+ "color": {
1073
+ "mode": "palette-classic"
1074
+ },
1075
+ "mappings": [],
1076
+ "max": 1,
1077
+ "min": 0,
1078
+ "thresholds": {
1079
+ "mode": "absolute",
1080
+ "steps": [
1081
+ {
1082
+ "color": "blue",
1083
+ "value": null
1084
+ }
1085
+ ]
1086
+ },
1087
+ "unit": "percentunit"
1088
+ },
1089
+ "overrides": []
1090
+ },
1091
+ "gridPos": {
1092
+ "h": 5,
1093
+ "w": 12,
1094
+ "x": 0,
1095
+ "y": 40
1096
+ },
1097
+ "id": 11,
1098
+ "options": {
1099
+ "colorMode": "value",
1100
+ "graphMode": "area",
1101
+ "justifyMode": "auto",
1102
+ "orientation": "auto",
1103
+ "reduceOptions": {
1104
+ "calcs": [
1105
+ "lastNotNull"
1106
+ ],
1107
+ "fields": "",
1108
+ "values": false
1109
+ },
1110
+ "showPercentChange": false,
1111
+ "textMode": "auto",
1112
+ "wideLayout": true
1113
+ },
1114
+ "pluginVersion": "11.1.0",
1115
+ "targets": [
1116
+ {
1117
+ "datasource": {
1118
+ "type": "prometheus",
1119
+ "uid": "prometheus"
1120
+ },
1121
+ "editorMode": "code",
1122
+ "expr": "clamp_max((sum(increase(observme_llm_cache_read_tokens_total[$__range])) or vector(0)) / clamp_min(((sum(increase(observme_llm_cache_read_tokens_total[$__range])) or vector(0)) + (sum(increase(observme_llm_input_tokens_total[$__range])) or vector(0))), 1), 1)",
1123
+ "instant": true,
1124
+ "range": false,
1125
+ "refId": "A"
1126
+ }
1127
+ ],
1128
+ "title": "Cache read ratio",
1129
+ "type": "stat"
1130
+ },
1131
+ {
1132
+ "datasource": {
1133
+ "type": "prometheus",
1134
+ "uid": "prometheus"
1135
+ },
1136
+ "description": "Provider-reported cache-read tokens over the selected range. Treat this as an estimated token saving only where cache-read accounting is supported; missing series render as zero.",
1137
+ "fieldConfig": {
1138
+ "defaults": {
1139
+ "color": {
1140
+ "mode": "thresholds"
1141
+ },
1142
+ "mappings": [],
1143
+ "thresholds": {
1144
+ "mode": "absolute",
1145
+ "steps": [
1146
+ {
1147
+ "color": "gray",
1148
+ "value": null
1149
+ },
1150
+ {
1151
+ "color": "green",
1152
+ "value": 1
1153
+ }
1154
+ ]
1155
+ },
1156
+ "unit": "short"
1157
+ },
1158
+ "overrides": []
1159
+ },
1160
+ "gridPos": {
1161
+ "h": 5,
1162
+ "w": 12,
1163
+ "x": 12,
1164
+ "y": 40
1165
+ },
1166
+ "id": 12,
1167
+ "options": {
1168
+ "colorMode": "value",
1169
+ "graphMode": "area",
1170
+ "justifyMode": "auto",
1171
+ "orientation": "auto",
1172
+ "reduceOptions": {
1173
+ "calcs": [
1174
+ "lastNotNull"
1175
+ ],
1176
+ "fields": "",
1177
+ "values": false
1178
+ },
1179
+ "showPercentChange": false,
1180
+ "textMode": "auto",
1181
+ "wideLayout": true
1182
+ },
1183
+ "pluginVersion": "11.1.0",
1184
+ "targets": [
1185
+ {
1186
+ "datasource": {
1187
+ "type": "prometheus",
1188
+ "uid": "prometheus"
1189
+ },
1190
+ "editorMode": "code",
1191
+ "expr": "sum(increase(observme_llm_cache_read_tokens_total[$__range])) or vector(0)",
1192
+ "instant": true,
1193
+ "range": false,
1194
+ "refId": "A"
1195
+ }
1196
+ ],
1197
+ "title": "Estimated cache savings (tokens)",
1198
+ "type": "stat"
1199
+ },
1200
+ {
1201
+ "collapsed": false,
1202
+ "gridPos": {
1203
+ "h": 1,
1204
+ "w": 24,
1205
+ "x": 0,
1206
+ "y": 45
1207
+ },
1208
+ "id": 104,
1209
+ "panels": [],
1210
+ "title": "Budget guidance",
1211
+ "type": "row"
1212
+ },
1213
+ {
1214
+ "description": "Visible interpretation guidance for the dashboard's default budget references and forecasts.",
1215
+ "gridPos": {
1216
+ "h": 5,
1217
+ "w": 24,
1218
+ "x": 0,
1219
+ "y": 46
1220
+ },
1221
+ "id": 13,
1222
+ "options": {
1223
+ "code": {
1224
+ "language": "markdown",
1225
+ "showLineNumbers": false,
1226
+ "showMiniMap": false
1227
+ },
1228
+ "content": "### Default budget references\n\n- **ObservMeCostSpike** alerts when LLM spend exceeds **$50 in 1 hour** for 15 minutes. Tune this budget-dependent threshold for your deployment.\n- **Selected-range budget usage** scales that $50/hour reference to the selected time range; 100% means spend matched the average envelope. It does not detect a short spike hidden inside a longer range.\n- **Projected 24h cost** extrapolates the average burn rate from the selected range. Very short, idle, or unrepresentative ranges can produce misleading forecasts.\n- Cost metrics use provider-reported usage. Missing cost series render as zero and do not prove that a provider supplied complete pricing data.",
1229
+ "mode": "markdown"
1230
+ },
1231
+ "pluginVersion": "11.1.0",
1232
+ "title": "How to interpret budget and forecast panels",
1233
+ "type": "text"
1234
+ }
1235
+ ],
1236
+ "refresh": "30s",
1237
+ "schemaVersion": 39,
1238
+ "tags": [
1239
+ "observme",
1240
+ "pi",
1241
+ "llm",
1242
+ "cost",
1243
+ "tokens"
1244
+ ],
1245
+ "time": {
1246
+ "from": "now-6h",
1247
+ "to": "now"
1248
+ },
1249
+ "timezone": "browser",
1250
+ "title": "ObservMe Cost",
1251
+ "uid": "observme-cost",
1252
+ "version": 8,
1253
+ "weekStart": ""
1254
+ }