@conquext/observa-loki 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dashboards/budgets.json +169 -0
- package/dashboards/cost-attribution.json +120 -0
- package/dashboards/errors.json +104 -0
- package/dashboards/forecasting.json +208 -0
- package/dashboards/model-comparison.json +152 -0
- package/dashboards/overview.json +90 -0
- package/dashboards/performance.json +119 -0
- package/dashboards/spend-analysis.json +101 -0
- package/dashboards/token-volume.json +114 -0
- package/dist/index.d.ts +24 -0
- package/dist/index.js +81 -0
- package/dist/index.js.map +1 -0
- package/package.json +28 -0
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
{
|
|
2
|
+
"__inputs": [
|
|
3
|
+
{ "name": "DS_OBSERVATORY", "label": "Observatory (Loki)", "type": "datasource", "pluginId": "loki" }
|
|
4
|
+
],
|
|
5
|
+
"title": "Observatory — Model Comparison",
|
|
6
|
+
"uid": "observatory-model-comparison",
|
|
7
|
+
"tags": ["observatory", "ai", "llm"],
|
|
8
|
+
"timezone": "browser",
|
|
9
|
+
"refresh": "30s",
|
|
10
|
+
"templating": {
|
|
11
|
+
"list": [
|
|
12
|
+
{ "name": "provider", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\"}, provider)", "multi": true, "includeAll": true },
|
|
13
|
+
{ "name": "model", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\", provider=~\"$provider\"}, model)", "multi": true, "includeAll": true },
|
|
14
|
+
{ "name": "team", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\"}, team)", "multi": true, "includeAll": true },
|
|
15
|
+
{ "name": "user_id", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\", team=~\"$team\"}, user_id)", "multi": true, "includeAll": true }
|
|
16
|
+
]
|
|
17
|
+
},
|
|
18
|
+
"panels": [
|
|
19
|
+
{
|
|
20
|
+
"title": "Cost vs Latency by Model",
|
|
21
|
+
"type": "scatter",
|
|
22
|
+
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 0 },
|
|
23
|
+
"targets": [{
|
|
24
|
+
"expr": "sum by (model) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", status=\"success\"} | json | unwrap cost [$__range])) / quantile_over_time(0.95, {job=\"observatory\", provider=~\"$provider\", model=~\"$model\", status=\"success\"} | json | unwrap latency_ms [$__range])",
|
|
25
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
26
|
+
}],
|
|
27
|
+
"fieldConfig": {
|
|
28
|
+
"overrides": [{
|
|
29
|
+
"matcher": { "id": "byName", "options": "Value" },
|
|
30
|
+
"properties": [{ "id": "unit", "value": "currencyUSD" }]
|
|
31
|
+
}]
|
|
32
|
+
}
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
"title": "Average Cost per Call",
|
|
36
|
+
"type": "stat",
|
|
37
|
+
"gridPos": { "h": 6, "w": 8, "x": 0, "y": 10 },
|
|
38
|
+
"targets": [{
|
|
39
|
+
"expr": "sum by (model) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap cost [$__range])) / sum by (model) (count_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} [$__range]))",
|
|
40
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
41
|
+
}],
|
|
42
|
+
"fieldConfig": { "defaults": { "unit": "currencyUSD" } }
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"title": "Average Latency (p95)",
|
|
46
|
+
"type": "stat",
|
|
47
|
+
"gridPos": { "h": 6, "w": 8, "x": 8, "y": 10 },
|
|
48
|
+
"targets": [{
|
|
49
|
+
"expr": "quantile_over_time(0.95, {job=\"observatory\", provider=~\"$provider\", model=~\"$model\", status=\"success\"} | json | unwrap latency_ms [$__range]) by (model)",
|
|
50
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
51
|
+
}],
|
|
52
|
+
"fieldConfig": { "defaults": { "unit": "ms" } }
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"title": "Total Calls",
|
|
56
|
+
"type": "stat",
|
|
57
|
+
"gridPos": { "h": 6, "w": 8, "x": 16, "y": 10 },
|
|
58
|
+
"targets": [{
|
|
59
|
+
"expr": "sum by (model) (count_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} [$__range]))",
|
|
60
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
61
|
+
}],
|
|
62
|
+
"fieldConfig": { "defaults": { "unit": "short" } }
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"title": "Model Performance Matrix",
|
|
66
|
+
"type": "table",
|
|
67
|
+
"gridPos": { "h": 12, "w": 24, "x": 0, "y": 16 },
|
|
68
|
+
"targets": [
|
|
69
|
+
{
|
|
70
|
+
"expr": "sum by (model) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap cost [$__range]))",
|
|
71
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
72
|
+
"refId": "cost"
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
"expr": "sum by (model) (count_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} [$__range]))",
|
|
76
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
77
|
+
"refId": "calls"
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
"expr": "quantile_over_time(0.95, {job=\"observatory\", provider=~\"$provider\", model=~\"$model\", status=\"success\"} | json | unwrap latency_ms [$__range]) by (model)",
|
|
81
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
82
|
+
"refId": "latency"
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
"expr": "sum by (model) (count_over_time({job=\"observatory\", status=\"error\", provider=~\"$provider\", model=~\"$model\"} [$__range])) / sum by (model) (count_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} [$__range]))",
|
|
86
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
87
|
+
"refId": "error_rate"
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
"expr": "sum by (model) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap total_tokens [$__range]))",
|
|
91
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
92
|
+
"refId": "tokens"
|
|
93
|
+
}
|
|
94
|
+
],
|
|
95
|
+
"transformations": [
|
|
96
|
+
{ "id": "merge", "options": {} },
|
|
97
|
+
{
|
|
98
|
+
"id": "organize",
|
|
99
|
+
"options": {
|
|
100
|
+
"excludeByName": {},
|
|
101
|
+
"indexByName": { "model": 0, "calls": 1, "cost": 2, "latency": 3, "error_rate": 4, "tokens": 5 },
|
|
102
|
+
"renameByName": {
|
|
103
|
+
"calls": "Total Calls",
|
|
104
|
+
"cost": "Total Cost",
|
|
105
|
+
"latency": "Latency p95",
|
|
106
|
+
"error_rate": "Error Rate",
|
|
107
|
+
"tokens": "Total Tokens"
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
],
|
|
112
|
+
"fieldConfig": {
|
|
113
|
+
"overrides": [
|
|
114
|
+
{
|
|
115
|
+
"matcher": { "id": "byName", "options": "Total Cost" },
|
|
116
|
+
"properties": [{ "id": "unit", "value": "currencyUSD" }]
|
|
117
|
+
},
|
|
118
|
+
{
|
|
119
|
+
"matcher": { "id": "byName", "options": "Latency p95" },
|
|
120
|
+
"properties": [{ "id": "unit", "value": "ms" }]
|
|
121
|
+
},
|
|
122
|
+
{
|
|
123
|
+
"matcher": { "id": "byName", "options": "Error Rate" },
|
|
124
|
+
"properties": [{ "id": "unit", "value": "percentunit" }]
|
|
125
|
+
}
|
|
126
|
+
]
|
|
127
|
+
}
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
"title": "Cost Efficiency (Cost per 1K Tokens)",
|
|
131
|
+
"type": "bar",
|
|
132
|
+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
|
|
133
|
+
"targets": [{
|
|
134
|
+
"expr": "sum by (model) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap cost [$__range])) / (sum by (model) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap total_tokens [$__range])) / 1000)",
|
|
135
|
+
"legendFormat": "{{model}}",
|
|
136
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
137
|
+
}],
|
|
138
|
+
"fieldConfig": { "defaults": { "unit": "currencyUSD" } }
|
|
139
|
+
},
|
|
140
|
+
{
|
|
141
|
+
"title": "Speed Efficiency (Tokens per Second)",
|
|
142
|
+
"type": "bar",
|
|
143
|
+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
|
|
144
|
+
"targets": [{
|
|
145
|
+
"expr": "sum by (model) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", status=\"success\"} | json | unwrap output_tokens [$__range])) / (sum by (model) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", status=\"success\"} | json | unwrap latency_ms [$__range])) / 1000)",
|
|
146
|
+
"legendFormat": "{{model}}",
|
|
147
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
148
|
+
}],
|
|
149
|
+
"fieldConfig": { "defaults": { "unit": "short", "decimals": 2 } }
|
|
150
|
+
}
|
|
151
|
+
]
|
|
152
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
{
|
|
2
|
+
"__inputs": [
|
|
3
|
+
{ "name": "DS_OBSERVATORY", "label": "Observatory (Loki)", "type": "datasource", "pluginId": "loki" }
|
|
4
|
+
],
|
|
5
|
+
"title": "Observatory — Overview",
|
|
6
|
+
"uid": "observatory-overview",
|
|
7
|
+
"tags": ["observatory", "ai", "llm"],
|
|
8
|
+
"timezone": "browser",
|
|
9
|
+
"refresh": "30s",
|
|
10
|
+
"templating": {
|
|
11
|
+
"list": [
|
|
12
|
+
{ "name": "provider", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\"}, provider)", "multi": true, "includeAll": true },
|
|
13
|
+
{ "name": "model", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\", provider=~\"$provider\"}, model)", "multi": true, "includeAll": true },
|
|
14
|
+
{ "name": "team", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\"}, team)", "multi": true, "includeAll": true },
|
|
15
|
+
{ "name": "user_id", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\", team=~\"$team\"}, user_id)", "multi": true, "includeAll": true }
|
|
16
|
+
]
|
|
17
|
+
},
|
|
18
|
+
"panels": [
|
|
19
|
+
{
|
|
20
|
+
"title": "Total Spend",
|
|
21
|
+
"type": "stat",
|
|
22
|
+
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
|
|
23
|
+
"targets": [{
|
|
24
|
+
"expr": "sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap cost [$__range]))",
|
|
25
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
26
|
+
}],
|
|
27
|
+
"fieldConfig": { "defaults": { "unit": "currencyUSD" } }
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"title": "Total Tokens",
|
|
31
|
+
"type": "stat",
|
|
32
|
+
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
|
|
33
|
+
"targets": [{
|
|
34
|
+
"expr": "sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap total_tokens [$__range]))",
|
|
35
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
36
|
+
}]
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"title": "Total Calls",
|
|
40
|
+
"type": "stat",
|
|
41
|
+
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
|
|
42
|
+
"targets": [{
|
|
43
|
+
"expr": "sum(count_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} [$__range]))",
|
|
44
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
45
|
+
}]
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
"title": "Error Rate",
|
|
49
|
+
"type": "stat",
|
|
50
|
+
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
|
|
51
|
+
"targets": [{
|
|
52
|
+
"expr": "sum(count_over_time({job=\"observatory\", status=\"error\", provider=~\"$provider\"} [$__range])) / sum(count_over_time({job=\"observatory\", provider=~\"$provider\"} [$__range]))",
|
|
53
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
54
|
+
}],
|
|
55
|
+
"fieldConfig": { "defaults": { "unit": "percentunit", "thresholds": { "steps": [{ "value": 0, "color": "green" }, { "value": 0.05, "color": "yellow" }, { "value": 0.1, "color": "red" }] } } }
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
"title": "Spend Over Time",
|
|
59
|
+
"type": "timeseries",
|
|
60
|
+
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 4 },
|
|
61
|
+
"targets": [{
|
|
62
|
+
"expr": "sum by (provider) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap cost [$__interval]))",
|
|
63
|
+
"legendFormat": "{{provider}}",
|
|
64
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
65
|
+
}],
|
|
66
|
+
"fieldConfig": { "defaults": { "unit": "currencyUSD" } }
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
"title": "Calls Over Time",
|
|
70
|
+
"type": "timeseries",
|
|
71
|
+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
|
|
72
|
+
"targets": [{
|
|
73
|
+
"expr": "sum by (model) (count_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} [$__interval]))",
|
|
74
|
+
"legendFormat": "{{model}}",
|
|
75
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
76
|
+
}]
|
|
77
|
+
},
|
|
78
|
+
{
|
|
79
|
+
"title": "Latency p95",
|
|
80
|
+
"type": "timeseries",
|
|
81
|
+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
|
|
82
|
+
"targets": [{
|
|
83
|
+
"expr": "quantile_over_time(0.95, {job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap latency_ms [$__interval]) by (model)",
|
|
84
|
+
"legendFormat": "{{model}} p95",
|
|
85
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
86
|
+
}],
|
|
87
|
+
"fieldConfig": { "defaults": { "unit": "ms" } }
|
|
88
|
+
}
|
|
89
|
+
]
|
|
90
|
+
}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
{
|
|
2
|
+
"__inputs": [
|
|
3
|
+
{ "name": "DS_OBSERVATORY", "label": "Observatory (Loki)", "type": "datasource", "pluginId": "loki" }
|
|
4
|
+
],
|
|
5
|
+
"title": "Observatory — Performance",
|
|
6
|
+
"uid": "observatory-performance",
|
|
7
|
+
"tags": ["observatory", "ai", "llm"],
|
|
8
|
+
"timezone": "browser",
|
|
9
|
+
"refresh": "30s",
|
|
10
|
+
"templating": {
|
|
11
|
+
"list": [
|
|
12
|
+
{ "name": "provider", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\"}, provider)", "multi": true, "includeAll": true },
|
|
13
|
+
{ "name": "model", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\", provider=~\"$provider\"}, model)", "multi": true, "includeAll": true },
|
|
14
|
+
{ "name": "team", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\"}, team)", "multi": true, "includeAll": true },
|
|
15
|
+
{ "name": "user_id", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\", team=~\"$team\"}, user_id)", "multi": true, "includeAll": true }
|
|
16
|
+
]
|
|
17
|
+
},
|
|
18
|
+
"panels": [
|
|
19
|
+
{
|
|
20
|
+
"title": "Latency p50",
|
|
21
|
+
"type": "stat",
|
|
22
|
+
"gridPos": { "h": 4, "w": 8, "x": 0, "y": 0 },
|
|
23
|
+
"targets": [{
|
|
24
|
+
"expr": "quantile_over_time(0.5, {job=\"observatory\", provider=~\"$provider\", model=~\"$model\", status=\"success\"} | json | unwrap latency_ms [$__range])",
|
|
25
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
26
|
+
}],
|
|
27
|
+
"fieldConfig": { "defaults": { "unit": "ms" } }
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"title": "Latency p95",
|
|
31
|
+
"type": "stat",
|
|
32
|
+
"gridPos": { "h": 4, "w": 8, "x": 8, "y": 0 },
|
|
33
|
+
"targets": [{
|
|
34
|
+
"expr": "quantile_over_time(0.95, {job=\"observatory\", provider=~\"$provider\", model=~\"$model\", status=\"success\"} | json | unwrap latency_ms [$__range])",
|
|
35
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
36
|
+
}],
|
|
37
|
+
"fieldConfig": { "defaults": { "unit": "ms" } }
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"title": "Latency p99",
|
|
41
|
+
"type": "stat",
|
|
42
|
+
"gridPos": { "h": 4, "w": 8, "x": 16, "y": 0 },
|
|
43
|
+
"targets": [{
|
|
44
|
+
"expr": "quantile_over_time(0.99, {job=\"observatory\", provider=~\"$provider\", model=~\"$model\", status=\"success\"} | json | unwrap latency_ms [$__range])",
|
|
45
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
46
|
+
}],
|
|
47
|
+
"fieldConfig": { "defaults": { "unit": "ms" } }
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"title": "Latency Over Time (p50, p95, p99)",
|
|
51
|
+
"type": "timeseries",
|
|
52
|
+
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 4 },
|
|
53
|
+
"targets": [
|
|
54
|
+
{
|
|
55
|
+
"expr": "quantile_over_time(0.5, {job=\"observatory\", provider=~\"$provider\", model=~\"$model\", status=\"success\"} | json | unwrap latency_ms [$__interval])",
|
|
56
|
+
"legendFormat": "p50",
|
|
57
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
58
|
+
"refId": "A"
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
"expr": "quantile_over_time(0.95, {job=\"observatory\", provider=~\"$provider\", model=~\"$model\", status=\"success\"} | json | unwrap latency_ms [$__interval])",
|
|
62
|
+
"legendFormat": "p95",
|
|
63
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
64
|
+
"refId": "B"
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
"expr": "quantile_over_time(0.99, {job=\"observatory\", provider=~\"$provider\", model=~\"$model\", status=\"success\"} | json | unwrap latency_ms [$__interval])",
|
|
68
|
+
"legendFormat": "p99",
|
|
69
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
70
|
+
"refId": "C"
|
|
71
|
+
}
|
|
72
|
+
],
|
|
73
|
+
"fieldConfig": { "defaults": { "unit": "ms" } }
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
"title": "Time to First Token (TTFT) p95",
|
|
77
|
+
"type": "timeseries",
|
|
78
|
+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
|
|
79
|
+
"targets": [{
|
|
80
|
+
"expr": "quantile_over_time(0.95, {job=\"observatory\", provider=~\"$provider\", model=~\"$model\", status=\"success\"} | json | unwrap ttft_ms [$__interval]) by (model)",
|
|
81
|
+
"legendFormat": "{{model}}",
|
|
82
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
83
|
+
}],
|
|
84
|
+
"fieldConfig": { "defaults": { "unit": "ms" } }
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
"title": "Latency by Model (p95)",
|
|
88
|
+
"type": "timeseries",
|
|
89
|
+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
|
|
90
|
+
"targets": [{
|
|
91
|
+
"expr": "quantile_over_time(0.95, {job=\"observatory\", provider=~\"$provider\", model=~\"$model\", status=\"success\"} | json | unwrap latency_ms [$__interval]) by (model)",
|
|
92
|
+
"legendFormat": "{{model}}",
|
|
93
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
94
|
+
}],
|
|
95
|
+
"fieldConfig": { "defaults": { "unit": "ms" } }
|
|
96
|
+
},
|
|
97
|
+
{
|
|
98
|
+
"title": "Latency Histogram",
|
|
99
|
+
"type": "histogram",
|
|
100
|
+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 },
|
|
101
|
+
"targets": [{
|
|
102
|
+
"expr": "{job=\"observatory\", provider=~\"$provider\", model=~\"$model\", status=\"success\"} | json | unwrap latency_ms",
|
|
103
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
104
|
+
}],
|
|
105
|
+
"fieldConfig": { "defaults": { "unit": "ms" } }
|
|
106
|
+
},
|
|
107
|
+
{
|
|
108
|
+
"title": "Throughput (Tokens/Second)",
|
|
109
|
+
"type": "timeseries",
|
|
110
|
+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 },
|
|
111
|
+
"targets": [{
|
|
112
|
+
"expr": "sum by (model) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", status=\"success\"} | json | unwrap output_tokens [$__interval])) / sum by (model) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", status=\"success\"} | json | unwrap latency_ms [$__interval]) / 1000)",
|
|
113
|
+
"legendFormat": "{{model}}",
|
|
114
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
115
|
+
}],
|
|
116
|
+
"fieldConfig": { "defaults": { "unit": "short", "decimals": 2 } }
|
|
117
|
+
}
|
|
118
|
+
]
|
|
119
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
{
|
|
2
|
+
"__inputs": [
|
|
3
|
+
{ "name": "DS_OBSERVATORY", "label": "Observatory (Loki)", "type": "datasource", "pluginId": "loki" }
|
|
4
|
+
],
|
|
5
|
+
"title": "Observatory — Spend Analysis",
|
|
6
|
+
"uid": "observatory-spend-analysis",
|
|
7
|
+
"tags": ["observatory", "ai", "llm"],
|
|
8
|
+
"timezone": "browser",
|
|
9
|
+
"refresh": "30s",
|
|
10
|
+
"templating": {
|
|
11
|
+
"list": [
|
|
12
|
+
{ "name": "provider", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\"}, provider)", "multi": true, "includeAll": true },
|
|
13
|
+
{ "name": "model", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\", provider=~\"$provider\"}, model)", "multi": true, "includeAll": true },
|
|
14
|
+
{ "name": "team", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\"}, team)", "multi": true, "includeAll": true },
|
|
15
|
+
{ "name": "user_id", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\", team=~\"$team\"}, user_id)", "multi": true, "includeAll": true }
|
|
16
|
+
]
|
|
17
|
+
},
|
|
18
|
+
"panels": [
|
|
19
|
+
{
|
|
20
|
+
"title": "Spend by Model",
|
|
21
|
+
"type": "timeseries",
|
|
22
|
+
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 },
|
|
23
|
+
"targets": [{
|
|
24
|
+
"expr": "sum by (model) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\", user_id=~\"$user_id\"} | json | unwrap cost [$__interval]))",
|
|
25
|
+
"legendFormat": "{{model}}",
|
|
26
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
27
|
+
}],
|
|
28
|
+
"fieldConfig": { "defaults": { "unit": "currencyUSD" } }
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"title": "Spend by Team",
|
|
32
|
+
"type": "timeseries",
|
|
33
|
+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
|
34
|
+
"targets": [{
|
|
35
|
+
"expr": "sum by (team) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} | json | unwrap cost [$__interval]))",
|
|
36
|
+
"legendFormat": "{{team}}",
|
|
37
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
38
|
+
}],
|
|
39
|
+
"fieldConfig": { "defaults": { "unit": "currencyUSD" } }
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
"title": "Spend by User",
|
|
43
|
+
"type": "timeseries",
|
|
44
|
+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
|
45
|
+
"targets": [{
|
|
46
|
+
"expr": "sum by (user_id) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\", user_id=~\"$user_id\"} | json | unwrap cost [$__interval]))",
|
|
47
|
+
"legendFormat": "{{user_id}}",
|
|
48
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
49
|
+
}],
|
|
50
|
+
"fieldConfig": { "defaults": { "unit": "currencyUSD" } }
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
"title": "Budget Burn Rate (Daily Avg)",
|
|
54
|
+
"type": "stat",
|
|
55
|
+
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 16 },
|
|
56
|
+
"targets": [{
|
|
57
|
+
"expr": "sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} | json | unwrap cost [24h])) / 1",
|
|
58
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
59
|
+
}],
|
|
60
|
+
"fieldConfig": { "defaults": { "unit": "currencyUSD" } }
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
"title": "Projected Monthly Spend",
|
|
64
|
+
"type": "stat",
|
|
65
|
+
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 16 },
|
|
66
|
+
"targets": [{
|
|
67
|
+
"expr": "sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} | json | unwrap cost [24h])) * 30",
|
|
68
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
69
|
+
}],
|
|
70
|
+
"fieldConfig": { "defaults": { "unit": "currencyUSD" } }
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
"title": "Top Models by Spend",
|
|
74
|
+
"type": "table",
|
|
75
|
+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 },
|
|
76
|
+
"targets": [{
|
|
77
|
+
"expr": "sum by (model) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap cost [$__range]))",
|
|
78
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
79
|
+
"refId": "A"
|
|
80
|
+
}],
|
|
81
|
+
"transformations": [{
|
|
82
|
+
"id": "sortBy",
|
|
83
|
+
"options": { "fields": {}, "sort": [{ "field": "Value", "desc": true }] }
|
|
84
|
+
}]
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
"title": "Top Teams by Spend",
|
|
88
|
+
"type": "table",
|
|
89
|
+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 },
|
|
90
|
+
"targets": [{
|
|
91
|
+
"expr": "sum by (team) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} | json | unwrap cost [$__range]))",
|
|
92
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
93
|
+
"refId": "A"
|
|
94
|
+
}],
|
|
95
|
+
"transformations": [{
|
|
96
|
+
"id": "sortBy",
|
|
97
|
+
"options": { "fields": {}, "sort": [{ "field": "Value", "desc": true }] }
|
|
98
|
+
}]
|
|
99
|
+
}
|
|
100
|
+
]
|
|
101
|
+
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
{
|
|
2
|
+
"__inputs": [
|
|
3
|
+
{ "name": "DS_OBSERVATORY", "label": "Observatory (Loki)", "type": "datasource", "pluginId": "loki" }
|
|
4
|
+
],
|
|
5
|
+
"title": "Observatory — Token Volume",
|
|
6
|
+
"uid": "observatory-token-volume",
|
|
7
|
+
"tags": ["observatory", "ai", "llm"],
|
|
8
|
+
"timezone": "browser",
|
|
9
|
+
"refresh": "30s",
|
|
10
|
+
"templating": {
|
|
11
|
+
"list": [
|
|
12
|
+
{ "name": "provider", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\"}, provider)", "multi": true, "includeAll": true },
|
|
13
|
+
{ "name": "model", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\", provider=~\"$provider\"}, model)", "multi": true, "includeAll": true },
|
|
14
|
+
{ "name": "team", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\"}, team)", "multi": true, "includeAll": true },
|
|
15
|
+
{ "name": "user_id", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\", team=~\"$team\"}, user_id)", "multi": true, "includeAll": true }
|
|
16
|
+
]
|
|
17
|
+
},
|
|
18
|
+
"panels": [
|
|
19
|
+
{
|
|
20
|
+
"title": "Input vs Output Tokens Over Time",
|
|
21
|
+
"type": "timeseries",
|
|
22
|
+
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 },
|
|
23
|
+
"targets": [
|
|
24
|
+
{
|
|
25
|
+
"expr": "sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\", user_id=~\"$user_id\"} | json | unwrap input_tokens [$__interval]))",
|
|
26
|
+
"legendFormat": "Input Tokens",
|
|
27
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
28
|
+
"refId": "A"
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"expr": "sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\", user_id=~\"$user_id\"} | json | unwrap output_tokens [$__interval]))",
|
|
32
|
+
"legendFormat": "Output Tokens",
|
|
33
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
34
|
+
"refId": "B"
|
|
35
|
+
}
|
|
36
|
+
],
|
|
37
|
+
"fieldConfig": { "defaults": { "unit": "short" } }
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"title": "Token Ratio by Model (Output/Input)",
|
|
41
|
+
"type": "timeseries",
|
|
42
|
+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
|
43
|
+
"targets": [{
|
|
44
|
+
"expr": "sum by (model) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap output_tokens [$__interval])) / sum by (model) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap input_tokens [$__interval]))",
|
|
45
|
+
"legendFormat": "{{model}}",
|
|
46
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
47
|
+
}],
|
|
48
|
+
"fieldConfig": { "defaults": { "unit": "short", "decimals": 2 } }
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
"title": "Cache Hit Rate",
|
|
52
|
+
"type": "timeseries",
|
|
53
|
+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
|
54
|
+
"targets": [{
|
|
55
|
+
"expr": "sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap cache_read_tokens [$__interval])) / sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap input_tokens [$__interval]))",
|
|
56
|
+
"legendFormat": "Cache Hit Rate",
|
|
57
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
58
|
+
}],
|
|
59
|
+
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1 } }
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
"title": "Total Token Volume",
|
|
63
|
+
"type": "stat",
|
|
64
|
+
"gridPos": { "h": 4, "w": 8, "x": 0, "y": 16 },
|
|
65
|
+
"targets": [{
|
|
66
|
+
"expr": "sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap total_tokens [$__range]))",
|
|
67
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
68
|
+
}],
|
|
69
|
+
"fieldConfig": { "defaults": { "unit": "short" } }
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
"title": "Cache Read Tokens",
|
|
73
|
+
"type": "stat",
|
|
74
|
+
"gridPos": { "h": 4, "w": 8, "x": 8, "y": 16 },
|
|
75
|
+
"targets": [{
|
|
76
|
+
"expr": "sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap cache_read_tokens [$__range]))",
|
|
77
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
78
|
+
}],
|
|
79
|
+
"fieldConfig": { "defaults": { "unit": "short" } }
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
"title": "Cache Creation Tokens",
|
|
83
|
+
"type": "stat",
|
|
84
|
+
"gridPos": { "h": 4, "w": 8, "x": 16, "y": 16 },
|
|
85
|
+
"targets": [{
|
|
86
|
+
"expr": "sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap cache_creation_tokens [$__range]))",
|
|
87
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
88
|
+
}],
|
|
89
|
+
"fieldConfig": { "defaults": { "unit": "short" } }
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
"title": "Token Distribution by Model",
|
|
93
|
+
"type": "piechart",
|
|
94
|
+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 },
|
|
95
|
+
"targets": [{
|
|
96
|
+
"expr": "sum by (model) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap total_tokens [$__range]))",
|
|
97
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
98
|
+
}]
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
"title": "Tokens by Team",
|
|
102
|
+
"type": "table",
|
|
103
|
+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 },
|
|
104
|
+
"targets": [{
|
|
105
|
+
"expr": "sum by (team) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} | json | unwrap total_tokens [$__range]))",
|
|
106
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
107
|
+
}],
|
|
108
|
+
"transformations": [{
|
|
109
|
+
"id": "sortBy",
|
|
110
|
+
"options": { "fields": {}, "sort": [{ "field": "Value", "desc": true }] }
|
|
111
|
+
}]
|
|
112
|
+
}
|
|
113
|
+
]
|
|
114
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { StorageBackend, UsageEvent, UsageQuery } from '@conquext/observa-core';
|
|
2
|
+
|
|
3
|
+
interface LokiConfig {
|
|
4
|
+
url: string;
|
|
5
|
+
labels?: Record<string, string>;
|
|
6
|
+
auth?: {
|
|
7
|
+
username: string;
|
|
8
|
+
password: string;
|
|
9
|
+
} | {
|
|
10
|
+
token: string;
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
declare class LokiBackend implements StorageBackend {
|
|
14
|
+
private readonly pushUrl;
|
|
15
|
+
private readonly staticLabels;
|
|
16
|
+
private readonly authHeader?;
|
|
17
|
+
constructor(config: LokiConfig);
|
|
18
|
+
write(events: UsageEvent[]): Promise<void>;
|
|
19
|
+
query(_query: UsageQuery): Promise<UsageEvent[]>;
|
|
20
|
+
close(): Promise<void>;
|
|
21
|
+
private eventToStream;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export { LokiBackend, type LokiConfig };
|