@conquext/loki 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dashboards/budgets.json +169 -0
- package/dashboards/cost-attribution.json +120 -0
- package/dashboards/errors.json +104 -0
- package/dashboards/forecasting.json +208 -0
- package/dashboards/model-comparison.json +152 -0
- package/dashboards/overview.json +90 -0
- package/dashboards/performance.json +119 -0
- package/dashboards/spend-analysis.json +101 -0
- package/dashboards/token-volume.json +114 -0
- package/dist/index.d.ts +24 -0
- package/dist/index.js +81 -0
- package/dist/index.js.map +1 -0
- package/package.json +28 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Rasheed Alabi
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
{
|
|
2
|
+
"__inputs": [
|
|
3
|
+
{ "name": "DS_OBSERVATORY", "label": "Observatory (Loki)", "type": "datasource", "pluginId": "loki" }
|
|
4
|
+
],
|
|
5
|
+
"title": "Observatory — Budgets",
|
|
6
|
+
"uid": "observatory-budgets",
|
|
7
|
+
"tags": ["observatory", "ai", "llm"],
|
|
8
|
+
"timezone": "browser",
|
|
9
|
+
"refresh": "30s",
|
|
10
|
+
"templating": {
|
|
11
|
+
"list": [
|
|
12
|
+
{ "name": "provider", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\"}, provider)", "multi": true, "includeAll": true },
|
|
13
|
+
{ "name": "model", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\", provider=~\"$provider\"}, model)", "multi": true, "includeAll": true },
|
|
14
|
+
{ "name": "team", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\"}, team)", "multi": true, "includeAll": true },
|
|
15
|
+
{ "name": "user_id", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\", team=~\"$team\"}, user_id)", "multi": true, "includeAll": true }
|
|
16
|
+
]
|
|
17
|
+
},
|
|
18
|
+
"panels": [
|
|
19
|
+
{
|
|
20
|
+
"title": "Budget Utilization (Current Period)",
|
|
21
|
+
"type": "gauge",
|
|
22
|
+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
|
|
23
|
+
"targets": [{
|
|
24
|
+
"expr": "sum by (user_id) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\", user_id=~\"$user_id\"} | json | unwrap cost [30d])) / sum by (user_id) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\", user_id=~\"$user_id\"} | json | unwrap budget_limit [30d]))",
|
|
25
|
+
"legendFormat": "{{user_id}}",
|
|
26
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
27
|
+
}],
|
|
28
|
+
"fieldConfig": {
|
|
29
|
+
"defaults": {
|
|
30
|
+
"unit": "percentunit",
|
|
31
|
+
"min": 0,
|
|
32
|
+
"max": 1,
|
|
33
|
+
"thresholds": {
|
|
34
|
+
"mode": "absolute",
|
|
35
|
+
"steps": [
|
|
36
|
+
{ "value": 0, "color": "green" },
|
|
37
|
+
{ "value": 0.7, "color": "yellow" },
|
|
38
|
+
{ "value": 0.9, "color": "red" }
|
|
39
|
+
]
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"title": "Team Budget Utilization",
|
|
46
|
+
"type": "gauge",
|
|
47
|
+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
|
|
48
|
+
"targets": [{
|
|
49
|
+
"expr": "sum by (team) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} | json | unwrap cost [30d])) / sum by (team) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} | json | unwrap budget_limit [30d]))",
|
|
50
|
+
"legendFormat": "{{team}}",
|
|
51
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
52
|
+
}],
|
|
53
|
+
"fieldConfig": {
|
|
54
|
+
"defaults": {
|
|
55
|
+
"unit": "percentunit",
|
|
56
|
+
"min": 0,
|
|
57
|
+
"max": 1,
|
|
58
|
+
"thresholds": {
|
|
59
|
+
"mode": "absolute",
|
|
60
|
+
"steps": [
|
|
61
|
+
{ "value": 0, "color": "green" },
|
|
62
|
+
{ "value": 0.7, "color": "yellow" },
|
|
63
|
+
{ "value": 0.9, "color": "red" }
|
|
64
|
+
]
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"title": "Budget Spend Over Time",
|
|
71
|
+
"type": "timeseries",
|
|
72
|
+
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 8 },
|
|
73
|
+
"targets": [
|
|
74
|
+
{
|
|
75
|
+
"expr": "sum by (user_id) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\", user_id=~\"$user_id\"} | json | unwrap cost [$__interval]))",
|
|
76
|
+
"legendFormat": "{{user_id}} (spend)",
|
|
77
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
78
|
+
"refId": "A"
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
"expr": "sum by (user_id) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\", user_id=~\"$user_id\"} | json | unwrap budget_limit [$__interval]))",
|
|
82
|
+
"legendFormat": "{{user_id}} (limit)",
|
|
83
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
84
|
+
"refId": "B"
|
|
85
|
+
}
|
|
86
|
+
],
|
|
87
|
+
"fieldConfig": { "defaults": { "unit": "currencyUSD" } }
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
"title": "Downgrade Events Over Time",
|
|
91
|
+
"type": "timeseries",
|
|
92
|
+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
|
|
93
|
+
"targets": [{
|
|
94
|
+
"expr": "sum by (user_id) (count_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\", user_id=~\"$user_id\"} | json | downgraded=\"true\" [$__interval]))",
|
|
95
|
+
"legendFormat": "{{user_id}}",
|
|
96
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
97
|
+
}],
|
|
98
|
+
"fieldConfig": { "defaults": { "unit": "short" } }
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
"title": "Downgrade Rate by Model",
|
|
102
|
+
"type": "timeseries",
|
|
103
|
+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
|
|
104
|
+
"targets": [{
|
|
105
|
+
"expr": "sum by (model) (count_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | downgraded=\"true\" [$__interval])) / sum by (model) (count_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} [$__interval]))",
|
|
106
|
+
"legendFormat": "{{model}}",
|
|
107
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
108
|
+
}],
|
|
109
|
+
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0 } }
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
"title": "Users Approaching Limits (>80%)",
|
|
113
|
+
"type": "table",
|
|
114
|
+
"gridPos": { "h": 10, "w": 12, "x": 0, "y": 24 },
|
|
115
|
+
"targets": [{
|
|
116
|
+
"expr": "sum by (user_id, team) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} | json | unwrap cost [30d])) / sum by (user_id, team) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} | json | unwrap budget_limit [30d])) > 0.8",
|
|
117
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
118
|
+
"refId": "A"
|
|
119
|
+
}],
|
|
120
|
+
"transformations": [{
|
|
121
|
+
"id": "sortBy",
|
|
122
|
+
"options": { "fields": {}, "sort": [{ "field": "Value", "desc": true }] }
|
|
123
|
+
}],
|
|
124
|
+
"fieldConfig": {
|
|
125
|
+
"overrides": [{
|
|
126
|
+
"matcher": { "id": "byName", "options": "Value" },
|
|
127
|
+
"properties": [
|
|
128
|
+
{ "id": "unit", "value": "percentunit" },
|
|
129
|
+
{ "id": "custom.displayMode", "value": "color-background" },
|
|
130
|
+
{ "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "value": 0.8, "color": "yellow" }, { "value": 0.95, "color": "red" }] } }
|
|
131
|
+
]
|
|
132
|
+
}]
|
|
133
|
+
}
|
|
134
|
+
},
|
|
135
|
+
{
|
|
136
|
+
"title": "Budget Limit Distribution",
|
|
137
|
+
"type": "piechart",
|
|
138
|
+
"gridPos": { "h": 10, "w": 12, "x": 12, "y": 24 },
|
|
139
|
+
"targets": [{
|
|
140
|
+
"expr": "sum by (user_id) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\", user_id=~\"$user_id\"} | json | unwrap budget_limit [30d]))",
|
|
141
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
142
|
+
}]
|
|
143
|
+
},
|
|
144
|
+
{
|
|
145
|
+
"title": "Days Until Budget Exhaustion",
|
|
146
|
+
"type": "table",
|
|
147
|
+
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 34 },
|
|
148
|
+
"targets": [{
|
|
149
|
+
"expr": "(sum by (user_id) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\", user_id=~\"$user_id\"} | json | unwrap budget_limit [30d])) - sum by (user_id) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\", user_id=~\"$user_id\"} | json | unwrap cost [30d]))) / (sum by (user_id) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\", user_id=~\"$user_id\"} | json | unwrap cost [24h])) / 1)",
|
|
150
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
151
|
+
"refId": "A"
|
|
152
|
+
}],
|
|
153
|
+
"transformations": [{
|
|
154
|
+
"id": "sortBy",
|
|
155
|
+
"options": { "fields": {}, "sort": [{ "field": "Value", "desc": false }] }
|
|
156
|
+
}],
|
|
157
|
+
"fieldConfig": {
|
|
158
|
+
"overrides": [{
|
|
159
|
+
"matcher": { "id": "byName", "options": "Value" },
|
|
160
|
+
"properties": [
|
|
161
|
+
{ "id": "unit", "value": "d" },
|
|
162
|
+
{ "id": "custom.displayMode", "value": "color-background" },
|
|
163
|
+
{ "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "value": 0, "color": "red" }, { "value": 7, "color": "yellow" }, { "value": 30, "color": "green" }] } }
|
|
164
|
+
]
|
|
165
|
+
}]
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
]
|
|
169
|
+
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
{
|
|
2
|
+
"__inputs": [
|
|
3
|
+
{ "name": "DS_OBSERVATORY", "label": "Observatory (Loki)", "type": "datasource", "pluginId": "loki" }
|
|
4
|
+
],
|
|
5
|
+
"title": "Observatory — Cost Attribution",
|
|
6
|
+
"uid": "observatory-cost-attribution",
|
|
7
|
+
"tags": ["observatory", "ai", "llm"],
|
|
8
|
+
"timezone": "browser",
|
|
9
|
+
"refresh": "30s",
|
|
10
|
+
"templating": {
|
|
11
|
+
"list": [
|
|
12
|
+
{ "name": "provider", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\"}, provider)", "multi": true, "includeAll": true },
|
|
13
|
+
{ "name": "model", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\", provider=~\"$provider\"}, model)", "multi": true, "includeAll": true },
|
|
14
|
+
{ "name": "team", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\"}, team)", "multi": true, "includeAll": true },
|
|
15
|
+
{ "name": "user_id", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\", team=~\"$team\"}, user_id)", "multi": true, "includeAll": true }
|
|
16
|
+
]
|
|
17
|
+
},
|
|
18
|
+
"panels": [
|
|
19
|
+
{
|
|
20
|
+
"title": "Top 10 Users by Spend",
|
|
21
|
+
"type": "table",
|
|
22
|
+
"gridPos": { "h": 10, "w": 12, "x": 0, "y": 0 },
|
|
23
|
+
"targets": [{
|
|
24
|
+
"expr": "topk(10, sum by (user_id) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} | json | unwrap cost [$__range])))",
|
|
25
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
26
|
+
"refId": "A"
|
|
27
|
+
}],
|
|
28
|
+
"transformations": [{
|
|
29
|
+
"id": "sortBy",
|
|
30
|
+
"options": { "fields": {}, "sort": [{ "field": "Value", "desc": true }] }
|
|
31
|
+
}],
|
|
32
|
+
"fieldConfig": {
|
|
33
|
+
"overrides": [{
|
|
34
|
+
"matcher": { "id": "byName", "options": "Value" },
|
|
35
|
+
"properties": [{ "id": "unit", "value": "currencyUSD" }]
|
|
36
|
+
}]
|
|
37
|
+
}
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"title": "Top 10 Tasks by Spend",
|
|
41
|
+
"type": "table",
|
|
42
|
+
"gridPos": { "h": 10, "w": 12, "x": 12, "y": 0 },
|
|
43
|
+
"targets": [{
|
|
44
|
+
"expr": "topk(10, sum by (task) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap cost [$__range])))",
|
|
45
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
46
|
+
"refId": "A"
|
|
47
|
+
}],
|
|
48
|
+
"transformations": [{
|
|
49
|
+
"id": "sortBy",
|
|
50
|
+
"options": { "fields": {}, "sort": [{ "field": "Value", "desc": true }] }
|
|
51
|
+
}],
|
|
52
|
+
"fieldConfig": {
|
|
53
|
+
"overrides": [{
|
|
54
|
+
"matcher": { "id": "byName", "options": "Value" },
|
|
55
|
+
"properties": [{ "id": "unit", "value": "currencyUSD" }]
|
|
56
|
+
}]
|
|
57
|
+
}
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
"title": "Cost per Conversation",
|
|
61
|
+
"type": "timeseries",
|
|
62
|
+
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 10 },
|
|
63
|
+
"targets": [{
|
|
64
|
+
"expr": "sum by (conversation_id) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | conversation_id != \"\" | unwrap cost [$__interval])) / count by (conversation_id) (count_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | conversation_id != \"\" [$__interval]))",
|
|
65
|
+
"legendFormat": "{{conversation_id}}",
|
|
66
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
67
|
+
}],
|
|
68
|
+
"fieldConfig": { "defaults": { "unit": "currencyUSD" } }
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
"title": "Spend by User (Heatmap)",
|
|
72
|
+
"type": "heatmap",
|
|
73
|
+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 },
|
|
74
|
+
"targets": [{
|
|
75
|
+
"expr": "sum by (user_id) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} | json | unwrap cost [$__interval]))",
|
|
76
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
77
|
+
}]
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
"title": "Cost Distribution by Team",
|
|
81
|
+
"type": "piechart",
|
|
82
|
+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 },
|
|
83
|
+
"targets": [{
|
|
84
|
+
"expr": "sum by (team) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} | json | unwrap cost [$__range]))",
|
|
85
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
86
|
+
}]
|
|
87
|
+
},
|
|
88
|
+
{
|
|
89
|
+
"title": "Average Cost per Call by Model",
|
|
90
|
+
"type": "table",
|
|
91
|
+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 },
|
|
92
|
+
"targets": [{
|
|
93
|
+
"expr": "sum by (model) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap cost [$__range])) / sum by (model) (count_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} [$__range]))",
|
|
94
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
95
|
+
"refId": "A"
|
|
96
|
+
}],
|
|
97
|
+
"transformations": [{
|
|
98
|
+
"id": "sortBy",
|
|
99
|
+
"options": { "fields": {}, "sort": [{ "field": "Value", "desc": true }] }
|
|
100
|
+
}],
|
|
101
|
+
"fieldConfig": {
|
|
102
|
+
"overrides": [{
|
|
103
|
+
"matcher": { "id": "byName", "options": "Value" },
|
|
104
|
+
"properties": [{ "id": "unit", "value": "currencyUSD" }]
|
|
105
|
+
}]
|
|
106
|
+
}
|
|
107
|
+
},
|
|
108
|
+
{
|
|
109
|
+
"title": "User Spend Over Time",
|
|
110
|
+
"type": "timeseries",
|
|
111
|
+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 },
|
|
112
|
+
"targets": [{
|
|
113
|
+
"expr": "sum by (user_id) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\", user_id=~\"$user_id\"} | json | unwrap cost [$__interval]))",
|
|
114
|
+
"legendFormat": "{{user_id}}",
|
|
115
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
116
|
+
}],
|
|
117
|
+
"fieldConfig": { "defaults": { "unit": "currencyUSD" } }
|
|
118
|
+
}
|
|
119
|
+
]
|
|
120
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
{
|
|
2
|
+
"__inputs": [
|
|
3
|
+
{ "name": "DS_OBSERVATORY", "label": "Observatory (Loki)", "type": "datasource", "pluginId": "loki" }
|
|
4
|
+
],
|
|
5
|
+
"title": "Observatory — Errors",
|
|
6
|
+
"uid": "observatory-errors",
|
|
7
|
+
"tags": ["observatory", "ai", "llm"],
|
|
8
|
+
"timezone": "browser",
|
|
9
|
+
"refresh": "30s",
|
|
10
|
+
"templating": {
|
|
11
|
+
"list": [
|
|
12
|
+
{ "name": "provider", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\"}, provider)", "multi": true, "includeAll": true },
|
|
13
|
+
{ "name": "model", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\", provider=~\"$provider\"}, model)", "multi": true, "includeAll": true },
|
|
14
|
+
{ "name": "team", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\"}, team)", "multi": true, "includeAll": true },
|
|
15
|
+
{ "name": "user_id", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\", team=~\"$team\"}, user_id)", "multi": true, "includeAll": true }
|
|
16
|
+
]
|
|
17
|
+
},
|
|
18
|
+
"panels": [
|
|
19
|
+
{
|
|
20
|
+
"title": "Error Rate",
|
|
21
|
+
"type": "stat",
|
|
22
|
+
"gridPos": { "h": 4, "w": 8, "x": 0, "y": 0 },
|
|
23
|
+
"targets": [{
|
|
24
|
+
"expr": "sum(count_over_time({job=\"observatory\", status=\"error\", provider=~\"$provider\", model=~\"$model\"} [$__range])) / sum(count_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} [$__range]))",
|
|
25
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
26
|
+
}],
|
|
27
|
+
"fieldConfig": { "defaults": { "unit": "percentunit", "thresholds": { "steps": [{ "value": 0, "color": "green" }, { "value": 0.05, "color": "yellow" }, { "value": 0.1, "color": "red" }] } } }
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"title": "Total Errors",
|
|
31
|
+
"type": "stat",
|
|
32
|
+
"gridPos": { "h": 4, "w": 8, "x": 8, "y": 0 },
|
|
33
|
+
"targets": [{
|
|
34
|
+
"expr": "sum(count_over_time({job=\"observatory\", status=\"error\", provider=~\"$provider\", model=~\"$model\"} [$__range]))",
|
|
35
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
36
|
+
}],
|
|
37
|
+
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "steps": [{ "value": 0, "color": "green" }, { "value": 10, "color": "yellow" }, { "value": 50, "color": "red" }] } } }
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"title": "Errors in Last Hour",
|
|
41
|
+
"type": "stat",
|
|
42
|
+
"gridPos": { "h": 4, "w": 8, "x": 16, "y": 0 },
|
|
43
|
+
"targets": [{
|
|
44
|
+
"expr": "sum(count_over_time({job=\"observatory\", status=\"error\", provider=~\"$provider\", model=~\"$model\"} [1h]))",
|
|
45
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
46
|
+
}],
|
|
47
|
+
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "steps": [{ "value": 0, "color": "green" }, { "value": 5, "color": "yellow" }, { "value": 20, "color": "red" }] } } }
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"title": "Error Rate by Provider",
|
|
51
|
+
"type": "timeseries",
|
|
52
|
+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
|
|
53
|
+
"targets": [{
|
|
54
|
+
"expr": "sum by (provider) (count_over_time({job=\"observatory\", status=\"error\", provider=~\"$provider\", model=~\"$model\"} [$__interval])) / sum by (provider) (count_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} [$__interval]))",
|
|
55
|
+
"legendFormat": "{{provider}}",
|
|
56
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
57
|
+
}],
|
|
58
|
+
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0 } }
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
"title": "Error Rate by Model",
|
|
62
|
+
"type": "timeseries",
|
|
63
|
+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
|
|
64
|
+
"targets": [{
|
|
65
|
+
"expr": "sum by (model) (count_over_time({job=\"observatory\", status=\"error\", provider=~\"$provider\", model=~\"$model\"} [$__interval])) / sum by (model) (count_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} [$__interval]))",
|
|
66
|
+
"legendFormat": "{{model}}",
|
|
67
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
68
|
+
}],
|
|
69
|
+
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0 } }
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
"title": "Error Messages",
|
|
73
|
+
"type": "logs",
|
|
74
|
+
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 12 },
|
|
75
|
+
"targets": [{
|
|
76
|
+
"expr": "{job=\"observatory\", status=\"error\", provider=~\"$provider\", model=~\"$model\"} | json | line_format \"{{.timestamp}} | {{.provider}}/{{.model}} | {{.error}}\"",
|
|
77
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
78
|
+
}]
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
"title": "Error Count by Error Message",
|
|
82
|
+
"type": "table",
|
|
83
|
+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 },
|
|
84
|
+
"targets": [{
|
|
85
|
+
"expr": "sum by (error) (count_over_time({job=\"observatory\", status=\"error\", provider=~\"$provider\", model=~\"$model\"} | json [$__range]))",
|
|
86
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
87
|
+
"refId": "A"
|
|
88
|
+
}],
|
|
89
|
+
"transformations": [{
|
|
90
|
+
"id": "sortBy",
|
|
91
|
+
"options": { "fields": {}, "sort": [{ "field": "Value", "desc": true }] }
|
|
92
|
+
}]
|
|
93
|
+
},
|
|
94
|
+
{
|
|
95
|
+
"title": "Errors by Team",
|
|
96
|
+
"type": "piechart",
|
|
97
|
+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 },
|
|
98
|
+
"targets": [{
|
|
99
|
+
"expr": "sum by (team) (count_over_time({job=\"observatory\", status=\"error\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} [$__range]))",
|
|
100
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
101
|
+
}]
|
|
102
|
+
}
|
|
103
|
+
]
|
|
104
|
+
}
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
{
|
|
2
|
+
"__inputs": [
|
|
3
|
+
{ "name": "DS_OBSERVATORY", "label": "Observatory (Loki)", "type": "datasource", "pluginId": "loki" }
|
|
4
|
+
],
|
|
5
|
+
"title": "Observatory — Forecasting",
|
|
6
|
+
"uid": "observatory-forecasting",
|
|
7
|
+
"tags": ["observatory", "ai", "llm"],
|
|
8
|
+
"timezone": "browser",
|
|
9
|
+
"refresh": "30s",
|
|
10
|
+
"templating": {
|
|
11
|
+
"list": [
|
|
12
|
+
{ "name": "provider", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\"}, provider)", "multi": true, "includeAll": true },
|
|
13
|
+
{ "name": "model", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\", provider=~\"$provider\"}, model)", "multi": true, "includeAll": true },
|
|
14
|
+
{ "name": "team", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\"}, team)", "multi": true, "includeAll": true },
|
|
15
|
+
{ "name": "user_id", "type": "query", "datasource": "${DS_OBSERVATORY}", "query": "label_values({job=\"observatory\", team=~\"$team\"}, user_id)", "multi": true, "includeAll": true }
|
|
16
|
+
]
|
|
17
|
+
},
|
|
18
|
+
"panels": [
|
|
19
|
+
{
|
|
20
|
+
"title": "Projected Monthly Spend",
|
|
21
|
+
"type": "stat",
|
|
22
|
+
"gridPos": { "h": 6, "w": 8, "x": 0, "y": 0 },
|
|
23
|
+
"targets": [{
|
|
24
|
+
"expr": "sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} | json | unwrap cost [7d])) / 7 * 30",
|
|
25
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
26
|
+
}],
|
|
27
|
+
"fieldConfig": {
|
|
28
|
+
"defaults": {
|
|
29
|
+
"unit": "currencyUSD",
|
|
30
|
+
"color": { "mode": "thresholds" },
|
|
31
|
+
"thresholds": {
|
|
32
|
+
"mode": "absolute",
|
|
33
|
+
"steps": [
|
|
34
|
+
{ "value": 0, "color": "green" },
|
|
35
|
+
{ "value": 1000, "color": "yellow" },
|
|
36
|
+
{ "value": 5000, "color": "red" }
|
|
37
|
+
]
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
"title": "Daily Average (Last 7d)",
|
|
44
|
+
"type": "stat",
|
|
45
|
+
"gridPos": { "h": 6, "w": 8, "x": 8, "y": 0 },
|
|
46
|
+
"targets": [{
|
|
47
|
+
"expr": "sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} | json | unwrap cost [7d])) / 7",
|
|
48
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
49
|
+
}],
|
|
50
|
+
"fieldConfig": { "defaults": { "unit": "currencyUSD" } }
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
"title": "Growth Rate (7d vs 30d)",
|
|
54
|
+
"type": "stat",
|
|
55
|
+
"gridPos": { "h": 6, "w": 8, "x": 16, "y": 0 },
|
|
56
|
+
"targets": [{
|
|
57
|
+
"expr": "(sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} | json | unwrap cost [7d])) / 7 - sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} | json | unwrap cost [30d])) / 30) / (sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} | json | unwrap cost [30d])) / 30)",
|
|
58
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
59
|
+
}],
|
|
60
|
+
"fieldConfig": {
|
|
61
|
+
"defaults": {
|
|
62
|
+
"unit": "percentunit",
|
|
63
|
+
"color": { "mode": "thresholds" },
|
|
64
|
+
"thresholds": {
|
|
65
|
+
"mode": "absolute",
|
|
66
|
+
"steps": [
|
|
67
|
+
{ "value": -0.2, "color": "green" },
|
|
68
|
+
{ "value": 0, "color": "yellow" },
|
|
69
|
+
{ "value": 0.2, "color": "red" }
|
|
70
|
+
]
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
"title": "Daily Spend Trend with 7-Day Moving Average",
|
|
77
|
+
"type": "timeseries",
|
|
78
|
+
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 6 },
|
|
79
|
+
"targets": [
|
|
80
|
+
{
|
|
81
|
+
"expr": "sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} | json | unwrap cost [24h]))",
|
|
82
|
+
"legendFormat": "Daily Spend",
|
|
83
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
84
|
+
"refId": "A"
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
"expr": "avg_over_time(sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} | json | unwrap cost [24h])) [7d])",
|
|
88
|
+
"legendFormat": "7-Day Moving Average",
|
|
89
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
90
|
+
"refId": "B"
|
|
91
|
+
},
|
|
92
|
+
{
|
|
93
|
+
"expr": "avg_over_time(sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} | json | unwrap cost [24h])) [30d])",
|
|
94
|
+
"legendFormat": "30-Day Moving Average",
|
|
95
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
96
|
+
"refId": "C"
|
|
97
|
+
}
|
|
98
|
+
],
|
|
99
|
+
"fieldConfig": { "defaults": { "unit": "currencyUSD" } }
|
|
100
|
+
},
|
|
101
|
+
{
|
|
102
|
+
"title": "Spend Forecast by Provider (Next 30d)",
|
|
103
|
+
"type": "timeseries",
|
|
104
|
+
"gridPos": { "h": 10, "w": 12, "x": 0, "y": 16 },
|
|
105
|
+
"targets": [{
|
|
106
|
+
"expr": "sum by (provider) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap cost [7d])) / 7 * 30",
|
|
107
|
+
"legendFormat": "{{provider}} (projected)",
|
|
108
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
109
|
+
}],
|
|
110
|
+
"fieldConfig": { "defaults": { "unit": "currencyUSD" } }
|
|
111
|
+
},
|
|
112
|
+
{
|
|
113
|
+
"title": "Spend Forecast by Team (Next 30d)",
|
|
114
|
+
"type": "timeseries",
|
|
115
|
+
"gridPos": { "h": 10, "w": 12, "x": 12, "y": 16 },
|
|
116
|
+
"targets": [{
|
|
117
|
+
"expr": "sum by (team) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\", team=~\"$team\"} | json | unwrap cost [7d])) / 7 * 30",
|
|
118
|
+
"legendFormat": "{{team}} (projected)",
|
|
119
|
+
"datasource": "${DS_OBSERVATORY}"
|
|
120
|
+
}],
|
|
121
|
+
"fieldConfig": { "defaults": { "unit": "currencyUSD" } }
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
"title": "Token Volume Trend",
|
|
125
|
+
"type": "timeseries",
|
|
126
|
+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 },
|
|
127
|
+
"targets": [
|
|
128
|
+
{
|
|
129
|
+
"expr": "sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap total_tokens [24h]))",
|
|
130
|
+
"legendFormat": "Daily Tokens",
|
|
131
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
132
|
+
"refId": "A"
|
|
133
|
+
},
|
|
134
|
+
{
|
|
135
|
+
"expr": "avg_over_time(sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap total_tokens [24h])) [7d])",
|
|
136
|
+
"legendFormat": "7-Day Average",
|
|
137
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
138
|
+
"refId": "B"
|
|
139
|
+
}
|
|
140
|
+
],
|
|
141
|
+
"fieldConfig": { "defaults": { "unit": "short" } }
|
|
142
|
+
},
|
|
143
|
+
{
|
|
144
|
+
"title": "Call Volume Trend",
|
|
145
|
+
"type": "timeseries",
|
|
146
|
+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 },
|
|
147
|
+
"targets": [
|
|
148
|
+
{
|
|
149
|
+
"expr": "sum(count_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} [24h]))",
|
|
150
|
+
"legendFormat": "Daily Calls",
|
|
151
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
152
|
+
"refId": "A"
|
|
153
|
+
},
|
|
154
|
+
{
|
|
155
|
+
"expr": "avg_over_time(sum(count_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} [24h])) [7d])",
|
|
156
|
+
"legendFormat": "7-Day Average",
|
|
157
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
158
|
+
"refId": "B"
|
|
159
|
+
}
|
|
160
|
+
],
|
|
161
|
+
"fieldConfig": { "defaults": { "unit": "short" } }
|
|
162
|
+
},
|
|
163
|
+
{
|
|
164
|
+
"title": "Projected Monthly Cost by Model",
|
|
165
|
+
"type": "table",
|
|
166
|
+
"gridPos": { "h": 10, "w": 12, "x": 0, "y": 34 },
|
|
167
|
+
"targets": [{
|
|
168
|
+
"expr": "sum by (model) (sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap cost [7d])) / 7 * 30",
|
|
169
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
170
|
+
"refId": "A"
|
|
171
|
+
}],
|
|
172
|
+
"transformations": [{
|
|
173
|
+
"id": "sortBy",
|
|
174
|
+
"options": { "fields": {}, "sort": [{ "field": "Value", "desc": true }] }
|
|
175
|
+
}],
|
|
176
|
+
"fieldConfig": {
|
|
177
|
+
"overrides": [{
|
|
178
|
+
"matcher": { "id": "byName", "options": "Value" },
|
|
179
|
+
"properties": [
|
|
180
|
+
{ "id": "unit", "value": "currencyUSD" },
|
|
181
|
+
{ "id": "custom.displayMode", "value": "color-background" },
|
|
182
|
+
{ "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "value": 0, "color": "green" }, { "value": 500, "color": "yellow" }, { "value": 2000, "color": "red" }] } }
|
|
183
|
+
]
|
|
184
|
+
}]
|
|
185
|
+
}
|
|
186
|
+
},
|
|
187
|
+
{
|
|
188
|
+
"title": "Cost Trend Comparison (Current vs Previous Period)",
|
|
189
|
+
"type": "bargauge",
|
|
190
|
+
"gridPos": { "h": 10, "w": 12, "x": 12, "y": 34 },
|
|
191
|
+
"targets": [
|
|
192
|
+
{
|
|
193
|
+
"expr": "sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap cost [7d]))",
|
|
194
|
+
"legendFormat": "Last 7 Days",
|
|
195
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
196
|
+
"refId": "A"
|
|
197
|
+
},
|
|
198
|
+
{
|
|
199
|
+
"expr": "sum(sum_over_time({job=\"observatory\", provider=~\"$provider\", model=~\"$model\"} | json | unwrap cost [7d] offset 7d))",
|
|
200
|
+
"legendFormat": "Previous 7 Days",
|
|
201
|
+
"datasource": "${DS_OBSERVATORY}",
|
|
202
|
+
"refId": "B"
|
|
203
|
+
}
|
|
204
|
+
],
|
|
205
|
+
"fieldConfig": { "defaults": { "unit": "currencyUSD" } }
|
|
206
|
+
}
|
|
207
|
+
]
|
|
208
|
+
}
|