inferock-bench 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +5 -0
- package/dist/adapters/anthropic.js +481 -0
- package/dist/adapters/anthropic.js.map +1 -0
- package/dist/adapters/canonical-v2.js +287 -0
- package/dist/adapters/canonical-v2.js.map +1 -0
- package/dist/adapters/openai.js +424 -0
- package/dist/adapters/openai.js.map +1 -0
- package/dist/adapters/types.js +4 -0
- package/dist/adapters/types.js.map +1 -0
- package/dist/cli.js +195 -0
- package/dist/cli.js.map +1 -0
- package/dist/config.js +240 -0
- package/dist/config.js.map +1 -0
- package/dist/dashboard.js +624 -0
- package/dist/dashboard.js.map +1 -0
- package/dist/index.js +12 -0
- package/dist/index.js.map +1 -0
- package/dist/init.js +130 -0
- package/dist/init.js.map +1 -0
- package/dist/provider.js +6 -0
- package/dist/provider.js.map +1 -0
- package/dist/proxy.js +338 -0
- package/dist/proxy.js.map +1 -0
- package/dist/receipt.js +59 -0
- package/dist/receipt.js.map +1 -0
- package/dist/record.js +78 -0
- package/dist/record.js.map +1 -0
- package/dist/server.js +48 -0
- package/dist/server.js.map +1 -0
- package/dist/sse.js +48 -0
- package/dist/sse.js.map +1 -0
- package/dist/storage.js +71 -0
- package/dist/storage.js.map +1 -0
- package/dist/summary.js +112 -0
- package/dist/summary.js.map +1 -0
- package/dist/telemetry.js +38 -0
- package/dist/telemetry.js.map +1 -0
- package/package.json +41 -0
|
@@ -0,0 +1,624 @@
|
|
|
1
|
+
import { normalizeCanonicalEvent } from "@inferock/measure/canonical-event";
|
|
2
|
+
import { estimateCostUsd } from "@inferock/measure/stateless";
|
|
3
|
+
import { benchKeyFromConfig, providerBaseUrl, providerKeyStatus, } from "./config.js";
|
|
4
|
+
import { createReceiptBundle, renderReceipt } from "./receipt.js";
|
|
5
|
+
import { summarizeBenchEvents } from "./summary.js";
|
|
6
|
+
export function dashboardSetupState(input) {
|
|
7
|
+
const env = input.env ?? process.env;
|
|
8
|
+
const benchKey = input.config.benchKey ?? benchKeyFromConfig(input.config, env);
|
|
9
|
+
return {
|
|
10
|
+
benchKey,
|
|
11
|
+
benchKeySource: env.INFEROCK_BENCH_KEY
|
|
12
|
+
? input.config.benchKey ? "config" : "env"
|
|
13
|
+
: input.config.benchKey
|
|
14
|
+
? "config"
|
|
15
|
+
: "missing",
|
|
16
|
+
configPath: input.paths?.configFile ?? null,
|
|
17
|
+
providers: {
|
|
18
|
+
openai: providerState("openai", input.config, env),
|
|
19
|
+
anthropic: providerState("anthropic", input.config, env),
|
|
20
|
+
},
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
export function dashboardStateFor(setup, summary) {
|
|
24
|
+
const hasProvider = setup.providers.openai.configured || setup.providers.anthropic.configured;
|
|
25
|
+
if (!hasProvider)
|
|
26
|
+
return "no-provider";
|
|
27
|
+
return summary.measuredCalls > 0 ? "calls-flowing" : "configured";
|
|
28
|
+
}
|
|
29
|
+
export function recentCallsFromRecords(records, limit) {
|
|
30
|
+
return records
|
|
31
|
+
.map((record) => normalizeCanonicalEvent(record.event))
|
|
32
|
+
.sort((left, right) => new Date(right.timing.startedAt).getTime() - new Date(left.timing.startedAt).getTime())
|
|
33
|
+
.slice(0, limit)
|
|
34
|
+
.map((event) => ({
|
|
35
|
+
time: event.timing.startedAt,
|
|
36
|
+
provider: event.request.provider,
|
|
37
|
+
model: event.response.servedModel || event.request.requestedModel || event.request.model,
|
|
38
|
+
statusCode: event.response.statusCode,
|
|
39
|
+
status: event.response.statusCode < 400 ? "ok" : "error",
|
|
40
|
+
inputTokens: event.usage.input,
|
|
41
|
+
outputTokens: event.usage.output,
|
|
42
|
+
totalTokens: totalTokens(event),
|
|
43
|
+
costUsd: estimateCostUsd(event),
|
|
44
|
+
}));
|
|
45
|
+
}
|
|
46
|
+
export function summaryPayload(input) {
|
|
47
|
+
const summary = summarizeBenchEvents(input.records);
|
|
48
|
+
const setup = dashboardSetupState(input);
|
|
49
|
+
return {
|
|
50
|
+
summary,
|
|
51
|
+
setup,
|
|
52
|
+
dashboardState: dashboardStateFor(setup, summary),
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
export function receiptPayload(records) {
|
|
56
|
+
const summary = summarizeBenchEvents(records);
|
|
57
|
+
const bundle = createReceiptBundle(summary);
|
|
58
|
+
return {
|
|
59
|
+
bundle,
|
|
60
|
+
compactText: renderReceipt(bundle, true),
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
export function renderDashboardHtml() {
|
|
64
|
+
return `<!doctype html>
|
|
65
|
+
<html lang="en">
|
|
66
|
+
<head>
|
|
67
|
+
<meta charset="utf-8">
|
|
68
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
69
|
+
<title>Inferock Bench</title>
|
|
70
|
+
<style>
|
|
71
|
+
:root {
|
|
72
|
+
color-scheme: light;
|
|
73
|
+
--ink: #17201b;
|
|
74
|
+
--muted: #637268;
|
|
75
|
+
--line: #d9e2dc;
|
|
76
|
+
--paper: #f7f8f5;
|
|
77
|
+
--panel: #ffffff;
|
|
78
|
+
--green: #137a43;
|
|
79
|
+
--green-bg: #e6f5ec;
|
|
80
|
+
--red: #b3261e;
|
|
81
|
+
--red-bg: #fdecea;
|
|
82
|
+
--amber: #8a5a00;
|
|
83
|
+
--amber-bg: #fff4d7;
|
|
84
|
+
--blue: #2457a6;
|
|
85
|
+
--blue-bg: #e9f1ff;
|
|
86
|
+
--shadow: 0 1px 2px rgba(23, 32, 27, 0.08);
|
|
87
|
+
}
|
|
88
|
+
* { box-sizing: border-box; }
|
|
89
|
+
body {
|
|
90
|
+
margin: 0;
|
|
91
|
+
background: var(--paper);
|
|
92
|
+
color: var(--ink);
|
|
93
|
+
font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
|
94
|
+
letter-spacing: 0;
|
|
95
|
+
}
|
|
96
|
+
button, input, textarea { font: inherit; }
|
|
97
|
+
button {
|
|
98
|
+
border: 1px solid var(--line);
|
|
99
|
+
background: var(--panel);
|
|
100
|
+
color: var(--ink);
|
|
101
|
+
border-radius: 6px;
|
|
102
|
+
padding: 8px 10px;
|
|
103
|
+
cursor: pointer;
|
|
104
|
+
min-height: 36px;
|
|
105
|
+
}
|
|
106
|
+
button.primary { background: var(--ink); border-color: var(--ink); color: white; }
|
|
107
|
+
button.danger { color: var(--red); border-color: #efbbb6; }
|
|
108
|
+
button:disabled { cursor: default; opacity: 0.55; }
|
|
109
|
+
input {
|
|
110
|
+
width: 100%;
|
|
111
|
+
min-height: 38px;
|
|
112
|
+
border: 1px solid var(--line);
|
|
113
|
+
border-radius: 6px;
|
|
114
|
+
padding: 8px 10px;
|
|
115
|
+
background: white;
|
|
116
|
+
color: var(--ink);
|
|
117
|
+
}
|
|
118
|
+
.shell { max-width: 1260px; margin: 0 auto; padding: 18px; }
|
|
119
|
+
header {
|
|
120
|
+
display: flex;
|
|
121
|
+
align-items: center;
|
|
122
|
+
justify-content: space-between;
|
|
123
|
+
gap: 16px;
|
|
124
|
+
margin-bottom: 14px;
|
|
125
|
+
}
|
|
126
|
+
h1, h2, h3, p { margin: 0; }
|
|
127
|
+
h1 { font-size: 18px; font-weight: 700; }
|
|
128
|
+
h2 { font-size: 15px; font-weight: 700; }
|
|
129
|
+
h3 { font-size: 13px; font-weight: 700; }
|
|
130
|
+
.status-line { color: var(--muted); font-size: 12px; }
|
|
131
|
+
.grid {
|
|
132
|
+
display: grid;
|
|
133
|
+
grid-template-columns: minmax(0, 1.4fr) minmax(320px, 0.8fr);
|
|
134
|
+
gap: 14px;
|
|
135
|
+
align-items: start;
|
|
136
|
+
}
|
|
137
|
+
.stack { display: grid; gap: 14px; min-width: 0; }
|
|
138
|
+
.band, .card {
|
|
139
|
+
background: var(--panel);
|
|
140
|
+
border: 1px solid var(--line);
|
|
141
|
+
border-radius: 8px;
|
|
142
|
+
box-shadow: var(--shadow);
|
|
143
|
+
min-width: 0;
|
|
144
|
+
}
|
|
145
|
+
.band { padding: 18px; }
|
|
146
|
+
.card { padding: 14px; }
|
|
147
|
+
.loss-number {
|
|
148
|
+
font-size: clamp(44px, 8vw, 86px);
|
|
149
|
+
line-height: 0.95;
|
|
150
|
+
font-weight: 800;
|
|
151
|
+
letter-spacing: 0;
|
|
152
|
+
}
|
|
153
|
+
.split {
|
|
154
|
+
display: flex;
|
|
155
|
+
flex-wrap: wrap;
|
|
156
|
+
gap: 8px;
|
|
157
|
+
margin-top: 12px;
|
|
158
|
+
}
|
|
159
|
+
.pill {
|
|
160
|
+
display: inline-flex;
|
|
161
|
+
align-items: center;
|
|
162
|
+
gap: 6px;
|
|
163
|
+
border-radius: 999px;
|
|
164
|
+
border: 1px solid var(--line);
|
|
165
|
+
padding: 5px 8px;
|
|
166
|
+
font-size: 12px;
|
|
167
|
+
color: var(--muted);
|
|
168
|
+
background: #fbfcfa;
|
|
169
|
+
min-height: 28px;
|
|
170
|
+
}
|
|
171
|
+
.pill.good { background: var(--green-bg); border-color: #a9d9ba; color: var(--green); }
|
|
172
|
+
.pill.bad { background: var(--red-bg); border-color: #f2b7b0; color: var(--red); }
|
|
173
|
+
.pill.warn { background: var(--amber-bg); border-color: #ecd18d; color: var(--amber); }
|
|
174
|
+
.metric-row {
|
|
175
|
+
display: grid;
|
|
176
|
+
grid-template-columns: repeat(3, minmax(0, 1fr));
|
|
177
|
+
gap: 10px;
|
|
178
|
+
margin-top: 14px;
|
|
179
|
+
}
|
|
180
|
+
.metric {
|
|
181
|
+
border: 1px solid var(--line);
|
|
182
|
+
border-radius: 8px;
|
|
183
|
+
padding: 10px;
|
|
184
|
+
min-height: 74px;
|
|
185
|
+
background: #fbfcfa;
|
|
186
|
+
}
|
|
187
|
+
.metric strong { display: block; font-size: 24px; line-height: 1.1; }
|
|
188
|
+
.metric span { display: block; margin-top: 6px; color: var(--muted); font-size: 12px; }
|
|
189
|
+
.setup-panel.needs-setup { border-color: #e1b95f; box-shadow: 0 0 0 2px rgba(225, 185, 95, 0.22); }
|
|
190
|
+
.provider-grid { display: grid; gap: 10px; margin-top: 12px; }
|
|
191
|
+
.provider-row {
|
|
192
|
+
display: grid;
|
|
193
|
+
grid-template-columns: minmax(0, 1fr) auto;
|
|
194
|
+
gap: 8px;
|
|
195
|
+
align-items: end;
|
|
196
|
+
}
|
|
197
|
+
.provider-meta {
|
|
198
|
+
display: flex;
|
|
199
|
+
flex-wrap: wrap;
|
|
200
|
+
gap: 6px;
|
|
201
|
+
margin-top: 7px;
|
|
202
|
+
min-height: 28px;
|
|
203
|
+
}
|
|
204
|
+
.actions { display: flex; flex-wrap: wrap; gap: 8px; margin-top: 12px; }
|
|
205
|
+
.key-card { display: none; }
|
|
206
|
+
.key-card.visible { display: block; }
|
|
207
|
+
.key-line {
|
|
208
|
+
display: grid;
|
|
209
|
+
grid-template-columns: minmax(0, 1fr) auto;
|
|
210
|
+
gap: 8px;
|
|
211
|
+
align-items: center;
|
|
212
|
+
margin-top: 10px;
|
|
213
|
+
}
|
|
214
|
+
code, pre {
|
|
215
|
+
font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, "Liberation Mono", monospace;
|
|
216
|
+
letter-spacing: 0;
|
|
217
|
+
}
|
|
218
|
+
code {
|
|
219
|
+
display: inline-block;
|
|
220
|
+
max-width: 100%;
|
|
221
|
+
overflow-wrap: anywhere;
|
|
222
|
+
background: #eef2ee;
|
|
223
|
+
border: 1px solid var(--line);
|
|
224
|
+
border-radius: 6px;
|
|
225
|
+
padding: 6px 8px;
|
|
226
|
+
font-size: 12px;
|
|
227
|
+
}
|
|
228
|
+
pre {
|
|
229
|
+
white-space: pre-wrap;
|
|
230
|
+
overflow-wrap: anywhere;
|
|
231
|
+
background: #101814;
|
|
232
|
+
color: #e7f0ea;
|
|
233
|
+
border-radius: 8px;
|
|
234
|
+
padding: 12px;
|
|
235
|
+
margin: 10px 0 0;
|
|
236
|
+
font-size: 12px;
|
|
237
|
+
line-height: 1.45;
|
|
238
|
+
max-height: 300px;
|
|
239
|
+
overflow: auto;
|
|
240
|
+
}
|
|
241
|
+
.snippet-list { display: grid; gap: 10px; margin-top: 12px; }
|
|
242
|
+
.table-wrap { overflow-x: auto; }
|
|
243
|
+
table { width: 100%; border-collapse: collapse; min-width: 680px; }
|
|
244
|
+
th, td {
|
|
245
|
+
text-align: left;
|
|
246
|
+
padding: 10px 8px;
|
|
247
|
+
border-bottom: 1px solid var(--line);
|
|
248
|
+
font-size: 13px;
|
|
249
|
+
vertical-align: middle;
|
|
250
|
+
}
|
|
251
|
+
th { color: var(--muted); font-weight: 700; font-size: 12px; }
|
|
252
|
+
td.money { font-variant-numeric: tabular-nums; }
|
|
253
|
+
.empty {
|
|
254
|
+
border: 1px dashed var(--line);
|
|
255
|
+
border-radius: 8px;
|
|
256
|
+
padding: 14px;
|
|
257
|
+
color: var(--muted);
|
|
258
|
+
background: #fbfcfa;
|
|
259
|
+
margin-top: 10px;
|
|
260
|
+
}
|
|
261
|
+
.calls-strip {
|
|
262
|
+
display: grid;
|
|
263
|
+
grid-auto-flow: column;
|
|
264
|
+
grid-auto-columns: minmax(240px, 1fr);
|
|
265
|
+
gap: 10px;
|
|
266
|
+
overflow-x: auto;
|
|
267
|
+
padding-bottom: 2px;
|
|
268
|
+
margin-top: 10px;
|
|
269
|
+
}
|
|
270
|
+
.call {
|
|
271
|
+
border: 1px solid var(--line);
|
|
272
|
+
border-radius: 8px;
|
|
273
|
+
padding: 10px;
|
|
274
|
+
background: #fbfcfa;
|
|
275
|
+
min-height: 126px;
|
|
276
|
+
}
|
|
277
|
+
.call-top { display: flex; justify-content: space-between; gap: 8px; align-items: center; }
|
|
278
|
+
.call dl {
|
|
279
|
+
display: grid;
|
|
280
|
+
grid-template-columns: auto 1fr;
|
|
281
|
+
gap: 5px 8px;
|
|
282
|
+
margin: 10px 0 0;
|
|
283
|
+
font-size: 12px;
|
|
284
|
+
}
|
|
285
|
+
.call dt { color: var(--muted); }
|
|
286
|
+
.call dd { margin: 0; overflow-wrap: anywhere; }
|
|
287
|
+
.receipt-actions { display: flex; gap: 8px; flex-wrap: wrap; margin-top: 10px; }
|
|
288
|
+
.fine-print { color: var(--muted); font-size: 12px; margin-top: 8px; }
|
|
289
|
+
@media (max-width: 900px) {
|
|
290
|
+
.shell { padding: 12px; }
|
|
291
|
+
.grid { grid-template-columns: 1fr; }
|
|
292
|
+
.metric-row { grid-template-columns: 1fr; }
|
|
293
|
+
header { align-items: flex-start; flex-direction: column; }
|
|
294
|
+
}
|
|
295
|
+
</style>
|
|
296
|
+
</head>
|
|
297
|
+
<body data-dashboard-state="no-provider">
|
|
298
|
+
<main class="shell">
|
|
299
|
+
<header>
|
|
300
|
+
<div>
|
|
301
|
+
<h1>Inferock Bench</h1>
|
|
302
|
+
<p class="status-line" id="refreshStatus">Loading local dashboard...</p>
|
|
303
|
+
</div>
|
|
304
|
+
<span class="pill" id="statePill">Waiting for setup</span>
|
|
305
|
+
</header>
|
|
306
|
+
|
|
307
|
+
<section class="grid">
|
|
308
|
+
<div class="stack">
|
|
309
|
+
<section class="band" aria-labelledby="lossHeading">
|
|
310
|
+
<h2 id="lossHeading">$ lost so far</h2>
|
|
311
|
+
<div class="loss-number" id="lostTotal">$0.00</div>
|
|
312
|
+
<div class="split">
|
|
313
|
+
<span class="pill" id="recognizedSplit">provider-recognized $0.00</span>
|
|
314
|
+
<span class="pill" id="unrecognizedSplit">unrecognized $0.00</span>
|
|
315
|
+
<span class="pill good" id="failurePill">measured 0 calls, 0 failures</span>
|
|
316
|
+
</div>
|
|
317
|
+
<div class="metric-row">
|
|
318
|
+
<div class="metric"><strong id="measuredCalls">0</strong><span>measured calls</span></div>
|
|
319
|
+
<div class="metric"><strong id="failureCount">0</strong><span>failures</span></div>
|
|
320
|
+
<div class="metric"><strong id="providerSpend">$0.00</strong><span>provider spend observed</span></div>
|
|
321
|
+
</div>
|
|
322
|
+
</section>
|
|
323
|
+
|
|
324
|
+
<section class="card" aria-labelledby="rowsHeading">
|
|
325
|
+
<h2 id="rowsHeading">Per-class loss rows</h2>
|
|
326
|
+
<div class="table-wrap">
|
|
327
|
+
<table>
|
|
328
|
+
<thead>
|
|
329
|
+
<tr>
|
|
330
|
+
<th>Signal class</th>
|
|
331
|
+
<th>Count</th>
|
|
332
|
+
<th>Evidence grade</th>
|
|
333
|
+
<th>Provider-recognized</th>
|
|
334
|
+
<th>Unrecognized</th>
|
|
335
|
+
</tr>
|
|
336
|
+
</thead>
|
|
337
|
+
<tbody id="rowsBody"></tbody>
|
|
338
|
+
</table>
|
|
339
|
+
</div>
|
|
340
|
+
<div class="empty" id="rowsEmpty">No loss rows. measured 0 calls, 0 failures.</div>
|
|
341
|
+
</section>
|
|
342
|
+
|
|
343
|
+
<section class="card" aria-labelledby="callsHeading">
|
|
344
|
+
<h2 id="callsHeading">Recent calls</h2>
|
|
345
|
+
<div class="calls-strip" id="callsStrip"></div>
|
|
346
|
+
<div class="empty" id="callsEmpty">No calls measured yet.</div>
|
|
347
|
+
</section>
|
|
348
|
+
</div>
|
|
349
|
+
|
|
350
|
+
<aside class="stack">
|
|
351
|
+
<section class="card setup-panel needs-setup" id="setupPanel" aria-labelledby="setupHeading">
|
|
352
|
+
<h2 id="setupHeading">Provider API keys</h2>
|
|
353
|
+
<div class="provider-grid">
|
|
354
|
+
<label>
|
|
355
|
+
<h3>OpenAI</h3>
|
|
356
|
+
<input id="openaiKeyInput" type="password" autocomplete="off" spellcheck="false" placeholder="Paste OpenAI key">
|
|
357
|
+
<div class="provider-meta" id="openaiMeta"></div>
|
|
358
|
+
</label>
|
|
359
|
+
<label>
|
|
360
|
+
<h3>Anthropic</h3>
|
|
361
|
+
<input id="anthropicKeyInput" type="password" autocomplete="off" spellcheck="false" placeholder="Paste Anthropic key">
|
|
362
|
+
<div class="provider-meta" id="anthropicMeta"></div>
|
|
363
|
+
</label>
|
|
364
|
+
</div>
|
|
365
|
+
<div class="actions">
|
|
366
|
+
<button class="primary" id="saveSetupButton" type="button">Save</button>
|
|
367
|
+
<button class="danger" data-remove-provider="openai" type="button">Remove OpenAI</button>
|
|
368
|
+
<button class="danger" data-remove-provider="anthropic" type="button">Remove Anthropic</button>
|
|
369
|
+
</div>
|
|
370
|
+
<p class="fine-print" id="setupPath"></p>
|
|
371
|
+
</section>
|
|
372
|
+
|
|
373
|
+
<section class="card key-card" id="keyCard" data-key-card aria-labelledby="keyHeading">
|
|
374
|
+
<h2 id="keyHeading">Your local bench key</h2>
|
|
375
|
+
<div class="key-line">
|
|
376
|
+
<code id="benchKeyValue">ibl_</code>
|
|
377
|
+
<button id="copyKeyButton" type="button">Copy</button>
|
|
378
|
+
</div>
|
|
379
|
+
<div class="snippet-list" id="snippetList"></div>
|
|
380
|
+
</section>
|
|
381
|
+
|
|
382
|
+
<section class="card" id="zeroHint" aria-labelledby="zeroHintHeading">
|
|
383
|
+
<h2 id="zeroHintHeading">Setup hint</h2>
|
|
384
|
+
<div class="snippet-list" id="zeroHintSnippets"></div>
|
|
385
|
+
</section>
|
|
386
|
+
|
|
387
|
+
<section class="card" aria-labelledby="receiptHeading">
|
|
388
|
+
<h2 id="receiptHeading">Receipt</h2>
|
|
389
|
+
<pre id="receiptText">$0.00
|
|
390
|
+
my AI provider cost me $0.00 in failures this period
|
|
391
|
+
measured 0 calls, 0 failures
|
|
392
|
+
no loss rows</pre>
|
|
393
|
+
<div class="receipt-actions">
|
|
394
|
+
<button id="copyReceiptButton" type="button">Copy receipt</button>
|
|
395
|
+
<button id="downloadReceiptButton" type="button">Download JSON bundle</button>
|
|
396
|
+
</div>
|
|
397
|
+
</section>
|
|
398
|
+
</aside>
|
|
399
|
+
</section>
|
|
400
|
+
</main>
|
|
401
|
+
|
|
402
|
+
<script>
|
|
403
|
+
const state = { setup: null, receipt: null };
|
|
404
|
+
const $ = (id) => document.getElementById(id);
|
|
405
|
+
const money = new Intl.NumberFormat("en-US", { style: "currency", currency: "USD", minimumFractionDigits: 2, maximumFractionDigits: 2 });
|
|
406
|
+
const integer = new Intl.NumberFormat("en-US");
|
|
407
|
+
|
|
408
|
+
function formatUsd(value) {
|
|
409
|
+
return money.format(Number(value || 0));
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
function providerLabel(provider) {
|
|
413
|
+
return provider === "openai" ? "OpenAI" : "Anthropic";
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
function sdkBaseUrl(provider) {
|
|
417
|
+
return provider === "openai" ? location.origin + "/v1" : location.origin;
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
function snippetFor(provider, key) {
|
|
421
|
+
const client = provider === "openai" ? "openai" : "anthropic";
|
|
422
|
+
const ctor = provider === "openai" ? "OpenAI" : "Anthropic";
|
|
423
|
+
return "const " + client + " = new " + ctor + "({\\n" +
|
|
424
|
+
" apiKey: process.env.INFEROCK_BENCH_KEY ?? \\"" + key + "\\",\\n" +
|
|
425
|
+
" baseURL: \\"" + sdkBaseUrl(provider) + "\\",\\n" +
|
|
426
|
+
"});";
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
function renderSummary(payload) {
|
|
430
|
+
const summary = payload.summary;
|
|
431
|
+
state.setup = payload.setup;
|
|
432
|
+
document.body.dataset.dashboardState = payload.dashboardState;
|
|
433
|
+
$("lostTotal").textContent = formatUsd(summary.totalLostUsd);
|
|
434
|
+
$("recognizedSplit").textContent = "provider-recognized " + formatUsd(summary.providerRecognizedUsd);
|
|
435
|
+
$("unrecognizedSplit").textContent = "unrecognized " + formatUsd(summary.unrecognizedUsd);
|
|
436
|
+
$("measuredCalls").textContent = integer.format(summary.measuredCalls);
|
|
437
|
+
$("failureCount").textContent = integer.format(summary.failureCount);
|
|
438
|
+
$("providerSpend").textContent = formatUsd(summary.providerSpendUsd);
|
|
439
|
+
$("failurePill").textContent = "measured " + integer.format(summary.measuredCalls) + " calls, " + integer.format(summary.failureCount) + " failures";
|
|
440
|
+
$("failurePill").className = "pill " + (summary.failureCount === 0 ? "good" : "bad");
|
|
441
|
+
$("statePill").textContent = payload.dashboardState === "no-provider"
|
|
442
|
+
? "Provider key needed"
|
|
443
|
+
: payload.dashboardState === "configured"
|
|
444
|
+
? "Ready for traffic"
|
|
445
|
+
: "Calls flowing";
|
|
446
|
+
$("statePill").className = "pill " + (payload.dashboardState === "no-provider" ? "warn" : "good");
|
|
447
|
+
renderSetup(payload.setup, payload.dashboardState, summary);
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
function renderSetup(setup, dashboardState, summary) {
|
|
451
|
+
const hasProvider = setup.providers.openai.configured || setup.providers.anthropic.configured;
|
|
452
|
+
$("setupPanel").classList.toggle("needs-setup", !hasProvider);
|
|
453
|
+
$("setupPath").textContent = setup.configPath ? "Saved locally at " + setup.configPath : "";
|
|
454
|
+
renderProviderMeta("openai", setup.providers.openai);
|
|
455
|
+
renderProviderMeta("anthropic", setup.providers.anthropic);
|
|
456
|
+
|
|
457
|
+
$("keyCard").classList.toggle("visible", hasProvider);
|
|
458
|
+
$("benchKeyValue").textContent = setup.benchKey || "";
|
|
459
|
+
const snippets = [];
|
|
460
|
+
for (const provider of ["openai", "anthropic"]) {
|
|
461
|
+
if (setup.providers[provider].configured) snippets.push(snippetBlock(provider, setup.benchKey));
|
|
462
|
+
}
|
|
463
|
+
$("snippetList").innerHTML = snippets.join("");
|
|
464
|
+
const showZeroHint = hasProvider && summary.measuredCalls === 0;
|
|
465
|
+
$("zeroHint").style.display = showZeroHint ? "" : "none";
|
|
466
|
+
$("zeroHintSnippets").innerHTML = showZeroHint ? snippets.join("") : "";
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
function renderProviderMeta(provider, info) {
|
|
470
|
+
const target = $(provider + "Meta");
|
|
471
|
+
if (!info.configured) {
|
|
472
|
+
target.innerHTML = '<span class="pill warn">not configured</span>';
|
|
473
|
+
return;
|
|
474
|
+
}
|
|
475
|
+
const source = info.source === "env" ? "env override" : "local config";
|
|
476
|
+
target.innerHTML = '<span class="pill good">' + escapeHtml(info.maskedKey || "configured") + '</span>' +
|
|
477
|
+
'<span class="pill">' + source + '</span>';
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
function snippetBlock(provider, key) {
|
|
481
|
+
return '<div><h3>' + providerLabel(provider) + '</h3><pre>' + escapeHtml(snippetFor(provider, key)) + '</pre></div>';
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
function renderRows(rows) {
|
|
485
|
+
const body = $("rowsBody");
|
|
486
|
+
body.innerHTML = rows.map((row) => '<tr>' +
|
|
487
|
+
'<td>' + escapeHtml(row.code + "/" + row.failureClass) + '</td>' +
|
|
488
|
+
'<td>' + integer.format(row.count) + '</td>' +
|
|
489
|
+
'<td><span class="pill">' + escapeHtml(row.evidenceGrade) + '</span></td>' +
|
|
490
|
+
'<td class="money">' + formatUsd(row.providerRecognizedUsd) + '</td>' +
|
|
491
|
+
'<td class="money">' + formatUsd(row.unrecognizedUsd) + '</td>' +
|
|
492
|
+
'</tr>').join("");
|
|
493
|
+
$("rowsEmpty").style.display = rows.length === 0 ? "" : "none";
|
|
494
|
+
if (rows.length === 0) {
|
|
495
|
+
const measured = $("measuredCalls").textContent || "0";
|
|
496
|
+
const failures = $("failureCount").textContent || "0";
|
|
497
|
+
$("rowsEmpty").textContent = "No loss rows. measured " + measured + " calls, " + failures + " failures.";
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
function renderCalls(calls) {
|
|
502
|
+
$("callsStrip").innerHTML = calls.map((call) => '<article class="call">' +
|
|
503
|
+
'<div class="call-top"><strong>' + escapeHtml(providerLabel(call.provider)) + '</strong><span class="pill ' + (call.status === "ok" ? "good" : "bad") + '">' + call.statusCode + '</span></div>' +
|
|
504
|
+
'<dl>' +
|
|
505
|
+
'<dt>time</dt><dd>' + escapeHtml(new Date(call.time).toLocaleTimeString()) + '</dd>' +
|
|
506
|
+
'<dt>model</dt><dd>' + escapeHtml(call.model) + '</dd>' +
|
|
507
|
+
'<dt>tokens</dt><dd>' + integer.format(call.totalTokens) + ' total (' + integer.format(call.inputTokens) + ' in, ' + integer.format(call.outputTokens) + ' out)</dd>' +
|
|
508
|
+
'<dt>cost</dt><dd>' + formatUsd(call.costUsd) + '</dd>' +
|
|
509
|
+
'</dl>' +
|
|
510
|
+
'</article>').join("");
|
|
511
|
+
$("callsEmpty").style.display = calls.length === 0 ? "" : "none";
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
function renderReceipt(payload) {
|
|
515
|
+
state.receipt = payload;
|
|
516
|
+
$("receiptText").textContent = payload.compactText;
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
async function refresh() {
|
|
520
|
+
try {
|
|
521
|
+
const responses = await Promise.all([
|
|
522
|
+
fetch("/api/summary"),
|
|
523
|
+
fetch("/api/rows"),
|
|
524
|
+
fetch("/api/calls?limit=8"),
|
|
525
|
+
fetch("/api/receipt"),
|
|
526
|
+
]);
|
|
527
|
+
for (const response of responses) {
|
|
528
|
+
if (!response.ok) throw new Error("HTTP " + response.status);
|
|
529
|
+
}
|
|
530
|
+
const summary = await responses[0].json();
|
|
531
|
+
const rows = await responses[1].json();
|
|
532
|
+
const calls = await responses[2].json();
|
|
533
|
+
const receipt = await responses[3].json();
|
|
534
|
+
renderSummary(summary);
|
|
535
|
+
renderRows(rows.rows);
|
|
536
|
+
renderCalls(calls.calls);
|
|
537
|
+
renderReceipt(receipt);
|
|
538
|
+
$("refreshStatus").textContent = "Updated " + new Date().toLocaleTimeString();
|
|
539
|
+
} catch (error) {
|
|
540
|
+
$("refreshStatus").textContent = "Refresh failed: " + (error && error.message ? error.message : "unknown error");
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
async function saveSetup() {
|
|
545
|
+
const body = {};
|
|
546
|
+
const openaiKey = $("openaiKeyInput").value.trim();
|
|
547
|
+
const anthropicKey = $("anthropicKeyInput").value.trim();
|
|
548
|
+
if (openaiKey) body.openaiApiKey = openaiKey;
|
|
549
|
+
if (anthropicKey) body.anthropicApiKey = anthropicKey;
|
|
550
|
+
await postSetup(body);
|
|
551
|
+
$("openaiKeyInput").value = "";
|
|
552
|
+
$("anthropicKeyInput").value = "";
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
async function removeProvider(provider) {
|
|
556
|
+
await postSetup(provider === "openai" ? { openaiApiKey: null } : { anthropicApiKey: null });
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
async function postSetup(body) {
|
|
560
|
+
const response = await fetch("/api/setup", {
|
|
561
|
+
method: "POST",
|
|
562
|
+
headers: { "content-type": "application/json" },
|
|
563
|
+
body: JSON.stringify(body),
|
|
564
|
+
});
|
|
565
|
+
if (!response.ok) throw new Error("setup failed");
|
|
566
|
+
const payload = await response.json();
|
|
567
|
+
renderSummary(payload);
|
|
568
|
+
await refresh();
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
async function copyText(text) {
|
|
572
|
+
await navigator.clipboard.writeText(text);
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
function downloadReceipt() {
|
|
576
|
+
if (!state.receipt) return;
|
|
577
|
+
const blob = new Blob([JSON.stringify(state.receipt.bundle, null, 2) + "\\n"], { type: "application/json" });
|
|
578
|
+
const link = document.createElement("a");
|
|
579
|
+
link.href = URL.createObjectURL(blob);
|
|
580
|
+
link.download = "inferock-bench-receipt.json";
|
|
581
|
+
link.click();
|
|
582
|
+
URL.revokeObjectURL(link.href);
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
function escapeHtml(value) {
|
|
586
|
+
return String(value)
|
|
587
|
+
.replaceAll("&", "&")
|
|
588
|
+
.replaceAll("<", "<")
|
|
589
|
+
.replaceAll(">", ">")
|
|
590
|
+
.replaceAll('"', """)
|
|
591
|
+
.replaceAll("'", "'");
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
$("saveSetupButton").addEventListener("click", () => saveSetup().catch((error) => {
|
|
595
|
+
$("refreshStatus").textContent = error.message || "setup failed";
|
|
596
|
+
}));
|
|
597
|
+
for (const button of document.querySelectorAll("[data-remove-provider]")) {
|
|
598
|
+
button.addEventListener("click", () => removeProvider(button.dataset.removeProvider).catch((error) => {
|
|
599
|
+
$("refreshStatus").textContent = error.message || "remove failed";
|
|
600
|
+
}));
|
|
601
|
+
}
|
|
602
|
+
$("copyKeyButton").addEventListener("click", () => state.setup && copyText(state.setup.benchKey));
|
|
603
|
+
$("copyReceiptButton").addEventListener("click", () => state.receipt && copyText(state.receipt.compactText));
|
|
604
|
+
$("downloadReceiptButton").addEventListener("click", downloadReceipt);
|
|
605
|
+
|
|
606
|
+
refresh();
|
|
607
|
+
setInterval(refresh, 3000);
|
|
608
|
+
</script>
|
|
609
|
+
</body>
|
|
610
|
+
</html>`;
|
|
611
|
+
}
|
|
612
|
+
function providerState(provider, config, env) {
|
|
613
|
+
return {
|
|
614
|
+
...providerKeyStatus(provider, config, env),
|
|
615
|
+
providerApiBaseUrl: providerBaseUrl(provider, config, env),
|
|
616
|
+
};
|
|
617
|
+
}
|
|
618
|
+
function totalTokens(event) {
|
|
619
|
+
return event.usage.input +
|
|
620
|
+
event.usage.output +
|
|
621
|
+
(event.usage.cache?.read ?? 0) +
|
|
622
|
+
(event.usage.cache?.creation ?? 0);
|
|
623
|
+
}
|
|
624
|
+
//# sourceMappingURL=dashboard.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dashboard.js","sourceRoot":"","sources":["../src/dashboard.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EACL,kBAAkB,EAClB,eAAe,EACf,iBAAiB,GAIlB,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAElE,OAAO,EAAE,oBAAoB,EAAqB,MAAM,cAAc,CAAC;AA2BvE,MAAM,UAAU,mBAAmB,CAAC,KAInC;IACC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACrC,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,IAAI,kBAAkB,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAChF,OAAO;QACL,QAAQ;QACR,cAAc,EAAE,GAAG,CAAC,kBAAkB;YACpC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK;YAC1C,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ;gBACrB,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,SAAS;QACf,UAAU,EAAE,KAAK,CAAC,KAAK,EAAE,UAAU,IAAI,IAAI;QAC3C,SAAS,EAAE;YACT,MAAM,EAAE,aAAa,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC;YAClD,SAAS,EAAE,aAAa,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC;SACzD;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,KAA0B,EAC1B,OAAqB;IAErB,MAAM,WAAW,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,IAAI,KAAK,CAAC,SAAS,CAAC,SAAS,CAAC,UAAU,CAAC;IAC9F,IAAI,CAAC,WAAW;QAAE,OAAO,aAAa,CAAC;IACvC,OAAO,OAAO,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,YAAY,CAAC;AACpE,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,OAAoC,EACpC,KAAa;IAEb,OAAO,OAAO;SACX,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,uBAAuB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;SACtD,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACpB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;SACxF,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;SACf,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACf,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;QAC5B,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ;QAChC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,WAAW,IAAI,KAAK,CAAC,OAAO,CAAC,cAAc,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK;QACxF,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU;QACrC,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO;QACxD,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK;QAC9B,YAAY,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM;QAChC,WAAW,EAAE,WAAW,CAAC,KAAK,CAAC;QAC/B,OAAO,EAAE,eAAe,CAAC,KAAK,CAAC;KAChC,CAAC,CAAC,CAAC;AACR,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAK9B;IACC,MAAM,OAAO,GAAG,oBAAoB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;IACzC,OAAO;QACL,OAAO;QACP,KAAK;QACL,cAAc,EAAE,iBAAiB,CAAC,KAAK,EAAE,OAAO,CAAC;KAClD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,OAAoC;IACjE,MAAM,OAAO,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAC5C,OAAO;QACL,MAAM;QACN,WAAW,EAAE,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC;KACzC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB;IACjC,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAkiBD,CAAC;AACT,CAAC;AAED,SAAS,aAAa,CACpB,QAAsB,EACtB,MAAmB,EACnB,GAAsB;IAEtB,OAAO;QACL,GAAG,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC;QAC3C,kBAAkB,EAAE,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC;KAC3D,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,KAAiD;IACpE,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK;QACtB,KAAK,CAAC,KAAK,CAAC,MAAM;QAClB,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC;QAC9B,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,IAAI,CAAC,CAAC,CAAC;AACvC,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { runCli } from "./cli.js";
|
|
3
|
+
runCli(process.argv.slice(2)).catch((error) => {
|
|
4
|
+
if (error instanceof Error && error.constructor.name === "CliUsageError") {
|
|
5
|
+
process.exitCode = 1;
|
|
6
|
+
return;
|
|
7
|
+
}
|
|
8
|
+
const message = error instanceof Error ? error.message : "inferock-bench failed";
|
|
9
|
+
console.error(message);
|
|
10
|
+
process.exitCode = 1;
|
|
11
|
+
});
|
|
12
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAElC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IACrD,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;QACzE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,uBAAuB,CAAC;IACjF,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACvB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;AACvB,CAAC,CAAC,CAAC"}
|