@schlessera/brain-ui-server 0.19.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/data/model-prices.json +63 -0
- package/dist/activity/digest.d.ts.map +1 -1
- package/dist/activity/digest.js +30 -2
- package/dist/activity/digest.js.map +1 -1
- package/dist/activity/notify.d.ts.map +1 -1
- package/dist/activity/notify.js +26 -4
- package/dist/activity/notify.js.map +1 -1
- package/dist/activity/query.d.ts.map +1 -1
- package/dist/activity/query.js +28 -1
- package/dist/activity/query.js.map +1 -1
- package/dist/activity/recorder.d.ts +5 -0
- package/dist/activity/recorder.d.ts.map +1 -1
- package/dist/activity/recorder.js +37 -4
- package/dist/activity/recorder.js.map +1 -1
- package/dist/activity/runtime.d.ts +7 -1
- package/dist/activity/runtime.d.ts.map +1 -1
- package/dist/activity/runtime.js +28 -10
- package/dist/activity/runtime.js.map +1 -1
- package/dist/activity/store.d.ts +51 -2
- package/dist/activity/store.d.ts.map +1 -1
- package/dist/activity/store.js +236 -19
- package/dist/activity/store.js.map +1 -1
- package/dist/activity/stream.d.ts +6 -2
- package/dist/activity/stream.d.ts.map +1 -1
- package/dist/activity/stream.js +42 -1
- package/dist/activity/stream.js.map +1 -1
- package/dist/agent/backend.d.ts +13 -4
- package/dist/agent/backend.d.ts.map +1 -1
- package/dist/agent/backend.js +68 -5
- package/dist/agent/backend.js.map +1 -1
- package/dist/app.d.ts.map +1 -1
- package/dist/app.js +18 -3
- package/dist/app.js.map +1 -1
- package/dist/config/env.d.ts +36 -0
- package/dist/config/env.d.ts.map +1 -1
- package/dist/config/env.js +66 -2
- package/dist/config/env.js.map +1 -1
- package/dist/db/settings.d.ts +15 -0
- package/dist/db/settings.d.ts.map +1 -1
- package/dist/db/settings.js +43 -0
- package/dist/db/settings.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/pricing/model-pricing.d.ts +61 -0
- package/dist/pricing/model-pricing.d.ts.map +1 -0
- package/dist/pricing/model-pricing.js +379 -0
- package/dist/pricing/model-pricing.js.map +1 -0
- package/dist/routes/activity.d.ts.map +1 -1
- package/dist/routes/activity.js +40 -32
- package/dist/routes/activity.js.map +1 -1
- package/dist/routes/models.d.ts +6 -0
- package/dist/routes/models.d.ts.map +1 -1
- package/dist/routes/models.js +38 -3
- package/dist/routes/models.js.map +1 -1
- package/dist/ws/run-session.d.ts.map +1 -1
- package/dist/ws/run-session.js +36 -2
- package/dist/ws/run-session.js.map +1 -1
- package/migrations/009_activity_followups.sql +22 -0
- package/migrations/010_effective_cost.sql +21 -0
- package/package.json +4 -3
- package/src/activity/digest.ts +30 -2
- package/src/activity/notify.ts +30 -8
- package/src/activity/query.ts +28 -1
- package/src/activity/recorder.ts +57 -5
- package/src/activity/runtime.ts +41 -13
- package/src/activity/store.ts +316 -21
- package/src/activity/stream.ts +39 -9
- package/src/agent/backend.ts +91 -10
- package/src/app.ts +19 -3
- package/src/config/env.ts +90 -2
- package/src/db/settings.ts +56 -0
- package/src/index.ts +1 -0
- package/src/pricing/model-pricing.ts +497 -0
- package/src/routes/activity.ts +41 -33
- package/src/routes/models.ts +57 -6
- package/src/ws/run-session.ts +45 -3
|
@@ -0,0 +1,497 @@
|
|
|
1
|
+
// Model pricing.
|
|
2
|
+
//
|
|
3
|
+
// The rollup layer needs per-token USD rates to turn a run's token usage into
|
|
4
|
+
// an effective cost, and prices change far faster than this package ships. So
|
|
5
|
+
// rates come from two remote catalogs, merged and TTL-cached, with a bundled
|
|
6
|
+
// snapshot as the offline fallback — the same shape as the model-discovery
|
|
7
|
+
// source in `ui-backend-claude` (versioned cache envelope, synchronous reads,
|
|
8
|
+
// cold-start-only await, single-flight refresh, serve-stale-on-error).
|
|
9
|
+
//
|
|
10
|
+
// The two sources cover different territory:
|
|
11
|
+
//
|
|
12
|
+
// 1. LITELLM (`model_prices_and_context_window.json`): community-maintained,
|
|
13
|
+
// per-single-token floats, keyed bare for Anthropic/OpenAI and
|
|
14
|
+
// `gemini/...` for Gemini. Broad coverage, known cross-key drift — so
|
|
15
|
+
// entries are validated on ingest and dropped, never trusted blindly.
|
|
16
|
+
//
|
|
17
|
+
// 2. OPENROUTER (`GET /api/v1/models`): the vendor's own catalog, keyed
|
|
18
|
+
// `vendor/model`, prices as STRING per-token USD. `"0"` is genuinely
|
|
19
|
+
// free, not unknown. OpenRouter wins for ids its catalog carries, since
|
|
20
|
+
// an openrouter-routed run is billed at OpenRouter's rate.
|
|
21
|
+
//
|
|
22
|
+
// Pricing is best-effort by construction: no network, a broken response, or a
|
|
23
|
+
// corrupt cache degrades to the last good table (or the snapshot), never to a
|
|
24
|
+
// throw. A model the table cannot price resolves to null — the caller renders
|
|
25
|
+
// unknown, never zero (the binding fail-loud decision).
|
|
26
|
+
|
|
27
|
+
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "fs";
|
|
28
|
+
import { dirname, join } from "path";
|
|
29
|
+
|
|
30
|
+
import { canonicalModelId } from "@schlessera/brain-ui-sdk/protocol";
|
|
31
|
+
|
|
32
|
+
const LITELLM_URL =
|
|
33
|
+
"https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json";
|
|
34
|
+
const OPENROUTER_URL = "https://openrouter.ai/api/v1/models";
|
|
35
|
+
const REQUEST_TIMEOUT_MS = 10_000;
|
|
36
|
+
const CACHE_VERSION = 1;
|
|
37
|
+
const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;
|
|
38
|
+
/** Hard cap on a catalog response — both real catalogs are ~2 MB; anything an
|
|
39
|
+
* order of magnitude past that is a broken or hostile endpoint, not data. */
|
|
40
|
+
const MAX_RESPONSE_BYTES = 20 * 1024 * 1024;
|
|
41
|
+
/** Plausibility ceiling on a per-token rate: $0.01/token = $10,000 per million
|
|
42
|
+
* tokens, far past any real price. A rate above it is a corrupt or hostile
|
|
43
|
+
* entry — once frozen into a rollup it would misreport spend forever. */
|
|
44
|
+
const MAX_RATE_PER_TOKEN = 0.01;
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Per-token USD rates for one model. `cacheRead`/`cacheWrite` are null when
|
|
48
|
+
* the source lacks them — a caller whose run consumed cache tokens must then
|
|
49
|
+
* treat the whole run as unpriced, never price the gap at zero.
|
|
50
|
+
*/
|
|
51
|
+
export interface PricingRates {
|
|
52
|
+
input: number;
|
|
53
|
+
output: number;
|
|
54
|
+
cacheRead: number | null;
|
|
55
|
+
cacheWrite: number | null;
|
|
56
|
+
/** True when the rates came from the bundled snapshot (genuinely dated). */
|
|
57
|
+
estimate: boolean;
|
|
58
|
+
source: "litellm" | "openrouter" | "snapshot";
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** The stored shape: rates without the resolution-time flags. */
|
|
62
|
+
interface RawRate {
|
|
63
|
+
input: number;
|
|
64
|
+
output: number;
|
|
65
|
+
cacheRead: number | null;
|
|
66
|
+
cacheWrite: number | null;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** One remote source's last successful fetch. */
|
|
70
|
+
interface SourceTable {
|
|
71
|
+
/** When this source last fetched successfully; null when it never has. */
|
|
72
|
+
fetchedAt: number | null;
|
|
73
|
+
rates: Record<string, RawRate>;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* The cache envelope — also the format of the bundled snapshot, which sets
|
|
78
|
+
* `snapshot: true` so a consumer reading the file can tell the two apart.
|
|
79
|
+
* Sources are stored separately (not pre-merged) so a partial refresh can
|
|
80
|
+
* replace one side while keeping the other's last data.
|
|
81
|
+
*/
|
|
82
|
+
interface PricingCacheFile {
|
|
83
|
+
version: number;
|
|
84
|
+
/** Most recent successful refresh (either source). */
|
|
85
|
+
fetchedAt: number;
|
|
86
|
+
litellm: SourceTable;
|
|
87
|
+
openrouter: SourceTable;
|
|
88
|
+
snapshot?: boolean;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// --- ingest validation -------------------------------------------------------
|
|
92
|
+
|
|
93
|
+
/** A finite, non-negative, plausible price from a float or an OpenRouter
|
|
94
|
+
* string; else null. The ceiling drops absurd rates at ingest, before they
|
|
95
|
+
* can freeze into any rollup. */
|
|
96
|
+
function asPrice(value: unknown): number | null {
|
|
97
|
+
if (typeof value !== "number" && typeof value !== "string") return null;
|
|
98
|
+
if (typeof value === "string" && value.trim() === "") return null;
|
|
99
|
+
const n = Number(value);
|
|
100
|
+
return Number.isFinite(n) && n >= 0 && n <= MAX_RATE_PER_TOKEN ? n : null;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Build a rate from raw fields, or null when input/output don't validate.
|
|
105
|
+
* Cache rates are optional — absent or malformed becomes null, which the
|
|
106
|
+
* resolution contract surfaces to the caller rather than papering over.
|
|
107
|
+
*/
|
|
108
|
+
function toRate(
|
|
109
|
+
input: unknown,
|
|
110
|
+
output: unknown,
|
|
111
|
+
cacheRead: unknown,
|
|
112
|
+
cacheWrite: unknown
|
|
113
|
+
): RawRate | null {
|
|
114
|
+
const inp = asPrice(input);
|
|
115
|
+
const out = asPrice(output);
|
|
116
|
+
if (inp === null || out === null) return null;
|
|
117
|
+
return { input: inp, output: out, cacheRead: asPrice(cacheRead), cacheWrite: asPrice(cacheWrite) };
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** LiteLLM: one big object keyed by model id, per-single-token floats. */
|
|
121
|
+
function parseLitellm(body: unknown): Record<string, RawRate> {
|
|
122
|
+
if (typeof body !== "object" || body === null || Array.isArray(body)) {
|
|
123
|
+
throw new Error("unexpected LiteLLM payload");
|
|
124
|
+
}
|
|
125
|
+
// Null-prototype: ids are foreign strings — a "__proto__" entry must be an
|
|
126
|
+
// ordinary key, never a prototype write (same rule for every table below).
|
|
127
|
+
const rates: Record<string, RawRate> = Object.create(null);
|
|
128
|
+
for (const [id, entry] of Object.entries(body)) {
|
|
129
|
+
// The field-set documentation row, not a model.
|
|
130
|
+
if (id === "sample_spec") continue;
|
|
131
|
+
if (typeof entry !== "object" || entry === null) continue;
|
|
132
|
+
const e = entry as Record<string, unknown>;
|
|
133
|
+
const rate = toRate(
|
|
134
|
+
e.input_cost_per_token,
|
|
135
|
+
e.output_cost_per_token,
|
|
136
|
+
e.cache_read_input_token_cost,
|
|
137
|
+
e.cache_creation_input_token_cost
|
|
138
|
+
);
|
|
139
|
+
// Entries without usable input/output costs (context-window-only rows,
|
|
140
|
+
// malformed values) are dropped — a bad community entry must not be fatal.
|
|
141
|
+
if (rate) rates[id] = rate;
|
|
142
|
+
}
|
|
143
|
+
return rates;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** OpenRouter: `{"data":[...]}`, `vendor/model` ids, string per-token prices. */
|
|
147
|
+
function parseOpenRouter(body: unknown): Record<string, RawRate> {
|
|
148
|
+
const data = (body as { data?: unknown } | null)?.data;
|
|
149
|
+
if (!Array.isArray(data)) throw new Error("unexpected OpenRouter payload");
|
|
150
|
+
const rates: Record<string, RawRate> = Object.create(null);
|
|
151
|
+
for (const row of data) {
|
|
152
|
+
if (typeof row !== "object" || row === null) continue;
|
|
153
|
+
const { id, pricing } = row as { id?: unknown; pricing?: unknown };
|
|
154
|
+
if (typeof id !== "string" || id.length === 0) continue;
|
|
155
|
+
if (typeof pricing !== "object" || pricing === null) continue;
|
|
156
|
+
const p = pricing as Record<string, unknown>;
|
|
157
|
+
const rate = toRate(p.prompt, p.completion, p.input_cache_read, p.input_cache_write);
|
|
158
|
+
if (rate) rates[id] = rate;
|
|
159
|
+
}
|
|
160
|
+
return rates;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// --- fetch -------------------------------------------------------------------
|
|
164
|
+
|
|
165
|
+
/** One GET-and-parse with a hard timeout and a single retry on 5xx / network failure. */
|
|
166
|
+
async function fetchSource(
|
|
167
|
+
url: string,
|
|
168
|
+
parse: (body: unknown) => Record<string, RawRate>,
|
|
169
|
+
fetchImpl: typeof fetch
|
|
170
|
+
): Promise<Record<string, RawRate>> {
|
|
171
|
+
let lastError: unknown;
|
|
172
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
173
|
+
try {
|
|
174
|
+
const res = await fetchImpl(url, {
|
|
175
|
+
headers: { accept: "application/json" },
|
|
176
|
+
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
|
|
177
|
+
});
|
|
178
|
+
if (res.ok) {
|
|
179
|
+
// Size-capped read: an unbounded body from a broken or hostile
|
|
180
|
+
// endpoint must not be streamed into memory (and onward to the brain
|
|
181
|
+
// volume). The declared length rejects early; the actual length
|
|
182
|
+
// catches a body without one. Oversize is terminal, not retryable —
|
|
183
|
+
// the caller serves its last good table.
|
|
184
|
+
const declared = Number(res.headers.get("content-length"));
|
|
185
|
+
if (Number.isFinite(declared) && declared > MAX_RESPONSE_BYTES) {
|
|
186
|
+
lastError = new Error(`response too large (${declared} bytes)`);
|
|
187
|
+
break;
|
|
188
|
+
}
|
|
189
|
+
const text = await res.text();
|
|
190
|
+
if (text.length > MAX_RESPONSE_BYTES) {
|
|
191
|
+
lastError = new Error(`response too large (${text.length} chars)`);
|
|
192
|
+
break;
|
|
193
|
+
}
|
|
194
|
+
return parse(JSON.parse(text));
|
|
195
|
+
}
|
|
196
|
+
lastError = new Error(`HTTP ${res.status}`);
|
|
197
|
+
// 4xx is terminal (moved or removed endpoint) — don't burn a retry.
|
|
198
|
+
if (res.status < 500) break;
|
|
199
|
+
} catch (err) {
|
|
200
|
+
lastError = err;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// --- cache -------------------------------------------------------------------
|
|
207
|
+
|
|
208
|
+
/** Cache path: alongside the discovery and keyterm caches, on the brain volume. */
|
|
209
|
+
export function pricingCachePath(brainPath: string): string {
|
|
210
|
+
return join(brainPath, ".brain-ui", "model-pricing.json");
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** Revalidate a stored table entry by entry; malformed entries are dropped. */
|
|
214
|
+
function sanitizeTable(raw: unknown): SourceTable | null {
|
|
215
|
+
if (typeof raw !== "object" || raw === null) return null;
|
|
216
|
+
const { fetchedAt, rates } = raw as { fetchedAt?: unknown; rates?: unknown };
|
|
217
|
+
if (typeof rates !== "object" || rates === null) return null;
|
|
218
|
+
const clean: Record<string, RawRate> = Object.create(null);
|
|
219
|
+
for (const [id, entry] of Object.entries(rates)) {
|
|
220
|
+
const e = (entry ?? {}) as Record<string, unknown>;
|
|
221
|
+
const rate = toRate(e.input, e.output, e.cacheRead, e.cacheWrite);
|
|
222
|
+
if (rate) clean[id] = rate;
|
|
223
|
+
}
|
|
224
|
+
return { fetchedAt: typeof fetchedAt === "number" ? fetchedAt : null, rates: clean };
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
interface RemoteTables {
|
|
228
|
+
fetchedAt: number;
|
|
229
|
+
litellm: SourceTable;
|
|
230
|
+
openrouter: SourceTable;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
function readCache(path: string): RemoteTables | null {
|
|
234
|
+
try {
|
|
235
|
+
if (!existsSync(path)) return null;
|
|
236
|
+
const parsed = JSON.parse(readFileSync(path, "utf-8")) as PricingCacheFile;
|
|
237
|
+
if (parsed?.version !== CACHE_VERSION || typeof parsed.fetchedAt !== "number") {
|
|
238
|
+
return null;
|
|
239
|
+
}
|
|
240
|
+
const litellm = sanitizeTable(parsed.litellm);
|
|
241
|
+
const openrouter = sanitizeTable(parsed.openrouter);
|
|
242
|
+
if (!litellm || !openrouter) return null;
|
|
243
|
+
return { fetchedAt: parsed.fetchedAt, litellm, openrouter };
|
|
244
|
+
} catch {
|
|
245
|
+
// A corrupt cache is not fatal — the next refresh rewrites it.
|
|
246
|
+
return null;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* Atomic cache write: temp file + rename, so a concurrent writer (server and
|
|
252
|
+
* cron wrapper share this file) or a crash mid-write can never leave a torn
|
|
253
|
+
* file behind — the reader sees the old cache or the new one, nothing between.
|
|
254
|
+
* Returns the failure message instead of throwing: a read-only volume costs
|
|
255
|
+
* persistence across restarts, nothing else — the in-memory table keeps
|
|
256
|
+
* serving — but the failure must be observable in state(). (No console here
|
|
257
|
+
* by package rule.)
|
|
258
|
+
*/
|
|
259
|
+
function writeCache(path: string, cache: PricingCacheFile): string | null {
|
|
260
|
+
const tmp = `${path}.tmp.${process.pid}`;
|
|
261
|
+
try {
|
|
262
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
263
|
+
writeFileSync(tmp, JSON.stringify(cache, null, 2), "utf-8");
|
|
264
|
+
renameSync(tmp, path);
|
|
265
|
+
return null;
|
|
266
|
+
} catch (err) {
|
|
267
|
+
try {
|
|
268
|
+
rmSync(tmp, { force: true });
|
|
269
|
+
} catch {
|
|
270
|
+
// Best-effort cleanup; the stray temp file is harmless.
|
|
271
|
+
}
|
|
272
|
+
return err instanceof Error ? err.message : String(err);
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* The bundled snapshot ships inside the package (`data/` is in `files`).
|
|
278
|
+
* src/pricing/ and dist/pricing/ sit at the same depth, so the relative hop
|
|
279
|
+
* to the package root resolves identically on both resolution paths — the
|
|
280
|
+
* same pattern the migrations dir uses in src/db/client.ts.
|
|
281
|
+
*/
|
|
282
|
+
function readSnapshot(): Record<string, RawRate> {
|
|
283
|
+
try {
|
|
284
|
+
const path = join(import.meta.dir, "../../data/model-prices.json");
|
|
285
|
+
const parsed = JSON.parse(readFileSync(path, "utf-8")) as PricingCacheFile;
|
|
286
|
+
if (parsed?.version !== CACHE_VERSION) return {};
|
|
287
|
+
const litellm = sanitizeTable(parsed.litellm);
|
|
288
|
+
const openrouter = sanitizeTable(parsed.openrouter);
|
|
289
|
+
return Object.assign(
|
|
290
|
+
Object.create(null),
|
|
291
|
+
litellm?.rates ?? {},
|
|
292
|
+
openrouter?.rates ?? {}
|
|
293
|
+
);
|
|
294
|
+
} catch {
|
|
295
|
+
// A missing snapshot only narrows the fallback to "nothing resolves",
|
|
296
|
+
// which every caller already handles as unknown.
|
|
297
|
+
return {};
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
// --- pricing service ---------------------------------------------------------
|
|
302
|
+
|
|
303
|
+
export interface ModelPricingOptions {
|
|
304
|
+
/** Brain repo path; the cache lives under its `.brain-ui/` directory. */
|
|
305
|
+
brainPath: string;
|
|
306
|
+
/** How long a fetched table is considered fresh. Default 24h. */
|
|
307
|
+
ttlMs?: number;
|
|
308
|
+
/** Set false to disable pricing entirely (resolve() always null). */
|
|
309
|
+
enabled?: boolean;
|
|
310
|
+
/** @internal Test seam — inject fetch. */
|
|
311
|
+
fetchImpl?: typeof fetch;
|
|
312
|
+
/** @internal Test seam — inject the clock. */
|
|
313
|
+
now?: () => number;
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
export interface ModelPricingState {
|
|
317
|
+
enabled: boolean;
|
|
318
|
+
/** When a refresh last succeeded (either source); null when none ever has. */
|
|
319
|
+
fetchedAt: number | null;
|
|
320
|
+
/** The current table is older than the TTL (or was never fetched). */
|
|
321
|
+
stale: boolean;
|
|
322
|
+
/** What the table is served from: remote data (cache included) or the snapshot. */
|
|
323
|
+
source: "remote" | "snapshot";
|
|
324
|
+
/** Per-source freshness — a partial refresh leaves the failed side behind. */
|
|
325
|
+
litellmFetchedAt: number | null;
|
|
326
|
+
openrouterFetchedAt: number | null;
|
|
327
|
+
/** Last refresh failure, if the current table is served despite one. */
|
|
328
|
+
error?: string;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* A lazily-refreshed model-price table.
|
|
333
|
+
*
|
|
334
|
+
* `resolve()` is synchronous and never touches the network, so the rollup
|
|
335
|
+
* transaction can price a run without blocking on a catalog being reachable.
|
|
336
|
+
* `ensureFresh()` awaits only on a cold start (no remote data at all — the
|
|
337
|
+
* snapshot still serves through the wait) and otherwise refreshes behind the
|
|
338
|
+
* current table.
|
|
339
|
+
*/
|
|
340
|
+
export interface ModelPricing {
|
|
341
|
+
resolve(modelId: string): PricingRates | null;
|
|
342
|
+
state(): ModelPricingState;
|
|
343
|
+
/** Refresh if stale. Awaits only when no remote data has ever been fetched. */
|
|
344
|
+
ensureFresh(): Promise<void>;
|
|
345
|
+
/** Force a refresh regardless of TTL. Rejects only when BOTH sources fail. */
|
|
346
|
+
refresh(): Promise<void>;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
export function createModelPricing(options: ModelPricingOptions): ModelPricing {
|
|
350
|
+
const ttlMs = options.ttlMs ?? DEFAULT_TTL_MS;
|
|
351
|
+
const enabled = options.enabled ?? true;
|
|
352
|
+
const now = options.now ?? Date.now;
|
|
353
|
+
const fetchImpl = options.fetchImpl ?? fetch;
|
|
354
|
+
const cachePath = pricingCachePath(options.brainPath);
|
|
355
|
+
|
|
356
|
+
// Remote data (disk cache or a completed refresh) and the bundled snapshot
|
|
357
|
+
// stay separate: the snapshot serves only while no remote table exists, and
|
|
358
|
+
// its resolutions are flagged estimate — mixing the two would silently
|
|
359
|
+
// launder dated rates as fresh ones.
|
|
360
|
+
let remote: RemoteTables | null = enabled ? readCache(cachePath) : null;
|
|
361
|
+
const snapshot: Record<string, RawRate> = enabled ? readSnapshot() : {};
|
|
362
|
+
let lastError: string | undefined;
|
|
363
|
+
let lastWriteError: string | undefined;
|
|
364
|
+
let inFlight: Promise<void> | null = null;
|
|
365
|
+
|
|
366
|
+
// Staleness keys on the OLDEST source, not the envelope: a partial refresh
|
|
367
|
+
// (one catalog up, one down) must keep retrying the failed side on the
|
|
368
|
+
// normal ensureFresh cadence instead of hiding behind the winner's
|
|
369
|
+
// timestamp for a full TTL — which would also suppress the client's
|
|
370
|
+
// staleness indicator exactly while one source is dark.
|
|
371
|
+
const isStale = () => {
|
|
372
|
+
if (remote === null) return true;
|
|
373
|
+
const oldest = Math.min(
|
|
374
|
+
remote.litellm.fetchedAt ?? 0,
|
|
375
|
+
remote.openrouter.fetchedAt ?? 0
|
|
376
|
+
);
|
|
377
|
+
return now() - oldest > ttlMs;
|
|
378
|
+
};
|
|
379
|
+
|
|
380
|
+
async function doRefresh(): Promise<void> {
|
|
381
|
+
const [litellm, openrouter] = await Promise.allSettled([
|
|
382
|
+
fetchSource(LITELLM_URL, parseLitellm, fetchImpl),
|
|
383
|
+
fetchSource(OPENROUTER_URL, parseOpenRouter, fetchImpl),
|
|
384
|
+
]);
|
|
385
|
+
|
|
386
|
+
const failures: string[] = [];
|
|
387
|
+
const describe = (reason: unknown) =>
|
|
388
|
+
reason instanceof Error ? reason.message : String(reason);
|
|
389
|
+
if (litellm.status === "rejected") failures.push(`litellm: ${describe(litellm.reason)}`);
|
|
390
|
+
if (openrouter.status === "rejected")
|
|
391
|
+
failures.push(`openrouter: ${describe(openrouter.reason)}`);
|
|
392
|
+
|
|
393
|
+
if (litellm.status === "rejected" && openrouter.status === "rejected") {
|
|
394
|
+
// Keep serving whatever we already had; surface the reason.
|
|
395
|
+
lastError = failures.join("; ");
|
|
396
|
+
throw new Error(lastError);
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
// Partial refresh is still a refresh: the failed source keeps its last
|
|
400
|
+
// table (empty if it never succeeded) and shows through its fetchedAt
|
|
401
|
+
// plus state().error.
|
|
402
|
+
remote = {
|
|
403
|
+
fetchedAt: now(),
|
|
404
|
+
litellm:
|
|
405
|
+
litellm.status === "fulfilled"
|
|
406
|
+
? { fetchedAt: now(), rates: litellm.value }
|
|
407
|
+
: (remote?.litellm ?? { fetchedAt: null, rates: {} }),
|
|
408
|
+
openrouter:
|
|
409
|
+
openrouter.status === "fulfilled"
|
|
410
|
+
? { fetchedAt: now(), rates: openrouter.value }
|
|
411
|
+
: (remote?.openrouter ?? { fetchedAt: null, rates: {} }),
|
|
412
|
+
};
|
|
413
|
+
lastError = failures.length > 0 ? failures.join("; ") : undefined;
|
|
414
|
+
lastWriteError =
|
|
415
|
+
writeCache(cachePath, { version: CACHE_VERSION, ...remote }) ?? undefined;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
function refresh(): Promise<void> {
|
|
419
|
+
if (!enabled) return Promise.resolve();
|
|
420
|
+
// Single-flight: a burst of requests triggers one fetch pair, not N.
|
|
421
|
+
if (!inFlight) {
|
|
422
|
+
inFlight = doRefresh().finally(() => {
|
|
423
|
+
inFlight = null;
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
return inFlight;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
function lookup(id: string): PricingRates | null {
|
|
430
|
+
if (remote) {
|
|
431
|
+
// OpenRouter wins for ids its catalog carries — an openrouter-routed
|
|
432
|
+
// run is billed at OpenRouter's rate, not LiteLLM's idea of it.
|
|
433
|
+
// Own-property lookups only: an id like "toString" must never resolve
|
|
434
|
+
// a prototype member as phantom rates.
|
|
435
|
+
const or = Object.hasOwn(remote.openrouter.rates, id)
|
|
436
|
+
? remote.openrouter.rates[id]
|
|
437
|
+
: undefined;
|
|
438
|
+
if (or) return { ...or, estimate: false, source: "openrouter" };
|
|
439
|
+
const ll = Object.hasOwn(remote.litellm.rates, id)
|
|
440
|
+
? remote.litellm.rates[id]
|
|
441
|
+
: undefined;
|
|
442
|
+
if (ll) return { ...ll, estimate: false, source: "litellm" };
|
|
443
|
+
return null;
|
|
444
|
+
}
|
|
445
|
+
const snap = Object.hasOwn(snapshot, id) ? snapshot[id] : undefined;
|
|
446
|
+
return snap ? { ...snap, estimate: true, source: "snapshot" } : null;
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
const resolveId = (id: string) => lookup(id) ?? lookup(canonicalModelId(id));
|
|
450
|
+
|
|
451
|
+
return {
|
|
452
|
+
resolve(modelId: string): PricingRates | null {
|
|
453
|
+
if (!enabled) return null;
|
|
454
|
+
const direct = resolveId(modelId);
|
|
455
|
+
if (direct) return direct;
|
|
456
|
+
// An OpenRouter routing suffix (`:nitro` / `:floor`) is a request-time
|
|
457
|
+
// shortcut with no catalog price of its own — price at the base id's
|
|
458
|
+
// rate, flagged ESTIMATE regardless of source: the routed premium is in
|
|
459
|
+
// no catalog (AE2).
|
|
460
|
+
const variant = modelId.match(/^(.+):(nitro|floor)$/);
|
|
461
|
+
if (variant) {
|
|
462
|
+
const base = resolveId(variant[1]!);
|
|
463
|
+
if (base) return { ...base, estimate: true };
|
|
464
|
+
}
|
|
465
|
+
return null;
|
|
466
|
+
},
|
|
467
|
+
state: () => {
|
|
468
|
+
// Cache-write failures ride the same error field as fetch failures —
|
|
469
|
+
// both mean "the table you see may not survive a restart / refresh".
|
|
470
|
+
const errors = [
|
|
471
|
+
lastError,
|
|
472
|
+
lastWriteError !== undefined ? `cache write failed: ${lastWriteError}` : undefined,
|
|
473
|
+
].filter((e): e is string => e !== undefined);
|
|
474
|
+
return {
|
|
475
|
+
enabled,
|
|
476
|
+
fetchedAt: remote?.fetchedAt ?? null,
|
|
477
|
+
stale: enabled ? isStale() : false,
|
|
478
|
+
source: remote ? "remote" : "snapshot",
|
|
479
|
+
litellmFetchedAt: remote?.litellm.fetchedAt ?? null,
|
|
480
|
+
openrouterFetchedAt: remote?.openrouter.fetchedAt ?? null,
|
|
481
|
+
...(errors.length > 0 ? { error: errors.join("; ") } : {}),
|
|
482
|
+
};
|
|
483
|
+
},
|
|
484
|
+
async ensureFresh() {
|
|
485
|
+
if (!enabled || !isStale()) return;
|
|
486
|
+
// Cold start: no remote data yet, so the caller waits (bounded by the
|
|
487
|
+
// request timeout); the snapshot serves anyone who won't. Otherwise
|
|
488
|
+
// refresh in the background.
|
|
489
|
+
if (remote === null) {
|
|
490
|
+
await refresh().catch(() => {});
|
|
491
|
+
return;
|
|
492
|
+
}
|
|
493
|
+
void refresh().catch(() => {});
|
|
494
|
+
},
|
|
495
|
+
refresh,
|
|
496
|
+
};
|
|
497
|
+
}
|
package/src/routes/activity.ts
CHANGED
|
@@ -5,12 +5,11 @@ import {
|
|
|
5
5
|
type ActivityAggregate,
|
|
6
6
|
type ActivityRollups,
|
|
7
7
|
type ActivityRunDetail,
|
|
8
|
-
type ActivityRunRollup,
|
|
9
8
|
type ActivityRunSummary,
|
|
10
9
|
} from "@schlessera/brain-ui-sdk/protocol";
|
|
11
10
|
|
|
12
11
|
import { rowToRunRollup, type ActivityStore } from "../activity/store.js";
|
|
13
|
-
import { toWireEvent, toWireSpan } from "../activity/stream.js";
|
|
12
|
+
import { toWireEvent, toWireRollup, toWireSpan } from "../activity/stream.js";
|
|
14
13
|
import type { ActivityNotifier } from "../activity/notify.js";
|
|
15
14
|
import {
|
|
16
15
|
dismissActivityDigest,
|
|
@@ -144,6 +143,8 @@ export function createActivityRoutes(deps: {
|
|
|
144
143
|
running: true,
|
|
145
144
|
durationMs: null,
|
|
146
145
|
costUsd: s.usage.costUsd ?? null,
|
|
146
|
+
// No rollup yet — effective cost is unknown until the run prices.
|
|
147
|
+
effectiveCostUsd: null,
|
|
147
148
|
failureReason: null,
|
|
148
149
|
detailPruned: false,
|
|
149
150
|
}));
|
|
@@ -179,19 +180,13 @@ export function createActivityRoutes(deps: {
|
|
|
179
180
|
.all(...(params as never[]), limit) as any[]
|
|
180
181
|
)
|
|
181
182
|
.map(rowToRunRollup)
|
|
183
|
+
// The wire mapping (explicit-null vs omit) lives in toWireRollup;
|
|
184
|
+
// a summary adds only its identity and liveness fields on top.
|
|
182
185
|
.map((r) => ({
|
|
183
|
-
|
|
186
|
+
...toWireRollup(r),
|
|
184
187
|
origin: r.origin,
|
|
185
|
-
|
|
186
|
-
sessionId: r.sessionId,
|
|
187
|
-
jobName: r.jobName,
|
|
188
|
-
startedAt: r.startedAt,
|
|
189
|
-
endedAt: r.endedAt,
|
|
190
|
-
outcome: r.outcome,
|
|
188
|
+
runId: r.runId,
|
|
191
189
|
running: false,
|
|
192
|
-
durationMs: r.durationMs,
|
|
193
|
-
costUsd: r.costUsd,
|
|
194
|
-
failureReason: r.failureReason,
|
|
195
190
|
detailPruned: r.detailPruned,
|
|
196
191
|
}));
|
|
197
192
|
|
|
@@ -208,39 +203,40 @@ export function createActivityRoutes(deps: {
|
|
|
208
203
|
const runId = c.req.param("runId");
|
|
209
204
|
try {
|
|
210
205
|
const snapshot = store.snapshotRun(runId);
|
|
206
|
+
const row = db
|
|
207
|
+
.query("SELECT * FROM activity_run_rollups WHERE run_id = ?")
|
|
208
|
+
.get(runId) as any;
|
|
211
209
|
if (snapshot) {
|
|
210
|
+
// Tool payload events (up to 16 KB each) dominate a run's byte
|
|
211
|
+
// size, and the history list opens details eagerly — so the REST
|
|
212
|
+
// detail ships WITHOUT them unless the drill-in asks
|
|
213
|
+
// (?include=payloads). The WS snapshot path is untouched: a live
|
|
214
|
+
// subscription already committed to the full stream.
|
|
215
|
+
const includePayloads = c.req.query("include") === "payloads";
|
|
216
|
+
const events = includePayloads
|
|
217
|
+
? snapshot.events
|
|
218
|
+
: snapshot.events.filter(
|
|
219
|
+
(e) => e.eventType !== "tool_input" && e.eventType !== "tool_output"
|
|
220
|
+
);
|
|
212
221
|
// Through the SAME wire mappers the live stream uses: a raw
|
|
213
222
|
// SpanRow serializes null fields where the wire contract omits
|
|
214
223
|
// them, which broke the client's `outcome === undefined`
|
|
215
|
-
// liveness test on REST-loaded runs.
|
|
224
|
+
// liveness test on REST-loaded runs. The rollup rides along when
|
|
225
|
+
// it exists (finished runs) so the detail view can show cost facts
|
|
226
|
+
// without waiting for detail pruning to force the rollup-only path.
|
|
216
227
|
const detail: ActivityRunDetail = {
|
|
217
228
|
runId,
|
|
218
229
|
detailPruned: false,
|
|
219
230
|
spans: snapshot.spans.map(toWireSpan),
|
|
220
|
-
events:
|
|
231
|
+
events: events.map(toWireEvent),
|
|
221
232
|
highWaterSeq: snapshot.highWaterSeq,
|
|
233
|
+
...(row ? { rollup: toWireRollup(rowToRunRollup(row)) } : {}),
|
|
222
234
|
};
|
|
223
235
|
return c.json(detail);
|
|
224
236
|
}
|
|
225
|
-
const row = db
|
|
226
|
-
.query("SELECT * FROM activity_run_rollups WHERE run_id = ?")
|
|
227
|
-
.get(runId) as any;
|
|
228
237
|
if (!row) return c.json({ error: "Unknown run" }, 404);
|
|
229
238
|
const r = rowToRunRollup(row);
|
|
230
|
-
const
|
|
231
|
-
origin: r.origin,
|
|
232
|
-
name: r.name,
|
|
233
|
-
sessionId: r.sessionId,
|
|
234
|
-
jobName: r.jobName,
|
|
235
|
-
startedAt: r.startedAt,
|
|
236
|
-
endedAt: r.endedAt,
|
|
237
|
-
outcome: r.outcome,
|
|
238
|
-
durationMs: r.durationMs,
|
|
239
|
-
spanCount: r.spanCount,
|
|
240
|
-
costUsd: r.costUsd,
|
|
241
|
-
failureReason: r.failureReason,
|
|
242
|
-
};
|
|
243
|
-
const detail: ActivityRunDetail = { runId, detailPruned: true, rollup };
|
|
239
|
+
const detail: ActivityRunDetail = { runId, detailPruned: true, rollup: toWireRollup(r) };
|
|
244
240
|
return c.json(detail);
|
|
245
241
|
} catch (err) {
|
|
246
242
|
return c.json(
|
|
@@ -263,8 +259,8 @@ export function createActivityRoutes(deps: {
|
|
|
263
259
|
// JS because only Intl knows the configured timezone's day boundary.
|
|
264
260
|
const rows = db
|
|
265
261
|
.query(
|
|
266
|
-
`SELECT started_at, outcome, duration_ms, cost_usd,
|
|
267
|
-
output_tokens, cache_read_tokens, cache_creation_tokens
|
|
262
|
+
`SELECT started_at, outcome, duration_ms, cost_usd, effective_cost_usd,
|
|
263
|
+
input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens
|
|
268
264
|
FROM activity_run_rollups WHERE started_at >= ?`
|
|
269
265
|
)
|
|
270
266
|
.all(since) as any[];
|
|
@@ -313,6 +309,8 @@ function groupedAggregates(
|
|
|
313
309
|
COUNT(*) AS runs,
|
|
314
310
|
SUM(CASE WHEN outcome IN ('error', 'timeout', 'interrupted') THEN 1 ELSE 0 END) AS failures,
|
|
315
311
|
SUM(COALESCE(cost_usd, 0)) AS costUsd,
|
|
312
|
+
SUM(COALESCE(effective_cost_usd, 0)) AS effectiveCostUsd,
|
|
313
|
+
SUM(CASE WHEN effective_cost_usd IS NULL THEN 1 ELSE 0 END) AS unpricedRuns,
|
|
316
314
|
SUM(COALESCE(input_tokens, 0)) AS inputTokens,
|
|
317
315
|
SUM(COALESCE(output_tokens, 0)) AS outputTokens,
|
|
318
316
|
SUM(COALESCE(cache_read_tokens, 0)) AS cacheReadTokens,
|
|
@@ -332,6 +330,8 @@ function add(map: Map<string, ActivityAggregate>, key: string, r: any): void {
|
|
|
332
330
|
runs: 0,
|
|
333
331
|
failures: 0,
|
|
334
332
|
costUsd: 0,
|
|
333
|
+
effectiveCostUsd: 0,
|
|
334
|
+
unpricedRuns: 0,
|
|
335
335
|
inputTokens: 0,
|
|
336
336
|
outputTokens: 0,
|
|
337
337
|
cacheReadTokens: 0,
|
|
@@ -342,7 +342,15 @@ function add(map: Map<string, ActivityAggregate>, key: string, r: any): void {
|
|
|
342
342
|
if (isFailureOutcome(r.outcome)) {
|
|
343
343
|
a.failures += 1;
|
|
344
344
|
}
|
|
345
|
+
// Both cost sums are sum-of-KNOWNS (a missing value contributes nothing);
|
|
346
|
+
// for effective cost the excluded rows are counted so no surface can pass
|
|
347
|
+
// an unknown off as $0 (AE3).
|
|
345
348
|
a.costUsd += r.cost_usd ?? 0;
|
|
349
|
+
if (r.effective_cost_usd == null) {
|
|
350
|
+
a.unpricedRuns = (a.unpricedRuns ?? 0) + 1;
|
|
351
|
+
} else {
|
|
352
|
+
a.effectiveCostUsd = (a.effectiveCostUsd ?? 0) + r.effective_cost_usd;
|
|
353
|
+
}
|
|
346
354
|
a.inputTokens += r.input_tokens ?? 0;
|
|
347
355
|
a.outputTokens += r.output_tokens ?? 0;
|
|
348
356
|
a.cacheReadTokens += r.cache_read_tokens ?? 0;
|