@trazum/core 1.81.0 → 1.82.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -18
- package/dist/advisories.d.ts +33 -1
- package/dist/advisories.d.ts.map +1 -1
- package/dist/advisories.js +103 -16
- package/dist/advisories.js.map +1 -1
- package/dist/band.d.ts +131 -0
- package/dist/band.d.ts.map +1 -0
- package/dist/band.js +210 -0
- package/dist/band.js.map +1 -0
- package/dist/conform.d.ts +1 -1
- package/dist/conform.d.ts.map +1 -1
- package/dist/conform.js +84 -1
- package/dist/conform.js.map +1 -1
- package/dist/helicone.d.ts +92 -0
- package/dist/helicone.d.ts.map +1 -0
- package/dist/helicone.js +194 -0
- package/dist/helicone.js.map +1 -0
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +9 -3
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/es.d.ts.map +1 -1
- package/dist/i18n/es.js +9 -3
- package/dist/i18n/es.js.map +1 -1
- package/dist/i18n/types.d.ts +43 -1
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/index.d.ts +10 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/litellm.d.ts +98 -0
- package/dist/litellm.d.ts.map +1 -0
- package/dist/litellm.js +207 -0
- package/dist/litellm.js.map +1 -0
- package/dist/measured-profile.d.ts +4 -0
- package/dist/measured-profile.d.ts.map +1 -1
- package/dist/measured-profile.js +22 -1
- package/dist/measured-profile.js.map +1 -1
- package/dist/schemas.d.ts.map +1 -1
- package/dist/schemas.js +36 -0
- package/dist/schemas.js.map +1 -1
- package/dist/tokenizer.d.ts +45 -49
- package/dist/tokenizer.d.ts.map +1 -1
- package/dist/tokenizer.js +100 -51
- package/dist/tokenizer.js.map +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/verdict-bridge.d.ts +113 -0
- package/dist/verdict-bridge.d.ts.map +1 -0
- package/dist/verdict-bridge.js +127 -0
- package/dist/verdict-bridge.js.map +1 -0
- package/package.json +1 -1
- package/src/advisories.ts +132 -15
- package/src/band.ts +213 -0
- package/src/conform.ts +100 -1
- package/src/helicone.ts +227 -0
- package/src/i18n/en.ts +10 -3
- package/src/i18n/es.ts +10 -3
- package/src/i18n/types.ts +45 -1
- package/src/index.ts +17 -1
- package/src/litellm.ts +243 -0
- package/src/measured-profile.ts +26 -2
- package/src/schemas.ts +36 -0
- package/src/tokenizer.ts +101 -51
- package/src/types.ts +1 -0
- package/src/verdict-bridge.ts +200 -0
package/src/helicone.ts
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Helicone's request export, read as a usage log.
|
|
3
|
+
*
|
|
4
|
+
* The third converter in the pattern `from-claude-code` started: a pure
|
|
5
|
+
* function turns one tool's export into Trazum's usage-log records, and every
|
|
6
|
+
* door prices it from there. Helicone sits as a proxy in front of the provider
|
|
7
|
+
* and keeps every request it saw, so a team using it already has the export.
|
|
8
|
+
*
|
|
9
|
+
* **The format is derived, not guessed.** The columns below are the SELECT
|
|
10
|
+
* that builds Helicone's own request table — `web/lib/api/request/request.ts`
|
|
11
|
+
* in Helicone/helicone — and the response shape its `POST /v1/request/query`
|
|
12
|
+
* endpoint documents. A converter written from memory of an API is a converter
|
|
13
|
+
* that silently mis-reads somebody's bill.
|
|
14
|
+
*
|
|
15
|
+
* ## The model, and why it takes three columns to answer
|
|
16
|
+
*
|
|
17
|
+
* Helicone carries `request_model`, `model_override` and `response_model`, and
|
|
18
|
+
* they can disagree: the override exists precisely because a proxy can send a
|
|
19
|
+
* different model than the caller asked for, and the response says what
|
|
20
|
+
* actually answered. **The response wins**, then the override, then the
|
|
21
|
+
* request — the bill is about what was billed, not what was intended — and
|
|
22
|
+
* `modelDisagreements` counts the rows where they differed so a reader can see
|
|
23
|
+
* the substitution happened rather than discover it in a total.
|
|
24
|
+
*
|
|
25
|
+
* ## What deliberately does not cross
|
|
26
|
+
*
|
|
27
|
+
* A Helicone row carries the request body and the response body — the prompt
|
|
28
|
+
* and the completion — and `request_user_id`, which is an email address in
|
|
29
|
+
* Helicone's own documented example. **None of it is read.** The converter
|
|
30
|
+
* names the fields it takes and takes nothing else, and a fixture plants a
|
|
31
|
+
* marker in each and greps the whole output.
|
|
32
|
+
*
|
|
33
|
+
* ## What it refuses to invent
|
|
34
|
+
*
|
|
35
|
+
* There is no cache-token split anywhere on the row: `cache_enabled` is a flag
|
|
36
|
+
* on the analytics table. A converted record therefore carries no cache fields
|
|
37
|
+
* and the caching questions come back `cannot-tell` rather than answered from
|
|
38
|
+
* a guess — the same refusal `from-otel` and `from-litellm` make.
|
|
39
|
+
*
|
|
40
|
+
* There is no conversation identity either. `request_id` is one call, so the
|
|
41
|
+
* records carry no `session` and the conversation-shaped findings stay
|
|
42
|
+
* unavailable. A custom property can carry one, and then it is there because
|
|
43
|
+
* the operator put it there — never because this file inferred it.
|
|
44
|
+
*/
|
|
45
|
+
|
|
46
|
+
/** One converted record, shaped exactly as `parseUsageLine` reads it. */
|
|
47
|
+
export interface HeliconeRecord {
|
|
48
|
+
model: string;
|
|
49
|
+
ts?: string;
|
|
50
|
+
label?: string;
|
|
51
|
+
usage: {
|
|
52
|
+
input_tokens: number;
|
|
53
|
+
output_tokens: number;
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export interface HeliconeConversion {
|
|
58
|
+
records: HeliconeRecord[];
|
|
59
|
+
/** Rows that were requests and converted. */
|
|
60
|
+
rows: number;
|
|
61
|
+
/** Rows naming no model in any of the three columns: counted, never guessed. */
|
|
62
|
+
unnamedModel: number;
|
|
63
|
+
/**
|
|
64
|
+
* Rows where the model that answered was not the model that was asked for.
|
|
65
|
+
*
|
|
66
|
+
* The response's model is what the bill rests on, and a proxy substituting
|
|
67
|
+
* one model for another is a fact worth seeing rather than a difference
|
|
68
|
+
* that only shows up as an unexplained total.
|
|
69
|
+
*/
|
|
70
|
+
modelDisagreements: number;
|
|
71
|
+
/** Rows carrying no token counts at all — a logged request nobody can price. */
|
|
72
|
+
noTokens: number;
|
|
73
|
+
/** Rows Helicone served from its cache, with no token split behind the flag. */
|
|
74
|
+
cacheFlagged: number;
|
|
75
|
+
/** Lines or documents that did not parse as JSON at all. */
|
|
76
|
+
unparseable: number;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const asObject = (value: unknown): Record<string, unknown> | null =>
|
|
80
|
+
typeof value === 'object' && value !== null && !Array.isArray(value)
|
|
81
|
+
? (value as Record<string, unknown>)
|
|
82
|
+
: null;
|
|
83
|
+
|
|
84
|
+
const tryParse = (text: string): unknown => {
|
|
85
|
+
try {
|
|
86
|
+
return JSON.parse(text);
|
|
87
|
+
} catch {
|
|
88
|
+
return undefined;
|
|
89
|
+
}
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
const asCount = (value: unknown): number | undefined => {
|
|
93
|
+
const n = typeof value === 'string' ? Number(value) : value;
|
|
94
|
+
return typeof n === 'number' && Number.isFinite(n) && n >= 0 ? n : undefined;
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
const asName = (value: unknown): string | null =>
|
|
98
|
+
typeof value === 'string' && value.trim() !== '' ? value.trim() : null;
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* The label, from Helicone's custom properties.
|
|
102
|
+
*
|
|
103
|
+
* `request_properties` is the documented place for whatever a team tags its
|
|
104
|
+
* calls with, and the keys are theirs. Trazum looks for the ones a workload
|
|
105
|
+
* would plausibly be named by, in order, and takes the first — never joins
|
|
106
|
+
* several, because a workload has one name in a bill.
|
|
107
|
+
*/
|
|
108
|
+
function labelOf(row: Record<string, unknown>): string | undefined {
|
|
109
|
+
const properties = asObject(row.request_properties) ?? asObject(row.properties);
|
|
110
|
+
if (properties === null) return undefined;
|
|
111
|
+
for (const key of ['Helicone-Property-Label', 'label', 'workload', 'app', 'environment']) {
|
|
112
|
+
const found = asName(properties[key]);
|
|
113
|
+
if (found !== null) return found;
|
|
114
|
+
}
|
|
115
|
+
return undefined;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** The request's creation time as an ISO instant, or undefined. */
|
|
119
|
+
function timeOf(row: Record<string, unknown>): string | undefined {
|
|
120
|
+
const raw = row.request_created_at ?? row.created_at ?? row.response_created_at;
|
|
121
|
+
if (typeof raw !== 'string' && typeof raw !== 'number') return undefined;
|
|
122
|
+
const ms = typeof raw === 'number' ? raw : Date.parse(raw);
|
|
123
|
+
if (!Number.isFinite(ms)) return undefined;
|
|
124
|
+
return new Date(ms).toISOString();
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Convert a Helicone request export.
|
|
129
|
+
*
|
|
130
|
+
* Accepts a JSON array of rows, a single row, `{ data: [...] }` as the query
|
|
131
|
+
* endpoint returns it, or newline-delimited rows. Pure over its input.
|
|
132
|
+
*/
|
|
133
|
+
export function heliconeRecords(text: string): HeliconeConversion {
|
|
134
|
+
const out: HeliconeConversion = {
|
|
135
|
+
records: [],
|
|
136
|
+
rows: 0,
|
|
137
|
+
unnamedModel: 0,
|
|
138
|
+
modelDisagreements: 0,
|
|
139
|
+
noTokens: 0,
|
|
140
|
+
cacheFlagged: 0,
|
|
141
|
+
unparseable: 0,
|
|
142
|
+
};
|
|
143
|
+
|
|
144
|
+
const rows: unknown[] = [];
|
|
145
|
+
const whole = tryParse(text);
|
|
146
|
+
if (whole !== undefined) {
|
|
147
|
+
if (Array.isArray(whole)) rows.push(...whole);
|
|
148
|
+
else {
|
|
149
|
+
const doc = asObject(whole);
|
|
150
|
+
if (doc !== null && Array.isArray(doc.data)) rows.push(...doc.data);
|
|
151
|
+
else if (doc !== null) rows.push(doc);
|
|
152
|
+
}
|
|
153
|
+
} else {
|
|
154
|
+
for (const line of text.split('\n')) {
|
|
155
|
+
if (line.trim() === '') continue;
|
|
156
|
+
const parsed = tryParse(line);
|
|
157
|
+
if (parsed === undefined) out.unparseable += 1;
|
|
158
|
+
else if (Array.isArray(parsed)) rows.push(...parsed);
|
|
159
|
+
else rows.push(parsed);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
for (const raw of rows) {
|
|
164
|
+
const row = asObject(raw);
|
|
165
|
+
if (row === null) continue;
|
|
166
|
+
const input = asCount(row.prompt_tokens);
|
|
167
|
+
const output = asCount(row.completion_tokens);
|
|
168
|
+
// A request row is one carrying at least one token column. Anything else
|
|
169
|
+
// in the same file is not a call and is skipped rather than recorded as a
|
|
170
|
+
// zero, which would be a call nobody made.
|
|
171
|
+
if (input === undefined && output === undefined) continue;
|
|
172
|
+
|
|
173
|
+
out.rows += 1;
|
|
174
|
+
if (input === 0 && output === 0) out.noTokens += 1;
|
|
175
|
+
if (row.cache_enabled === true || row.cache_enabled === 1 || row.cache_enabled === '1') {
|
|
176
|
+
out.cacheFlagged += 1;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/*
|
|
180
|
+
The response first, and this order is the whole point of counting the
|
|
181
|
+
disagreement. A proxy that substituted a model billed for the model that
|
|
182
|
+
answered, so that is what a bill has to price; taking the requested one
|
|
183
|
+
would report the intention and call it the cost.
|
|
184
|
+
*/
|
|
185
|
+
const answered = asName(row.response_model);
|
|
186
|
+
const overridden = asName(row.model_override);
|
|
187
|
+
const requested = asName(row.request_model);
|
|
188
|
+
const model = answered ?? overridden ?? requested;
|
|
189
|
+
if (model === null) {
|
|
190
|
+
out.unnamedModel += 1;
|
|
191
|
+
continue;
|
|
192
|
+
}
|
|
193
|
+
if (requested !== null && answered !== null && requested !== answered) {
|
|
194
|
+
out.modelDisagreements += 1;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
const ts = timeOf(row);
|
|
198
|
+
const label = labelOf(row);
|
|
199
|
+
out.records.push({
|
|
200
|
+
model,
|
|
201
|
+
...(ts !== undefined ? { ts } : {}),
|
|
202
|
+
...(label !== undefined ? { label } : {}),
|
|
203
|
+
usage: { input_tokens: input ?? 0, output_tokens: output ?? 0 },
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
return out;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* Whether a file looks like a Helicone request export, by shape.
|
|
212
|
+
*
|
|
213
|
+
* Deliberately narrow: a token column beside one of the names only Helicone's
|
|
214
|
+
* own SELECT produces. A looser test would claim an OpenAI usage response or a
|
|
215
|
+
* LiteLLM spend log — both of which also carry `prompt_tokens` — and refuse it
|
|
216
|
+
* in this file's words instead of letting the code that can read it try.
|
|
217
|
+
*/
|
|
218
|
+
export function looksLikeHelicone(text: string, prefixBytes = 8192): boolean {
|
|
219
|
+
const head = text.slice(0, prefixBytes);
|
|
220
|
+
if (!head.includes('prompt_tokens') && !head.includes('completion_tokens')) return false;
|
|
221
|
+
return (
|
|
222
|
+
head.includes('"request_created_at"') ||
|
|
223
|
+
head.includes('"request_model"') ||
|
|
224
|
+
head.includes('"response_model"') ||
|
|
225
|
+
head.includes('"request_properties"')
|
|
226
|
+
);
|
|
227
|
+
}
|
package/src/i18n/en.ts
CHANGED
|
@@ -154,10 +154,12 @@ export const en: CoreMessages = {
|
|
|
154
154
|
};
|
|
155
155
|
},
|
|
156
156
|
|
|
157
|
-
promptCachingNotWorthIt: () => ({
|
|
157
|
+
promptCachingNotWorthIt: ({ readPct, writePct, breakEvenPct }) => ({
|
|
158
158
|
title: 'At that hit rate, caching does not pay off',
|
|
159
159
|
detail:
|
|
160
|
-
|
|
160
|
+
breakEvenPct === null
|
|
161
|
+
? `A cache write costs ${writePct}% of the input price on this model, the same as not caching, and a read costs ${readPct}%. So caching cannot lose you money here: at this hit rate it simply buys nothing. Raise prefix reuse and it starts paying immediately.`
|
|
162
|
+
: `A cache write costs ${writePct}% of the input price on this model and a read costs ${readPct}%. Below a ${breakEvenPct}% hit rate you pay more than you save. Either raise prefix reuse or leave caching off.`,
|
|
161
163
|
}),
|
|
162
164
|
|
|
163
165
|
belowCacheMinimum: ({
|
|
@@ -198,7 +200,12 @@ export const en: CoreMessages = {
|
|
|
198
200
|
|
|
199
201
|
modelDowngrade: ({ modelName, tier, candidateName, currentUsd, candidateUsd }) => ({
|
|
200
202
|
title: `This task may not need ${modelName}`,
|
|
201
|
-
detail: `By length and vocabulary the prompt looks like "${tier}" complexity. On ${candidateName} you would go from ${currentUsd} to ${candidateUsd} per month. This is a keyword heuristic, not a judgement about answer quality:
|
|
203
|
+
detail: `By length and vocabulary the prompt looks like "${tier}" complexity. On ${candidateName} you would go from ${currentUsd} to ${candidateUsd} per month. This is a keyword heuristic, not a judgement about answer quality. Measure it before switching in production: \`trazum route <log.jsonl> --prompt-file <prompt> --cases <cases> --yes\` sends the same cases to both models and reports whether the cheaper one still does the job.`,
|
|
204
|
+
}),
|
|
205
|
+
|
|
206
|
+
tierSignalsConflict: ({ complexSignals, simpleSignals }) => ({
|
|
207
|
+
title: 'The prompt asks for depth and for brevity at once',
|
|
208
|
+
detail: `It carries ${complexSignals} signal${complexSignals === 1 ? '' : 's'} of a hard task and ${simpleSignals} of an easy one, and this heuristic scores those against each other, so they cancel. No tier is recommended here: a number produced by two opposite readings cancelling out looks exactly like a prompt with no signals at all, and they are not the same thing. Decide it by measurement rather than by vocabulary: \`trazum route <log.jsonl> --prompt-file <prompt> --cases <cases> --yes\`.`,
|
|
202
209
|
}),
|
|
203
210
|
|
|
204
211
|
outputDominated: ({ outputUsd, inputUsd }) => ({
|
package/src/i18n/es.ts
CHANGED
|
@@ -138,10 +138,12 @@ export const es: CoreMessages = {
|
|
|
138
138
|
};
|
|
139
139
|
},
|
|
140
140
|
|
|
141
|
-
promptCachingNotWorthIt: () => ({
|
|
141
|
+
promptCachingNotWorthIt: ({ readPct, writePct, breakEvenPct }) => ({
|
|
142
142
|
title: 'Con esa tasa de acierto, la caché no compensa',
|
|
143
143
|
detail:
|
|
144
|
-
|
|
144
|
+
breakEvenPct === null
|
|
145
|
+
? `En este modelo escribir en caché cuesta un ${writePct}% del precio de entrada, lo mismo que no usarla, y leer un ${readPct}%. Así que aquí la caché no puede costarte dinero: con esta tasa de acierto simplemente no te aporta nada. En cuanto subas la reutilización del prefijo, empieza a compensar.`
|
|
146
|
+
: `En este modelo escribir en caché cuesta un ${writePct}% del precio de entrada y leer un ${readPct}%. Por debajo de un ${breakEvenPct}% de aciertos pagas más de lo que ahorras. Sube la reutilización del prefijo o deja la caché desactivada.`,
|
|
145
147
|
}),
|
|
146
148
|
|
|
147
149
|
belowCacheMinimum: ({
|
|
@@ -182,7 +184,12 @@ export const es: CoreMessages = {
|
|
|
182
184
|
|
|
183
185
|
modelDowngrade: ({ modelName, tier, candidateName, currentUsd, candidateUsd }) => ({
|
|
184
186
|
title: `Esta tarea quizá no necesite ${modelName}`,
|
|
185
|
-
detail: `Por longitud y vocabulario, el prompt parece de complejidad "${tier}". Con ${candidateName} pasarías de ${currentUsd} a ${candidateUsd} al mes. Es una heurística por palabras clave, no un juicio de calidad:
|
|
187
|
+
detail: `Por longitud y vocabulario, el prompt parece de complejidad "${tier}". Con ${candidateName} pasarías de ${currentUsd} a ${candidateUsd} al mes. Es una heurística por palabras clave, no un juicio de calidad. Mídelo antes de cambiar en producción: \`trazum route <log.jsonl> --prompt-file <prompt> --cases <casos> --yes\` envía los mismos casos a ambos modelos y dice si el barato sigue haciendo el trabajo.`,
|
|
188
|
+
}),
|
|
189
|
+
|
|
190
|
+
tierSignalsConflict: ({ complexSignals, simpleSignals }) => ({
|
|
191
|
+
title: 'El prompt pide profundidad y brevedad a la vez',
|
|
192
|
+
detail: `Lleva ${complexSignals} señal${complexSignals === 1 ? '' : 'es'} de tarea difícil y ${simpleSignals} de tarea fácil, y esta heurística las resta entre sí, así que se anulan. Aquí no se recomienda ningún nivel: un número que sale de dos lecturas opuestas cancelándose se parece exactamente a un prompt sin ninguna señal, y no son lo mismo. Decídelo midiendo y no por vocabulario: \`trazum route <log.jsonl> --prompt-file <prompt> --cases <casos> --yes\`.`,
|
|
186
193
|
}),
|
|
187
194
|
|
|
188
195
|
outputDominated: ({ outputUsd, inputUsd }) => ({
|
package/src/i18n/types.ts
CHANGED
|
@@ -215,6 +215,49 @@ export interface LlmMessages {
|
|
|
215
215
|
suspiciousShrink(retainedPct: number): string;
|
|
216
216
|
}
|
|
217
217
|
|
|
218
|
+
/**
|
|
219
|
+
* What the caching-does-not-pay advisory needs to state a threshold that is
|
|
220
|
+
* true about **this** model.
|
|
221
|
+
*
|
|
222
|
+
* It used to take nothing and print one sentence for every model: *"a cache
|
|
223
|
+
* write costs 125% of the input price and a read costs 10%. Below roughly a 28%
|
|
224
|
+
* hit rate you pay more than you save."* Two of those three numbers were the
|
|
225
|
+
* Anthropic multipliers stated as universal, and the third was not derivable
|
|
226
|
+
* from any model in the catalogue: at 125% and 10% the break-even is 21.74%,
|
|
227
|
+
* not 28%.
|
|
228
|
+
*
|
|
229
|
+
* The multipliers were already in hand at the call site and already passed to
|
|
230
|
+
* the advisory next door. Only this message invented its own.
|
|
231
|
+
*/
|
|
232
|
+
export interface PromptCachingNotWorthItParams {
|
|
233
|
+
/** The model's cache read multiplier as a percentage of the input price. */
|
|
234
|
+
readPct: number;
|
|
235
|
+
/** Its 5-minute cache write multiplier, likewise. */
|
|
236
|
+
writePct: number;
|
|
237
|
+
/**
|
|
238
|
+
* The hit rate at which caching stops costing money, derived rather than
|
|
239
|
+
* quoted: `(1 - write) / (read - write)`.
|
|
240
|
+
*
|
|
241
|
+
* `null` when the write multiplier is 1, which is not a smaller threshold but
|
|
242
|
+
* a different situation: writing costs exactly what not caching costs, so
|
|
243
|
+
* caching can never lose money and there is no rate below which it does. Eight
|
|
244
|
+
* of the eighteen models in the catalogue are that shape, and the old sentence
|
|
245
|
+
* told all eight to consider turning caching off.
|
|
246
|
+
*/
|
|
247
|
+
breakEvenPct: number | null;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* What the tier heuristic saw when it declined to answer.
|
|
252
|
+
*
|
|
253
|
+
* Both counts, because "three of each" and "one of each" are different degrees
|
|
254
|
+
* of the same refusal and the reader can act on the difference.
|
|
255
|
+
*/
|
|
256
|
+
export interface TierSignalsConflictParams {
|
|
257
|
+
complexSignals: number;
|
|
258
|
+
simpleSignals: number;
|
|
259
|
+
}
|
|
260
|
+
|
|
218
261
|
/** Why a proposed rewrite did not survive checking against the prompt. */
|
|
219
262
|
export interface SuggestMessages {
|
|
220
263
|
'not-found'(): string;
|
|
@@ -236,11 +279,12 @@ export interface CoreMessages {
|
|
|
236
279
|
contextOverflow(p: ContextOverflowParams): LocalizedMessage;
|
|
237
280
|
contextNearLimit(p: ContextNearLimitParams): LocalizedMessage;
|
|
238
281
|
promptCaching(p: PromptCachingParams): LocalizedMessage;
|
|
239
|
-
promptCachingNotWorthIt(): LocalizedMessage;
|
|
282
|
+
promptCachingNotWorthIt(p: PromptCachingNotWorthItParams): LocalizedMessage;
|
|
240
283
|
belowCacheMinimum(p: BelowCacheMinimumParams): LocalizedMessage;
|
|
241
284
|
cachePrefixReorder(p: CachePrefixReorderParams): LocalizedMessage;
|
|
242
285
|
batchApi(): LocalizedMessage;
|
|
243
286
|
modelDowngrade(p: ModelDowngradeParams): LocalizedMessage;
|
|
287
|
+
tierSignalsConflict(p: TierSignalsConflictParams): LocalizedMessage;
|
|
244
288
|
outputDominated(p: OutputDominatedParams): LocalizedMessage;
|
|
245
289
|
promoPricing(p: PromoPricingParams): LocalizedMessage;
|
|
246
290
|
contradictoryInstructions(p: ContradictoryInstructionsParams): LocalizedMessage;
|
package/src/index.ts
CHANGED
|
@@ -166,7 +166,18 @@ export type {
|
|
|
166
166
|
GatewayStanding,
|
|
167
167
|
RefuseReason,
|
|
168
168
|
} from './gateway.js';
|
|
169
|
+
export {
|
|
170
|
+
BANDS,
|
|
171
|
+
MEASURED_FOREIGN_ERROR_PCT,
|
|
172
|
+
bandFor,
|
|
173
|
+
bucketFor,
|
|
174
|
+
foreignTokenizer,
|
|
175
|
+
measuredForeignError,
|
|
176
|
+
} from './band.js';
|
|
177
|
+
export type { BandBucket } from './band.js';
|
|
169
178
|
export { conform, CONTRACT_NAMES, requiredFieldsOf } from './conform.js';
|
|
179
|
+
export { readDroppedVerdict, verdictMatchesSlice } from './verdict-bridge.js';
|
|
180
|
+
export type { DroppedVerdict, BridgeReading } from './verdict-bridge.js';
|
|
170
181
|
export { CONTRACT_SCHEMAS, contractSchema } from './schemas.js';
|
|
171
182
|
export type {
|
|
172
183
|
ConformOptions,
|
|
@@ -203,6 +214,10 @@ export type { MeasuredIndex } from './position.js';
|
|
|
203
214
|
export { claudeCodeRecords, looksLikeClaudeCodeTranscript } from './claude-code.js';
|
|
204
215
|
export { otelRecords, looksLikeOtel } from './otel.js';
|
|
205
216
|
export type { OtelConversion, OtelRecord } from './otel.js';
|
|
217
|
+
export { litellmRecords, looksLikeLiteLlm } from './litellm.js';
|
|
218
|
+
export type { LiteLlmConversion, LiteLlmRecord } from './litellm.js';
|
|
219
|
+
export { heliconeRecords, looksLikeHelicone } from './helicone.js';
|
|
220
|
+
export type { HeliconeConversion, HeliconeRecord } from './helicone.js';
|
|
206
221
|
export type { ClaudeCodeConversion, ClaudeCodeRecord } from './claude-code.js';
|
|
207
222
|
export { positionReport } from './position-report.js';
|
|
208
223
|
export type {
|
|
@@ -424,7 +439,8 @@ export type {
|
|
|
424
439
|
export { RULES, getRule } from './rules.js';
|
|
425
440
|
export { segment, join, protectedTexts } from './segment.js';
|
|
426
441
|
export { computeSavings, costOfCall, formatUsd, formatSignedUsd } from './savings.js';
|
|
427
|
-
export { buildAdvisories, recommendTier } from './advisories.js';
|
|
442
|
+
export { buildAdvisories, recommendTier, recommendTierDetailed } from './advisories.js';
|
|
443
|
+
export type { TierRecommendation } from './advisories.js';
|
|
428
444
|
export type { AdvisoryOptions } from './advisories.js';
|
|
429
445
|
export { analyzeCachePrefix } from './cache.js';
|
|
430
446
|
|
package/src/litellm.ts
ADDED
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LiteLLM's spend log, read as a usage log.
|
|
3
|
+
*
|
|
4
|
+
* `from-claude-code` proved the pattern and `from-otel` generalised it: a pure
|
|
5
|
+
* converter turns one tool's export into Trazum's usage-log records, and every
|
|
6
|
+
* door prices it from there. LiteLLM is the gateway a great many teams already
|
|
7
|
+
* put in front of every provider, so its `LiteLLM_SpendLogs` table is the one
|
|
8
|
+
* export most likely to already exist on somebody's disk.
|
|
9
|
+
*
|
|
10
|
+
* **The format is derived, not guessed.** Every field below is read off
|
|
11
|
+
* `litellm/proxy/schema.prisma` in BerriAI/litellm, model `LiteLLM_SpendLogs`.
|
|
12
|
+
* A converter written from memory of an API is a converter that silently
|
|
13
|
+
* mis-reads somebody's bill, and this project has one rule above all others
|
|
14
|
+
* about numbers it cannot justify.
|
|
15
|
+
*
|
|
16
|
+
* ## What maps
|
|
17
|
+
*
|
|
18
|
+
* | LiteLLM | Trazum |
|
|
19
|
+
* | --- | --- |
|
|
20
|
+
* | `model` | `model` |
|
|
21
|
+
* | `prompt_tokens` | `usage.input_tokens` |
|
|
22
|
+
* | `completion_tokens` | `usage.output_tokens` |
|
|
23
|
+
* | `startTime` | `ts` |
|
|
24
|
+
* | `session_id` | `session` |
|
|
25
|
+
* | `request_tags[0]`, else `metadata.tags[0]`, else `model_group` | `label` |
|
|
26
|
+
*
|
|
27
|
+
* ## What deliberately does not cross
|
|
28
|
+
*
|
|
29
|
+
* The row carries `messages` and `response` — the prompt itself and the
|
|
30
|
+
* completion — plus `api_key` (hashed, still credential-shaped),
|
|
31
|
+
* `requester_ip_address`, `user` and `end_user`. **None of it is read.** The
|
|
32
|
+
* converter names the fields it takes and takes nothing else, and a fixture
|
|
33
|
+
* plants a marker in each of them and greps the whole output.
|
|
34
|
+
*
|
|
35
|
+
* ## What it refuses to invent
|
|
36
|
+
*
|
|
37
|
+
* `cache_hit` is a flag and `cache_key` an identifier; neither is a token
|
|
38
|
+
* count. A converted record therefore carries no cache split at all, and the
|
|
39
|
+
* caching questions come back `cannot-tell` rather than answered from a
|
|
40
|
+
* guess — the same refusal as inventing a price. `cacheFlagged` counts the
|
|
41
|
+
* rows that said "hit" so the operator can see the gap is real rather than
|
|
42
|
+
* empty.
|
|
43
|
+
*
|
|
44
|
+
* `spend` is LiteLLM's own priced figure and is **never** merged into
|
|
45
|
+
* Trazum's catalogue-priced total. It is a second measurement of the same
|
|
46
|
+
* calls, kept apart the way the store's provider-billed standing is kept
|
|
47
|
+
* apart from the log's, and returned on its own so a caller who wants to
|
|
48
|
+
* compare the two can, deliberately.
|
|
49
|
+
*/
|
|
50
|
+
|
|
51
|
+
/** One converted record, shaped exactly as `parseUsageLine` reads it. */
|
|
52
|
+
export interface LiteLlmRecord {
|
|
53
|
+
model: string;
|
|
54
|
+
ts?: string;
|
|
55
|
+
label?: string;
|
|
56
|
+
session?: string;
|
|
57
|
+
usage: {
|
|
58
|
+
input_tokens: number;
|
|
59
|
+
output_tokens: number;
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export interface LiteLlmConversion {
|
|
64
|
+
records: LiteLlmRecord[];
|
|
65
|
+
/** Rows that were spend logs and converted. */
|
|
66
|
+
rows: number;
|
|
67
|
+
/** Rows naming no model: counted, never guessed at, never dropped silently. */
|
|
68
|
+
unnamedModel: number;
|
|
69
|
+
/** Rows carrying no token counts at all — a logged call nobody can price. */
|
|
70
|
+
noTokens: number;
|
|
71
|
+
/** Rows LiteLLM marked as a cache hit, with no token split to act on. */
|
|
72
|
+
cacheFlagged: number;
|
|
73
|
+
/**
|
|
74
|
+
* LiteLLM's own total for the converted rows, in USD, or null when no row
|
|
75
|
+
* carried one. **Never** added to anything Trazum computes: it is the
|
|
76
|
+
* gateway's arithmetic over the gateway's price table, and merging two
|
|
77
|
+
* price tables into one figure is how a report becomes quietly wrong.
|
|
78
|
+
*/
|
|
79
|
+
reportedSpendUsd: number | null;
|
|
80
|
+
/** Lines or documents that did not parse as JSON at all. */
|
|
81
|
+
unparseable: number;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const asObject = (value: unknown): Record<string, unknown> | null =>
|
|
85
|
+
typeof value === 'object' && value !== null && !Array.isArray(value)
|
|
86
|
+
? (value as Record<string, unknown>)
|
|
87
|
+
: null;
|
|
88
|
+
|
|
89
|
+
const tryParse = (text: string): unknown => {
|
|
90
|
+
try {
|
|
91
|
+
return JSON.parse(text);
|
|
92
|
+
} catch {
|
|
93
|
+
return undefined;
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
const asCount = (value: unknown): number | undefined => {
|
|
98
|
+
const n = typeof value === 'string' ? Number(value) : value;
|
|
99
|
+
return typeof n === 'number' && Number.isFinite(n) && n >= 0 ? n : undefined;
|
|
100
|
+
};
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* The first tag LiteLLM carries for a row, from the two places it puts them.
|
|
104
|
+
*
|
|
105
|
+
* `request_tags` is the documented column and `metadata.tags` is where the
|
|
106
|
+
* proxy writes them when the caller passes them in the request body. First
|
|
107
|
+
* only: a workload has one name in a bill, and joining several tags into one
|
|
108
|
+
* label would invent a workload that does not exist.
|
|
109
|
+
*/
|
|
110
|
+
function labelOf(row: Record<string, unknown>): string | undefined {
|
|
111
|
+
const fromColumn = Array.isArray(row.request_tags) ? row.request_tags : null;
|
|
112
|
+
if (fromColumn !== null && typeof fromColumn[0] === 'string' && fromColumn[0] !== '') {
|
|
113
|
+
return fromColumn[0];
|
|
114
|
+
}
|
|
115
|
+
const metadata = asObject(row.metadata);
|
|
116
|
+
const fromMetadata = metadata !== null && Array.isArray(metadata.tags) ? metadata.tags : null;
|
|
117
|
+
if (fromMetadata !== null && typeof fromMetadata[0] === 'string' && fromMetadata[0] !== '') {
|
|
118
|
+
return fromMetadata[0];
|
|
119
|
+
}
|
|
120
|
+
// The public model name a proxy route is known by. A weaker label than a
|
|
121
|
+
// tag — it groups by route rather than by workload — and better than none.
|
|
122
|
+
if (typeof row.model_group === 'string' && row.model_group !== '') return row.model_group;
|
|
123
|
+
return undefined;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** `startTime` as an ISO instant, or undefined when it is not a time. */
|
|
127
|
+
function timeOf(row: Record<string, unknown>): string | undefined {
|
|
128
|
+
const raw = row.startTime ?? row.start_time;
|
|
129
|
+
if (typeof raw !== 'string' && typeof raw !== 'number') return undefined;
|
|
130
|
+
const ms = typeof raw === 'number' ? raw : Date.parse(raw);
|
|
131
|
+
if (!Number.isFinite(ms)) return undefined;
|
|
132
|
+
return new Date(ms).toISOString();
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Convert a LiteLLM spend-log export.
|
|
137
|
+
*
|
|
138
|
+
* Accepts a JSON array of rows, a single row, `{ data: [...] }` as the proxy's
|
|
139
|
+
* own endpoints return it, or newline-delimited rows — so a `psql --json`
|
|
140
|
+
* dump, an API response and a streamed capture all work. Pure over its input.
|
|
141
|
+
*/
|
|
142
|
+
export function litellmRecords(text: string): LiteLlmConversion {
|
|
143
|
+
const out: LiteLlmConversion = {
|
|
144
|
+
records: [],
|
|
145
|
+
rows: 0,
|
|
146
|
+
unnamedModel: 0,
|
|
147
|
+
noTokens: 0,
|
|
148
|
+
cacheFlagged: 0,
|
|
149
|
+
reportedSpendUsd: null,
|
|
150
|
+
unparseable: 0,
|
|
151
|
+
};
|
|
152
|
+
|
|
153
|
+
const rows: unknown[] = [];
|
|
154
|
+
const whole = tryParse(text);
|
|
155
|
+
if (whole !== undefined) {
|
|
156
|
+
if (Array.isArray(whole)) rows.push(...whole);
|
|
157
|
+
else {
|
|
158
|
+
const doc = asObject(whole);
|
|
159
|
+
// `{ data: [...] }` is what the proxy's spend endpoints return.
|
|
160
|
+
if (doc !== null && Array.isArray(doc.data)) rows.push(...doc.data);
|
|
161
|
+
else if (doc !== null) rows.push(doc);
|
|
162
|
+
}
|
|
163
|
+
} else {
|
|
164
|
+
for (const line of text.split('\n')) {
|
|
165
|
+
if (line.trim() === '') continue;
|
|
166
|
+
const parsed = tryParse(line);
|
|
167
|
+
if (parsed === undefined) out.unparseable += 1;
|
|
168
|
+
else if (Array.isArray(parsed)) rows.push(...parsed);
|
|
169
|
+
else rows.push(parsed);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
let spend = 0;
|
|
174
|
+
let sawSpend = false;
|
|
175
|
+
|
|
176
|
+
for (const raw of rows) {
|
|
177
|
+
const row = asObject(raw);
|
|
178
|
+
if (row === null) continue;
|
|
179
|
+
// A spend log row is one that carries at least one of the two token
|
|
180
|
+
// columns. Anything else in the same file is not a call and is skipped
|
|
181
|
+
// rather than converted into a record of zero.
|
|
182
|
+
const input = asCount(row.prompt_tokens);
|
|
183
|
+
const output = asCount(row.completion_tokens);
|
|
184
|
+
if (input === undefined && output === undefined) continue;
|
|
185
|
+
|
|
186
|
+
out.rows += 1;
|
|
187
|
+
|
|
188
|
+
if (input === 0 && output === 0) out.noTokens += 1;
|
|
189
|
+
if (row.cache_hit === true || row.cache_hit === 'true' || row.cache_hit === 'True') {
|
|
190
|
+
out.cacheFlagged += 1;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
const reported = asCount(row.spend);
|
|
194
|
+
if (reported !== undefined) {
|
|
195
|
+
spend += reported;
|
|
196
|
+
sawSpend = true;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
const model = typeof row.model === 'string' ? row.model.trim() : '';
|
|
200
|
+
if (model === '') {
|
|
201
|
+
// Counted, never guessed at. `model_group` is a route name, not a model,
|
|
202
|
+
// and pricing a call by the route it took would be a figure attributed
|
|
203
|
+
// to something it does not describe.
|
|
204
|
+
out.unnamedModel += 1;
|
|
205
|
+
continue;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
const ts = timeOf(row);
|
|
209
|
+
const label = labelOf(row);
|
|
210
|
+
const session = typeof row.session_id === 'string' && row.session_id !== ''
|
|
211
|
+
? row.session_id
|
|
212
|
+
: undefined;
|
|
213
|
+
|
|
214
|
+
out.records.push({
|
|
215
|
+
model,
|
|
216
|
+
...(ts !== undefined ? { ts } : {}),
|
|
217
|
+
...(label !== undefined ? { label } : {}),
|
|
218
|
+
...(session !== undefined ? { session } : {}),
|
|
219
|
+
usage: { input_tokens: input ?? 0, output_tokens: output ?? 0 },
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
out.reportedSpendUsd = sawSpend ? spend : null;
|
|
224
|
+
return out;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* Whether a file looks like a LiteLLM spend log, by shape.
|
|
229
|
+
*
|
|
230
|
+
* Deliberately narrow: `request_id` beside one of the two token columns, and
|
|
231
|
+
* a field no other export in this project carries. A looser test would claim
|
|
232
|
+
* an OpenAI usage response, whose rows also have `prompt_tokens`, and refuse
|
|
233
|
+
* it in this file's words instead of letting the code that can read it try.
|
|
234
|
+
*/
|
|
235
|
+
export function looksLikeLiteLlm(text: string, prefixBytes = 8192): boolean {
|
|
236
|
+
const head = text.slice(0, prefixBytes);
|
|
237
|
+
if (!head.includes('prompt_tokens') && !head.includes('completion_tokens')) return false;
|
|
238
|
+
return (
|
|
239
|
+
head.includes('"request_id"') ||
|
|
240
|
+
head.includes('"custom_llm_provider"') ||
|
|
241
|
+
head.includes('"model_group"')
|
|
242
|
+
);
|
|
243
|
+
}
|