@trazum/core 1.10.0 → 1.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/against.d.ts +32 -0
- package/dist/against.d.ts.map +1 -0
- package/dist/against.js +34 -0
- package/dist/against.js.map +1 -0
- package/dist/config-schema.d.ts +42 -1
- package/dist/config-schema.d.ts.map +1 -1
- package/dist/config-schema.js +61 -0
- package/dist/config-schema.js.map +1 -1
- package/dist/conversation.d.ts +121 -0
- package/dist/conversation.d.ts.map +1 -0
- package/dist/conversation.js +157 -0
- package/dist/conversation.js.map +1 -0
- package/dist/csv.d.ts +61 -0
- package/dist/csv.d.ts.map +1 -0
- package/dist/csv.js +149 -0
- package/dist/csv.js.map +1 -0
- package/dist/evaluate.d.ts +24 -0
- package/dist/evaluate.d.ts.map +1 -1
- package/dist/evaluate.js +5 -2
- package/dist/evaluate.js.map +1 -1
- package/dist/index.d.ts +25 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +28 -1
- package/dist/index.js.map +1 -1
- package/dist/input-shape.d.ts +104 -0
- package/dist/input-shape.d.ts.map +1 -0
- package/dist/input-shape.js +132 -0
- package/dist/input-shape.js.map +1 -0
- package/dist/levers.d.ts +151 -0
- package/dist/levers.d.ts.map +1 -0
- package/dist/levers.js +160 -0
- package/dist/levers.js.map +1 -0
- package/dist/node.d.ts +1 -1
- package/dist/node.d.ts.map +1 -1
- package/dist/output-shape.d.ts +96 -0
- package/dist/output-shape.d.ts.map +1 -0
- package/dist/output-shape.js +145 -0
- package/dist/output-shape.js.map +1 -0
- package/dist/pricing-overlay.d.ts +1 -1
- package/dist/pricing-overlay.d.ts.map +1 -1
- package/dist/pricing-overlay.js +46 -0
- package/dist/pricing-overlay.js.map +1 -1
- package/dist/repeats.d.ts +75 -0
- package/dist/repeats.d.ts.map +1 -0
- package/dist/repeats.js +82 -0
- package/dist/repeats.js.map +1 -0
- package/dist/reprice.d.ts +143 -0
- package/dist/reprice.d.ts.map +1 -0
- package/dist/reprice.js +82 -0
- package/dist/reprice.js.map +1 -0
- package/dist/session-cost.d.ts +70 -0
- package/dist/session-cost.d.ts.map +1 -0
- package/dist/session-cost.js +90 -0
- package/dist/session-cost.js.map +1 -0
- package/dist/session-ledger.d.ts +77 -0
- package/dist/session-ledger.d.ts.map +1 -0
- package/dist/session-ledger.js +99 -0
- package/dist/session-ledger.js.map +1 -0
- package/dist/ttl-fit.d.ts +103 -0
- package/dist/ttl-fit.d.ts.map +1 -0
- package/dist/ttl-fit.js +184 -0
- package/dist/ttl-fit.js.map +1 -0
- package/dist/usage.d.ts +434 -16
- package/dist/usage.d.ts.map +1 -1
- package/dist/usage.js +383 -23
- package/dist/usage.js.map +1 -1
- package/package.json +1 -1
- package/src/against.ts +48 -0
- package/src/config-schema.ts +106 -0
- package/src/conversation.ts +305 -0
- package/src/csv.ts +184 -0
- package/src/evaluate.ts +33 -3
- package/src/index.ts +51 -1
- package/src/input-shape.ts +259 -0
- package/src/levers.ts +331 -0
- package/src/node.ts +1 -1
- package/src/output-shape.ts +254 -0
- package/src/pricing-overlay.ts +52 -1
- package/src/repeats.ts +166 -0
- package/src/reprice.ts +227 -0
- package/src/session-cost.ts +170 -0
- package/src/session-ledger.ts +189 -0
- package/src/ttl-fit.ts +251 -0
- package/src/usage.ts +795 -7
package/dist/usage.d.ts
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
import type { SessionCostShape } from './session-cost.js';
|
|
2
|
+
import type { CacheTtlFit } from './ttl-fit.js';
|
|
3
|
+
import type { SingleTurnCacheWrites } from './session-ledger.js';
|
|
4
|
+
import type { ConversationGrowth } from './conversation.js';
|
|
5
|
+
import type { OutputShape } from './output-shape.js';
|
|
6
|
+
import type { InputShape } from './input-shape.js';
|
|
7
|
+
import type { RepeatedTurns } from './repeats.js';
|
|
1
8
|
import type { PricingCatalogue } from './pricing.js';
|
|
2
9
|
/**
|
|
3
10
|
* Where the money actually went, from calls that actually happened.
|
|
@@ -53,6 +60,13 @@ import type { PricingCatalogue } from './pricing.js';
|
|
|
53
60
|
* exists precisely because guessing is what the rest of the package has to do.
|
|
54
61
|
* It reports what was spent, split by where it went. What to do about it is a
|
|
55
62
|
* different question and belongs to the advisories.
|
|
63
|
+
*
|
|
64
|
+
* `cacheEconomics` is the one counterfactual here, and it is not an exception to
|
|
65
|
+
* that rule — it is the line the rule draws. A saving requires imagining a prompt
|
|
66
|
+
* nobody wrote; this requires imagining the **same tokens at a different rate**,
|
|
67
|
+
* which is arithmetic. Caching does not change what is sent, only the multiplier
|
|
68
|
+
* it is billed at, so "these tokens cost 1.25x instead of 1x" is as measured as
|
|
69
|
+
* the total itself. Anything that would need a guess about content stays out.
|
|
56
70
|
*/
|
|
57
71
|
/** One recorded call, after parsing. All counts, no content. */
|
|
58
72
|
export interface UsageRecord {
|
|
@@ -85,6 +99,42 @@ export interface UsageRecord {
|
|
|
85
99
|
* refuses to read a log until it is annotated is a profile nobody runs.
|
|
86
100
|
*/
|
|
87
101
|
label: string | null;
|
|
102
|
+
/**
|
|
103
|
+
* Optional conversation identifier, for measuring what re-sent history costs.
|
|
104
|
+
*
|
|
105
|
+
* On a chat or agent workload the input grows with every turn, because the whole
|
|
106
|
+
* conversation goes back up on each call. That is frequently the largest line on
|
|
107
|
+
* the bill and nothing watches it — a prompt file cannot show it, and a total
|
|
108
|
+
* cannot either.
|
|
109
|
+
*
|
|
110
|
+
* **Trazum never prints this value.** A session key is somebody's conversation
|
|
111
|
+
* and could easily be an account id or an email; it is used to group calls and
|
|
112
|
+
* to count turns, and every figure derived from it is reported per *label*. That
|
|
113
|
+
* keeps the guarantee this module is built on: a usage log handed to Trazum
|
|
114
|
+
* carries no content, and nothing identifying comes back out of it either.
|
|
115
|
+
*/
|
|
116
|
+
session: string | null;
|
|
117
|
+
/**
|
|
118
|
+
* When the call happened, as epoch milliseconds, or `null` when the log does
|
|
119
|
+
* not say.
|
|
120
|
+
*
|
|
121
|
+
* Read from `ts`, `timestamp`, `created_at` or OpenAI's `created`; ISO 8601
|
|
122
|
+
* strings and epoch numbers both work, with seconds told from milliseconds by
|
|
123
|
+
* magnitude. The clock unlocks the two findings counts alone cannot make:
|
|
124
|
+
* what period this log actually covers, and whether the cache TTL fits how
|
|
125
|
+
* fast the turns arrive — the single most common reason a cache loses money.
|
|
126
|
+
*/
|
|
127
|
+
ts: number | null;
|
|
128
|
+
/**
|
|
129
|
+
* Whether the answer hit the output ceiling, when the log says.
|
|
130
|
+
*
|
|
131
|
+
* `true` for Anthropic's `stop_reason: "max_tokens"` and OpenAI's
|
|
132
|
+
* `finish_reason: "length"`; `false` for any other recorded reason; `null`
|
|
133
|
+
* when the log does not carry the field. Three states, because "no truncation
|
|
134
|
+
* recorded" and "no truncation happened" are different answers — the report
|
|
135
|
+
* must not congratulate a log that never measured.
|
|
136
|
+
*/
|
|
137
|
+
truncated: boolean | null;
|
|
88
138
|
}
|
|
89
139
|
/** What a set of calls cost, split by where the money went. */
|
|
90
140
|
export interface UsageBreakdown {
|
|
@@ -92,17 +142,78 @@ export interface UsageBreakdown {
|
|
|
92
142
|
inputTokens: number;
|
|
93
143
|
cacheReadTokens: number;
|
|
94
144
|
cacheWriteTokens: number;
|
|
145
|
+
/**
|
|
146
|
+
* The two write TTLs kept apart, because they are billed at different rates
|
|
147
|
+
* — 1.25x input for a 5-minute entry and **2x** for a 1-hour one.
|
|
148
|
+
*
|
|
149
|
+
* `cacheWriteTokens` is their sum and stays the figure to read for volume.
|
|
150
|
+
* These exist so the same tokens can be priced again at another model's
|
|
151
|
+
* rates without the ratio between the two being invented: it is not a
|
|
152
|
+
* constant across providers, so a total that has lost the split cannot be
|
|
153
|
+
* repriced, only guessed at.
|
|
154
|
+
*
|
|
155
|
+
* Writes whose TTL the log did not state are in the 5-minute bucket, the
|
|
156
|
+
* same assumption `cacheWriteUsdIfAssumed1h` measures the cost of.
|
|
157
|
+
*/
|
|
158
|
+
cacheWrite5mTokens: number;
|
|
159
|
+
cacheWrite1hTokens: number;
|
|
95
160
|
outputTokens: number;
|
|
96
161
|
/**
|
|
97
162
|
* Calls whose cache-write TTL the log did not state, so the cheaper rate was
|
|
98
163
|
* assumed. Non-zero means this total is a floor on those calls, not a figure.
|
|
99
164
|
*/
|
|
100
165
|
assumedWriteTtlCalls: number;
|
|
166
|
+
/**
|
|
167
|
+
* The largest single call's input, cache reads and writes included — the one
|
|
168
|
+
* number that says whether these calls would fit somewhere else.
|
|
169
|
+
*
|
|
170
|
+
* A cheaper model with a smaller context window does not make this traffic
|
|
171
|
+
* cheaper; it makes some of it impossible, and a price comparison that only
|
|
172
|
+
* multiplies rates would call that a saving. The maximum is the right
|
|
173
|
+
* statistic rather than the mean: one call over the ceiling is a failed
|
|
174
|
+
* call, and an average hides it.
|
|
175
|
+
*/
|
|
176
|
+
maxCallInputTokens: number;
|
|
101
177
|
inputUsd: number;
|
|
102
178
|
cacheReadUsd: number;
|
|
103
179
|
cacheWriteUsd: number;
|
|
104
180
|
outputUsd: number;
|
|
105
181
|
totalUsd: number;
|
|
182
|
+
/**
|
|
183
|
+
* What the cache-touched tokens would have cost as ordinary input.
|
|
184
|
+
*
|
|
185
|
+
* Reads plus writes, at each model's own full input rate, accumulated per call
|
|
186
|
+
* because the rate is per model and a total loses that. Not part of `totalUsd`
|
|
187
|
+
* and not a bill — it is the other half of `cacheEconomics`, kept here because
|
|
188
|
+
* it can only be computed while the model is still in hand.
|
|
189
|
+
*/
|
|
190
|
+
cachedTokensAtInputRateUsd: number;
|
|
191
|
+
/**
|
|
192
|
+
* `cacheWriteUsd` with every **unstated-TTL** write priced at the 1-hour rate.
|
|
193
|
+
*
|
|
194
|
+
* Equal to `cacheWriteUsd` when the log recorded which TTL each write used.
|
|
195
|
+
* When it did not, the cheaper 5-minute rate is assumed for the headline figure
|
|
196
|
+
* — and this is what the same calls cost if that assumption is wrong.
|
|
197
|
+
*
|
|
198
|
+
* It exists because the assumption reaches further than the total. It moves the
|
|
199
|
+
* *verdict*: a workload reading back between 0.28 and 1.11 tokens per token
|
|
200
|
+
* written is reported as paying for itself at 1.25x and as losing money at 2x,
|
|
201
|
+
* and the log is silent about which. A verdict that cannot see the assumption
|
|
202
|
+
* behind it states the flattering half as a fact.
|
|
203
|
+
*/
|
|
204
|
+
cacheWriteUsdIfAssumed1h: number;
|
|
205
|
+
/**
|
|
206
|
+
* Calls whose answer hit the output ceiling, and what their output cost.
|
|
207
|
+
*
|
|
208
|
+
* The one category of a bill that is waste without a counterpart: an answer
|
|
209
|
+
* cut off mid-generation was paid for in full, is frequently retried — billed
|
|
210
|
+
* again — and the truncated attempt bought nothing. Output is the largest
|
|
211
|
+
* line on most bills, and this is the slice of it nobody sees.
|
|
212
|
+
*/
|
|
213
|
+
truncatedCalls: number;
|
|
214
|
+
truncatedOutputUsd: number;
|
|
215
|
+
/** Calls that recorded a stop reason at all, truncated or not. */
|
|
216
|
+
stopReasonCalls: number;
|
|
106
217
|
}
|
|
107
218
|
export interface UsageProfileReport {
|
|
108
219
|
/** Everything, combined. */
|
|
@@ -117,6 +228,20 @@ export interface UsageProfileReport {
|
|
|
117
228
|
model: string;
|
|
118
229
|
breakdown: UsageBreakdown;
|
|
119
230
|
}>;
|
|
231
|
+
/**
|
|
232
|
+
* Per label **and** model, largest bill first.
|
|
233
|
+
*
|
|
234
|
+
* The grouping a decision is actually made at. "Route `classify` to something
|
|
235
|
+
* cheaper" is a question about the calls `classify` makes to one model, and a
|
|
236
|
+
* label that spans two models has no single answer — pricing it against a
|
|
237
|
+
* cheaper candidate would mean picking one of the two current prices and
|
|
238
|
+
* applying it to tokens that were never billed at it.
|
|
239
|
+
*/
|
|
240
|
+
byLabelAndModel: Array<{
|
|
241
|
+
label: string;
|
|
242
|
+
model: string;
|
|
243
|
+
breakdown: UsageBreakdown;
|
|
244
|
+
}>;
|
|
120
245
|
/**
|
|
121
246
|
* Models in the log that the pricing catalogue does not know.
|
|
122
247
|
*
|
|
@@ -148,6 +273,201 @@ export interface UsageProfileReport {
|
|
|
148
273
|
* produces a total that is wrong by an unknown amount.
|
|
149
274
|
*/
|
|
150
275
|
skippedLines: number[];
|
|
276
|
+
/**
|
|
277
|
+
* What re-sending the conversation costs, where the log carries a session.
|
|
278
|
+
*
|
|
279
|
+
* Empty when it does not, which is a different statement from zero growth — the
|
|
280
|
+
* report says which, because "nothing to report" and "nothing recorded" are the
|
|
281
|
+
* two answers a reader would act on differently.
|
|
282
|
+
*/
|
|
283
|
+
conversations: ConversationGrowth[];
|
|
284
|
+
/** Whether any record carried a session at all. */
|
|
285
|
+
hasSessions: boolean;
|
|
286
|
+
/**
|
|
287
|
+
* Where the output spend concentrates, for slices whose output is a real share
|
|
288
|
+
* of the bill. The actionable half of "output dominates": six per cent of calls
|
|
289
|
+
* holding half the spend is a tail worth hunting, forty-five per cent is a task
|
|
290
|
+
* whose answers are inherently long — and the total cannot tell them apart.
|
|
291
|
+
*/
|
|
292
|
+
outputShapes: OutputShape[];
|
|
293
|
+
/**
|
|
294
|
+
* How big a call's input is, and how uneven that is across a slice — the
|
|
295
|
+
* half of the bill a total could only name. "Input is 63% of this bill" is
|
|
296
|
+
* unactionable; whether the p95 call carries twelve times the median call's
|
|
297
|
+
* input decides between capping something and rewriting a prompt.
|
|
298
|
+
*
|
|
299
|
+
* Every figure is a bucket ceiling rather than an interpolated percentile,
|
|
300
|
+
* and slices with too few calls for a percentile to mean anything are left
|
|
301
|
+
* out entirely rather than reported at a precision they do not have.
|
|
302
|
+
*/
|
|
303
|
+
inputShapes: InputShape[];
|
|
304
|
+
/**
|
|
305
|
+
* Calls that re-sent the previous call's exact input size, in the same
|
|
306
|
+
* conversation, seconds later — the shape of a retry or a loop.
|
|
307
|
+
*
|
|
308
|
+
* A conversation's input grows with every turn, so two consecutive calls
|
|
309
|
+
* carrying the same size a moment apart is a thing going wrong rather than
|
|
310
|
+
* a thing working. Needs both a session and a clock; empty when the log
|
|
311
|
+
* carries neither, which is a different statement from "none happened".
|
|
312
|
+
*/
|
|
313
|
+
repeatedTurns: RepeatedTurns[];
|
|
314
|
+
/**
|
|
315
|
+
* The period the log covers, when its records carry a clock, over every
|
|
316
|
+
* parsed record — priced and unpriced alike, because when a call happened is
|
|
317
|
+
* a fact about the log rather than about the catalogue.
|
|
318
|
+
*
|
|
319
|
+
* `calls` is how many records carried a timestamp; compared against the
|
|
320
|
+
* parsed total it says whether the span describes the whole log or a slice
|
|
321
|
+
* of it, and the report states which. **The span is stated, never
|
|
322
|
+
* extrapolated**: "this log covers 13 days" makes the reader's own monthly
|
|
323
|
+
* arithmetic valid, while a per-month figure printed from a partial month
|
|
324
|
+
* would be this module doing the guessing it exists to end.
|
|
325
|
+
*/
|
|
326
|
+
span: {
|
|
327
|
+
fromMs: number;
|
|
328
|
+
toMs: number;
|
|
329
|
+
calls: number;
|
|
330
|
+
} | null;
|
|
331
|
+
/**
|
|
332
|
+
* Spend per UTC day, oldest first, over priced records that carry a clock.
|
|
333
|
+
*
|
|
334
|
+
* The shape of a bill over time is the finding the total hides: a steady $3 a
|
|
335
|
+
* day and a quiet week broken by one $40 spike sum to the same number and
|
|
336
|
+
* call for opposite responses. Each day carries its most expensive label so a
|
|
337
|
+
* spike arrives with a suspect attached — per *label*, never per session.
|
|
338
|
+
*
|
|
339
|
+
* UTC deliberately: the log's timestamps carry no zone once parsed, and
|
|
340
|
+
* bucketing by the reader's local midnight would make the same log answer
|
|
341
|
+
* differently in two offices.
|
|
342
|
+
*/
|
|
343
|
+
spendByDay: Array<{
|
|
344
|
+
/** `YYYY-MM-DD`, UTC. */
|
|
345
|
+
day: string;
|
|
346
|
+
usd: number;
|
|
347
|
+
calls: number;
|
|
348
|
+
/** The label that spent the most this day, or null when nothing had one. */
|
|
349
|
+
topLabel: string | null;
|
|
350
|
+
topLabelUsd: number;
|
|
351
|
+
}>;
|
|
352
|
+
/**
|
|
353
|
+
* Lines that are exact duplicates of an earlier line, and what they added
|
|
354
|
+
* to the total.
|
|
355
|
+
*
|
|
356
|
+
* Reading a directory of rotated logs — or catting them together by hand —
|
|
357
|
+
* makes double-counting easy: a log exported twice, an overlapping export,
|
|
358
|
+
* a copy left in the folder. The bill then reads high, and nothing else in
|
|
359
|
+
* this report can see it, because two identical calls are indistinguishable
|
|
360
|
+
* from one call recorded twice *unless* the record carries a clock.
|
|
361
|
+
*
|
|
362
|
+
* So this counts only records with a `ts`: identical token counts, identical
|
|
363
|
+
* label and session, and the same millisecond. Two real calls colliding on
|
|
364
|
+
* all of that is possible and vanishingly unlikely; without a clock it is
|
|
365
|
+
* ordinary, which is why clockless records are excluded rather than guessed
|
|
366
|
+
* at. The report states the count and the money and stops — whether it is a
|
|
367
|
+
* double export or a busy millisecond is the reader's to know.
|
|
368
|
+
*
|
|
369
|
+
* The comparison is over the **raw line**, not a hash of it: a hash
|
|
370
|
+
* collision would report a duplicate that is not one, and this figure exists
|
|
371
|
+
* to make somebody distrust a total.
|
|
372
|
+
*/
|
|
373
|
+
duplicateLines: {
|
|
374
|
+
count: number;
|
|
375
|
+
usd: number;
|
|
376
|
+
};
|
|
377
|
+
/**
|
|
378
|
+
* How many parsed records carried each optional field.
|
|
379
|
+
*
|
|
380
|
+
* Every finding this module makes beyond the totals needs a field the log
|
|
381
|
+
* format does not require, and a reader who never adds them sees a report
|
|
382
|
+
* quietly missing half of itself. Counting them turns "Trazum did not tell
|
|
383
|
+
* me about conversation growth" into "none of your 40,000 records carry a
|
|
384
|
+
* session", which is a fact somebody can act on in an afternoon.
|
|
385
|
+
*
|
|
386
|
+
* Counted over records that **parsed**, priced or not: whether a field is
|
|
387
|
+
* present is a property of the log, not of the price catalogue. Partial
|
|
388
|
+
* coverage is the interesting case and is why these are counts rather than
|
|
389
|
+
* booleans — 12 records out of 40,000 carrying a label is not "labelled",
|
|
390
|
+
* and a boolean would call it that.
|
|
391
|
+
*/
|
|
392
|
+
fieldCoverage: {
|
|
393
|
+
/** Records with a usable `label`. */
|
|
394
|
+
label: number;
|
|
395
|
+
/** Records with a usable `session` or `conversation_id`. */
|
|
396
|
+
session: number;
|
|
397
|
+
/** Records with a readable timestamp. */
|
|
398
|
+
ts: number;
|
|
399
|
+
/** Records with a `stop_reason` or `finish_reason`. */
|
|
400
|
+
stopReason: number;
|
|
401
|
+
/** Records whose cache writes stated which TTL they used. */
|
|
402
|
+
cacheTtl: number;
|
|
403
|
+
/** Records that wrote to the cache at all — the denominator for `cacheTtl`. */
|
|
404
|
+
cacheWrites: number;
|
|
405
|
+
/** Every record that parsed, priced or not — the denominator for the rest. */
|
|
406
|
+
parsed: number;
|
|
407
|
+
};
|
|
408
|
+
/**
|
|
409
|
+
* Spend per hour of the UTC day, 0–23, over priced records that carry a
|
|
410
|
+
* clock — and only the hours that saw traffic.
|
|
411
|
+
*
|
|
412
|
+
* The shape a day has says what kind of workload this is. Spend packed into
|
|
413
|
+
* the hours a country is awake is interactive traffic somebody is waiting
|
|
414
|
+
* on; spend spread evenly across all twenty-four is background work, and
|
|
415
|
+
* background work is exactly what the Batch API halves the price of. The
|
|
416
|
+
* total cannot tell those apart, and neither can the per-day series.
|
|
417
|
+
*
|
|
418
|
+
* UTC deliberately, like `spendByDay`: bucketing by the reader's local hour
|
|
419
|
+
* would make the same log answer differently in two offices, and the log's
|
|
420
|
+
* timestamps carry no zone once parsed. A reader who knows their traffic is
|
|
421
|
+
* in one region can shift the hours themselves; Trazum inventing an offset
|
|
422
|
+
* would be guessing.
|
|
423
|
+
*/
|
|
424
|
+
spendByHour: Array<{
|
|
425
|
+
hour: number;
|
|
426
|
+
usd: number;
|
|
427
|
+
calls: number;
|
|
428
|
+
}>;
|
|
429
|
+
/**
|
|
430
|
+
* Whether each slice's cache TTL fits how fast its turns arrive — the
|
|
431
|
+
* mechanism behind a losing cache verdict, and the one place an overlong TTL
|
|
432
|
+
* (2x writes surviving gaps measured in seconds) is ever visible. Needs
|
|
433
|
+
* `session` and a timestamp on the records; empty otherwise, which the
|
|
434
|
+
* report distinguishes from "measured and fine".
|
|
435
|
+
*/
|
|
436
|
+
cacheTtlFit: CacheTtlFit[];
|
|
437
|
+
/**
|
|
438
|
+
* The time filter this report was computed under, or `null` when there was
|
|
439
|
+
* none — so a rendering can say "this is a window, not the log" instead of
|
|
440
|
+
* presenting a slice as the whole.
|
|
441
|
+
*
|
|
442
|
+
* `undatedExcluded` is the honesty cost of filtering by a clock some records
|
|
443
|
+
* do not carry: calls that passed every other filter but could not be
|
|
444
|
+
* placed in or out of the window. Non-zero means the window's figures are a
|
|
445
|
+
* floor on the period, and every rendering says so out loud.
|
|
446
|
+
*/
|
|
447
|
+
timeWindow: {
|
|
448
|
+
sinceMs: number | null;
|
|
449
|
+
untilMs: number | null;
|
|
450
|
+
undatedExcluded: number;
|
|
451
|
+
} | null;
|
|
452
|
+
/**
|
|
453
|
+
* Cache writes made by conversations that ended after one turn — reuse paid
|
|
454
|
+
* for that their own conversation never made. A ceiling named as one: the
|
|
455
|
+
* provider's cache is keyed by prefix, so another session sharing the
|
|
456
|
+
* prefix within the TTL could have read these writes, and the log cannot
|
|
457
|
+
* see whose write a read hit. When the slice's `cacheReadTokens` is zero
|
|
458
|
+
* the ceiling collapses into a fact — nothing read those writes at all —
|
|
459
|
+
* and the renderings say which of the two they are stating.
|
|
460
|
+
*/
|
|
461
|
+
singleTurnCacheWrites: SingleTurnCacheWrites[];
|
|
462
|
+
/**
|
|
463
|
+
* What one conversation costs — median, p95 and maximum per slice, exact
|
|
464
|
+
* and billed. The question a total cannot answer: whether $4,000 is forty
|
|
465
|
+
* thousand cheap conversations or four hundred expensive ones, which is
|
|
466
|
+
* what a per-seat price or a quota is set from. Empty when the log carries
|
|
467
|
+
* no session, or when no slice has enough conversations for a median to
|
|
468
|
+
* mean anything.
|
|
469
|
+
*/
|
|
470
|
+
sessionCosts: SessionCostShape[];
|
|
151
471
|
}
|
|
152
472
|
/** The share of the bill each part accounts for, as fractions of 1. */
|
|
153
473
|
export interface UsageShares {
|
|
@@ -156,30 +476,47 @@ export interface UsageShares {
|
|
|
156
476
|
cacheWrite: number;
|
|
157
477
|
output: number;
|
|
158
478
|
}
|
|
479
|
+
export declare function parseUsageLine(line: string): UsageRecord | null;
|
|
159
480
|
/**
|
|
160
|
-
*
|
|
161
|
-
*
|
|
162
|
-
* Accepts the Anthropic shape and the OpenAI one, because those are the two
|
|
163
|
-
* things people actually have. The alternative — a Trazum-specific schema — asks
|
|
164
|
-
* for a transformation step before the tool will read anything, and a tool with a
|
|
165
|
-
* setup cost that exceeds its payoff does not get run twice.
|
|
481
|
+
* The bucket unlabelled calls land in.
|
|
166
482
|
*
|
|
167
|
-
*
|
|
483
|
+
* The empty string, because it is the one value a parsed label can never be —
|
|
484
|
+
* `parseUsageLine` trims and rejects empty. The first version used the literal
|
|
485
|
+
* string `'unlabelled'`, and a workload somebody had actually named `unlabelled`
|
|
486
|
+
* merged silently into the missing-label bucket: 200 labelled calls and 200
|
|
487
|
+
* unlabelled ones reported as one row of 400, and the "none of these calls
|
|
488
|
+
* carried a label" warning fired over a log where half of them had.
|
|
168
489
|
*
|
|
169
|
-
*
|
|
170
|
-
*
|
|
171
|
-
* contributing nothing, which lowers every per-call figure.
|
|
172
|
-
* 3. **Any** count present but unreadable. A field that is there and unusable is
|
|
173
|
-
* corruption, and a corrupt line belongs in `skippedLines` where the report
|
|
174
|
-
* names it, not in the totals as a silent zero.
|
|
490
|
+
* Presentation stays in the CLI, which translates this sentinel through the
|
|
491
|
+
* message catalogue; data consumers can tell `''` from any real label.
|
|
175
492
|
*/
|
|
176
|
-
export declare
|
|
177
|
-
/** The bucket unlabelled calls land in, named so a report can say so. */
|
|
178
|
-
export declare const UNLABELLED = "unlabelled";
|
|
493
|
+
export declare const UNLABELLED = "";
|
|
179
494
|
export interface UsageProfileOptions {
|
|
180
495
|
catalogue: PricingCatalogue;
|
|
181
496
|
/** Date the prices are read at, so a promotional rate resolves the same way. */
|
|
182
497
|
on?: Date;
|
|
498
|
+
/**
|
|
499
|
+
* Profile only the records carrying this label — the drill-down, once the
|
|
500
|
+
* full report has named a suspect. `UNLABELLED` (the empty string) selects
|
|
501
|
+
* the records with no label at all. Unreadable lines still land in
|
|
502
|
+
* `skippedLines` whatever they might have been labelled: a filter must not
|
|
503
|
+
* make corruption disappear.
|
|
504
|
+
*/
|
|
505
|
+
label?: string;
|
|
506
|
+
/**
|
|
507
|
+
* Profile only the records whose clock falls in `[sinceMs, untilMs)` — the
|
|
508
|
+
* drill-down in time, once the peak day or the span has named a period.
|
|
509
|
+
*
|
|
510
|
+
* Epoch milliseconds, half-open on the right so two adjacent windows share
|
|
511
|
+
* no record. Either bound alone works. **A record with no clock cannot be
|
|
512
|
+
* placed inside or outside a window**, so under a time filter it is
|
|
513
|
+
* excluded and counted in `timeWindow.undatedExcluded` — excluded, because
|
|
514
|
+
* including it would put unknown-time spend inside a window it may not
|
|
515
|
+
* belong to; counted, because dropping it silently would understate the
|
|
516
|
+
* period's bill by an invisible amount, which is the flattering direction.
|
|
517
|
+
*/
|
|
518
|
+
sinceMs?: number;
|
|
519
|
+
untilMs?: number;
|
|
183
520
|
}
|
|
184
521
|
/**
|
|
185
522
|
* Reads a usage log and says where the money went.
|
|
@@ -216,4 +553,85 @@ export declare function sharesOf(breakdown: UsageBreakdown): UsageShares;
|
|
|
216
553
|
* on the day it warms.
|
|
217
554
|
*/
|
|
218
555
|
export declare function cacheHitRate(breakdown: UsageBreakdown): number | null;
|
|
556
|
+
/** What caching did to this bill, measured against the same tokens uncached. */
|
|
557
|
+
export interface CacheEconomics {
|
|
558
|
+
/** What the cache-touched tokens actually cost: reads plus writes. */
|
|
559
|
+
spentUsd: number;
|
|
560
|
+
/** What those same tokens would have cost billed as ordinary input. */
|
|
561
|
+
withoutCachingUsd: number;
|
|
562
|
+
/**
|
|
563
|
+
* `spentUsd - withoutCachingUsd`.
|
|
564
|
+
*
|
|
565
|
+
* **Positive means caching cost more than it saved** — the opposite of the sign
|
|
566
|
+
* convention everywhere else in Trazum, and deliberately so, because this is the
|
|
567
|
+
* number nobody expects to come out positive and the one worth interrupting for.
|
|
568
|
+
*/
|
|
569
|
+
deltaUsd: number;
|
|
570
|
+
/**
|
|
571
|
+
* Read tokens per write token, or `null` when nothing was written.
|
|
572
|
+
*
|
|
573
|
+
* Context for the delta, not a verdict of its own: the delta already decides,
|
|
574
|
+
* and it decides at the real per-model rates. This says *why* — a ratio near
|
|
575
|
+
* zero on an Anthropic workload is a prefix being rebuilt faster than it is
|
|
576
|
+
* reused, which is the shape of a cache that never gets to work.
|
|
577
|
+
*/
|
|
578
|
+
readsPerWrite: number | null;
|
|
579
|
+
verdict: CacheVerdict;
|
|
580
|
+
/**
|
|
581
|
+
* `deltaUsd` with every write whose TTL the log did not state priced at the
|
|
582
|
+
* 1-hour rate instead of the assumed 5-minute one.
|
|
583
|
+
*
|
|
584
|
+
* Equal to `deltaUsd` when every TTL was recorded, and never smaller: the
|
|
585
|
+
* 1-hour multiplier is at or above the 5-minute one on every model in the
|
|
586
|
+
* catalogue, so this is a genuine worst case rather than the other end of a
|
|
587
|
+
* range.
|
|
588
|
+
*/
|
|
589
|
+
worstCaseDeltaUsd: number;
|
|
590
|
+
/**
|
|
591
|
+
* The verdict at that worst case.
|
|
592
|
+
*
|
|
593
|
+
* **When this differs from `verdict`, the log cannot settle the question** and
|
|
594
|
+
* neither can any report built from it. That is not a rare shape: a workload
|
|
595
|
+
* reading back between 0.28 and 1.11 tokens per token written flips between
|
|
596
|
+
* `paid-off` and `lost-money` on the TTL alone, and a log carrying only the flat
|
|
597
|
+
* `cache_creation_input_tokens` never says which. Measured on a million written
|
|
598
|
+
* tokens against three hundred thousand read back, the difference was a $0.10
|
|
599
|
+
* saving against a $3.65 loss — a $3.75 swing across the sign, and the assumed
|
|
600
|
+
* half is the flattering one.
|
|
601
|
+
*/
|
|
602
|
+
worstCaseVerdict: CacheVerdict;
|
|
603
|
+
}
|
|
604
|
+
/**
|
|
605
|
+
* - `paid-off` — caching took money off the bill.
|
|
606
|
+
* - `lost-money` — caching added to it. Possible on Anthropic, where a write
|
|
607
|
+
* costs 1.25x input (5-minute) or 2x (1-hour); a prefix that never gets read
|
|
608
|
+
* back is billed at a premium for nothing.
|
|
609
|
+
* - `no-difference` — the multipliers cancelled out. This is where automatic
|
|
610
|
+
* caching with a 1x write rate lands when nothing was ever read.
|
|
611
|
+
* - `not-attempted` — no cache tokens at all, in either direction.
|
|
612
|
+
* - `unpriced` — cache tokens with no prices behind them, so there is no
|
|
613
|
+
* comparison to make. Saying nothing is the only honest answer.
|
|
614
|
+
*/
|
|
615
|
+
export type CacheVerdict = 'paid-off' | 'lost-money' | 'no-difference' | 'not-attempted' | 'unpriced';
|
|
616
|
+
/**
|
|
617
|
+
* Did caching pay for itself?
|
|
618
|
+
*
|
|
619
|
+
* The question nothing else in this package can answer, and the one that decides
|
|
620
|
+
* whether the advice the rest of it gives was right. Trazum tells people to cache;
|
|
621
|
+
* on Anthropic a cache **write** costs 1.25x plain input at the 5-minute TTL and
|
|
622
|
+
* **2x** at the 1-hour one, so a prefix that changes faster than it is reused is
|
|
623
|
+
* billed at a premium and returns nothing. That workload would be cheaper with
|
|
624
|
+
* caching switched off, and no other report in this repository would ever say so.
|
|
625
|
+
*
|
|
626
|
+
* The counterfactual is exact, which is why this is allowed to exist here at all:
|
|
627
|
+
* caching changes the multiplier on a token, never the token. Had `cache_control`
|
|
628
|
+
* not been set, the identical prefix would have gone up as ordinary input at 1x.
|
|
629
|
+
* So `withoutCachingUsd` is not an estimate of a different call — it is the same
|
|
630
|
+
* call, arithmetic away.
|
|
631
|
+
*
|
|
632
|
+
* Worth running per label as well as over the whole log. A profitable cache on one
|
|
633
|
+
* workload and a bleeding one on another net out to a comfortable-looking total,
|
|
634
|
+
* and the aggregate is exactly where a loss hides.
|
|
635
|
+
*/
|
|
636
|
+
export declare function cacheEconomics(breakdown: UsageBreakdown): CacheEconomics;
|
|
219
637
|
//# sourceMappingURL=usage.d.ts.map
|
package/dist/usage.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"usage.d.ts","sourceRoot":"","sources":["../src/usage.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"usage.d.ts","sourceRoot":"","sources":["../src/usage.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAC5D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAErD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6DG;AAEH,gEAAgE;AAChE,MAAM,WAAW,WAAW;IAC1B,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,WAAW,EAAE,MAAM,CAAC;IACpB,0EAA0E;IAC1E,eAAe,EAAE,MAAM,CAAC;IACxB,oEAAoE;IACpE,kBAAkB,EAAE,MAAM,CAAC;IAC3B,yEAAyE;IACzE,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;;;;;;OAOG;IACH,aAAa,EAAE,OAAO,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB;;;;;;;OAOG;IACH,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB;;;;;;;;;;;;;OAaG;IACH,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB;;;;;;;;;OASG;IACH,EAAE,EAAE,MAAM,GAAG,IAAI,CAAC;IAClB;;;;;;;;OAQG;IACH,SAAS,EAAE,OAAO,GAAG,IAAI,CAAC;CAC3B;AAED,+DAA+D;AAC/D,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;;;;;;;;;;OAYG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;;;;;;OASG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;;;;OAOG;IACH,0BAA0B,EAAE,MAAM,CAAC;IACnC;;;;;;;;;;;;OAYG;IACH,wBAAwB,EAAE,MAAM,CAAC;IACjC;;;;;;;OAOG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kEAAkE;IAClE,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,kBAAkB;IACjC,4BAA4B;IAC5B,KAAK,EAAE,cAAc,CAAC;IACtB,yEAAyE;IACzE,OAAO,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,cAAc,CAAA;KAAE,CAAC,CAAC;IAC7D,qCAAqC;IACrC,OAAO,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,cAAc,CAAA;KAAE,CAAC,CAAC;IAC7D;;;;;;;;OAQG;IACH,eAAe,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,cAAc,CAAA;KAAE,CAAC,CAAC;IACpF;;;;;;OAMG;IACH,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB;;;;;;;;;;;;OAYG;IACH,QAAQ,EAAE,cAAc,CAAC;IACzB;;;;;;;OAOG;IACH,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB;;;;;;OAMG;IACH,aAAa,EAAE,kBAAkB,EAAE,CAAC;IACpC,mDAAmD;IACnD,WAAW,EAAE,OAAO,CAAC;IACrB;;;;;OAKG;IACH,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B;;;;;;;;;OASG;IACH,WAAW,EAAE,UAAU,EAAE,CAAC;IAC1B;;;;;;;;OAQG;IACH,aAAa,EAAE,aAAa,EAAE,CAAC;IAC/B;;;;;;;;;;;OAWG;IACH,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAC7D;;;;;;;;;;;OAWG;IACH,UAAU,EAAE,KAAK,CAAC;QAChB,yBAAyB;QACzB,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,4EAA4E;QAC5E,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC,CAAC;IACH;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,cAAc,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/C;;;;;;;;;;;;;;OAcG;IACH,aAAa,EAAE;QACb,qCAAqC;QACrC,KAAK,EAAE,MAAM,CAAC;QACd,4DAA4D;QAC5D,OAAO,EAAE,MAAM,CAAC;QAChB,yCAAyC;QACzC,EAAE,EAAE,MAAM,CAAC;QACX,uDAAuD;QACvD,UAAU,EAAE,MAAM,CAAC;QACnB,6DAA6D;QAC7D,QAAQ,EAAE,MAAM,CAAC;QACjB,+EAA+E;QAC/E,WAAW,EAAE,MAAM,CAAC;QACpB,8EAA8E;QAC9E,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF;;;;;;;;;;;;;;;OAeG;IACH,WAAW,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjE;;;;;;OAMG;IACH,WAAW,EAAE,WAAW,EAAE,CAAC;IAC3B;;;;;;;;;OASG;IACH,UAAU,EAAE;QAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,eAAe,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAC/F;;;;;;;;OAQG;IACH,qBAAqB,EAAE,qBAAqB,EAAE,CAAC;IAC/C;;;;;;;OAOG;IACH,YAAY,EAAE,gBAAgB,EAAE,CAAC;CAClC;AAED,uEAAuE;AACvE,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;CAChB;AAkHD,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI,CAwH/D;AAuBD;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,UAAU,KAAK,CAAC;AAoF7B,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,gBAAgB,CAAC;IAC5B,gFAAgF;IAChF,EAAE,CAAC,EAAE,IAAI,CAAC;IACV;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,mBAAmB,GAAG,kBAAkB,CA+N3F;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,QAAQ,CAAC,SAAS,EAAE,cAAc,GAAG,WAAW,CAS/D;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,cAAc,GAAG,MAAM,GAAG,IAAI,CAKrE;AAED,gFAAgF;AAChF,MAAM,WAAW,cAAc;IAC7B,sEAAsE;IACtE,QAAQ,EAAE,MAAM,CAAC;IACjB,uEAAuE;IACvE,iBAAiB,EAAE,MAAM,CAAC;IAC1B;;;;;;OAMG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;;;;OAOG;IACH,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,OAAO,EAAE,YAAY,CAAC;IACtB;;;;;;;;OAQG;IACH,iBAAiB,EAAE,MAAM,CAAC;IAC1B;;;;;;;;;;;OAWG;IACH,gBAAgB,EAAE,YAAY,CAAC;CAChC;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,YAAY,GACpB,UAAU,GACV,YAAY,GACZ,eAAe,GACf,eAAe,GACf,UAAU,CAAC;AAcf;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,cAAc,CAAC,SAAS,EAAE,cAAc,GAAG,cAAc,CA6CxE"}
|