@trazum/core 1.10.0 → 1.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/against.d.ts +32 -0
- package/dist/against.d.ts.map +1 -0
- package/dist/against.js +34 -0
- package/dist/against.js.map +1 -0
- package/dist/config-schema.d.ts +42 -1
- package/dist/config-schema.d.ts.map +1 -1
- package/dist/config-schema.js +61 -0
- package/dist/config-schema.js.map +1 -1
- package/dist/conversation.d.ts +121 -0
- package/dist/conversation.d.ts.map +1 -0
- package/dist/conversation.js +157 -0
- package/dist/conversation.js.map +1 -0
- package/dist/csv.d.ts +61 -0
- package/dist/csv.d.ts.map +1 -0
- package/dist/csv.js +149 -0
- package/dist/csv.js.map +1 -0
- package/dist/evaluate.d.ts +24 -0
- package/dist/evaluate.d.ts.map +1 -1
- package/dist/evaluate.js +5 -2
- package/dist/evaluate.js.map +1 -1
- package/dist/index.d.ts +25 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +28 -1
- package/dist/index.js.map +1 -1
- package/dist/input-shape.d.ts +104 -0
- package/dist/input-shape.d.ts.map +1 -0
- package/dist/input-shape.js +132 -0
- package/dist/input-shape.js.map +1 -0
- package/dist/levers.d.ts +151 -0
- package/dist/levers.d.ts.map +1 -0
- package/dist/levers.js +160 -0
- package/dist/levers.js.map +1 -0
- package/dist/node.d.ts +1 -1
- package/dist/node.d.ts.map +1 -1
- package/dist/output-shape.d.ts +96 -0
- package/dist/output-shape.d.ts.map +1 -0
- package/dist/output-shape.js +145 -0
- package/dist/output-shape.js.map +1 -0
- package/dist/pricing-overlay.d.ts +1 -1
- package/dist/pricing-overlay.d.ts.map +1 -1
- package/dist/pricing-overlay.js +46 -0
- package/dist/pricing-overlay.js.map +1 -1
- package/dist/repeats.d.ts +75 -0
- package/dist/repeats.d.ts.map +1 -0
- package/dist/repeats.js +82 -0
- package/dist/repeats.js.map +1 -0
- package/dist/reprice.d.ts +143 -0
- package/dist/reprice.d.ts.map +1 -0
- package/dist/reprice.js +82 -0
- package/dist/reprice.js.map +1 -0
- package/dist/session-cost.d.ts +70 -0
- package/dist/session-cost.d.ts.map +1 -0
- package/dist/session-cost.js +90 -0
- package/dist/session-cost.js.map +1 -0
- package/dist/session-ledger.d.ts +77 -0
- package/dist/session-ledger.d.ts.map +1 -0
- package/dist/session-ledger.js +99 -0
- package/dist/session-ledger.js.map +1 -0
- package/dist/ttl-fit.d.ts +103 -0
- package/dist/ttl-fit.d.ts.map +1 -0
- package/dist/ttl-fit.js +184 -0
- package/dist/ttl-fit.js.map +1 -0
- package/dist/usage.d.ts +434 -16
- package/dist/usage.d.ts.map +1 -1
- package/dist/usage.js +383 -23
- package/dist/usage.js.map +1 -1
- package/package.json +1 -1
- package/src/against.ts +48 -0
- package/src/config-schema.ts +106 -0
- package/src/conversation.ts +305 -0
- package/src/csv.ts +184 -0
- package/src/evaluate.ts +33 -3
- package/src/index.ts +51 -1
- package/src/input-shape.ts +259 -0
- package/src/levers.ts +331 -0
- package/src/node.ts +1 -1
- package/src/output-shape.ts +254 -0
- package/src/pricing-overlay.ts +52 -1
- package/src/repeats.ts +166 -0
- package/src/reprice.ts +227 -0
- package/src/session-cost.ts +170 -0
- package/src/session-ledger.ts +189 -0
- package/src/ttl-fit.ts +251 -0
- package/src/usage.ts +795 -7
package/src/config-schema.ts
CHANGED
|
@@ -48,6 +48,21 @@ export interface BaselineConfig {
|
|
|
48
48
|
maxGrowthPct?: number;
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
+
/**
|
|
52
|
+
* Money budgets, in dollars, for the log-reading side of the tool.
|
|
53
|
+
*
|
|
54
|
+
* A budget for a workload that made no calls is **not** a pass and not a
|
|
55
|
+
* failure: it is a measurement that did not happen, and the report says so
|
|
56
|
+
* rather than reporting green over an absence. That is the same three-state
|
|
57
|
+
* rule the counts, the timestamps and the stop reasons all follow.
|
|
58
|
+
*/
|
|
59
|
+
export interface SpendConfig {
|
|
60
|
+
/** Whole-log budget. `--max-usd` overrides it. */
|
|
61
|
+
maxUsd?: number;
|
|
62
|
+
/** Per-label budgets, each gated against that label's own spend. */
|
|
63
|
+
byLabel?: Record<string, number>;
|
|
64
|
+
}
|
|
65
|
+
|
|
51
66
|
export interface TrazumConfig {
|
|
52
67
|
level?: RuleLevel;
|
|
53
68
|
locale?: Locale;
|
|
@@ -59,6 +74,32 @@ export interface TrazumConfig {
|
|
|
59
74
|
* leaving it to be inferred.
|
|
60
75
|
*/
|
|
61
76
|
budgets?: Record<string, number>;
|
|
77
|
+
/**
|
|
78
|
+
* Which prompt file each usage-log label sends, so `profile` can close the
|
|
79
|
+
* loop it opens.
|
|
80
|
+
*
|
|
81
|
+
* `profile` can say "caching loses money on `support-rag`" and nothing more —
|
|
82
|
+
* the log carries counts, not content. With this map it reads the named file
|
|
83
|
+
* and says *why*: where the first placeholder sits, how many stable tokens
|
|
84
|
+
* never reach the cacheable prefix, and whether the model's minimum is met at
|
|
85
|
+
* all. The file is whatever is in the repository today, which may not be what
|
|
86
|
+
* produced the log, and the report says so.
|
|
87
|
+
*/
|
|
88
|
+
labels?: Record<string, string>;
|
|
89
|
+
/**
|
|
90
|
+
* Money budgets for `trazum profile`, in dollars.
|
|
91
|
+
*
|
|
92
|
+
* `budgets` gates the tokens a prompt file may hold; this gates the dollars
|
|
93
|
+
* a usage log records — the same difference `check` and `profile` have
|
|
94
|
+
* everywhere else. Written in the repository rather than passed as a flag
|
|
95
|
+
* because a per-workload budget is a policy several people agree on, and a
|
|
96
|
+
* policy that lives in one CI invocation is a policy nobody can read.
|
|
97
|
+
*
|
|
98
|
+
* `maxUsd` is the default for `--max-usd`; `byLabel` gates each named
|
|
99
|
+
* workload against its own limit in the same run. A flag still wins over
|
|
100
|
+
* the config, as everywhere in this tool.
|
|
101
|
+
*/
|
|
102
|
+
spend?: SpendConfig;
|
|
62
103
|
/** Default for `trazum diff --max-growth`, in tokens. */
|
|
63
104
|
maxGrowth?: number;
|
|
64
105
|
/**
|
|
@@ -94,6 +135,8 @@ export const CONFIG_KEYS = [
|
|
|
94
135
|
'disable',
|
|
95
136
|
'usage',
|
|
96
137
|
'budgets',
|
|
138
|
+
'labels',
|
|
139
|
+
'spend',
|
|
97
140
|
'maxGrowth',
|
|
98
141
|
'baseline',
|
|
99
142
|
'extensions',
|
|
@@ -102,6 +145,8 @@ export const CONFIG_KEYS = [
|
|
|
102
145
|
|
|
103
146
|
export const CONFIG_BASELINE_KEYS = ['path', 'maxGrowthTokens', 'maxGrowthPct'] as const;
|
|
104
147
|
|
|
148
|
+
export const CONFIG_SPEND_KEYS = ['maxUsd', 'byLabel'] as const;
|
|
149
|
+
|
|
105
150
|
export const CONFIG_USAGE_KEYS = [
|
|
106
151
|
'model',
|
|
107
152
|
'callsPerMonth',
|
|
@@ -217,6 +262,31 @@ function parseUsage(raw: unknown, source: string): Partial<UsageProfile> {
|
|
|
217
262
|
*/
|
|
218
263
|
const IS_ABSOLUTE = /^(?:[/\\]|[A-Za-z]:[/\\])/;
|
|
219
264
|
|
|
265
|
+
function parseLabels(raw: unknown, source: string): Record<string, string> {
|
|
266
|
+
if (!isPlainObject(raw)) throw new ConfigError('"labels" must be an object', source);
|
|
267
|
+
|
|
268
|
+
const labels: Record<string, string> = {};
|
|
269
|
+
for (const [label, value] of Object.entries(raw)) {
|
|
270
|
+
if (label.length === 0) {
|
|
271
|
+
throw new ConfigError('"labels" has an empty label', source);
|
|
272
|
+
}
|
|
273
|
+
if (typeof value !== 'string' || value.length === 0) {
|
|
274
|
+
throw new ConfigError(`labels["${label}"] must be a file path`, source);
|
|
275
|
+
}
|
|
276
|
+
// Same boundary as budgets, for the same reason: an absolute path or one
|
|
277
|
+
// that climbs out with ".." points outside the project, and both are
|
|
278
|
+
// mistakes worth naming rather than files worth reading.
|
|
279
|
+
if (IS_ABSOLUTE.test(value) || value.includes('..')) {
|
|
280
|
+
throw new ConfigError(
|
|
281
|
+
`labels["${label}"] must be a relative path inside the project`,
|
|
282
|
+
source,
|
|
283
|
+
);
|
|
284
|
+
}
|
|
285
|
+
labels[label] = value;
|
|
286
|
+
}
|
|
287
|
+
return labels;
|
|
288
|
+
}
|
|
289
|
+
|
|
220
290
|
function parseBudgets(raw: unknown, source: string): Record<string, number> {
|
|
221
291
|
if (!isPlainObject(raw)) throw new ConfigError('"budgets" must be an object', source);
|
|
222
292
|
|
|
@@ -243,6 +313,40 @@ function parseBudgets(raw: unknown, source: string): Record<string, number> {
|
|
|
243
313
|
return budgets;
|
|
244
314
|
}
|
|
245
315
|
|
|
316
|
+
/**
|
|
317
|
+
* Validates the `spend` block.
|
|
318
|
+
*
|
|
319
|
+
* Dollars, not tokens, so non-integers are legitimate — $0.50 is a budget
|
|
320
|
+
* somebody means. Negative is not: a budget below zero can only fail, which
|
|
321
|
+
* makes it a mistake dressed as a policy. An empty label is rejected for the
|
|
322
|
+
* reason the empty string is the unlabelled bucket's sentinel: a config that
|
|
323
|
+
* meant "calls with no label" should say so through a real key, not through a
|
|
324
|
+
* value that collides with an internal one.
|
|
325
|
+
*/
|
|
326
|
+
function parseSpend(raw: unknown, source: string): SpendConfig {
|
|
327
|
+
if (!isPlainObject(raw)) throw new ConfigError('"spend" must be an object', source);
|
|
328
|
+
rejectUnknownKeys(raw, CONFIG_SPEND_KEYS, source, 'spend.');
|
|
329
|
+
|
|
330
|
+
const spend: SpendConfig = {};
|
|
331
|
+
if (raw.maxUsd !== undefined) {
|
|
332
|
+
spend.maxUsd = requireNonNegativeNumber(raw.maxUsd, 'spend.maxUsd', source);
|
|
333
|
+
}
|
|
334
|
+
if (raw.byLabel !== undefined) {
|
|
335
|
+
if (!isPlainObject(raw.byLabel)) {
|
|
336
|
+
throw new ConfigError('"spend.byLabel" must be an object', source);
|
|
337
|
+
}
|
|
338
|
+
const byLabel: Record<string, number> = {};
|
|
339
|
+
for (const [label, value] of Object.entries(raw.byLabel)) {
|
|
340
|
+
if (label.trim().length === 0) {
|
|
341
|
+
throw new ConfigError('"spend.byLabel" has an empty label', source);
|
|
342
|
+
}
|
|
343
|
+
byLabel[label] = requireNonNegativeNumber(value, `spend.byLabel["${label}"]`, source);
|
|
344
|
+
}
|
|
345
|
+
spend.byLabel = byLabel;
|
|
346
|
+
}
|
|
347
|
+
return spend;
|
|
348
|
+
}
|
|
349
|
+
|
|
246
350
|
/**
|
|
247
351
|
* Validates the `baseline` block.
|
|
248
352
|
*
|
|
@@ -390,6 +494,8 @@ export function parseConfig(raw: string, source = CONFIG_FILENAME): TrazumConfig
|
|
|
390
494
|
|
|
391
495
|
if (document.usage !== undefined) config.usage = parseUsage(document.usage, source);
|
|
392
496
|
if (document.budgets !== undefined) config.budgets = parseBudgets(document.budgets, source);
|
|
497
|
+
if (document.labels !== undefined) config.labels = parseLabels(document.labels, source);
|
|
498
|
+
if (document.spend !== undefined) config.spend = parseSpend(document.spend, source);
|
|
393
499
|
if (document.baseline !== undefined) {
|
|
394
500
|
config.baseline = parseBaselineConfig(document.baseline, source);
|
|
395
501
|
}
|
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
import { effectivePricing, multipliersFor } from './pricing.js';
|
|
2
|
+
import { UNLABELLED } from './usage.js';
|
|
3
|
+
import type { PricingCatalogue } from './pricing.js';
|
|
4
|
+
import type { UsageRecord } from './usage.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* What re-sending the conversation costs.
|
|
8
|
+
*
|
|
9
|
+
* ## The line nothing was watching
|
|
10
|
+
*
|
|
11
|
+
* A chat or agent workload sends the whole conversation back on every turn. Turn
|
|
12
|
+
* one is a system prompt and a question; turn twenty is a system prompt and
|
|
13
|
+
* nineteen previous exchanges and a question. The input grows linearly with the
|
|
14
|
+
* turn count, and on an agent bill that growth is routinely the largest single
|
|
15
|
+
* line — larger than the prompt, larger than the answers.
|
|
16
|
+
*
|
|
17
|
+
* Nothing in this package could see it. A prompt file shows the system prompt and
|
|
18
|
+
* not the history. A total shows the sum and not the shape. Even `profile` reported
|
|
19
|
+
* "input is 71% of this bill" without being able to say that most of that input was
|
|
20
|
+
* the same sentences, sent again.
|
|
21
|
+
*
|
|
22
|
+
* ## What it will and will not claim
|
|
23
|
+
*
|
|
24
|
+
* The honest figure is a **ceiling**, and the token half of it is exact: the
|
|
25
|
+
* input tokens beyond every turn being the size of the session's smallest turn.
|
|
26
|
+
* That quantity is order-independent — the first-seen anchor made the identical
|
|
27
|
+
* workload vanish when exported newest-first — and it is immune to billing
|
|
28
|
+
* rates: a cost-based anchor charged an ordinary 5-minute-TTL agent 77.5%
|
|
29
|
+
* "growth" on a conversation that never grew, because a cache-miss turn costs
|
|
30
|
+
* 12.5x a cache-hit turn of the same size. The dollars are that token share of
|
|
31
|
+
* what the session actually spent, at its own blended rate.
|
|
32
|
+
*
|
|
33
|
+
* It is a ceiling and not a saving because part of that growth is the user's own
|
|
34
|
+
* new messages, which nobody can truncate away, and this module cannot tell those
|
|
35
|
+
* apart from re-sent history — it sees counts, not content. Reporting the ceiling
|
|
36
|
+
* as an opportunity would be the flattering direction; reporting nothing because
|
|
37
|
+
* the exact split is unknowable would be worse. So it reports the bound and says
|
|
38
|
+
* what it is.
|
|
39
|
+
*
|
|
40
|
+
* ## The session key never leaves this module
|
|
41
|
+
*
|
|
42
|
+
* A session identifier is somebody's conversation, and in a real log it is often an
|
|
43
|
+
* account id, a ticket number or an email. It is used to group calls and count
|
|
44
|
+
* turns; **no figure reported anywhere carries it**, and every result is aggregated
|
|
45
|
+
* per label. The promise that a usage log handed to Trazum contains no content is
|
|
46
|
+
* only worth something if nothing identifying comes back out either.
|
|
47
|
+
*/
|
|
48
|
+
|
|
49
|
+
/** How one label-and-model slice grows across a conversation. */
|
|
50
|
+
export interface ConversationGrowth {
|
|
51
|
+
label: string;
|
|
52
|
+
model: string;
|
|
53
|
+
modelName: string;
|
|
54
|
+
/** How many distinct conversations were seen. Never which ones. */
|
|
55
|
+
sessions: number;
|
|
56
|
+
calls: number;
|
|
57
|
+
/** Mean input tokens on the smallest turn of a conversation. */
|
|
58
|
+
minTurnTokens: number;
|
|
59
|
+
/** Mean input tokens on the largest turn. */
|
|
60
|
+
maxTurnTokens: number;
|
|
61
|
+
/** Turns in the longest conversation seen. */
|
|
62
|
+
longestSession: number;
|
|
63
|
+
/** Input-side spend: plain input, cache reads and cache writes. */
|
|
64
|
+
inputUsd: number;
|
|
65
|
+
/** What that would have been if every turn had cost what its cheapest turn did. */
|
|
66
|
+
flatUsd: number;
|
|
67
|
+
/**
|
|
68
|
+
* `inputUsd - flatUsd`. **A ceiling on what removing conversation growth could
|
|
69
|
+
* be worth, not a saving** — part of it is the user's own new messages.
|
|
70
|
+
*/
|
|
71
|
+
growthUsd: number;
|
|
72
|
+
/** `growthUsd` as a fraction of the whole bill in the log. */
|
|
73
|
+
shareOfBill: number;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export interface ConversationOptions {
|
|
77
|
+
catalogue: PricingCatalogue;
|
|
78
|
+
on?: Date;
|
|
79
|
+
/**
|
|
80
|
+
* Slices whose growth is below this share of the bill are dropped, and slices
|
|
81
|
+
* shorter than `minTurns` never count as conversations at all.
|
|
82
|
+
*/
|
|
83
|
+
minShare?: number;
|
|
84
|
+
/**
|
|
85
|
+
* Conversations shorter than this are ignored.
|
|
86
|
+
*
|
|
87
|
+
* Two turns is not a conversation, it is a retry — and a workload that never
|
|
88
|
+
* exceeds two turns has no growth to measure, so including it would put a row on
|
|
89
|
+
* screen whose figure is arithmetic noise. Default 3.
|
|
90
|
+
*/
|
|
91
|
+
minTurns?: number;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Input-side cost of one call at its own model's rates. */
|
|
95
|
+
function inputCostOf(record: UsageRecord, catalogue: PricingCatalogue, on: Date): number | null {
|
|
96
|
+
const model = catalogue.byId.get(record.model);
|
|
97
|
+
if (!model) return null;
|
|
98
|
+
const { inputPerMTok } = effectivePricing(model, on);
|
|
99
|
+
const rates = multipliersFor(model);
|
|
100
|
+
const per = (tokens: number, rate: number): number => (tokens / 1_000_000) * rate;
|
|
101
|
+
return (
|
|
102
|
+
per(record.inputTokens, inputPerMTok) +
|
|
103
|
+
per(record.cacheReadTokens, inputPerMTok * rates.cacheRead) +
|
|
104
|
+
per(record.cacheWrite5mTokens, inputPerMTok * rates.cacheWrite5m) +
|
|
105
|
+
per(record.cacheWrite1hTokens, inputPerMTok * rates.cacheWrite1h)
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Every input-side token of one call, whatever rate it was billed at. */
|
|
110
|
+
const inputTokensOf = (r: UsageRecord): number =>
|
|
111
|
+
r.inputTokens + r.cacheReadTokens + r.cacheWrite5mTokens + r.cacheWrite1hTokens;
|
|
112
|
+
|
|
113
|
+
interface Session {
|
|
114
|
+
turns: number;
|
|
115
|
+
/**
|
|
116
|
+
* The smallest turn by **tokens**, not by billed cost.
|
|
117
|
+
*
|
|
118
|
+
* Two faults taught this shape. Anchoring on the first record seen made the
|
|
119
|
+
* measurement depend on the order of the log: the identical workload exported
|
|
120
|
+
* newest-first computed a *negative* growth and the section silently vanished.
|
|
121
|
+
* Anchoring on the cheapest turn's *cost* fixed the ordering and introduced
|
|
122
|
+
* the second fault: per-turn cost varies with the cache multiplier even when
|
|
123
|
+
* the input never grows — an identical 10,000-token turn costs 12.5x more as
|
|
124
|
+
* a cache write than as a cache read — so an ordinary 5-minute-TTL agent
|
|
125
|
+
* whose conversation stayed flat reported 77.5% of its bill as "conversation
|
|
126
|
+
* growth", and the report recommended trimming history that was not there.
|
|
127
|
+
*
|
|
128
|
+
* Tokens are what growth *is*, and they are immune to both: order-independent,
|
|
129
|
+
* and identical however each turn happened to be billed.
|
|
130
|
+
*/
|
|
131
|
+
minTokens: number;
|
|
132
|
+
maxTokens: number;
|
|
133
|
+
totalTokens: number;
|
|
134
|
+
totalUsd: number;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Measures what conversation growth costs, from records that carry a session.
|
|
139
|
+
*
|
|
140
|
+
* Records without one are skipped rather than lumped together: calls from
|
|
141
|
+
* different conversations pushed into a single bucket would report a turn count
|
|
142
|
+
* that is really a call count, and a growth figure derived from it would be
|
|
143
|
+
* arithmetic performed on a fiction.
|
|
144
|
+
*
|
|
145
|
+
* Takes records rather than a report because turn order is the whole measurement,
|
|
146
|
+
* and a breakdown has already thrown it away.
|
|
147
|
+
*/
|
|
148
|
+
export interface ConversationTracker {
|
|
149
|
+
/** Feed one parsed record. Records without a session are ignored. */
|
|
150
|
+
add(record: UsageRecord): void;
|
|
151
|
+
/** The finished measurement, once the whole bill is known. */
|
|
152
|
+
finish(totalUsd: number): ConversationGrowth[];
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* An accumulator, so a profile can measure this in the pass it already makes.
|
|
157
|
+
*
|
|
158
|
+
* The alternative was holding every record to hand to a pure function afterwards,
|
|
159
|
+
* and a usage log is measured in megabytes — a profile that needs the whole file in
|
|
160
|
+
* memory to answer one question is a profile that stops working on the logs most
|
|
161
|
+
* worth reading. What this holds is bounded by the number of **conversations**, and
|
|
162
|
+
* only ever four numbers each.
|
|
163
|
+
*/
|
|
164
|
+
export function createConversationTracker(options: ConversationOptions): ConversationTracker {
|
|
165
|
+
const { catalogue, on = new Date(), minShare = 0.01, minTurns = 3 } = options;
|
|
166
|
+
|
|
167
|
+
// Keyed on the pair, then on the session inside it. A newline cannot occur in a
|
|
168
|
+
// model id, and both halves are trimmed strings.
|
|
169
|
+
const slices = new Map<string, Map<string, Session>>();
|
|
170
|
+
|
|
171
|
+
const add = (record: UsageRecord): void => {
|
|
172
|
+
if (record.session === null) return;
|
|
173
|
+
const cost = inputCostOf(record, catalogue, on);
|
|
174
|
+
// An unpriced model contributes no dollars anywhere else either; including it
|
|
175
|
+
// here would report growth of zero on a workload that grew.
|
|
176
|
+
if (cost === null) return;
|
|
177
|
+
|
|
178
|
+
const sliceKey = `${record.label ?? UNLABELLED}\n${record.model}`;
|
|
179
|
+
let sessions = slices.get(sliceKey);
|
|
180
|
+
if (!sessions) {
|
|
181
|
+
sessions = new Map();
|
|
182
|
+
slices.set(sliceKey, sessions);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
const tokens = inputTokensOf(record);
|
|
186
|
+
const existing = sessions.get(record.session);
|
|
187
|
+
if (!existing) {
|
|
188
|
+
sessions.set(record.session, {
|
|
189
|
+
turns: 1,
|
|
190
|
+
minTokens: tokens,
|
|
191
|
+
maxTokens: tokens,
|
|
192
|
+
totalTokens: tokens,
|
|
193
|
+
totalUsd: cost,
|
|
194
|
+
});
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
197
|
+
existing.turns += 1;
|
|
198
|
+
existing.minTokens = Math.min(existing.minTokens, tokens);
|
|
199
|
+
existing.maxTokens = Math.max(existing.maxTokens, tokens);
|
|
200
|
+
existing.totalTokens += tokens;
|
|
201
|
+
existing.totalUsd += cost;
|
|
202
|
+
};
|
|
203
|
+
|
|
204
|
+
const finish = (totalUsd: number): ConversationGrowth[] => {
|
|
205
|
+
const out: ConversationGrowth[] = [];
|
|
206
|
+
|
|
207
|
+
for (const [sliceKey, sessions] of slices) {
|
|
208
|
+
const split = sliceKey.indexOf('\n');
|
|
209
|
+
const label = sliceKey.slice(0, split);
|
|
210
|
+
const modelId = sliceKey.slice(split + 1);
|
|
211
|
+
const model = catalogue.byId.get(modelId);
|
|
212
|
+
if (!model) continue;
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Only conversations long enough to have grown. A two-turn session has one
|
|
216
|
+
* step of growth and is as likely to be a retry, and averaging it in drags the
|
|
217
|
+
* measured shape towards flat — understating the real thing, which is the
|
|
218
|
+
* direction that flatters.
|
|
219
|
+
*/
|
|
220
|
+
const long = [...sessions.values()].filter((s) => s.turns >= minTurns);
|
|
221
|
+
if (long.length === 0) continue;
|
|
222
|
+
|
|
223
|
+
let inputUsd = 0;
|
|
224
|
+
let flatUsd = 0;
|
|
225
|
+
let minTokens = 0;
|
|
226
|
+
let maxTokens = 0;
|
|
227
|
+
let calls = 0;
|
|
228
|
+
let longestSession = 0;
|
|
229
|
+
|
|
230
|
+
for (const session of long) {
|
|
231
|
+
inputUsd += session.totalUsd;
|
|
232
|
+
/**
|
|
233
|
+
* The growth is measured in tokens — `totalTokens - minTokens·turns`,
|
|
234
|
+
* which is exact, order-independent, and zero for a conversation whose
|
|
235
|
+
* turns never change size however each one was billed. The money is that
|
|
236
|
+
* token share of what the session actually spent: pricing the excess at
|
|
237
|
+
* any single rate would either overstate it (full input rate, when most
|
|
238
|
+
* re-sent history is cache-read cheap) or move with billing noise (the
|
|
239
|
+
* cheapest turn's rate, which is what mis-billed a flat cached agent).
|
|
240
|
+
*/
|
|
241
|
+
const flatTokens = session.minTokens * session.turns;
|
|
242
|
+
flatUsd +=
|
|
243
|
+
session.totalTokens > 0
|
|
244
|
+
? session.totalUsd * (flatTokens / session.totalTokens)
|
|
245
|
+
: session.totalUsd;
|
|
246
|
+
minTokens += session.minTokens;
|
|
247
|
+
maxTokens += session.maxTokens;
|
|
248
|
+
calls += session.turns;
|
|
249
|
+
longestSession = Math.max(longestSession, session.turns);
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
const growthUsd = inputUsd - flatUsd;
|
|
253
|
+
const shareOfBill = totalUsd > 0 ? growthUsd / totalUsd : 0;
|
|
254
|
+
/**
|
|
255
|
+
* Below the attention threshold — **and that covers shrinking conversations
|
|
256
|
+
* too**, because a negative share is below any threshold at or above zero.
|
|
257
|
+
*
|
|
258
|
+
* There was a separate `growthUsd <= 0` check here. No mutation could break
|
|
259
|
+
* it: every case it caught, this one caught first. A guard nothing can
|
|
260
|
+
* distinguish is not defence in depth, it is a second place for the intent to
|
|
261
|
+
* drift from the code, so the intent lives in this comment instead.
|
|
262
|
+
*
|
|
263
|
+
* The case it was written for is ordinary: an opening turn carrying an
|
|
264
|
+
* attachment or a retrieved document is bigger than everything after it, and
|
|
265
|
+
* reporting that as "conversation growth" would be a negative ceiling
|
|
266
|
+
* presented as an opportunity.
|
|
267
|
+
*/
|
|
268
|
+
if (shareOfBill < minShare) continue;
|
|
269
|
+
|
|
270
|
+
out.push({
|
|
271
|
+
label,
|
|
272
|
+
model: modelId,
|
|
273
|
+
modelName: model.displayName,
|
|
274
|
+
sessions: long.length,
|
|
275
|
+
calls,
|
|
276
|
+
minTurnTokens: minTokens / long.length,
|
|
277
|
+
maxTurnTokens: maxTokens / long.length,
|
|
278
|
+
longestSession,
|
|
279
|
+
inputUsd,
|
|
280
|
+
flatUsd,
|
|
281
|
+
growthUsd,
|
|
282
|
+
shareOfBill,
|
|
283
|
+
});
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
return out.sort((a, b) => b.growthUsd - a.growthUsd);
|
|
287
|
+
};
|
|
288
|
+
|
|
289
|
+
return { add, finish };
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* The same measurement over a list of records, for a caller holding one already.
|
|
294
|
+
*
|
|
295
|
+
* `profileUsage` uses the tracker instead, so it never has to keep the log.
|
|
296
|
+
*/
|
|
297
|
+
export function conversationGrowth(
|
|
298
|
+
records: readonly UsageRecord[],
|
|
299
|
+
totalUsd: number,
|
|
300
|
+
options: ConversationOptions,
|
|
301
|
+
): ConversationGrowth[] {
|
|
302
|
+
const tracker = createConversationTracker(options);
|
|
303
|
+
for (const record of records) tracker.add(record);
|
|
304
|
+
return tracker.finish(totalUsd);
|
|
305
|
+
}
|
package/src/csv.ts
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import { UNLABELLED } from './usage.js';
|
|
2
|
+
import type { UsageProfileReport } from './usage.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The profile as a spreadsheet.
|
|
6
|
+
*
|
|
7
|
+
* ## Why a file format is a feature
|
|
8
|
+
*
|
|
9
|
+
* The terminal report is read once and closed. The people who decide what a
|
|
10
|
+
* workload is allowed to cost live in spreadsheets, and handing them a
|
|
11
|
+
* screenshot of a terminal is how a finding stops at the person who ran the
|
|
12
|
+
* command. `--json` is for machines; this is for the pivot table that gets
|
|
13
|
+
* shown to whoever signs off the bill.
|
|
14
|
+
*
|
|
15
|
+
* ## One row per label and model, and no total row
|
|
16
|
+
*
|
|
17
|
+
* `byLabelAndModel` is the grouping a decision is actually made at — routing
|
|
18
|
+
* `classify` to a cheaper model is a question about one label's calls to one
|
|
19
|
+
* model — so it is the grain of the file.
|
|
20
|
+
*
|
|
21
|
+
* **There is deliberately no TOTAL row.** A total inside a data file is the
|
|
22
|
+
* oldest spreadsheet trap there is: somebody sums the column, the total row is
|
|
23
|
+
* included, and every figure downstream is exactly twice what it should be.
|
|
24
|
+
* The sum of this file is the bill, and a spreadsheet can compute it.
|
|
25
|
+
*
|
|
26
|
+
* ## Unpriced models get empty cells, never zeros
|
|
27
|
+
*
|
|
28
|
+
* A model the catalogue does not know has real tokens and unknown dollars.
|
|
29
|
+
* Writing `0` there would be a claim — that those calls were free — and it
|
|
30
|
+
* would survive into every chart built on the file. An empty cell is the
|
|
31
|
+
* absence it actually is, and spreadsheets already know how to skip one.
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
/** Columns, in order. Exported so a test can pin the header rather than a string. */
|
|
35
|
+
export const PROFILE_CSV_COLUMNS = [
|
|
36
|
+
'label',
|
|
37
|
+
'model',
|
|
38
|
+
'calls',
|
|
39
|
+
'input_tokens',
|
|
40
|
+
'cache_read_tokens',
|
|
41
|
+
'cache_write_tokens',
|
|
42
|
+
'output_tokens',
|
|
43
|
+
'input_usd',
|
|
44
|
+
'cache_read_usd',
|
|
45
|
+
'cache_write_usd',
|
|
46
|
+
'output_usd',
|
|
47
|
+
'total_usd',
|
|
48
|
+
] as const;
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* One CSV field, RFC 4180.
|
|
52
|
+
*
|
|
53
|
+
* Labels are arbitrary strings out of somebody's log: a label containing a
|
|
54
|
+
* comma would shift every column after it, and one containing a quote would
|
|
55
|
+
* break the row it sits in. Both are quoted here rather than sanitised,
|
|
56
|
+
* because changing the label would make the file disagree with every other
|
|
57
|
+
* rendering about what the workload is called.
|
|
58
|
+
*
|
|
59
|
+
* A leading `=`, `+`, `-` or `@` is prefixed with an apostrophe: those are
|
|
60
|
+
* how a spreadsheet is told a cell is a formula, and a label out of a log is
|
|
61
|
+
* data. This is the one place a value is altered, and it is altered to stop
|
|
62
|
+
* a log from executing anything when the file is opened.
|
|
63
|
+
*/
|
|
64
|
+
function field(value: string): string {
|
|
65
|
+
const guarded = /^[=+\-@\t\r]/.test(value) ? `'${value}` : value;
|
|
66
|
+
return /[",\n\r]/.test(guarded) ? `"${guarded.replace(/"/g, '""')}"` : guarded;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** A dollar figure with enough places to survive being summed. */
|
|
70
|
+
const usd = (value: number): string => value.toFixed(6);
|
|
71
|
+
|
|
72
|
+
export interface ProfileCsvOptions {
|
|
73
|
+
/** What to call the bucket for calls carrying no label. */
|
|
74
|
+
unlabelled: string;
|
|
75
|
+
/**
|
|
76
|
+
* Which table to write.
|
|
77
|
+
*
|
|
78
|
+
* `slice` is one row per label and model — the grain a routing or budget
|
|
79
|
+
* decision is made at. `day` and `hour` are the time series, which is what
|
|
80
|
+
* a spreadsheet gets asked to chart; keeping them behind a choice rather
|
|
81
|
+
* than in extra columns means every file has one row shape, and a
|
|
82
|
+
* spreadsheet that has to filter before it can sum is a spreadsheet
|
|
83
|
+
* somebody sums wrong.
|
|
84
|
+
*/
|
|
85
|
+
shape?: ProfileCsvShape;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export type ProfileCsvShape = 'slice' | 'day' | 'hour';
|
|
89
|
+
|
|
90
|
+
/** Columns for the per-day series. */
|
|
91
|
+
export const PROFILE_CSV_DAY_COLUMNS = ['day', 'usd', 'calls', 'top_label', 'top_label_usd'] as const;
|
|
92
|
+
|
|
93
|
+
/** Columns for the per-hour-of-UTC-day series. */
|
|
94
|
+
export const PROFILE_CSV_HOUR_COLUMNS = ['hour_utc', 'usd', 'calls'] as const;
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* The report as CSV text, one row per label and model.
|
|
98
|
+
*
|
|
99
|
+
* Rows arrive in the report's own order — largest bill first — because a
|
|
100
|
+
* spreadsheet can re-sort and a reader opening the file should see the
|
|
101
|
+
* expensive workload at the top either way.
|
|
102
|
+
*/
|
|
103
|
+
export function profileToCsv(report: UsageProfileReport, options: ProfileCsvOptions): string {
|
|
104
|
+
if (options.shape === 'day') {
|
|
105
|
+
const rows: string[] = [PROFILE_CSV_DAY_COLUMNS.join(',')];
|
|
106
|
+
for (const day of report.spendByDay) {
|
|
107
|
+
rows.push(
|
|
108
|
+
[
|
|
109
|
+
day.day,
|
|
110
|
+
usd(day.usd),
|
|
111
|
+
String(day.calls),
|
|
112
|
+
// A day whose calls carried no label at all has no top label, and an
|
|
113
|
+
// empty cell is that absence. Naming the unlabelled bucket here
|
|
114
|
+
// would claim a label the log never carried.
|
|
115
|
+
day.topLabel === null
|
|
116
|
+
? ''
|
|
117
|
+
: field(day.topLabel === UNLABELLED ? options.unlabelled : day.topLabel),
|
|
118
|
+
day.topLabel === null ? '' : usd(day.topLabelUsd),
|
|
119
|
+
].join(','),
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
return `${rows.join('\n')}\n`;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
if (options.shape === 'hour') {
|
|
126
|
+
const rows: string[] = [PROFILE_CSV_HOUR_COLUMNS.join(',')];
|
|
127
|
+
for (const hour of report.spendByHour) {
|
|
128
|
+
rows.push([String(hour.hour), usd(hour.usd), String(hour.calls)].join(','));
|
|
129
|
+
}
|
|
130
|
+
return `${rows.join('\n')}\n`;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const rows: string[] = [PROFILE_CSV_COLUMNS.join(',')];
|
|
134
|
+
|
|
135
|
+
for (const { label, model, breakdown } of report.byLabelAndModel) {
|
|
136
|
+
rows.push(
|
|
137
|
+
[
|
|
138
|
+
field(label === UNLABELLED ? options.unlabelled : label),
|
|
139
|
+
field(model),
|
|
140
|
+
String(breakdown.calls),
|
|
141
|
+
String(breakdown.inputTokens),
|
|
142
|
+
String(breakdown.cacheReadTokens),
|
|
143
|
+
String(breakdown.cacheWriteTokens),
|
|
144
|
+
String(breakdown.outputTokens),
|
|
145
|
+
usd(breakdown.inputUsd),
|
|
146
|
+
usd(breakdown.cacheReadUsd),
|
|
147
|
+
usd(breakdown.cacheWriteUsd),
|
|
148
|
+
usd(breakdown.outputUsd),
|
|
149
|
+
usd(breakdown.totalUsd),
|
|
150
|
+
].join(','),
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* The unpriced calls, with their tokens and no dollars.
|
|
156
|
+
*
|
|
157
|
+
* They are absent from `byLabelAndModel` — which holds what could be priced
|
|
158
|
+
* — and leaving them out of the file entirely would make its token columns
|
|
159
|
+
* disagree with the log. `byModel` keeps them, so they are recovered from
|
|
160
|
+
* there, with empty dollar cells rather than zeros.
|
|
161
|
+
*/
|
|
162
|
+
for (const model of report.unpricedModels) {
|
|
163
|
+
const row = report.byModel.find((entry) => entry.model === model);
|
|
164
|
+
if (!row) continue;
|
|
165
|
+
rows.push(
|
|
166
|
+
[
|
|
167
|
+
field(options.unlabelled),
|
|
168
|
+
field(model),
|
|
169
|
+
String(row.breakdown.calls),
|
|
170
|
+
String(row.breakdown.inputTokens),
|
|
171
|
+
String(row.breakdown.cacheReadTokens),
|
|
172
|
+
String(row.breakdown.cacheWriteTokens),
|
|
173
|
+
String(row.breakdown.outputTokens),
|
|
174
|
+
'',
|
|
175
|
+
'',
|
|
176
|
+
'',
|
|
177
|
+
'',
|
|
178
|
+
'',
|
|
179
|
+
].join(','),
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
return `${rows.join('\n')}\n`;
|
|
184
|
+
}
|