@trazum/cli 1.80.1 → 1.81.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/gateway-server.d.ts +38 -1
- package/dist/gateway-server.d.ts.map +1 -1
- package/dist/gateway-server.js +111 -9
- package/dist/gateway-server.js.map +1 -1
- package/dist/i18n/en.d.ts +1 -1
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +377 -372
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/es.d.ts.map +1 -1
- package/dist/i18n/es.js +5 -0
- package/dist/i18n/es.js.map +1 -1
- package/dist/i18n/types.d.ts +6 -0
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/index.js +156 -3
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
- package/src/gateway-server.ts +146 -9
- package/src/i18n/en.ts +379 -372
- package/src/i18n/es.ts +7 -0
- package/src/i18n/types.ts +6 -0
- package/src/index.ts +198 -3
package/src/i18n/en.ts
CHANGED
|
@@ -6,8 +6,8 @@ const count = (value: number): string => value.toLocaleString('en-US');
|
|
|
6
6
|
/**
|
|
7
7
|
* A grouped count and its noun, agreeing.
|
|
8
8
|
*
|
|
9
|
-
* `1 calls` and `1 prompts` were reachable on ordinary input
|
|
10
|
-
* repository with a single prompt
|
|
9
|
+
* `1 calls` and `1 prompts` were reachable on ordinary input, a one-call log, a
|
|
10
|
+
* repository with a single prompt, and two messages in this file already did the
|
|
11
11
|
* agreement by hand while a dozen did not. A helper rather than a dozen ternaries,
|
|
12
12
|
* so the next message written gets it for free.
|
|
13
13
|
*/
|
|
@@ -20,7 +20,7 @@ const ago = (days: number | null): string =>
|
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
/**
|
|
23
|
-
* English catalogue
|
|
23
|
+
* English catalogue, the source of truth.
|
|
24
24
|
*
|
|
25
25
|
* When a message changes here, the other catalogues need the same change.
|
|
26
26
|
* `test/i18n.test.js` in the core enforces the equivalent guarantee for the
|
|
@@ -30,7 +30,7 @@ export const en: CliMessages = {
|
|
|
30
30
|
locale: 'en',
|
|
31
31
|
numberLocale: 'en-US',
|
|
32
32
|
|
|
33
|
-
help: (d, bold) => `${bold('trazum')}
|
|
33
|
+
help: (d, bold) => `${bold('trazum')}, cut the cost of your prompts without losing what they ask for.
|
|
34
34
|
|
|
35
35
|
${bold('USAGE')}
|
|
36
36
|
trazum init [dir] [--dry-run | --yes]
|
|
@@ -92,7 +92,7 @@ ${bold('OPTIONS FOR init')}
|
|
|
92
92
|
found, arithmetic first.
|
|
93
93
|
|
|
94
94
|
It never invents a threshold. A budget is a policy, so init hands you the
|
|
95
|
-
measured figure and leaves the limit to you
|
|
95
|
+
measured figure and leaves the limit to you: a generated config full of
|
|
96
96
|
guessed numbers is one nobody trusts.
|
|
97
97
|
|
|
98
98
|
${bold('OPTIONS FOR conform')}
|
|
@@ -100,7 +100,7 @@ ${bold('OPTIONS FOR conform')}
|
|
|
100
100
|
detecting one: ${d.contracts.join(', ')}.
|
|
101
101
|
--json The report as data.
|
|
102
102
|
|
|
103
|
-
Answers two questions and keeps them apart. Does this document conform
|
|
103
|
+
Answers two questions and keeps them apart. Does this document conform,
|
|
104
104
|
required fields, present and the right type, exits 1 when not. And what can a
|
|
105
105
|
valid document of this shape not answer, with the field that would unlock
|
|
106
106
|
each.
|
|
@@ -111,7 +111,7 @@ ${bold('OPTIONS FOR conform')}
|
|
|
111
111
|
${bold('OPTIONS FOR rollup')}
|
|
112
112
|
--json The roll-up document as data. See docs/format.md.
|
|
113
113
|
--html-out <file> Also write the roll-up as one self-contained HTML
|
|
114
|
-
file
|
|
114
|
+
file, the team-facing report, for whoever pays
|
|
115
115
|
the bill. Each contributor's gaps stay under that
|
|
116
116
|
contributor, and what no merge can measure is in
|
|
117
117
|
the caveat block, impossible to crop.
|
|
@@ -119,13 +119,13 @@ ${bold('OPTIONS FOR rollup')}
|
|
|
119
119
|
Several people's bills, one roll-up. Each contributor runs
|
|
120
120
|
\`trazum profile --json\` where their traffic already is; this merges the
|
|
121
121
|
documents. A format and a merge, not a service: nothing is uploaded and there
|
|
122
|
-
is no account
|
|
122
|
+
is no account: the documents arrive however your team already moves files.
|
|
123
123
|
|
|
124
124
|
Most of the output is what the merge could not do. Each contributor's own
|
|
125
125
|
gaps stay with that contributor rather than being averaged into one figure,
|
|
126
126
|
the findings that need individual calls are named instead of dropped, and
|
|
127
127
|
every roll-up of more than one document says that overlap between
|
|
128
|
-
contributors is unmeasurable
|
|
128
|
+
contributors is unmeasurable: two people exporting the same traffic double
|
|
129
129
|
the bill and no merge of summaries can see it.
|
|
130
130
|
|
|
131
131
|
Exits 1 when a contribution was handed over and could not be merged.
|
|
@@ -148,11 +148,11 @@ ${bold('OPTIONS FOR switch')}
|
|
|
148
148
|
Unlocks the line pricing that evaluation.
|
|
149
149
|
|
|
150
150
|
The decision every what-if serves, priced: what this measured mix costs on
|
|
151
|
-
the candidate, the saving with its sign stated, and
|
|
152
|
-
migration cost
|
|
151
|
+
the candidate, the saving with its sign stated, and, with a declared
|
|
152
|
+
migration cost, the break-even as division on the measured past, days of
|
|
153
153
|
window attached, never a forecast. The evaluation the switch requires is
|
|
154
154
|
priced at this log's own mean call, two calls on the incumbent and one on
|
|
155
|
-
the candidate per case. What it refuses: any sentence about quality
|
|
155
|
+
the candidate per case. What it refuses: any sentence about quality, that
|
|
156
156
|
verdict belongs to trazum route, and the report ends by printing it.
|
|
157
157
|
|
|
158
158
|
${bold('OPTIONS FOR ownrate')}
|
|
@@ -161,7 +161,7 @@ ${bold('OPTIONS FOR ownrate')}
|
|
|
161
161
|
--utilization <0-1> Fraction of the hour actually serving.
|
|
162
162
|
Defaults to 1.
|
|
163
163
|
|
|
164
|
-
Your self-hosted model's $/MTok, derived from your own two numbers
|
|
164
|
+
Your self-hosted model's $/MTok, derived from your own two numbers,
|
|
165
165
|
division, nothing else: no amortisation, no energy model, no guessed
|
|
166
166
|
efficiency. Prints the figure and the pricing-overlay snippet to paste
|
|
167
167
|
into trazum.config.json, so the model you run yourself becomes a
|
|
@@ -173,7 +173,7 @@ ${bold('OPTIONS FOR from-otel')}
|
|
|
173
173
|
-o, --out <file> Write the usage log there instead of stdout.
|
|
174
174
|
|
|
175
175
|
OpenTelemetry GenAI spans, read as a usage log: model, timestamp, label and
|
|
176
|
-
the token counts from each gen_ai.* span
|
|
176
|
+
the token counts from each gen_ai.* span, and nothing else. Prompt content
|
|
177
177
|
and trace ids stay in the span. OTel has no cache TTL split, so cache
|
|
178
178
|
verdicts read "cannot tell" rather than a guessed one. Non-LLM spans are
|
|
179
179
|
counted and skipped, said on stderr.
|
|
@@ -181,19 +181,19 @@ ${bold('OPTIONS FOR from-otel')}
|
|
|
181
181
|
${bold('OPTIONS FOR from-claude-code')}
|
|
182
182
|
--label <name> Stamp every record with one label.
|
|
183
183
|
--label-from-project Label each record with its transcript's project
|
|
184
|
-
directory name, verbatim
|
|
184
|
+
directory name, verbatim, map it to a workload
|
|
185
185
|
name with the config's \`labels\` block.
|
|
186
186
|
-o, --out <file> Write the usage log there instead of stdout.
|
|
187
187
|
|
|
188
188
|
Claude Code's transcripts, read as a usage log: model, timestamp, session
|
|
189
|
-
and the API's own usage object
|
|
189
|
+
and the API's own usage object, the cache TTL split included, and
|
|
190
190
|
nothing else. No message text, no paths, no branches cross the
|
|
191
191
|
conversion. One API call is written as one line per content block, so
|
|
192
192
|
lines are collapsed by request id; what was collapsed or passed over is
|
|
193
193
|
said on stderr, never silently.
|
|
194
194
|
|
|
195
195
|
${bold('OPTIONS FOR position')}
|
|
196
|
-
--html-out <file> The position as one self-contained page
|
|
196
|
+
--html-out <file> The position as one self-contained page, the
|
|
197
197
|
same sentences the terminal prints, caveats
|
|
198
198
|
first, for the person who does not run CLIs.
|
|
199
199
|
--json The position document, contract-checked like
|
|
@@ -212,7 +212,7 @@ ${bold('OPTIONS FOR pulse')}
|
|
|
212
212
|
--json The report as data.
|
|
213
213
|
|
|
214
214
|
Did the things that are supposed to run, run? \`watch --once\` is built for a
|
|
215
|
-
scheduler, and its state file is read by exactly one thing
|
|
215
|
+
scheduler, and its state file is read by exactly one thing, the next cycle.
|
|
216
216
|
So nothing could tell you the watcher had stopped, because the thing that
|
|
217
217
|
would tell you was the thing that stopped.
|
|
218
218
|
|
|
@@ -220,8 +220,8 @@ ${bold('OPTIONS FOR pulse')}
|
|
|
220
220
|
is your CI: a step running this on the schedule you already have turns a dead
|
|
221
221
|
cron into a red build, without Trazum holding anybody's metrics.
|
|
222
222
|
|
|
223
|
-
Never gates on a first run that never happened
|
|
224
|
-
late against
|
|
223
|
+
Never gates on a first run that never happened: that has no cadence to be
|
|
224
|
+
late against, and never on how far the measurements reach, which is a
|
|
225
225
|
provider reporting on its own schedule rather than a job that failed.
|
|
226
226
|
|
|
227
227
|
${bold('OPTIONS FOR bench')}
|
|
@@ -236,7 +236,7 @@ ${bold('OPTIONS FOR bench')}
|
|
|
236
236
|
--against: how much regression is too much is a
|
|
237
237
|
policy, and this tool does not write yours.
|
|
238
238
|
--json The measurements as data. The shape never changes
|
|
239
|
-
with the gate flags
|
|
239
|
+
with the gate flags: a verdict is the exit code
|
|
240
240
|
and sentences on stderr, the way check gates.
|
|
241
241
|
|
|
242
242
|
Measures this machine: the standard workloads, one shot each, wall time and
|
|
@@ -270,7 +270,7 @@ ${bold('OPTIONS FOR feedback')}
|
|
|
270
270
|
(none)
|
|
271
271
|
|
|
272
272
|
Prints where to report a wrong optimisation, a bug, a question or a security
|
|
273
|
-
problem
|
|
273
|
+
problem, and a blank issue with the version, runtime and platform already
|
|
274
274
|
filled in, printed in full first so nothing travels you have not read.
|
|
275
275
|
|
|
276
276
|
It sends nothing. Trazum has no telemetry: no ping, no install hook, no
|
|
@@ -279,17 +279,17 @@ ${bold('OPTIONS FOR feedback')}
|
|
|
279
279
|
|
|
280
280
|
${bold('OPTIONS FOR gateway')}
|
|
281
281
|
--on-cannot-tell <policy> Required, no default: fail-open or fail-closed.
|
|
282
|
-
What happens when the gateway cannot judge a call
|
|
283
|
-
|
|
282
|
+
What happens when the gateway cannot judge a call:
|
|
283
|
+
no budget, nothing measured, an unpriced model.
|
|
284
284
|
--port <n> | --socket <p> Where to listen. Loopback only, always.
|
|
285
285
|
--log <usage.jsonl> The usage log the limits policy is measured
|
|
286
|
-
against, read once at start
|
|
286
|
+
against, read once at start, same flag as
|
|
287
287
|
serve's, same reason: label and session spend
|
|
288
288
|
live in a usage log, not in the store.
|
|
289
289
|
|
|
290
290
|
Stands between your SDK and the provider, speaking their wire format, so no
|
|
291
291
|
code changes. Usage is measured from the provider's own response as it comes
|
|
292
|
-
back
|
|
292
|
+
back, no export, no connector lag, no missing day.
|
|
293
293
|
|
|
294
294
|
It refuses and never substitutes: a call over budget gets HTTP 402 with the
|
|
295
295
|
cheaper alternatives named. Substitution happens only where you wrote it down
|
|
@@ -306,7 +306,7 @@ ${bold('OPTIONS FOR experiment')}
|
|
|
306
306
|
honoured.
|
|
307
307
|
|
|
308
308
|
Judges recorded outcomes and cost together. Three-valued: A wins, B wins, or
|
|
309
|
-
not separable
|
|
309
|
+
not separable, with the number of outcomes per arm that would settle it, so
|
|
310
310
|
"run it longer" is an instruction rather than a shrug. Nothing is promoted.
|
|
311
311
|
|
|
312
312
|
${bold('OPTIONS FOR quality')}
|
|
@@ -321,7 +321,7 @@ ${bold('OPTIONS FOR quality')}
|
|
|
321
321
|
|
|
322
322
|
A before-and-after rather than an experiment, so it reports "cannot tell"
|
|
323
323
|
whenever the model mix, the call volume or the outcome coverage moved across
|
|
324
|
-
the boundary
|
|
324
|
+
the boundary: the prompt is not the only variable and it says so.
|
|
325
325
|
|
|
326
326
|
${bold('OPTIONS FOR semantic')}
|
|
327
327
|
--yes Required to send anything. Without it the command
|
|
@@ -342,14 +342,14 @@ ${bold('OPTIONS FOR commitment')}
|
|
|
342
342
|
|
|
343
343
|
Replays measured whole months against the deal's terms. Both directions are
|
|
344
344
|
priced: the months that cleared the floor, and what the floor cost in the
|
|
345
|
-
months that fell short
|
|
345
|
+
months that fell short, kept as its own figure, because netted into the
|
|
346
346
|
saving it disappears. An as-if calculation about the past, never a forecast.
|
|
347
347
|
|
|
348
348
|
${bold('OPTIONS FOR report')}
|
|
349
349
|
--year <yyyy> Required. Which year to assemble.
|
|
350
350
|
--json The record as data.
|
|
351
351
|
|
|
352
|
-
Everything comes from the store and the plans you already keep
|
|
352
|
+
Everything comes from the store and the plans you already keep: nothing is
|
|
353
353
|
computed that cannot be checked against a document that already exists.
|
|
354
354
|
Months with no record are named rather than filled, promises are kept in
|
|
355
355
|
three outcomes rather than two, and the report lists what it cannot say.
|
|
@@ -374,14 +374,14 @@ ${bold('OPTIONS FOR eval')}
|
|
|
374
374
|
--concurrency <n> Calls in flight at once. Default: 3.
|
|
375
375
|
--export promptfoo Write a promptfoo suite instead of running anything:
|
|
376
376
|
both prompts, every case, no API key needed and no
|
|
377
|
-
call made. The assertions are yours
|
|
377
|
+
call made. The assertions are yours: this exists
|
|
378
378
|
for the question agreement cannot answer.
|
|
379
379
|
-o, --out <file> Where to write it. Defaults to stdout.
|
|
380
380
|
|
|
381
381
|
Runs both prompt versions over your cases and reports whether the
|
|
382
382
|
optimisation changed the answers. Costs THREE provider calls per case: the
|
|
383
383
|
original twice, to measure the model's own run-to-run variance, and the
|
|
384
|
-
optimised once. That baseline is the yardstick
|
|
384
|
+
optimised once. That baseline is the yardstick, without it, a divergence
|
|
385
385
|
figure means nothing. Exits with code 1 when the answers genuinely diverge.
|
|
386
386
|
|
|
387
387
|
${bold('OPTIONS FOR rank')}
|
|
@@ -443,7 +443,7 @@ ${bold('OPTIONS FOR optimize')}
|
|
|
443
443
|
--all-labels With --from-log: every prompt in the config's
|
|
444
444
|
"labels" map, optimised and priced against its
|
|
445
445
|
own measured traffic, ranked by what the change
|
|
446
|
-
is worth
|
|
446
|
+
is worth, plus the mismatches in both
|
|
447
447
|
directions (mapped prompts with no traffic,
|
|
448
448
|
traffic with no mapped prompt).
|
|
449
449
|
--from-log <usage.jsonl> Measure the three figures above from a usage log
|
|
@@ -457,7 +457,7 @@ ${bold('OPTIONS FOR optimize')}
|
|
|
457
457
|
--batch The work tolerates latency (Batch API, 50% off).
|
|
458
458
|
--disable <id,id> Turn off specific rules (see "trazum rules").
|
|
459
459
|
--suggest Ask the LLM for phrase-level rewrites and list them
|
|
460
|
-
with what each saves. Changes nothing on its own
|
|
460
|
+
with what each saves. Changes nothing on its own,
|
|
461
461
|
every proposal is checked against your prompt first
|
|
462
462
|
and dropped if it does not survive.
|
|
463
463
|
--apply-suggestions Take them. Only with --suggest; alone it is an error
|
|
@@ -478,7 +478,7 @@ ${bold('OPTIONS FOR optimize')}
|
|
|
478
478
|
--tokens-only Report the token saving and no money at all. The
|
|
479
479
|
default inside Claude Code, Codex or Cursor, where
|
|
480
480
|
a subscription means there is no bill to reduce.
|
|
481
|
-
--cost Show the money even there
|
|
481
|
+
--cost Show the money even there: the host says where
|
|
482
482
|
Trazum runs, not where your prompt goes.
|
|
483
483
|
--prompt <name> Which marked prompt to optimise, when a source file
|
|
484
484
|
holds more than one. See "trazum where".
|
|
@@ -505,7 +505,7 @@ ${bold('OPTIONS FOR check')}
|
|
|
505
505
|
--baseline Gate on the recorded cost baseline. On by default whenever
|
|
506
506
|
the config declares one, so CI needs no argument; the useful
|
|
507
507
|
spelling is --no-baseline, which skips it for one run.
|
|
508
|
-
--files-from <file|-> Check only the files listed, one path per line
|
|
508
|
+
--files-from <file|-> Check only the files listed, one path per line, the shape
|
|
509
509
|
git diff --name-only produces, so a pre-commit hook is one
|
|
510
510
|
pipe: git diff --cached --name-only | trazum check --files-from -
|
|
511
511
|
Non-prompt paths and deletions are dropped and counted out
|
|
@@ -518,7 +518,7 @@ ${bold('OPTIONS FOR check')}
|
|
|
518
518
|
|
|
519
519
|
${bold('OPTIONS FOR baseline')}
|
|
520
520
|
Records what the prompts in a directory cost right now, to a file you commit.
|
|
521
|
-
Then "check" fails the build when the repository drifts past it
|
|
521
|
+
Then "check" fails the build when the repository drifts past it: the question
|
|
522
522
|
budgets cannot answer, because a repository at 95% of every budget passes
|
|
523
523
|
forever while a pull request adds four hundred tokens across a dozen files.
|
|
524
524
|
|
|
@@ -527,7 +527,7 @@ ${bold('OPTIONS FOR baseline')}
|
|
|
527
527
|
--model, --calls, --output-tokens, --cache-hit-rate, --batch
|
|
528
528
|
The scenario the monthly figure is recorded under. It is
|
|
529
529
|
recorded so a later comparison can say whether the money is
|
|
530
|
-
comparable
|
|
530
|
+
comparable: the gate itself is in tokens, so a repriced
|
|
531
531
|
model never fails a build on its own.
|
|
532
532
|
--exact-tokens Exact counts (needs ANTHROPIC_API_KEY).
|
|
533
533
|
--json Result as JSON.
|
|
@@ -536,7 +536,7 @@ ${bold('OPTIONS FOR baseline')}
|
|
|
536
536
|
writing the thing you would fix the failure with is a loop.
|
|
537
537
|
|
|
538
538
|
Given a directory it checks every prompt inside it against the "budgets"
|
|
539
|
-
patterns in ${bold('trazum.config.json')}
|
|
539
|
+
patterns in ${bold('trazum.config.json')}: one CI step for a whole repository of
|
|
540
540
|
prompts. A file no pattern covers is listed as unbudgeted rather than
|
|
541
541
|
skipped quietly, and a run where nothing at all was budgeted is an error:
|
|
542
542
|
"0 failures" from a check that measured nothing is the most misleading
|
|
@@ -546,7 +546,7 @@ ${bold('OPTIONS FOR ladder')}
|
|
|
546
546
|
--since <when> Read only calls in this window. A UTC day
|
|
547
547
|
--until <when> (2026-08-14), a full ISO 8601 timestamp, a
|
|
548
548
|
relative window (7d, 24h) or "now". Calls with no
|
|
549
|
-
"ts" cannot be placed and are left out
|
|
549
|
+
"ts" cannot be placed and are left out, counted
|
|
550
550
|
out loud, never dropped silently.
|
|
551
551
|
--label <name> Judge one workload's ladder rather than the whole
|
|
552
552
|
log. A label that matches nothing is an error
|
|
@@ -561,9 +561,9 @@ ${bold('OPTIONS FOR owners')}
|
|
|
561
561
|
${bold('OPTIONS FOR profile')}
|
|
562
562
|
--against <log.jsonl> Compare this bill to a previous log. Positive
|
|
563
563
|
means the bill grew; drivers are ranked by their
|
|
564
|
-
contribution to the change. No period is assumed
|
|
564
|
+
contribution to the change. No period is assumed,
|
|
565
565
|
judge the call counts before judging the money.
|
|
566
|
-
--label <name> Profile only the calls carrying this label
|
|
566
|
+
--label <name> Profile only the calls carrying this label, the
|
|
567
567
|
drill-down once the full report named a suspect.
|
|
568
568
|
A label that matches nothing is an error naming
|
|
569
569
|
the labels that exist. With --against, both logs
|
|
@@ -577,16 +577,16 @@ ${bold('OPTIONS FOR profile')}
|
|
|
577
577
|
an error, not a flag that silently gates nothing.
|
|
578
578
|
--max-cache-loss-usd <n> Exit 1 when caching added more than n dollars to
|
|
579
579
|
this bill. Reads the worst case when the log did
|
|
580
|
-
not record the write TTL
|
|
580
|
+
not record the write TTL: a gate reading the
|
|
581
581
|
flattering half would pass the bills it exists
|
|
582
|
-
to catch
|
|
582
|
+
to catch, and says which claim fired.
|
|
583
583
|
--max-day-usd <n> Fail when any single UTC day inside the log spent
|
|
584
584
|
more than this. A month under budget can hide the
|
|
585
585
|
afternoon a loop burned a quarter of it. A log
|
|
586
586
|
with no timestamps fails: not measured is not
|
|
587
587
|
under budget.
|
|
588
588
|
--max-session-usd <n> Fail when any single conversation in the log
|
|
589
|
-
cost more than this
|
|
589
|
+
cost more than this, the unit an agent product
|
|
590
590
|
blows up in. A log with no sessions fails: not
|
|
591
591
|
measured is not under budget.
|
|
592
592
|
--since <when> Profile only calls at or after this moment. A UTC
|
|
@@ -594,22 +594,22 @@ ${bold('OPTIONS FOR profile')}
|
|
|
594
594
|
relative window (7d, 24h) or "now";
|
|
595
595
|
--until with a bare date includes that whole day.
|
|
596
596
|
Calls with no "ts" cannot be placed and are left
|
|
597
|
-
out
|
|
597
|
+
out, counted out loud, never dropped silently.
|
|
598
598
|
With --against, both logs get the same window.
|
|
599
599
|
--what-if <model> Price these exact calls on another model. The
|
|
600
|
-
same token counts at a different rate card
|
|
600
|
+
same token counts at a different rate card,
|
|
601
601
|
multiplication, not advice, and it says so.
|
|
602
602
|
Calls larger than that model's context window
|
|
603
603
|
are named as impossible, not priced as cheap.
|
|
604
604
|
--markdown-out <file> Also write the report as Markdown, for a CI job
|
|
605
605
|
summary or a pull request comment.
|
|
606
606
|
--html-out <file> Also write the report as one self-contained HTML
|
|
607
|
-
file
|
|
607
|
+
file, the one to email to whoever pays the bill.
|
|
608
608
|
Same figures as --json, caveats rendered as
|
|
609
609
|
prominently as the totals they qualify.
|
|
610
610
|
--csv-shape <shape> Which table --csv-out writes: slice (default),
|
|
611
611
|
day, hour, or model-day (one row per day and
|
|
612
|
-
model
|
|
612
|
+
model, charts the mix moving). One row shape
|
|
613
613
|
per file, so nothing has to be filtered before
|
|
614
614
|
it can be summed.
|
|
615
615
|
--csv-out <file> Also write the report as CSV, one row per label
|
|
@@ -620,20 +620,20 @@ ${bold('OPTIONS FOR profile')}
|
|
|
620
620
|
--pricing <file> Local price overlay, as everywhere else.
|
|
621
621
|
--json The full report as data, including the levers.
|
|
622
622
|
|
|
623
|
-
Takes a log file or a directory of them
|
|
623
|
+
Takes a log file or a directory of them, rotated daily logs are read in
|
|
624
624
|
name order as one bill, and how many were read is stated. Gzipped files
|
|
625
625
|
(.jsonl.gz and the rest) are read too, because that is what a rotated log
|
|
626
626
|
looks like a day later; one that will not decompress is an error naming the
|
|
627
627
|
file, never a bill quietly missing a day.
|
|
628
628
|
|
|
629
629
|
Reads what the provider actually charged. Optional fields unlock findings:
|
|
630
|
-
"label" (which workload), "session" (which conversation
|
|
630
|
+
"label" (which workload), "session" (which conversation, grouped by, never
|
|
631
631
|
printed), "stop_reason"/"finish_reason" (answers cut off at max_tokens).
|
|
632
632
|
|
|
633
633
|
${bold('OPTIONS FOR plan')}
|
|
634
634
|
--min-usd <n> Leave out actions worth less than n dollars. How
|
|
635
635
|
many were left out is stated, never silent.
|
|
636
|
-
-o, --out <file> Save the plan as dated JSON
|
|
636
|
+
-o, --out <file> Save the plan as dated JSON: the file
|
|
637
637
|
"trazum verify" will later hold it to.
|
|
638
638
|
--markdown-out <file> Also write the plan as Markdown, for a CI job
|
|
639
639
|
summary or a pull request comment.
|
|
@@ -642,8 +642,8 @@ ${bold('OPTIONS FOR plan')}
|
|
|
642
642
|
|
|
643
643
|
Reads a usage log and turns the report's findings into a ranked plan: route
|
|
644
644
|
this, batch that, fix the truncation pair, look at the cache. The money is
|
|
645
|
-
composed correctly
|
|
646
|
-
never summed
|
|
645
|
+
composed correctly, route and batch on the same slice arrive combined,
|
|
646
|
+
never summed, and every action names what the log cannot confirm, because
|
|
647
647
|
a plan that hides its assumptions is advice pretending to be arithmetic.
|
|
648
648
|
Projected savings and money already spent are separate totals throughout.
|
|
649
649
|
|
|
@@ -656,8 +656,8 @@ ${bold('OPTIONS FOR serve')}
|
|
|
656
656
|
the store; without this, every ceiling in
|
|
657
657
|
"limits" answers cannot-tell.
|
|
658
658
|
|
|
659
|
-
Answers the two questions that matter at call time
|
|
660
|
-
and is there budget left
|
|
659
|
+
Answers the two questions that matter at call time, what will this cost,
|
|
660
|
+
and is there budget left, in single-digit milliseconds, so an agent or a
|
|
661
661
|
wrapper can ask before it spends rather than reading a report afterwards.
|
|
662
662
|
|
|
663
663
|
POST /cost with {"model": "...", "inputTokens": n, "outputTokens": n};
|
|
@@ -669,7 +669,7 @@ ${bold('OPTIONS FOR serve')}
|
|
|
669
669
|
It listens on 127.0.0.1 and nowhere else, and there is no flag to change
|
|
670
670
|
that: a cost oracle on a network interface holds a company's spend, its
|
|
671
671
|
model mix and its budgets, and answers whoever asks. There is no auth for
|
|
672
|
-
the same reason there is no --host
|
|
672
|
+
the same reason there is no --host: a token checked over loopback is
|
|
673
673
|
theatre, and the honest posture is a surface small enough not to need one.
|
|
674
674
|
|
|
675
675
|
With no store and no budget it still prices the call and says the budget
|
|
@@ -688,15 +688,15 @@ ${bold('OPTIONS FOR watch')}
|
|
|
688
688
|
instead of the store.
|
|
689
689
|
--json The cycle as data: crossings, abstentions, gap.
|
|
690
690
|
|
|
691
|
-
Evaluates the spend gates from your config
|
|
692
|
-
maxCacheLossUsd
|
|
691
|
+
Evaluates the spend gates from your config, maxUsd, maxDayUsd,
|
|
692
|
+
maxCacheLossUsd, against what has been measured, and tells you the
|
|
693
693
|
afternoon it happens rather than three weeks later. Exits 1 when something
|
|
694
694
|
crossed, so cron mails it and CI fails.
|
|
695
695
|
|
|
696
696
|
An alert fires on a measured crossing and never on a projection: "you have
|
|
697
697
|
spent $412 of a $400 budget" is a fact and "you will exceed" is a forecast,
|
|
698
698
|
which this tool does not make at any window length. A day still being
|
|
699
|
-
measured is reported as not yet judgeable rather than passed
|
|
699
|
+
measured is reported as not yet judgeable rather than passed, but a day
|
|
700
700
|
already over budget fires whatever the hour, because it does not become less
|
|
701
701
|
over budget at midnight.
|
|
702
702
|
|
|
@@ -715,8 +715,8 @@ ${bold('OPTIONS FOR store')}
|
|
|
715
715
|
|
|
716
716
|
Says what the local store holds: how many measurements, over what span, per
|
|
717
717
|
provider, and what a prune would take. The store keeps aggregates and
|
|
718
|
-
billing fields
|
|
719
|
-
credential
|
|
718
|
+
billing fields, never prompt text, never completion text, never a
|
|
719
|
+
credential, so it is a file a team can back up without a privacy review.
|
|
720
720
|
|
|
721
721
|
Pruning is the one operation here that destroys something, so it refuses to
|
|
722
722
|
run without a retention policy: set "store": {"keepDays": 90} in the config
|
|
@@ -731,7 +731,7 @@ ${bold('OPTIONS FOR connect')}
|
|
|
731
731
|
variable the key would come from. Sends nothing
|
|
732
732
|
and needs no credential.
|
|
733
733
|
--payload <file> Price a usage payload you already have, instead of
|
|
734
|
-
pulling one. No credential, no network
|
|
734
|
+
pulling one. No credential, no network, the same
|
|
735
735
|
arithmetic on the same shape.
|
|
736
736
|
--store Keep what was pulled in the local store, so the
|
|
737
737
|
next run does not download it again and "trazum
|
|
@@ -749,8 +749,8 @@ ${bold('OPTIONS FOR connect')}
|
|
|
749
749
|
|
|
750
750
|
These APIs serve sums over a window, not one row per call, so a connected
|
|
751
751
|
report is a restricted one and says so: the totals, the model split, the day
|
|
752
|
-
series and the cache verdict are all available, and the per-call findings
|
|
753
|
-
input shapes, truncation retries, conversations, context pressure
|
|
752
|
+
series and the cache verdict are all available, and the per-call findings,
|
|
753
|
+
input shapes, truncation retries, conversations, context pressure, are
|
|
754
754
|
listed as unavailable with what would unlock them. A rate limit, a page cap
|
|
755
755
|
or an expired cursor returns what arrived with the gap named, never a total
|
|
756
756
|
that quietly describes less traffic than you asked about.
|
|
@@ -759,15 +759,15 @@ ${bold('OPTIONS FOR history')}
|
|
|
759
759
|
--store Build the series from the local store instead of
|
|
760
760
|
a directory of stored reports. Bucketed sources
|
|
761
761
|
carry no label, so the label series is absent and
|
|
762
|
-
said to be
|
|
762
|
+
said to be: the model-share and cache-share
|
|
763
763
|
series are what a series exists for, and both
|
|
764
764
|
work.
|
|
765
765
|
--markdown-out <file> Also write the series as Markdown, for a CI job
|
|
766
766
|
summary or a pull request comment.
|
|
767
767
|
--json The history as data.
|
|
768
768
|
|
|
769
|
-
Takes a directory of stored reports
|
|
770
|
-
writes
|
|
769
|
+
Takes a directory of stored reports, the --json documents profile already
|
|
770
|
+
writes, plus any saved plans beside them, and builds the series no
|
|
771
771
|
pairwise comparison can see: a workload climbing a little every period, a
|
|
772
772
|
model share rising since a date, a cache share decaying slowly enough that
|
|
773
773
|
no single week's report called it a finding, and the same action planned
|
|
@@ -779,7 +779,7 @@ ${bold('OPTIONS FOR verify')}
|
|
|
779
779
|
--against <log|dir> The newer usage log the plan is held to. Required.
|
|
780
780
|
--gate Exit 1 when an action did not produce what the
|
|
781
781
|
plan promised, or its fields stopped being
|
|
782
|
-
recorded
|
|
782
|
+
recorded, "not recorded" must not read as
|
|
783
783
|
"fixed". A vanished workload fails nothing.
|
|
784
784
|
--markdown-out <file> Also write the verdicts as Markdown, for a CI
|
|
785
785
|
job summary or a pull request comment.
|
|
@@ -787,7 +787,7 @@ ${bold('OPTIONS FOR verify')}
|
|
|
787
787
|
--json The verification as data.
|
|
788
788
|
|
|
789
789
|
Holds a saved plan to the log that came after it, with three outcomes and
|
|
790
|
-
never two: the change arrived, it did not, or it cannot be told
|
|
790
|
+
never two: the change arrived, it did not, or it cannot be told, because
|
|
791
791
|
the workload vanished, the fields the detection needs stopped being
|
|
792
792
|
recorded, or tokens cannot say which tier billed them. Differences carry
|
|
793
793
|
the world's measured movement (calls, output per call) from the plan's own
|
|
@@ -806,7 +806,7 @@ ${bold('OPTIONS FOR route')}
|
|
|
806
806
|
|
|
807
807
|
Reads a usage log, finds the slice where routing to a cheaper model is worth
|
|
808
808
|
the most, and measures whether that model still does the job. The same prompt
|
|
809
|
-
goes to both, and the original runs twice per case
|
|
809
|
+
goes to both, and the original runs twice per case, so the verdict is judged
|
|
810
810
|
against that model's own variance rather than a threshold somebody picked.
|
|
811
811
|
|
|
812
812
|
Costs three provider calls per case and needs TRAZUM_LLM_* configured.
|
|
@@ -833,7 +833,7 @@ ${bold('OPTIONS FOR diff')}
|
|
|
833
833
|
is given: deciding that growth is unacceptable is your call, not ours.
|
|
834
834
|
|
|
835
835
|
${bold('trazum where')}
|
|
836
|
-
Says which provider a file's prompts are actually sent to, and how it knows
|
|
836
|
+
Says which provider a file's prompts are actually sent to, and how it knows,
|
|
837
837
|
an SDK import, a base URL, a quoted model id, or "model=" on a trazum:prompt
|
|
838
838
|
marker. Every answer names the line it came from.
|
|
839
839
|
|
|
@@ -845,7 +845,7 @@ ${bold('trazum where')}
|
|
|
845
845
|
are all called through the OpenAI SDK with a different base_url, so treating
|
|
846
846
|
that as a contradiction would refuse to price an ordinary client.
|
|
847
847
|
|
|
848
|
-
With no file, it reports only which tool Trazum is running inside
|
|
848
|
+
With no file, it reports only which tool Trazum is running inside, and warns
|
|
849
849
|
when that tool bills by subscription, because a monthly saving is arithmetic
|
|
850
850
|
about tokens there, not money you get back.
|
|
851
851
|
|
|
@@ -857,14 +857,14 @@ ${bold('CONFIG FILE')}
|
|
|
857
857
|
usage { model, callsPerMonth, avgOutputTokens, cacheHitRate, batchEligible }
|
|
858
858
|
budgets { "prompts/**": 2000, "prompts/system.txt": 4000 }
|
|
859
859
|
baseline { "path": "trazum.baseline.json", "maxGrowthTokens": 0, "maxGrowthPct": 5 }
|
|
860
|
-
pricing "./prices.json"
|
|
861
|
-
labels { "support-rag": "prompts/support.txt" }
|
|
860
|
+
pricing "./prices.json": local price corrections, see below
|
|
861
|
+
labels { "support-rag": "prompts/support.txt" }: which prompt file each
|
|
862
862
|
usage-log label sends, so "trazum profile" can read the file and
|
|
863
863
|
say why a failing cache fails
|
|
864
|
-
spend { "maxUsd": 200, "byLabel": { "chat": 40 } }
|
|
864
|
+
spend { "maxUsd": 200, "byLabel": { "chat": 40 } }: money budgets for
|
|
865
865
|
"trazum profile", in dollars. A budgeted label with no calls in
|
|
866
866
|
the log is reported as not measured, never as a pass
|
|
867
|
-
limits { "dayUsd": 25, "sessionUsd": 1.5, "byLabel": { "chat": 5 } }
|
|
867
|
+
limits { "dayUsd": 25, "sessionUsd": 1.5, "byLabel": { "chat": 5 } }:
|
|
868
868
|
the enforcement policy: USD ceilings judged before a call is
|
|
869
869
|
made, at whichever door it arrives (the gateway's 402, "serve",
|
|
870
870
|
spend_guard over MCP). Separate from "spend" because a report
|
|
@@ -873,17 +873,17 @@ ${bold('CONFIG FILE')}
|
|
|
873
873
|
outage written as a policy
|
|
874
874
|
owners { "patterns": { "payments": ["billing-*"] },
|
|
875
875
|
"shared": { "search": { "payments": 0.6, "support": 0.4 } },
|
|
876
|
-
"budgets": { "payments": 400 } }
|
|
876
|
+
"budgets": { "payments": 400 } }: whose budget each workload
|
|
877
877
|
lands on. Spend matching no owner is its own line and is NEVER
|
|
878
878
|
divided between the others; a shared split must sum to 1, and the
|
|
879
879
|
rule travels with the report so the argument is about the rule
|
|
880
880
|
ladders { "support": { "tiers": ["claude-haiku-4-5", "claude-opus-5"],
|
|
881
|
-
"escalateOn": ["escalated"] } }
|
|
881
|
+
"escalateOn": ["escalated"] } }: cheap model first, escalate a
|
|
882
882
|
recorded failure to a dearer one. Both fields required.
|
|
883
883
|
"trazum ladder <log>" prints the break-even escalation rate
|
|
884
884
|
beside the measured one: an escalation pays twice, so above that
|
|
885
885
|
rate the ladder costs more than never having built it
|
|
886
|
-
outcomes { "values": ["resolved", "escalated"], "success": ["resolved"] }
|
|
886
|
+
outcomes { "values": ["resolved", "escalated"], "success": ["resolved"] }:
|
|
887
887
|
your own vocabulary for what happened, and which of it counts as
|
|
888
888
|
a win. Both required: which words mean success is a judgement
|
|
889
889
|
about your product rather than your bill, and this tool has no
|
|
@@ -891,27 +891,27 @@ ${bold('CONFIG FILE')}
|
|
|
891
891
|
value in a log that "values" never declares is named as
|
|
892
892
|
undeclared, never counted as a failure
|
|
893
893
|
waive [{ "gate": "maxUsd", "reason": "August migration", "until":
|
|
894
|
-
"2026-09-15" }]
|
|
894
|
+
"2026-09-15" }]: a gate failure decided about, on the record.
|
|
895
895
|
All three fields required: a waiver with no end date is a
|
|
896
896
|
finding deleted with extra steps. Waived failures still print,
|
|
897
897
|
and the day the waiver expires the gate fails again
|
|
898
898
|
|
|
899
899
|
Flags beat the config; the config beats the defaults. Budgets resolve to the
|
|
900
|
-
most specific matching pattern
|
|
900
|
+
most specific matching pattern: most literal characters wins. A boolean the
|
|
901
901
|
config switched on comes back off with --no-<flag>, e.g. --no-batch.
|
|
902
902
|
|
|
903
903
|
${bold('budgets')} is a ceiling; ${bold('baseline')} is a gate. One asks whether a file fits,
|
|
904
904
|
the other whether the repository got worse than the commit somebody recorded
|
|
905
905
|
with "trazum baseline". A repository at 95% of every budget passes forever
|
|
906
906
|
while a pull request adds four hundred tokens across a dozen files. With
|
|
907
|
-
baseline in the config, "check" on a directory reads it and gates on it
|
|
907
|
+
baseline in the config, "check" on a directory reads it and gates on it, no
|
|
908
908
|
flag, because a gate you have to remember to pass an argument to runs in the
|
|
909
909
|
author's terminal and not in CI. Thresholds are in tokens, never dollars: a
|
|
910
910
|
repriced model would otherwise fail a build for a change nobody made.
|
|
911
911
|
|
|
912
912
|
A config that will not validate is an error, including an unknown key. A
|
|
913
913
|
lenient parser would silently restore defaults, and for a budget the default
|
|
914
|
-
is "no budget"
|
|
914
|
+
is "no budget", a green build for a prompt nobody measured.
|
|
915
915
|
|
|
916
916
|
--config <file> Use this config instead of searching for one.
|
|
917
917
|
|
|
@@ -935,7 +935,7 @@ ${bold('PRICES')}
|
|
|
935
935
|
--pricing-live Take prices from OpenRouter instead of the bundled
|
|
936
936
|
table: today's figures for hundreds of models across
|
|
937
937
|
dozens of providers. Opt-in, because it is a network
|
|
938
|
-
call
|
|
938
|
+
call: the deterministic core never makes one.
|
|
939
939
|
A --pricing file wins over this.
|
|
940
940
|
|
|
941
941
|
What that source does not publish is whether a model
|
|
@@ -959,7 +959,7 @@ ${bold('OPTIONAL LLM')}
|
|
|
959
959
|
|
|
960
960
|
${bold('LANGUAGE')}
|
|
961
961
|
The report language follows --locale, then TRAZUM_LOCALE, then LANG, and last
|
|
962
|
-
the config file
|
|
962
|
+
the config file, so a project can set the language its CI logs read in
|
|
963
963
|
without overriding the language of whoever is at the keyboard. It changes the
|
|
964
964
|
report only: the same prompt always optimises the same way.
|
|
965
965
|
|
|
@@ -984,7 +984,7 @@ ${bold('EXAMPLES')}
|
|
|
984
984
|
|
|
985
985
|
errors: {
|
|
986
986
|
livePricingFailed: (url: string, detail: string) =>
|
|
987
|
-
`Could not load live prices from ${url}: ${detail}. The bundled prices are still there
|
|
987
|
+
`Could not load live prices from ${url}: ${detail}. The bundled prices are still there. Drop --pricing-live to use them.`,
|
|
988
988
|
optionNeedsValue: (name) => `Option --${name} needs a value.`,
|
|
989
989
|
mustBeNonNegative: (name, raw) =>
|
|
990
990
|
`--${name} must be a non-negative number (received: "${raw}").`,
|
|
@@ -997,7 +997,7 @@ ${bold('EXAMPLES')}
|
|
|
997
997
|
allLabelsNeedsLog: () =>
|
|
998
998
|
'--all-labels ranks prompts by measured traffic, so it needs --from-log <usage.jsonl>. Without a log every saving would be multiplied by the same typed guess, which ranks prompts by length and calls it a priority.',
|
|
999
999
|
allLabelsNeedsMap: () =>
|
|
1000
|
-
'--all-labels reads the "labels" map in trazum.config.json
|
|
1000
|
+
'--all-labels reads the "labels" map in trazum.config.json, label to prompt file, and this config has none. Map at least one workload to its prompt.',
|
|
1001
1001
|
fromLogConflict: (flag) =>
|
|
1002
1002
|
`--from-log measures the figure --${flag} types, and merging a measurement with a guess produces a number that is neither. Pass one or the other.`,
|
|
1003
1003
|
fromLogNeedsLabel: (available) =>
|
|
@@ -1005,7 +1005,7 @@ ${bold('EXAMPLES')}
|
|
|
1005
1005
|
fromLogAmbiguousLabel: (target, labels) =>
|
|
1006
1006
|
`${target} is mapped to more than one label in trazum.config.json (${labels}), so --from-log cannot pick one silently. Pass --label.`,
|
|
1007
1007
|
fromLogLabelEmpty: (label, available) =>
|
|
1008
|
-
`No priced call in this log carries the label "${label}", so there is nothing to measure
|
|
1008
|
+
`No priced call in this log carries the label "${label}", so there is nothing to measure: a zero-call profile would price this change as worthless rather than as unmeasured. Labels with traffic: ${available}.`,
|
|
1009
1009
|
unknownRuleInDisable: (id) => `Unknown rule in --disable: "${id}". Full list: trazum rules`,
|
|
1010
1010
|
unknownCommand: (command) => `Unknown command: "${command}". Try "trazum --help".`,
|
|
1011
1011
|
missingInputFile: () => 'Missing input file. Use "-" to read from standard input.',
|
|
@@ -1021,15 +1021,15 @@ ${bold('EXAMPLES')}
|
|
|
1021
1021
|
* Named, and told what it cannot do rather than merely that it will not.
|
|
1022
1022
|
*
|
|
1023
1023
|
* The alternative on offer has to be somebody else's tooling, because
|
|
1024
|
-
* Trazum has none for this family
|
|
1024
|
+
* Trazum has none for this family, saying so is the whole refusal. A
|
|
1025
1025
|
* refusal never arrives bare.
|
|
1026
1026
|
*/
|
|
1027
1027
|
exactTokensWrongFamily: (model, provider) =>
|
|
1028
1028
|
`--exact-tokens counts with Anthropic's endpoint, which counts Claude's tokenizer${
|
|
1029
1029
|
provider === null
|
|
1030
|
-
? `, and the catalogue records no provider for "${model}"
|
|
1030
|
+
? `, and the catalogue records no provider for "${model}", so which tokenizer it uses is not something Trazum knows`
|
|
1031
1031
|
: `, and ${model} is a model from ${provider}`
|
|
1032
|
-
}. Counting it there would either be refused upstream or return a number for a different tokenizer, and this tool will not label that exact. Drop --exact-tokens for the estimate
|
|
1032
|
+
}. Counting it there would either be refused upstream or return a number for a different tokenizer, and this tool will not label that exact. Drop --exact-tokens for the estimate, which is honest about being one, or count with ${
|
|
1033
1033
|
provider === null ? "that model's" : provider + "'s"
|
|
1034
1034
|
} own tooling.`,
|
|
1035
1035
|
checkNeedsMaxTokens: () => 'trazum check needs --max-tokens <n>.',
|
|
@@ -1060,7 +1060,7 @@ ${bold('EXAMPLES')}
|
|
|
1060
1060
|
estimated: (offFamily) =>
|
|
1061
1061
|
offFamily === null
|
|
1062
1062
|
? ' (estimated, ±10%)'
|
|
1063
|
-
: ` (estimated
|
|
1063
|
+
: ` (estimated: the counter is calibrated on Claude, not ${offFamily})`,
|
|
1064
1064
|
exactCount: () => ' (exact count)',
|
|
1065
1065
|
rulesApplied: () => 'Rules applied',
|
|
1066
1066
|
nothingToTrim: () => ' No rule found anything to trim.',
|
|
@@ -1070,14 +1070,14 @@ ${bold('EXAMPLES')}
|
|
|
1070
1070
|
// and naming the coverage cannot be wrong.
|
|
1071
1071
|
dictionaryCoverage: (languages) =>
|
|
1072
1072
|
` The phrase dictionaries cover ${languages}. A prompt in another language `
|
|
1073
|
-
+ 'is not necessarily efficient
|
|
1073
|
+
+ 'is not necessarily efficient: it may just be one Trazum cannot read yet.',
|
|
1074
1074
|
dictionaryUnreviewed: (languages) =>
|
|
1075
1075
|
` Of those, ${languages} carry entries nobody here reads: written by the same `
|
|
1076
1076
|
+ 'process that wrote the rules, never agreed by a speaker of the language.',
|
|
1077
1077
|
dictionaryAppliedUnreviewed: (language) =>
|
|
1078
1078
|
` These changes came from the ${language} dictionary, which nobody here reads. `
|
|
1079
1079
|
+ 'Its entries were written by the same process that wrote the rules and never '
|
|
1080
|
-
+ 'agreed by a speaker
|
|
1080
|
+
+ 'agreed by a speaker. Read the diff before trusting it.',
|
|
1081
1081
|
levelAggressive: () => '[aggressive]',
|
|
1082
1082
|
levelSafe: () => '[safe]',
|
|
1083
1083
|
ruleHits: (hits, tokensSaved) => `(${hits}×, ~${tokensSaved} tokens)`,
|
|
@@ -1094,37 +1094,37 @@ ${bold('EXAMPLES')}
|
|
|
1094
1094
|
costWith: (modelName) => `Cost with ${modelName}`,
|
|
1095
1095
|
usageLine: (calls, outputTokens, batch) =>
|
|
1096
1096
|
`${calls} calls/month · ${outputTokens} output tokens per call${batch ? ' · Batch API' : ''}`,
|
|
1097
|
-
allLabelsHeading: (count) => `Every mapped prompt against its own measured traffic
|
|
1097
|
+
allLabelsHeading: (count) => `Every mapped prompt against its own measured traffic, ${count} ranked by what the change is worth`,
|
|
1098
1098
|
allLabelsRow: (saving) => `${saving}/month if optimised`,
|
|
1099
1099
|
allLabelsRowPeriod: (saving) => `${saving} over the measured period if optimised`,
|
|
1100
1100
|
allLabelsFooter: () =>
|
|
1101
1101
|
'Ranked by measured traffic, not by prompt length: a big prompt on a dead workload is worth less than a small one on a busy one. Each figure is this prompt\'s token delta at its own label\'s measured rate.',
|
|
1102
1102
|
allLabelsUnmapped: (label, usd) =>
|
|
1103
|
-
`${label} carries ${usd} of measured spend and no prompt file is mapped to it
|
|
1103
|
+
`${label} carries ${usd} of measured spend and no prompt file is mapped to it: the workload nobody can optimise because nobody said where it lives. Map it under "labels" in trazum.config.json.`,
|
|
1104
1104
|
allLabelsDead: (label, path) =>
|
|
1105
|
-
`${label} is mapped to ${path} and has no traffic in this log
|
|
1105
|
+
`${label} is mapped to ${path} and has no traffic in this log, a retired workload, a renamed label, or a typo that has been silently doing nothing.`,
|
|
1106
1106
|
allLabelsUnreadable: (label, path) =>
|
|
1107
1107
|
`${label} is mapped to ${path}, which could not be read. The mapping exists; the file does not.`,
|
|
1108
1108
|
usageLineMeasured: (calls, days, scaled, outputTokens, batch) =>
|
|
1109
|
-
`${calls} calls measured over ${days} days
|
|
1109
|
+
`${calls} calls measured over ${days} days, ${scaled}/month at that rate · ${outputTokens} output tokens per call, measured${batch ? ' · Batch API' : ''}`,
|
|
1110
1110
|
usageLineMeasuredPeriod: (calls, days, outputTokens, batch) =>
|
|
1111
1111
|
`${calls} calls measured${days === null ? ' (the log carries no clock)' : ` over ${days} days`} · ${outputTokens} output tokens per call, measured${batch ? ' · Batch API' : ''}`,
|
|
1112
1112
|
measuredModelShare: (model, share, count) =>
|
|
1113
1113
|
`This label ran on ${count} models; the figures use ${model}, which carried ${share} of its spend.`,
|
|
1114
1114
|
measuredNoOutput: () =>
|
|
1115
|
-
'No call in this slice recorded output tokens, so the output half of every figure below is $0 measured
|
|
1115
|
+
'No call in this slice recorded output tokens, so the output half of every figure below is $0 measured, not $0 assumed.',
|
|
1116
1116
|
perPeriodSaving: (saving, pct) => `saving ${saving} over the measured period (${pct}%)`,
|
|
1117
1117
|
periodNotScaled: (days) =>
|
|
1118
1118
|
days === null
|
|
1119
1119
|
? 'Not scaled to a month: the log carries no clock, so there is no rate to scale. These figures cover exactly the calls measured.'
|
|
1120
|
-
: `Not scaled to a month: ${days} days is under the week a scaling needs
|
|
1120
|
+
: `Not scaled to a month: ${days} days is under the week a scaling needs, and being shorter than one weekly cycle multiplies whichever part of the cycle it caught. These figures cover exactly the period measured.`,
|
|
1121
1121
|
perMonthSaving: (saving, pct) => `saving ${saving}/month (${pct}%)`,
|
|
1122
1122
|
beyondShortening: () => 'Beyond shortening the prompt',
|
|
1123
1123
|
biggestLever: () => 'Start here:',
|
|
1124
1124
|
biggestLeverDetail: (title, amount, times) =>
|
|
1125
1125
|
// The title keeps its case: lowercasing it turned "Claude Opus 5" into
|
|
1126
1126
|
// "claude opus 5", which is a product name mangled to fit a sentence.
|
|
1127
|
-
`"${title}"
|
|
1127
|
+
`"${title}", ${amount}/month` +
|
|
1128
1128
|
(times !== null && times >= 2 ? `, ${times}× what the rules saved.` : '.'),
|
|
1129
1129
|
perMonthSuffix: (amount) => ` ~${amount}/month`,
|
|
1130
1130
|
diff: () => 'Diff',
|
|
@@ -1136,17 +1136,17 @@ ${bold('EXAMPLES')}
|
|
|
1136
1136
|
// decided on the string rather than on a number that is no longer here.
|
|
1137
1137
|
tokensSaved: (tokens) => `${tokens} token${tokens === '1' ? '' : 's'} back, every call.`,
|
|
1138
1138
|
windowNegligible: (tokens, model, window) =>
|
|
1139
|
-
`${tokens} tokens of ${model}'s ${window}-token window
|
|
1139
|
+
`${tokens} tokens of ${model}'s ${window}-token window, under a tenth of a percent, so the window is not what constrains this prompt.`,
|
|
1140
1140
|
windowUnmoved: (share, model, window) =>
|
|
1141
1141
|
`${share} of ${model}'s ${window}-token window, before and after: this change is too small to move it.`,
|
|
1142
1142
|
beyondThisPromptTokensOnly: () =>
|
|
1143
1143
|
'Shortening a prompt is the smallest lever there is: measured on an ordinary support prompt, the rules recover about 1% of a monthly bill. If any of your prompts go to a metered API, "trazum profile <usage.jsonl>" reads what the provider actually charged and prices the levers that are not the prompt. Recording that log is a few lines and it never contains prompt text.',
|
|
1144
1144
|
beyondThisPrompt: () =>
|
|
1145
|
-
'Shortening a prompt is the smallest lever there is: measured on an ordinary support prompt, the rules recover about 1% of a monthly bill. On a metered API the things that move 40% to 80% are which model the call goes to, the Batch API, prompt caching, and what re-sending the conversation costs
|
|
1145
|
+
'Shortening a prompt is the smallest lever there is: measured on an ordinary support prompt, the rules recover about 1% of a monthly bill. On a metered API the things that move 40% to 80% are which model the call goes to, the Batch API, prompt caching, and what re-sending the conversation costs, and "trazum profile <usage.jsonl>" prices all four from what the provider actually charged. Recording that log is a few lines and it never contains prompt text.',
|
|
1146
1146
|
windowUse: (before, after, model, window) =>
|
|
1147
|
-
`Context window: ${before} → ${after} of ${model}'s ${window} tokens
|
|
1147
|
+
`Context window: ${before} → ${after} of ${model}'s ${window} tokens, room the conversation gets instead.`,
|
|
1148
1148
|
tokensOnlyAskedFor: () =>
|
|
1149
|
-
'You named a scenario, and it was not priced: Trazum is running somewhere that bills by subscription, so there is no bill here to reduce. Add --cost to price it anyway
|
|
1149
|
+
'You named a scenario, and it was not priced: Trazum is running somewhere that bills by subscription, so there is no bill here to reduce. Add --cost to price it anyway: the host says where Trazum runs, not where your prompt goes.',
|
|
1150
1150
|
tokensOnlyCost: () => 'Pass --cost if this prompt is bound for a metered API.',
|
|
1151
1151
|
pricingOverlaid: (models, lastReviewed) =>
|
|
1152
1152
|
`Prices for ${models} came from a local overlay reviewed ${lastReviewed}, not from the bundled catalogue.`,
|
|
@@ -1197,7 +1197,7 @@ ${bold('EXAMPLES')}
|
|
|
1197
1197
|
prompts: (count) =>
|
|
1198
1198
|
`${count} prompt ${count === 1 ? 'file' : 'files'} found.`,
|
|
1199
1199
|
noPrompts: () =>
|
|
1200
|
-
'No prompt files found. Directory mode reads .txt, .md, .prompt and .tmpl by default
|
|
1200
|
+
'No prompt files found. Directory mode reads .txt, .md, .prompt and .tmpl by default. Set "extensions" if yours are named otherwise.',
|
|
1201
1201
|
sourcesTruncated: (cap) =>
|
|
1202
1202
|
`Stopped after ${cap} source files, so a provider named further down was not seen.`,
|
|
1203
1203
|
usageFound: (kind, where) =>
|
|
@@ -1207,7 +1207,7 @@ ${bold('EXAMPLES')}
|
|
|
1207
1207
|
? `A local store of past measurements is in ${where}.`
|
|
1208
1208
|
: `Usage log found: ${where}.`,
|
|
1209
1209
|
noUsage: () =>
|
|
1210
|
-
'No usage found. Point trazum at a log (trazum profile <log.jsonl>) or pull one (trazum connect anthropic)
|
|
1210
|
+
'No usage found. Point trazum at a log (trazum profile <log.jsonl>) or pull one (trazum connect anthropic): every money figure in this tool comes from one.',
|
|
1211
1211
|
usageUnreadable: (where, because) =>
|
|
1212
1212
|
`${where} is there and could not be read: ${because}. That is a different problem from having no usage, and it is the one to fix first.`,
|
|
1213
1213
|
|
|
@@ -1228,7 +1228,7 @@ ${bold('EXAMPLES')}
|
|
|
1228
1228
|
noModelEvidence: () => 'nothing measured or written in the source names one model',
|
|
1229
1229
|
modelConflict: (files) => `these files name more than one provider: ${files}`,
|
|
1230
1230
|
modelProviderOnly: (provider, file) =>
|
|
1231
|
-
`${file} names ${provider} and no model
|
|
1231
|
+
`${file} names ${provider} and no model: a provider default would read as your decision six weeks from now`,
|
|
1232
1232
|
nothingMeasured: () => 'nothing has been measured yet',
|
|
1233
1233
|
windowTooShort: (days, minimum) =>
|
|
1234
1234
|
`${days} days measured; ${minimum} are needed before that is a monthly rate rather than a forecast`,
|
|
@@ -1242,7 +1242,7 @@ ${bold('EXAMPLES')}
|
|
|
1242
1242
|
`${labels} ${labels === 1 ? 'label' : 'labels'} in the log, and nothing here proves which prompt file sends which`,
|
|
1243
1243
|
budgetIsPolicy: () => 'a budget is a policy, and there is no measured figure to write one against yet',
|
|
1244
1244
|
budgetIsPolicyMeasured: (usd, days) =>
|
|
1245
|
-
`a budget is a policy, so it is yours to set
|
|
1245
|
+
`a budget is a policy, so it is yours to set: the measured figure is $${usd} over ${days} days`,
|
|
1246
1246
|
|
|
1247
1247
|
findingHeading: () => 'The most valuable thing found',
|
|
1248
1248
|
noFinding: (why) =>
|
|
@@ -1258,7 +1258,7 @@ ${bold('EXAMPLES')}
|
|
|
1258
1258
|
findingBatch: () => 'The Batch API halves both halves of the bill, for work that can wait.',
|
|
1259
1259
|
findingTotal: (usd, days) => `Together: $${usd} over the same ${days} days.`,
|
|
1260
1260
|
findingNext: () =>
|
|
1261
|
-
'trazum plan <your log> ranks every action, not just this one. If a figure here looks wrong, trazum feedback says where to tell us
|
|
1261
|
+
'trazum plan <your log> ranks every action, not just this one. If a figure here looks wrong, trazum feedback says where to tell us, it sends nothing on its own.',
|
|
1262
1262
|
|
|
1263
1263
|
wouldOverwrite: (keys) => `This replaces keys you already set: ${keys}. Pass --yes to write anyway.`,
|
|
1264
1264
|
nothingToWrite: () => 'No config written: nothing above could be justified from what is here.',
|
|
@@ -1277,14 +1277,14 @@ ${bold('EXAMPLES')}
|
|
|
1277
1277
|
missing: (months) =>
|
|
1278
1278
|
`No record at all for ${months}. Those months are named rather than filled: a year that quietly covers part of itself and prints an annual total is wrong by the rest and says nothing about it.`,
|
|
1279
1279
|
promises: (planned, arrived, notArrived, cannotTell) =>
|
|
1280
|
-
`${planned} actions planned. ${arrived} arrived, ${notArrived} did not, and ${cannotTell} could not be judged. Three outcomes, never two
|
|
1280
|
+
`${planned} actions planned. ${arrived} arrived, ${notArrived} did not, and ${cannotTell} could not be judged. Three outcomes, never two, the third is the one an ordinary annual report folds into the flattering one.`,
|
|
1281
1281
|
projected: (usd) => `The plans projected ${usd} of savings.`,
|
|
1282
1282
|
noArrivedFigure: () =>
|
|
1283
|
-
'There is deliberately no figure beside it for what arrived. A verification says WHETHER each action landed; it has never carried a per-action dollar figure for the saving. Assembling one here would mean deciding which of several observed numbers is "the saving"
|
|
1283
|
+
'There is deliberately no figure beside it for what arrived. A verification says WHETHER each action landed; it has never carried a per-action dollar figure for the saving. Assembling one here would mean deciding which of several observed numbers is "the saving", a judgement the verification refused to make, and exactly the annual-report arithmetic this document replaces.',
|
|
1284
1284
|
outcomes: (recorded, parsed, unrecordedUsd) =>
|
|
1285
1285
|
`${recorded} of ${parsed} calls recorded an outcome; ${unrecordedUsd} of the year\u2019s spend did not.`,
|
|
1286
1286
|
noOutcomes: () =>
|
|
1287
|
-
'No outcome was recorded this year, so nothing here says what the money bought. That is not a success rate of zero
|
|
1287
|
+
'No outcome was recorded this year, so nothing here says what the money bought. That is not a success rate of zero, an uninstrumented year and a failing year are different sentences.',
|
|
1288
1288
|
cannotSayHeading: () => 'What this record cannot say',
|
|
1289
1289
|
cannotSay: (kind) =>
|
|
1290
1290
|
kind === 'months-missing'
|
|
@@ -1294,14 +1294,14 @@ ${bold('EXAMPLES')}
|
|
|
1294
1294
|
: kind === 'nothing-was-verified'
|
|
1295
1295
|
? 'whether the promises were kept, because no plan was verified'
|
|
1296
1296
|
: kind === 'some-promises-unjudgeable'
|
|
1297
|
-
? 'whether some promises were kept
|
|
1297
|
+
? 'whether some promises were kept, they could not be judged from the logs that exist'
|
|
1298
1298
|
: kind === 'arrived-savings-not-quantified'
|
|
1299
1299
|
? 'how many dollars the kept promises were worth'
|
|
1300
1300
|
: kind === 'no-outcomes-recorded'
|
|
1301
1301
|
? 'what any of the money bought'
|
|
1302
1302
|
: 'what the uninstrumented share of the traffic did',
|
|
1303
1303
|
noNewData: () =>
|
|
1304
|
-
'Every figure above comes from the store and the plans you already keep. Nothing was computed here that cannot be checked against a document that already exists
|
|
1304
|
+
'Every figure above comes from the store and the plans you already keep. Nothing was computed here that cannot be checked against a document that already exists, which is the only reason an annual report is worth trusting, since it is the document most likely to be quoted out of the room it was written in.',
|
|
1305
1305
|
},
|
|
1306
1306
|
|
|
1307
1307
|
commitment: {
|
|
@@ -1310,7 +1310,7 @@ ${bold('EXAMPLES')}
|
|
|
1310
1310
|
needsTerms: () =>
|
|
1311
1311
|
'--floor and --discount are required, and --months defaults to 12. Take them from the contract the provider is offering you: the floor is what you commit to spending each month after the discount, and the discount is what you get off list.',
|
|
1312
1312
|
asIf: () =>
|
|
1313
|
-
'This is what the deal WOULD HAVE done on the traffic you actually had. It is a measurement of the past, not a prediction
|
|
1313
|
+
'This is what the deal WOULD HAVE done on the traffic you actually had. It is a measurement of the past, not a prediction, every month below happened.',
|
|
1314
1314
|
columns: { month: 'month', list: 'list', paid: 'would pay', saving: 'saving' },
|
|
1315
1315
|
net: (usd, months) => `Net over ${months} measured months: ${usd}.`,
|
|
1316
1316
|
good: (usd) => `The months that cleared the floor saved ${usd}.`,
|
|
@@ -1320,15 +1320,15 @@ ${bold('EXAMPLES')}
|
|
|
1320
1320
|
breakEven: (usd) =>
|
|
1321
1321
|
`Break-even is ${usd} of monthly spend. Below it you pay the floor for less usage than it buys; above it the saving grows.`,
|
|
1322
1322
|
spread: (low, high, median) =>
|
|
1323
|
-
`Measured spread: ${low} to ${high}, median ${median}. That is the shortfall risk in the only form a log can state it
|
|
1323
|
+
`Measured spread: ${low} to ${high}, median ${median}. That is the shortfall risk in the only form a log can state it, a count of real months and the range they covered, rather than a probability from a distribution nobody fitted.`,
|
|
1324
1324
|
shortTerm: (covered, term) =>
|
|
1325
1325
|
`This replays ${covered} months against a ${term}-month commitment. It is a real answer about ${covered} months and not about ${term}.`,
|
|
1326
1326
|
cannotTell: (why, needed) =>
|
|
1327
1327
|
why === 'no-history'
|
|
1328
1328
|
? 'Nothing to replay: this log has no whole months in it.'
|
|
1329
|
-
: `Cannot judge this from ${needed === '0' ? 'what exists' : `what exists
|
|
1329
|
+
: `Cannot judge this from ${needed === '0' ? 'what exists' : `what exists, ${needed} more whole month(s) would settle it`}. A commitment is signed for a year, and an answer from one month is a year-long decision made on a fortnight of evidence.`,
|
|
1330
1330
|
neverAForecast: () =>
|
|
1331
|
-
'Nothing here is annualised, extrapolated or fitted to a trend. If next quarter looks nothing like last quarter, this arithmetic says nothing about it
|
|
1331
|
+
'Nothing here is annualised, extrapolated or fitted to a trend. If next quarter looks nothing like last quarter, this arithmetic says nothing about it, which is the honest limit of what a log can tell you about a contract.',
|
|
1332
1332
|
},
|
|
1333
1333
|
|
|
1334
1334
|
owners: {
|
|
@@ -1337,39 +1337,39 @@ ${bold('EXAMPLES')}
|
|
|
1337
1337
|
'No owners configured. Add "owners" to trazum.config.json to attribute spend, for example {"patterns": {"payments": ["billing-*"]}, "budgets": {"payments": 400}}.',
|
|
1338
1338
|
columns: { owner: 'owner', spend: 'spend', budget: 'budget', calls: 'calls' },
|
|
1339
1339
|
verdict: (kind) =>
|
|
1340
|
-
kind === 'over' ? 'over' : kind === 'within' ? 'within' : kind === 'not-measured' ? 'not measured' : '
|
|
1340
|
+
kind === 'over' ? 'over' : kind === 'within' ? 'within' : kind === 'not-measured' ? 'not measured' : 'no verdict',
|
|
1341
1341
|
unallocated: (usd, share, labels) =>
|
|
1342
1342
|
`Unallocated: ${usd} (${share} of the bill), from ${labels}.`,
|
|
1343
1343
|
neverSpread: () =>
|
|
1344
|
-
'It is not divided between the owners above, and it never will be. Spreading unattributed spend proportionally is the most common lie in cost reporting: it makes every line add up and every team\u2019s number wrong by an amount nobody can see
|
|
1344
|
+
'It is not divided between the owners above, and it never will be. Spreading unattributed spend proportionally is the most common lie in cost reporting: it makes every line add up and every team\u2019s number wrong by an amount nobody can see, hardest on whoever instruments best, because their known spend is largest and they absorb the biggest share of somebody else\u2019s mystery. Claim it with a pattern.',
|
|
1345
1345
|
nothingUnallocated: () => 'Every workload in this log has an owner.',
|
|
1346
1346
|
sharedHeading: () => 'Shared, by a rule somebody wrote',
|
|
1347
1347
|
sharedRule: (label, rule) => `${label}: ${rule}`,
|
|
1348
1348
|
problemsHeading: () => 'This ownership config will not do what it looks like it does',
|
|
1349
1349
|
problem: (kind, detail) =>
|
|
1350
1350
|
kind === 'split-does-not-sum'
|
|
1351
|
-
? `${detail}
|
|
1351
|
+
? `${detail}, a split that does not sum to 1 loses money or invents it, silently. That workload is left whole in the unallocated line until this is fixed.`
|
|
1352
1352
|
: kind === 'split-names-unknown-owner'
|
|
1353
1353
|
? `${detail} names an owner that "owners.patterns" does not declare.`
|
|
1354
1354
|
: kind === 'split-has-one-owner'
|
|
1355
|
-
? `${detail} is "shared" with one owner
|
|
1355
|
+
? `${detail} is "shared" with one owner, that is a pattern written the long way, and reading it as a share invites a second owner to be added without the first being adjusted.`
|
|
1356
1356
|
: kind === 'negative-share'
|
|
1357
1357
|
? `${detail} has a negative share.`
|
|
1358
1358
|
: `${detail} has a budget but no patterns, so nothing can ever land on it.`,
|
|
1359
1359
|
notMeasured: (owner) =>
|
|
1360
|
-
`${owner} has a budget and no measured calls. That is NOT under budget
|
|
1360
|
+
`${owner} has a budget and no measured calls. That is NOT under budget, a team whose logs never arrived passes every budget it has, forever, and a green tick beside their name says the opposite of the truth.`,
|
|
1361
1361
|
},
|
|
1362
1362
|
|
|
1363
1363
|
semantic: {
|
|
1364
1364
|
heading: (path) => `Semantic pass on ${path}`,
|
|
1365
1365
|
willCost: (usd, input, output, model) =>
|
|
1366
|
-
`This will send the prompt to ${model}: about ${input} tokens in and ${output} out, roughly ${usd}. Estimated, not measured
|
|
1366
|
+
`This will send the prompt to ${model}: about ${input} tokens in and ${output} out, roughly ${usd}. Estimated, not measured, a tool that spends your money to tell you how to spend less should be the first thing audited by its own arithmetic. Pass --yes to run it.`,
|
|
1367
1367
|
needsYes: () => 'Nothing was sent. Add --yes once you have read the price above.',
|
|
1368
1368
|
finding: (kind, because) =>
|
|
1369
1369
|
`${kind === 'contradiction' ? 'Contradiction' : kind === 'restated-instruction' ? 'Restated' : 'Same thing, different words'}: ${because}`,
|
|
1370
1370
|
span: (line, text) => `line ${line}: ${text}`,
|
|
1371
1371
|
ceiling: (tokens) =>
|
|
1372
|
-
`At most ${tokens} tokens
|
|
1372
|
+
`At most ${tokens} tokens, a ceiling, not a saving. Merging these two means writing one passage that does the work of both, and nobody knows yet how long that is.`,
|
|
1373
1373
|
noCeiling: () =>
|
|
1374
1374
|
'No tokens attached. A contradiction is worth fixing because the prompt is wrong, not because it is long, and putting a figure on it would sell the wrong reason.',
|
|
1375
1375
|
nothingFound: () => 'Nothing survived checking. That is a real answer.',
|
|
@@ -1387,7 +1387,7 @@ ${bold('EXAMPLES')}
|
|
|
1387
1387
|
disposes: () =>
|
|
1388
1388
|
'The model proposes and the deterministic layer disposes: every quoted passage is checked character for character against the prompt, pairs the rules engine already catches are dropped, and every token figure is counted here rather than believed.',
|
|
1389
1389
|
optIn: () =>
|
|
1390
|
-
'This pass is optional and always will be. Trazum works with no key, no network and no model
|
|
1390
|
+
'This pass is optional and always will be. Trazum works with no key, no network and no model, that has been true since 0.1.0 and this does not change it.',
|
|
1391
1391
|
},
|
|
1392
1392
|
|
|
1393
1393
|
quality: {
|
|
@@ -1400,14 +1400,14 @@ ${bold('EXAMPLES')}
|
|
|
1400
1400
|
dropped: (from, to, outcomes, cost) =>
|
|
1401
1401
|
`The resolution rate moved from ${from} to ${to} on ${outcomes} measured outcomes, and this change ${cost}. Both halves are measured; neither is an estimate.`,
|
|
1402
1402
|
held: (from, to, outcomes) =>
|
|
1403
|
-
`The resolution rate moved from ${from} to ${to} on ${outcomes} measured outcomes
|
|
1403
|
+
`The resolution rate moved from ${from} to ${to} on ${outcomes} measured outcomes, up, measurably.`,
|
|
1404
1404
|
cannotTell: (why, need) =>
|
|
1405
1405
|
why === 'too-few-before'
|
|
1406
1406
|
? `Cannot tell: only ${need} outcomes before the change, and this gate needs 100 a side. It fails builds, so the threshold is not the one a rate uses elsewhere.`
|
|
1407
1407
|
: why === 'too-few-after'
|
|
1408
1408
|
? `Cannot tell yet: only ${need} outcomes since the change, and this gate needs 100 a side. Run it again once the traffic has caught up.`
|
|
1409
1409
|
: why === 'not-separable'
|
|
1410
|
-
? 'Cannot tell: the rate did not measurably move. That is NOT the same as "it held"
|
|
1410
|
+
? 'Cannot tell: the rate did not measurably move. That is NOT the same as "it held", a gate that spelled them the same way would pass a real regression it merely lacked the power to see.'
|
|
1411
1411
|
: why === 'no-vocabulary'
|
|
1412
1412
|
? 'Cannot tell: "outcomes.success" declares nothing, so there is no resolution rate to compare.'
|
|
1413
1413
|
: 'Cannot tell: something other than the prompt moved across the boundary. See below.',
|
|
@@ -1416,15 +1416,15 @@ ${bold('EXAMPLES')}
|
|
|
1416
1416
|
kind === 'model-mix-moved'
|
|
1417
1417
|
? `The model mix moved by ${detail}. The drop may be entirely somebody else's migration, and this tool cannot separate the two.`
|
|
1418
1418
|
: kind === 'volume-moved'
|
|
1419
|
-
? `The call volume moved ${detail}. A workload whose traffic moved that much is usually a workload whose population changed
|
|
1419
|
+
? `The call volume moved ${detail}. A workload whose traffic moved that much is usually a workload whose population changed, a new surface, a new customer, a campaign, and the questions being asked are not the questions from before.`
|
|
1420
1420
|
: `Outcome coverage moved from ${detail}. The two rates describe different populations: a team that starts instrumenting its hard cases sees its measured rate fall without anything having got worse.`,
|
|
1421
1421
|
notRandomised: () =>
|
|
1422
|
-
'This is a before-and-after, not an experiment. It splits traffic by time rather than at random, so everything else that changed at the same time is in the difference too
|
|
1422
|
+
'This is a before-and-after, not an experiment. It splits traffic by time rather than at random, so everything else that changed at the same time is in the difference too, which is why it says "cannot tell" far more readily than an A/B would.',
|
|
1423
1423
|
cannotSee: () =>
|
|
1424
1424
|
'It cannot see anything else you deployed that day. A "dropped" verdict says the rate fell and the three things it can check did not move. That is a smaller claim than "the prompt did it", and it is the largest one the evidence supports.',
|
|
1425
1425
|
gateFailed: () => 'Gate failed: a measured drop with nothing else to explain it.',
|
|
1426
1426
|
gateHeldOpen: () =>
|
|
1427
|
-
'Gate not passed and not failed. "Cannot tell" holds the claim open rather than exiting green
|
|
1427
|
+
'Gate not passed and not failed. "Cannot tell" holds the claim open rather than exiting green, the posture verify has had since 1.39.',
|
|
1428
1428
|
},
|
|
1429
1429
|
|
|
1430
1430
|
experiment: {
|
|
@@ -1436,10 +1436,10 @@ ${bold('EXAMPLES')}
|
|
|
1436
1436
|
arm: (name, rate, successes, recorded, interval) =>
|
|
1437
1437
|
`${name} ${rate} (${successes} of ${recorded} recorded) 95% ${interval}`,
|
|
1438
1438
|
wins: (name, low, high) =>
|
|
1439
|
-
`${name} wins. The difference is between ${low} and ${high} at 95% confidence
|
|
1439
|
+
`${name} wins. The difference is between ${low} and ${high} at 95% confidence, the whole interval is on one side of zero, which is what "wins" means here.`,
|
|
1440
1440
|
notSeparable: (why, needed) =>
|
|
1441
1441
|
why === 'no-difference-observed'
|
|
1442
|
-
? 'Not separable: both arms recorded the same rate. No sample size separates a difference of zero, so there is no "run it longer" to offer
|
|
1442
|
+
? 'Not separable: both arms recorded the same rate. No sample size separates a difference of zero, so there is no "run it longer" to offer, there is nothing here to find.'
|
|
1443
1443
|
: why === 'nothing-recorded'
|
|
1444
1444
|
? 'Not separable: an arm recorded no outcomes at all, so there is no rate to compare.'
|
|
1445
1445
|
: `Not separable on this traffic: the 95% interval on the difference includes zero. One number is larger, and that is not a finding. About ${needed} outcomes per arm would settle the difference observed so far.`,
|
|
@@ -1447,7 +1447,7 @@ ${bold('EXAMPLES')}
|
|
|
1447
1447
|
`Read early. The declared rule was ${declared} outcomes per arm and ${short} has ${recorded}. Nothing can stop a number being read early; this line exists so whoever reads the result later can see that it was.`,
|
|
1448
1448
|
honoured: (declared) => `Stopping rule honoured: both arms cleared ${declared} recorded outcomes.`,
|
|
1449
1449
|
marginalDearer: (better, usd) =>
|
|
1450
|
-
`${better} resolves more and costs more. One extra success costs ${usd}
|
|
1450
|
+
`${better} resolves more and costs more. One extra success costs ${usd}, that figure, not the rate, is what the decision turns on.`,
|
|
1451
1451
|
marginalCheaper: (better) => `${better} resolves more AND costs less per call. Nothing is being traded.`,
|
|
1452
1452
|
neverPromotes: () =>
|
|
1453
1453
|
'Nothing was changed. A winner is a finding; taking it is a decision with a name attached, and it belongs in the plan like everything else.',
|
|
@@ -1456,7 +1456,7 @@ ${bold('EXAMPLES')}
|
|
|
1456
1456
|
ladder: {
|
|
1457
1457
|
heading: () => 'Escalation ladders',
|
|
1458
1458
|
noLadders: () =>
|
|
1459
|
-
'No ladders configured. A ladder sends a workload to a cheap model first and escalates a recorded failure to a dearer one
|
|
1459
|
+
'No ladders configured. A ladder sends a workload to a cheap model first and escalates a recorded failure to a dearer one. Add "ladders" to trazum.config.json, for example {"support": {"tiers": ["claude-haiku-4-5", "claude-opus-5"], "escalateOn": ["escalated"]}}.',
|
|
1460
1460
|
workload: (label) => label,
|
|
1461
1461
|
arithmetic: (cheap, dear, breakEven) =>
|
|
1462
1462
|
`${cheap} a call cheap, ${dear} dear. Break-even escalation rate: ${breakEven}.`,
|
|
@@ -1475,7 +1475,7 @@ ${bold('EXAMPLES')}
|
|
|
1475
1475
|
: why === 'tier-unpriced'
|
|
1476
1476
|
? 'Cannot tell: one of the tiers is not in the price catalogue, so the break-even rate cannot be computed.'
|
|
1477
1477
|
: 'Cannot tell: this ladder declares no escalation values.',
|
|
1478
|
-
problemsHeading: (label) => `${label}
|
|
1478
|
+
problemsHeading: (label) => `${label}: this ladder will not do what it looks like it does`,
|
|
1479
1479
|
problem: (kind, detail) =>
|
|
1480
1480
|
kind === 'escalate-on-a-success'
|
|
1481
1481
|
? `escalateOn names "${detail}", which "outcomes.success" declares a SUCCESS. This ladder pays twice for work that already worked, on every call, while looking exactly like a cost-saving measure.`
|
|
@@ -1489,7 +1489,7 @@ ${bold('EXAMPLES')}
|
|
|
1489
1489
|
? `"${detail}" is not in the price catalogue.`
|
|
1490
1490
|
: `a ladder needs at least two tiers; this one has ${detail}.`,
|
|
1491
1491
|
theDoubleSpend: () =>
|
|
1492
|
-
'An escalation pays twice: the cheap attempt is not refunded. So a ladder saves money only below its break-even escalation rate, and above it costs more than never having built one
|
|
1492
|
+
'An escalation pays twice: the cheap attempt is not refunded. So a ladder saves money only below its break-even escalation rate, and above it costs more than never having built one, which is why the rate is printed beside the measurement rather than left to be worked out in somebody\u2019s head.',
|
|
1493
1493
|
notExecuted: () =>
|
|
1494
1494
|
'Trazum does not run the escalation. A ladder escalates after a failure is known, which is after the answer came back and usually after something downstream judged it, so the retry belongs in your own loop. What is here is the policy and the arithmetic that says whether the policy is worth running.',
|
|
1495
1495
|
},
|
|
@@ -1500,9 +1500,9 @@ ${bold('EXAMPLES')}
|
|
|
1500
1500
|
? `Name the provider to stand in front of. Known: ${known}.`
|
|
1501
1501
|
: `"${given}" is not a provider this gateway speaks for. Known: ${known}.`,
|
|
1502
1502
|
pricedNotFronted: (given, known) =>
|
|
1503
|
-
`Trazum prices ${given} but this gateway does not stand in front of it yet
|
|
1503
|
+
`Trazum prices ${given} but this gateway does not stand in front of it yet: it fronts ${known}. That is a real gap and it is on the roadmap, not a typo on your part. Until then: "trazum profile" prices a ${given} log you export, and its --max-usd gate fails a build on the bill after the fact. What you do not get is a refusal before the money is spent, which is the whole point of the gateway and the reason this is worth saying rather than listing ${given} as unknown.`,
|
|
1504
1504
|
needsPolicy: (policies) =>
|
|
1505
|
-
`--on-cannot-tell is required, and there is no default: ${policies}. When the gateway cannot judge a call
|
|
1505
|
+
`--on-cannot-tell is required, and there is no default: ${policies}. When the gateway cannot judge a call, no budget, nothing measured, an unpriced model, one of these happens, and only you know which failure your product can survive. fail-open keeps it working and lets the bill run; fail-closed stops the bill and takes it down with it. Picking one for you would be the most consequential decision in your architecture, made silently at install time.`,
|
|
1506
1506
|
listening: (where, provider) => `Gateway on ${where}, in front of ${provider}`,
|
|
1507
1507
|
/**
|
|
1508
1508
|
* One sentence per cause, and no `else`.
|
|
@@ -1510,26 +1510,26 @@ ${bold('EXAMPLES')}
|
|
|
1510
1510
|
* This was a two-branch ternary whose second arm explained OpenAI's
|
|
1511
1511
|
* `stream_options`. A third cause added later would have inherited that
|
|
1512
1512
|
* sentence and told a Gemini user about a setting that does not exist in
|
|
1513
|
-
* their SDK
|
|
1513
|
+
* their SDK, a message bounded by its neighbour rather than by its
|
|
1514
1514
|
* subject, which is this project's most repeated mistake.
|
|
1515
1515
|
*/
|
|
1516
1516
|
unmeasured: (cause, sofar) => {
|
|
1517
1517
|
if (cause === 'stream-broke') {
|
|
1518
|
-
return `unmeasured: the stream broke before its usage event
|
|
1518
|
+
return `unmeasured: the stream broke before its usage event, so the provider billed this call and this session cannot see it (${sofar} unmeasured so far)`;
|
|
1519
1519
|
}
|
|
1520
1520
|
if (cause === 'no-usage-event') {
|
|
1521
|
-
return `unmeasured: the stream carried no usage event
|
|
1521
|
+
return `unmeasured: the stream carried no usage event, on OpenAI that is every streaming call without stream_options.include_usage, so the total below is short by these (${sofar} unmeasured so far)`;
|
|
1522
1522
|
}
|
|
1523
|
-
return `unmeasured: the provider answered without any usage in the body
|
|
1523
|
+
return `unmeasured: the provider answered without any usage in the body, so the call was made and its cost is not knowable from the response, so the total below is short by these (${sofar} unmeasured so far)`;
|
|
1524
1524
|
},
|
|
1525
1525
|
pointYourSdk: (where) =>
|
|
1526
1526
|
`Point your SDK's base URL at ${where} and change nothing else. It speaks the provider's own wire format, so no code changes and no new client.`,
|
|
1527
1527
|
credential: () =>
|
|
1528
1528
|
'Your credential is forwarded untouched and never read, never stored, never logged and never put in a URL. Trazum holds no key here and cannot make a call of its own through this.',
|
|
1529
1529
|
neverSubstitutes: () =>
|
|
1530
|
-
'A call over budget is refused with HTTP 402 and the cheaper alternatives named
|
|
1530
|
+
'A call over budget is refused with HTTP 402 and the cheaper alternatives named, never silently swapped, trimmed or downgraded. 402 rather than 429 on purpose: every provider SDK retries a 429, which would turn one refusal into a retry storm.',
|
|
1531
1531
|
standing: (consumed, limit) =>
|
|
1532
|
-
`Judging against ${consumed} of ${limit}, measured, read once at start
|
|
1532
|
+
`Judging against ${consumed} of ${limit}, measured, read once at start: a file read in the request path would put this tool's latency between you and your provider on every call.`,
|
|
1533
1533
|
noStanding: () =>
|
|
1534
1534
|
'Nothing measured for this period, so every call is unjudged and the failure policy below decides. Set spend.monthlyUsd and pull with trazum connect.',
|
|
1535
1535
|
policy: (policy) =>
|
|
@@ -1537,7 +1537,7 @@ ${bold('EXAMPLES')}
|
|
|
1537
1537
|
? 'When it cannot judge: the call goes through, and the record says it was unjudged rather than within budget.'
|
|
1538
1538
|
: 'When it cannot judge: the call is refused. Nothing gets through unmeasured.',
|
|
1539
1539
|
measured: (model, label, input, output, substituted) =>
|
|
1540
|
-
` ${model}${label === null ? '' : ` [${label}]`}: ${input} in, ${output} out${substituted ? ' (substituted
|
|
1540
|
+
` ${model}${label === null ? '' : ` [${label}]`}: ${input} in, ${output} out${substituted ? ' (substituted, marked, and never counted as the call that was asked for)' : ''}`,
|
|
1541
1541
|
},
|
|
1542
1542
|
|
|
1543
1543
|
feedback: {
|
|
@@ -1546,19 +1546,19 @@ ${bold('EXAMPLES')}
|
|
|
1546
1546
|
'This command sends nothing, and neither does anything else here. Trazum has no telemetry: no ping, no install hook, no anonymous counter. A tool whose whole argument is that it reads your bill without uploading it cannot also be quietly reporting on you, and a test fails the build if this command ever reaches the network.',
|
|
1547
1547
|
whereHeading: () => 'Where',
|
|
1548
1548
|
wrongOptimisation: () =>
|
|
1549
|
-
'A rule changed what a prompt asks for
|
|
1549
|
+
'A rule changed what a prompt asks for, the report that matters most, and the failure this product exists to avoid:',
|
|
1550
1550
|
bug: () => 'Anything else that is wrong:',
|
|
1551
1551
|
question: () => 'A question, or an idea you are not sure about:',
|
|
1552
|
-
security: () => 'A security problem
|
|
1552
|
+
security: () => 'A security problem, privately, never a public issue:',
|
|
1553
1553
|
environmentHeading: () => 'What a maintainer will ask for',
|
|
1554
1554
|
environmentOnly: () =>
|
|
1555
|
-
'That is the whole of it. Nothing about your work is here
|
|
1555
|
+
'That is the whole of it. Nothing about your work is here, not the config, not a prompt, not a label, not a figure. Those are what a good report needs and what only you can decide to share.',
|
|
1556
1556
|
linkHeading: () => 'A blank issue with the above already filled in',
|
|
1557
1557
|
},
|
|
1558
1558
|
|
|
1559
1559
|
conform: {
|
|
1560
1560
|
noTarget: () =>
|
|
1561
|
-
'Pass a file to check
|
|
1561
|
+
'Pass a file to check, a usage log, or any document Trazum emits. Use "-" to read from stdin.',
|
|
1562
1562
|
badContract: (given, known) => `"${given}" is not a contract. Known contracts: ${known}.`,
|
|
1563
1563
|
unrecognised: (path) => `${path} does not match any contract Trazum knows.`,
|
|
1564
1564
|
/**
|
|
@@ -1583,49 +1583,49 @@ ${bold('EXAMPLES')}
|
|
|
1583
1583
|
problem: (at, kind, detail) => `${at}: ${detail} (${kind})`,
|
|
1584
1584
|
moreProblems: (count) => `…and ${count} more. Fix these first; they are often the same mistake.`,
|
|
1585
1585
|
unavailableHeading: () => 'What this cannot answer, and what would unlock it',
|
|
1586
|
-
unavailable: (finding, because, unlockedBy) => `${finding}
|
|
1586
|
+
unavailable: (finding, because, unlockedBy) => `${finding}, ${because}. Add ${unlockedBy}.`,
|
|
1587
1587
|
unavailableNeverGates: () =>
|
|
1588
1588
|
'None of those failed anything. Choosing not to record a field is a decision, not a defect, and a gate that failed on it would be this tool telling you what to log.',
|
|
1589
1589
|
},
|
|
1590
1590
|
|
|
1591
1591
|
rollup: {
|
|
1592
1592
|
noTargets: () =>
|
|
1593
|
-
'Pass the profile documents to roll up
|
|
1593
|
+
'Pass the profile documents to roll up, each one written by `trazum profile --json`, or a directory of them.',
|
|
1594
1594
|
noSuchTarget: (path) => `${path} is not there. A roll-up never guesses at a contribution it could not read.`,
|
|
1595
1595
|
emptyDirectory: (path) =>
|
|
1596
1596
|
`${path} holds no .json documents. An empty folder rolled up silently would report a team that spent nothing.`,
|
|
1597
1597
|
heading: (contributors, usd, calls) =>
|
|
1598
|
-
`Roll-up of ${plural(contributors, 'contributor')}
|
|
1598
|
+
`Roll-up of ${plural(contributors, 'contributor')}, ${usd} over ${plural(calls, 'call')}`,
|
|
1599
1599
|
span: (from, to) => `Covering ${from} to ${to}, stated and never extrapolated from.`,
|
|
1600
1600
|
noSpan: () => 'No contributor carried a clock, so this roll-up covers no stated period.',
|
|
1601
1601
|
contributorsHeading: () => 'Contributors, and what each one could not see',
|
|
1602
1602
|
contributor: (name, usd, calls, spanDays) =>
|
|
1603
|
-
`${name}
|
|
1603
|
+
`${name}, ${usd}, ${plural(calls, 'call')}${spanDays === null ? ', no clock' : `, ${plural(spanDays, 'day')}`}`,
|
|
1604
1604
|
claimedSpan: (from, to, contributors) =>
|
|
1605
|
-
`Asked for ${from} to ${to} by ${plural(contributors, 'contributor')}
|
|
1605
|
+
`Asked for ${from} to ${to} by ${plural(contributors, 'contributor')}: a claim about what was gone looking for, kept apart from what the records showed.`,
|
|
1606
1606
|
claimedRow: (from, to) => `asked for ${from} to ${to}`,
|
|
1607
1607
|
silentRuns: (runs) => runs,
|
|
1608
1608
|
undated: (count) =>
|
|
1609
1609
|
`${plural(count, 'record')} this contributor's own window could not place, because they carried no clock`,
|
|
1610
1610
|
rejectedHeading: () => 'Handed over and not merged',
|
|
1611
|
-
rejected: (name, because) => `${name}
|
|
1611
|
+
rejected: (name, because) => `${name}, ${because}`,
|
|
1612
1612
|
via: (rollup) => `via ${rollup}`,
|
|
1613
|
-
rejectedVia: (name, via, because) => `${name}, via ${via}
|
|
1613
|
+
rejectedVia: (name, via, because) => `${name}, via ${via}, ${because}`,
|
|
1614
1614
|
repeated: (names) =>
|
|
1615
|
-
`Handed over more than once, by name: ${names}. Its money is counted every time. Two machines genuinely sharing a name is possible, so nothing was subtracted
|
|
1615
|
+
`Handed over more than once, by name: ${names}. Its money is counted every time. Two machines genuinely sharing a name is possible, so nothing was subtracted. Check whether a roll-up and one of the machines inside it were both handed over.`,
|
|
1616
1616
|
identical: (names) => `The same document arrived more than once: ${names}.`,
|
|
1617
1617
|
identicalUsd: (usd) =>
|
|
1618
|
-
`${usd} of the total above is the repeat. It is counted, not discarded
|
|
1618
|
+
`${usd} of the total above is the repeat. It is counted, not discarded, whether it is one export handed over twice or two machines that agreed exactly is yours to know.`,
|
|
1619
1619
|
byLabelHeading: () => 'The merged bill, per workload',
|
|
1620
|
-
labelRow: (label, usd, calls) => `${label}
|
|
1620
|
+
labelRow: (label, usd, calls) => `${label}, ${usd}, ${plural(calls, 'call')}`,
|
|
1621
1621
|
notMergedHeading: () => 'Findings that do not roll up',
|
|
1622
|
-
notMerged: (finding, because) => `${finding}
|
|
1622
|
+
notMerged: (finding, because) => `${finding}, ${because}.`,
|
|
1623
1623
|
presentIn: (names) => `Present in: ${names}. Read it there.`,
|
|
1624
1624
|
cannotSayHeading: () => 'What this roll-up cannot say about itself',
|
|
1625
1625
|
caveat: (code) => {
|
|
1626
1626
|
switch (code) {
|
|
1627
1627
|
case 'overlap-invisible':
|
|
1628
|
-
return 'Overlap between contributors is unmeasurable here. Two people exporting the same traffic double the bill, and a merge of summaries cannot see it
|
|
1628
|
+
return 'Overlap between contributors is unmeasurable here. Two people exporting the same traffic double the bill, and a merge of summaries cannot see it: the raw lines a duplicate check needs are in no document.';
|
|
1629
1629
|
case 'mismatched-spans':
|
|
1630
1630
|
return 'The contributors cover meaningfully different periods. The sum is a sum; reading a share of it as a comparison of rates is the mistake this names.';
|
|
1631
1631
|
case 'contributor-without-clock':
|
|
@@ -1633,7 +1633,7 @@ ${bold('EXAMPLES')}
|
|
|
1633
1633
|
case 'day-top-label-unknown':
|
|
1634
1634
|
return "A day drew from more than one contributor, so its dearest label is unknown: each contributor knows its own, and the merged answer needs per-label-per-day spend no document carries.";
|
|
1635
1635
|
case 'no-claimed-period':
|
|
1636
|
-
return 'A contributor stated no window, so its span is all that is known of it
|
|
1636
|
+
return 'A contributor stated no window, so its span is all that is known of it, and a log whose latest record is the 5th may be a log of a quiet week or a log that stopped being written on the 5th.';
|
|
1637
1637
|
case 'silence-inside-a-claim':
|
|
1638
1638
|
return 'A contributor recorded nothing on days it asked for. Whether that is a quiet stretch or an export that stopped is yours to know; the days are named above.';
|
|
1639
1639
|
case 'claim-not-bounded':
|
|
@@ -1661,18 +1661,18 @@ ${bold('EXAMPLES')}
|
|
|
1661
1661
|
`The catalogue has no price for "${id}", and a comparison against a price nobody has is worse than none. trazum models lists what it knows; a pricing overlay adds what it does not.`,
|
|
1662
1662
|
heading: (target) => `Switching this traffic to ${target}, measured`,
|
|
1663
1663
|
saves: (current, target, saving) =>
|
|
1664
|
-
`${current} on this log becomes ${target} on the candidate
|
|
1664
|
+
`${current} on this log becomes ${target} on the candidate, ${saving} less, on the same tokens.`,
|
|
1665
1665
|
costs: (current, target, extra) =>
|
|
1666
|
-
`${current} on this log becomes ${target} on the candidate
|
|
1666
|
+
`${current} on this log becomes ${target} on the candidate, ${extra} MORE. This switch loses money before quality is even asked.`,
|
|
1667
1667
|
nothingMovable: () => 'Nothing here can move: every slice is already on the candidate or exceeds its context window.',
|
|
1668
1668
|
movable: (calls, slices) => `${calls} call(s) across ${slices} slice(s) could move.`,
|
|
1669
1669
|
overContext: (slices, usd) =>
|
|
1670
|
-
`${slices} slice(s) worth ${usd} cannot move
|
|
1670
|
+
`${slices} slice(s) worth ${usd} cannot move, at least one call exceeds the candidate's context window. Their money is in none of the figures above.`,
|
|
1671
1671
|
alreadyOnTarget: (calls, usd) => `${calls} call(s) worth ${usd} are already on the candidate.`,
|
|
1672
|
-
window: (days) => `Measured over ${days} day(s) of this log
|
|
1672
|
+
window: (days) => `Measured over ${days} day(s) of this log: every figure above is arithmetic on that window.`,
|
|
1673
1673
|
noWindow: () => 'No record carries a timestamp, so this log has no measured rate.',
|
|
1674
1674
|
breakEvenDays: (migration, days, measuredDays) =>
|
|
1675
|
-
`A declared migration cost of ${migration} is recovered after ~${days} day(s) at the saving measured over ${measuredDays} day(s). Division on the past
|
|
1675
|
+
`A declared migration cost of ${migration} is recovered after ~${days} day(s) at the saving measured over ${measuredDays} day(s). Division on the past, what next month does is next month's business.`,
|
|
1676
1676
|
breakEvenNoSaving: (migration) =>
|
|
1677
1677
|
`Break-even refused: the switch saves nothing, so the declared ${migration} has nothing to recover.`,
|
|
1678
1678
|
breakEvenNoClock: (migration) =>
|
|
@@ -1689,7 +1689,7 @@ ${bold('EXAMPLES')}
|
|
|
1689
1689
|
'Both numbers are yours to declare: trazum ownrate --gpu-usd-hour <n> --tokens-per-second <n> [--utilization <0-1>]',
|
|
1690
1690
|
invalid: (flag) => `--${flag} must be a positive number (utilization: greater than 0, at most 1).`,
|
|
1691
1691
|
result: (usdPerMTok, tokensPerSecond, gpuUsdPerHour, utilizationPct) =>
|
|
1692
|
-
`${usdPerMTok} per million tokens
|
|
1692
|
+
`${usdPerMTok} per million tokens, ${gpuUsdPerHour}/hour over ${tokensPerSecond} tokens/second at ${utilizationPct}% serving.`,
|
|
1693
1693
|
declared: () =>
|
|
1694
1694
|
'Derived from your declaration, honest only if the throughput was measured. Every report that prices this model will rest on these two numbers.',
|
|
1695
1695
|
snippetHeading: () => 'Paste into trazum.config.json (the "pricing" overlay file):',
|
|
@@ -1700,7 +1700,7 @@ ${bold('EXAMPLES')}
|
|
|
1700
1700
|
notFound: (path) => `${path}: not found`,
|
|
1701
1701
|
noExports: (path) => `${path}: no .json/.jsonl/.ndjson OTLP exports under it.`,
|
|
1702
1702
|
summary: (files, llmSpans) => `${files} file(s), ${llmSpans} LLM span(s) priced.`,
|
|
1703
|
-
skipped: (otherSpans) => `${otherSpans} non-LLM span(s) skipped
|
|
1703
|
+
skipped: (otherSpans) => `${otherSpans} non-LLM span(s) skipped, the trace's other work, not a bill.`,
|
|
1704
1704
|
noCache: (count) => `${count} span(s) carried no cache data. OpenTelemetry has not standardised the cache TTL split, so the cache verdicts read "cannot tell" on this log rather than a fabricated one.`,
|
|
1705
1705
|
unparseable: (count) => `${count} line(s) did not parse as JSON.`,
|
|
1706
1706
|
written: (file) => `Wrote ${file}.`,
|
|
@@ -1711,14 +1711,21 @@ ${bold('EXAMPLES')}
|
|
|
1711
1711
|
notFound: (path) => `${path}: not found`,
|
|
1712
1712
|
noTranscripts: (path) => `${path}: no .jsonl transcripts under it. Claude Code writes them per session under ~/.claude/projects/<project>/.`,
|
|
1713
1713
|
summary: (files, records) => `${files} transcript(s), ${records} priced call(s).`,
|
|
1714
|
-
collapsed: (lines) => `${lines} extra line(s) of already-counted calls collapsed
|
|
1714
|
+
collapsed: (lines) => `${lines} extra line(s) of already-counted calls collapsed: one API call is written as one line per content block, and counting each line would overbill.`,
|
|
1715
1715
|
streamed: (count) => `${count} call(s) were written while still streaming; the final line's counts stood.`,
|
|
1716
|
-
disagreements: (count) => `${count} call(s) had lines that disagreed beyond streaming growth
|
|
1716
|
+
disagreements: (count) => `${count} call(s) had lines that disagreed beyond streaming growth: a count shrank, or another field changed. The last line stood, but that is a finding: worth a look at the transcript.`,
|
|
1717
1717
|
noRequestId: (count) => `${count} call(s) carried no request id, so nothing could be collapsed against them; each is priced as recorded.`,
|
|
1718
1718
|
synthetic: (count) => `${count} turn(s) Claude Code wrote itself (model "<synthetic>") are not in the output: no provider ever billed them, so they are not calls.`,
|
|
1719
1719
|
labelled: () => 'Labelled by project folder. Pass --no-label-from-project to leave the label off, or --label <name> to set one.',
|
|
1720
1720
|
skipped: (other, unparseable, withoutUsage) => `Passed over: ${other} non-assistant line(s) (the transcript's other business), ${unparseable} unparseable, ${withoutUsage} assistant line(s) without usage.`,
|
|
1721
1721
|
written: (file) => `Wrote ${file}.`,
|
|
1722
|
+
stateNeedsFile: () =>
|
|
1723
|
+
'--state reads one transcript, not a folder: the state ties an offset in the transcript to a length of the output, and several transcripts appending to one output have no single such length. Point it at the .jsonl file.',
|
|
1724
|
+
stateNeedsOut: () => '--state needs --out: there is nothing to resume against when the records go to stdout.',
|
|
1725
|
+
resumed: (skipped, offset) =>
|
|
1726
|
+
skipped === 0
|
|
1727
|
+
? `Read the transcript in full and recorded a resume point at byte ${offset}.`
|
|
1728
|
+
: `Resumed at byte ${skipped}, so ${skipped} byte(s) were not re-read. Next resume point: byte ${offset}.`,
|
|
1722
1729
|
},
|
|
1723
1730
|
|
|
1724
1731
|
position: {
|
|
@@ -1727,12 +1734,12 @@ ${bold('EXAMPLES')}
|
|
|
1727
1734
|
scopeDay: () => 'today',
|
|
1728
1735
|
scopeLabel: (label) => `"${label}"`,
|
|
1729
1736
|
within: (scope, measured, limit, remaining, days, elapsed) =>
|
|
1730
|
-
`${scope}: ${measured} of ${limit} measured
|
|
1737
|
+
`${scope}: ${measured} of ${limit} measured, ${remaining} left (${days} of ${elapsed} elapsed days measured)`,
|
|
1731
1738
|
over: (scope, measured, limit, overBy) =>
|
|
1732
|
-
`${scope}: over
|
|
1733
|
-
cannotTell: (scope) => `${scope}: cannot tell
|
|
1739
|
+
`${scope}: over, ${measured} measured against ${limit}, ${overBy} past the ceiling`,
|
|
1740
|
+
cannotTell: (scope) => `${scope}: cannot tell, because nothing was measured in this window`,
|
|
1734
1741
|
distance: (days, rate, overDays) =>
|
|
1735
|
-
`at ${rate}/day over ${overDays} measured days, the ceiling is ${days} days away
|
|
1742
|
+
`at ${rate}/day over ${overDays} measured days, the ceiling is ${days} days away, division on the past, not a forecast`,
|
|
1736
1743
|
unmeasuredHeading: () => 'Configured and not measurable from this log',
|
|
1737
1744
|
unmeasured: (scope, why) => `${scope}: ${why}`,
|
|
1738
1745
|
why: (reason) =>
|
|
@@ -1741,8 +1748,8 @@ ${bold('EXAMPLES')}
|
|
|
1741
1748
|
: reason === 'no-labels'
|
|
1742
1749
|
? 'no record carries a label, so per-label spend is unknowable here'
|
|
1743
1750
|
: reason === 'nothing-recorded'
|
|
1744
|
-
? 'the log is empty
|
|
1745
|
-
: 'the log records labels and has never seen this one this month
|
|
1751
|
+
? 'the log is empty: nothing was recorded'
|
|
1752
|
+
: 'the log records labels and has never seen this one this month, possibly renamed, possibly idle, and neither is "under budget"',
|
|
1746
1753
|
cannotSayHeading: () => 'What this deliberately does not answer',
|
|
1747
1754
|
cannotSay: (code) => {
|
|
1748
1755
|
switch (code) {
|
|
@@ -1755,9 +1762,9 @@ ${bold('EXAMPLES')}
|
|
|
1755
1762
|
}
|
|
1756
1763
|
},
|
|
1757
1764
|
unpriced: (count) =>
|
|
1758
|
-
`${count} record(s) name a model the catalogue cannot price. They contribute nothing to any figure above
|
|
1765
|
+
`${count} record(s) name a model the catalogue cannot price. They contribute nothing to any figure above: money nobody can see, said here rather than hidden.`,
|
|
1759
1766
|
source: () =>
|
|
1760
|
-
'Measured from this log alone, priced record by record. The store\'s provider-billed monthly standing is a different measurement
|
|
1767
|
+
'Measured from this log alone, priced record by record. The store\'s provider-billed monthly standing is a different measurement, "trazum store" prints it, and the two are never merged into one figure.',
|
|
1761
1768
|
noLog: () => 'position needs a usage log: trazum position usage.jsonl',
|
|
1762
1769
|
},
|
|
1763
1770
|
pulse: {
|
|
@@ -1774,7 +1781,7 @@ ${bold('EXAMPLES')}
|
|
|
1774
1781
|
return kind;
|
|
1775
1782
|
}
|
|
1776
1783
|
},
|
|
1777
|
-
neverRun: (name) => `${name}: never here. Not late
|
|
1784
|
+
neverRun: (name) => `${name}: never here. Not late: there is no cadence to be late against.`,
|
|
1778
1785
|
age: (name, when, hours) => `${name}: ${when}Z, ${plural(hours, 'hour')} ago`,
|
|
1779
1786
|
noThreshold: () =>
|
|
1780
1787
|
'Nothing was judged. Pass --max-stale-hours <n> to make a run that stopped a failing exit code; how stale is too stale is a policy, and this tool does not write yours.',
|
|
@@ -1782,12 +1789,12 @@ ${bold('EXAMPLES')}
|
|
|
1782
1789
|
stale: (hours) =>
|
|
1783
1790
|
`Something that runs here has not run in over ${plural(hours, 'hour')}. Silence from a scheduled job and silence from a job with nothing to report look identical; this is which.`,
|
|
1784
1791
|
notAService: () =>
|
|
1785
|
-
'This command runs nothing and hosts nothing. Something has to notice, and the something is your CI: a step that runs this on the schedule you already have turns a dead cron into a red build, without Trazum holding anybody\'s metrics. How far the measurements reach is reported and never judged
|
|
1792
|
+
'This command runs nothing and hosts nothing. Something has to notice, and the something is your CI: a step that runs this on the schedule you already have turns a dead cron into a red build, without Trazum holding anybody\'s metrics. How far the measurements reach is reported and never judged: that is a provider reporting on its own schedule, not a job that failed.',
|
|
1786
1793
|
},
|
|
1787
1794
|
|
|
1788
1795
|
html: {
|
|
1789
1796
|
caveatsHeading: () => 'What this report cannot say',
|
|
1790
|
-
noClock: () => 'No record carried a timestamp, so days, rates and drift are not in this report
|
|
1797
|
+
noClock: () => 'No record carried a timestamp, so days, rates and drift are not in this report: a total is still a total.',
|
|
1791
1798
|
noSessions: () => 'No record carried a session, so conversation growth and per-conversation costs are not in this report.',
|
|
1792
1799
|
byLabelHeading: () => 'By workload',
|
|
1793
1800
|
byModelHeading: () => 'By model',
|
|
@@ -1810,7 +1817,7 @@ ${bold('EXAMPLES')}
|
|
|
1810
1817
|
|
|
1811
1818
|
schema: {
|
|
1812
1819
|
noTarget: (known) =>
|
|
1813
|
-
`Name a contract to print its JSON Schema. Known: ${known}. The schema validates with any draft 2020-12 validator
|
|
1820
|
+
`Name a contract to print its JSON Schema. Known: ${known}. The schema validates with any draft 2020-12 validator, no Trazum required.`,
|
|
1814
1821
|
unknown: (name, known) => `"${name}" is not a contract. Known: ${known}.`,
|
|
1815
1822
|
},
|
|
1816
1823
|
|
|
@@ -1823,7 +1830,7 @@ ${bold('EXAMPLES')}
|
|
|
1823
1830
|
colRatio: () => 'ratio',
|
|
1824
1831
|
colPeakRss: () => 'peak RSS',
|
|
1825
1832
|
note: () =>
|
|
1826
|
-
'One shot each, this machine, today. The wall clock is for a person reading two tables side by side; the ratio
|
|
1833
|
+
'One shot each, this machine, today. The wall clock is for a person reading two tables side by side; the ratio, workload over an in-process calibration loop, is the number a gate can hold, because the machine cancels out of it. Record one with --record, hold a build to it with --against and a stated --max-ratio.',
|
|
1827
1834
|
unknownWorkload: (id, known) => `Unknown workload "${id}". Known: ${known}.`,
|
|
1828
1835
|
recorded: (path) => `Baseline written to ${path}. Commit it; --against reads it.`,
|
|
1829
1836
|
needsMaxRatio: () =>
|
|
@@ -1846,21 +1853,21 @@ ${bold('EXAMPLES')}
|
|
|
1846
1853
|
where: {
|
|
1847
1854
|
hostHeading: () => 'Running inside',
|
|
1848
1855
|
subscription: (host) =>
|
|
1849
|
-
`${host} bills by subscription, not by the token. A monthly saving below is arithmetic about tokens, not money you get back
|
|
1856
|
+
`${host} bills by subscription, not by the token. A monthly saving below is arithmetic about tokens, not money you get back, what you gain is context window and rate-limit headroom.`,
|
|
1850
1857
|
noTarget: () => 'Pass a source file to see which provider its prompts are sent to.',
|
|
1851
1858
|
sourceHeading: (path) => `Prompts in ${path} go to`,
|
|
1852
1859
|
conflict: () => 'Cannot tell: the file names more than one provider.',
|
|
1853
1860
|
conflictFallback: () =>
|
|
1854
1861
|
'Nothing was assumed. Set "usage.model" in trazum.config.json, or pass --model.',
|
|
1855
1862
|
nothingFound: () => 'Nothing in this file says which provider it calls.',
|
|
1856
|
-
providerOnly: () => ' (provider only
|
|
1863
|
+
providerOnly: () => ' (provider only, nothing named a model)',
|
|
1857
1864
|
evidenceLine: (line, kind, detail) => `line ${line} ${kind}: ${detail}`,
|
|
1858
1865
|
pricedAs: () => 'Priced as',
|
|
1859
1866
|
fromConfig: () => '(from trazum.config.json)',
|
|
1860
1867
|
fromDetection: () => '(read from the source)',
|
|
1861
1868
|
fromProviderDefault: (provider) =>
|
|
1862
1869
|
`(${provider} was read from the source; nothing named a model, so this is theirs)`,
|
|
1863
|
-
fromDefault: () => '(the built-in default
|
|
1870
|
+
fromDefault: () => '(the built-in default, nothing said otherwise)',
|
|
1864
1871
|
},
|
|
1865
1872
|
|
|
1866
1873
|
pricing: {
|
|
@@ -1901,17 +1908,17 @@ ${bold('EXAMPLES')}
|
|
|
1901
1908
|
noteFormat: (tokens) => `~${tokens} tokens restating the output format`,
|
|
1902
1909
|
noteProtected: (pct) => `${pct}% is code or URLs, which cannot be trimmed`,
|
|
1903
1910
|
skipped: (count) =>
|
|
1904
|
-
`Skipped ${count} source ${count === 1 ? 'file' : 'files'} with no \`// trazum:prompt\` marker
|
|
1911
|
+
`Skipped ${count} source ${count === 1 ? 'file' : 'files'} with no \`// trazum:prompt\` marker, `
|
|
1905
1912
|
+ 'their prompts are not in this ranking.',
|
|
1906
1913
|
densityNote: () =>
|
|
1907
|
-
'Tok/sen is tokens per sentence: verbosity independent of length. There is no score
|
|
1914
|
+
'Tok/sen is tokens per sentence: verbosity independent of length. There is no score. Every column is a measurement you can check against the file.',
|
|
1908
1915
|
recoverableNote: () =>
|
|
1909
|
-
'Recover is what the deterministic rules would take at this level, priced by the usage profile, with the token count beside it
|
|
1916
|
+
'Recover is what the deterministic rules would take at this level, priced by the usage profile, with the token count beside it: a saving of one token is twenty-five cents and no work worth doing. It is measured by running the rules, not by a formula.',
|
|
1910
1917
|
},
|
|
1911
1918
|
|
|
1912
1919
|
blame: {
|
|
1913
1920
|
heading: (path, revisions) =>
|
|
1914
|
-
`${path}
|
|
1921
|
+
`${path}, ${revisions} ${revisions === 1 ? 'revision' : 'revisions'}`,
|
|
1915
1922
|
notARepository: () =>
|
|
1916
1923
|
'blame reads a file\'s history from git, and this directory is not inside a repository.',
|
|
1917
1924
|
outsideRepository: (path) =>
|
|
@@ -1925,7 +1932,7 @@ ${bold('EXAMPLES')}
|
|
|
1925
1932
|
`That movement is ${amount} a month on ${model} at ${calls} calls.`,
|
|
1926
1933
|
biggestRise: () => 'Biggest single increase',
|
|
1927
1934
|
biggestRiseDetail: (tokens, author, subject, sha) =>
|
|
1928
|
-
`+${tokens} tokens
|
|
1935
|
+
`+${tokens} tokens, ${author}, "${subject}" (${sha})`,
|
|
1929
1936
|
addedAt: () => 'added',
|
|
1930
1937
|
goneAt: () => 'not present',
|
|
1931
1938
|
truncated: (shown) =>
|
|
@@ -1950,17 +1957,17 @@ ${bold('EXAMPLES')}
|
|
|
1950
1957
|
title: () => 'Available rules',
|
|
1951
1958
|
disableHint: () => ' Turn off the ones you do not want with --disable id1,id2',
|
|
1952
1959
|
measureHeading: (root, prompts, level) =>
|
|
1953
|
-
`What each rule recovers in ${root}
|
|
1960
|
+
`What each rule recovers in ${root}, ${plural(prompts, 'prompt')}, level ${level}`,
|
|
1954
1961
|
measureTotals: (before, saved, floor) =>
|
|
1955
1962
|
`${before} tokens before. The rules recover ${saved}; normalisation recovers ${floor} whether a rule is enabled or not, and that is not the rules' work.`,
|
|
1956
1963
|
measureRow: (id, marginal, alone, prompts) =>
|
|
1957
1964
|
`${id.padEnd(22)} lost if removed ${marginal.padStart(6)} alone ${alone.padStart(6)} ${plural(prompts, 'prompt')}`,
|
|
1958
1965
|
measureOverlap: (sumOfAlone, saved) =>
|
|
1959
|
-
`The rules recover ${sumOfAlone} between them one at a time and ${saved} together. The gap is the overlap
|
|
1966
|
+
`The rules recover ${sumOfAlone} between them one at a time and ${saved} together. The gap is the overlap: two rules finding the same tokens, and it is stated rather than resolved into a total, because a single figure here is the one number that cannot be true.`,
|
|
1960
1967
|
measureRedundant: (ids) =>
|
|
1961
|
-
`Every token these find, something else finds too, here: ${ids}. An overlap, not a defect
|
|
1968
|
+
`Every token these find, something else finds too, here: ${ids}. An overlap, not a defect, and not a reason to delete one without deciding which is the better to keep.`,
|
|
1962
1969
|
measureFiredWithoutSaving: (ids) =>
|
|
1963
|
-
`These changed the prompt and recovered no tokens: ${ids}. Not the same as inert
|
|
1970
|
+
`These changed the prompt and recovered no tokens: ${ids}. Not the same as inert: they were exercised, and they are altering somebody's instruction for no measured benefit.`,
|
|
1964
1971
|
measureInert: (ids) =>
|
|
1965
1972
|
`These changed nothing in this corpus: ${ids}. That is a fact about these files, not about the rules: a rule that finds nothing here has not been shown to find nothing anywhere.`,
|
|
1966
1973
|
measureBand: (source) =>
|
|
@@ -1972,7 +1979,7 @@ ${bold('EXAMPLES')}
|
|
|
1972
1979
|
|
|
1973
1980
|
doctor: {
|
|
1974
1981
|
heading: (root, prompts) =>
|
|
1975
|
-
`${root}
|
|
1982
|
+
`${root}, ${prompts} ${prompts === 1 ? 'prompt' : 'prompts'}`,
|
|
1976
1983
|
subheading: (model, calls) => `Priced on ${model} at ${calls} calls a month.`,
|
|
1977
1984
|
pricesReviewed: (date, days) => `Prices reviewed ${date}${ago(days)}.`,
|
|
1978
1985
|
budgetsHeading: () => 'Budgets',
|
|
@@ -1984,8 +1991,8 @@ ${bold('EXAMPLES')}
|
|
|
1984
1991
|
`${count} of ${total} ${count === 1 ? 'prompt has' : 'prompts have'} no budget, so nothing is watching ${count === 1 ? 'it' : 'them'}`,
|
|
1985
1992
|
overBudget: (count) =>
|
|
1986
1993
|
count === 1
|
|
1987
|
-
? '1 prompt is already over its budget
|
|
1988
|
-
: `${count} prompts are already over their budget
|
|
1994
|
+
? '1 prompt is already over its budget: trazum check would fail on it'
|
|
1995
|
+
: `${count} prompts are already over their budget: trazum check would fail on them`,
|
|
1989
1996
|
andMore: (count) => `and ${count} more`,
|
|
1990
1997
|
findingsHeading: () => 'What it would be worth fixing',
|
|
1991
1998
|
acrossPrompts: (count) => `${count} ${count === 1 ? 'prompt' : 'prompts'}`,
|
|
@@ -1993,7 +2000,7 @@ ${bold('EXAMPLES')}
|
|
|
1993
2000
|
'Each line is the same advisory trazum optimize raises on those prompts, summed. '
|
|
1994
2001
|
+ 'Run it on any one of them to see the figure on its own.',
|
|
1995
2002
|
notAGate: () =>
|
|
1996
|
-
'Nothing here fails a build. trazum check is the gate; this is the survey
|
|
2003
|
+
'Nothing here fails a build. trazum check is the gate; this is the survey: the '
|
|
1997
2004
|
+ 'model recommendation is a keyword heuristic, and a build gated on one teaches '
|
|
1998
2005
|
+ 'people to re-run until green.',
|
|
1999
2006
|
|
|
@@ -2006,10 +2013,10 @@ ${bold('EXAMPLES')}
|
|
|
2006
2013
|
sharedPrefixFix: (drift) =>
|
|
2007
2014
|
drift === 'whitespace'
|
|
2008
2015
|
? 'A formatter fixes this: the text already agrees, only the spacing does not.'
|
|
2009
|
-
: 'Someone has to pick one wording
|
|
2016
|
+
: 'Someone has to pick one wording, the text itself differs, not just its spacing.',
|
|
2010
2017
|
sharedPrefixNoFigure: () =>
|
|
2011
2018
|
'No figure is attached, deliberately. Caching matches bytes, so these prompts hold '
|
|
2012
|
-
+ 'one cache entry each instead of one between them
|
|
2019
|
+
+ 'one cache entry each instead of one between them, but what that costs depends on '
|
|
2013
2020
|
+ 'how the calls are spread across the group, and Trazum applies a single '
|
|
2014
2021
|
+ '--cache-hit-rate to every prompt. Pricing it would mean inventing your traffic.',
|
|
2015
2022
|
},
|
|
@@ -2027,7 +2034,7 @@ ${bold('EXAMPLES')}
|
|
|
2027
2034
|
selfAgreement: (pct) =>
|
|
2028
2035
|
`The prompt agrees with itself ${pct} of the time. That is the yardstick: a removal `
|
|
2029
2036
|
+ 'that moves the answer less than this moved nothing attributable to the example.',
|
|
2030
|
-
line: (n, tokens, pct) => `example ${n}
|
|
2037
|
+
line: (n, tokens, pct) => `example ${n}, ${tokens} tokens, ${pct} agreement without it`,
|
|
2031
2038
|
verdictNeeded: () => 'needed here',
|
|
2032
2039
|
verdictRecoverable: () => 'no effect on these inputs',
|
|
2033
2040
|
verdictUnknown: () => 'inconclusive',
|
|
@@ -2035,7 +2042,7 @@ ${bold('EXAMPLES')}
|
|
|
2035
2042
|
`${tokens} tokens sit in examples whose removal changed nothing measurable here.`,
|
|
2036
2043
|
caveat: () =>
|
|
2037
2044
|
'Which is not the same as "delete them". An example may exist for a case these '
|
|
2038
|
-
+ 'inputs do not contain
|
|
2045
|
+
+ 'inputs do not contain, the boundary condition somebody hit in production and '
|
|
2039
2046
|
+ 'added a demonstration for. This measures the inputs you gave it, and only you '
|
|
2040
2047
|
+ 'know whether they cover what matters. Nothing was edited.',
|
|
2041
2048
|
},
|
|
@@ -2101,7 +2108,7 @@ ${bold('EXAMPLES')}
|
|
|
2101
2108
|
allTotal: (delta, prompts) =>
|
|
2102
2109
|
`${delta} tokens across ${prompts} ${prompts === 1 ? 'prompt' : 'prompts'}`,
|
|
2103
2110
|
signConvention: () =>
|
|
2104
|
-
'Every figure is after minus before, so positive means worse
|
|
2111
|
+
'Every figure is after minus before, so positive means worse, the opposite of the rest of Trazum.',
|
|
2105
2112
|
onlyBefore: () => 'only before',
|
|
2106
2113
|
onlyAfter: () => 'only after ',
|
|
2107
2114
|
onlyOneSideNote: () =>
|
|
@@ -2109,7 +2116,7 @@ ${bold('EXAMPLES')}
|
|
|
2109
2116
|
},
|
|
2110
2117
|
|
|
2111
2118
|
markdown: {
|
|
2112
|
-
checkHeading: (target) => `Trazum
|
|
2119
|
+
checkHeading: (target) => `Trazum, token budgets for ${target}`,
|
|
2113
2120
|
baselineGrew: (delta, pct) => `This branch adds ${delta} tokens (${pct}) to the prompts here`,
|
|
2114
2121
|
baselineShrank: (delta, pct) => `This branch removes ${delta} tokens (${pct}) from the prompts here`,
|
|
2115
2122
|
baselineUnchanged: () => 'No change against the recorded baseline',
|
|
@@ -2123,10 +2130,10 @@ ${bold('EXAMPLES')}
|
|
|
2123
2130
|
'The scenario or the price list moved since the baseline was recorded, so the two monthly figures are not the same measurement and are not subtracted here. The token comparison above is unaffected.',
|
|
2124
2131
|
baselineReRecord: (command, path) =>
|
|
2125
2132
|
`If this growth is intended, re-record with \`${command}\` and commit \`${path}\`.`,
|
|
2126
|
-
diffHeading: (before, after) => `Trazum
|
|
2133
|
+
diffHeading: (before, after) => `Trazum, ${before} → ${after}`,
|
|
2127
2134
|
rankHeading: (root, count) =>
|
|
2128
|
-
`Trazum
|
|
2129
|
-
blameHeading: (path) => `Trazum
|
|
2135
|
+
`Trazum, what to fix first in ${root} (${count} ${count === 1 ? "prompt" : "prompts"})`,
|
|
2136
|
+
blameHeading: (path) => `Trazum, token history for ${path}`,
|
|
2130
2137
|
rankLevel: (level) => `Measured at rule level \`${level}\`.`,
|
|
2131
2138
|
columnFile: () => 'Prompt',
|
|
2132
2139
|
columnTokens: () => 'Tokens',
|
|
@@ -2138,7 +2145,7 @@ ${bold('EXAMPLES')}
|
|
|
2138
2145
|
? 'The prompt is within budget.'
|
|
2139
2146
|
: `All ${budgeted} budgeted prompts are within budget.`,
|
|
2140
2147
|
overBudget: (failures, budgeted) => `${failures} of ${budgeted} over budget`,
|
|
2141
|
-
noBudget: () => '
|
|
2148
|
+
noBudget: () => 'none',
|
|
2142
2149
|
unbudgetedNote: (count) =>
|
|
2143
2150
|
count === 1
|
|
2144
2151
|
? '1 prompt is not covered by any budget pattern, so nothing is watching it.'
|
|
@@ -2175,7 +2182,7 @@ ${bold('EXAMPLES')}
|
|
|
2175
2182
|
check: {
|
|
2176
2183
|
okLabel: () => 'OK',
|
|
2177
2184
|
embeddedHeading: (path, count) =>
|
|
2178
|
-
`${path}
|
|
2185
|
+
`${path}, ${count} marked ${count === 1 ? 'prompt' : 'prompts'}`,
|
|
2179
2186
|
declinedHeading: (count) =>
|
|
2180
2187
|
`${count} ${count === 1 ? 'marker' : 'markers'} could not be read:`,
|
|
2181
2188
|
declinedAt: (line, detail) => `line ${line}: ${detail}`,
|
|
@@ -2187,7 +2194,7 @@ ${bold('EXAMPLES')}
|
|
|
2187
2194
|
stillTooBig: (optimizedTokens) =>
|
|
2188
2195
|
` Even optimised it does not fit (~${optimizedTokens} tokens): content has to be cut by hand.`,
|
|
2189
2196
|
directoryHeading: (directory, files) =>
|
|
2190
|
-
`${directory}
|
|
2197
|
+
`${directory}, ${files} ${files === 1 ? 'prompt' : 'prompts'}`,
|
|
2191
2198
|
directorySummary: (failures, files) =>
|
|
2192
2199
|
failures === 0
|
|
2193
2200
|
? `All ${files} within budget.`
|
|
@@ -2196,7 +2203,7 @@ ${bold('EXAMPLES')}
|
|
|
2196
2203
|
walkTruncated: () =>
|
|
2197
2204
|
'Stopped early: the directory is larger than the walk limit, so this is not the whole picture.',
|
|
2198
2205
|
filesFromSummary: (checked, listed, dropped) =>
|
|
2199
|
-
`Checking ${checked} of ${listed} listed file(s)
|
|
2206
|
+
`Checking ${checked} of ${listed} listed file(s), ${dropped} dropped: not a prompt extension, ignored by config, or no longer on disk.`,
|
|
2200
2207
|
filesFromNoBaseline: () =>
|
|
2201
2208
|
'The baseline gate is skipped under --files-from: it compares the whole repository against the committed record, and a partial list would read as removals. Budgets are checked per file as always; run "trazum check ." for the baseline.',
|
|
2202
2209
|
exactCountsCost: (files) =>
|
|
@@ -2204,7 +2211,7 @@ ${bold('EXAMPLES')}
|
|
|
2204
2211
|
},
|
|
2205
2212
|
profile: {
|
|
2206
2213
|
noTarget: () =>
|
|
2207
|
-
'Point this at a usage log: trazum profile usage.jsonl
|
|
2214
|
+
'Point this at a usage log: trazum profile usage.jsonl, one JSON object per line, each with a "model" and the "usage" object the API returned. Add "label" (which workload), "session" (which conversation) and "ts" (when) while you are there: without them every call looks alike, and the largest findings this command makes, conversation growth, and whether the cache TTL fits how fast your turns come, cannot be made at all. Recording it is four lines in your own code, it never contains prompt text, and the session key is grouped by and never printed.',
|
|
2208
2215
|
heading: () => 'Where the money went',
|
|
2209
2216
|
calls: (n) => plural(n, 'call'),
|
|
2210
2217
|
spent: (calls, total) => `${calls} · ${total}`,
|
|
@@ -2217,12 +2224,12 @@ ${bold('EXAMPLES')}
|
|
|
2217
2224
|
byModelHeading: () => 'By model',
|
|
2218
2225
|
row: (name, usd, pct, calls) => `${usd.padStart(11)} ${pct.padStart(5)} ${name} (${calls})`,
|
|
2219
2226
|
// Parenthesised so a real label named "unlabelled" cannot read identically
|
|
2220
|
-
// beside it
|
|
2227
|
+
// beside it, the data already keeps them apart; the display should too.
|
|
2221
2228
|
unlabelled: () => '(no label)',
|
|
2222
2229
|
cacheHit: (pct) => `Cache hit rate ${pct} of billable input.`,
|
|
2223
2230
|
cacheNever: () => 'Caching was never used on these calls. If any prefix repeats, that is the largest saving available.',
|
|
2224
2231
|
cacheLost: (usd, writes, reads) =>
|
|
2225
|
-
`Caching added ${usd} to this bill instead of taking it off. ${writes} tokens were written to the cache and ${reads} read back
|
|
2232
|
+
`Caching added ${usd} to this bill instead of taking it off. ${writes} tokens were written to the cache and ${reads} read back, and a write costs 1.25x plain input, or 2x at the 1-hour TTL. A prefix that changes faster than it is reused pays that premium for nothing. Either cache a prefix that holds still, or turn caching off here.`,
|
|
2226
2233
|
cachePaidOff: (usd) => `Caching took ${usd} off this bill, against the same tokens uncached.`,
|
|
2227
2234
|
cacheNoDifference: () =>
|
|
2228
2235
|
'Caching came out level on this bill: what it charged for these tokens is what they would have cost as ordinary input. It is neither paying for itself nor costing you anything.',
|
|
@@ -2240,7 +2247,7 @@ ${bold('EXAMPLES')}
|
|
|
2240
2247
|
outputDominates: (pct) =>
|
|
2241
2248
|
`Output is ${pct} of this bill, so shortening prompts has a low ceiling here. What moves it is asking for shorter answers and capping max_tokens.`,
|
|
2242
2249
|
unpriced: (models, calls) =>
|
|
2243
|
-
`${count(calls)} ${calls === 1 ? 'call is' : 'calls are'} not in these totals
|
|
2250
|
+
`${count(calls)} ${calls === 1 ? 'call is' : 'calls are'} not in these totals, because the pricing catalogue does not know: ${models}. Add them with a pricing overlay (--pricing) to include them.`,
|
|
2244
2251
|
skipped: (lineCount, lines) =>
|
|
2245
2252
|
`${count(lineCount)} ${lineCount === 1 ? 'line' : 'lines'} could not be read and ${lineCount === 1 ? 'was' : 'were'} left out (${lineCount === 1 ? 'line' : 'lines'} ${lines}).`,
|
|
2246
2253
|
empty: () => 'No usage records in that file.',
|
|
@@ -2248,78 +2255,78 @@ ${bold('EXAMPLES')}
|
|
|
2248
2255
|
'None of the models in that log are in the pricing catalogue, so there is no bill to report. Add them with a pricing overlay (--pricing) and run this again.',
|
|
2249
2256
|
leversHeading: () => 'What would actually move this bill',
|
|
2250
2257
|
leverSlice: (label, model, usd, pct) =>
|
|
2251
|
-
`${label} on ${model}
|
|
2258
|
+
`${label} on ${model}, up to ${usd} of this bill (${pct})`,
|
|
2252
2259
|
leverRoute: (candidate, usd) => `route it to ${candidate}, ${usd}`,
|
|
2253
2260
|
leverRouteVerify: (candidate) =>
|
|
2254
2261
|
`Whether that holds is an evaluation question, not an arithmetic one, and nothing here has seen a single answer. Measure it: trazum route <log> --prompt-file <prompt> --cases <cases> --yes`,
|
|
2255
2262
|
leverBatch: (usd) => `send it through the Batch API, ${usd}`,
|
|
2256
2263
|
leverCalls: (calls, spent) => `${calls}, ${spent} spent`,
|
|
2257
2264
|
leverPromptCeiling: (usd, pct) =>
|
|
2258
|
-
`For comparison: shortening the prompt text can touch ${usd} at the very most
|
|
2265
|
+
`For comparison: shortening the prompt text can touch ${usd} at the very most, ${pct} of this bill, and only if you deleted every input token. The real figure is far below that, because most of those tokens are retrieved context, conversation history and tool results that no prompt file contains.`,
|
|
2259
2266
|
historyHeading: () => 'What re-sending the conversation costs',
|
|
2260
2267
|
historyGrowth: (label, model, first, last, turns) =>
|
|
2261
2268
|
`${label} on ${model}: input ranges from ${first} tokens on the smallest turn to ${last} on the largest, over conversations of up to ${turns} turns.`,
|
|
2262
2269
|
historyCeiling: (usd, pct, flat, spent) =>
|
|
2263
|
-
`If every turn had been the size of its smallest one, that input would have cost ${flat} instead of ${spent}
|
|
2270
|
+
`If every turn had been the size of its smallest one, that input would have cost ${flat} instead of ${spent}, so at most ${usd} of this bill is conversation growth (${pct}). It is a ceiling and not a saving: some of that is the user's own new messages, which nothing can truncate away, and this reads counts rather than content so it cannot tell the two apart. What moves it is capping the history you replay, or summarising it.`,
|
|
2264
2271
|
truncatedWaste: (calls, usd, pct) =>
|
|
2265
|
-
`${calls} hit the max_tokens ceiling: ${usd} of the output spend (${pct}) bought answers that were cut off mid-generation
|
|
2272
|
+
`${calls} hit the max_tokens ceiling: ${usd} of the output spend (${pct}) bought answers that were cut off mid-generation, paid in full, frequently retried and billed again. Where the answer genuinely needs the room, raise max_tokens; where it does not, ask for less. Either way this is the one slice of a bill that is waste without a counterpart.`,
|
|
2266
2273
|
againstHeading: () => 'Against the previous log',
|
|
2267
2274
|
againstTotals: (before, after, delta, pct, callsBefore, callsAfter) =>
|
|
2268
|
-
`${before} → ${after} ${delta} (${pct}) ${callsBefore} → ${callsAfter}. Positive means the bill grew. Both figures are exactly what each file holds
|
|
2275
|
+
`${before} → ${after} ${delta} (${pct}) ${callsBefore} → ${callsAfter}. Positive means the bill grew. Both figures are exactly what each file holds, no period is assumed, so judge the call counts before judging the money.`,
|
|
2269
2276
|
againstDriver: (delta, label, before, after) => `${delta} ${label} (${before} → ${after})`,
|
|
2270
2277
|
againstDriverNew: (delta, label) => `${delta} ${label} (new since the previous log)`,
|
|
2271
2278
|
againstDriverGone: (delta, label) => `${delta} ${label} (gone since the previous log)`,
|
|
2272
2279
|
againstByModel: () =>
|
|
2273
|
-
'The same change, by model
|
|
2280
|
+
'The same change, by model, where the mix moved:',
|
|
2274
2281
|
labelPrefixBelowMinimum: (file, prefix, minimum, model) =>
|
|
2275
|
-
`${file} (as it is today
|
|
2282
|
+
`${file} (as it is today, and the log may predate it): the stable prefix is ${prefix} tokens and ${model} caches nothing under ${minimum}. Setting cache_control there does not error, it simply never caches, which is what a cache that only writes looks like from the bill.`,
|
|
2276
2283
|
labelPrefixMovable: (file, movable, prefix) =>
|
|
2277
|
-
`${file} (as it is today
|
|
2284
|
+
`${file} (as it is today, and the log may predate it): ~${movable} stable tokens sit after the first placeholder, where caching cannot reach them; the cacheable prefix is ${prefix}. "trazum optimize ${file} --reorder" moves them in front and shows the diff.`,
|
|
2278
2285
|
labelPrefixHealthy: (file, prefix, minimum) =>
|
|
2279
|
-
`${file} (as it is today
|
|
2286
|
+
`${file} (as it is today, and the log may predate it): the stable prefix is ${prefix} tokens, over the ${minimum} minimum. The prompt file is not the problem; look at whether the prefix is byte-identical between calls.`,
|
|
2280
2287
|
labelFileMissing: (label, file) =>
|
|
2281
|
-
`labels["${label}"] points at ${file}, which does not exist
|
|
2288
|
+
`labels["${label}"] points at ${file}, which does not exist: the mapping was skipped.`,
|
|
2282
2289
|
againstNothingPriced: () =>
|
|
2283
2290
|
'The previous log has nothing the pricing catalogue knows, so there is no comparison to make.',
|
|
2284
2291
|
truncatedNotRecorded: () =>
|
|
2285
|
-
'Whether any answers were cut off could not be measured
|
|
2292
|
+
'Whether any answers were cut off could not be measured, no call in this log carries a stop reason. Add "stop_reason" (Anthropic) or "finish_reason" (OpenAI) to the record; the API already returns it beside "usage".',
|
|
2286
2293
|
historyNoSessions: () =>
|
|
2287
|
-
'No call in this log carried a session, so what re-sending the conversation costs could not be measured
|
|
2294
|
+
'No call in this log carried a session, so what re-sending the conversation costs could not be measured, usually the largest line on a chat or agent bill. Add "session" (or "conversation_id") to the record and run this again. Trazum groups by it and never prints it.',
|
|
2288
2295
|
leversUnlabelled: () =>
|
|
2289
|
-
'None of these calls carried a label, so this is every workload in one row
|
|
2296
|
+
'None of these calls carried a label, so this is every workload in one row, a classifier and a RAG pipeline merged into a single figure, with one route suggested for both. Add "label" to the record and the levers split by workload, which is the grouping a decision is actually made at.',
|
|
2290
2297
|
outputShapeHeading: () => 'Where the output spend concentrates',
|
|
2291
2298
|
outputTail: (label, model, callPct, spendPct, above, usd) =>
|
|
2292
|
-
`${label} on ${model}: ${callPct} of calls hold ${spendPct} of the output spend
|
|
2299
|
+
`${label} on ${model}: ${callPct} of calls hold ${spendPct} of the output spend, the ones answering with more than ${above} tokens, out of ${usd} of output on this slice.`,
|
|
2293
2300
|
outputTailAdvice: () =>
|
|
2294
2301
|
'That is a tail, and a tail has a cause: a path through the prompt that invites an essay, a call with no max_tokens, a retrieval that returned a book. Finding it is a morning; it is not "make everything shorter".',
|
|
2295
2302
|
outputFlat: (label, model, callPct, spendPct, usd) =>
|
|
2296
|
-
`${label} on ${model}: the output spend sits where the calls are
|
|
2303
|
+
`${label} on ${model}: the output spend sits where the calls are, ${callPct} of them hold ${spendPct} of ${usd}. There is no tail to hunt.`,
|
|
2297
2304
|
outputFlatAdvice: () =>
|
|
2298
2305
|
'The answer length is the task here, so the levers are the blunt ones: ask for shorter answers in the prompt, and cap max_tokens.',
|
|
2299
2306
|
outputPercentiles: (p50, p95) =>
|
|
2300
|
-
`Half the measured answers fit within ${p50} output tokens, and 95% within ${p95}
|
|
2307
|
+
`Half the measured answers fit within ${p50} output tokens, and 95% within ${p95}, the number a max_tokens cap actually wants. Measured on these calls, promised for nothing.`,
|
|
2301
2308
|
inputShapeHeading: () => 'How big these calls are',
|
|
2302
2309
|
inputSkewed: (label, model, p50, p95, ratio, usd) =>
|
|
2303
|
-
`${label} on ${model} is uneven: half its calls fit within ${p50} input tokens and 95% within ${p95}
|
|
2310
|
+
`${label} on ${model} is uneven: half its calls fit within ${p50} input tokens and 95% within ${p95}, about ${ratio}x the ordinary call, over ${usd} of input spend.`,
|
|
2304
2311
|
inputSkewedAdvice: () =>
|
|
2305
2312
|
'Past four times the median, the ordinary call is fine and something is growing on top of it: a conversation nobody truncates, a retrieval with no cap, a tool result pasted in whole. The fix is a limit on the large calls, not a rewrite of the prompt every call sends.',
|
|
2306
2313
|
inputEven: (label, model, p50, p95, usd) =>
|
|
2307
2314
|
`${label} on ${model} is even: half its calls fit within ${p50} input tokens and 95% within ${p95}, over ${usd} of input spend.`,
|
|
2308
2315
|
inputEvenAdvice: () =>
|
|
2309
|
-
'The large calls are not much larger than the ordinary one, so there is no tail to cap
|
|
2316
|
+
'The large calls are not much larger than the ordinary one, so there is no tail to cap: the prompt is simply big. The levers are fewer retrieved documents, a shorter system block, and caching if the prefix repeats.',
|
|
2310
2317
|
inputHuge: (label, model, calls, usd) =>
|
|
2311
|
-
`${label} on ${model}: every one of its ${calls} is larger than this tool measures precisely, over ${usd} of input spend. No ceiling is named because there is none to name honestly
|
|
2318
|
+
`${label} on ${model}: every one of its ${calls} is larger than this tool measures precisely, over ${usd} of input spend. No ceiling is named because there is none to name honestly: that size is itself the finding.`,
|
|
2312
2319
|
repeatsHeading: () => 'The same request, sent again',
|
|
2313
2320
|
repeatsFound: (label, model, repeats, checked, seconds, usd) =>
|
|
2314
2321
|
`${label} on ${model}: ${repeats} of ${checked} calls re-sent the previous call's exact input size within ${seconds} seconds, in the same conversation, costing ${usd}.`,
|
|
2315
2322
|
pressureHeading: () => 'Approaching the context window',
|
|
2316
2323
|
pressureLine: (label, model, tokens, window, share) =>
|
|
2317
|
-
`${label} on ${model}: the largest call carried ${tokens} input tokens against a ${window}-token window
|
|
2324
|
+
`${label} on ${model}: the largest call carried ${tokens} input tokens against a ${window}-token window, ${share} of the ceiling.`,
|
|
2318
2325
|
mixDriftHeading: () => 'The mix moved inside this log',
|
|
2319
2326
|
mixDriftLine: (model, firstShare, lastShare, firstDays, lastDays, lastUsd) =>
|
|
2320
|
-
`${model} went from ${firstShare} of the spend in the first ${firstDays} days to ${lastShare} in the last ${lastDays}
|
|
2327
|
+
`${model} went from ${firstShare} of the spend in the first ${firstDays} days to ${lastShare} in the last ${lastDays}, ${lastUsd} of the recent half.`,
|
|
2321
2328
|
truncationRetryLine: (label, model, retried, truncated, seconds, wasted, retry) =>
|
|
2322
|
-
`${label} on ${model}: ${retried} of ${truncated} truncated answers were followed within ${seconds} seconds by another call in the same conversation
|
|
2329
|
+
`${label} on ${model}: ${retried} of ${truncated} truncated answers were followed within ${seconds} seconds by another call in the same conversation, ${wasted} spent on the cut attempts, plus ${retry} on the follow-ups.`,
|
|
2323
2330
|
truncationRetryNote: () =>
|
|
2324
2331
|
'The pair is the shape a retry has; the log cannot see content, so whether each was one is yours to know. Both sides of it are real money, and the fix is the same either way: a max_tokens the answers actually fit in.',
|
|
2325
2332
|
mixDriftNote: () =>
|
|
@@ -2327,9 +2334,9 @@ ${bold('EXAMPLES')}
|
|
|
2327
2334
|
pressureAdvice: () =>
|
|
2328
2335
|
'At 100% the call fails outright, and nothing on the bill changes until that day. The levers are a cap on retrieved context, truncating conversation history, or a larger-window model. When it crosses is not predicted here: the share is a fact, the trajectory is yours to know.',
|
|
2329
2336
|
repeatsAdvice: () =>
|
|
2330
|
-
'A conversation\'s input grows with every turn, so the same size twice in a row seconds apart is usually a retry after a timeout, an agent step repeating, or a loop
|
|
2337
|
+
'A conversation\'s input grows with every turn, so the same size twice in a row seconds apart is usually a retry after a timeout, an agent step repeating, or a loop: this reads counts and cannot see content, so it names the pattern and stops. Whatever it is, the money bought nothing the call before it had not already paid for.',
|
|
2331
2338
|
inputMostlyCached: (share) =>
|
|
2332
|
-
`${share} of those tokens were cache reads, billed at a tenth of the input rate
|
|
2339
|
+
`${share} of those tokens were cache reads, billed at a tenth of the input rate: the size is real and most of it is cheap.`,
|
|
2333
2340
|
inputFullRate: () =>
|
|
2334
2341
|
'Almost none of that was a cache read, so every one of those tokens was billed at the full input rate. If any prefix repeats across these calls, caching is the lever with the largest ceiling here.',
|
|
2335
2342
|
leversNone: () =>
|
|
@@ -2337,60 +2344,60 @@ ${bold('EXAMPLES')}
|
|
|
2337
2344
|
assumedWriteTtl: (calls) =>
|
|
2338
2345
|
`${count(calls)} ${calls === 1 ? 'call did' : 'calls did'} not say which cache-write TTL was used, so the cheaper 5-minute rate was assumed. A 1-hour entry costs 2x input rather than 1.25x, so this total is a floor for those calls. Record the "cache_creation" object the API returns to remove the assumption.`,
|
|
2339
2346
|
spanLine: (from, to, days) =>
|
|
2340
|
-
`This log covers ${from} → ${to} (${days} days). The span is stated, never extrapolated
|
|
2347
|
+
`This log covers ${from} → ${to} (${days} days). The span is stated, never extrapolated: the monthly arithmetic is yours to do, and now it is valid.`,
|
|
2341
2348
|
spanPartial: (withTs, total) =>
|
|
2342
2349
|
`Only ${withTs} of ${total} calls carry a timestamp; the span describes those.`,
|
|
2343
2350
|
ttlFitExpires: (label, model, gap) =>
|
|
2344
|
-
`${label} on ${model}: turns arrive a median of ${gap} apart and the 5-minute entry is gone by then
|
|
2351
|
+
`${label} on ${model}: turns arrive a median of ${gap} apart and the 5-minute entry is gone by then, writes expire before the next turn reads them, which from the bill is a cache that only writes. The 1-hour TTL costs 2x input to write and would survive these gaps; the other honest option is caching switched off here.`,
|
|
2345
2352
|
ttlFitExpiresBoth: (label, model, gap) =>
|
|
2346
|
-
`${label} on ${model}: turns arrive a median of ${gap} apart, and no cache entry lives that long
|
|
2353
|
+
`${label} on ${model}: turns arrive a median of ${gap} apart, and no cache entry lives that long, even the 1-hour TTL is gone by the next turn. Caching cannot work at this pace; turn it off here and stop paying the write premium.`,
|
|
2347
2354
|
ttlFitOverlong: (label, model, gap, usd) =>
|
|
2348
|
-
`${label} on ${model}: turns arrive a median of ${gap} apart
|
|
2355
|
+
`${label} on ${model}: turns arrive a median of ${gap} apart, comfortably inside the 5-minute window, and these writes pay the 1-hour rate, 2x input against 1.25x, for endurance the gaps never use. The same writes at the 5-minute TTL are ${usd} cheaper on this log, and that figure is exact: the same tokens at the other published rate.`,
|
|
2349
2356
|
ttlFitUnsettledGap: (label, model, gap) =>
|
|
2350
|
-
`${label} on ${model}: turns arrive a median of ${gap} apart
|
|
2357
|
+
`${label} on ${model}: turns arrive a median of ${gap} apart, which a 5-minute entry does not survive and a 1-hour one does, and the log did not record which these writes were, so whether they ever get read back cannot be settled. Record the "cache_creation" object the API returns and it settles itself.`,
|
|
2351
2358
|
ttlFitFits: (label, model, gap) =>
|
|
2352
2359
|
`${label} on ${model}: turns arrive a median of ${gap} apart, inside the lifetime these writes use. The TTL is not the problem here.`,
|
|
2353
2360
|
ttlFitUnmeasured: () =>
|
|
2354
|
-
'Whether the cache TTL fits how fast the turns arrive could not be measured
|
|
2361
|
+
'Whether the cache TTL fits how fast the turns arrive could not be measured, it needs both "session" and "ts" on the record. A 5-minute entry on a workload whose turns come nine minutes apart expires unread on every write, and only the clock can see it. Trazum groups by the session and never shows it.',
|
|
2355
2362
|
dayPeak: (day, usd, xMedian) =>
|
|
2356
2363
|
`The most expensive day in this log was ${day}: ${usd}, ${xMedian}x the median day.`,
|
|
2357
2364
|
dayPeakLabel: (label, usd) => `Most of it was ${label} (${usd}).`,
|
|
2358
2365
|
maxUsdOk: (total, max) => `Within budget: ${total} spent against --max-usd ${max}.`,
|
|
2359
2366
|
maxUsdFailed: (total, max) =>
|
|
2360
|
-
`FAILED
|
|
2367
|
+
`FAILED: this log spent ${total} against a --max-usd of ${max}. The figures are the provider's own billed counts over exactly this log; no period was assumed.`,
|
|
2361
2368
|
maxGrowthUsdFailed: (delta, max) =>
|
|
2362
|
-
`FAILED
|
|
2369
|
+
`FAILED, the bill grew ${delta} against the previous log, over the --max-growth-usd limit of ${max}.`,
|
|
2363
2370
|
maxGrowthNeedsAgainst: () =>
|
|
2364
2371
|
'--max-growth-usd has nothing to compare without --against <previous.jsonl>. On its own it would have run silently and gated nothing, which is not an answer.',
|
|
2365
2372
|
maxCacheLossOk: (worst, max) =>
|
|
2366
2373
|
`Cache within budget: caching cost at most ${worst} against --max-cache-loss-usd ${max}, worst case included.`,
|
|
2367
2374
|
maxCacheLossFailed: (delta, max) =>
|
|
2368
|
-
`FAILED
|
|
2375
|
+
`FAILED, caching added ${delta} to this bill (the same tokens as plain input would have cost less), over the --max-cache-loss-usd limit of ${max}. The counterfactual is exact: the same tokens at the published input rate.`,
|
|
2369
2376
|
maxDayOk: (day, usd, max) =>
|
|
2370
2377
|
`No single day over budget: the worst was ${day} at ${usd}, against --max-day-usd ${max}.`,
|
|
2371
2378
|
maxDayFailed: (day, usd, max) =>
|
|
2372
|
-
`FAILED
|
|
2379
|
+
`FAILED, ${day} spent ${usd}, over the --max-day-usd limit of ${max}. A total under budget can hide a single runaway day, which is what this gate exists to catch.`,
|
|
2373
2380
|
maxDayNoClock: () =>
|
|
2374
|
-
'FAILED
|
|
2381
|
+
'FAILED, --max-day-usd was asked for and no record in this log carries a timestamp, so there are no days to judge. That is not a pass: a bill nobody could measure by day is not a bill that stayed under a daily budget. Add "ts" to the record and the gate arms.',
|
|
2375
2382
|
maxSessionOk: (worst, max, sessions) =>
|
|
2376
2383
|
`No conversation over budget: the most expensive of ${sessions} cost ${worst}, against --max-session-usd ${max}. A conversation that started before this log is counted only for the turns recorded here, so this is a floor.`,
|
|
2377
2384
|
maxSessionFailed: (worst, max, sessions) =>
|
|
2378
|
-
`FAILED
|
|
2385
|
+
`FAILED: the most expensive of ${sessions} conversations cost ${worst}, over the --max-session-usd limit of ${max}. A month's budget and a day's budget both pass while one conversation loops its way through this; the per-conversation figure is the one that catches it.`,
|
|
2379
2386
|
maxSessionNoSessions: () =>
|
|
2380
|
-
'FAILED
|
|
2387
|
+
'FAILED, --max-session-usd was asked for and no record in this log carries a session, so there are no conversations to judge. That is not a pass. Add "session" (or "conversation_id") to the record and the gate arms; Trazum groups by it and never prints it.',
|
|
2381
2388
|
maxDayUndated: (calls) =>
|
|
2382
|
-
`${calls} calls carry no timestamp, so they are in the bill and in none of the days above
|
|
2389
|
+
`${calls} calls carry no timestamp, so they are in the bill and in none of the days above: the worst day is a floor by whatever they held. A failure would stand regardless; this pass is over the part that could be dated.`,
|
|
2383
2390
|
maxCacheLossWorstCase: (calls, worst, max) =>
|
|
2384
|
-
`FAILED
|
|
2391
|
+
`FAILED, ${count(calls)} ${calls === 1 ? 'call' : 'calls'} did not record which cache-write TTL was paid, and at the 1-hour rate caching added up to ${worst}, over the --max-cache-loss-usd limit of ${max}. The gate reads the worst case on purpose: a gate reading the flattering half would pass exactly the bills it exists to catch. Record the "cache_creation" object the API returns and the ceiling becomes a figure.`,
|
|
2385
2392
|
pricesStale: (date, days) =>
|
|
2386
|
-
`The price table behind every dollar here was last reviewed ${date}
|
|
2393
|
+
`The price table behind every dollar here was last reviewed ${date}, ${count(days)} days ago, past the 45 this tool considers current. If the provider changed prices since, this report is wrong by exactly that change. --pricing-live fetches today's prices; --pricing overlays your own.`,
|
|
2387
2394
|
dayTableDay: () => 'Day (UTC)',
|
|
2388
2395
|
dayTableCalls: () => 'calls',
|
|
2389
2396
|
dayTableTop: () => 'biggest that day',
|
|
2390
2397
|
dayTableEarlier: (days) =>
|
|
2391
2398
|
`…and ${count(days)} earlier ${days === 1 ? 'day' : 'days'} not shown here. The full series rides --json as spendByDay.`,
|
|
2392
2399
|
gateOnFloor: (reasons) =>
|
|
2393
|
-
`Note: the gated figure is a floor, not the bill
|
|
2400
|
+
`Note: the gated figure is a floor, not the bill, ${reasons}. Whatever those calls cost is not in the number the gate just judged, so a pass here means "the part I could read fits", never "the bill fits".`,
|
|
2394
2401
|
floorSkipped: (lines) =>
|
|
2395
2402
|
`${count(lines)} ${lines === 1 ? 'line was' : 'lines were'} unreadable and left out`,
|
|
2396
2403
|
floorUnpriced: (calls) =>
|
|
@@ -2398,37 +2405,37 @@ ${bold('EXAMPLES')}
|
|
|
2398
2405
|
floorUndated: (calls) =>
|
|
2399
2406
|
`${count(calls)} ${calls === 1 ? 'call carries' : 'calls carry'} no timestamp and fell outside the window`,
|
|
2400
2407
|
sessionCost: (label, model, sessions, median, medianTurns, p95, max) =>
|
|
2401
|
-
`${label} on ${model}: across ${sessions} conversations, the median one costs ${median} over ${medianTurns} turns, 95% come in under ${p95}, and the most expensive was ${max}. Exact billed counts, per conversation
|
|
2408
|
+
`${label} on ${model}: across ${sessions} conversations, the median one costs ${median} over ${medianTurns} turns, 95% come in under ${p95}, and the most expensive was ${max}. Exact billed counts, per conversation: the figure a per-seat price or a quota is set from. A conversation that started before this log or continues after it is counted only for the turns recorded here.`,
|
|
2402
2409
|
labelBudgetOk: (label, usd, max) => `Within budget: ${label} spent ${usd} against ${max}.`,
|
|
2403
2410
|
labelBudgetFailed: (label, usd, max) =>
|
|
2404
|
-
`FAILED
|
|
2411
|
+
`FAILED, ${label} spent ${usd} against its budget of ${max} in trazum.config.json.`,
|
|
2405
2412
|
labelBudgetMissing: (label) =>
|
|
2406
2413
|
`${label} has a budget in trazum.config.json and no calls in this log, so nothing was measured for it. Not a pass: a workload that did not appear is not a workload that came in under budget.`,
|
|
2407
2414
|
labelBudgetWindowed: () =>
|
|
2408
2415
|
'Per-label budgets in trazum.config.json were not applied: --since/--until make "what this label spent" mean a slice, and a budget written for the whole period would be gating against something it does not describe.',
|
|
2409
2416
|
duplicateLines: (calls, usd) =>
|
|
2410
|
-
`${plural(calls, 'line is an exact duplicate', `lines are exact duplicates`)} of an earlier line
|
|
2417
|
+
`${plural(calls, 'line is an exact duplicate', `lines are exact duplicates`)} of an earlier line, same counts, same label and session, same millisecond, and that adds ${usd} to the total above. If a log was exported twice or two files in a directory overlap, this bill is overstated by that much. Two real calls colliding on all of that is possible; it is just unlikely.`,
|
|
2411
2418
|
budgetVsWire: (label, file, budget, perCall, share) =>
|
|
2412
|
-
`The budget on ${file} is ${budget} tokens, and calls labelled ${label} carry about ${perCall} input tokens each
|
|
2419
|
+
`The budget on ${file} is ${budget} tokens, and calls labelled ${label} carry about ${perCall} input tokens each, so that gate governs roughly ${share} of what actually goes up the wire. The rest is retrieved context, conversation history and tool results, which no prompt file contains and no budget on one can see. The budget is not wrong; it is just smaller than the bill.`,
|
|
2413
2420
|
badCsvShape: (value) =>
|
|
2414
|
-
`--csv-shape does not know "${value}". It takes "slice" (one row per label and model, the default), "day", "hour" or "model-day" (one row per day and model
|
|
2421
|
+
`--csv-shape does not know "${value}". It takes "slice" (one row per label and model, the default), "day", "hour" or "model-day" (one row per day and model, the long format a pivot table wants).`,
|
|
2415
2422
|
whatIfHeading: (model) => `These exact calls on ${model}`,
|
|
2416
2423
|
whatIfAssumption: () =>
|
|
2417
2424
|
'This is multiplication, not advice: the same token counts at another rate card. It says nothing about whether that model could do the work, and a model that answers at greater length or gets retried would not send these counts at all.',
|
|
2418
2425
|
whatIfTotal: (current, target, delta) =>
|
|
2419
|
-
`${current} of movable spend would have been ${target}
|
|
2426
|
+
`${current} of movable spend would have been ${target}, a difference of ${delta}.`,
|
|
2420
2427
|
whatIfCheaper: () =>
|
|
2421
2428
|
'Verify before you move anything: trazum route measures one prompt against both models on your own examples.',
|
|
2422
2429
|
whatIfDearer: () => 'That direction costs more. The arithmetic is here so the number is not a guess.',
|
|
2423
2430
|
whatIfBatchOnTarget: (batched, moved) =>
|
|
2424
|
-
`If those calls can also wait, the target's Batch API takes the moved bill from ${moved} to ${batched}
|
|
2431
|
+
`If those calls can also wait, the target's Batch API takes the moved bill from ${moved} to ${batched}, the discount applies to the target's rates, not the ones you left. Whether they can wait is not in the log; that half of the decision is yours.`,
|
|
2425
2432
|
whatIfCacheBeyond: (largest, min, noCache) =>
|
|
2426
|
-
`Its cache traffic could not exist there: the largest call is ${largest} tokens against the target's ${min}-token cache minimum, so no call in this slice could create an entry. Without the cache the same tokens cost ${noCache}
|
|
2433
|
+
`Its cache traffic could not exist there: the largest call is ${largest} tokens against the target's ${min}-token cache minimum, so no call in this slice could create an entry. Without the cache the same tokens cost ${noCache}: that is the figure the target would actually bill, and the row above flatters the move.`,
|
|
2427
2434
|
whatIfSlice: (label, model, current, target) => `${label} on ${model}: ${current} → ${target}`,
|
|
2428
2435
|
whatIfOverContext: (label, tokens, window, usd) =>
|
|
2429
2436
|
`${label} cannot move: its largest call carries ${tokens} input tokens and that model's window is ${window}. Those calls would fail, not cost less, so their ${usd} is excluded from the figures above.`,
|
|
2430
2437
|
whatIfAlreadyThere: (calls, usd) =>
|
|
2431
|
-
`Already on that model: ${calls} worth ${usd}, left out of the figures above
|
|
2438
|
+
`Already on that model: ${calls} worth ${usd}, left out of the figures above, because money that cannot move would make the difference look smaller than it is.`,
|
|
2432
2439
|
whatIfUnpriced: (calls, models) =>
|
|
2433
2440
|
`Excluded: ${calls} whose model has no price here (${models}). Their cost on the target is knowable; the difference is not, because there is no current figure to subtract from.`,
|
|
2434
2441
|
whatIfNothingToMove: () =>
|
|
@@ -2437,30 +2444,30 @@ ${bold('EXAMPLES')}
|
|
|
2437
2444
|
`--what-if does not know "${value}". Priced models: ${available}. Add it with --pricing if you have its rates.`,
|
|
2438
2445
|
badGzip: (file, detail) =>
|
|
2439
2446
|
`${file} is gzipped and would not decompress: ${detail}. Reading the rest and saying nothing would report a bill missing whatever that file held, so this stops instead. Check the file, or move it out of the directory.`,
|
|
2440
|
-
dryRunHeading: () => 'What this log can answer
|
|
2447
|
+
dryRunHeading: () => 'What this log can answer, no bill was produced',
|
|
2441
2448
|
dryRunParsed: (parsed, skipped) =>
|
|
2442
2449
|
`${parsed} records parse; ${skipped} lines could not be read. A dry run prices nothing: the point is what the gates you are about to wire would stand on.`,
|
|
2443
2450
|
dryRunUnpriced: (models) =>
|
|
2444
2451
|
`Models the price table does not know: ${models}. Their tokens parse; their dollars need a --pricing overlay.`,
|
|
2445
2452
|
dryRunTotals: () => 'The bill itself: totals, per-model split, cache economics, the levers.',
|
|
2446
2453
|
dryRunLabels: (share) =>
|
|
2447
|
-
`Per-workload findings and label budgets
|
|
2454
|
+
`Per-workload findings and label budgets, "label" on ${share} of records.`,
|
|
2448
2455
|
dryRunClock: (share) =>
|
|
2449
|
-
`The period, per-day and per-hour shape, --max-day-usd, --since/--until
|
|
2456
|
+
`The period, per-day and per-hour shape, --max-day-usd, --since/--until, a timestamp on ${share} of records.`,
|
|
2450
2457
|
dryRunSessions: (share) =>
|
|
2451
|
-
`Conversation growth, per-conversation cost, --max-session-usd
|
|
2458
|
+
`Conversation growth, per-conversation cost, --max-session-usd, a session on ${share} of records. Grouped by, never printed.`,
|
|
2452
2459
|
dryRunStopReason: (share) =>
|
|
2453
|
-
`Truncated answers and their retry bill
|
|
2460
|
+
`Truncated answers and their retry bill, a stop reason on ${share} of records.`,
|
|
2454
2461
|
dryRunCacheTtl: (ttl, writes) =>
|
|
2455
|
-
`Settled cache verdicts
|
|
2462
|
+
`Settled cache verdicts, the "cache_creation" split on ${ttl} of ${writes} cache-writing records.`,
|
|
2456
2463
|
dryRunNoCacheTraffic: () =>
|
|
2457
|
-
'Cache verdicts: nothing wrote to the cache in this log, so there is nothing to settle
|
|
2464
|
+
'Cache verdicts: nothing wrote to the cache in this log, so there is nothing to settle, not a missing field.',
|
|
2458
2465
|
dryRunFooter: () =>
|
|
2459
2466
|
'A ✗ is not a defect in the log; it is a finding this log cannot support yet. The README\'s recording recipe carries every field above.',
|
|
2460
2467
|
dryRunNoGates: () =>
|
|
2461
2468
|
'--dry-run produces no bill, so a gate beside it would exit green having judged nothing. Run the gates without --dry-run.',
|
|
2462
2469
|
bySourceNeedsConfig: () =>
|
|
2463
|
-
'--by-source reads the "sources" block in trazum.config.json
|
|
2470
|
+
'--by-source reads the "sources" block in trazum.config.json, a name per service, each with glob patterns over log paths, and this config has none. Name at least one source.',
|
|
2464
2471
|
bySourceNothingMatched: (sources) =>
|
|
2465
2472
|
'No log file matched any source pattern. The sources configured: ${sources}. Patterns match the paths as given on the command line.'.replace('${sources}', sources),
|
|
2466
2473
|
fleetHeading: (count, total, calls) => `The fleet: ${count} sources · ${total} · ${calls}`,
|
|
@@ -2470,38 +2477,38 @@ ${bold('EXAMPLES')}
|
|
|
2470
2477
|
fleetWorst: (name, usd, share) =>
|
|
2471
2478
|
`${name} is where the money is: ${usd}, ${share} of the fleet's total.`,
|
|
2472
2479
|
fleetMismatchedSpans: () =>
|
|
2473
|
-
'These sources cover different periods, so the shares above compare totals, not rates
|
|
2480
|
+
'These sources cover different periods, so the shares above compare totals, not rates: a 3-day log looks cheap next to a 30-day one for reasons that have nothing to do with cost. Each row states its own span.',
|
|
2474
2481
|
fleetSplitBrain: (label, detail) =>
|
|
2475
|
-
`The same workload runs on different models in different sources
|
|
2482
|
+
`The same workload runs on different models in different sources, ${label}: ${detail}. Same job, different rate cards; whether that is a decision or an accident is not in the logs.`,
|
|
2476
2483
|
fleetCacheUnderwater: (name, usd) =>
|
|
2477
|
-
`Caching pays for the fleet overall but loses ${usd} in ${name}
|
|
2484
|
+
`Caching pays for the fleet overall but loses ${usd} in ${name}: the aggregate verdict was hiding it.`,
|
|
2478
2485
|
fleetUnmatched: (file) =>
|
|
2479
|
-
`${file} matched no source pattern, so it is in no report above
|
|
2486
|
+
`${file} matched no source pattern, so it is in no report above, spend missing from every bill until a pattern covers it.`,
|
|
2480
2487
|
fleetFooter: () =>
|
|
2481
2488
|
'Summaries per source; the full report for one service is "trazum profile <its logs>". Same thresholds, same findings, one source at a time.',
|
|
2482
2489
|
fleetBudgetOk: (name, usd, max) => `Within budget: ${name} spent ${usd} against its ${max} in spend.bySource.`,
|
|
2483
2490
|
fleetBudgetFailed: (name, usd, max) =>
|
|
2484
|
-
`FAILED
|
|
2491
|
+
`FAILED, ${name} spent ${usd} against its budget of ${max} in spend.bySource. The fleet total can be fine while one service bleeds; this gate names which.`,
|
|
2485
2492
|
fleetBudgetMissing: (name) =>
|
|
2486
2493
|
`${name} has a budget in spend.bySource and no logs matched it in this run, so nothing was measured for it. Not a pass: a service that did not appear is not a service under budget.`,
|
|
2487
2494
|
coverageHeading: () => 'What this log cannot answer yet',
|
|
2488
2495
|
needsLabel: (seen) =>
|
|
2489
2496
|
`"label" on ${seen} records: without it every workload is one row, so no per-workload spend, no drill-down, and the levers describe a mixture rather than a decision.`,
|
|
2490
2497
|
needsOutcome: (seen) =>
|
|
2491
|
-
`an "outcome"
|
|
2498
|
+
`an "outcome", ${seen}. The one field that changes what every other figure here means: without it this tool can say a workload got 40% cheaper and cannot say whether it stopped working. Record your own word for what happened and declare the vocabulary under "outcomes".`,
|
|
2492
2499
|
dryRunOutcomes: (share) =>
|
|
2493
2500
|
`cost per outcome and a success rate (${share} of records carry an "outcome")`,
|
|
2494
2501
|
outcomeHeading: () => 'Outcomes',
|
|
2495
2502
|
outcomeRate: (rate, ofUsd) =>
|
|
2496
|
-
`${rate} of ${ofUsd} in declared outcomes succeeded
|
|
2503
|
+
`${rate} of ${ofUsd} in declared outcomes succeeded, by spend rather than by call, because the two diverge exactly when the expensive half is the half that fails.`,
|
|
2497
2504
|
outcomeNoRate: (why) =>
|
|
2498
2505
|
why === 'nothing-recorded'
|
|
2499
|
-
? 'No success rate: nothing in this log recorded an outcome. That is not a rate of zero
|
|
2506
|
+
? 'No success rate: nothing in this log recorded an outcome. That is not a rate of zero, a rate of zero is a real and terrible measurement, and this is nobody having told us.'
|
|
2500
2507
|
: 'No success rate: "outcomes.success" declares no values, so nothing here counts as one. A legitimate thing to declare, and it means the rate is not this tool\u2019s to compute.',
|
|
2501
2508
|
outcomeUnrecorded: (share, usd) =>
|
|
2502
2509
|
`${share} of the bill (${usd}) carried no outcome, and is in neither half of the rate above.`,
|
|
2503
2510
|
outcomeUndeclared: (values) =>
|
|
2504
|
-
`Not declared in "outcomes.values": ${values}. Named rather than counted as failures
|
|
2511
|
+
`Not declared in "outcomes.values": ${values}. Named rather than counted as failures, a typo in an exporter should look like a typo, not like a product regression.`,
|
|
2505
2512
|
perOutcomeHeading: () => 'What an outcome costs',
|
|
2506
2513
|
perOutcomeRow: (key, perCall, perOutcome, coverage) => `${key} ${perCall} ${perOutcome} ${coverage}`,
|
|
2507
2514
|
perOutcomeColumns: {
|
|
@@ -2521,29 +2528,29 @@ ${bold('EXAMPLES')}
|
|
|
2521
2528
|
? 'not recorded'
|
|
2522
2529
|
: 'no vocabulary',
|
|
2523
2530
|
perOutcomeNumerator: () =>
|
|
2524
|
-
'Per success divides the spend on calls that recorded an outcome, never the whole bill
|
|
2531
|
+
'Per success divides the spend on calls that recorded an outcome, never the whole bill, dividing everything would charge your uninstrumented traffic to your measured successes and report a figure too high by exactly the uncovered share, silently, in the direction that gets a working feature killed. "recorded" is what share of each workload\u2019s spend the figure covers.',
|
|
2525
2532
|
perOutcomeDisagreement: (key, callRank, outcomeRank) =>
|
|
2526
2533
|
`${key} is #${callRank} by cost per call and #${outcomeRank} by cost per success.`,
|
|
2527
2534
|
perOutcomeBothOrders: () =>
|
|
2528
2535
|
'Cheapest per call and cheapest per success are different orders, and both are printed rather than one being picked. A workload can move up one while moving down the other, and somebody optimising on the first number would be moving the wrong one.',
|
|
2529
2536
|
outcomeColumns: { outcome: 'outcome', calls: 'calls', spend: 'spend' },
|
|
2530
2537
|
verdictSuccess: () => 'success',
|
|
2531
|
-
verdictOther: () => '
|
|
2538
|
+
verdictOther: () => 'other',
|
|
2532
2539
|
verdictUndeclared: () => 'undeclared',
|
|
2533
2540
|
needsSession: (seen) =>
|
|
2534
2541
|
`"session" on ${seen} records: without it there is no conversation growth, no per-conversation cost, and no cache-TTL fit. It is grouped by and never printed.`,
|
|
2535
2542
|
needsTs: (seen) =>
|
|
2536
2543
|
`"ts" on ${seen} records: without it the log has no period, no per-day or per-hour shape, and the cache-TTL question cannot be asked at all.`,
|
|
2537
2544
|
needsStopReason: (seen) =>
|
|
2538
|
-
`"stop_reason" (Anthropic) or "finish_reason" (OpenAI) on ${seen} records: without it, answers cut off at max_tokens are invisible
|
|
2545
|
+
`"stop_reason" (Anthropic) or "finish_reason" (OpenAI) on ${seen} records: without it, answers cut off at max_tokens are invisible, and silence there is not the same as none.`,
|
|
2539
2546
|
needsCacheTtl: (seen) =>
|
|
2540
2547
|
`the "cache_creation" object on ${seen} of the records that wrote to the cache: without it the 5-minute rate is assumed, so those totals are a floor and some cache verdicts cannot be settled.`,
|
|
2541
2548
|
hoursConcentrated: (hours, list) =>
|
|
2542
|
-
`80% of this spend lands in ${hours} hours of the UTC day (${list})
|
|
2549
|
+
`80% of this spend lands in ${hours} hours of the UTC day (${list}), interactive traffic somebody is waiting on, where the Batch API's 24-hour turnaround does not fit. Hours are UTC; shift them yourself if your traffic sits in one region.`,
|
|
2543
2550
|
hoursFlat: (hours) =>
|
|
2544
|
-
`The spend is spread across the day: it takes ${hours} hours of the UTC day to cover 80% of it. That is the shape background work has, and background work is what the Batch API halves the price of
|
|
2551
|
+
`The spend is spread across the day: it takes ${hours} hours of the UTC day to cover 80% of it. That is the shape background work has, and background work is what the Batch API halves the price of. See the levers above for what it would be worth here. Whether these calls can wait is yours to say; the log only shows when they happened.`,
|
|
2545
2552
|
truncatedBy: (label, calls, measured, rate, usd) =>
|
|
2546
|
-
`${label}: ${calls} of ${measured} calls that recorded a stop reason were cut off (${rate}), ${usd} of output. The denominator is the calls that measured, not every call
|
|
2553
|
+
`${label}: ${calls} of ${measured} calls that recorded a stop reason were cut off (${rate}), ${usd} of output. The denominator is the calls that measured, not every call: a workload logging the field half the time is not a workload whose other half completed.`,
|
|
2547
2554
|
truncatedCeiling: (p95) =>
|
|
2548
2555
|
`95% of the answers that finished fit within ${p95} output tokens, so a cap around there would stop cutting them off. Measured on these calls, promised for nothing.`,
|
|
2549
2556
|
readFiles: (files, directory) =>
|
|
@@ -2553,28 +2560,28 @@ ${bold('EXAMPLES')}
|
|
|
2553
2560
|
sessionCostTail: (ratio) =>
|
|
2554
2561
|
`The 95th percentile is ${ratio}x the median there: most conversations are cheap and a few are not, which is a tail a quota can catch. Where median and p95 sit close together the workload is simply expensive and there is no tail to hunt.`,
|
|
2555
2562
|
sessionSpendOnly: (sessions, max) =>
|
|
2556
|
-
`${sessions} ${sessions === '1' ? 'conversation' : 'conversations'} in this log; the most expensive cost ${max}. Too few per workload for a percentile
|
|
2563
|
+
`${sessions} ${sessions === '1' ? 'conversation' : 'conversations'} in this log; the most expensive cost ${max}. Too few per workload for a percentile: a maximum is a fact at any count, and it is the figure --max-session-usd judges.`,
|
|
2557
2564
|
waiveActive: (gate, reason, until, daysLeft) =>
|
|
2558
|
-
`WAIVED
|
|
2565
|
+
`WAIVED: the ${gate} failure above is on the record and silenced until ${until} (${daysLeft} days left): "${reason}". The bill still counts it; only the exit code is quiet, and the day the waiver expires this gate fails again.`,
|
|
2559
2566
|
waiveExpired: (gate, until, reason) =>
|
|
2560
|
-
`The waiver on ${gate} expired on ${until} and no longer silences anything. It was written for: "${reason}". Renew it with a new date and a current reason, or fix what it was covering
|
|
2567
|
+
`The waiver on ${gate} expired on ${until} and no longer silences anything. It was written for: "${reason}". Renew it with a new date and a current reason, or fix what it was covering: an expired waiver left in place is a finding deleted with extra steps.`,
|
|
2561
2568
|
waiveNotRecorded: (path, because) =>
|
|
2562
2569
|
` (This waiver was not written to ${path}: ${because}. The gate's verdict above is unaffected.)`,
|
|
2563
2570
|
summaryNoComparison: () =>
|
|
2564
|
-
'No previous log was given, so nothing here says whether the bill moved
|
|
2571
|
+
'No previous log was given, so nothing here says whether the bill moved, a summary without a comparison states the bill, not its stability.',
|
|
2565
2572
|
summaryFooter: () =>
|
|
2566
|
-
'The short form: what changed and the largest lever, nothing else. Run trazum profile for the full report
|
|
2573
|
+
'The short form: what changed and the largest lever, nothing else. Run trazum profile for the full report: every figure here comes from it.',
|
|
2567
2574
|
gateLargest: (label, model, usd, share) =>
|
|
2568
2575
|
`Most of it is ${label} on ${model}: ${usd}, ${share} of the bill. That is where the money is, not necessarily where the fix is.`,
|
|
2569
2576
|
gateLever: (label, action, saving, overage, covers) =>
|
|
2570
|
-
`The largest lever the report priced would save ${saving} on ${label} by ${action}
|
|
2577
|
+
`The largest lever the report priced would save ${saving} on ${label} by ${action}, ${covers ? `enough to cover the ${overage} this is over by` : `short of the ${overage} this is over by, so it is part of the answer rather than all of it`}. Whether that is the right call for this workload is yours to judge; the figure is arithmetic, not advice.`,
|
|
2571
2578
|
gateLeverRoute: (model) => `moving it to ${model}`,
|
|
2572
2579
|
gateLeverBatch: () => 'sending it through the Batch API',
|
|
2573
2580
|
gateLeverBoth: (model) => `moving it to ${model} and sending it through the Batch API`,
|
|
2574
2581
|
gateMarginTight: (margin, room) =>
|
|
2575
|
-
`Passed with ${margin} of the budget left
|
|
2582
|
+
`Passed with ${margin} of the budget left, ${room}. Under a tenth is close enough that an ordinary week crosses it; a pass this tight is worth knowing about before it becomes a failure.`,
|
|
2576
2583
|
maxGrowthCoverageLost: (fields, was, now) =>
|
|
2577
|
-
`FAILED
|
|
2584
|
+
`FAILED: this log stopped recording ${fields} (${was} of records before, ${now} now), so the comparison cannot be made. That is not a pass: a bill whose growth nobody could measure is not a bill that stayed flat, and every finding that needed the field went quiet for a reason that has nothing to do with spend.`,
|
|
2578
2585
|
coverageField: (field) =>
|
|
2579
2586
|
({ label: 'label', session: 'session', ts: 'timestamp', stopReason: 'stop reason' })[field] ?? field,
|
|
2580
2587
|
coverageSilenced: (field) =>
|
|
@@ -2587,39 +2594,39 @@ ${bold('EXAMPLES')}
|
|
|
2587
2594
|
coverageDrift: (field, was, now) =>
|
|
2588
2595
|
`Coverage moved: ${field} was on ${was} of records and is now on ${now}.`,
|
|
2589
2596
|
coverageDriftWhy: () =>
|
|
2590
|
-
'A field the log stopped recording is not a finding that got fixed
|
|
2597
|
+
'A field the log stopped recording is not a finding that got fixed: every finding that needed it has gone quiet for a reason that has nothing to do with the bill. Reported from a 20-point move in either direction; a field that appeared means this report can see what the previous one could not.',
|
|
2591
2598
|
againstOverlap: (from, to) =>
|
|
2592
|
-
`These two logs both cover ${from} → ${to}, so some of the same calls sit on both sides of this subtraction and part of the change is the same money counted twice. Compare periods that do not overlap
|
|
2599
|
+
`These two logs both cover ${from} → ${to}, so some of the same calls sit on both sides of this subtraction and part of the change is the same money counted twice. Compare periods that do not overlap, or window both logs with --since/--until.`,
|
|
2593
2600
|
windowLine: (since, until) =>
|
|
2594
2601
|
`Filtered to --since ${since} --until ${until}. Everything below describes this window, not the whole log; a bare date means the whole of that UTC day.`,
|
|
2595
2602
|
windowUndated: (calls) =>
|
|
2596
|
-
`${count(calls)} ${calls === 1 ? 'call carries' : 'calls carry'} no timestamp and cannot be placed inside or outside this window, so ${calls === 1 ? 'it was' : 'they were'} left out. Their spend is in the log and not in this report
|
|
2603
|
+
`${count(calls)} ${calls === 1 ? 'call carries' : 'calls carry'} no timestamp and cannot be placed inside or outside this window, so ${calls === 1 ? 'it was' : 'they were'} left out. Their spend is in the log and not in this report: the window's figures are a floor on the period.`,
|
|
2597
2604
|
windowRelative: () =>
|
|
2598
|
-
'That window is relative to this machine\'s clock, not to the log\'s last record
|
|
2605
|
+
'That window is relative to this machine\'s clock, not to the log\'s last record: a log exported a month ago will answer "the last 7 days" with nothing.',
|
|
2599
2606
|
windowRelativeEmpty: () =>
|
|
2600
2607
|
'A relative window is measured from this machine\'s clock: if this log was exported earlier, ask for the dates it covers instead.',
|
|
2601
2608
|
windowNeedsClock: () =>
|
|
2602
|
-
'No record in this log carries a timestamp, so --since/--until have nothing to filter by. A time window over a clockless log would gate nothing, which is not an answer. Add "ts" to the records
|
|
2609
|
+
'No record in this log carries a timestamp, so --since/--until have nothing to filter by. A time window over a clockless log would gate nothing, which is not an answer. Add "ts" to the records, the recipe in the README shows where.',
|
|
2603
2610
|
windowMatchesNothing: (from, to) =>
|
|
2604
|
-
`No record falls inside this window. The log covers ${from} → ${to}. A window matching nothing must not become a $0 report
|
|
2611
|
+
`No record falls inside this window. The log covers ${from} → ${to}. A window matching nothing must not become a $0 report, under --max-usd it would pass a budget gate over a period the log does not cover.`,
|
|
2605
2612
|
sinceAfterUntil: () =>
|
|
2606
2613
|
'--since is at or after --until, so the window contains no time at all. Check the two dates.',
|
|
2607
2614
|
badWhen: (flag, value) =>
|
|
2608
2615
|
`--${flag} could not read "${value}". It takes a UTC day (2026-08-14) or a full ISO 8601 timestamp (2026-08-14T09:30:00Z).`,
|
|
2609
2616
|
singleTurnCeiling: (label, model, single, sessions, usd) =>
|
|
2610
|
-
`${label} on ${model}: ${single} of ${sessions} conversations ended after their first turn, and their cache writes
|
|
2617
|
+
`${label} on ${model}: ${single} of ${sessions} conversations ended after their first turn, and their cache writes, ${usd}, paid for reuse their own conversation never made. Another conversation sharing the same prefix within the TTL could have read them; the log cannot see whose write a read hit, so that figure is a ceiling on the waste, not a bill.`,
|
|
2611
2618
|
singleTurnConfirmed: (label, model, single, sessions, usd) =>
|
|
2612
|
-
`${label} on ${model}: ${single} of ${sessions} conversations ended after their first turn and spent ${usd} writing a cache that nothing in this log ever read. Within the conversation, across conversations
|
|
2619
|
+
`${label} on ${model}: ${single} of ${sessions} conversations ended after their first turn and spent ${usd} writing a cache that nothing in this log ever read. Within the conversation, across conversations, no read anywhere, so those writes bought nothing. Caching a one-shot call is pure write premium; stop marking these calls with cache_control.`,
|
|
2613
2620
|
},
|
|
2614
2621
|
|
|
2615
2622
|
plan: {
|
|
2616
2623
|
noTarget: () =>
|
|
2617
|
-
'Point this at a usage log or a directory of them: trazum plan usage.jsonl. It turns the report into a ranked plan
|
|
2624
|
+
'Point this at a usage log or a directory of them: trazum plan usage.jsonl. It turns the report into a ranked plan, what to do first, what each action is worth, and what the log cannot confirm about it.',
|
|
2618
2625
|
nothingPriced: () =>
|
|
2619
|
-
'This log priced nothing
|
|
2626
|
+
'This log priced nothing, no call in it matched a model in the catalogue. A plan over zero dollars would be advice about nothing; check the log with "trazum profile" first.',
|
|
2620
2627
|
heading: (actions, total) => `The plan: ${actions} actions against a ${total} bill`,
|
|
2621
2628
|
totals: (projected, staked) =>
|
|
2622
|
-
`${projected} projected savings, on assumptions listed below. ${staked} already spent on problems this plan names
|
|
2629
|
+
`${projected} projected savings, on assumptions listed below. ${staked} already spent on problems this plan names, measured, not projected.`,
|
|
2623
2630
|
noClock: () =>
|
|
2624
2631
|
'No timestamps in this log, so every figure is per this log, not per any period.',
|
|
2625
2632
|
projected: (usd) => `${usd} projected`,
|
|
@@ -2637,24 +2644,24 @@ ${bold('EXAMPLES')}
|
|
|
2637
2644
|
: 'Look at the cache on';
|
|
2638
2645
|
return `${verb} ${label} (${model})`;
|
|
2639
2646
|
},
|
|
2640
|
-
routeTo: (model) => `to ${model}
|
|
2647
|
+
routeTo: (model) => `to ${model}, combined with batching where both apply, never summed`,
|
|
2641
2648
|
assume: (assumption) => {
|
|
2642
2649
|
switch (assumption.kind) {
|
|
2643
2650
|
case 'model-capability':
|
|
2644
|
-
return `assumes ${assumption.model} can do this work
|
|
2651
|
+
return `assumes ${assumption.model} can do this work, the log prices the move, it cannot judge the answers`;
|
|
2645
2652
|
case 'batch-window':
|
|
2646
2653
|
return 'assumes these calls can wait for a batch window';
|
|
2647
2654
|
case 'retry-pattern-real':
|
|
2648
|
-
return 'assumes the retry pattern is real
|
|
2655
|
+
return 'assumes the retry pattern is real, the log sees shapes, not content';
|
|
2649
2656
|
case 'max-tokens-fits':
|
|
2650
2657
|
return 'assumes a max_tokens the answers fit in removes the pair';
|
|
2651
2658
|
case 'traffic-pattern-holds':
|
|
2652
|
-
return 'assumes the traffic pattern holds
|
|
2659
|
+
return 'assumes the traffic pattern holds: a cache that lost money on this log may pay on different traffic';
|
|
2653
2660
|
}
|
|
2654
2661
|
},
|
|
2655
2662
|
check: (command) => `check it: ${command}`,
|
|
2656
2663
|
filtered: (count, minUsd, worth) =>
|
|
2657
|
-
`${count} actions under ${minUsd}, worth ${worth} together, left out by --min-usd
|
|
2664
|
+
`${count} actions under ${minUsd}, worth ${worth} together, left out by --min-usd, left out of this document entirely, not disproved.`,
|
|
2658
2665
|
footer: () =>
|
|
2659
2666
|
'Ranked by money, projected or already spent alike. The assumptions are yours to answer: this plan is arithmetic over the log, not knowledge of your product.',
|
|
2660
2667
|
wrote: (path) =>
|
|
@@ -2666,11 +2673,11 @@ ${bold('EXAMPLES')}
|
|
|
2666
2673
|
limitsNoLog: () =>
|
|
2667
2674
|
'A limits policy is configured and no --log was given, so every ceiling will answer cannot-tell: per-label and per-session spend live in a usage log, not in the store. Point --log at your usage log to make the ceilings judgeable.',
|
|
2668
2675
|
limitsUnpriced: (count) =>
|
|
2669
|
-
`${count} record(s) in the --log name a model the catalogue cannot price. They contribute nothing to any measured figure
|
|
2676
|
+
`${count} record(s) in the --log name a model the catalogue cannot price. They contribute nothing to any measured figure: money the limits policy cannot see, said here rather than hidden.`,
|
|
2670
2677
|
loopbackOnly: () =>
|
|
2671
|
-
'Loopback only, and there is no flag to change that: this holds your spend, your model mix and your budgets, and would answer whoever asked. There is no auth for the same reason
|
|
2678
|
+
'Loopback only, and there is no flag to change that: this holds your spend, your model mix and your budgets, and would answer whoever asked. There is no auth for the same reason: a token checked over loopback is theatre.',
|
|
2672
2679
|
measuredFrom: (usd) =>
|
|
2673
|
-
`Budget answers are measured against ${usd} from the store, read once at start. Every answer carries the window that figure covers rather than implying it is current to the second
|
|
2680
|
+
`Budget answers are measured against ${usd} from the store, read once at start. Every answer carries the window that figure covers rather than implying it is current to the second, restart to refresh it.`,
|
|
2674
2681
|
nothingMeasured: (dir) =>
|
|
2675
2682
|
`Nothing is measured yet (the store at ${dir} is empty), so the budget half of every answer will say so. The cost half still answers from the catalogue: offline is a mode, not a failure.`,
|
|
2676
2683
|
noBudget: () =>
|
|
@@ -2682,14 +2689,14 @@ ${bold('EXAMPLES')}
|
|
|
2682
2689
|
|
|
2683
2690
|
watch: {
|
|
2684
2691
|
noThresholds: () =>
|
|
2685
|
-
'Watching needs something to watch for. Set spend.maxUsd, spend.maxDayUsd or spend.maxCacheLossUsd in trazum.config.json
|
|
2692
|
+
'Watching needs something to watch for. Set spend.maxUsd, spend.maxDayUsd or spend.maxCacheLossUsd in trazum.config.json: a watcher with no threshold is a green light nobody earned.',
|
|
2686
2693
|
nothingToWatch: (dir) =>
|
|
2687
|
-
`Nothing has been measured yet: the store at ${dir} is empty. Fill it with "trazum connect <provider> --store" first
|
|
2694
|
+
`Nothing has been measured yet: the store at ${dir} is empty. Fill it with "trazum connect <provider> --store" first, watching nothing would report that everything is fine.`,
|
|
2688
2695
|
intervalTooTight: () =>
|
|
2689
2696
|
'--interval must be at least 5m. Usage APIs are rate limited, and a tight loop is a way to get your own key throttled by a tool that exists to save you money.',
|
|
2690
2697
|
badWebhook: (reason) =>
|
|
2691
2698
|
reason === 'credentials-in-url'
|
|
2692
|
-
? 'That webhook URL carries credentials. URLs end up in logs, shell history and error messages, so this one is refused
|
|
2699
|
+
? 'That webhook URL carries credentials. URLs end up in logs, shell history and error messages, so this one is refused, put the secret in a header your receiver checks, or in the receiver itself.'
|
|
2693
2700
|
: reason === 'insecure-scheme'
|
|
2694
2701
|
? 'A webhook must be https, except on loopback. An alert carries your spend figures, and sending them in the clear across a network is a leak you did not ask for.'
|
|
2695
2702
|
: 'That webhook is not a URL this tool can parse.',
|
|
@@ -2700,7 +2707,7 @@ ${bold('EXAMPLES')}
|
|
|
2700
2707
|
: gate === 'maxDayUsd'
|
|
2701
2708
|
? `Spend on ${day}`
|
|
2702
2709
|
: 'Money lost to caching';
|
|
2703
|
-
return `CROSSED
|
|
2710
|
+
return `CROSSED, ${what} is ${measured} against a limit of ${limit}. Measured, not projected.`;
|
|
2704
2711
|
},
|
|
2705
2712
|
stillOver: (gate, measured, limit, day) => {
|
|
2706
2713
|
const what =
|
|
@@ -2709,36 +2716,36 @@ ${bold('EXAMPLES')}
|
|
|
2709
2716
|
: gate === 'maxDayUsd'
|
|
2710
2717
|
? `Spend on ${day}`
|
|
2711
2718
|
: 'Money lost to caching';
|
|
2712
|
-
return `STILL OVER
|
|
2719
|
+
return `STILL OVER, ${what} is ${measured} against a limit of ${limit}, and was already reported. Quiet is not clean.`;
|
|
2713
2720
|
},
|
|
2714
2721
|
notJudgeable: (gate, reason, covered) =>
|
|
2715
2722
|
reason === 'window-too-short'
|
|
2716
|
-
? `${gate} cannot be judged yet: this period is ${covered ?? 'partly'} measured, and a threshold over part of a day is a threshold over something else. Not a pass
|
|
2723
|
+
? `${gate} cannot be judged yet: this period is ${covered ?? 'partly'} measured, and a threshold over part of a day is a threshold over something else. Not a pass: it will be judged when the day is in.`
|
|
2717
2724
|
: `${gate} cannot be judged on this source, which does not serve what the gate is written against. Not a pass: a gate silently skipped reads exactly like a gate that keeps passing.`,
|
|
2718
2725
|
gap: (from, to) =>
|
|
2719
2726
|
`Nothing was watching between ${from} and ${to}. Whatever crossed in that stretch was not seen, and this line exists so a resumed watcher does not imply coverage it did not have.`,
|
|
2720
2727
|
allWithin: (gates) => `Within every threshold: ${gates} gates evaluated against measured spend.`,
|
|
2721
2728
|
webhookFailed: (status) =>
|
|
2722
|
-
`The webhook did not deliver (${status}). The crossing is still in the exit code and in the output above
|
|
2729
|
+
`The webhook did not deliver (${status}). The crossing is still in the exit code and in the output above: a receiver being down must not be the quietest failure in the room.`,
|
|
2723
2730
|
watching: (minutes) => `Watching every ${minutes} minutes. Ctrl-C stops it.`,
|
|
2724
2731
|
},
|
|
2725
2732
|
|
|
2726
2733
|
store: {
|
|
2727
2734
|
appended: (count, dir) => `Kept ${count} measurements in ${dir}.`,
|
|
2728
2735
|
empty: (dir) =>
|
|
2729
|
-
`The store at ${dir} is empty. Fill it with "trazum connect <provider> --store"
|
|
2736
|
+
`The store at ${dir} is empty. Fill it with "trazum connect <provider> --store": that is a state, not an error.`,
|
|
2730
2737
|
heading: (records, usd, from, to) =>
|
|
2731
2738
|
`The store: ${records} measurements · ${usd} · ${from} → ${to}`,
|
|
2732
2739
|
providerRow: (provider, records, span, models) =>
|
|
2733
2740
|
`${provider} ${records} measurements · ${span} · ${models} models`,
|
|
2734
2741
|
holds: (files) =>
|
|
2735
|
-
`Held in ${files} files: token counts, billed dollars and the account's own workspace and key identifiers. Never prompt text, never completion text, never a credential
|
|
2742
|
+
`Held in ${files} files: token counts, billed dollars and the account's own workspace and key identifiers. Never prompt text, never completion text, never a credential: this is a file you can back up without a privacy review.`,
|
|
2736
2743
|
possiblyDouble: (count) =>
|
|
2737
|
-
`${count} records could not be told apart from another
|
|
2744
|
+
`${count} records could not be told apart from another, a window of no length, or a record naming no model. They are kept whole rather than merged, so a total built on them may count the same spend twice. Saying so beats a smaller number nobody can check.`,
|
|
2738
2745
|
unknownVersion: (count) =>
|
|
2739
2746
|
`${count} records come from a newer schema than this version knows, so they are kept and left out of the figures above rather than guessed at. Upgrade to read them.`,
|
|
2740
2747
|
unreadable: (file, line) =>
|
|
2741
|
-
`${file} line ${line} would not parse, so it is not in the figures above. The rest of the file was read
|
|
2748
|
+
`${file} line ${line} would not parse, so it is not in the figures above. The rest of the file was read: one broken line must not lose a month.`,
|
|
2742
2749
|
retention: (days) => `Retention: ${days} days, from "store.keepDays". Run "trazum store --prune" to apply it.`,
|
|
2743
2750
|
noRetention: () =>
|
|
2744
2751
|
'No retention policy is configured, so nothing is ever deleted on its own. Set "store": {"keepDays": 90} when you want one.',
|
|
@@ -2747,7 +2754,7 @@ ${bold('EXAMPLES')}
|
|
|
2747
2754
|
pruneDryRun: (count, days, span, usd) =>
|
|
2748
2755
|
span === null
|
|
2749
2756
|
? `Nothing is older than ${days} days, so a prune would delete nothing.`
|
|
2750
|
-
: `A prune would delete ${count} measurements older than ${days} days, covering ${span} and ${usd} of measured spend. Nothing was deleted
|
|
2757
|
+
: `A prune would delete ${count} measurements older than ${days} days, covering ${span} and ${usd} of measured spend. Nothing was deleted: this was --dry-run.`,
|
|
2751
2758
|
pruned: (count, days, span, usd, kept) =>
|
|
2752
2759
|
span === null
|
|
2753
2760
|
? `Nothing was older than ${days} days. ${kept} measurements kept, and the append log compacted.`
|
|
@@ -2768,7 +2775,7 @@ ${bold('EXAMPLES')}
|
|
|
2768
2775
|
budgetNeverForecast: () =>
|
|
2769
2776
|
'That is a shape, not a forecast. Where this goes next depends on what you do next, and no arithmetic here knows that.',
|
|
2770
2777
|
budgetNothingMeasured: (elapsedDays) =>
|
|
2771
|
-
`Nothing has been measured this month, across ${elapsedDays} elapsed ${elapsedDays === 1 ? 'day' : 'days'}. That is not a budget under control
|
|
2778
|
+
`Nothing has been measured this month, across ${elapsedDays} elapsed ${elapsedDays === 1 ? 'day' : 'days'}. That is not a budget under control: it is a budget nobody is watching. Run trazum connect to pull what the provider has.`,
|
|
2772
2779
|
budgetPartial: (measuredDays, elapsedDays, days) =>
|
|
2773
2780
|
`Only ${measuredDays} of ${elapsedDays} elapsed days carry any measurement, so the figure below is a floor on the month rather than the month. Missing: ${days}.`,
|
|
2774
2781
|
budgetScopesUnmeasured: (count) =>
|
|
@@ -2777,11 +2784,11 @@ ${bold('EXAMPLES')}
|
|
|
2777
2784
|
|
|
2778
2785
|
connect: {
|
|
2779
2786
|
noTarget: (providers) =>
|
|
2780
|
-
`Name a provider to read your bill from: trazum connect anthropic. Available: ${providers}. The credential comes from the environment and is never stored
|
|
2787
|
+
`Name a provider to read your bill from: trazum connect anthropic. Available: ${providers}. The credential comes from the environment and is never stored. Add --dry-run to see exactly what would be called and which variable it would be read from.`,
|
|
2781
2788
|
unknownProvider: (id, providers) =>
|
|
2782
2789
|
`There is no connector for "${id}". The ones that exist are: ${providers}.`,
|
|
2783
2790
|
pricedNoConnector: (id, providers) =>
|
|
2784
|
-
`Trazum prices ${id} but has no connector for it yet
|
|
2791
|
+
`Trazum prices ${id} but has no connector for it yet, it connects to ${providers}. Either that provider publishes no usage API, or one has not been written; both are gaps rather than mistakes of yours. Until then, export a log and run "trazum profile" on it: every figure works except the ones a usage API would supply on its own.`,
|
|
2785
2792
|
dryRun: (provider, from, to, envVars, keyKind) =>
|
|
2786
2793
|
`Would read ${provider} usage from ${from} to ${to}, using ${keyKind} taken from ${envVars}. Nothing was sent and no credential was needed to print this.`,
|
|
2787
2794
|
heading: (provider, from, to, usd, calls) =>
|
|
@@ -2791,7 +2798,7 @@ ${bold('EXAMPLES')}
|
|
|
2791
2798
|
modelRow: (model, usd, share, calls) =>
|
|
2792
2799
|
calls === null ? `${model} ${usd} ${share}` : `${model} ${usd} ${share} · ${calls} calls`,
|
|
2793
2800
|
nothingBilled: () =>
|
|
2794
|
-
'The provider billed nothing in this window. That is a measurement, not an error
|
|
2801
|
+
'The provider billed nothing in this window. That is a measurement, not an error, widen it with --since if you expected traffic.',
|
|
2795
2802
|
cachePaid: (saved) => `Caching paid for itself: ${saved} less than these tokens would have cost as ordinary input.`,
|
|
2796
2803
|
cacheLost: (added) => `Caching added ${added} to this bill against what the same tokens would have cost as ordinary input.`,
|
|
2797
2804
|
cacheUnsettled: () =>
|
|
@@ -2802,7 +2809,7 @@ ${bold('EXAMPLES')}
|
|
|
2802
2809
|
`${model} is not in the price catalogue, so its ${tokens} tokens are counted and its money is not. Add it with --pricing rather than reading the total as complete.`,
|
|
2803
2810
|
gap: (detail) => `This window is incomplete: ${detail}.`,
|
|
2804
2811
|
unavailable: (findings) =>
|
|
2805
|
-
`Findings this source cannot support: ${findings}. They need one row per call, and a sum has lost the rows
|
|
2812
|
+
`Findings this source cannot support: ${findings}. They need one row per call, and a sum has lost the rows, a per-call log still answers them.`,
|
|
2806
2813
|
wrote: (path) => `Report written to ${path}.`,
|
|
2807
2814
|
footer: () =>
|
|
2808
2815
|
'Every figure here is the provider\u2019s own billed token count at the catalogue\u2019s rates. Nothing was estimated, and nothing the provider did not serve was filled in.',
|
|
@@ -2821,7 +2828,7 @@ ${bold('EXAMPLES')}
|
|
|
2821
2828
|
runModel: (model, periods, sinceName, from, to) =>
|
|
2822
2829
|
`${model}'s share of the bill has climbed for ${periods} consecutive periods since ${sinceName}: ${from} → ${to}. The totals can look flat while the mix moves under them.`,
|
|
2823
2830
|
runCache: (periods, sinceName, from, to) =>
|
|
2824
|
-
`The cache share has decayed for ${periods} consecutive periods since ${sinceName}: ${from} → ${to}
|
|
2831
|
+
`The cache share has decayed for ${periods} consecutive periods since ${sinceName}: ${from} → ${to}, slowly enough that no single report called it a finding, which is exactly why a series exists.`,
|
|
2825
2832
|
repeated: (kind, label, model, appearances, first, last) => {
|
|
2826
2833
|
const what =
|
|
2827
2834
|
kind === 'route'
|
|
@@ -2834,26 +2841,26 @@ ${bold('EXAMPLES')}
|
|
|
2834
2841
|
? `Fixing the truncation retries on ${label} (${model})`
|
|
2835
2842
|
: `Fixing the cache on ${label} (${model})`;
|
|
2836
2843
|
const span = first !== null && last !== null ? ` (${first} → ${last})` : '';
|
|
2837
|
-
return `${what} has been planned ${appearances} times${span} and is still in the newest plan
|
|
2844
|
+
return `${what} has been planned ${appearances} times${span} and is still in the newest plan: a decision nobody is revisiting.`;
|
|
2838
2845
|
},
|
|
2839
2846
|
storeNoLabels: () =>
|
|
2840
|
-
'This series comes from the store, and a usage API groups by model and workspace rather than by workload
|
|
2847
|
+
'This series comes from the store, and a usage API groups by model and workspace rather than by workload, so there is no label series here at all. Absent, not empty: nothing above says a workload did or did not move.',
|
|
2841
2848
|
unmeasured: (days, from, to, after, before) =>
|
|
2842
|
-
`Nothing covers ${from} to ${to}
|
|
2849
|
+
`Nothing covers ${from} to ${to}, ${plural(days, 'day')} between ${after} and ${before}.`,
|
|
2843
2850
|
unmeasuredTotal: (days) =>
|
|
2844
2851
|
`${plural(days, 'day')} of this stretch are covered by no report. A series with a hole in it and a shorter series read identically; this is which.`,
|
|
2845
2852
|
overlap: (a, b, days) =>
|
|
2846
|
-
`${a} and ${b} both cover ${plural(days, 'day')}. Named, never merged
|
|
2847
|
-
runHole: (days) =>
|
|
2848
|
-
undated: (name) => `${name} carries no span, so it is on no timeline above
|
|
2853
|
+
`${a} and ${b} both cover ${plural(days, 'day')}. Named, never merged, which is the better measurement is not knowable from here, and adding both totals counts those days twice.`,
|
|
2854
|
+
runHole: (days) => `, ${plural(days, 'day')} of this run are covered by no report`,
|
|
2855
|
+
undated: (name) => `${name} carries no span, so it is on no timeline above, named, never silently absorbed.`,
|
|
2849
2856
|
unrecognized: (name) => `${name} is neither a stored report nor a saved plan, so it is in no series above.`,
|
|
2850
2857
|
footer: () =>
|
|
2851
|
-
'A series names shapes, not futures. Twenty points make a trend visible; they do not make next month knowable
|
|
2858
|
+
'A series names shapes, not futures. Twenty points make a trend visible; they do not make next month knowable, where these lines go next is yours to judge.',
|
|
2852
2859
|
waiverHeading: () => 'What this repository has been living with',
|
|
2853
2860
|
waiverSince: (day, uses) =>
|
|
2854
2861
|
`${uses} recorded ${uses === 1 ? 'use' : 'uses'} since ${day}, when recording started.`,
|
|
2855
2862
|
waiverNoneRecorded: () =>
|
|
2856
|
-
'No waiver has silenced a gate since recording started. Nothing here is inferred from the config
|
|
2863
|
+
'No waiver has silenced a gate since recording started. Nothing here is inferred from the config: a waiver written down and never hit is not a decision anyone is living with.',
|
|
2857
2864
|
waiverHabit: (gate, uses, days, firstDay, lastDay) =>
|
|
2858
2865
|
`${gate}: ${uses} ${uses === 1 ? 'use' : 'uses'} across ${days} ${days === 1 ? 'day' : 'days'}, ${firstDay} to ${lastDay}`,
|
|
2859
2866
|
waiverVerdict: (verdict) =>
|
|
@@ -2862,7 +2869,7 @@ ${bold('EXAMPLES')}
|
|
|
2862
2869
|
: verdict === 'recurring'
|
|
2863
2870
|
? 'The same decision, holding. The gate keeps firing and the reason has not moved.'
|
|
2864
2871
|
: verdict === 'renewed-without-revisiting'
|
|
2865
|
-
? 'The expiry has moved and the reason has not. That is the shape a decision takes when nobody is revisiting it
|
|
2872
|
+
? 'The expiry has moved and the reason has not. That is the shape a decision takes when nobody is revisiting it, which is sometimes exactly right, and worth saying out loud either way.'
|
|
2866
2873
|
: 'The reason changed between uses. Somebody looked again.',
|
|
2867
2874
|
waiverReasonNow: (reason) => `Reason: ${reason}`,
|
|
2868
2875
|
waiverReasonsChanged: (count) => `${count} different reasons given over that time.`,
|
|
@@ -2871,7 +2878,7 @@ ${bold('EXAMPLES')}
|
|
|
2871
2878
|
waiverNoLongerConfigured: () =>
|
|
2872
2879
|
'Not in the config any more. The decision was reversed; the record keeps it.',
|
|
2873
2880
|
waiverNeverUsed: (gates) =>
|
|
2874
|
-
`Waived in the config and never hit by a recorded run: ${gates}. Either the gate stopped failing
|
|
2881
|
+
`Waived in the config and never hit by a recorded run: ${gates}. Either the gate stopped failing, good news nobody wrote down, or the waiver names a situation that does not arise. Both are worth deleting.`,
|
|
2875
2882
|
waiverUnreadable: (count, path) =>
|
|
2876
2883
|
`${count} ${count === 1 ? 'line' : 'lines'} in ${path} could not be read and are not counted above.`,
|
|
2877
2884
|
waiverStartsHere: () =>
|
|
@@ -2880,7 +2887,7 @@ ${bold('EXAMPLES')}
|
|
|
2880
2887
|
|
|
2881
2888
|
verify: {
|
|
2882
2889
|
noTarget: () =>
|
|
2883
|
-
'Point this at a saved plan and a newer log: trazum verify plan.json --against usage.jsonl. It says, per action, whether the change arrived, did not arrive, or cannot be told
|
|
2890
|
+
'Point this at a saved plan and a newer log: trazum verify plan.json --against usage.jsonl. It says, per action, whether the change arrived, did not arrive, or cannot be told, and never fewer than those three.',
|
|
2884
2891
|
needsAgainst: () =>
|
|
2885
2892
|
'--against <newer.jsonl|dir> is required. A plan can only be verified against a log that came after it; without one there is nothing to hold the prediction to.',
|
|
2886
2893
|
badPlan: (path, why) =>
|
|
@@ -2902,7 +2909,7 @@ ${bold('EXAMPLES')}
|
|
|
2902
2909
|
counts: (arrived, notArrived, cannotTell) =>
|
|
2903
2910
|
`${arrived} arrived · ${notArrived} did not arrive · ${cannotTell} cannot be told. The third is not a soft version of the second: it means this log cannot answer, which is its own finding.`,
|
|
2904
2911
|
pricesChanged: (planReviewed, nowReviewed) =>
|
|
2905
|
-
`Prices were reviewed ${planReviewed} when the plan was made and ${nowReviewed} now, so every dollar comparison here is two price lists, not one measurement
|
|
2912
|
+
`Prices were reviewed ${planReviewed} when the plan was made and ${nowReviewed} now, so every dollar comparison here is two price lists, not one measurement: a team must not be blamed for a saving that arithmetic revoked.`,
|
|
2906
2913
|
action: (kind, label, model, outcome) => {
|
|
2907
2914
|
const what =
|
|
2908
2915
|
kind === 'route'
|
|
@@ -2916,14 +2923,14 @@ ${bold('EXAMPLES')}
|
|
|
2916
2923
|
: `Fix the cache on ${label} (${model})`;
|
|
2917
2924
|
const verdict =
|
|
2918
2925
|
outcome === 'arrived' ? 'ARRIVED' : outcome === 'not-arrived' ? 'DID NOT ARRIVE' : 'CANNOT BE TOLD';
|
|
2919
|
-
return `${what}
|
|
2926
|
+
return `${what}, ${verdict}`;
|
|
2920
2927
|
},
|
|
2921
2928
|
reason: (reason) =>
|
|
2922
2929
|
reason === 'workload-vanished'
|
|
2923
|
-
? 'the label carries no priced traffic in this log
|
|
2930
|
+
? 'the label carries no priced traffic in this log: a vanished workload is not a fixed one, and not a broken one either'
|
|
2924
2931
|
: reason === 'fields-stopped'
|
|
2925
|
-
? 'the fields the detection needs are not in this log
|
|
2926
|
-
: 'the log records tokens, and tokens do not say which tier billed them
|
|
2932
|
+
? 'the fields the detection needs are not in this log, "not recorded" must not read as "fixed", so with --gate this fails'
|
|
2933
|
+
: 'the log records tokens, and tokens do not say which tier billed them: the Batch API cannot be seen from here',
|
|
2927
2934
|
routeObserved: (dearestModel, onTargetUsd, onOldUsd) =>
|
|
2928
2935
|
`the label's dearest model is now ${dearestModel} · ${onTargetUsd} on the target, ${onOldUsd} still on the old model`,
|
|
2929
2936
|
batchUnobservable: () =>
|
|
@@ -2935,9 +2942,9 @@ ${bold('EXAMPLES')}
|
|
|
2935
2942
|
? `caching now pays for itself on this slice (${deltaUsd} against the no-cache bill)`
|
|
2936
2943
|
: `caching still adds ${deltaUsd} to this slice's bill`,
|
|
2937
2944
|
attribution: (callsBefore, callsAfter, outBefore, outAfter) =>
|
|
2938
|
-
`the world moved too: calls ${callsBefore} → ${callsAfter}, output/call ${outBefore} → ${outAfter} tokens
|
|
2945
|
+
`the world moved too: calls ${callsBefore} → ${callsAfter}, output/call ${outBefore} → ${outAfter} tokens, stated so the verdict is not read as the whole story`,
|
|
2939
2946
|
gateFailed: (failures, total) =>
|
|
2940
|
-
`GATE FAILED
|
|
2947
|
+
`GATE FAILED, ${failures} of ${total} actions did not produce what the plan promised, or stopped being measurable by the team's own log.`,
|
|
2941
2948
|
gateOk: () => 'Gate passed: every verifiable action arrived, and nothing became unverifiable.',
|
|
2942
2949
|
footer: () =>
|
|
2943
2950
|
'Arrived and did-not-arrive are measurements; cannot-be-told is the log refusing to guess. All three are the verification working, not failing.',
|
|
@@ -2955,26 +2962,26 @@ ${bold('EXAMPLES')}
|
|
|
2955
2962
|
picked: (label, model, candidate, usd, pct) =>
|
|
2956
2963
|
`${label} on ${model} → ${candidate}, worth ${usd} of this bill (${pct}).`,
|
|
2957
2964
|
willSpend: (calls, model, candidate) =>
|
|
2958
|
-
`This will make ${count(calls)} provider calls: two per case on ${model} to measure its own variance, one per case on ${candidate}. Nothing has been spent yet
|
|
2965
|
+
`This will make ${count(calls)} provider calls: two per case on ${model} to measure its own variance, one per case on ${candidate}. Nothing has been spent yet. Add --yes to run it.`,
|
|
2959
2966
|
dryRun: () => 'Nothing was called.',
|
|
2960
2967
|
running: (cases) => `Running ${count(cases)} cases...`,
|
|
2961
2968
|
agreement: (cross, self) =>
|
|
2962
|
-
`The cheaper model agrees with the original ${cross} of the time. The original agrees with itself ${self} of the time
|
|
2969
|
+
`The cheaper model agrees with the original ${cross} of the time. The original agrees with itself ${self} of the time: that is the yardstick, not 100%.`,
|
|
2963
2970
|
holds: (usd) =>
|
|
2964
|
-
`HOLDS
|
|
2971
|
+
`HOLDS: the difference is inside the original model's own noise. On this bill that route is worth ${usd}.`,
|
|
2965
2972
|
diverges: (usd) =>
|
|
2966
|
-
`DIVERGES
|
|
2973
|
+
`DIVERGES: the cheaper model gives materially different answers. The ${usd} is real and so is the change in behaviour; this one is not free money.`,
|
|
2967
2974
|
inconclusive: () =>
|
|
2968
|
-
'INCONCLUSIVE
|
|
2975
|
+
'INCONCLUSIVE: the original model was too inconsistent with itself on these cases to judge anything against. Add cases, or pick ones with less room for the model to wander.',
|
|
2969
2976
|
unlabelledSlice: () =>
|
|
2970
|
-
'These calls carry no label, so Trazum cannot tell whether they are all this prompt. If they are not, the figure above covers calls this measurement never touched
|
|
2977
|
+
'These calls carry no label, so Trazum cannot tell whether they are all this prompt. If they are not, the figure above covers calls this measurement never touched. Add "label" to the record and the slice becomes one workload, which is what makes the number attributable.',
|
|
2971
2978
|
yours: () =>
|
|
2972
|
-
'Agreement is not correctness. This measures whether the answers moved, not whether they were ever right
|
|
2979
|
+
'Agreement is not correctness. This measures whether the answers moved, not whether they were ever right: the decision is still yours.',
|
|
2973
2980
|
},
|
|
2974
2981
|
|
|
2975
2982
|
baseline: {
|
|
2976
2983
|
recorded: (path, files, tokens) =>
|
|
2977
|
-
`Recorded ${files} prompts, ${tokens} tokens, to ${path}. Commit it
|
|
2984
|
+
`Recorded ${files} prompts, ${tokens} tokens, to ${path}. Commit it: the gate compares the tree against what is committed.`,
|
|
2978
2985
|
recordedMoney: (monthly, model, calls) =>
|
|
2979
2986
|
`That is ${monthly} per month with ${model} at ${calls} calls. Reported, not gated on: thresholds are in tokens, so a repriced model never fails a build on its own.`,
|
|
2980
2987
|
heading: () => 'Against the baseline',
|
|
@@ -3000,7 +3007,7 @@ ${bold('EXAMPLES')}
|
|
|
3000
3007
|
slots: {
|
|
3001
3008
|
task: {
|
|
3002
3009
|
question: 'What should the model do, in one sentence?',
|
|
3003
|
-
unlocks: 'the whole prompt
|
|
3010
|
+
unlocks: 'the whole prompt, without it there is nothing to write',
|
|
3004
3011
|
},
|
|
3005
3012
|
role: {
|
|
3006
3013
|
question: 'Who is the model being while it does that?',
|
|
@@ -3008,7 +3015,7 @@ ${bold('EXAMPLES')}
|
|
|
3008
3015
|
},
|
|
3009
3016
|
inputs: {
|
|
3010
3017
|
question: 'What changes from one call to the next?',
|
|
3011
|
-
unlocks: 'the varying part
|
|
3018
|
+
unlocks: 'the varying part, without it the prompt hard-codes one case and is rewritten for the next',
|
|
3012
3019
|
},
|
|
3013
3020
|
'output-shape': {
|
|
3014
3021
|
question: 'What should come back: prose, json, list or table?',
|
|
@@ -3024,7 +3031,7 @@ ${bold('EXAMPLES')}
|
|
|
3024
3031
|
},
|
|
3025
3032
|
audience: {
|
|
3026
3033
|
question: 'Who reads the output?',
|
|
3027
|
-
unlocks: 'the register
|
|
3034
|
+
unlocks: 'the register: an answer for an engineer and one for a customer are different answers',
|
|
3028
3035
|
},
|
|
3029
3036
|
constraints: {
|
|
3030
3037
|
question: 'What must it never do?',
|
|
@@ -3040,7 +3047,7 @@ ${bold('EXAMPLES')}
|
|
|
3040
3047
|
},
|
|
3041
3048
|
'example-inputs': {
|
|
3042
3049
|
question: 'What input produced that example?',
|
|
3043
|
-
unlocks: 'the pairing
|
|
3050
|
+
unlocks: 'the pairing, an example answer with no input teaches the shape and not the mapping',
|
|
3044
3051
|
},
|
|
3045
3052
|
'failure-modes': {
|
|
3046
3053
|
question: 'What has gone wrong with this before?',
|
|
@@ -3048,11 +3055,11 @@ ${bold('EXAMPLES')}
|
|
|
3048
3055
|
},
|
|
3049
3056
|
model: {
|
|
3050
3057
|
question: 'Which model is this for?',
|
|
3051
|
-
unlocks: 'the cost estimate
|
|
3058
|
+
unlocks: 'the cost estimate, this changes the report and never the prompt',
|
|
3052
3059
|
},
|
|
3053
3060
|
budget: {
|
|
3054
3061
|
question: 'What is the monthly ceiling for this prompt?',
|
|
3055
|
-
unlocks: 'the budget check
|
|
3062
|
+
unlocks: 'the budget check, this changes the report and never the prompt',
|
|
3056
3063
|
},
|
|
3057
3064
|
},
|
|
3058
3065
|
missing: (count) =>
|
|
@@ -3068,7 +3075,7 @@ ${bold('EXAMPLES')}
|
|
|
3068
3075
|
: `"${id}" is not a slot. Did you mean "${nearest}"?`,
|
|
3069
3076
|
answerNotText: (id) => `"${id}" must be a string, or null to decline it.`,
|
|
3070
3077
|
tokens: (count) => `${count} tokens`,
|
|
3071
|
-
monthly: (usd) => `${usd} per month, estimated
|
|
3078
|
+
monthly: (usd) => `${usd} per month, estimated: nobody has sent this prompt yet`,
|
|
3072
3079
|
budget: (verdict, limit) =>
|
|
3073
3080
|
verdict === 'over' ? `over the budget of ${limit}` : `within the budget of ${limit}`,
|
|
3074
3081
|
noVerdict: (reason) =>
|
|
@@ -3079,7 +3086,7 @@ ${bold('EXAMPLES')}
|
|
|
3079
3086
|
: 'That model is not in the price catalogue, so it cannot be priced.',
|
|
3080
3087
|
clean: () => 'trazum optimize recovers nothing from this.',
|
|
3081
3088
|
notClean: (rules, tokens) =>
|
|
3082
|
-
`trazum optimize would still recover ${tokens} tokens here (${rules})
|
|
3089
|
+
`trazum optimize would still recover ${tokens} tokens here (${rules}), from your answers, not from the structure.`,
|
|
3083
3090
|
declined: (ids) => `Declined, and left out: ${ids}`,
|
|
3084
3091
|
},
|
|
3085
3092
|
};
|