@trazum/cli 1.80.1 → 1.82.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/gateway-server.d.ts +38 -1
- package/dist/gateway-server.d.ts.map +1 -1
- package/dist/gateway-server.js +125 -9
- package/dist/gateway-server.js.map +1 -1
- package/dist/i18n/en.d.ts +1 -1
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +437 -376
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/es.d.ts.map +1 -1
- package/dist/i18n/es.js +68 -5
- package/dist/i18n/es.js.map +1 -1
- package/dist/i18n/types.d.ts +74 -6
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/index.d.ts +54 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +545 -55
- package/dist/index.js.map +1 -1
- package/dist/markdown.d.ts +10 -0
- package/dist/markdown.d.ts.map +1 -1
- package/dist/markdown.js +4 -4
- package/dist/markdown.js.map +1 -1
- package/package.json +2 -2
- package/src/gateway-server.ts +160 -9
- package/src/i18n/en.ts +454 -378
- package/src/i18n/es.ts +85 -7
- package/src/i18n/types.ts +76 -6
- package/src/index.ts +602 -54
- package/src/markdown.ts +14 -4
package/src/i18n/en.ts
CHANGED
|
@@ -6,8 +6,8 @@ const count = (value: number): string => value.toLocaleString('en-US');
|
|
|
6
6
|
/**
|
|
7
7
|
* A grouped count and its noun, agreeing.
|
|
8
8
|
*
|
|
9
|
-
* `1 calls` and `1 prompts` were reachable on ordinary input
|
|
10
|
-
* repository with a single prompt
|
|
9
|
+
* `1 calls` and `1 prompts` were reachable on ordinary input, a one-call log, a
|
|
10
|
+
* repository with a single prompt, and two messages in this file already did the
|
|
11
11
|
* agreement by hand while a dozen did not. A helper rather than a dozen ternaries,
|
|
12
12
|
* so the next message written gets it for free.
|
|
13
13
|
*/
|
|
@@ -20,7 +20,7 @@ const ago = (days: number | null): string =>
|
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
/**
|
|
23
|
-
* English catalogue
|
|
23
|
+
* English catalogue, the source of truth.
|
|
24
24
|
*
|
|
25
25
|
* When a message changes here, the other catalogues need the same change.
|
|
26
26
|
* `test/i18n.test.js` in the core enforces the equivalent guarantee for the
|
|
@@ -30,7 +30,7 @@ export const en: CliMessages = {
|
|
|
30
30
|
locale: 'en',
|
|
31
31
|
numberLocale: 'en-US',
|
|
32
32
|
|
|
33
|
-
help: (d, bold) => `${bold('trazum')}
|
|
33
|
+
help: (d, bold) => `${bold('trazum')}, cut the cost of your prompts without losing what they ask for.
|
|
34
34
|
|
|
35
35
|
${bold('USAGE')}
|
|
36
36
|
trazum init [dir] [--dry-run | --yes]
|
|
@@ -61,6 +61,8 @@ ${bold('USAGE')}
|
|
|
61
61
|
trazum position <usage.jsonl>
|
|
62
62
|
trazum from-claude-code <file|dir> [--label <name>] [-o <file>]
|
|
63
63
|
trazum from-otel <file|dir> [--label-from-service] [-o <file>]
|
|
64
|
+
trazum from-litellm <file|dir> [-o <file>]
|
|
65
|
+
trazum from-helicone <file|dir> [-o <file>]
|
|
64
66
|
trazum switch <usage.jsonl> --to <model> [--migration-usd <n>] [--cases <n>]
|
|
65
67
|
trazum ownrate --gpu-usd-hour <n> --tokens-per-second <n> [--utilization <0-1>]
|
|
66
68
|
trazum pulse [--max-stale-hours <n>]
|
|
@@ -92,7 +94,7 @@ ${bold('OPTIONS FOR init')}
|
|
|
92
94
|
found, arithmetic first.
|
|
93
95
|
|
|
94
96
|
It never invents a threshold. A budget is a policy, so init hands you the
|
|
95
|
-
measured figure and leaves the limit to you
|
|
97
|
+
measured figure and leaves the limit to you: a generated config full of
|
|
96
98
|
guessed numbers is one nobody trusts.
|
|
97
99
|
|
|
98
100
|
${bold('OPTIONS FOR conform')}
|
|
@@ -100,7 +102,7 @@ ${bold('OPTIONS FOR conform')}
|
|
|
100
102
|
detecting one: ${d.contracts.join(', ')}.
|
|
101
103
|
--json The report as data.
|
|
102
104
|
|
|
103
|
-
Answers two questions and keeps them apart. Does this document conform
|
|
105
|
+
Answers two questions and keeps them apart. Does this document conform,
|
|
104
106
|
required fields, present and the right type, exits 1 when not. And what can a
|
|
105
107
|
valid document of this shape not answer, with the field that would unlock
|
|
106
108
|
each.
|
|
@@ -111,7 +113,7 @@ ${bold('OPTIONS FOR conform')}
|
|
|
111
113
|
${bold('OPTIONS FOR rollup')}
|
|
112
114
|
--json The roll-up document as data. See docs/format.md.
|
|
113
115
|
--html-out <file> Also write the roll-up as one self-contained HTML
|
|
114
|
-
file
|
|
116
|
+
file, the team-facing report, for whoever pays
|
|
115
117
|
the bill. Each contributor's gaps stay under that
|
|
116
118
|
contributor, and what no merge can measure is in
|
|
117
119
|
the caveat block, impossible to crop.
|
|
@@ -119,13 +121,13 @@ ${bold('OPTIONS FOR rollup')}
|
|
|
119
121
|
Several people's bills, one roll-up. Each contributor runs
|
|
120
122
|
\`trazum profile --json\` where their traffic already is; this merges the
|
|
121
123
|
documents. A format and a merge, not a service: nothing is uploaded and there
|
|
122
|
-
is no account
|
|
124
|
+
is no account: the documents arrive however your team already moves files.
|
|
123
125
|
|
|
124
126
|
Most of the output is what the merge could not do. Each contributor's own
|
|
125
127
|
gaps stay with that contributor rather than being averaged into one figure,
|
|
126
128
|
the findings that need individual calls are named instead of dropped, and
|
|
127
129
|
every roll-up of more than one document says that overlap between
|
|
128
|
-
contributors is unmeasurable
|
|
130
|
+
contributors is unmeasurable: two people exporting the same traffic double
|
|
129
131
|
the bill and no merge of summaries can see it.
|
|
130
132
|
|
|
131
133
|
Exits 1 when a contribution was handed over and could not be merged.
|
|
@@ -148,11 +150,11 @@ ${bold('OPTIONS FOR switch')}
|
|
|
148
150
|
Unlocks the line pricing that evaluation.
|
|
149
151
|
|
|
150
152
|
The decision every what-if serves, priced: what this measured mix costs on
|
|
151
|
-
the candidate, the saving with its sign stated, and
|
|
152
|
-
migration cost
|
|
153
|
+
the candidate, the saving with its sign stated, and, with a declared
|
|
154
|
+
migration cost, the break-even as division on the measured past, days of
|
|
153
155
|
window attached, never a forecast. The evaluation the switch requires is
|
|
154
156
|
priced at this log's own mean call, two calls on the incumbent and one on
|
|
155
|
-
the candidate per case. What it refuses: any sentence about quality
|
|
157
|
+
the candidate per case. What it refuses: any sentence about quality, that
|
|
156
158
|
verdict belongs to trazum route, and the report ends by printing it.
|
|
157
159
|
|
|
158
160
|
${bold('OPTIONS FOR ownrate')}
|
|
@@ -161,19 +163,42 @@ ${bold('OPTIONS FOR ownrate')}
|
|
|
161
163
|
--utilization <0-1> Fraction of the hour actually serving.
|
|
162
164
|
Defaults to 1.
|
|
163
165
|
|
|
164
|
-
Your self-hosted model's $/MTok, derived from your own two numbers
|
|
166
|
+
Your self-hosted model's $/MTok, derived from your own two numbers,
|
|
165
167
|
division, nothing else: no amortisation, no energy model, no guessed
|
|
166
168
|
efficiency. Prints the figure and the pricing-overlay snippet to paste
|
|
167
169
|
into trazum.config.json, so the model you run yourself becomes a
|
|
168
170
|
first-class row in every report, priced by you and marked as such.
|
|
169
171
|
|
|
172
|
+
${bold('OPTIONS FOR from-helicone')}
|
|
173
|
+
-o, --out <file> Write the usage log there instead of stdout.
|
|
174
|
+
|
|
175
|
+
A Helicone request export, read as a usage log: model, timestamp, label and
|
|
176
|
+
the token counts from each row, and nothing else. The request body, the
|
|
177
|
+
response body and the user id stay in the row. The model that answered is
|
|
178
|
+
what is priced, and rows answered by a different model than the one
|
|
179
|
+
requested are counted. cache_enabled is a flag and never a token split, so
|
|
180
|
+
cache verdicts read "cannot tell". A request id is one call and never a
|
|
181
|
+
conversation, so the records carry no session and the conversation findings
|
|
182
|
+
cannot be answered from this log; that is said on every run.
|
|
183
|
+
|
|
184
|
+
${bold('OPTIONS FOR from-litellm')}
|
|
185
|
+
-o, --out <file> Write the usage log there instead of stdout.
|
|
186
|
+
|
|
187
|
+
A LiteLLM spend log, read as a usage log: model, timestamp, label, session
|
|
188
|
+
and the token counts from each row, and nothing else. The messages, the
|
|
189
|
+
response, the hashed key, the requester address and the end user stay in the
|
|
190
|
+
row. cache_hit is a flag and never a token split, so cache verdicts read
|
|
191
|
+
"cannot tell" rather than a guessed one. Rows naming no model are counted
|
|
192
|
+
and left out: model_group is a route, not a model. LiteLLM's own spend
|
|
193
|
+
figure is printed beside Trazum's and never merged into it.
|
|
194
|
+
|
|
170
195
|
${bold('OPTIONS FOR from-otel')}
|
|
171
196
|
--label-from-service Label each record with its span's service name,
|
|
172
197
|
so a per-service bill falls out.
|
|
173
198
|
-o, --out <file> Write the usage log there instead of stdout.
|
|
174
199
|
|
|
175
200
|
OpenTelemetry GenAI spans, read as a usage log: model, timestamp, label and
|
|
176
|
-
the token counts from each gen_ai.* span
|
|
201
|
+
the token counts from each gen_ai.* span, and nothing else. Prompt content
|
|
177
202
|
and trace ids stay in the span. OTel has no cache TTL split, so cache
|
|
178
203
|
verdicts read "cannot tell" rather than a guessed one. Non-LLM spans are
|
|
179
204
|
counted and skipped, said on stderr.
|
|
@@ -181,19 +206,19 @@ ${bold('OPTIONS FOR from-otel')}
|
|
|
181
206
|
${bold('OPTIONS FOR from-claude-code')}
|
|
182
207
|
--label <name> Stamp every record with one label.
|
|
183
208
|
--label-from-project Label each record with its transcript's project
|
|
184
|
-
directory name, verbatim
|
|
209
|
+
directory name, verbatim, map it to a workload
|
|
185
210
|
name with the config's \`labels\` block.
|
|
186
211
|
-o, --out <file> Write the usage log there instead of stdout.
|
|
187
212
|
|
|
188
213
|
Claude Code's transcripts, read as a usage log: model, timestamp, session
|
|
189
|
-
and the API's own usage object
|
|
214
|
+
and the API's own usage object, the cache TTL split included, and
|
|
190
215
|
nothing else. No message text, no paths, no branches cross the
|
|
191
216
|
conversion. One API call is written as one line per content block, so
|
|
192
217
|
lines are collapsed by request id; what was collapsed or passed over is
|
|
193
218
|
said on stderr, never silently.
|
|
194
219
|
|
|
195
220
|
${bold('OPTIONS FOR position')}
|
|
196
|
-
--html-out <file> The position as one self-contained page
|
|
221
|
+
--html-out <file> The position as one self-contained page, the
|
|
197
222
|
same sentences the terminal prints, caveats
|
|
198
223
|
first, for the person who does not run CLIs.
|
|
199
224
|
--json The position document, contract-checked like
|
|
@@ -212,7 +237,7 @@ ${bold('OPTIONS FOR pulse')}
|
|
|
212
237
|
--json The report as data.
|
|
213
238
|
|
|
214
239
|
Did the things that are supposed to run, run? \`watch --once\` is built for a
|
|
215
|
-
scheduler, and its state file is read by exactly one thing
|
|
240
|
+
scheduler, and its state file is read by exactly one thing, the next cycle.
|
|
216
241
|
So nothing could tell you the watcher had stopped, because the thing that
|
|
217
242
|
would tell you was the thing that stopped.
|
|
218
243
|
|
|
@@ -220,8 +245,8 @@ ${bold('OPTIONS FOR pulse')}
|
|
|
220
245
|
is your CI: a step running this on the schedule you already have turns a dead
|
|
221
246
|
cron into a red build, without Trazum holding anybody's metrics.
|
|
222
247
|
|
|
223
|
-
Never gates on a first run that never happened
|
|
224
|
-
late against
|
|
248
|
+
Never gates on a first run that never happened: that has no cadence to be
|
|
249
|
+
late against, and never on how far the measurements reach, which is a
|
|
225
250
|
provider reporting on its own schedule rather than a job that failed.
|
|
226
251
|
|
|
227
252
|
${bold('OPTIONS FOR bench')}
|
|
@@ -236,7 +261,7 @@ ${bold('OPTIONS FOR bench')}
|
|
|
236
261
|
--against: how much regression is too much is a
|
|
237
262
|
policy, and this tool does not write yours.
|
|
238
263
|
--json The measurements as data. The shape never changes
|
|
239
|
-
with the gate flags
|
|
264
|
+
with the gate flags: a verdict is the exit code
|
|
240
265
|
and sentences on stderr, the way check gates.
|
|
241
266
|
|
|
242
267
|
Measures this machine: the standard workloads, one shot each, wall time and
|
|
@@ -270,7 +295,7 @@ ${bold('OPTIONS FOR feedback')}
|
|
|
270
295
|
(none)
|
|
271
296
|
|
|
272
297
|
Prints where to report a wrong optimisation, a bug, a question or a security
|
|
273
|
-
problem
|
|
298
|
+
problem, and a blank issue with the version, runtime and platform already
|
|
274
299
|
filled in, printed in full first so nothing travels you have not read.
|
|
275
300
|
|
|
276
301
|
It sends nothing. Trazum has no telemetry: no ping, no install hook, no
|
|
@@ -279,17 +304,17 @@ ${bold('OPTIONS FOR feedback')}
|
|
|
279
304
|
|
|
280
305
|
${bold('OPTIONS FOR gateway')}
|
|
281
306
|
--on-cannot-tell <policy> Required, no default: fail-open or fail-closed.
|
|
282
|
-
What happens when the gateway cannot judge a call
|
|
283
|
-
|
|
307
|
+
What happens when the gateway cannot judge a call:
|
|
308
|
+
no budget, nothing measured, an unpriced model.
|
|
284
309
|
--port <n> | --socket <p> Where to listen. Loopback only, always.
|
|
285
310
|
--log <usage.jsonl> The usage log the limits policy is measured
|
|
286
|
-
against, read once at start
|
|
311
|
+
against, read once at start, same flag as
|
|
287
312
|
serve's, same reason: label and session spend
|
|
288
313
|
live in a usage log, not in the store.
|
|
289
314
|
|
|
290
315
|
Stands between your SDK and the provider, speaking their wire format, so no
|
|
291
316
|
code changes. Usage is measured from the provider's own response as it comes
|
|
292
|
-
back
|
|
317
|
+
back, no export, no connector lag, no missing day.
|
|
293
318
|
|
|
294
319
|
It refuses and never substitutes: a call over budget gets HTTP 402 with the
|
|
295
320
|
cheaper alternatives named. Substitution happens only where you wrote it down
|
|
@@ -306,7 +331,7 @@ ${bold('OPTIONS FOR experiment')}
|
|
|
306
331
|
honoured.
|
|
307
332
|
|
|
308
333
|
Judges recorded outcomes and cost together. Three-valued: A wins, B wins, or
|
|
309
|
-
not separable
|
|
334
|
+
not separable, with the number of outcomes per arm that would settle it, so
|
|
310
335
|
"run it longer" is an instruction rather than a shrug. Nothing is promoted.
|
|
311
336
|
|
|
312
337
|
${bold('OPTIONS FOR quality')}
|
|
@@ -321,7 +346,7 @@ ${bold('OPTIONS FOR quality')}
|
|
|
321
346
|
|
|
322
347
|
A before-and-after rather than an experiment, so it reports "cannot tell"
|
|
323
348
|
whenever the model mix, the call volume or the outcome coverage moved across
|
|
324
|
-
the boundary
|
|
349
|
+
the boundary: the prompt is not the only variable and it says so.
|
|
325
350
|
|
|
326
351
|
${bold('OPTIONS FOR semantic')}
|
|
327
352
|
--yes Required to send anything. Without it the command
|
|
@@ -342,14 +367,14 @@ ${bold('OPTIONS FOR commitment')}
|
|
|
342
367
|
|
|
343
368
|
Replays measured whole months against the deal's terms. Both directions are
|
|
344
369
|
priced: the months that cleared the floor, and what the floor cost in the
|
|
345
|
-
months that fell short
|
|
370
|
+
months that fell short, kept as its own figure, because netted into the
|
|
346
371
|
saving it disappears. An as-if calculation about the past, never a forecast.
|
|
347
372
|
|
|
348
373
|
${bold('OPTIONS FOR report')}
|
|
349
374
|
--year <yyyy> Required. Which year to assemble.
|
|
350
375
|
--json The record as data.
|
|
351
376
|
|
|
352
|
-
Everything comes from the store and the plans you already keep
|
|
377
|
+
Everything comes from the store and the plans you already keep: nothing is
|
|
353
378
|
computed that cannot be checked against a document that already exists.
|
|
354
379
|
Months with no record are named rather than filled, promises are kept in
|
|
355
380
|
three outcomes rather than two, and the report lists what it cannot say.
|
|
@@ -374,14 +399,14 @@ ${bold('OPTIONS FOR eval')}
|
|
|
374
399
|
--concurrency <n> Calls in flight at once. Default: 3.
|
|
375
400
|
--export promptfoo Write a promptfoo suite instead of running anything:
|
|
376
401
|
both prompts, every case, no API key needed and no
|
|
377
|
-
call made. The assertions are yours
|
|
402
|
+
call made. The assertions are yours: this exists
|
|
378
403
|
for the question agreement cannot answer.
|
|
379
404
|
-o, --out <file> Where to write it. Defaults to stdout.
|
|
380
405
|
|
|
381
406
|
Runs both prompt versions over your cases and reports whether the
|
|
382
407
|
optimisation changed the answers. Costs THREE provider calls per case: the
|
|
383
408
|
original twice, to measure the model's own run-to-run variance, and the
|
|
384
|
-
optimised once. That baseline is the yardstick
|
|
409
|
+
optimised once. That baseline is the yardstick, without it, a divergence
|
|
385
410
|
figure means nothing. Exits with code 1 when the answers genuinely diverge.
|
|
386
411
|
|
|
387
412
|
${bold('OPTIONS FOR rank')}
|
|
@@ -443,7 +468,7 @@ ${bold('OPTIONS FOR optimize')}
|
|
|
443
468
|
--all-labels With --from-log: every prompt in the config's
|
|
444
469
|
"labels" map, optimised and priced against its
|
|
445
470
|
own measured traffic, ranked by what the change
|
|
446
|
-
is worth
|
|
471
|
+
is worth, plus the mismatches in both
|
|
447
472
|
directions (mapped prompts with no traffic,
|
|
448
473
|
traffic with no mapped prompt).
|
|
449
474
|
--from-log <usage.jsonl> Measure the three figures above from a usage log
|
|
@@ -457,7 +482,7 @@ ${bold('OPTIONS FOR optimize')}
|
|
|
457
482
|
--batch The work tolerates latency (Batch API, 50% off).
|
|
458
483
|
--disable <id,id> Turn off specific rules (see "trazum rules").
|
|
459
484
|
--suggest Ask the LLM for phrase-level rewrites and list them
|
|
460
|
-
with what each saves. Changes nothing on its own
|
|
485
|
+
with what each saves. Changes nothing on its own,
|
|
461
486
|
every proposal is checked against your prompt first
|
|
462
487
|
and dropped if it does not survive.
|
|
463
488
|
--apply-suggestions Take them. Only with --suggest; alone it is an error
|
|
@@ -478,7 +503,7 @@ ${bold('OPTIONS FOR optimize')}
|
|
|
478
503
|
--tokens-only Report the token saving and no money at all. The
|
|
479
504
|
default inside Claude Code, Codex or Cursor, where
|
|
480
505
|
a subscription means there is no bill to reduce.
|
|
481
|
-
--cost Show the money even there
|
|
506
|
+
--cost Show the money even there: the host says where
|
|
482
507
|
Trazum runs, not where your prompt goes.
|
|
483
508
|
--prompt <name> Which marked prompt to optimise, when a source file
|
|
484
509
|
holds more than one. See "trazum where".
|
|
@@ -505,7 +530,7 @@ ${bold('OPTIONS FOR check')}
|
|
|
505
530
|
--baseline Gate on the recorded cost baseline. On by default whenever
|
|
506
531
|
the config declares one, so CI needs no argument; the useful
|
|
507
532
|
spelling is --no-baseline, which skips it for one run.
|
|
508
|
-
--files-from <file|-> Check only the files listed, one path per line
|
|
533
|
+
--files-from <file|-> Check only the files listed, one path per line, the shape
|
|
509
534
|
git diff --name-only produces, so a pre-commit hook is one
|
|
510
535
|
pipe: git diff --cached --name-only | trazum check --files-from -
|
|
511
536
|
Non-prompt paths and deletions are dropped and counted out
|
|
@@ -518,7 +543,7 @@ ${bold('OPTIONS FOR check')}
|
|
|
518
543
|
|
|
519
544
|
${bold('OPTIONS FOR baseline')}
|
|
520
545
|
Records what the prompts in a directory cost right now, to a file you commit.
|
|
521
|
-
Then "check" fails the build when the repository drifts past it
|
|
546
|
+
Then "check" fails the build when the repository drifts past it: the question
|
|
522
547
|
budgets cannot answer, because a repository at 95% of every budget passes
|
|
523
548
|
forever while a pull request adds four hundred tokens across a dozen files.
|
|
524
549
|
|
|
@@ -527,7 +552,7 @@ ${bold('OPTIONS FOR baseline')}
|
|
|
527
552
|
--model, --calls, --output-tokens, --cache-hit-rate, --batch
|
|
528
553
|
The scenario the monthly figure is recorded under. It is
|
|
529
554
|
recorded so a later comparison can say whether the money is
|
|
530
|
-
comparable
|
|
555
|
+
comparable: the gate itself is in tokens, so a repriced
|
|
531
556
|
model never fails a build on its own.
|
|
532
557
|
--exact-tokens Exact counts (needs ANTHROPIC_API_KEY).
|
|
533
558
|
--json Result as JSON.
|
|
@@ -536,7 +561,7 @@ ${bold('OPTIONS FOR baseline')}
|
|
|
536
561
|
writing the thing you would fix the failure with is a loop.
|
|
537
562
|
|
|
538
563
|
Given a directory it checks every prompt inside it against the "budgets"
|
|
539
|
-
patterns in ${bold('trazum.config.json')}
|
|
564
|
+
patterns in ${bold('trazum.config.json')}: one CI step for a whole repository of
|
|
540
565
|
prompts. A file no pattern covers is listed as unbudgeted rather than
|
|
541
566
|
skipped quietly, and a run where nothing at all was budgeted is an error:
|
|
542
567
|
"0 failures" from a check that measured nothing is the most misleading
|
|
@@ -546,7 +571,7 @@ ${bold('OPTIONS FOR ladder')}
|
|
|
546
571
|
--since <when> Read only calls in this window. A UTC day
|
|
547
572
|
--until <when> (2026-08-14), a full ISO 8601 timestamp, a
|
|
548
573
|
relative window (7d, 24h) or "now". Calls with no
|
|
549
|
-
"ts" cannot be placed and are left out
|
|
574
|
+
"ts" cannot be placed and are left out, counted
|
|
550
575
|
out loud, never dropped silently.
|
|
551
576
|
--label <name> Judge one workload's ladder rather than the whole
|
|
552
577
|
log. A label that matches nothing is an error
|
|
@@ -561,9 +586,9 @@ ${bold('OPTIONS FOR owners')}
|
|
|
561
586
|
${bold('OPTIONS FOR profile')}
|
|
562
587
|
--against <log.jsonl> Compare this bill to a previous log. Positive
|
|
563
588
|
means the bill grew; drivers are ranked by their
|
|
564
|
-
contribution to the change. No period is assumed
|
|
589
|
+
contribution to the change. No period is assumed,
|
|
565
590
|
judge the call counts before judging the money.
|
|
566
|
-
--label <name> Profile only the calls carrying this label
|
|
591
|
+
--label <name> Profile only the calls carrying this label, the
|
|
567
592
|
drill-down once the full report named a suspect.
|
|
568
593
|
A label that matches nothing is an error naming
|
|
569
594
|
the labels that exist. With --against, both logs
|
|
@@ -577,16 +602,16 @@ ${bold('OPTIONS FOR profile')}
|
|
|
577
602
|
an error, not a flag that silently gates nothing.
|
|
578
603
|
--max-cache-loss-usd <n> Exit 1 when caching added more than n dollars to
|
|
579
604
|
this bill. Reads the worst case when the log did
|
|
580
|
-
not record the write TTL
|
|
605
|
+
not record the write TTL: a gate reading the
|
|
581
606
|
flattering half would pass the bills it exists
|
|
582
|
-
to catch
|
|
607
|
+
to catch, and says which claim fired.
|
|
583
608
|
--max-day-usd <n> Fail when any single UTC day inside the log spent
|
|
584
609
|
more than this. A month under budget can hide the
|
|
585
610
|
afternoon a loop burned a quarter of it. A log
|
|
586
611
|
with no timestamps fails: not measured is not
|
|
587
612
|
under budget.
|
|
588
613
|
--max-session-usd <n> Fail when any single conversation in the log
|
|
589
|
-
cost more than this
|
|
614
|
+
cost more than this, the unit an agent product
|
|
590
615
|
blows up in. A log with no sessions fails: not
|
|
591
616
|
measured is not under budget.
|
|
592
617
|
--since <when> Profile only calls at or after this moment. A UTC
|
|
@@ -594,22 +619,22 @@ ${bold('OPTIONS FOR profile')}
|
|
|
594
619
|
relative window (7d, 24h) or "now";
|
|
595
620
|
--until with a bare date includes that whole day.
|
|
596
621
|
Calls with no "ts" cannot be placed and are left
|
|
597
|
-
out
|
|
622
|
+
out, counted out loud, never dropped silently.
|
|
598
623
|
With --against, both logs get the same window.
|
|
599
624
|
--what-if <model> Price these exact calls on another model. The
|
|
600
|
-
same token counts at a different rate card
|
|
625
|
+
same token counts at a different rate card,
|
|
601
626
|
multiplication, not advice, and it says so.
|
|
602
627
|
Calls larger than that model's context window
|
|
603
628
|
are named as impossible, not priced as cheap.
|
|
604
629
|
--markdown-out <file> Also write the report as Markdown, for a CI job
|
|
605
630
|
summary or a pull request comment.
|
|
606
631
|
--html-out <file> Also write the report as one self-contained HTML
|
|
607
|
-
file
|
|
632
|
+
file, the one to email to whoever pays the bill.
|
|
608
633
|
Same figures as --json, caveats rendered as
|
|
609
634
|
prominently as the totals they qualify.
|
|
610
635
|
--csv-shape <shape> Which table --csv-out writes: slice (default),
|
|
611
636
|
day, hour, or model-day (one row per day and
|
|
612
|
-
model
|
|
637
|
+
model, charts the mix moving). One row shape
|
|
613
638
|
per file, so nothing has to be filtered before
|
|
614
639
|
it can be summed.
|
|
615
640
|
--csv-out <file> Also write the report as CSV, one row per label
|
|
@@ -620,20 +645,20 @@ ${bold('OPTIONS FOR profile')}
|
|
|
620
645
|
--pricing <file> Local price overlay, as everywhere else.
|
|
621
646
|
--json The full report as data, including the levers.
|
|
622
647
|
|
|
623
|
-
Takes a log file or a directory of them
|
|
648
|
+
Takes a log file or a directory of them, rotated daily logs are read in
|
|
624
649
|
name order as one bill, and how many were read is stated. Gzipped files
|
|
625
650
|
(.jsonl.gz and the rest) are read too, because that is what a rotated log
|
|
626
651
|
looks like a day later; one that will not decompress is an error naming the
|
|
627
652
|
file, never a bill quietly missing a day.
|
|
628
653
|
|
|
629
654
|
Reads what the provider actually charged. Optional fields unlock findings:
|
|
630
|
-
"label" (which workload), "session" (which conversation
|
|
655
|
+
"label" (which workload), "session" (which conversation, grouped by, never
|
|
631
656
|
printed), "stop_reason"/"finish_reason" (answers cut off at max_tokens).
|
|
632
657
|
|
|
633
658
|
${bold('OPTIONS FOR plan')}
|
|
634
659
|
--min-usd <n> Leave out actions worth less than n dollars. How
|
|
635
660
|
many were left out is stated, never silent.
|
|
636
|
-
-o, --out <file> Save the plan as dated JSON
|
|
661
|
+
-o, --out <file> Save the plan as dated JSON: the file
|
|
637
662
|
"trazum verify" will later hold it to.
|
|
638
663
|
--markdown-out <file> Also write the plan as Markdown, for a CI job
|
|
639
664
|
summary or a pull request comment.
|
|
@@ -642,8 +667,8 @@ ${bold('OPTIONS FOR plan')}
|
|
|
642
667
|
|
|
643
668
|
Reads a usage log and turns the report's findings into a ranked plan: route
|
|
644
669
|
this, batch that, fix the truncation pair, look at the cache. The money is
|
|
645
|
-
composed correctly
|
|
646
|
-
never summed
|
|
670
|
+
composed correctly, route and batch on the same slice arrive combined,
|
|
671
|
+
never summed, and every action names what the log cannot confirm, because
|
|
647
672
|
a plan that hides its assumptions is advice pretending to be arithmetic.
|
|
648
673
|
Projected savings and money already spent are separate totals throughout.
|
|
649
674
|
|
|
@@ -656,8 +681,8 @@ ${bold('OPTIONS FOR serve')}
|
|
|
656
681
|
the store; without this, every ceiling in
|
|
657
682
|
"limits" answers cannot-tell.
|
|
658
683
|
|
|
659
|
-
Answers the two questions that matter at call time
|
|
660
|
-
and is there budget left
|
|
684
|
+
Answers the two questions that matter at call time, what will this cost,
|
|
685
|
+
and is there budget left, in single-digit milliseconds, so an agent or a
|
|
661
686
|
wrapper can ask before it spends rather than reading a report afterwards.
|
|
662
687
|
|
|
663
688
|
POST /cost with {"model": "...", "inputTokens": n, "outputTokens": n};
|
|
@@ -669,7 +694,7 @@ ${bold('OPTIONS FOR serve')}
|
|
|
669
694
|
It listens on 127.0.0.1 and nowhere else, and there is no flag to change
|
|
670
695
|
that: a cost oracle on a network interface holds a company's spend, its
|
|
671
696
|
model mix and its budgets, and answers whoever asks. There is no auth for
|
|
672
|
-
the same reason there is no --host
|
|
697
|
+
the same reason there is no --host: a token checked over loopback is
|
|
673
698
|
theatre, and the honest posture is a surface small enough not to need one.
|
|
674
699
|
|
|
675
700
|
With no store and no budget it still prices the call and says the budget
|
|
@@ -688,15 +713,15 @@ ${bold('OPTIONS FOR watch')}
|
|
|
688
713
|
instead of the store.
|
|
689
714
|
--json The cycle as data: crossings, abstentions, gap.
|
|
690
715
|
|
|
691
|
-
Evaluates the spend gates from your config
|
|
692
|
-
maxCacheLossUsd
|
|
716
|
+
Evaluates the spend gates from your config, maxUsd, maxDayUsd,
|
|
717
|
+
maxCacheLossUsd, against what has been measured, and tells you the
|
|
693
718
|
afternoon it happens rather than three weeks later. Exits 1 when something
|
|
694
719
|
crossed, so cron mails it and CI fails.
|
|
695
720
|
|
|
696
721
|
An alert fires on a measured crossing and never on a projection: "you have
|
|
697
722
|
spent $412 of a $400 budget" is a fact and "you will exceed" is a forecast,
|
|
698
723
|
which this tool does not make at any window length. A day still being
|
|
699
|
-
measured is reported as not yet judgeable rather than passed
|
|
724
|
+
measured is reported as not yet judgeable rather than passed, but a day
|
|
700
725
|
already over budget fires whatever the hour, because it does not become less
|
|
701
726
|
over budget at midnight.
|
|
702
727
|
|
|
@@ -715,8 +740,8 @@ ${bold('OPTIONS FOR store')}
|
|
|
715
740
|
|
|
716
741
|
Says what the local store holds: how many measurements, over what span, per
|
|
717
742
|
provider, and what a prune would take. The store keeps aggregates and
|
|
718
|
-
billing fields
|
|
719
|
-
credential
|
|
743
|
+
billing fields, never prompt text, never completion text, never a
|
|
744
|
+
credential, so it is a file a team can back up without a privacy review.
|
|
720
745
|
|
|
721
746
|
Pruning is the one operation here that destroys something, so it refuses to
|
|
722
747
|
run without a retention policy: set "store": {"keepDays": 90} in the config
|
|
@@ -731,7 +756,7 @@ ${bold('OPTIONS FOR connect')}
|
|
|
731
756
|
variable the key would come from. Sends nothing
|
|
732
757
|
and needs no credential.
|
|
733
758
|
--payload <file> Price a usage payload you already have, instead of
|
|
734
|
-
pulling one. No credential, no network
|
|
759
|
+
pulling one. No credential, no network, the same
|
|
735
760
|
arithmetic on the same shape.
|
|
736
761
|
--store Keep what was pulled in the local store, so the
|
|
737
762
|
next run does not download it again and "trazum
|
|
@@ -749,8 +774,8 @@ ${bold('OPTIONS FOR connect')}
|
|
|
749
774
|
|
|
750
775
|
These APIs serve sums over a window, not one row per call, so a connected
|
|
751
776
|
report is a restricted one and says so: the totals, the model split, the day
|
|
752
|
-
series and the cache verdict are all available, and the per-call findings
|
|
753
|
-
input shapes, truncation retries, conversations, context pressure
|
|
777
|
+
series and the cache verdict are all available, and the per-call findings,
|
|
778
|
+
input shapes, truncation retries, conversations, context pressure, are
|
|
754
779
|
listed as unavailable with what would unlock them. A rate limit, a page cap
|
|
755
780
|
or an expired cursor returns what arrived with the gap named, never a total
|
|
756
781
|
that quietly describes less traffic than you asked about.
|
|
@@ -759,15 +784,15 @@ ${bold('OPTIONS FOR history')}
|
|
|
759
784
|
--store Build the series from the local store instead of
|
|
760
785
|
a directory of stored reports. Bucketed sources
|
|
761
786
|
carry no label, so the label series is absent and
|
|
762
|
-
said to be
|
|
787
|
+
said to be: the model-share and cache-share
|
|
763
788
|
series are what a series exists for, and both
|
|
764
789
|
work.
|
|
765
790
|
--markdown-out <file> Also write the series as Markdown, for a CI job
|
|
766
791
|
summary or a pull request comment.
|
|
767
792
|
--json The history as data.
|
|
768
793
|
|
|
769
|
-
Takes a directory of stored reports
|
|
770
|
-
writes
|
|
794
|
+
Takes a directory of stored reports, the --json documents profile already
|
|
795
|
+
writes, plus any saved plans beside them, and builds the series no
|
|
771
796
|
pairwise comparison can see: a workload climbing a little every period, a
|
|
772
797
|
model share rising since a date, a cache share decaying slowly enough that
|
|
773
798
|
no single week's report called it a finding, and the same action planned
|
|
@@ -779,7 +804,7 @@ ${bold('OPTIONS FOR verify')}
|
|
|
779
804
|
--against <log|dir> The newer usage log the plan is held to. Required.
|
|
780
805
|
--gate Exit 1 when an action did not produce what the
|
|
781
806
|
plan promised, or its fields stopped being
|
|
782
|
-
recorded
|
|
807
|
+
recorded, "not recorded" must not read as
|
|
783
808
|
"fixed". A vanished workload fails nothing.
|
|
784
809
|
--markdown-out <file> Also write the verdicts as Markdown, for a CI
|
|
785
810
|
job summary or a pull request comment.
|
|
@@ -787,7 +812,7 @@ ${bold('OPTIONS FOR verify')}
|
|
|
787
812
|
--json The verification as data.
|
|
788
813
|
|
|
789
814
|
Holds a saved plan to the log that came after it, with three outcomes and
|
|
790
|
-
never two: the change arrived, it did not, or it cannot be told
|
|
815
|
+
never two: the change arrived, it did not, or it cannot be told, because
|
|
791
816
|
the workload vanished, the fields the detection needs stopped being
|
|
792
817
|
recorded, or tokens cannot say which tier billed them. Differences carry
|
|
793
818
|
the world's measured movement (calls, output per call) from the plan's own
|
|
@@ -806,7 +831,7 @@ ${bold('OPTIONS FOR route')}
|
|
|
806
831
|
|
|
807
832
|
Reads a usage log, finds the slice where routing to a cheaper model is worth
|
|
808
833
|
the most, and measures whether that model still does the job. The same prompt
|
|
809
|
-
goes to both, and the original runs twice per case
|
|
834
|
+
goes to both, and the original runs twice per case, so the verdict is judged
|
|
810
835
|
against that model's own variance rather than a threshold somebody picked.
|
|
811
836
|
|
|
812
837
|
Costs three provider calls per case and needs TRAZUM_LLM_* configured.
|
|
@@ -833,7 +858,7 @@ ${bold('OPTIONS FOR diff')}
|
|
|
833
858
|
is given: deciding that growth is unacceptable is your call, not ours.
|
|
834
859
|
|
|
835
860
|
${bold('trazum where')}
|
|
836
|
-
Says which provider a file's prompts are actually sent to, and how it knows
|
|
861
|
+
Says which provider a file's prompts are actually sent to, and how it knows,
|
|
837
862
|
an SDK import, a base URL, a quoted model id, or "model=" on a trazum:prompt
|
|
838
863
|
marker. Every answer names the line it came from.
|
|
839
864
|
|
|
@@ -845,7 +870,7 @@ ${bold('trazum where')}
|
|
|
845
870
|
are all called through the OpenAI SDK with a different base_url, so treating
|
|
846
871
|
that as a contradiction would refuse to price an ordinary client.
|
|
847
872
|
|
|
848
|
-
With no file, it reports only which tool Trazum is running inside
|
|
873
|
+
With no file, it reports only which tool Trazum is running inside, and warns
|
|
849
874
|
when that tool bills by subscription, because a monthly saving is arithmetic
|
|
850
875
|
about tokens there, not money you get back.
|
|
851
876
|
|
|
@@ -857,14 +882,14 @@ ${bold('CONFIG FILE')}
|
|
|
857
882
|
usage { model, callsPerMonth, avgOutputTokens, cacheHitRate, batchEligible }
|
|
858
883
|
budgets { "prompts/**": 2000, "prompts/system.txt": 4000 }
|
|
859
884
|
baseline { "path": "trazum.baseline.json", "maxGrowthTokens": 0, "maxGrowthPct": 5 }
|
|
860
|
-
pricing "./prices.json"
|
|
861
|
-
labels { "support-rag": "prompts/support.txt" }
|
|
885
|
+
pricing "./prices.json": local price corrections, see below
|
|
886
|
+
labels { "support-rag": "prompts/support.txt" }: which prompt file each
|
|
862
887
|
usage-log label sends, so "trazum profile" can read the file and
|
|
863
888
|
say why a failing cache fails
|
|
864
|
-
spend { "maxUsd": 200, "byLabel": { "chat": 40 } }
|
|
889
|
+
spend { "maxUsd": 200, "byLabel": { "chat": 40 } }: money budgets for
|
|
865
890
|
"trazum profile", in dollars. A budgeted label with no calls in
|
|
866
891
|
the log is reported as not measured, never as a pass
|
|
867
|
-
limits { "dayUsd": 25, "sessionUsd": 1.5, "byLabel": { "chat": 5 } }
|
|
892
|
+
limits { "dayUsd": 25, "sessionUsd": 1.5, "byLabel": { "chat": 5 } }:
|
|
868
893
|
the enforcement policy: USD ceilings judged before a call is
|
|
869
894
|
made, at whichever door it arrives (the gateway's 402, "serve",
|
|
870
895
|
spend_guard over MCP). Separate from "spend" because a report
|
|
@@ -873,17 +898,17 @@ ${bold('CONFIG FILE')}
|
|
|
873
898
|
outage written as a policy
|
|
874
899
|
owners { "patterns": { "payments": ["billing-*"] },
|
|
875
900
|
"shared": { "search": { "payments": 0.6, "support": 0.4 } },
|
|
876
|
-
"budgets": { "payments": 400 } }
|
|
901
|
+
"budgets": { "payments": 400 } }: whose budget each workload
|
|
877
902
|
lands on. Spend matching no owner is its own line and is NEVER
|
|
878
903
|
divided between the others; a shared split must sum to 1, and the
|
|
879
904
|
rule travels with the report so the argument is about the rule
|
|
880
905
|
ladders { "support": { "tiers": ["claude-haiku-4-5", "claude-opus-5"],
|
|
881
|
-
"escalateOn": ["escalated"] } }
|
|
906
|
+
"escalateOn": ["escalated"] } }: cheap model first, escalate a
|
|
882
907
|
recorded failure to a dearer one. Both fields required.
|
|
883
908
|
"trazum ladder <log>" prints the break-even escalation rate
|
|
884
909
|
beside the measured one: an escalation pays twice, so above that
|
|
885
910
|
rate the ladder costs more than never having built it
|
|
886
|
-
outcomes { "values": ["resolved", "escalated"], "success": ["resolved"] }
|
|
911
|
+
outcomes { "values": ["resolved", "escalated"], "success": ["resolved"] }:
|
|
887
912
|
your own vocabulary for what happened, and which of it counts as
|
|
888
913
|
a win. Both required: which words mean success is a judgement
|
|
889
914
|
about your product rather than your bill, and this tool has no
|
|
@@ -891,27 +916,27 @@ ${bold('CONFIG FILE')}
|
|
|
891
916
|
value in a log that "values" never declares is named as
|
|
892
917
|
undeclared, never counted as a failure
|
|
893
918
|
waive [{ "gate": "maxUsd", "reason": "August migration", "until":
|
|
894
|
-
"2026-09-15" }]
|
|
919
|
+
"2026-09-15" }]: a gate failure decided about, on the record.
|
|
895
920
|
All three fields required: a waiver with no end date is a
|
|
896
921
|
finding deleted with extra steps. Waived failures still print,
|
|
897
922
|
and the day the waiver expires the gate fails again
|
|
898
923
|
|
|
899
924
|
Flags beat the config; the config beats the defaults. Budgets resolve to the
|
|
900
|
-
most specific matching pattern
|
|
925
|
+
most specific matching pattern: most literal characters wins. A boolean the
|
|
901
926
|
config switched on comes back off with --no-<flag>, e.g. --no-batch.
|
|
902
927
|
|
|
903
928
|
${bold('budgets')} is a ceiling; ${bold('baseline')} is a gate. One asks whether a file fits,
|
|
904
929
|
the other whether the repository got worse than the commit somebody recorded
|
|
905
930
|
with "trazum baseline". A repository at 95% of every budget passes forever
|
|
906
931
|
while a pull request adds four hundred tokens across a dozen files. With
|
|
907
|
-
baseline in the config, "check" on a directory reads it and gates on it
|
|
932
|
+
baseline in the config, "check" on a directory reads it and gates on it, no
|
|
908
933
|
flag, because a gate you have to remember to pass an argument to runs in the
|
|
909
934
|
author's terminal and not in CI. Thresholds are in tokens, never dollars: a
|
|
910
935
|
repriced model would otherwise fail a build for a change nobody made.
|
|
911
936
|
|
|
912
937
|
A config that will not validate is an error, including an unknown key. A
|
|
913
938
|
lenient parser would silently restore defaults, and for a budget the default
|
|
914
|
-
is "no budget"
|
|
939
|
+
is "no budget", a green build for a prompt nobody measured.
|
|
915
940
|
|
|
916
941
|
--config <file> Use this config instead of searching for one.
|
|
917
942
|
|
|
@@ -935,7 +960,7 @@ ${bold('PRICES')}
|
|
|
935
960
|
--pricing-live Take prices from OpenRouter instead of the bundled
|
|
936
961
|
table: today's figures for hundreds of models across
|
|
937
962
|
dozens of providers. Opt-in, because it is a network
|
|
938
|
-
call
|
|
963
|
+
call: the deterministic core never makes one.
|
|
939
964
|
A --pricing file wins over this.
|
|
940
965
|
|
|
941
966
|
What that source does not publish is whether a model
|
|
@@ -959,7 +984,7 @@ ${bold('OPTIONAL LLM')}
|
|
|
959
984
|
|
|
960
985
|
${bold('LANGUAGE')}
|
|
961
986
|
The report language follows --locale, then TRAZUM_LOCALE, then LANG, and last
|
|
962
|
-
the config file
|
|
987
|
+
the config file, so a project can set the language its CI logs read in
|
|
963
988
|
without overriding the language of whoever is at the keyboard. It changes the
|
|
964
989
|
report only: the same prompt always optimises the same way.
|
|
965
990
|
|
|
@@ -984,7 +1009,7 @@ ${bold('EXAMPLES')}
|
|
|
984
1009
|
|
|
985
1010
|
errors: {
|
|
986
1011
|
livePricingFailed: (url: string, detail: string) =>
|
|
987
|
-
`Could not load live prices from ${url}: ${detail}. The bundled prices are still there
|
|
1012
|
+
`Could not load live prices from ${url}: ${detail}. The bundled prices are still there. Drop --pricing-live to use them.`,
|
|
988
1013
|
optionNeedsValue: (name) => `Option --${name} needs a value.`,
|
|
989
1014
|
mustBeNonNegative: (name, raw) =>
|
|
990
1015
|
`--${name} must be a non-negative number (received: "${raw}").`,
|
|
@@ -997,7 +1022,7 @@ ${bold('EXAMPLES')}
|
|
|
997
1022
|
allLabelsNeedsLog: () =>
|
|
998
1023
|
'--all-labels ranks prompts by measured traffic, so it needs --from-log <usage.jsonl>. Without a log every saving would be multiplied by the same typed guess, which ranks prompts by length and calls it a priority.',
|
|
999
1024
|
allLabelsNeedsMap: () =>
|
|
1000
|
-
'--all-labels reads the "labels" map in trazum.config.json
|
|
1025
|
+
'--all-labels reads the "labels" map in trazum.config.json, label to prompt file, and this config has none. Map at least one workload to its prompt.',
|
|
1001
1026
|
fromLogConflict: (flag) =>
|
|
1002
1027
|
`--from-log measures the figure --${flag} types, and merging a measurement with a guess produces a number that is neither. Pass one or the other.`,
|
|
1003
1028
|
fromLogNeedsLabel: (available) =>
|
|
@@ -1005,7 +1030,7 @@ ${bold('EXAMPLES')}
|
|
|
1005
1030
|
fromLogAmbiguousLabel: (target, labels) =>
|
|
1006
1031
|
`${target} is mapped to more than one label in trazum.config.json (${labels}), so --from-log cannot pick one silently. Pass --label.`,
|
|
1007
1032
|
fromLogLabelEmpty: (label, available) =>
|
|
1008
|
-
`No priced call in this log carries the label "${label}", so there is nothing to measure
|
|
1033
|
+
`No priced call in this log carries the label "${label}", so there is nothing to measure: a zero-call profile would price this change as worthless rather than as unmeasured. Labels with traffic: ${available}.`,
|
|
1009
1034
|
unknownRuleInDisable: (id) => `Unknown rule in --disable: "${id}". Full list: trazum rules`,
|
|
1010
1035
|
unknownCommand: (command) => `Unknown command: "${command}". Try "trazum --help".`,
|
|
1011
1036
|
missingInputFile: () => 'Missing input file. Use "-" to read from standard input.',
|
|
@@ -1021,15 +1046,15 @@ ${bold('EXAMPLES')}
|
|
|
1021
1046
|
* Named, and told what it cannot do rather than merely that it will not.
|
|
1022
1047
|
*
|
|
1023
1048
|
* The alternative on offer has to be somebody else's tooling, because
|
|
1024
|
-
* Trazum has none for this family
|
|
1049
|
+
* Trazum has none for this family, saying so is the whole refusal. A
|
|
1025
1050
|
* refusal never arrives bare.
|
|
1026
1051
|
*/
|
|
1027
1052
|
exactTokensWrongFamily: (model, provider) =>
|
|
1028
1053
|
`--exact-tokens counts with Anthropic's endpoint, which counts Claude's tokenizer${
|
|
1029
1054
|
provider === null
|
|
1030
|
-
? `, and the catalogue records no provider for "${model}"
|
|
1055
|
+
? `, and the catalogue records no provider for "${model}", so which tokenizer it uses is not something Trazum knows`
|
|
1031
1056
|
: `, and ${model} is a model from ${provider}`
|
|
1032
|
-
}. Counting it there would either be refused upstream or return a number for a different tokenizer, and this tool will not label that exact. Drop --exact-tokens for the estimate
|
|
1057
|
+
}. Counting it there would either be refused upstream or return a number for a different tokenizer, and this tool will not label that exact. Drop --exact-tokens for the estimate, which is honest about being one, or count with ${
|
|
1033
1058
|
provider === null ? "that model's" : provider + "'s"
|
|
1034
1059
|
} own tooling.`,
|
|
1035
1060
|
checkNeedsMaxTokens: () => 'trazum check needs --max-tokens <n>.',
|
|
@@ -1057,10 +1082,14 @@ ${bold('EXAMPLES')}
|
|
|
1057
1082
|
|
|
1058
1083
|
report: {
|
|
1059
1084
|
inputTokens: () => 'Input tokens',
|
|
1060
|
-
estimated: (offFamily) =>
|
|
1061
|
-
offFamily === null
|
|
1062
|
-
|
|
1063
|
-
|
|
1085
|
+
estimated: (offFamily, band, measuredOff) => {
|
|
1086
|
+
if (offFamily === null) return ` (estimated, ±${band}%)`;
|
|
1087
|
+
return measuredOff === null
|
|
1088
|
+
? ` (estimated: the counter is calibrated on Claude, not ${offFamily}, and nobody has`
|
|
1089
|
+
+ ' measured how far off it is there)'
|
|
1090
|
+
: ` (estimated: the counter is calibrated on Claude, not ${offFamily}, where it has`
|
|
1091
|
+
+ ` measured up to ${measuredOff}% out)`;
|
|
1092
|
+
},
|
|
1064
1093
|
exactCount: () => ' (exact count)',
|
|
1065
1094
|
rulesApplied: () => 'Rules applied',
|
|
1066
1095
|
nothingToTrim: () => ' No rule found anything to trim.',
|
|
@@ -1070,14 +1099,14 @@ ${bold('EXAMPLES')}
|
|
|
1070
1099
|
// and naming the coverage cannot be wrong.
|
|
1071
1100
|
dictionaryCoverage: (languages) =>
|
|
1072
1101
|
` The phrase dictionaries cover ${languages}. A prompt in another language `
|
|
1073
|
-
+ 'is not necessarily efficient
|
|
1102
|
+
+ 'is not necessarily efficient: it may just be one Trazum cannot read yet.',
|
|
1074
1103
|
dictionaryUnreviewed: (languages) =>
|
|
1075
1104
|
` Of those, ${languages} carry entries nobody here reads: written by the same `
|
|
1076
1105
|
+ 'process that wrote the rules, never agreed by a speaker of the language.',
|
|
1077
1106
|
dictionaryAppliedUnreviewed: (language) =>
|
|
1078
1107
|
` These changes came from the ${language} dictionary, which nobody here reads. `
|
|
1079
1108
|
+ 'Its entries were written by the same process that wrote the rules and never '
|
|
1080
|
-
+ 'agreed by a speaker
|
|
1109
|
+
+ 'agreed by a speaker. Read the diff before trusting it.',
|
|
1081
1110
|
levelAggressive: () => '[aggressive]',
|
|
1082
1111
|
levelSafe: () => '[safe]',
|
|
1083
1112
|
ruleHits: (hits, tokensSaved) => `(${hits}×, ~${tokensSaved} tokens)`,
|
|
@@ -1094,37 +1123,37 @@ ${bold('EXAMPLES')}
|
|
|
1094
1123
|
costWith: (modelName) => `Cost with ${modelName}`,
|
|
1095
1124
|
usageLine: (calls, outputTokens, batch) =>
|
|
1096
1125
|
`${calls} calls/month · ${outputTokens} output tokens per call${batch ? ' · Batch API' : ''}`,
|
|
1097
|
-
allLabelsHeading: (count) => `Every mapped prompt against its own measured traffic
|
|
1126
|
+
allLabelsHeading: (count) => `Every mapped prompt against its own measured traffic, ${count} ranked by what the change is worth`,
|
|
1098
1127
|
allLabelsRow: (saving) => `${saving}/month if optimised`,
|
|
1099
1128
|
allLabelsRowPeriod: (saving) => `${saving} over the measured period if optimised`,
|
|
1100
1129
|
allLabelsFooter: () =>
|
|
1101
1130
|
'Ranked by measured traffic, not by prompt length: a big prompt on a dead workload is worth less than a small one on a busy one. Each figure is this prompt\'s token delta at its own label\'s measured rate.',
|
|
1102
1131
|
allLabelsUnmapped: (label, usd) =>
|
|
1103
|
-
`${label} carries ${usd} of measured spend and no prompt file is mapped to it
|
|
1132
|
+
`${label} carries ${usd} of measured spend and no prompt file is mapped to it: the workload nobody can optimise because nobody said where it lives. Map it under "labels" in trazum.config.json.`,
|
|
1104
1133
|
allLabelsDead: (label, path) =>
|
|
1105
|
-
`${label} is mapped to ${path} and has no traffic in this log
|
|
1134
|
+
`${label} is mapped to ${path} and has no traffic in this log, a retired workload, a renamed label, or a typo that has been silently doing nothing.`,
|
|
1106
1135
|
allLabelsUnreadable: (label, path) =>
|
|
1107
1136
|
`${label} is mapped to ${path}, which could not be read. The mapping exists; the file does not.`,
|
|
1108
1137
|
usageLineMeasured: (calls, days, scaled, outputTokens, batch) =>
|
|
1109
|
-
`${calls} calls measured over ${days} days
|
|
1138
|
+
`${calls} calls measured over ${days} days, ${scaled}/month at that rate · ${outputTokens} output tokens per call, measured${batch ? ' · Batch API' : ''}`,
|
|
1110
1139
|
usageLineMeasuredPeriod: (calls, days, outputTokens, batch) =>
|
|
1111
1140
|
`${calls} calls measured${days === null ? ' (the log carries no clock)' : ` over ${days} days`} · ${outputTokens} output tokens per call, measured${batch ? ' · Batch API' : ''}`,
|
|
1112
1141
|
measuredModelShare: (model, share, count) =>
|
|
1113
1142
|
`This label ran on ${count} models; the figures use ${model}, which carried ${share} of its spend.`,
|
|
1114
1143
|
measuredNoOutput: () =>
|
|
1115
|
-
'No call in this slice recorded output tokens, so the output half of every figure below is $0 measured
|
|
1144
|
+
'No call in this slice recorded output tokens, so the output half of every figure below is $0 measured, not $0 assumed.',
|
|
1116
1145
|
perPeriodSaving: (saving, pct) => `saving ${saving} over the measured period (${pct}%)`,
|
|
1117
1146
|
periodNotScaled: (days) =>
|
|
1118
1147
|
days === null
|
|
1119
1148
|
? 'Not scaled to a month: the log carries no clock, so there is no rate to scale. These figures cover exactly the calls measured.'
|
|
1120
|
-
: `Not scaled to a month: ${days} days is under the week a scaling needs
|
|
1149
|
+
: `Not scaled to a month: ${days} days is under the week a scaling needs, and being shorter than one weekly cycle multiplies whichever part of the cycle it caught. These figures cover exactly the period measured.`,
|
|
1121
1150
|
perMonthSaving: (saving, pct) => `saving ${saving}/month (${pct}%)`,
|
|
1122
1151
|
beyondShortening: () => 'Beyond shortening the prompt',
|
|
1123
1152
|
biggestLever: () => 'Start here:',
|
|
1124
1153
|
biggestLeverDetail: (title, amount, times) =>
|
|
1125
1154
|
// The title keeps its case: lowercasing it turned "Claude Opus 5" into
|
|
1126
1155
|
// "claude opus 5", which is a product name mangled to fit a sentence.
|
|
1127
|
-
`"${title}"
|
|
1156
|
+
`"${title}", ${amount}/month` +
|
|
1128
1157
|
(times !== null && times >= 2 ? `, ${times}× what the rules saved.` : '.'),
|
|
1129
1158
|
perMonthSuffix: (amount) => ` ~${amount}/month`,
|
|
1130
1159
|
diff: () => 'Diff',
|
|
@@ -1136,17 +1165,17 @@ ${bold('EXAMPLES')}
|
|
|
1136
1165
|
// decided on the string rather than on a number that is no longer here.
|
|
1137
1166
|
tokensSaved: (tokens) => `${tokens} token${tokens === '1' ? '' : 's'} back, every call.`,
|
|
1138
1167
|
windowNegligible: (tokens, model, window) =>
|
|
1139
|
-
`${tokens} tokens of ${model}'s ${window}-token window
|
|
1168
|
+
`${tokens} tokens of ${model}'s ${window}-token window, under a tenth of a percent, so the window is not what constrains this prompt.`,
|
|
1140
1169
|
windowUnmoved: (share, model, window) =>
|
|
1141
1170
|
`${share} of ${model}'s ${window}-token window, before and after: this change is too small to move it.`,
|
|
1142
1171
|
beyondThisPromptTokensOnly: () =>
|
|
1143
1172
|
'Shortening a prompt is the smallest lever there is: measured on an ordinary support prompt, the rules recover about 1% of a monthly bill. If any of your prompts go to a metered API, "trazum profile <usage.jsonl>" reads what the provider actually charged and prices the levers that are not the prompt. Recording that log is a few lines and it never contains prompt text.',
|
|
1144
1173
|
beyondThisPrompt: () =>
|
|
1145
|
-
'Shortening a prompt is the smallest lever there is: measured on an ordinary support prompt, the rules recover about 1% of a monthly bill. On a metered API the things that move 40% to 80% are which model the call goes to, the Batch API, prompt caching, and what re-sending the conversation costs
|
|
1174
|
+
'Shortening a prompt is the smallest lever there is: measured on an ordinary support prompt, the rules recover about 1% of a monthly bill. On a metered API the things that move 40% to 80% are which model the call goes to, the Batch API, prompt caching, and what re-sending the conversation costs, and "trazum profile <usage.jsonl>" prices all four from what the provider actually charged. Recording that log is a few lines and it never contains prompt text.',
|
|
1146
1175
|
windowUse: (before, after, model, window) =>
|
|
1147
|
-
`Context window: ${before} → ${after} of ${model}'s ${window} tokens
|
|
1176
|
+
`Context window: ${before} → ${after} of ${model}'s ${window} tokens, room the conversation gets instead.`,
|
|
1148
1177
|
tokensOnlyAskedFor: () =>
|
|
1149
|
-
'You named a scenario, and it was not priced: Trazum is running somewhere that bills by subscription, so there is no bill here to reduce. Add --cost to price it anyway
|
|
1178
|
+
'You named a scenario, and it was not priced: Trazum is running somewhere that bills by subscription, so there is no bill here to reduce. Add --cost to price it anyway: the host says where Trazum runs, not where your prompt goes.',
|
|
1150
1179
|
tokensOnlyCost: () => 'Pass --cost if this prompt is bound for a metered API.',
|
|
1151
1180
|
pricingOverlaid: (models, lastReviewed) =>
|
|
1152
1181
|
`Prices for ${models} came from a local overlay reviewed ${lastReviewed}, not from the bundled catalogue.`,
|
|
@@ -1197,7 +1226,7 @@ ${bold('EXAMPLES')}
|
|
|
1197
1226
|
prompts: (count) =>
|
|
1198
1227
|
`${count} prompt ${count === 1 ? 'file' : 'files'} found.`,
|
|
1199
1228
|
noPrompts: () =>
|
|
1200
|
-
'No prompt files found. Directory mode reads .txt, .md, .prompt and .tmpl by default
|
|
1229
|
+
'No prompt files found. Directory mode reads .txt, .md, .prompt and .tmpl by default. Set "extensions" if yours are named otherwise.',
|
|
1201
1230
|
sourcesTruncated: (cap) =>
|
|
1202
1231
|
`Stopped after ${cap} source files, so a provider named further down was not seen.`,
|
|
1203
1232
|
usageFound: (kind, where) =>
|
|
@@ -1207,7 +1236,7 @@ ${bold('EXAMPLES')}
|
|
|
1207
1236
|
? `A local store of past measurements is in ${where}.`
|
|
1208
1237
|
: `Usage log found: ${where}.`,
|
|
1209
1238
|
noUsage: () =>
|
|
1210
|
-
'No usage found. Point trazum at a log (trazum profile <log.jsonl>) or pull one (trazum connect anthropic)
|
|
1239
|
+
'No usage found. Point trazum at a log (trazum profile <log.jsonl>) or pull one (trazum connect anthropic): every money figure in this tool comes from one.',
|
|
1211
1240
|
usageUnreadable: (where, because) =>
|
|
1212
1241
|
`${where} is there and could not be read: ${because}. That is a different problem from having no usage, and it is the one to fix first.`,
|
|
1213
1242
|
|
|
@@ -1228,7 +1257,7 @@ ${bold('EXAMPLES')}
|
|
|
1228
1257
|
noModelEvidence: () => 'nothing measured or written in the source names one model',
|
|
1229
1258
|
modelConflict: (files) => `these files name more than one provider: ${files}`,
|
|
1230
1259
|
modelProviderOnly: (provider, file) =>
|
|
1231
|
-
`${file} names ${provider} and no model
|
|
1260
|
+
`${file} names ${provider} and no model: a provider default would read as your decision six weeks from now`,
|
|
1232
1261
|
nothingMeasured: () => 'nothing has been measured yet',
|
|
1233
1262
|
windowTooShort: (days, minimum) =>
|
|
1234
1263
|
`${days} days measured; ${minimum} are needed before that is a monthly rate rather than a forecast`,
|
|
@@ -1242,7 +1271,7 @@ ${bold('EXAMPLES')}
|
|
|
1242
1271
|
`${labels} ${labels === 1 ? 'label' : 'labels'} in the log, and nothing here proves which prompt file sends which`,
|
|
1243
1272
|
budgetIsPolicy: () => 'a budget is a policy, and there is no measured figure to write one against yet',
|
|
1244
1273
|
budgetIsPolicyMeasured: (usd, days) =>
|
|
1245
|
-
`a budget is a policy, so it is yours to set
|
|
1274
|
+
`a budget is a policy, so it is yours to set: the measured figure is $${usd} over ${days} days`,
|
|
1246
1275
|
|
|
1247
1276
|
findingHeading: () => 'The most valuable thing found',
|
|
1248
1277
|
noFinding: (why) =>
|
|
@@ -1258,7 +1287,7 @@ ${bold('EXAMPLES')}
|
|
|
1258
1287
|
findingBatch: () => 'The Batch API halves both halves of the bill, for work that can wait.',
|
|
1259
1288
|
findingTotal: (usd, days) => `Together: $${usd} over the same ${days} days.`,
|
|
1260
1289
|
findingNext: () =>
|
|
1261
|
-
'trazum plan <your log> ranks every action, not just this one. If a figure here looks wrong, trazum feedback says where to tell us
|
|
1290
|
+
'trazum plan <your log> ranks every action, not just this one. If a figure here looks wrong, trazum feedback says where to tell us, it sends nothing on its own.',
|
|
1262
1291
|
|
|
1263
1292
|
wouldOverwrite: (keys) => `This replaces keys you already set: ${keys}. Pass --yes to write anyway.`,
|
|
1264
1293
|
nothingToWrite: () => 'No config written: nothing above could be justified from what is here.',
|
|
@@ -1277,14 +1306,14 @@ ${bold('EXAMPLES')}
|
|
|
1277
1306
|
missing: (months) =>
|
|
1278
1307
|
`No record at all for ${months}. Those months are named rather than filled: a year that quietly covers part of itself and prints an annual total is wrong by the rest and says nothing about it.`,
|
|
1279
1308
|
promises: (planned, arrived, notArrived, cannotTell) =>
|
|
1280
|
-
`${planned} actions planned. ${arrived} arrived, ${notArrived} did not, and ${cannotTell} could not be judged. Three outcomes, never two
|
|
1309
|
+
`${planned} actions planned. ${arrived} arrived, ${notArrived} did not, and ${cannotTell} could not be judged. Three outcomes, never two, the third is the one an ordinary annual report folds into the flattering one.`,
|
|
1281
1310
|
projected: (usd) => `The plans projected ${usd} of savings.`,
|
|
1282
1311
|
noArrivedFigure: () =>
|
|
1283
|
-
'There is deliberately no figure beside it for what arrived. A verification says WHETHER each action landed; it has never carried a per-action dollar figure for the saving. Assembling one here would mean deciding which of several observed numbers is "the saving"
|
|
1312
|
+
'There is deliberately no figure beside it for what arrived. A verification says WHETHER each action landed; it has never carried a per-action dollar figure for the saving. Assembling one here would mean deciding which of several observed numbers is "the saving", a judgement the verification refused to make, and exactly the annual-report arithmetic this document replaces.',
|
|
1284
1313
|
outcomes: (recorded, parsed, unrecordedUsd) =>
|
|
1285
1314
|
`${recorded} of ${parsed} calls recorded an outcome; ${unrecordedUsd} of the year\u2019s spend did not.`,
|
|
1286
1315
|
noOutcomes: () =>
|
|
1287
|
-
'No outcome was recorded this year, so nothing here says what the money bought. That is not a success rate of zero
|
|
1316
|
+
'No outcome was recorded this year, so nothing here says what the money bought. That is not a success rate of zero, an uninstrumented year and a failing year are different sentences.',
|
|
1288
1317
|
cannotSayHeading: () => 'What this record cannot say',
|
|
1289
1318
|
cannotSay: (kind) =>
|
|
1290
1319
|
kind === 'months-missing'
|
|
@@ -1294,14 +1323,14 @@ ${bold('EXAMPLES')}
|
|
|
1294
1323
|
: kind === 'nothing-was-verified'
|
|
1295
1324
|
? 'whether the promises were kept, because no plan was verified'
|
|
1296
1325
|
: kind === 'some-promises-unjudgeable'
|
|
1297
|
-
? 'whether some promises were kept
|
|
1326
|
+
? 'whether some promises were kept, they could not be judged from the logs that exist'
|
|
1298
1327
|
: kind === 'arrived-savings-not-quantified'
|
|
1299
1328
|
? 'how many dollars the kept promises were worth'
|
|
1300
1329
|
: kind === 'no-outcomes-recorded'
|
|
1301
1330
|
? 'what any of the money bought'
|
|
1302
1331
|
: 'what the uninstrumented share of the traffic did',
|
|
1303
1332
|
noNewData: () =>
|
|
1304
|
-
'Every figure above comes from the store and the plans you already keep. Nothing was computed here that cannot be checked against a document that already exists
|
|
1333
|
+
'Every figure above comes from the store and the plans you already keep. Nothing was computed here that cannot be checked against a document that already exists, which is the only reason an annual report is worth trusting, since it is the document most likely to be quoted out of the room it was written in.',
|
|
1305
1334
|
},
|
|
1306
1335
|
|
|
1307
1336
|
commitment: {
|
|
@@ -1310,7 +1339,7 @@ ${bold('EXAMPLES')}
|
|
|
1310
1339
|
needsTerms: () =>
|
|
1311
1340
|
'--floor and --discount are required, and --months defaults to 12. Take them from the contract the provider is offering you: the floor is what you commit to spending each month after the discount, and the discount is what you get off list.',
|
|
1312
1341
|
asIf: () =>
|
|
1313
|
-
'This is what the deal WOULD HAVE done on the traffic you actually had. It is a measurement of the past, not a prediction
|
|
1342
|
+
'This is what the deal WOULD HAVE done on the traffic you actually had. It is a measurement of the past, not a prediction, every month below happened.',
|
|
1314
1343
|
columns: { month: 'month', list: 'list', paid: 'would pay', saving: 'saving' },
|
|
1315
1344
|
net: (usd, months) => `Net over ${months} measured months: ${usd}.`,
|
|
1316
1345
|
good: (usd) => `The months that cleared the floor saved ${usd}.`,
|
|
@@ -1320,15 +1349,15 @@ ${bold('EXAMPLES')}
|
|
|
1320
1349
|
breakEven: (usd) =>
|
|
1321
1350
|
`Break-even is ${usd} of monthly spend. Below it you pay the floor for less usage than it buys; above it the saving grows.`,
|
|
1322
1351
|
spread: (low, high, median) =>
|
|
1323
|
-
`Measured spread: ${low} to ${high}, median ${median}. That is the shortfall risk in the only form a log can state it
|
|
1352
|
+
`Measured spread: ${low} to ${high}, median ${median}. That is the shortfall risk in the only form a log can state it, a count of real months and the range they covered, rather than a probability from a distribution nobody fitted.`,
|
|
1324
1353
|
shortTerm: (covered, term) =>
|
|
1325
1354
|
`This replays ${covered} months against a ${term}-month commitment. It is a real answer about ${covered} months and not about ${term}.`,
|
|
1326
1355
|
cannotTell: (why, needed) =>
|
|
1327
1356
|
why === 'no-history'
|
|
1328
1357
|
? 'Nothing to replay: this log has no whole months in it.'
|
|
1329
|
-
: `Cannot judge this from ${needed === '0' ? 'what exists' : `what exists
|
|
1358
|
+
: `Cannot judge this from ${needed === '0' ? 'what exists' : `what exists, ${needed} more whole month(s) would settle it`}. A commitment is signed for a year, and an answer from one month is a year-long decision made on a fortnight of evidence.`,
|
|
1330
1359
|
neverAForecast: () =>
|
|
1331
|
-
'Nothing here is annualised, extrapolated or fitted to a trend. If next quarter looks nothing like last quarter, this arithmetic says nothing about it
|
|
1360
|
+
'Nothing here is annualised, extrapolated or fitted to a trend. If next quarter looks nothing like last quarter, this arithmetic says nothing about it, which is the honest limit of what a log can tell you about a contract.',
|
|
1332
1361
|
},
|
|
1333
1362
|
|
|
1334
1363
|
owners: {
|
|
@@ -1337,39 +1366,39 @@ ${bold('EXAMPLES')}
|
|
|
1337
1366
|
'No owners configured. Add "owners" to trazum.config.json to attribute spend, for example {"patterns": {"payments": ["billing-*"]}, "budgets": {"payments": 400}}.',
|
|
1338
1367
|
columns: { owner: 'owner', spend: 'spend', budget: 'budget', calls: 'calls' },
|
|
1339
1368
|
verdict: (kind) =>
|
|
1340
|
-
kind === 'over' ? 'over' : kind === 'within' ? 'within' : kind === 'not-measured' ? 'not measured' : '
|
|
1369
|
+
kind === 'over' ? 'over' : kind === 'within' ? 'within' : kind === 'not-measured' ? 'not measured' : 'no verdict',
|
|
1341
1370
|
unallocated: (usd, share, labels) =>
|
|
1342
1371
|
`Unallocated: ${usd} (${share} of the bill), from ${labels}.`,
|
|
1343
1372
|
neverSpread: () =>
|
|
1344
|
-
'It is not divided between the owners above, and it never will be. Spreading unattributed spend proportionally is the most common lie in cost reporting: it makes every line add up and every team\u2019s number wrong by an amount nobody can see
|
|
1373
|
+
'It is not divided between the owners above, and it never will be. Spreading unattributed spend proportionally is the most common lie in cost reporting: it makes every line add up and every team\u2019s number wrong by an amount nobody can see, hardest on whoever instruments best, because their known spend is largest and they absorb the biggest share of somebody else\u2019s mystery. Claim it with a pattern.',
|
|
1345
1374
|
nothingUnallocated: () => 'Every workload in this log has an owner.',
|
|
1346
1375
|
sharedHeading: () => 'Shared, by a rule somebody wrote',
|
|
1347
1376
|
sharedRule: (label, rule) => `${label}: ${rule}`,
|
|
1348
1377
|
problemsHeading: () => 'This ownership config will not do what it looks like it does',
|
|
1349
1378
|
problem: (kind, detail) =>
|
|
1350
1379
|
kind === 'split-does-not-sum'
|
|
1351
|
-
? `${detail}
|
|
1380
|
+
? `${detail}, a split that does not sum to 1 loses money or invents it, silently. That workload is left whole in the unallocated line until this is fixed.`
|
|
1352
1381
|
: kind === 'split-names-unknown-owner'
|
|
1353
1382
|
? `${detail} names an owner that "owners.patterns" does not declare.`
|
|
1354
1383
|
: kind === 'split-has-one-owner'
|
|
1355
|
-
? `${detail} is "shared" with one owner
|
|
1384
|
+
? `${detail} is "shared" with one owner, that is a pattern written the long way, and reading it as a share invites a second owner to be added without the first being adjusted.`
|
|
1356
1385
|
: kind === 'negative-share'
|
|
1357
1386
|
? `${detail} has a negative share.`
|
|
1358
1387
|
: `${detail} has a budget but no patterns, so nothing can ever land on it.`,
|
|
1359
1388
|
notMeasured: (owner) =>
|
|
1360
|
-
`${owner} has a budget and no measured calls. That is NOT under budget
|
|
1389
|
+
`${owner} has a budget and no measured calls. That is NOT under budget, a team whose logs never arrived passes every budget it has, forever, and a green tick beside their name says the opposite of the truth.`,
|
|
1361
1390
|
},
|
|
1362
1391
|
|
|
1363
1392
|
semantic: {
|
|
1364
1393
|
heading: (path) => `Semantic pass on ${path}`,
|
|
1365
1394
|
willCost: (usd, input, output, model) =>
|
|
1366
|
-
`This will send the prompt to ${model}: about ${input} tokens in and ${output} out, roughly ${usd}. Estimated, not measured
|
|
1395
|
+
`This will send the prompt to ${model}: about ${input} tokens in and ${output} out, roughly ${usd}. Estimated, not measured, a tool that spends your money to tell you how to spend less should be the first thing audited by its own arithmetic. Pass --yes to run it.`,
|
|
1367
1396
|
needsYes: () => 'Nothing was sent. Add --yes once you have read the price above.',
|
|
1368
1397
|
finding: (kind, because) =>
|
|
1369
1398
|
`${kind === 'contradiction' ? 'Contradiction' : kind === 'restated-instruction' ? 'Restated' : 'Same thing, different words'}: ${because}`,
|
|
1370
1399
|
span: (line, text) => `line ${line}: ${text}`,
|
|
1371
1400
|
ceiling: (tokens) =>
|
|
1372
|
-
`At most ${tokens} tokens
|
|
1401
|
+
`At most ${tokens} tokens, a ceiling, not a saving. Merging these two means writing one passage that does the work of both, and nobody knows yet how long that is.`,
|
|
1373
1402
|
noCeiling: () =>
|
|
1374
1403
|
'No tokens attached. A contradiction is worth fixing because the prompt is wrong, not because it is long, and putting a figure on it would sell the wrong reason.',
|
|
1375
1404
|
nothingFound: () => 'Nothing survived checking. That is a real answer.',
|
|
@@ -1387,7 +1416,7 @@ ${bold('EXAMPLES')}
|
|
|
1387
1416
|
disposes: () =>
|
|
1388
1417
|
'The model proposes and the deterministic layer disposes: every quoted passage is checked character for character against the prompt, pairs the rules engine already catches are dropped, and every token figure is counted here rather than believed.',
|
|
1389
1418
|
optIn: () =>
|
|
1390
|
-
'This pass is optional and always will be. Trazum works with no key, no network and no model
|
|
1419
|
+
'This pass is optional and always will be. Trazum works with no key, no network and no model, that has been true since 0.1.0 and this does not change it.',
|
|
1391
1420
|
},
|
|
1392
1421
|
|
|
1393
1422
|
quality: {
|
|
@@ -1400,14 +1429,14 @@ ${bold('EXAMPLES')}
|
|
|
1400
1429
|
dropped: (from, to, outcomes, cost) =>
|
|
1401
1430
|
`The resolution rate moved from ${from} to ${to} on ${outcomes} measured outcomes, and this change ${cost}. Both halves are measured; neither is an estimate.`,
|
|
1402
1431
|
held: (from, to, outcomes) =>
|
|
1403
|
-
`The resolution rate moved from ${from} to ${to} on ${outcomes} measured outcomes
|
|
1432
|
+
`The resolution rate moved from ${from} to ${to} on ${outcomes} measured outcomes, up, measurably.`,
|
|
1404
1433
|
cannotTell: (why, need) =>
|
|
1405
1434
|
why === 'too-few-before'
|
|
1406
1435
|
? `Cannot tell: only ${need} outcomes before the change, and this gate needs 100 a side. It fails builds, so the threshold is not the one a rate uses elsewhere.`
|
|
1407
1436
|
: why === 'too-few-after'
|
|
1408
1437
|
? `Cannot tell yet: only ${need} outcomes since the change, and this gate needs 100 a side. Run it again once the traffic has caught up.`
|
|
1409
1438
|
: why === 'not-separable'
|
|
1410
|
-
? 'Cannot tell: the rate did not measurably move. That is NOT the same as "it held"
|
|
1439
|
+
? 'Cannot tell: the rate did not measurably move. That is NOT the same as "it held", a gate that spelled them the same way would pass a real regression it merely lacked the power to see.'
|
|
1411
1440
|
: why === 'no-vocabulary'
|
|
1412
1441
|
? 'Cannot tell: "outcomes.success" declares nothing, so there is no resolution rate to compare.'
|
|
1413
1442
|
: 'Cannot tell: something other than the prompt moved across the boundary. See below.',
|
|
@@ -1416,15 +1445,15 @@ ${bold('EXAMPLES')}
|
|
|
1416
1445
|
kind === 'model-mix-moved'
|
|
1417
1446
|
? `The model mix moved by ${detail}. The drop may be entirely somebody else's migration, and this tool cannot separate the two.`
|
|
1418
1447
|
: kind === 'volume-moved'
|
|
1419
|
-
? `The call volume moved ${detail}. A workload whose traffic moved that much is usually a workload whose population changed
|
|
1448
|
+
? `The call volume moved ${detail}. A workload whose traffic moved that much is usually a workload whose population changed, a new surface, a new customer, a campaign, and the questions being asked are not the questions from before.`
|
|
1420
1449
|
: `Outcome coverage moved from ${detail}. The two rates describe different populations: a team that starts instrumenting its hard cases sees its measured rate fall without anything having got worse.`,
|
|
1421
1450
|
notRandomised: () =>
|
|
1422
|
-
'This is a before-and-after, not an experiment. It splits traffic by time rather than at random, so everything else that changed at the same time is in the difference too
|
|
1451
|
+
'This is a before-and-after, not an experiment. It splits traffic by time rather than at random, so everything else that changed at the same time is in the difference too, which is why it says "cannot tell" far more readily than an A/B would.',
|
|
1423
1452
|
cannotSee: () =>
|
|
1424
1453
|
'It cannot see anything else you deployed that day. A "dropped" verdict says the rate fell and the three things it can check did not move. That is a smaller claim than "the prompt did it", and it is the largest one the evidence supports.',
|
|
1425
1454
|
gateFailed: () => 'Gate failed: a measured drop with nothing else to explain it.',
|
|
1426
1455
|
gateHeldOpen: () =>
|
|
1427
|
-
'Gate not passed and not failed. "Cannot tell" holds the claim open rather than exiting green
|
|
1456
|
+
'Gate not passed and not failed. "Cannot tell" holds the claim open rather than exiting green, the posture verify has had since 1.39.',
|
|
1428
1457
|
},
|
|
1429
1458
|
|
|
1430
1459
|
experiment: {
|
|
@@ -1436,10 +1465,10 @@ ${bold('EXAMPLES')}
|
|
|
1436
1465
|
arm: (name, rate, successes, recorded, interval) =>
|
|
1437
1466
|
`${name} ${rate} (${successes} of ${recorded} recorded) 95% ${interval}`,
|
|
1438
1467
|
wins: (name, low, high) =>
|
|
1439
|
-
`${name} wins. The difference is between ${low} and ${high} at 95% confidence
|
|
1468
|
+
`${name} wins. The difference is between ${low} and ${high} at 95% confidence, the whole interval is on one side of zero, which is what "wins" means here.`,
|
|
1440
1469
|
notSeparable: (why, needed) =>
|
|
1441
1470
|
why === 'no-difference-observed'
|
|
1442
|
-
? 'Not separable: both arms recorded the same rate. No sample size separates a difference of zero, so there is no "run it longer" to offer
|
|
1471
|
+
? 'Not separable: both arms recorded the same rate. No sample size separates a difference of zero, so there is no "run it longer" to offer, there is nothing here to find.'
|
|
1443
1472
|
: why === 'nothing-recorded'
|
|
1444
1473
|
? 'Not separable: an arm recorded no outcomes at all, so there is no rate to compare.'
|
|
1445
1474
|
: `Not separable on this traffic: the 95% interval on the difference includes zero. One number is larger, and that is not a finding. About ${needed} outcomes per arm would settle the difference observed so far.`,
|
|
@@ -1447,7 +1476,7 @@ ${bold('EXAMPLES')}
|
|
|
1447
1476
|
`Read early. The declared rule was ${declared} outcomes per arm and ${short} has ${recorded}. Nothing can stop a number being read early; this line exists so whoever reads the result later can see that it was.`,
|
|
1448
1477
|
honoured: (declared) => `Stopping rule honoured: both arms cleared ${declared} recorded outcomes.`,
|
|
1449
1478
|
marginalDearer: (better, usd) =>
|
|
1450
|
-
`${better} resolves more and costs more. One extra success costs ${usd}
|
|
1479
|
+
`${better} resolves more and costs more. One extra success costs ${usd}, that figure, not the rate, is what the decision turns on.`,
|
|
1451
1480
|
marginalCheaper: (better) => `${better} resolves more AND costs less per call. Nothing is being traded.`,
|
|
1452
1481
|
neverPromotes: () =>
|
|
1453
1482
|
'Nothing was changed. A winner is a finding; taking it is a decision with a name attached, and it belongs in the plan like everything else.',
|
|
@@ -1456,7 +1485,7 @@ ${bold('EXAMPLES')}
|
|
|
1456
1485
|
ladder: {
|
|
1457
1486
|
heading: () => 'Escalation ladders',
|
|
1458
1487
|
noLadders: () =>
|
|
1459
|
-
'No ladders configured. A ladder sends a workload to a cheap model first and escalates a recorded failure to a dearer one
|
|
1488
|
+
'No ladders configured. A ladder sends a workload to a cheap model first and escalates a recorded failure to a dearer one. Add "ladders" to trazum.config.json, for example {"support": {"tiers": ["claude-haiku-4-5", "claude-opus-5"], "escalateOn": ["escalated"]}}.',
|
|
1460
1489
|
workload: (label) => label,
|
|
1461
1490
|
arithmetic: (cheap, dear, breakEven) =>
|
|
1462
1491
|
`${cheap} a call cheap, ${dear} dear. Break-even escalation rate: ${breakEven}.`,
|
|
@@ -1475,7 +1504,7 @@ ${bold('EXAMPLES')}
|
|
|
1475
1504
|
: why === 'tier-unpriced'
|
|
1476
1505
|
? 'Cannot tell: one of the tiers is not in the price catalogue, so the break-even rate cannot be computed.'
|
|
1477
1506
|
: 'Cannot tell: this ladder declares no escalation values.',
|
|
1478
|
-
problemsHeading: (label) => `${label}
|
|
1507
|
+
problemsHeading: (label) => `${label}: this ladder will not do what it looks like it does`,
|
|
1479
1508
|
problem: (kind, detail) =>
|
|
1480
1509
|
kind === 'escalate-on-a-success'
|
|
1481
1510
|
? `escalateOn names "${detail}", which "outcomes.success" declares a SUCCESS. This ladder pays twice for work that already worked, on every call, while looking exactly like a cost-saving measure.`
|
|
@@ -1489,7 +1518,7 @@ ${bold('EXAMPLES')}
|
|
|
1489
1518
|
? `"${detail}" is not in the price catalogue.`
|
|
1490
1519
|
: `a ladder needs at least two tiers; this one has ${detail}.`,
|
|
1491
1520
|
theDoubleSpend: () =>
|
|
1492
|
-
'An escalation pays twice: the cheap attempt is not refunded. So a ladder saves money only below its break-even escalation rate, and above it costs more than never having built one
|
|
1521
|
+
'An escalation pays twice: the cheap attempt is not refunded. So a ladder saves money only below its break-even escalation rate, and above it costs more than never having built one, which is why the rate is printed beside the measurement rather than left to be worked out in somebody\u2019s head.',
|
|
1493
1522
|
notExecuted: () =>
|
|
1494
1523
|
'Trazum does not run the escalation. A ladder escalates after a failure is known, which is after the answer came back and usually after something downstream judged it, so the retry belongs in your own loop. What is here is the policy and the arithmetic that says whether the policy is worth running.',
|
|
1495
1524
|
},
|
|
@@ -1500,9 +1529,9 @@ ${bold('EXAMPLES')}
|
|
|
1500
1529
|
? `Name the provider to stand in front of. Known: ${known}.`
|
|
1501
1530
|
: `"${given}" is not a provider this gateway speaks for. Known: ${known}.`,
|
|
1502
1531
|
pricedNotFronted: (given, known) =>
|
|
1503
|
-
`Trazum prices ${given} but this gateway does not stand in front of it yet
|
|
1532
|
+
`Trazum prices ${given} but this gateway does not stand in front of it yet: it fronts ${known}. That is a real gap and it is on the roadmap, not a typo on your part. Until then: "trazum profile" prices a ${given} log you export, and its --max-usd gate fails a build on the bill after the fact. What you do not get is a refusal before the money is spent, which is the whole point of the gateway and the reason this is worth saying rather than listing ${given} as unknown.`,
|
|
1504
1533
|
needsPolicy: (policies) =>
|
|
1505
|
-
`--on-cannot-tell is required, and there is no default: ${policies}. When the gateway cannot judge a call
|
|
1534
|
+
`--on-cannot-tell is required, and there is no default: ${policies}. When the gateway cannot judge a call, no budget, nothing measured, an unpriced model, one of these happens, and only you know which failure your product can survive. fail-open keeps it working and lets the bill run; fail-closed stops the bill and takes it down with it. Picking one for you would be the most consequential decision in your architecture, made silently at install time.`,
|
|
1506
1535
|
listening: (where, provider) => `Gateway on ${where}, in front of ${provider}`,
|
|
1507
1536
|
/**
|
|
1508
1537
|
* One sentence per cause, and no `else`.
|
|
@@ -1510,26 +1539,26 @@ ${bold('EXAMPLES')}
|
|
|
1510
1539
|
* This was a two-branch ternary whose second arm explained OpenAI's
|
|
1511
1540
|
* `stream_options`. A third cause added later would have inherited that
|
|
1512
1541
|
* sentence and told a Gemini user about a setting that does not exist in
|
|
1513
|
-
* their SDK
|
|
1542
|
+
* their SDK, a message bounded by its neighbour rather than by its
|
|
1514
1543
|
* subject, which is this project's most repeated mistake.
|
|
1515
1544
|
*/
|
|
1516
1545
|
unmeasured: (cause, sofar) => {
|
|
1517
1546
|
if (cause === 'stream-broke') {
|
|
1518
|
-
return `unmeasured: the stream broke before its usage event
|
|
1547
|
+
return `unmeasured: the stream broke before its usage event, so the provider billed this call and this session cannot see it (${sofar} unmeasured so far)`;
|
|
1519
1548
|
}
|
|
1520
1549
|
if (cause === 'no-usage-event') {
|
|
1521
|
-
return `unmeasured: the stream carried no usage event
|
|
1550
|
+
return `unmeasured: the stream carried no usage event, on OpenAI that is every streaming call without stream_options.include_usage, so the total below is short by these (${sofar} unmeasured so far)`;
|
|
1522
1551
|
}
|
|
1523
|
-
return `unmeasured: the provider answered without any usage in the body
|
|
1552
|
+
return `unmeasured: the provider answered without any usage in the body, so the call was made and its cost is not knowable from the response, so the total below is short by these (${sofar} unmeasured so far)`;
|
|
1524
1553
|
},
|
|
1525
1554
|
pointYourSdk: (where) =>
|
|
1526
1555
|
`Point your SDK's base URL at ${where} and change nothing else. It speaks the provider's own wire format, so no code changes and no new client.`,
|
|
1527
1556
|
credential: () =>
|
|
1528
1557
|
'Your credential is forwarded untouched and never read, never stored, never logged and never put in a URL. Trazum holds no key here and cannot make a call of its own through this.',
|
|
1529
1558
|
neverSubstitutes: () =>
|
|
1530
|
-
'A call over budget is refused with HTTP 402 and the cheaper alternatives named
|
|
1559
|
+
'A call over budget is refused with HTTP 402 and the cheaper alternatives named, never silently swapped, trimmed or downgraded. 402 rather than 429 on purpose: every provider SDK retries a 429, which would turn one refusal into a retry storm.',
|
|
1531
1560
|
standing: (consumed, limit) =>
|
|
1532
|
-
`Judging against ${consumed} of ${limit}, measured, read once at start
|
|
1561
|
+
`Judging against ${consumed} of ${limit}, measured, read once at start: a file read in the request path would put this tool's latency between you and your provider on every call.`,
|
|
1533
1562
|
noStanding: () =>
|
|
1534
1563
|
'Nothing measured for this period, so every call is unjudged and the failure policy below decides. Set spend.monthlyUsd and pull with trazum connect.',
|
|
1535
1564
|
policy: (policy) =>
|
|
@@ -1537,7 +1566,7 @@ ${bold('EXAMPLES')}
|
|
|
1537
1566
|
? 'When it cannot judge: the call goes through, and the record says it was unjudged rather than within budget.'
|
|
1538
1567
|
: 'When it cannot judge: the call is refused. Nothing gets through unmeasured.',
|
|
1539
1568
|
measured: (model, label, input, output, substituted) =>
|
|
1540
|
-
` ${model}${label === null ? '' : ` [${label}]`}: ${input} in, ${output} out${substituted ? ' (substituted
|
|
1569
|
+
` ${model}${label === null ? '' : ` [${label}]`}: ${input} in, ${output} out${substituted ? ' (substituted, marked, and never counted as the call that was asked for)' : ''}`,
|
|
1541
1570
|
},
|
|
1542
1571
|
|
|
1543
1572
|
feedback: {
|
|
@@ -1546,19 +1575,19 @@ ${bold('EXAMPLES')}
|
|
|
1546
1575
|
'This command sends nothing, and neither does anything else here. Trazum has no telemetry: no ping, no install hook, no anonymous counter. A tool whose whole argument is that it reads your bill without uploading it cannot also be quietly reporting on you, and a test fails the build if this command ever reaches the network.',
|
|
1547
1576
|
whereHeading: () => 'Where',
|
|
1548
1577
|
wrongOptimisation: () =>
|
|
1549
|
-
'A rule changed what a prompt asks for
|
|
1578
|
+
'A rule changed what a prompt asks for, the report that matters most, and the failure this product exists to avoid:',
|
|
1550
1579
|
bug: () => 'Anything else that is wrong:',
|
|
1551
1580
|
question: () => 'A question, or an idea you are not sure about:',
|
|
1552
|
-
security: () => 'A security problem
|
|
1581
|
+
security: () => 'A security problem, privately, never a public issue:',
|
|
1553
1582
|
environmentHeading: () => 'What a maintainer will ask for',
|
|
1554
1583
|
environmentOnly: () =>
|
|
1555
|
-
'That is the whole of it. Nothing about your work is here
|
|
1584
|
+
'That is the whole of it. Nothing about your work is here, not the config, not a prompt, not a label, not a figure. Those are what a good report needs and what only you can decide to share.',
|
|
1556
1585
|
linkHeading: () => 'A blank issue with the above already filled in',
|
|
1557
1586
|
},
|
|
1558
1587
|
|
|
1559
1588
|
conform: {
|
|
1560
1589
|
noTarget: () =>
|
|
1561
|
-
'Pass a file to check
|
|
1590
|
+
'Pass a file to check, a usage log, or any document Trazum emits. Use "-" to read from stdin.',
|
|
1562
1591
|
badContract: (given, known) => `"${given}" is not a contract. Known contracts: ${known}.`,
|
|
1563
1592
|
unrecognised: (path) => `${path} does not match any contract Trazum knows.`,
|
|
1564
1593
|
/**
|
|
@@ -1583,49 +1612,49 @@ ${bold('EXAMPLES')}
|
|
|
1583
1612
|
problem: (at, kind, detail) => `${at}: ${detail} (${kind})`,
|
|
1584
1613
|
moreProblems: (count) => `…and ${count} more. Fix these first; they are often the same mistake.`,
|
|
1585
1614
|
unavailableHeading: () => 'What this cannot answer, and what would unlock it',
|
|
1586
|
-
unavailable: (finding, because, unlockedBy) => `${finding}
|
|
1615
|
+
unavailable: (finding, because, unlockedBy) => `${finding}, ${because}. Add ${unlockedBy}.`,
|
|
1587
1616
|
unavailableNeverGates: () =>
|
|
1588
1617
|
'None of those failed anything. Choosing not to record a field is a decision, not a defect, and a gate that failed on it would be this tool telling you what to log.',
|
|
1589
1618
|
},
|
|
1590
1619
|
|
|
1591
1620
|
rollup: {
|
|
1592
1621
|
noTargets: () =>
|
|
1593
|
-
'Pass the profile documents to roll up
|
|
1622
|
+
'Pass the profile documents to roll up, each one written by `trazum profile --json`, or a directory of them.',
|
|
1594
1623
|
noSuchTarget: (path) => `${path} is not there. A roll-up never guesses at a contribution it could not read.`,
|
|
1595
1624
|
emptyDirectory: (path) =>
|
|
1596
1625
|
`${path} holds no .json documents. An empty folder rolled up silently would report a team that spent nothing.`,
|
|
1597
1626
|
heading: (contributors, usd, calls) =>
|
|
1598
|
-
`Roll-up of ${plural(contributors, 'contributor')}
|
|
1627
|
+
`Roll-up of ${plural(contributors, 'contributor')}, ${usd} over ${plural(calls, 'call')}`,
|
|
1599
1628
|
span: (from, to) => `Covering ${from} to ${to}, stated and never extrapolated from.`,
|
|
1600
1629
|
noSpan: () => 'No contributor carried a clock, so this roll-up covers no stated period.',
|
|
1601
1630
|
contributorsHeading: () => 'Contributors, and what each one could not see',
|
|
1602
1631
|
contributor: (name, usd, calls, spanDays) =>
|
|
1603
|
-
`${name}
|
|
1632
|
+
`${name}, ${usd}, ${plural(calls, 'call')}${spanDays === null ? ', no clock' : `, ${plural(spanDays, 'day')}`}`,
|
|
1604
1633
|
claimedSpan: (from, to, contributors) =>
|
|
1605
|
-
`Asked for ${from} to ${to} by ${plural(contributors, 'contributor')}
|
|
1634
|
+
`Asked for ${from} to ${to} by ${plural(contributors, 'contributor')}: a claim about what was gone looking for, kept apart from what the records showed.`,
|
|
1606
1635
|
claimedRow: (from, to) => `asked for ${from} to ${to}`,
|
|
1607
1636
|
silentRuns: (runs) => runs,
|
|
1608
1637
|
undated: (count) =>
|
|
1609
1638
|
`${plural(count, 'record')} this contributor's own window could not place, because they carried no clock`,
|
|
1610
1639
|
rejectedHeading: () => 'Handed over and not merged',
|
|
1611
|
-
rejected: (name, because) => `${name}
|
|
1640
|
+
rejected: (name, because) => `${name}, ${because}`,
|
|
1612
1641
|
via: (rollup) => `via ${rollup}`,
|
|
1613
|
-
rejectedVia: (name, via, because) => `${name}, via ${via}
|
|
1642
|
+
rejectedVia: (name, via, because) => `${name}, via ${via}, ${because}`,
|
|
1614
1643
|
repeated: (names) =>
|
|
1615
|
-
`Handed over more than once, by name: ${names}. Its money is counted every time. Two machines genuinely sharing a name is possible, so nothing was subtracted
|
|
1644
|
+
`Handed over more than once, by name: ${names}. Its money is counted every time. Two machines genuinely sharing a name is possible, so nothing was subtracted. Check whether a roll-up and one of the machines inside it were both handed over.`,
|
|
1616
1645
|
identical: (names) => `The same document arrived more than once: ${names}.`,
|
|
1617
1646
|
identicalUsd: (usd) =>
|
|
1618
|
-
`${usd} of the total above is the repeat. It is counted, not discarded
|
|
1647
|
+
`${usd} of the total above is the repeat. It is counted, not discarded, whether it is one export handed over twice or two machines that agreed exactly is yours to know.`,
|
|
1619
1648
|
byLabelHeading: () => 'The merged bill, per workload',
|
|
1620
|
-
labelRow: (label, usd, calls) => `${label}
|
|
1649
|
+
labelRow: (label, usd, calls) => `${label}, ${usd}, ${plural(calls, 'call')}`,
|
|
1621
1650
|
notMergedHeading: () => 'Findings that do not roll up',
|
|
1622
|
-
notMerged: (finding, because) => `${finding}
|
|
1651
|
+
notMerged: (finding, because) => `${finding}, ${because}.`,
|
|
1623
1652
|
presentIn: (names) => `Present in: ${names}. Read it there.`,
|
|
1624
1653
|
cannotSayHeading: () => 'What this roll-up cannot say about itself',
|
|
1625
1654
|
caveat: (code) => {
|
|
1626
1655
|
switch (code) {
|
|
1627
1656
|
case 'overlap-invisible':
|
|
1628
|
-
return 'Overlap between contributors is unmeasurable here. Two people exporting the same traffic double the bill, and a merge of summaries cannot see it
|
|
1657
|
+
return 'Overlap between contributors is unmeasurable here. Two people exporting the same traffic double the bill, and a merge of summaries cannot see it: the raw lines a duplicate check needs are in no document.';
|
|
1629
1658
|
case 'mismatched-spans':
|
|
1630
1659
|
return 'The contributors cover meaningfully different periods. The sum is a sum; reading a share of it as a comparison of rates is the mistake this names.';
|
|
1631
1660
|
case 'contributor-without-clock':
|
|
@@ -1633,7 +1662,7 @@ ${bold('EXAMPLES')}
|
|
|
1633
1662
|
case 'day-top-label-unknown':
|
|
1634
1663
|
return "A day drew from more than one contributor, so its dearest label is unknown: each contributor knows its own, and the merged answer needs per-label-per-day spend no document carries.";
|
|
1635
1664
|
case 'no-claimed-period':
|
|
1636
|
-
return 'A contributor stated no window, so its span is all that is known of it
|
|
1665
|
+
return 'A contributor stated no window, so its span is all that is known of it, and a log whose latest record is the 5th may be a log of a quiet week or a log that stopped being written on the 5th.';
|
|
1637
1666
|
case 'silence-inside-a-claim':
|
|
1638
1667
|
return 'A contributor recorded nothing on days it asked for. Whether that is a quiet stretch or an export that stopped is yours to know; the days are named above.';
|
|
1639
1668
|
case 'claim-not-bounded':
|
|
@@ -1661,18 +1690,18 @@ ${bold('EXAMPLES')}
|
|
|
1661
1690
|
`The catalogue has no price for "${id}", and a comparison against a price nobody has is worse than none. trazum models lists what it knows; a pricing overlay adds what it does not.`,
|
|
1662
1691
|
heading: (target) => `Switching this traffic to ${target}, measured`,
|
|
1663
1692
|
saves: (current, target, saving) =>
|
|
1664
|
-
`${current} on this log becomes ${target} on the candidate
|
|
1693
|
+
`${current} on this log becomes ${target} on the candidate, ${saving} less, on the same tokens.`,
|
|
1665
1694
|
costs: (current, target, extra) =>
|
|
1666
|
-
`${current} on this log becomes ${target} on the candidate
|
|
1695
|
+
`${current} on this log becomes ${target} on the candidate, ${extra} MORE. This switch loses money before quality is even asked.`,
|
|
1667
1696
|
nothingMovable: () => 'Nothing here can move: every slice is already on the candidate or exceeds its context window.',
|
|
1668
1697
|
movable: (calls, slices) => `${calls} call(s) across ${slices} slice(s) could move.`,
|
|
1669
1698
|
overContext: (slices, usd) =>
|
|
1670
|
-
`${slices} slice(s) worth ${usd} cannot move
|
|
1699
|
+
`${slices} slice(s) worth ${usd} cannot move, at least one call exceeds the candidate's context window. Their money is in none of the figures above.`,
|
|
1671
1700
|
alreadyOnTarget: (calls, usd) => `${calls} call(s) worth ${usd} are already on the candidate.`,
|
|
1672
|
-
window: (days) => `Measured over ${days} day(s) of this log
|
|
1701
|
+
window: (days) => `Measured over ${days} day(s) of this log: every figure above is arithmetic on that window.`,
|
|
1673
1702
|
noWindow: () => 'No record carries a timestamp, so this log has no measured rate.',
|
|
1674
1703
|
breakEvenDays: (migration, days, measuredDays) =>
|
|
1675
|
-
`A declared migration cost of ${migration} is recovered after ~${days} day(s) at the saving measured over ${measuredDays} day(s). Division on the past
|
|
1704
|
+
`A declared migration cost of ${migration} is recovered after ~${days} day(s) at the saving measured over ${measuredDays} day(s). Division on the past, what next month does is next month's business.`,
|
|
1676
1705
|
breakEvenNoSaving: (migration) =>
|
|
1677
1706
|
`Break-even refused: the switch saves nothing, so the declared ${migration} has nothing to recover.`,
|
|
1678
1707
|
breakEvenNoClock: (migration) =>
|
|
@@ -1689,7 +1718,7 @@ ${bold('EXAMPLES')}
|
|
|
1689
1718
|
'Both numbers are yours to declare: trazum ownrate --gpu-usd-hour <n> --tokens-per-second <n> [--utilization <0-1>]',
|
|
1690
1719
|
invalid: (flag) => `--${flag} must be a positive number (utilization: greater than 0, at most 1).`,
|
|
1691
1720
|
result: (usdPerMTok, tokensPerSecond, gpuUsdPerHour, utilizationPct) =>
|
|
1692
|
-
`${usdPerMTok} per million tokens
|
|
1721
|
+
`${usdPerMTok} per million tokens, ${gpuUsdPerHour}/hour over ${tokensPerSecond} tokens/second at ${utilizationPct}% serving.`,
|
|
1693
1722
|
declared: () =>
|
|
1694
1723
|
'Derived from your declaration, honest only if the throughput was measured. Every report that prices this model will rest on these two numbers.',
|
|
1695
1724
|
snippetHeading: () => 'Paste into trazum.config.json (the "pricing" overlay file):',
|
|
@@ -1700,25 +1729,72 @@ ${bold('EXAMPLES')}
|
|
|
1700
1729
|
notFound: (path) => `${path}: not found`,
|
|
1701
1730
|
noExports: (path) => `${path}: no .json/.jsonl/.ndjson OTLP exports under it.`,
|
|
1702
1731
|
summary: (files, llmSpans) => `${files} file(s), ${llmSpans} LLM span(s) priced.`,
|
|
1703
|
-
skipped: (otherSpans) => `${otherSpans} non-LLM span(s) skipped
|
|
1732
|
+
skipped: (otherSpans) => `${otherSpans} non-LLM span(s) skipped, the trace's other work, not a bill.`,
|
|
1704
1733
|
noCache: (count) => `${count} span(s) carried no cache data. OpenTelemetry has not standardised the cache TTL split, so the cache verdicts read "cannot tell" on this log rather than a fabricated one.`,
|
|
1705
1734
|
unparseable: (count) => `${count} line(s) did not parse as JSON.`,
|
|
1706
1735
|
written: (file) => `Wrote ${file}.`,
|
|
1707
1736
|
},
|
|
1708
1737
|
|
|
1738
|
+
fromHelicone: {
|
|
1739
|
+
noPath: () =>
|
|
1740
|
+
'from-helicone needs a request export or a directory: trazum from-helicone requests.json',
|
|
1741
|
+
notFound: (path) => `${path}: not found`,
|
|
1742
|
+
noExports: (path) =>
|
|
1743
|
+
`${path}: no .json/.jsonl/.ndjson exports under it. Helicone's requests come out of its POST /v1/request/query endpoint, or the export button on the requests table.`,
|
|
1744
|
+
summary: (files, rows) => `${files} file(s), ${rows} request(s) read.`,
|
|
1745
|
+
unnamedModel: (count) =>
|
|
1746
|
+
`${count} row(s) named no model in any of the three columns and are not in the output: nothing on the row says what answered, so nothing can price it.`,
|
|
1747
|
+
disagreements: (count) =>
|
|
1748
|
+
`${count} row(s) were answered by a different model than the one requested. The model that answered is what is priced, because a bill is about what was billed; the count is here so a substitution is something you see rather than find inside a total.`,
|
|
1749
|
+
noTokens: (count) =>
|
|
1750
|
+
`${count} row(s) carried zero tokens on both sides: logged requests nobody can price. Counted here and worth a look at Helicone.`,
|
|
1751
|
+
cacheFlagged: (count) =>
|
|
1752
|
+
`${count} row(s) were served from Helicone's cache. That is a flag on the row and never a token split, so the cache verdicts read "cannot tell" on this log rather than a fabricated one.`,
|
|
1753
|
+
noSessions: () =>
|
|
1754
|
+
'No session identity: a Helicone request id names one call, not a conversation, so the conversation findings (single-turn cache waste, context pressure) cannot be answered from this log. Passing a per-call id off as a conversation would report every call as a conversation of one.',
|
|
1755
|
+
unparseable: (count) => `${count} line(s) did not parse as JSON.`,
|
|
1756
|
+
written: (file) => `Wrote ${file}.`,
|
|
1757
|
+
},
|
|
1758
|
+
|
|
1759
|
+
fromLiteLlm: {
|
|
1760
|
+
noPath: () =>
|
|
1761
|
+
'from-litellm needs a spend-log export or a directory: trazum from-litellm spend.json',
|
|
1762
|
+
notFound: (path) => `${path}: not found`,
|
|
1763
|
+
noExports: (path) =>
|
|
1764
|
+
`${path}: no .json/.jsonl/.ndjson exports under it. LiteLLM's spend logs come out of the LiteLLM_SpendLogs table, or from the proxy's own /spend endpoints.`,
|
|
1765
|
+
summary: (files, rows) => `${files} file(s), ${rows} logged call(s) read.`,
|
|
1766
|
+
unnamedModel: (count) =>
|
|
1767
|
+
`${count} row(s) named no model and are not in the output. "model_group" is the name of a proxy route and several models can sit behind one, so pricing those calls by the route they took would attribute a figure to something it does not describe.`,
|
|
1768
|
+
noTokens: (count) =>
|
|
1769
|
+
`${count} row(s) carried zero tokens on both sides: logged calls nobody can price. They are counted here and worth a look at the proxy.`,
|
|
1770
|
+
cacheFlagged: (count) =>
|
|
1771
|
+
`${count} row(s) are marked as a cache hit. LiteLLM records that as a flag and never as a token split, so the cache verdicts read "cannot tell" on this log rather than a fabricated one.`,
|
|
1772
|
+
reportedSpend: (usd) =>
|
|
1773
|
+
`LiteLLM priced these same calls at ${usd} with its own table. That figure is not merged into anything Trazum computes: two price tables summed into one total is how a report becomes quietly wrong. Compare them deliberately or not at all.`,
|
|
1774
|
+
unparseable: (count) => `${count} line(s) did not parse as JSON.`,
|
|
1775
|
+
written: (file) => `Wrote ${file}.`,
|
|
1776
|
+
},
|
|
1777
|
+
|
|
1709
1778
|
fromClaudeCode: {
|
|
1710
1779
|
noPath: () => 'from-claude-code needs a transcript file or directory: trazum from-claude-code ~/.claude/projects',
|
|
1711
1780
|
notFound: (path) => `${path}: not found`,
|
|
1712
1781
|
noTranscripts: (path) => `${path}: no .jsonl transcripts under it. Claude Code writes them per session under ~/.claude/projects/<project>/.`,
|
|
1713
1782
|
summary: (files, records) => `${files} transcript(s), ${records} priced call(s).`,
|
|
1714
|
-
collapsed: (lines) => `${lines} extra line(s) of already-counted calls collapsed
|
|
1783
|
+
collapsed: (lines) => `${lines} extra line(s) of already-counted calls collapsed: one API call is written as one line per content block, and counting each line would overbill.`,
|
|
1715
1784
|
streamed: (count) => `${count} call(s) were written while still streaming; the final line's counts stood.`,
|
|
1716
|
-
disagreements: (count) => `${count} call(s) had lines that disagreed beyond streaming growth
|
|
1785
|
+
disagreements: (count) => `${count} call(s) had lines that disagreed beyond streaming growth: a count shrank, or another field changed. The last line stood, but that is a finding: worth a look at the transcript.`,
|
|
1717
1786
|
noRequestId: (count) => `${count} call(s) carried no request id, so nothing could be collapsed against them; each is priced as recorded.`,
|
|
1718
1787
|
synthetic: (count) => `${count} turn(s) Claude Code wrote itself (model "<synthetic>") are not in the output: no provider ever billed them, so they are not calls.`,
|
|
1719
1788
|
labelled: () => 'Labelled by project folder. Pass --no-label-from-project to leave the label off, or --label <name> to set one.',
|
|
1720
1789
|
skipped: (other, unparseable, withoutUsage) => `Passed over: ${other} non-assistant line(s) (the transcript's other business), ${unparseable} unparseable, ${withoutUsage} assistant line(s) without usage.`,
|
|
1721
1790
|
written: (file) => `Wrote ${file}.`,
|
|
1791
|
+
stateNeedsFile: () =>
|
|
1792
|
+
'--state reads one transcript, not a folder: the state ties an offset in the transcript to a length of the output, and several transcripts appending to one output have no single such length. Point it at the .jsonl file.',
|
|
1793
|
+
stateNeedsOut: () => '--state needs --out: there is nothing to resume against when the records go to stdout.',
|
|
1794
|
+
resumed: (skipped, offset) =>
|
|
1795
|
+
skipped === 0
|
|
1796
|
+
? `Read the transcript in full and recorded a resume point at byte ${offset}.`
|
|
1797
|
+
: `Resumed at byte ${skipped}, so ${skipped} byte(s) were not re-read. Next resume point: byte ${offset}.`,
|
|
1722
1798
|
},
|
|
1723
1799
|
|
|
1724
1800
|
position: {
|
|
@@ -1727,12 +1803,12 @@ ${bold('EXAMPLES')}
|
|
|
1727
1803
|
scopeDay: () => 'today',
|
|
1728
1804
|
scopeLabel: (label) => `"${label}"`,
|
|
1729
1805
|
within: (scope, measured, limit, remaining, days, elapsed) =>
|
|
1730
|
-
`${scope}: ${measured} of ${limit} measured
|
|
1806
|
+
`${scope}: ${measured} of ${limit} measured, ${remaining} left (${days} of ${elapsed} elapsed days measured)`,
|
|
1731
1807
|
over: (scope, measured, limit, overBy) =>
|
|
1732
|
-
`${scope}: over
|
|
1733
|
-
cannotTell: (scope) => `${scope}: cannot tell
|
|
1808
|
+
`${scope}: over, ${measured} measured against ${limit}, ${overBy} past the ceiling`,
|
|
1809
|
+
cannotTell: (scope) => `${scope}: cannot tell, because nothing was measured in this window`,
|
|
1734
1810
|
distance: (days, rate, overDays) =>
|
|
1735
|
-
`at ${rate}/day over ${overDays} measured days, the ceiling is ${days} days away
|
|
1811
|
+
`at ${rate}/day over ${overDays} measured days, the ceiling is ${days} days away, division on the past, not a forecast`,
|
|
1736
1812
|
unmeasuredHeading: () => 'Configured and not measurable from this log',
|
|
1737
1813
|
unmeasured: (scope, why) => `${scope}: ${why}`,
|
|
1738
1814
|
why: (reason) =>
|
|
@@ -1741,8 +1817,8 @@ ${bold('EXAMPLES')}
|
|
|
1741
1817
|
: reason === 'no-labels'
|
|
1742
1818
|
? 'no record carries a label, so per-label spend is unknowable here'
|
|
1743
1819
|
: reason === 'nothing-recorded'
|
|
1744
|
-
? 'the log is empty
|
|
1745
|
-
: 'the log records labels and has never seen this one this month
|
|
1820
|
+
? 'the log is empty: nothing was recorded'
|
|
1821
|
+
: 'the log records labels and has never seen this one this month, possibly renamed, possibly idle, and neither is "under budget"',
|
|
1746
1822
|
cannotSayHeading: () => 'What this deliberately does not answer',
|
|
1747
1823
|
cannotSay: (code) => {
|
|
1748
1824
|
switch (code) {
|
|
@@ -1755,9 +1831,9 @@ ${bold('EXAMPLES')}
|
|
|
1755
1831
|
}
|
|
1756
1832
|
},
|
|
1757
1833
|
unpriced: (count) =>
|
|
1758
|
-
`${count} record(s) name a model the catalogue cannot price. They contribute nothing to any figure above
|
|
1834
|
+
`${count} record(s) name a model the catalogue cannot price. They contribute nothing to any figure above: money nobody can see, said here rather than hidden.`,
|
|
1759
1835
|
source: () =>
|
|
1760
|
-
'Measured from this log alone, priced record by record. The store\'s provider-billed monthly standing is a different measurement
|
|
1836
|
+
'Measured from this log alone, priced record by record. The store\'s provider-billed monthly standing is a different measurement, "trazum store" prints it, and the two are never merged into one figure.',
|
|
1761
1837
|
noLog: () => 'position needs a usage log: trazum position usage.jsonl',
|
|
1762
1838
|
},
|
|
1763
1839
|
pulse: {
|
|
@@ -1774,7 +1850,7 @@ ${bold('EXAMPLES')}
|
|
|
1774
1850
|
return kind;
|
|
1775
1851
|
}
|
|
1776
1852
|
},
|
|
1777
|
-
neverRun: (name) => `${name}: never here. Not late
|
|
1853
|
+
neverRun: (name) => `${name}: never here. Not late: there is no cadence to be late against.`,
|
|
1778
1854
|
age: (name, when, hours) => `${name}: ${when}Z, ${plural(hours, 'hour')} ago`,
|
|
1779
1855
|
noThreshold: () =>
|
|
1780
1856
|
'Nothing was judged. Pass --max-stale-hours <n> to make a run that stopped a failing exit code; how stale is too stale is a policy, and this tool does not write yours.',
|
|
@@ -1782,12 +1858,12 @@ ${bold('EXAMPLES')}
|
|
|
1782
1858
|
stale: (hours) =>
|
|
1783
1859
|
`Something that runs here has not run in over ${plural(hours, 'hour')}. Silence from a scheduled job and silence from a job with nothing to report look identical; this is which.`,
|
|
1784
1860
|
notAService: () =>
|
|
1785
|
-
'This command runs nothing and hosts nothing. Something has to notice, and the something is your CI: a step that runs this on the schedule you already have turns a dead cron into a red build, without Trazum holding anybody\'s metrics. How far the measurements reach is reported and never judged
|
|
1861
|
+
'This command runs nothing and hosts nothing. Something has to notice, and the something is your CI: a step that runs this on the schedule you already have turns a dead cron into a red build, without Trazum holding anybody\'s metrics. How far the measurements reach is reported and never judged: that is a provider reporting on its own schedule, not a job that failed.',
|
|
1786
1862
|
},
|
|
1787
1863
|
|
|
1788
1864
|
html: {
|
|
1789
1865
|
caveatsHeading: () => 'What this report cannot say',
|
|
1790
|
-
noClock: () => 'No record carried a timestamp, so days, rates and drift are not in this report
|
|
1866
|
+
noClock: () => 'No record carried a timestamp, so days, rates and drift are not in this report: a total is still a total.',
|
|
1791
1867
|
noSessions: () => 'No record carried a session, so conversation growth and per-conversation costs are not in this report.',
|
|
1792
1868
|
byLabelHeading: () => 'By workload',
|
|
1793
1869
|
byModelHeading: () => 'By model',
|
|
@@ -1810,7 +1886,7 @@ ${bold('EXAMPLES')}
|
|
|
1810
1886
|
|
|
1811
1887
|
schema: {
|
|
1812
1888
|
noTarget: (known) =>
|
|
1813
|
-
`Name a contract to print its JSON Schema. Known: ${known}. The schema validates with any draft 2020-12 validator
|
|
1889
|
+
`Name a contract to print its JSON Schema. Known: ${known}. The schema validates with any draft 2020-12 validator, no Trazum required.`,
|
|
1814
1890
|
unknown: (name, known) => `"${name}" is not a contract. Known: ${known}.`,
|
|
1815
1891
|
},
|
|
1816
1892
|
|
|
@@ -1823,7 +1899,7 @@ ${bold('EXAMPLES')}
|
|
|
1823
1899
|
colRatio: () => 'ratio',
|
|
1824
1900
|
colPeakRss: () => 'peak RSS',
|
|
1825
1901
|
note: () =>
|
|
1826
|
-
'One shot each, this machine, today. The wall clock is for a person reading two tables side by side; the ratio
|
|
1902
|
+
'One shot each, this machine, today. The wall clock is for a person reading two tables side by side; the ratio, workload over an in-process calibration loop, is the number a gate can hold, because the machine cancels out of it. Record one with --record, hold a build to it with --against and a stated --max-ratio.',
|
|
1827
1903
|
unknownWorkload: (id, known) => `Unknown workload "${id}". Known: ${known}.`,
|
|
1828
1904
|
recorded: (path) => `Baseline written to ${path}. Commit it; --against reads it.`,
|
|
1829
1905
|
needsMaxRatio: () =>
|
|
@@ -1846,21 +1922,21 @@ ${bold('EXAMPLES')}
|
|
|
1846
1922
|
where: {
|
|
1847
1923
|
hostHeading: () => 'Running inside',
|
|
1848
1924
|
subscription: (host) =>
|
|
1849
|
-
`${host} bills by subscription, not by the token. A monthly saving below is arithmetic about tokens, not money you get back
|
|
1925
|
+
`${host} bills by subscription, not by the token. A monthly saving below is arithmetic about tokens, not money you get back, what you gain is context window and rate-limit headroom.`,
|
|
1850
1926
|
noTarget: () => 'Pass a source file to see which provider its prompts are sent to.',
|
|
1851
1927
|
sourceHeading: (path) => `Prompts in ${path} go to`,
|
|
1852
1928
|
conflict: () => 'Cannot tell: the file names more than one provider.',
|
|
1853
1929
|
conflictFallback: () =>
|
|
1854
1930
|
'Nothing was assumed. Set "usage.model" in trazum.config.json, or pass --model.',
|
|
1855
1931
|
nothingFound: () => 'Nothing in this file says which provider it calls.',
|
|
1856
|
-
providerOnly: () => ' (provider only
|
|
1932
|
+
providerOnly: () => ' (provider only, nothing named a model)',
|
|
1857
1933
|
evidenceLine: (line, kind, detail) => `line ${line} ${kind}: ${detail}`,
|
|
1858
1934
|
pricedAs: () => 'Priced as',
|
|
1859
1935
|
fromConfig: () => '(from trazum.config.json)',
|
|
1860
1936
|
fromDetection: () => '(read from the source)',
|
|
1861
1937
|
fromProviderDefault: (provider) =>
|
|
1862
1938
|
`(${provider} was read from the source; nothing named a model, so this is theirs)`,
|
|
1863
|
-
fromDefault: () => '(the built-in default
|
|
1939
|
+
fromDefault: () => '(the built-in default, nothing said otherwise)',
|
|
1864
1940
|
},
|
|
1865
1941
|
|
|
1866
1942
|
pricing: {
|
|
@@ -1901,17 +1977,17 @@ ${bold('EXAMPLES')}
|
|
|
1901
1977
|
noteFormat: (tokens) => `~${tokens} tokens restating the output format`,
|
|
1902
1978
|
noteProtected: (pct) => `${pct}% is code or URLs, which cannot be trimmed`,
|
|
1903
1979
|
skipped: (count) =>
|
|
1904
|
-
`Skipped ${count} source ${count === 1 ? 'file' : 'files'} with no \`// trazum:prompt\` marker
|
|
1980
|
+
`Skipped ${count} source ${count === 1 ? 'file' : 'files'} with no \`// trazum:prompt\` marker, `
|
|
1905
1981
|
+ 'their prompts are not in this ranking.',
|
|
1906
1982
|
densityNote: () =>
|
|
1907
|
-
'Tok/sen is tokens per sentence: verbosity independent of length. There is no score
|
|
1983
|
+
'Tok/sen is tokens per sentence: verbosity independent of length. There is no score. Every column is a measurement you can check against the file.',
|
|
1908
1984
|
recoverableNote: () =>
|
|
1909
|
-
'Recover is what the deterministic rules would take at this level, priced by the usage profile, with the token count beside it
|
|
1985
|
+
'Recover is what the deterministic rules would take at this level, priced by the usage profile, with the token count beside it: a saving of one token is twenty-five cents and no work worth doing. It is measured by running the rules, not by a formula.',
|
|
1910
1986
|
},
|
|
1911
1987
|
|
|
1912
1988
|
blame: {
|
|
1913
1989
|
heading: (path, revisions) =>
|
|
1914
|
-
`${path}
|
|
1990
|
+
`${path}, ${revisions} ${revisions === 1 ? 'revision' : 'revisions'}`,
|
|
1915
1991
|
notARepository: () =>
|
|
1916
1992
|
'blame reads a file\'s history from git, and this directory is not inside a repository.',
|
|
1917
1993
|
outsideRepository: (path) =>
|
|
@@ -1925,14 +2001,14 @@ ${bold('EXAMPLES')}
|
|
|
1925
2001
|
`That movement is ${amount} a month on ${model} at ${calls} calls.`,
|
|
1926
2002
|
biggestRise: () => 'Biggest single increase',
|
|
1927
2003
|
biggestRiseDetail: (tokens, author, subject, sha) =>
|
|
1928
|
-
`+${tokens} tokens
|
|
2004
|
+
`+${tokens} tokens, ${author}, "${subject}" (${sha})`,
|
|
1929
2005
|
addedAt: () => 'added',
|
|
1930
2006
|
goneAt: () => 'not present',
|
|
1931
2007
|
truncated: (shown) =>
|
|
1932
2008
|
`Showing the most recent ${shown}. Pass --limit for more.`,
|
|
1933
2009
|
followedRename: (from) => `Followed a rename: earlier revisions are ${from}.`,
|
|
1934
|
-
estimateNote: () =>
|
|
1935
|
-
|
|
2010
|
+
estimateNote: (band) =>
|
|
2011
|
+
`Token counts are estimates (±${band}% for text of this kind). The trend is the point; the absolute figures are not.`,
|
|
1936
2012
|
},
|
|
1937
2013
|
|
|
1938
2014
|
languages: {
|
|
@@ -1950,17 +2026,17 @@ ${bold('EXAMPLES')}
|
|
|
1950
2026
|
title: () => 'Available rules',
|
|
1951
2027
|
disableHint: () => ' Turn off the ones you do not want with --disable id1,id2',
|
|
1952
2028
|
measureHeading: (root, prompts, level) =>
|
|
1953
|
-
`What each rule recovers in ${root}
|
|
2029
|
+
`What each rule recovers in ${root}, ${plural(prompts, 'prompt')}, level ${level}`,
|
|
1954
2030
|
measureTotals: (before, saved, floor) =>
|
|
1955
2031
|
`${before} tokens before. The rules recover ${saved}; normalisation recovers ${floor} whether a rule is enabled or not, and that is not the rules' work.`,
|
|
1956
2032
|
measureRow: (id, marginal, alone, prompts) =>
|
|
1957
2033
|
`${id.padEnd(22)} lost if removed ${marginal.padStart(6)} alone ${alone.padStart(6)} ${plural(prompts, 'prompt')}`,
|
|
1958
2034
|
measureOverlap: (sumOfAlone, saved) =>
|
|
1959
|
-
`The rules recover ${sumOfAlone} between them one at a time and ${saved} together. The gap is the overlap
|
|
2035
|
+
`The rules recover ${sumOfAlone} between them one at a time and ${saved} together. The gap is the overlap: two rules finding the same tokens, and it is stated rather than resolved into a total, because a single figure here is the one number that cannot be true.`,
|
|
1960
2036
|
measureRedundant: (ids) =>
|
|
1961
|
-
`Every token these find, something else finds too, here: ${ids}. An overlap, not a defect
|
|
2037
|
+
`Every token these find, something else finds too, here: ${ids}. An overlap, not a defect, and not a reason to delete one without deciding which is the better to keep.`,
|
|
1962
2038
|
measureFiredWithoutSaving: (ids) =>
|
|
1963
|
-
`These changed the prompt and recovered no tokens: ${ids}. Not the same as inert
|
|
2039
|
+
`These changed the prompt and recovered no tokens: ${ids}. Not the same as inert: they were exercised, and they are altering somebody's instruction for no measured benefit.`,
|
|
1964
2040
|
measureInert: (ids) =>
|
|
1965
2041
|
`These changed nothing in this corpus: ${ids}. That is a fact about these files, not about the rules: a rule that finds nothing here has not been shown to find nothing anywhere.`,
|
|
1966
2042
|
measureBand: (source) =>
|
|
@@ -1972,7 +2048,7 @@ ${bold('EXAMPLES')}
|
|
|
1972
2048
|
|
|
1973
2049
|
doctor: {
|
|
1974
2050
|
heading: (root, prompts) =>
|
|
1975
|
-
`${root}
|
|
2051
|
+
`${root}, ${prompts} ${prompts === 1 ? 'prompt' : 'prompts'}`,
|
|
1976
2052
|
subheading: (model, calls) => `Priced on ${model} at ${calls} calls a month.`,
|
|
1977
2053
|
pricesReviewed: (date, days) => `Prices reviewed ${date}${ago(days)}.`,
|
|
1978
2054
|
budgetsHeading: () => 'Budgets',
|
|
@@ -1984,8 +2060,8 @@ ${bold('EXAMPLES')}
|
|
|
1984
2060
|
`${count} of ${total} ${count === 1 ? 'prompt has' : 'prompts have'} no budget, so nothing is watching ${count === 1 ? 'it' : 'them'}`,
|
|
1985
2061
|
overBudget: (count) =>
|
|
1986
2062
|
count === 1
|
|
1987
|
-
? '1 prompt is already over its budget
|
|
1988
|
-
: `${count} prompts are already over their budget
|
|
2063
|
+
? '1 prompt is already over its budget: trazum check would fail on it'
|
|
2064
|
+
: `${count} prompts are already over their budget: trazum check would fail on them`,
|
|
1989
2065
|
andMore: (count) => `and ${count} more`,
|
|
1990
2066
|
findingsHeading: () => 'What it would be worth fixing',
|
|
1991
2067
|
acrossPrompts: (count) => `${count} ${count === 1 ? 'prompt' : 'prompts'}`,
|
|
@@ -1993,7 +2069,7 @@ ${bold('EXAMPLES')}
|
|
|
1993
2069
|
'Each line is the same advisory trazum optimize raises on those prompts, summed. '
|
|
1994
2070
|
+ 'Run it on any one of them to see the figure on its own.',
|
|
1995
2071
|
notAGate: () =>
|
|
1996
|
-
'Nothing here fails a build. trazum check is the gate; this is the survey
|
|
2072
|
+
'Nothing here fails a build. trazum check is the gate; this is the survey: the '
|
|
1997
2073
|
+ 'model recommendation is a keyword heuristic, and a build gated on one teaches '
|
|
1998
2074
|
+ 'people to re-run until green.',
|
|
1999
2075
|
|
|
@@ -2006,10 +2082,10 @@ ${bold('EXAMPLES')}
|
|
|
2006
2082
|
sharedPrefixFix: (drift) =>
|
|
2007
2083
|
drift === 'whitespace'
|
|
2008
2084
|
? 'A formatter fixes this: the text already agrees, only the spacing does not.'
|
|
2009
|
-
: 'Someone has to pick one wording
|
|
2085
|
+
: 'Someone has to pick one wording, the text itself differs, not just its spacing.',
|
|
2010
2086
|
sharedPrefixNoFigure: () =>
|
|
2011
2087
|
'No figure is attached, deliberately. Caching matches bytes, so these prompts hold '
|
|
2012
|
-
+ 'one cache entry each instead of one between them
|
|
2088
|
+
+ 'one cache entry each instead of one between them, but what that costs depends on '
|
|
2013
2089
|
+ 'how the calls are spread across the group, and Trazum applies a single '
|
|
2014
2090
|
+ '--cache-hit-rate to every prompt. Pricing it would mean inventing your traffic.',
|
|
2015
2091
|
},
|
|
@@ -2027,7 +2103,7 @@ ${bold('EXAMPLES')}
|
|
|
2027
2103
|
selfAgreement: (pct) =>
|
|
2028
2104
|
`The prompt agrees with itself ${pct} of the time. That is the yardstick: a removal `
|
|
2029
2105
|
+ 'that moves the answer less than this moved nothing attributable to the example.',
|
|
2030
|
-
line: (n, tokens, pct) => `example ${n}
|
|
2106
|
+
line: (n, tokens, pct) => `example ${n}, ${tokens} tokens, ${pct} agreement without it`,
|
|
2031
2107
|
verdictNeeded: () => 'needed here',
|
|
2032
2108
|
verdictRecoverable: () => 'no effect on these inputs',
|
|
2033
2109
|
verdictUnknown: () => 'inconclusive',
|
|
@@ -2035,7 +2111,7 @@ ${bold('EXAMPLES')}
|
|
|
2035
2111
|
`${tokens} tokens sit in examples whose removal changed nothing measurable here.`,
|
|
2036
2112
|
caveat: () =>
|
|
2037
2113
|
'Which is not the same as "delete them". An example may exist for a case these '
|
|
2038
|
-
+ 'inputs do not contain
|
|
2114
|
+
+ 'inputs do not contain, the boundary condition somebody hit in production and '
|
|
2039
2115
|
+ 'added a demonstration for. This measures the inputs you gave it, and only you '
|
|
2040
2116
|
+ 'know whether they cover what matters. Nothing was edited.',
|
|
2041
2117
|
},
|
|
@@ -2101,7 +2177,7 @@ ${bold('EXAMPLES')}
|
|
|
2101
2177
|
allTotal: (delta, prompts) =>
|
|
2102
2178
|
`${delta} tokens across ${prompts} ${prompts === 1 ? 'prompt' : 'prompts'}`,
|
|
2103
2179
|
signConvention: () =>
|
|
2104
|
-
'Every figure is after minus before, so positive means worse
|
|
2180
|
+
'Every figure is after minus before, so positive means worse, the opposite of the rest of Trazum.',
|
|
2105
2181
|
onlyBefore: () => 'only before',
|
|
2106
2182
|
onlyAfter: () => 'only after ',
|
|
2107
2183
|
onlyOneSideNote: () =>
|
|
@@ -2109,7 +2185,7 @@ ${bold('EXAMPLES')}
|
|
|
2109
2185
|
},
|
|
2110
2186
|
|
|
2111
2187
|
markdown: {
|
|
2112
|
-
checkHeading: (target) => `Trazum
|
|
2188
|
+
checkHeading: (target) => `Trazum, token budgets for ${target}`,
|
|
2113
2189
|
baselineGrew: (delta, pct) => `This branch adds ${delta} tokens (${pct}) to the prompts here`,
|
|
2114
2190
|
baselineShrank: (delta, pct) => `This branch removes ${delta} tokens (${pct}) from the prompts here`,
|
|
2115
2191
|
baselineUnchanged: () => 'No change against the recorded baseline',
|
|
@@ -2123,10 +2199,10 @@ ${bold('EXAMPLES')}
|
|
|
2123
2199
|
'The scenario or the price list moved since the baseline was recorded, so the two monthly figures are not the same measurement and are not subtracted here. The token comparison above is unaffected.',
|
|
2124
2200
|
baselineReRecord: (command, path) =>
|
|
2125
2201
|
`If this growth is intended, re-record with \`${command}\` and commit \`${path}\`.`,
|
|
2126
|
-
diffHeading: (before, after) => `Trazum
|
|
2202
|
+
diffHeading: (before, after) => `Trazum, ${before} → ${after}`,
|
|
2127
2203
|
rankHeading: (root, count) =>
|
|
2128
|
-
`Trazum
|
|
2129
|
-
blameHeading: (path) => `Trazum
|
|
2204
|
+
`Trazum, what to fix first in ${root} (${count} ${count === 1 ? "prompt" : "prompts"})`,
|
|
2205
|
+
blameHeading: (path) => `Trazum, token history for ${path}`,
|
|
2130
2206
|
rankLevel: (level) => `Measured at rule level \`${level}\`.`,
|
|
2131
2207
|
columnFile: () => 'Prompt',
|
|
2132
2208
|
columnTokens: () => 'Tokens',
|
|
@@ -2138,7 +2214,7 @@ ${bold('EXAMPLES')}
|
|
|
2138
2214
|
? 'The prompt is within budget.'
|
|
2139
2215
|
: `All ${budgeted} budgeted prompts are within budget.`,
|
|
2140
2216
|
overBudget: (failures, budgeted) => `${failures} of ${budgeted} over budget`,
|
|
2141
|
-
noBudget: () => '
|
|
2217
|
+
noBudget: () => 'none',
|
|
2142
2218
|
unbudgetedNote: (count) =>
|
|
2143
2219
|
count === 1
|
|
2144
2220
|
? '1 prompt is not covered by any budget pattern, so nothing is watching it.'
|
|
@@ -2153,7 +2229,7 @@ ${bold('EXAMPLES')}
|
|
|
2153
2229
|
footer: (source, level) => `Token counts ${source} · rule level \`${level}\``,
|
|
2154
2230
|
pricingOverlaid: (count, lastReviewed) =>
|
|
2155
2231
|
`Prices for ${count} ${count === 1 ? 'model' : 'models'} came from a local overlay reviewed ${lastReviewed}.`,
|
|
2156
|
-
sourceEstimated: () =>
|
|
2232
|
+
sourceEstimated: (band) => `estimated, ±${band}%`,
|
|
2157
2233
|
sourceExact: () => 'counted exactly',
|
|
2158
2234
|
measuringOptimised: () =>
|
|
2159
2235
|
'Measuring what the rules would leave, not what is written in the file.',
|
|
@@ -2175,7 +2251,7 @@ ${bold('EXAMPLES')}
|
|
|
2175
2251
|
check: {
|
|
2176
2252
|
okLabel: () => 'OK',
|
|
2177
2253
|
embeddedHeading: (path, count) =>
|
|
2178
|
-
`${path}
|
|
2254
|
+
`${path}, ${count} marked ${count === 1 ? 'prompt' : 'prompts'}`,
|
|
2179
2255
|
declinedHeading: (count) =>
|
|
2180
2256
|
`${count} ${count === 1 ? 'marker' : 'markers'} could not be read:`,
|
|
2181
2257
|
declinedAt: (line, detail) => `line ${line}: ${detail}`,
|
|
@@ -2187,7 +2263,7 @@ ${bold('EXAMPLES')}
|
|
|
2187
2263
|
stillTooBig: (optimizedTokens) =>
|
|
2188
2264
|
` Even optimised it does not fit (~${optimizedTokens} tokens): content has to be cut by hand.`,
|
|
2189
2265
|
directoryHeading: (directory, files) =>
|
|
2190
|
-
`${directory}
|
|
2266
|
+
`${directory}, ${files} ${files === 1 ? 'prompt' : 'prompts'}`,
|
|
2191
2267
|
directorySummary: (failures, files) =>
|
|
2192
2268
|
failures === 0
|
|
2193
2269
|
? `All ${files} within budget.`
|
|
@@ -2196,7 +2272,7 @@ ${bold('EXAMPLES')}
|
|
|
2196
2272
|
walkTruncated: () =>
|
|
2197
2273
|
'Stopped early: the directory is larger than the walk limit, so this is not the whole picture.',
|
|
2198
2274
|
filesFromSummary: (checked, listed, dropped) =>
|
|
2199
|
-
`Checking ${checked} of ${listed} listed file(s)
|
|
2275
|
+
`Checking ${checked} of ${listed} listed file(s), ${dropped} dropped: not a prompt extension, ignored by config, or no longer on disk.`,
|
|
2200
2276
|
filesFromNoBaseline: () =>
|
|
2201
2277
|
'The baseline gate is skipped under --files-from: it compares the whole repository against the committed record, and a partial list would read as removals. Budgets are checked per file as always; run "trazum check ." for the baseline.',
|
|
2202
2278
|
exactCountsCost: (files) =>
|
|
@@ -2204,7 +2280,7 @@ ${bold('EXAMPLES')}
|
|
|
2204
2280
|
},
|
|
2205
2281
|
profile: {
|
|
2206
2282
|
noTarget: () =>
|
|
2207
|
-
'Point this at a usage log: trazum profile usage.jsonl
|
|
2283
|
+
'Point this at a usage log: trazum profile usage.jsonl, one JSON object per line, each with a "model" and the "usage" object the API returned. Add "label" (which workload), "session" (which conversation) and "ts" (when) while you are there: without them every call looks alike, and the largest findings this command makes, conversation growth, and whether the cache TTL fits how fast your turns come, cannot be made at all. Recording it is four lines in your own code, it never contains prompt text, and the session key is grouped by and never printed.',
|
|
2208
2284
|
heading: () => 'Where the money went',
|
|
2209
2285
|
calls: (n) => plural(n, 'call'),
|
|
2210
2286
|
spent: (calls, total) => `${calls} · ${total}`,
|
|
@@ -2217,12 +2293,12 @@ ${bold('EXAMPLES')}
|
|
|
2217
2293
|
byModelHeading: () => 'By model',
|
|
2218
2294
|
row: (name, usd, pct, calls) => `${usd.padStart(11)} ${pct.padStart(5)} ${name} (${calls})`,
|
|
2219
2295
|
// Parenthesised so a real label named "unlabelled" cannot read identically
|
|
2220
|
-
// beside it
|
|
2296
|
+
// beside it, the data already keeps them apart; the display should too.
|
|
2221
2297
|
unlabelled: () => '(no label)',
|
|
2222
2298
|
cacheHit: (pct) => `Cache hit rate ${pct} of billable input.`,
|
|
2223
2299
|
cacheNever: () => 'Caching was never used on these calls. If any prefix repeats, that is the largest saving available.',
|
|
2224
2300
|
cacheLost: (usd, writes, reads) =>
|
|
2225
|
-
`Caching added ${usd} to this bill instead of taking it off. ${writes} tokens were written to the cache and ${reads} read back
|
|
2301
|
+
`Caching added ${usd} to this bill instead of taking it off. ${writes} tokens were written to the cache and ${reads} read back, and a write costs 1.25x plain input, or 2x at the 1-hour TTL. A prefix that changes faster than it is reused pays that premium for nothing. Either cache a prefix that holds still, or turn caching off here.`,
|
|
2226
2302
|
cachePaidOff: (usd) => `Caching took ${usd} off this bill, against the same tokens uncached.`,
|
|
2227
2303
|
cacheNoDifference: () =>
|
|
2228
2304
|
'Caching came out level on this bill: what it charged for these tokens is what they would have cost as ordinary input. It is neither paying for itself nor costing you anything.',
|
|
@@ -2240,7 +2316,7 @@ ${bold('EXAMPLES')}
|
|
|
2240
2316
|
outputDominates: (pct) =>
|
|
2241
2317
|
`Output is ${pct} of this bill, so shortening prompts has a low ceiling here. What moves it is asking for shorter answers and capping max_tokens.`,
|
|
2242
2318
|
unpriced: (models, calls) =>
|
|
2243
|
-
`${count(calls)} ${calls === 1 ? 'call is' : 'calls are'} not in these totals
|
|
2319
|
+
`${count(calls)} ${calls === 1 ? 'call is' : 'calls are'} not in these totals, because the pricing catalogue does not know: ${models}. Add them with a pricing overlay (--pricing) to include them.`,
|
|
2244
2320
|
skipped: (lineCount, lines) =>
|
|
2245
2321
|
`${count(lineCount)} ${lineCount === 1 ? 'line' : 'lines'} could not be read and ${lineCount === 1 ? 'was' : 'were'} left out (${lineCount === 1 ? 'line' : 'lines'} ${lines}).`,
|
|
2246
2322
|
empty: () => 'No usage records in that file.',
|
|
@@ -2248,78 +2324,78 @@ ${bold('EXAMPLES')}
|
|
|
2248
2324
|
'None of the models in that log are in the pricing catalogue, so there is no bill to report. Add them with a pricing overlay (--pricing) and run this again.',
|
|
2249
2325
|
leversHeading: () => 'What would actually move this bill',
|
|
2250
2326
|
leverSlice: (label, model, usd, pct) =>
|
|
2251
|
-
`${label} on ${model}
|
|
2327
|
+
`${label} on ${model}, up to ${usd} of this bill (${pct})`,
|
|
2252
2328
|
leverRoute: (candidate, usd) => `route it to ${candidate}, ${usd}`,
|
|
2253
2329
|
leverRouteVerify: (candidate) =>
|
|
2254
2330
|
`Whether that holds is an evaluation question, not an arithmetic one, and nothing here has seen a single answer. Measure it: trazum route <log> --prompt-file <prompt> --cases <cases> --yes`,
|
|
2255
2331
|
leverBatch: (usd) => `send it through the Batch API, ${usd}`,
|
|
2256
2332
|
leverCalls: (calls, spent) => `${calls}, ${spent} spent`,
|
|
2257
2333
|
leverPromptCeiling: (usd, pct) =>
|
|
2258
|
-
`For comparison: shortening the prompt text can touch ${usd} at the very most
|
|
2334
|
+
`For comparison: shortening the prompt text can touch ${usd} at the very most, ${pct} of this bill, and only if you deleted every input token. The real figure is far below that, because most of those tokens are retrieved context, conversation history and tool results that no prompt file contains.`,
|
|
2259
2335
|
historyHeading: () => 'What re-sending the conversation costs',
|
|
2260
2336
|
historyGrowth: (label, model, first, last, turns) =>
|
|
2261
2337
|
`${label} on ${model}: input ranges from ${first} tokens on the smallest turn to ${last} on the largest, over conversations of up to ${turns} turns.`,
|
|
2262
2338
|
historyCeiling: (usd, pct, flat, spent) =>
|
|
2263
|
-
`If every turn had been the size of its smallest one, that input would have cost ${flat} instead of ${spent}
|
|
2339
|
+
`If every turn had been the size of its smallest one, that input would have cost ${flat} instead of ${spent}, so at most ${usd} of this bill is conversation growth (${pct}). It is a ceiling and not a saving: some of that is the user's own new messages, which nothing can truncate away, and this reads counts rather than content so it cannot tell the two apart. What moves it is capping the history you replay, or summarising it.`,
|
|
2264
2340
|
truncatedWaste: (calls, usd, pct) =>
|
|
2265
|
-
`${calls} hit the max_tokens ceiling: ${usd} of the output spend (${pct}) bought answers that were cut off mid-generation
|
|
2341
|
+
`${calls} hit the max_tokens ceiling: ${usd} of the output spend (${pct}) bought answers that were cut off mid-generation, paid in full, frequently retried and billed again. Where the answer genuinely needs the room, raise max_tokens; where it does not, ask for less. Either way this is the one slice of a bill that is waste without a counterpart.`,
|
|
2266
2342
|
againstHeading: () => 'Against the previous log',
|
|
2267
2343
|
againstTotals: (before, after, delta, pct, callsBefore, callsAfter) =>
|
|
2268
|
-
`${before} → ${after} ${delta} (${pct}) ${callsBefore} → ${callsAfter}. Positive means the bill grew. Both figures are exactly what each file holds
|
|
2344
|
+
`${before} → ${after} ${delta} (${pct}) ${callsBefore} → ${callsAfter}. Positive means the bill grew. Both figures are exactly what each file holds, no period is assumed, so judge the call counts before judging the money.`,
|
|
2269
2345
|
againstDriver: (delta, label, before, after) => `${delta} ${label} (${before} → ${after})`,
|
|
2270
2346
|
againstDriverNew: (delta, label) => `${delta} ${label} (new since the previous log)`,
|
|
2271
2347
|
againstDriverGone: (delta, label) => `${delta} ${label} (gone since the previous log)`,
|
|
2272
2348
|
againstByModel: () =>
|
|
2273
|
-
'The same change, by model
|
|
2349
|
+
'The same change, by model, where the mix moved:',
|
|
2274
2350
|
labelPrefixBelowMinimum: (file, prefix, minimum, model) =>
|
|
2275
|
-
`${file} (as it is today
|
|
2351
|
+
`${file} (as it is today, and the log may predate it): the stable prefix is ${prefix} tokens and ${model} caches nothing under ${minimum}. Setting cache_control there does not error, it simply never caches, which is what a cache that only writes looks like from the bill.`,
|
|
2276
2352
|
labelPrefixMovable: (file, movable, prefix) =>
|
|
2277
|
-
`${file} (as it is today
|
|
2353
|
+
`${file} (as it is today, and the log may predate it): ~${movable} stable tokens sit after the first placeholder, where caching cannot reach them; the cacheable prefix is ${prefix}. "trazum optimize ${file} --reorder" moves them in front and shows the diff.`,
|
|
2278
2354
|
labelPrefixHealthy: (file, prefix, minimum) =>
|
|
2279
|
-
`${file} (as it is today
|
|
2355
|
+
`${file} (as it is today, and the log may predate it): the stable prefix is ${prefix} tokens, over the ${minimum} minimum. The prompt file is not the problem; look at whether the prefix is byte-identical between calls.`,
|
|
2280
2356
|
labelFileMissing: (label, file) =>
|
|
2281
|
-
`labels["${label}"] points at ${file}, which does not exist
|
|
2357
|
+
`labels["${label}"] points at ${file}, which does not exist: the mapping was skipped.`,
|
|
2282
2358
|
againstNothingPriced: () =>
|
|
2283
2359
|
'The previous log has nothing the pricing catalogue knows, so there is no comparison to make.',
|
|
2284
2360
|
truncatedNotRecorded: () =>
|
|
2285
|
-
'Whether any answers were cut off could not be measured
|
|
2361
|
+
'Whether any answers were cut off could not be measured, no call in this log carries a stop reason. Add "stop_reason" (Anthropic) or "finish_reason" (OpenAI) to the record; the API already returns it beside "usage".',
|
|
2286
2362
|
historyNoSessions: () =>
|
|
2287
|
-
'No call in this log carried a session, so what re-sending the conversation costs could not be measured
|
|
2363
|
+
'No call in this log carried a session, so what re-sending the conversation costs could not be measured, usually the largest line on a chat or agent bill. Add "session" (or "conversation_id") to the record and run this again. Trazum groups by it and never prints it.',
|
|
2288
2364
|
leversUnlabelled: () =>
|
|
2289
|
-
'None of these calls carried a label, so this is every workload in one row
|
|
2365
|
+
'None of these calls carried a label, so this is every workload in one row, a classifier and a RAG pipeline merged into a single figure, with one route suggested for both. Add "label" to the record and the levers split by workload, which is the grouping a decision is actually made at.',
|
|
2290
2366
|
outputShapeHeading: () => 'Where the output spend concentrates',
|
|
2291
2367
|
outputTail: (label, model, callPct, spendPct, above, usd) =>
|
|
2292
|
-
`${label} on ${model}: ${callPct} of calls hold ${spendPct} of the output spend
|
|
2368
|
+
`${label} on ${model}: ${callPct} of calls hold ${spendPct} of the output spend, the ones answering with more than ${above} tokens, out of ${usd} of output on this slice.`,
|
|
2293
2369
|
outputTailAdvice: () =>
|
|
2294
2370
|
'That is a tail, and a tail has a cause: a path through the prompt that invites an essay, a call with no max_tokens, a retrieval that returned a book. Finding it is a morning; it is not "make everything shorter".',
|
|
2295
2371
|
outputFlat: (label, model, callPct, spendPct, usd) =>
|
|
2296
|
-
`${label} on ${model}: the output spend sits where the calls are
|
|
2372
|
+
`${label} on ${model}: the output spend sits where the calls are, ${callPct} of them hold ${spendPct} of ${usd}. There is no tail to hunt.`,
|
|
2297
2373
|
outputFlatAdvice: () =>
|
|
2298
2374
|
'The answer length is the task here, so the levers are the blunt ones: ask for shorter answers in the prompt, and cap max_tokens.',
|
|
2299
2375
|
outputPercentiles: (p50, p95) =>
|
|
2300
|
-
`Half the measured answers fit within ${p50} output tokens, and 95% within ${p95}
|
|
2376
|
+
`Half the measured answers fit within ${p50} output tokens, and 95% within ${p95}, the number a max_tokens cap actually wants. Measured on these calls, promised for nothing.`,
|
|
2301
2377
|
inputShapeHeading: () => 'How big these calls are',
|
|
2302
2378
|
inputSkewed: (label, model, p50, p95, ratio, usd) =>
|
|
2303
|
-
`${label} on ${model} is uneven: half its calls fit within ${p50} input tokens and 95% within ${p95}
|
|
2379
|
+
`${label} on ${model} is uneven: half its calls fit within ${p50} input tokens and 95% within ${p95}, about ${ratio}x the ordinary call, over ${usd} of input spend.`,
|
|
2304
2380
|
inputSkewedAdvice: () =>
|
|
2305
2381
|
'Past four times the median, the ordinary call is fine and something is growing on top of it: a conversation nobody truncates, a retrieval with no cap, a tool result pasted in whole. The fix is a limit on the large calls, not a rewrite of the prompt every call sends.',
|
|
2306
2382
|
inputEven: (label, model, p50, p95, usd) =>
|
|
2307
2383
|
`${label} on ${model} is even: half its calls fit within ${p50} input tokens and 95% within ${p95}, over ${usd} of input spend.`,
|
|
2308
2384
|
inputEvenAdvice: () =>
|
|
2309
|
-
'The large calls are not much larger than the ordinary one, so there is no tail to cap
|
|
2385
|
+
'The large calls are not much larger than the ordinary one, so there is no tail to cap: the prompt is simply big. The levers are fewer retrieved documents, a shorter system block, and caching if the prefix repeats.',
|
|
2310
2386
|
inputHuge: (label, model, calls, usd) =>
|
|
2311
|
-
`${label} on ${model}: every one of its ${calls} is larger than this tool measures precisely, over ${usd} of input spend. No ceiling is named because there is none to name honestly
|
|
2387
|
+
`${label} on ${model}: every one of its ${calls} is larger than this tool measures precisely, over ${usd} of input spend. No ceiling is named because there is none to name honestly: that size is itself the finding.`,
|
|
2312
2388
|
repeatsHeading: () => 'The same request, sent again',
|
|
2313
2389
|
repeatsFound: (label, model, repeats, checked, seconds, usd) =>
|
|
2314
2390
|
`${label} on ${model}: ${repeats} of ${checked} calls re-sent the previous call's exact input size within ${seconds} seconds, in the same conversation, costing ${usd}.`,
|
|
2315
2391
|
pressureHeading: () => 'Approaching the context window',
|
|
2316
2392
|
pressureLine: (label, model, tokens, window, share) =>
|
|
2317
|
-
`${label} on ${model}: the largest call carried ${tokens} input tokens against a ${window}-token window
|
|
2393
|
+
`${label} on ${model}: the largest call carried ${tokens} input tokens against a ${window}-token window, ${share} of the ceiling.`,
|
|
2318
2394
|
mixDriftHeading: () => 'The mix moved inside this log',
|
|
2319
2395
|
mixDriftLine: (model, firstShare, lastShare, firstDays, lastDays, lastUsd) =>
|
|
2320
|
-
`${model} went from ${firstShare} of the spend in the first ${firstDays} days to ${lastShare} in the last ${lastDays}
|
|
2396
|
+
`${model} went from ${firstShare} of the spend in the first ${firstDays} days to ${lastShare} in the last ${lastDays}, ${lastUsd} of the recent half.`,
|
|
2321
2397
|
truncationRetryLine: (label, model, retried, truncated, seconds, wasted, retry) =>
|
|
2322
|
-
`${label} on ${model}: ${retried} of ${truncated} truncated answers were followed within ${seconds} seconds by another call in the same conversation
|
|
2398
|
+
`${label} on ${model}: ${retried} of ${truncated} truncated answers were followed within ${seconds} seconds by another call in the same conversation, ${wasted} spent on the cut attempts, plus ${retry} on the follow-ups.`,
|
|
2323
2399
|
truncationRetryNote: () =>
|
|
2324
2400
|
'The pair is the shape a retry has; the log cannot see content, so whether each was one is yours to know. Both sides of it are real money, and the fix is the same either way: a max_tokens the answers actually fit in.',
|
|
2325
2401
|
mixDriftNote: () =>
|
|
@@ -2327,9 +2403,9 @@ ${bold('EXAMPLES')}
|
|
|
2327
2403
|
pressureAdvice: () =>
|
|
2328
2404
|
'At 100% the call fails outright, and nothing on the bill changes until that day. The levers are a cap on retrieved context, truncating conversation history, or a larger-window model. When it crosses is not predicted here: the share is a fact, the trajectory is yours to know.',
|
|
2329
2405
|
repeatsAdvice: () =>
|
|
2330
|
-
'A conversation\'s input grows with every turn, so the same size twice in a row seconds apart is usually a retry after a timeout, an agent step repeating, or a loop
|
|
2406
|
+
'A conversation\'s input grows with every turn, so the same size twice in a row seconds apart is usually a retry after a timeout, an agent step repeating, or a loop: this reads counts and cannot see content, so it names the pattern and stops. Whatever it is, the money bought nothing the call before it had not already paid for.',
|
|
2331
2407
|
inputMostlyCached: (share) =>
|
|
2332
|
-
`${share} of those tokens were cache reads, billed at a tenth of the input rate
|
|
2408
|
+
`${share} of those tokens were cache reads, billed at a tenth of the input rate: the size is real and most of it is cheap.`,
|
|
2333
2409
|
inputFullRate: () =>
|
|
2334
2410
|
'Almost none of that was a cache read, so every one of those tokens was billed at the full input rate. If any prefix repeats across these calls, caching is the lever with the largest ceiling here.',
|
|
2335
2411
|
leversNone: () =>
|
|
@@ -2337,60 +2413,60 @@ ${bold('EXAMPLES')}
|
|
|
2337
2413
|
assumedWriteTtl: (calls) =>
|
|
2338
2414
|
`${count(calls)} ${calls === 1 ? 'call did' : 'calls did'} not say which cache-write TTL was used, so the cheaper 5-minute rate was assumed. A 1-hour entry costs 2x input rather than 1.25x, so this total is a floor for those calls. Record the "cache_creation" object the API returns to remove the assumption.`,
|
|
2339
2415
|
spanLine: (from, to, days) =>
|
|
2340
|
-
`This log covers ${from} → ${to} (${days} days). The span is stated, never extrapolated
|
|
2416
|
+
`This log covers ${from} → ${to} (${days} days). The span is stated, never extrapolated: the monthly arithmetic is yours to do, and now it is valid.`,
|
|
2341
2417
|
spanPartial: (withTs, total) =>
|
|
2342
2418
|
`Only ${withTs} of ${total} calls carry a timestamp; the span describes those.`,
|
|
2343
2419
|
ttlFitExpires: (label, model, gap) =>
|
|
2344
|
-
`${label} on ${model}: turns arrive a median of ${gap} apart and the 5-minute entry is gone by then
|
|
2420
|
+
`${label} on ${model}: turns arrive a median of ${gap} apart and the 5-minute entry is gone by then, writes expire before the next turn reads them, which from the bill is a cache that only writes. The 1-hour TTL costs 2x input to write and would survive these gaps; the other honest option is caching switched off here.`,
|
|
2345
2421
|
ttlFitExpiresBoth: (label, model, gap) =>
|
|
2346
|
-
`${label} on ${model}: turns arrive a median of ${gap} apart, and no cache entry lives that long
|
|
2422
|
+
`${label} on ${model}: turns arrive a median of ${gap} apart, and no cache entry lives that long, even the 1-hour TTL is gone by the next turn. Caching cannot work at this pace; turn it off here and stop paying the write premium.`,
|
|
2347
2423
|
ttlFitOverlong: (label, model, gap, usd) =>
|
|
2348
|
-
`${label} on ${model}: turns arrive a median of ${gap} apart
|
|
2424
|
+
`${label} on ${model}: turns arrive a median of ${gap} apart, comfortably inside the 5-minute window, and these writes pay the 1-hour rate, 2x input against 1.25x, for endurance the gaps never use. The same writes at the 5-minute TTL are ${usd} cheaper on this log, and that figure is exact: the same tokens at the other published rate.`,
|
|
2349
2425
|
ttlFitUnsettledGap: (label, model, gap) =>
|
|
2350
|
-
`${label} on ${model}: turns arrive a median of ${gap} apart
|
|
2426
|
+
`${label} on ${model}: turns arrive a median of ${gap} apart, which a 5-minute entry does not survive and a 1-hour one does, and the log did not record which these writes were, so whether they ever get read back cannot be settled. Record the "cache_creation" object the API returns and it settles itself.`,
|
|
2351
2427
|
ttlFitFits: (label, model, gap) =>
|
|
2352
2428
|
`${label} on ${model}: turns arrive a median of ${gap} apart, inside the lifetime these writes use. The TTL is not the problem here.`,
|
|
2353
2429
|
ttlFitUnmeasured: () =>
|
|
2354
|
-
'Whether the cache TTL fits how fast the turns arrive could not be measured
|
|
2430
|
+
'Whether the cache TTL fits how fast the turns arrive could not be measured, it needs both "session" and "ts" on the record. A 5-minute entry on a workload whose turns come nine minutes apart expires unread on every write, and only the clock can see it. Trazum groups by the session and never shows it.',
|
|
2355
2431
|
dayPeak: (day, usd, xMedian) =>
|
|
2356
2432
|
`The most expensive day in this log was ${day}: ${usd}, ${xMedian}x the median day.`,
|
|
2357
2433
|
dayPeakLabel: (label, usd) => `Most of it was ${label} (${usd}).`,
|
|
2358
2434
|
maxUsdOk: (total, max) => `Within budget: ${total} spent against --max-usd ${max}.`,
|
|
2359
2435
|
maxUsdFailed: (total, max) =>
|
|
2360
|
-
`FAILED
|
|
2436
|
+
`FAILED: this log spent ${total} against a --max-usd of ${max}. The figures are the provider's own billed counts over exactly this log; no period was assumed.`,
|
|
2361
2437
|
maxGrowthUsdFailed: (delta, max) =>
|
|
2362
|
-
`FAILED
|
|
2438
|
+
`FAILED, the bill grew ${delta} against the previous log, over the --max-growth-usd limit of ${max}.`,
|
|
2363
2439
|
maxGrowthNeedsAgainst: () =>
|
|
2364
2440
|
'--max-growth-usd has nothing to compare without --against <previous.jsonl>. On its own it would have run silently and gated nothing, which is not an answer.',
|
|
2365
2441
|
maxCacheLossOk: (worst, max) =>
|
|
2366
2442
|
`Cache within budget: caching cost at most ${worst} against --max-cache-loss-usd ${max}, worst case included.`,
|
|
2367
2443
|
maxCacheLossFailed: (delta, max) =>
|
|
2368
|
-
`FAILED
|
|
2444
|
+
`FAILED, caching added ${delta} to this bill (the same tokens as plain input would have cost less), over the --max-cache-loss-usd limit of ${max}. The counterfactual is exact: the same tokens at the published input rate.`,
|
|
2369
2445
|
maxDayOk: (day, usd, max) =>
|
|
2370
2446
|
`No single day over budget: the worst was ${day} at ${usd}, against --max-day-usd ${max}.`,
|
|
2371
2447
|
maxDayFailed: (day, usd, max) =>
|
|
2372
|
-
`FAILED
|
|
2448
|
+
`FAILED, ${day} spent ${usd}, over the --max-day-usd limit of ${max}. A total under budget can hide a single runaway day, which is what this gate exists to catch.`,
|
|
2373
2449
|
maxDayNoClock: () =>
|
|
2374
|
-
'FAILED
|
|
2450
|
+
'FAILED, --max-day-usd was asked for and no record in this log carries a timestamp, so there are no days to judge. That is not a pass: a bill nobody could measure by day is not a bill that stayed under a daily budget. Add "ts" to the record and the gate arms.',
|
|
2375
2451
|
maxSessionOk: (worst, max, sessions) =>
|
|
2376
2452
|
`No conversation over budget: the most expensive of ${sessions} cost ${worst}, against --max-session-usd ${max}. A conversation that started before this log is counted only for the turns recorded here, so this is a floor.`,
|
|
2377
2453
|
maxSessionFailed: (worst, max, sessions) =>
|
|
2378
|
-
`FAILED
|
|
2454
|
+
`FAILED: the most expensive of ${sessions} conversations cost ${worst}, over the --max-session-usd limit of ${max}. A month's budget and a day's budget both pass while one conversation loops its way through this; the per-conversation figure is the one that catches it.`,
|
|
2379
2455
|
maxSessionNoSessions: () =>
|
|
2380
|
-
'FAILED
|
|
2456
|
+
'FAILED, --max-session-usd was asked for and no record in this log carries a session, so there are no conversations to judge. That is not a pass. Add "session" (or "conversation_id") to the record and the gate arms; Trazum groups by it and never prints it.',
|
|
2381
2457
|
maxDayUndated: (calls) =>
|
|
2382
|
-
`${calls} calls carry no timestamp, so they are in the bill and in none of the days above
|
|
2458
|
+
`${calls} calls carry no timestamp, so they are in the bill and in none of the days above: the worst day is a floor by whatever they held. A failure would stand regardless; this pass is over the part that could be dated.`,
|
|
2383
2459
|
maxCacheLossWorstCase: (calls, worst, max) =>
|
|
2384
|
-
`FAILED
|
|
2460
|
+
`FAILED, ${count(calls)} ${calls === 1 ? 'call' : 'calls'} did not record which cache-write TTL was paid, and at the 1-hour rate caching added up to ${worst}, over the --max-cache-loss-usd limit of ${max}. The gate reads the worst case on purpose: a gate reading the flattering half would pass exactly the bills it exists to catch. Record the "cache_creation" object the API returns and the ceiling becomes a figure.`,
|
|
2385
2461
|
pricesStale: (date, days) =>
|
|
2386
|
-
`The price table behind every dollar here was last reviewed ${date}
|
|
2462
|
+
`The price table behind every dollar here was last reviewed ${date}, ${count(days)} days ago, past the 45 this tool considers current. If the provider changed prices since, this report is wrong by exactly that change. --pricing-live fetches today's prices; --pricing overlays your own.`,
|
|
2387
2463
|
dayTableDay: () => 'Day (UTC)',
|
|
2388
2464
|
dayTableCalls: () => 'calls',
|
|
2389
2465
|
dayTableTop: () => 'biggest that day',
|
|
2390
2466
|
dayTableEarlier: (days) =>
|
|
2391
2467
|
`…and ${count(days)} earlier ${days === 1 ? 'day' : 'days'} not shown here. The full series rides --json as spendByDay.`,
|
|
2392
2468
|
gateOnFloor: (reasons) =>
|
|
2393
|
-
`Note: the gated figure is a floor, not the bill
|
|
2469
|
+
`Note: the gated figure is a floor, not the bill, ${reasons}. Whatever those calls cost is not in the number the gate just judged, so a pass here means "the part I could read fits", never "the bill fits".`,
|
|
2394
2470
|
floorSkipped: (lines) =>
|
|
2395
2471
|
`${count(lines)} ${lines === 1 ? 'line was' : 'lines were'} unreadable and left out`,
|
|
2396
2472
|
floorUnpriced: (calls) =>
|
|
@@ -2398,37 +2474,37 @@ ${bold('EXAMPLES')}
|
|
|
2398
2474
|
floorUndated: (calls) =>
|
|
2399
2475
|
`${count(calls)} ${calls === 1 ? 'call carries' : 'calls carry'} no timestamp and fell outside the window`,
|
|
2400
2476
|
sessionCost: (label, model, sessions, median, medianTurns, p95, max) =>
|
|
2401
|
-
`${label} on ${model}: across ${sessions} conversations, the median one costs ${median} over ${medianTurns} turns, 95% come in under ${p95}, and the most expensive was ${max}. Exact billed counts, per conversation
|
|
2477
|
+
`${label} on ${model}: across ${sessions} conversations, the median one costs ${median} over ${medianTurns} turns, 95% come in under ${p95}, and the most expensive was ${max}. Exact billed counts, per conversation: the figure a per-seat price or a quota is set from. A conversation that started before this log or continues after it is counted only for the turns recorded here.`,
|
|
2402
2478
|
labelBudgetOk: (label, usd, max) => `Within budget: ${label} spent ${usd} against ${max}.`,
|
|
2403
2479
|
labelBudgetFailed: (label, usd, max) =>
|
|
2404
|
-
`FAILED
|
|
2480
|
+
`FAILED, ${label} spent ${usd} against its budget of ${max} in trazum.config.json.`,
|
|
2405
2481
|
labelBudgetMissing: (label) =>
|
|
2406
2482
|
`${label} has a budget in trazum.config.json and no calls in this log, so nothing was measured for it. Not a pass: a workload that did not appear is not a workload that came in under budget.`,
|
|
2407
2483
|
labelBudgetWindowed: () =>
|
|
2408
2484
|
'Per-label budgets in trazum.config.json were not applied: --since/--until make "what this label spent" mean a slice, and a budget written for the whole period would be gating against something it does not describe.',
|
|
2409
2485
|
duplicateLines: (calls, usd) =>
|
|
2410
|
-
`${plural(calls, 'line is an exact duplicate', `lines are exact duplicates`)} of an earlier line
|
|
2486
|
+
`${plural(calls, 'line is an exact duplicate', `lines are exact duplicates`)} of an earlier line, same counts, same label and session, same millisecond, and that adds ${usd} to the total above. If a log was exported twice or two files in a directory overlap, this bill is overstated by that much. Two real calls colliding on all of that is possible; it is just unlikely.`,
|
|
2411
2487
|
budgetVsWire: (label, file, budget, perCall, share) =>
|
|
2412
|
-
`The budget on ${file} is ${budget} tokens, and calls labelled ${label} carry about ${perCall} input tokens each
|
|
2488
|
+
`The budget on ${file} is ${budget} tokens, and calls labelled ${label} carry about ${perCall} input tokens each, so that gate governs roughly ${share} of what actually goes up the wire. The rest is retrieved context, conversation history and tool results, which no prompt file contains and no budget on one can see. The budget is not wrong; it is just smaller than the bill.`,
|
|
2413
2489
|
badCsvShape: (value) =>
|
|
2414
|
-
`--csv-shape does not know "${value}". It takes "slice" (one row per label and model, the default), "day", "hour" or "model-day" (one row per day and model
|
|
2490
|
+
`--csv-shape does not know "${value}". It takes "slice" (one row per label and model, the default), "day", "hour" or "model-day" (one row per day and model, the long format a pivot table wants).`,
|
|
2415
2491
|
whatIfHeading: (model) => `These exact calls on ${model}`,
|
|
2416
2492
|
whatIfAssumption: () =>
|
|
2417
2493
|
'This is multiplication, not advice: the same token counts at another rate card. It says nothing about whether that model could do the work, and a model that answers at greater length or gets retried would not send these counts at all.',
|
|
2418
2494
|
whatIfTotal: (current, target, delta) =>
|
|
2419
|
-
`${current} of movable spend would have been ${target}
|
|
2495
|
+
`${current} of movable spend would have been ${target}, a difference of ${delta}.`,
|
|
2420
2496
|
whatIfCheaper: () =>
|
|
2421
2497
|
'Verify before you move anything: trazum route measures one prompt against both models on your own examples.',
|
|
2422
2498
|
whatIfDearer: () => 'That direction costs more. The arithmetic is here so the number is not a guess.',
|
|
2423
2499
|
whatIfBatchOnTarget: (batched, moved) =>
|
|
2424
|
-
`If those calls can also wait, the target's Batch API takes the moved bill from ${moved} to ${batched}
|
|
2500
|
+
`If those calls can also wait, the target's Batch API takes the moved bill from ${moved} to ${batched}, the discount applies to the target's rates, not the ones you left. Whether they can wait is not in the log; that half of the decision is yours.`,
|
|
2425
2501
|
whatIfCacheBeyond: (largest, min, noCache) =>
|
|
2426
|
-
`Its cache traffic could not exist there: the largest call is ${largest} tokens against the target's ${min}-token cache minimum, so no call in this slice could create an entry. Without the cache the same tokens cost ${noCache}
|
|
2502
|
+
`Its cache traffic could not exist there: the largest call is ${largest} tokens against the target's ${min}-token cache minimum, so no call in this slice could create an entry. Without the cache the same tokens cost ${noCache}: that is the figure the target would actually bill, and the row above flatters the move.`,
|
|
2427
2503
|
whatIfSlice: (label, model, current, target) => `${label} on ${model}: ${current} → ${target}`,
|
|
2428
2504
|
whatIfOverContext: (label, tokens, window, usd) =>
|
|
2429
2505
|
`${label} cannot move: its largest call carries ${tokens} input tokens and that model's window is ${window}. Those calls would fail, not cost less, so their ${usd} is excluded from the figures above.`,
|
|
2430
2506
|
whatIfAlreadyThere: (calls, usd) =>
|
|
2431
|
-
`Already on that model: ${calls} worth ${usd}, left out of the figures above
|
|
2507
|
+
`Already on that model: ${calls} worth ${usd}, left out of the figures above, because money that cannot move would make the difference look smaller than it is.`,
|
|
2432
2508
|
whatIfUnpriced: (calls, models) =>
|
|
2433
2509
|
`Excluded: ${calls} whose model has no price here (${models}). Their cost on the target is knowable; the difference is not, because there is no current figure to subtract from.`,
|
|
2434
2510
|
whatIfNothingToMove: () =>
|
|
@@ -2437,30 +2513,30 @@ ${bold('EXAMPLES')}
|
|
|
2437
2513
|
`--what-if does not know "${value}". Priced models: ${available}. Add it with --pricing if you have its rates.`,
|
|
2438
2514
|
badGzip: (file, detail) =>
|
|
2439
2515
|
`${file} is gzipped and would not decompress: ${detail}. Reading the rest and saying nothing would report a bill missing whatever that file held, so this stops instead. Check the file, or move it out of the directory.`,
|
|
2440
|
-
dryRunHeading: () => 'What this log can answer
|
|
2516
|
+
dryRunHeading: () => 'What this log can answer, no bill was produced',
|
|
2441
2517
|
dryRunParsed: (parsed, skipped) =>
|
|
2442
2518
|
`${parsed} records parse; ${skipped} lines could not be read. A dry run prices nothing: the point is what the gates you are about to wire would stand on.`,
|
|
2443
2519
|
dryRunUnpriced: (models) =>
|
|
2444
2520
|
`Models the price table does not know: ${models}. Their tokens parse; their dollars need a --pricing overlay.`,
|
|
2445
2521
|
dryRunTotals: () => 'The bill itself: totals, per-model split, cache economics, the levers.',
|
|
2446
2522
|
dryRunLabels: (share) =>
|
|
2447
|
-
`Per-workload findings and label budgets
|
|
2523
|
+
`Per-workload findings and label budgets, "label" on ${share} of records.`,
|
|
2448
2524
|
dryRunClock: (share) =>
|
|
2449
|
-
`The period, per-day and per-hour shape, --max-day-usd, --since/--until
|
|
2525
|
+
`The period, per-day and per-hour shape, --max-day-usd, --since/--until, a timestamp on ${share} of records.`,
|
|
2450
2526
|
dryRunSessions: (share) =>
|
|
2451
|
-
`Conversation growth, per-conversation cost, --max-session-usd
|
|
2527
|
+
`Conversation growth, per-conversation cost, --max-session-usd, a session on ${share} of records. Grouped by, never printed.`,
|
|
2452
2528
|
dryRunStopReason: (share) =>
|
|
2453
|
-
`Truncated answers and their retry bill
|
|
2529
|
+
`Truncated answers and their retry bill, a stop reason on ${share} of records.`,
|
|
2454
2530
|
dryRunCacheTtl: (ttl, writes) =>
|
|
2455
|
-
`Settled cache verdicts
|
|
2531
|
+
`Settled cache verdicts, the "cache_creation" split on ${ttl} of ${writes} cache-writing records.`,
|
|
2456
2532
|
dryRunNoCacheTraffic: () =>
|
|
2457
|
-
'Cache verdicts: nothing wrote to the cache in this log, so there is nothing to settle
|
|
2533
|
+
'Cache verdicts: nothing wrote to the cache in this log, so there is nothing to settle, not a missing field.',
|
|
2458
2534
|
dryRunFooter: () =>
|
|
2459
2535
|
'A ✗ is not a defect in the log; it is a finding this log cannot support yet. The README\'s recording recipe carries every field above.',
|
|
2460
2536
|
dryRunNoGates: () =>
|
|
2461
2537
|
'--dry-run produces no bill, so a gate beside it would exit green having judged nothing. Run the gates without --dry-run.',
|
|
2462
2538
|
bySourceNeedsConfig: () =>
|
|
2463
|
-
'--by-source reads the "sources" block in trazum.config.json
|
|
2539
|
+
'--by-source reads the "sources" block in trazum.config.json, a name per service, each with glob patterns over log paths, and this config has none. Name at least one source.',
|
|
2464
2540
|
bySourceNothingMatched: (sources) =>
|
|
2465
2541
|
'No log file matched any source pattern. The sources configured: ${sources}. Patterns match the paths as given on the command line.'.replace('${sources}', sources),
|
|
2466
2542
|
fleetHeading: (count, total, calls) => `The fleet: ${count} sources · ${total} · ${calls}`,
|
|
@@ -2470,38 +2546,38 @@ ${bold('EXAMPLES')}
|
|
|
2470
2546
|
fleetWorst: (name, usd, share) =>
|
|
2471
2547
|
`${name} is where the money is: ${usd}, ${share} of the fleet's total.`,
|
|
2472
2548
|
fleetMismatchedSpans: () =>
|
|
2473
|
-
'These sources cover different periods, so the shares above compare totals, not rates
|
|
2549
|
+
'These sources cover different periods, so the shares above compare totals, not rates: a 3-day log looks cheap next to a 30-day one for reasons that have nothing to do with cost. Each row states its own span.',
|
|
2474
2550
|
fleetSplitBrain: (label, detail) =>
|
|
2475
|
-
`The same workload runs on different models in different sources
|
|
2551
|
+
`The same workload runs on different models in different sources, ${label}: ${detail}. Same job, different rate cards; whether that is a decision or an accident is not in the logs.`,
|
|
2476
2552
|
fleetCacheUnderwater: (name, usd) =>
|
|
2477
|
-
`Caching pays for the fleet overall but loses ${usd} in ${name}
|
|
2553
|
+
`Caching pays for the fleet overall but loses ${usd} in ${name}: the aggregate verdict was hiding it.`,
|
|
2478
2554
|
fleetUnmatched: (file) =>
|
|
2479
|
-
`${file} matched no source pattern, so it is in no report above
|
|
2555
|
+
`${file} matched no source pattern, so it is in no report above, spend missing from every bill until a pattern covers it.`,
|
|
2480
2556
|
fleetFooter: () =>
|
|
2481
2557
|
'Summaries per source; the full report for one service is "trazum profile <its logs>". Same thresholds, same findings, one source at a time.',
|
|
2482
2558
|
fleetBudgetOk: (name, usd, max) => `Within budget: ${name} spent ${usd} against its ${max} in spend.bySource.`,
|
|
2483
2559
|
fleetBudgetFailed: (name, usd, max) =>
|
|
2484
|
-
`FAILED
|
|
2560
|
+
`FAILED, ${name} spent ${usd} against its budget of ${max} in spend.bySource. The fleet total can be fine while one service bleeds; this gate names which.`,
|
|
2485
2561
|
fleetBudgetMissing: (name) =>
|
|
2486
2562
|
`${name} has a budget in spend.bySource and no logs matched it in this run, so nothing was measured for it. Not a pass: a service that did not appear is not a service under budget.`,
|
|
2487
2563
|
coverageHeading: () => 'What this log cannot answer yet',
|
|
2488
2564
|
needsLabel: (seen) =>
|
|
2489
2565
|
`"label" on ${seen} records: without it every workload is one row, so no per-workload spend, no drill-down, and the levers describe a mixture rather than a decision.`,
|
|
2490
2566
|
needsOutcome: (seen) =>
|
|
2491
|
-
`an "outcome"
|
|
2567
|
+
`an "outcome", ${seen}. The one field that changes what every other figure here means: without it this tool can say a workload got 40% cheaper and cannot say whether it stopped working. Record your own word for what happened and declare the vocabulary under "outcomes".`,
|
|
2492
2568
|
dryRunOutcomes: (share) =>
|
|
2493
2569
|
`cost per outcome and a success rate (${share} of records carry an "outcome")`,
|
|
2494
2570
|
outcomeHeading: () => 'Outcomes',
|
|
2495
2571
|
outcomeRate: (rate, ofUsd) =>
|
|
2496
|
-
`${rate} of ${ofUsd} in declared outcomes succeeded
|
|
2572
|
+
`${rate} of ${ofUsd} in declared outcomes succeeded, by spend rather than by call, because the two diverge exactly when the expensive half is the half that fails.`,
|
|
2497
2573
|
outcomeNoRate: (why) =>
|
|
2498
2574
|
why === 'nothing-recorded'
|
|
2499
|
-
? 'No success rate: nothing in this log recorded an outcome. That is not a rate of zero
|
|
2575
|
+
? 'No success rate: nothing in this log recorded an outcome. That is not a rate of zero, a rate of zero is a real and terrible measurement, and this is nobody having told us.'
|
|
2500
2576
|
: 'No success rate: "outcomes.success" declares no values, so nothing here counts as one. A legitimate thing to declare, and it means the rate is not this tool\u2019s to compute.',
|
|
2501
2577
|
outcomeUnrecorded: (share, usd) =>
|
|
2502
2578
|
`${share} of the bill (${usd}) carried no outcome, and is in neither half of the rate above.`,
|
|
2503
2579
|
outcomeUndeclared: (values) =>
|
|
2504
|
-
`Not declared in "outcomes.values": ${values}. Named rather than counted as failures
|
|
2580
|
+
`Not declared in "outcomes.values": ${values}. Named rather than counted as failures, a typo in an exporter should look like a typo, not like a product regression.`,
|
|
2505
2581
|
perOutcomeHeading: () => 'What an outcome costs',
|
|
2506
2582
|
perOutcomeRow: (key, perCall, perOutcome, coverage) => `${key} ${perCall} ${perOutcome} ${coverage}`,
|
|
2507
2583
|
perOutcomeColumns: {
|
|
@@ -2521,29 +2597,29 @@ ${bold('EXAMPLES')}
|
|
|
2521
2597
|
? 'not recorded'
|
|
2522
2598
|
: 'no vocabulary',
|
|
2523
2599
|
perOutcomeNumerator: () =>
|
|
2524
|
-
'Per success divides the spend on calls that recorded an outcome, never the whole bill
|
|
2600
|
+
'Per success divides the spend on calls that recorded an outcome, never the whole bill, dividing everything would charge your uninstrumented traffic to your measured successes and report a figure too high by exactly the uncovered share, silently, in the direction that gets a working feature killed. "recorded" is what share of each workload\u2019s spend the figure covers.',
|
|
2525
2601
|
perOutcomeDisagreement: (key, callRank, outcomeRank) =>
|
|
2526
2602
|
`${key} is #${callRank} by cost per call and #${outcomeRank} by cost per success.`,
|
|
2527
2603
|
perOutcomeBothOrders: () =>
|
|
2528
2604
|
'Cheapest per call and cheapest per success are different orders, and both are printed rather than one being picked. A workload can move up one while moving down the other, and somebody optimising on the first number would be moving the wrong one.',
|
|
2529
2605
|
outcomeColumns: { outcome: 'outcome', calls: 'calls', spend: 'spend' },
|
|
2530
2606
|
verdictSuccess: () => 'success',
|
|
2531
|
-
verdictOther: () => '
|
|
2607
|
+
verdictOther: () => 'other',
|
|
2532
2608
|
verdictUndeclared: () => 'undeclared',
|
|
2533
2609
|
needsSession: (seen) =>
|
|
2534
2610
|
`"session" on ${seen} records: without it there is no conversation growth, no per-conversation cost, and no cache-TTL fit. It is grouped by and never printed.`,
|
|
2535
2611
|
needsTs: (seen) =>
|
|
2536
2612
|
`"ts" on ${seen} records: without it the log has no period, no per-day or per-hour shape, and the cache-TTL question cannot be asked at all.`,
|
|
2537
2613
|
needsStopReason: (seen) =>
|
|
2538
|
-
`"stop_reason" (Anthropic) or "finish_reason" (OpenAI) on ${seen} records: without it, answers cut off at max_tokens are invisible
|
|
2614
|
+
`"stop_reason" (Anthropic) or "finish_reason" (OpenAI) on ${seen} records: without it, answers cut off at max_tokens are invisible, and silence there is not the same as none.`,
|
|
2539
2615
|
needsCacheTtl: (seen) =>
|
|
2540
2616
|
`the "cache_creation" object on ${seen} of the records that wrote to the cache: without it the 5-minute rate is assumed, so those totals are a floor and some cache verdicts cannot be settled.`,
|
|
2541
2617
|
hoursConcentrated: (hours, list) =>
|
|
2542
|
-
`80% of this spend lands in ${hours} hours of the UTC day (${list})
|
|
2618
|
+
`80% of this spend lands in ${hours} hours of the UTC day (${list}), interactive traffic somebody is waiting on, where the Batch API's 24-hour turnaround does not fit. Hours are UTC; shift them yourself if your traffic sits in one region.`,
|
|
2543
2619
|
hoursFlat: (hours) =>
|
|
2544
|
-
`The spend is spread across the day: it takes ${hours} hours of the UTC day to cover 80% of it. That is the shape background work has, and background work is what the Batch API halves the price of
|
|
2620
|
+
`The spend is spread across the day: it takes ${hours} hours of the UTC day to cover 80% of it. That is the shape background work has, and background work is what the Batch API halves the price of. See the levers above for what it would be worth here. Whether these calls can wait is yours to say; the log only shows when they happened.`,
|
|
2545
2621
|
truncatedBy: (label, calls, measured, rate, usd) =>
|
|
2546
|
-
`${label}: ${calls} of ${measured} calls that recorded a stop reason were cut off (${rate}), ${usd} of output. The denominator is the calls that measured, not every call
|
|
2622
|
+
`${label}: ${calls} of ${measured} calls that recorded a stop reason were cut off (${rate}), ${usd} of output. The denominator is the calls that measured, not every call: a workload logging the field half the time is not a workload whose other half completed.`,
|
|
2547
2623
|
truncatedCeiling: (p95) =>
|
|
2548
2624
|
`95% of the answers that finished fit within ${p95} output tokens, so a cap around there would stop cutting them off. Measured on these calls, promised for nothing.`,
|
|
2549
2625
|
readFiles: (files, directory) =>
|
|
@@ -2553,28 +2629,28 @@ ${bold('EXAMPLES')}
|
|
|
2553
2629
|
sessionCostTail: (ratio) =>
|
|
2554
2630
|
`The 95th percentile is ${ratio}x the median there: most conversations are cheap and a few are not, which is a tail a quota can catch. Where median and p95 sit close together the workload is simply expensive and there is no tail to hunt.`,
|
|
2555
2631
|
sessionSpendOnly: (sessions, max) =>
|
|
2556
|
-
`${sessions} ${sessions === '1' ? 'conversation' : 'conversations'} in this log; the most expensive cost ${max}. Too few per workload for a percentile
|
|
2632
|
+
`${sessions} ${sessions === '1' ? 'conversation' : 'conversations'} in this log; the most expensive cost ${max}. Too few per workload for a percentile: a maximum is a fact at any count, and it is the figure --max-session-usd judges.`,
|
|
2557
2633
|
waiveActive: (gate, reason, until, daysLeft) =>
|
|
2558
|
-
`WAIVED
|
|
2634
|
+
`WAIVED: the ${gate} failure above is on the record and silenced until ${until} (${daysLeft} days left): "${reason}". The bill still counts it; only the exit code is quiet, and the day the waiver expires this gate fails again.`,
|
|
2559
2635
|
waiveExpired: (gate, until, reason) =>
|
|
2560
|
-
`The waiver on ${gate} expired on ${until} and no longer silences anything. It was written for: "${reason}". Renew it with a new date and a current reason, or fix what it was covering
|
|
2636
|
+
`The waiver on ${gate} expired on ${until} and no longer silences anything. It was written for: "${reason}". Renew it with a new date and a current reason, or fix what it was covering: an expired waiver left in place is a finding deleted with extra steps.`,
|
|
2561
2637
|
waiveNotRecorded: (path, because) =>
|
|
2562
2638
|
` (This waiver was not written to ${path}: ${because}. The gate's verdict above is unaffected.)`,
|
|
2563
2639
|
summaryNoComparison: () =>
|
|
2564
|
-
'No previous log was given, so nothing here says whether the bill moved
|
|
2640
|
+
'No previous log was given, so nothing here says whether the bill moved, a summary without a comparison states the bill, not its stability.',
|
|
2565
2641
|
summaryFooter: () =>
|
|
2566
|
-
'The short form: what changed and the largest lever, nothing else. Run trazum profile for the full report
|
|
2642
|
+
'The short form: what changed and the largest lever, nothing else. Run trazum profile for the full report: every figure here comes from it.',
|
|
2567
2643
|
gateLargest: (label, model, usd, share) =>
|
|
2568
2644
|
`Most of it is ${label} on ${model}: ${usd}, ${share} of the bill. That is where the money is, not necessarily where the fix is.`,
|
|
2569
2645
|
gateLever: (label, action, saving, overage, covers) =>
|
|
2570
|
-
`The largest lever the report priced would save ${saving} on ${label} by ${action}
|
|
2646
|
+
`The largest lever the report priced would save ${saving} on ${label} by ${action}, ${covers ? `enough to cover the ${overage} this is over by` : `short of the ${overage} this is over by, so it is part of the answer rather than all of it`}. Whether that is the right call for this workload is yours to judge; the figure is arithmetic, not advice.`,
|
|
2571
2647
|
gateLeverRoute: (model) => `moving it to ${model}`,
|
|
2572
2648
|
gateLeverBatch: () => 'sending it through the Batch API',
|
|
2573
2649
|
gateLeverBoth: (model) => `moving it to ${model} and sending it through the Batch API`,
|
|
2574
2650
|
gateMarginTight: (margin, room) =>
|
|
2575
|
-
`Passed with ${margin} of the budget left
|
|
2651
|
+
`Passed with ${margin} of the budget left, ${room}. Under a tenth is close enough that an ordinary week crosses it; a pass this tight is worth knowing about before it becomes a failure.`,
|
|
2576
2652
|
maxGrowthCoverageLost: (fields, was, now) =>
|
|
2577
|
-
`FAILED
|
|
2653
|
+
`FAILED: this log stopped recording ${fields} (${was} of records before, ${now} now), so the comparison cannot be made. That is not a pass: a bill whose growth nobody could measure is not a bill that stayed flat, and every finding that needed the field went quiet for a reason that has nothing to do with spend.`,
|
|
2578
2654
|
coverageField: (field) =>
|
|
2579
2655
|
({ label: 'label', session: 'session', ts: 'timestamp', stopReason: 'stop reason' })[field] ?? field,
|
|
2580
2656
|
coverageSilenced: (field) =>
|
|
@@ -2587,39 +2663,39 @@ ${bold('EXAMPLES')}
|
|
|
2587
2663
|
coverageDrift: (field, was, now) =>
|
|
2588
2664
|
`Coverage moved: ${field} was on ${was} of records and is now on ${now}.`,
|
|
2589
2665
|
coverageDriftWhy: () =>
|
|
2590
|
-
'A field the log stopped recording is not a finding that got fixed
|
|
2666
|
+
'A field the log stopped recording is not a finding that got fixed: every finding that needed it has gone quiet for a reason that has nothing to do with the bill. Reported from a 20-point move in either direction; a field that appeared means this report can see what the previous one could not.',
|
|
2591
2667
|
againstOverlap: (from, to) =>
|
|
2592
|
-
`These two logs both cover ${from} → ${to}, so some of the same calls sit on both sides of this subtraction and part of the change is the same money counted twice. Compare periods that do not overlap
|
|
2668
|
+
`These two logs both cover ${from} → ${to}, so some of the same calls sit on both sides of this subtraction and part of the change is the same money counted twice. Compare periods that do not overlap, or window both logs with --since/--until.`,
|
|
2593
2669
|
windowLine: (since, until) =>
|
|
2594
2670
|
`Filtered to --since ${since} --until ${until}. Everything below describes this window, not the whole log; a bare date means the whole of that UTC day.`,
|
|
2595
2671
|
windowUndated: (calls) =>
|
|
2596
|
-
`${count(calls)} ${calls === 1 ? 'call carries' : 'calls carry'} no timestamp and cannot be placed inside or outside this window, so ${calls === 1 ? 'it was' : 'they were'} left out. Their spend is in the log and not in this report
|
|
2672
|
+
`${count(calls)} ${calls === 1 ? 'call carries' : 'calls carry'} no timestamp and cannot be placed inside or outside this window, so ${calls === 1 ? 'it was' : 'they were'} left out. Their spend is in the log and not in this report: the window's figures are a floor on the period.`,
|
|
2597
2673
|
windowRelative: () =>
|
|
2598
|
-
'That window is relative to this machine\'s clock, not to the log\'s last record
|
|
2674
|
+
'That window is relative to this machine\'s clock, not to the log\'s last record: a log exported a month ago will answer "the last 7 days" with nothing.',
|
|
2599
2675
|
windowRelativeEmpty: () =>
|
|
2600
2676
|
'A relative window is measured from this machine\'s clock: if this log was exported earlier, ask for the dates it covers instead.',
|
|
2601
2677
|
windowNeedsClock: () =>
|
|
2602
|
-
'No record in this log carries a timestamp, so --since/--until have nothing to filter by. A time window over a clockless log would gate nothing, which is not an answer. Add "ts" to the records
|
|
2678
|
+
'No record in this log carries a timestamp, so --since/--until have nothing to filter by. A time window over a clockless log would gate nothing, which is not an answer. Add "ts" to the records, the recipe in the README shows where.',
|
|
2603
2679
|
windowMatchesNothing: (from, to) =>
|
|
2604
|
-
`No record falls inside this window. The log covers ${from} → ${to}. A window matching nothing must not become a $0 report
|
|
2680
|
+
`No record falls inside this window. The log covers ${from} → ${to}. A window matching nothing must not become a $0 report, under --max-usd it would pass a budget gate over a period the log does not cover.`,
|
|
2605
2681
|
sinceAfterUntil: () =>
|
|
2606
2682
|
'--since is at or after --until, so the window contains no time at all. Check the two dates.',
|
|
2607
2683
|
badWhen: (flag, value) =>
|
|
2608
2684
|
`--${flag} could not read "${value}". It takes a UTC day (2026-08-14) or a full ISO 8601 timestamp (2026-08-14T09:30:00Z).`,
|
|
2609
2685
|
singleTurnCeiling: (label, model, single, sessions, usd) =>
|
|
2610
|
-
`${label} on ${model}: ${single} of ${sessions} conversations ended after their first turn, and their cache writes
|
|
2686
|
+
`${label} on ${model}: ${single} of ${sessions} conversations ended after their first turn, and their cache writes, ${usd}, paid for reuse their own conversation never made. Another conversation sharing the same prefix within the TTL could have read them; the log cannot see whose write a read hit, so that figure is a ceiling on the waste, not a bill.`,
|
|
2611
2687
|
singleTurnConfirmed: (label, model, single, sessions, usd) =>
|
|
2612
|
-
`${label} on ${model}: ${single} of ${sessions} conversations ended after their first turn and spent ${usd} writing a cache that nothing in this log ever read. Within the conversation, across conversations
|
|
2688
|
+
`${label} on ${model}: ${single} of ${sessions} conversations ended after their first turn and spent ${usd} writing a cache that nothing in this log ever read. Within the conversation, across conversations, no read anywhere, so those writes bought nothing. Caching a one-shot call is pure write premium; stop marking these calls with cache_control.`,
|
|
2613
2689
|
},
|
|
2614
2690
|
|
|
2615
2691
|
plan: {
|
|
2616
2692
|
noTarget: () =>
|
|
2617
|
-
'Point this at a usage log or a directory of them: trazum plan usage.jsonl. It turns the report into a ranked plan
|
|
2693
|
+
'Point this at a usage log or a directory of them: trazum plan usage.jsonl. It turns the report into a ranked plan, what to do first, what each action is worth, and what the log cannot confirm about it.',
|
|
2618
2694
|
nothingPriced: () =>
|
|
2619
|
-
'This log priced nothing
|
|
2695
|
+
'This log priced nothing, no call in it matched a model in the catalogue. A plan over zero dollars would be advice about nothing; check the log with "trazum profile" first.',
|
|
2620
2696
|
heading: (actions, total) => `The plan: ${actions} actions against a ${total} bill`,
|
|
2621
2697
|
totals: (projected, staked) =>
|
|
2622
|
-
`${projected} projected savings, on assumptions listed below. ${staked} already spent on problems this plan names
|
|
2698
|
+
`${projected} projected savings, on assumptions listed below. ${staked} already spent on problems this plan names, measured, not projected.`,
|
|
2623
2699
|
noClock: () =>
|
|
2624
2700
|
'No timestamps in this log, so every figure is per this log, not per any period.',
|
|
2625
2701
|
projected: (usd) => `${usd} projected`,
|
|
@@ -2637,24 +2713,24 @@ ${bold('EXAMPLES')}
|
|
|
2637
2713
|
: 'Look at the cache on';
|
|
2638
2714
|
return `${verb} ${label} (${model})`;
|
|
2639
2715
|
},
|
|
2640
|
-
routeTo: (model) => `to ${model}
|
|
2716
|
+
routeTo: (model) => `to ${model}, combined with batching where both apply, never summed`,
|
|
2641
2717
|
assume: (assumption) => {
|
|
2642
2718
|
switch (assumption.kind) {
|
|
2643
2719
|
case 'model-capability':
|
|
2644
|
-
return `assumes ${assumption.model} can do this work
|
|
2720
|
+
return `assumes ${assumption.model} can do this work, the log prices the move, it cannot judge the answers`;
|
|
2645
2721
|
case 'batch-window':
|
|
2646
2722
|
return 'assumes these calls can wait for a batch window';
|
|
2647
2723
|
case 'retry-pattern-real':
|
|
2648
|
-
return 'assumes the retry pattern is real
|
|
2724
|
+
return 'assumes the retry pattern is real, the log sees shapes, not content';
|
|
2649
2725
|
case 'max-tokens-fits':
|
|
2650
2726
|
return 'assumes a max_tokens the answers fit in removes the pair';
|
|
2651
2727
|
case 'traffic-pattern-holds':
|
|
2652
|
-
return 'assumes the traffic pattern holds
|
|
2728
|
+
return 'assumes the traffic pattern holds: a cache that lost money on this log may pay on different traffic';
|
|
2653
2729
|
}
|
|
2654
2730
|
},
|
|
2655
2731
|
check: (command) => `check it: ${command}`,
|
|
2656
2732
|
filtered: (count, minUsd, worth) =>
|
|
2657
|
-
`${count} actions under ${minUsd}, worth ${worth} together, left out by --min-usd
|
|
2733
|
+
`${count} actions under ${minUsd}, worth ${worth} together, left out by --min-usd, left out of this document entirely, not disproved.`,
|
|
2658
2734
|
footer: () =>
|
|
2659
2735
|
'Ranked by money, projected or already spent alike. The assumptions are yours to answer: this plan is arithmetic over the log, not knowledge of your product.',
|
|
2660
2736
|
wrote: (path) =>
|
|
@@ -2666,11 +2742,11 @@ ${bold('EXAMPLES')}
|
|
|
2666
2742
|
limitsNoLog: () =>
|
|
2667
2743
|
'A limits policy is configured and no --log was given, so every ceiling will answer cannot-tell: per-label and per-session spend live in a usage log, not in the store. Point --log at your usage log to make the ceilings judgeable.',
|
|
2668
2744
|
limitsUnpriced: (count) =>
|
|
2669
|
-
`${count} record(s) in the --log name a model the catalogue cannot price. They contribute nothing to any measured figure
|
|
2745
|
+
`${count} record(s) in the --log name a model the catalogue cannot price. They contribute nothing to any measured figure: money the limits policy cannot see, said here rather than hidden.`,
|
|
2670
2746
|
loopbackOnly: () =>
|
|
2671
|
-
'Loopback only, and there is no flag to change that: this holds your spend, your model mix and your budgets, and would answer whoever asked. There is no auth for the same reason
|
|
2747
|
+
'Loopback only, and there is no flag to change that: this holds your spend, your model mix and your budgets, and would answer whoever asked. There is no auth for the same reason: a token checked over loopback is theatre.',
|
|
2672
2748
|
measuredFrom: (usd) =>
|
|
2673
|
-
`Budget answers are measured against ${usd} from the store, read once at start. Every answer carries the window that figure covers rather than implying it is current to the second
|
|
2749
|
+
`Budget answers are measured against ${usd} from the store, read once at start. Every answer carries the window that figure covers rather than implying it is current to the second, restart to refresh it.`,
|
|
2674
2750
|
nothingMeasured: (dir) =>
|
|
2675
2751
|
`Nothing is measured yet (the store at ${dir} is empty), so the budget half of every answer will say so. The cost half still answers from the catalogue: offline is a mode, not a failure.`,
|
|
2676
2752
|
noBudget: () =>
|
|
@@ -2682,14 +2758,14 @@ ${bold('EXAMPLES')}
|
|
|
2682
2758
|
|
|
2683
2759
|
watch: {
|
|
2684
2760
|
noThresholds: () =>
|
|
2685
|
-
'Watching needs something to watch for. Set spend.maxUsd, spend.maxDayUsd or spend.maxCacheLossUsd in trazum.config.json
|
|
2761
|
+
'Watching needs something to watch for. Set spend.maxUsd, spend.maxDayUsd or spend.maxCacheLossUsd in trazum.config.json: a watcher with no threshold is a green light nobody earned.',
|
|
2686
2762
|
nothingToWatch: (dir) =>
|
|
2687
|
-
`Nothing has been measured yet: the store at ${dir} is empty. Fill it with "trazum connect <provider> --store" first
|
|
2763
|
+
`Nothing has been measured yet: the store at ${dir} is empty. Fill it with "trazum connect <provider> --store" first, watching nothing would report that everything is fine.`,
|
|
2688
2764
|
intervalTooTight: () =>
|
|
2689
2765
|
'--interval must be at least 5m. Usage APIs are rate limited, and a tight loop is a way to get your own key throttled by a tool that exists to save you money.',
|
|
2690
2766
|
badWebhook: (reason) =>
|
|
2691
2767
|
reason === 'credentials-in-url'
|
|
2692
|
-
? 'That webhook URL carries credentials. URLs end up in logs, shell history and error messages, so this one is refused
|
|
2768
|
+
? 'That webhook URL carries credentials. URLs end up in logs, shell history and error messages, so this one is refused, put the secret in a header your receiver checks, or in the receiver itself.'
|
|
2693
2769
|
: reason === 'insecure-scheme'
|
|
2694
2770
|
? 'A webhook must be https, except on loopback. An alert carries your spend figures, and sending them in the clear across a network is a leak you did not ask for.'
|
|
2695
2771
|
: 'That webhook is not a URL this tool can parse.',
|
|
@@ -2700,7 +2776,7 @@ ${bold('EXAMPLES')}
|
|
|
2700
2776
|
: gate === 'maxDayUsd'
|
|
2701
2777
|
? `Spend on ${day}`
|
|
2702
2778
|
: 'Money lost to caching';
|
|
2703
|
-
return `CROSSED
|
|
2779
|
+
return `CROSSED, ${what} is ${measured} against a limit of ${limit}. Measured, not projected.`;
|
|
2704
2780
|
},
|
|
2705
2781
|
stillOver: (gate, measured, limit, day) => {
|
|
2706
2782
|
const what =
|
|
@@ -2709,36 +2785,36 @@ ${bold('EXAMPLES')}
|
|
|
2709
2785
|
: gate === 'maxDayUsd'
|
|
2710
2786
|
? `Spend on ${day}`
|
|
2711
2787
|
: 'Money lost to caching';
|
|
2712
|
-
return `STILL OVER
|
|
2788
|
+
return `STILL OVER, ${what} is ${measured} against a limit of ${limit}, and was already reported. Quiet is not clean.`;
|
|
2713
2789
|
},
|
|
2714
2790
|
notJudgeable: (gate, reason, covered) =>
|
|
2715
2791
|
reason === 'window-too-short'
|
|
2716
|
-
? `${gate} cannot be judged yet: this period is ${covered ?? 'partly'} measured, and a threshold over part of a day is a threshold over something else. Not a pass
|
|
2792
|
+
? `${gate} cannot be judged yet: this period is ${covered ?? 'partly'} measured, and a threshold over part of a day is a threshold over something else. Not a pass: it will be judged when the day is in.`
|
|
2717
2793
|
: `${gate} cannot be judged on this source, which does not serve what the gate is written against. Not a pass: a gate silently skipped reads exactly like a gate that keeps passing.`,
|
|
2718
2794
|
gap: (from, to) =>
|
|
2719
2795
|
`Nothing was watching between ${from} and ${to}. Whatever crossed in that stretch was not seen, and this line exists so a resumed watcher does not imply coverage it did not have.`,
|
|
2720
2796
|
allWithin: (gates) => `Within every threshold: ${gates} gates evaluated against measured spend.`,
|
|
2721
2797
|
webhookFailed: (status) =>
|
|
2722
|
-
`The webhook did not deliver (${status}). The crossing is still in the exit code and in the output above
|
|
2798
|
+
`The webhook did not deliver (${status}). The crossing is still in the exit code and in the output above: a receiver being down must not be the quietest failure in the room.`,
|
|
2723
2799
|
watching: (minutes) => `Watching every ${minutes} minutes. Ctrl-C stops it.`,
|
|
2724
2800
|
},
|
|
2725
2801
|
|
|
2726
2802
|
store: {
|
|
2727
2803
|
appended: (count, dir) => `Kept ${count} measurements in ${dir}.`,
|
|
2728
2804
|
empty: (dir) =>
|
|
2729
|
-
`The store at ${dir} is empty. Fill it with "trazum connect <provider> --store"
|
|
2805
|
+
`The store at ${dir} is empty. Fill it with "trazum connect <provider> --store": that is a state, not an error.`,
|
|
2730
2806
|
heading: (records, usd, from, to) =>
|
|
2731
2807
|
`The store: ${records} measurements · ${usd} · ${from} → ${to}`,
|
|
2732
2808
|
providerRow: (provider, records, span, models) =>
|
|
2733
2809
|
`${provider} ${records} measurements · ${span} · ${models} models`,
|
|
2734
2810
|
holds: (files) =>
|
|
2735
|
-
`Held in ${files} files: token counts, billed dollars and the account's own workspace and key identifiers. Never prompt text, never completion text, never a credential
|
|
2811
|
+
`Held in ${files} files: token counts, billed dollars and the account's own workspace and key identifiers. Never prompt text, never completion text, never a credential: this is a file you can back up without a privacy review.`,
|
|
2736
2812
|
possiblyDouble: (count) =>
|
|
2737
|
-
`${count} records could not be told apart from another
|
|
2813
|
+
`${count} records could not be told apart from another, a window of no length, or a record naming no model. They are kept whole rather than merged, so a total built on them may count the same spend twice. Saying so beats a smaller number nobody can check.`,
|
|
2738
2814
|
unknownVersion: (count) =>
|
|
2739
2815
|
`${count} records come from a newer schema than this version knows, so they are kept and left out of the figures above rather than guessed at. Upgrade to read them.`,
|
|
2740
2816
|
unreadable: (file, line) =>
|
|
2741
|
-
`${file} line ${line} would not parse, so it is not in the figures above. The rest of the file was read
|
|
2817
|
+
`${file} line ${line} would not parse, so it is not in the figures above. The rest of the file was read: one broken line must not lose a month.`,
|
|
2742
2818
|
retention: (days) => `Retention: ${days} days, from "store.keepDays". Run "trazum store --prune" to apply it.`,
|
|
2743
2819
|
noRetention: () =>
|
|
2744
2820
|
'No retention policy is configured, so nothing is ever deleted on its own. Set "store": {"keepDays": 90} when you want one.',
|
|
@@ -2747,7 +2823,7 @@ ${bold('EXAMPLES')}
|
|
|
2747
2823
|
pruneDryRun: (count, days, span, usd) =>
|
|
2748
2824
|
span === null
|
|
2749
2825
|
? `Nothing is older than ${days} days, so a prune would delete nothing.`
|
|
2750
|
-
: `A prune would delete ${count} measurements older than ${days} days, covering ${span} and ${usd} of measured spend. Nothing was deleted
|
|
2826
|
+
: `A prune would delete ${count} measurements older than ${days} days, covering ${span} and ${usd} of measured spend. Nothing was deleted: this was --dry-run.`,
|
|
2751
2827
|
pruned: (count, days, span, usd, kept) =>
|
|
2752
2828
|
span === null
|
|
2753
2829
|
? `Nothing was older than ${days} days. ${kept} measurements kept, and the append log compacted.`
|
|
@@ -2768,7 +2844,7 @@ ${bold('EXAMPLES')}
|
|
|
2768
2844
|
budgetNeverForecast: () =>
|
|
2769
2845
|
'That is a shape, not a forecast. Where this goes next depends on what you do next, and no arithmetic here knows that.',
|
|
2770
2846
|
budgetNothingMeasured: (elapsedDays) =>
|
|
2771
|
-
`Nothing has been measured this month, across ${elapsedDays} elapsed ${elapsedDays === 1 ? 'day' : 'days'}. That is not a budget under control
|
|
2847
|
+
`Nothing has been measured this month, across ${elapsedDays} elapsed ${elapsedDays === 1 ? 'day' : 'days'}. That is not a budget under control: it is a budget nobody is watching. Run trazum connect to pull what the provider has.`,
|
|
2772
2848
|
budgetPartial: (measuredDays, elapsedDays, days) =>
|
|
2773
2849
|
`Only ${measuredDays} of ${elapsedDays} elapsed days carry any measurement, so the figure below is a floor on the month rather than the month. Missing: ${days}.`,
|
|
2774
2850
|
budgetScopesUnmeasured: (count) =>
|
|
@@ -2777,11 +2853,11 @@ ${bold('EXAMPLES')}
|
|
|
2777
2853
|
|
|
2778
2854
|
connect: {
|
|
2779
2855
|
noTarget: (providers) =>
|
|
2780
|
-
`Name a provider to read your bill from: trazum connect anthropic. Available: ${providers}. The credential comes from the environment and is never stored
|
|
2856
|
+
`Name a provider to read your bill from: trazum connect anthropic. Available: ${providers}. The credential comes from the environment and is never stored. Add --dry-run to see exactly what would be called and which variable it would be read from.`,
|
|
2781
2857
|
unknownProvider: (id, providers) =>
|
|
2782
2858
|
`There is no connector for "${id}". The ones that exist are: ${providers}.`,
|
|
2783
2859
|
pricedNoConnector: (id, providers) =>
|
|
2784
|
-
`Trazum prices ${id} but has no connector for it yet
|
|
2860
|
+
`Trazum prices ${id} but has no connector for it yet, it connects to ${providers}. Either that provider publishes no usage API, or one has not been written; both are gaps rather than mistakes of yours. Until then, export a log and run "trazum profile" on it: every figure works except the ones a usage API would supply on its own.`,
|
|
2785
2861
|
dryRun: (provider, from, to, envVars, keyKind) =>
|
|
2786
2862
|
`Would read ${provider} usage from ${from} to ${to}, using ${keyKind} taken from ${envVars}. Nothing was sent and no credential was needed to print this.`,
|
|
2787
2863
|
heading: (provider, from, to, usd, calls) =>
|
|
@@ -2791,7 +2867,7 @@ ${bold('EXAMPLES')}
|
|
|
2791
2867
|
modelRow: (model, usd, share, calls) =>
|
|
2792
2868
|
calls === null ? `${model} ${usd} ${share}` : `${model} ${usd} ${share} · ${calls} calls`,
|
|
2793
2869
|
nothingBilled: () =>
|
|
2794
|
-
'The provider billed nothing in this window. That is a measurement, not an error
|
|
2870
|
+
'The provider billed nothing in this window. That is a measurement, not an error, widen it with --since if you expected traffic.',
|
|
2795
2871
|
cachePaid: (saved) => `Caching paid for itself: ${saved} less than these tokens would have cost as ordinary input.`,
|
|
2796
2872
|
cacheLost: (added) => `Caching added ${added} to this bill against what the same tokens would have cost as ordinary input.`,
|
|
2797
2873
|
cacheUnsettled: () =>
|
|
@@ -2802,7 +2878,7 @@ ${bold('EXAMPLES')}
|
|
|
2802
2878
|
`${model} is not in the price catalogue, so its ${tokens} tokens are counted and its money is not. Add it with --pricing rather than reading the total as complete.`,
|
|
2803
2879
|
gap: (detail) => `This window is incomplete: ${detail}.`,
|
|
2804
2880
|
unavailable: (findings) =>
|
|
2805
|
-
`Findings this source cannot support: ${findings}. They need one row per call, and a sum has lost the rows
|
|
2881
|
+
`Findings this source cannot support: ${findings}. They need one row per call, and a sum has lost the rows, a per-call log still answers them.`,
|
|
2806
2882
|
wrote: (path) => `Report written to ${path}.`,
|
|
2807
2883
|
footer: () =>
|
|
2808
2884
|
'Every figure here is the provider\u2019s own billed token count at the catalogue\u2019s rates. Nothing was estimated, and nothing the provider did not serve was filled in.',
|
|
@@ -2821,7 +2897,7 @@ ${bold('EXAMPLES')}
|
|
|
2821
2897
|
runModel: (model, periods, sinceName, from, to) =>
|
|
2822
2898
|
`${model}'s share of the bill has climbed for ${periods} consecutive periods since ${sinceName}: ${from} → ${to}. The totals can look flat while the mix moves under them.`,
|
|
2823
2899
|
runCache: (periods, sinceName, from, to) =>
|
|
2824
|
-
`The cache share has decayed for ${periods} consecutive periods since ${sinceName}: ${from} → ${to}
|
|
2900
|
+
`The cache share has decayed for ${periods} consecutive periods since ${sinceName}: ${from} → ${to}, slowly enough that no single report called it a finding, which is exactly why a series exists.`,
|
|
2825
2901
|
repeated: (kind, label, model, appearances, first, last) => {
|
|
2826
2902
|
const what =
|
|
2827
2903
|
kind === 'route'
|
|
@@ -2834,26 +2910,26 @@ ${bold('EXAMPLES')}
|
|
|
2834
2910
|
? `Fixing the truncation retries on ${label} (${model})`
|
|
2835
2911
|
: `Fixing the cache on ${label} (${model})`;
|
|
2836
2912
|
const span = first !== null && last !== null ? ` (${first} → ${last})` : '';
|
|
2837
|
-
return `${what} has been planned ${appearances} times${span} and is still in the newest plan
|
|
2913
|
+
return `${what} has been planned ${appearances} times${span} and is still in the newest plan: a decision nobody is revisiting.`;
|
|
2838
2914
|
},
|
|
2839
2915
|
storeNoLabels: () =>
|
|
2840
|
-
'This series comes from the store, and a usage API groups by model and workspace rather than by workload
|
|
2916
|
+
'This series comes from the store, and a usage API groups by model and workspace rather than by workload, so there is no label series here at all. Absent, not empty: nothing above says a workload did or did not move.',
|
|
2841
2917
|
unmeasured: (days, from, to, after, before) =>
|
|
2842
|
-
`Nothing covers ${from} to ${to}
|
|
2918
|
+
`Nothing covers ${from} to ${to}, ${plural(days, 'day')} between ${after} and ${before}.`,
|
|
2843
2919
|
unmeasuredTotal: (days) =>
|
|
2844
2920
|
`${plural(days, 'day')} of this stretch are covered by no report. A series with a hole in it and a shorter series read identically; this is which.`,
|
|
2845
2921
|
overlap: (a, b, days) =>
|
|
2846
|
-
`${a} and ${b} both cover ${plural(days, 'day')}. Named, never merged
|
|
2847
|
-
runHole: (days) =>
|
|
2848
|
-
undated: (name) => `${name} carries no span, so it is on no timeline above
|
|
2922
|
+
`${a} and ${b} both cover ${plural(days, 'day')}. Named, never merged, which is the better measurement is not knowable from here, and adding both totals counts those days twice.`,
|
|
2923
|
+
runHole: (days) => `, ${plural(days, 'day')} of this run are covered by no report`,
|
|
2924
|
+
undated: (name) => `${name} carries no span, so it is on no timeline above, named, never silently absorbed.`,
|
|
2849
2925
|
unrecognized: (name) => `${name} is neither a stored report nor a saved plan, so it is in no series above.`,
|
|
2850
2926
|
footer: () =>
|
|
2851
|
-
'A series names shapes, not futures. Twenty points make a trend visible; they do not make next month knowable
|
|
2927
|
+
'A series names shapes, not futures. Twenty points make a trend visible; they do not make next month knowable, where these lines go next is yours to judge.',
|
|
2852
2928
|
waiverHeading: () => 'What this repository has been living with',
|
|
2853
2929
|
waiverSince: (day, uses) =>
|
|
2854
2930
|
`${uses} recorded ${uses === 1 ? 'use' : 'uses'} since ${day}, when recording started.`,
|
|
2855
2931
|
waiverNoneRecorded: () =>
|
|
2856
|
-
'No waiver has silenced a gate since recording started. Nothing here is inferred from the config
|
|
2932
|
+
'No waiver has silenced a gate since recording started. Nothing here is inferred from the config: a waiver written down and never hit is not a decision anyone is living with.',
|
|
2857
2933
|
waiverHabit: (gate, uses, days, firstDay, lastDay) =>
|
|
2858
2934
|
`${gate}: ${uses} ${uses === 1 ? 'use' : 'uses'} across ${days} ${days === 1 ? 'day' : 'days'}, ${firstDay} to ${lastDay}`,
|
|
2859
2935
|
waiverVerdict: (verdict) =>
|
|
@@ -2862,7 +2938,7 @@ ${bold('EXAMPLES')}
|
|
|
2862
2938
|
: verdict === 'recurring'
|
|
2863
2939
|
? 'The same decision, holding. The gate keeps firing and the reason has not moved.'
|
|
2864
2940
|
: verdict === 'renewed-without-revisiting'
|
|
2865
|
-
? 'The expiry has moved and the reason has not. That is the shape a decision takes when nobody is revisiting it
|
|
2941
|
+
? 'The expiry has moved and the reason has not. That is the shape a decision takes when nobody is revisiting it, which is sometimes exactly right, and worth saying out loud either way.'
|
|
2866
2942
|
: 'The reason changed between uses. Somebody looked again.',
|
|
2867
2943
|
waiverReasonNow: (reason) => `Reason: ${reason}`,
|
|
2868
2944
|
waiverReasonsChanged: (count) => `${count} different reasons given over that time.`,
|
|
@@ -2871,7 +2947,7 @@ ${bold('EXAMPLES')}
|
|
|
2871
2947
|
waiverNoLongerConfigured: () =>
|
|
2872
2948
|
'Not in the config any more. The decision was reversed; the record keeps it.',
|
|
2873
2949
|
waiverNeverUsed: (gates) =>
|
|
2874
|
-
`Waived in the config and never hit by a recorded run: ${gates}. Either the gate stopped failing
|
|
2950
|
+
`Waived in the config and never hit by a recorded run: ${gates}. Either the gate stopped failing, good news nobody wrote down, or the waiver names a situation that does not arise. Both are worth deleting.`,
|
|
2875
2951
|
waiverUnreadable: (count, path) =>
|
|
2876
2952
|
`${count} ${count === 1 ? 'line' : 'lines'} in ${path} could not be read and are not counted above.`,
|
|
2877
2953
|
waiverStartsHere: () =>
|
|
@@ -2880,7 +2956,7 @@ ${bold('EXAMPLES')}
|
|
|
2880
2956
|
|
|
2881
2957
|
verify: {
|
|
2882
2958
|
noTarget: () =>
|
|
2883
|
-
'Point this at a saved plan and a newer log: trazum verify plan.json --against usage.jsonl. It says, per action, whether the change arrived, did not arrive, or cannot be told
|
|
2959
|
+
'Point this at a saved plan and a newer log: trazum verify plan.json --against usage.jsonl. It says, per action, whether the change arrived, did not arrive, or cannot be told, and never fewer than those three.',
|
|
2884
2960
|
needsAgainst: () =>
|
|
2885
2961
|
'--against <newer.jsonl|dir> is required. A plan can only be verified against a log that came after it; without one there is nothing to hold the prediction to.',
|
|
2886
2962
|
badPlan: (path, why) =>
|
|
@@ -2902,7 +2978,7 @@ ${bold('EXAMPLES')}
|
|
|
2902
2978
|
counts: (arrived, notArrived, cannotTell) =>
|
|
2903
2979
|
`${arrived} arrived · ${notArrived} did not arrive · ${cannotTell} cannot be told. The third is not a soft version of the second: it means this log cannot answer, which is its own finding.`,
|
|
2904
2980
|
pricesChanged: (planReviewed, nowReviewed) =>
|
|
2905
|
-
`Prices were reviewed ${planReviewed} when the plan was made and ${nowReviewed} now, so every dollar comparison here is two price lists, not one measurement
|
|
2981
|
+
`Prices were reviewed ${planReviewed} when the plan was made and ${nowReviewed} now, so every dollar comparison here is two price lists, not one measurement: a team must not be blamed for a saving that arithmetic revoked.`,
|
|
2906
2982
|
action: (kind, label, model, outcome) => {
|
|
2907
2983
|
const what =
|
|
2908
2984
|
kind === 'route'
|
|
@@ -2916,14 +2992,14 @@ ${bold('EXAMPLES')}
|
|
|
2916
2992
|
: `Fix the cache on ${label} (${model})`;
|
|
2917
2993
|
const verdict =
|
|
2918
2994
|
outcome === 'arrived' ? 'ARRIVED' : outcome === 'not-arrived' ? 'DID NOT ARRIVE' : 'CANNOT BE TOLD';
|
|
2919
|
-
return `${what}
|
|
2995
|
+
return `${what}, ${verdict}`;
|
|
2920
2996
|
},
|
|
2921
2997
|
reason: (reason) =>
|
|
2922
2998
|
reason === 'workload-vanished'
|
|
2923
|
-
? 'the label carries no priced traffic in this log
|
|
2999
|
+
? 'the label carries no priced traffic in this log: a vanished workload is not a fixed one, and not a broken one either'
|
|
2924
3000
|
: reason === 'fields-stopped'
|
|
2925
|
-
? 'the fields the detection needs are not in this log
|
|
2926
|
-
: 'the log records tokens, and tokens do not say which tier billed them
|
|
3001
|
+
? 'the fields the detection needs are not in this log, "not recorded" must not read as "fixed", so with --gate this fails'
|
|
3002
|
+
: 'the log records tokens, and tokens do not say which tier billed them: the Batch API cannot be seen from here',
|
|
2927
3003
|
routeObserved: (dearestModel, onTargetUsd, onOldUsd) =>
|
|
2928
3004
|
`the label's dearest model is now ${dearestModel} · ${onTargetUsd} on the target, ${onOldUsd} still on the old model`,
|
|
2929
3005
|
batchUnobservable: () =>
|
|
@@ -2935,9 +3011,9 @@ ${bold('EXAMPLES')}
|
|
|
2935
3011
|
? `caching now pays for itself on this slice (${deltaUsd} against the no-cache bill)`
|
|
2936
3012
|
: `caching still adds ${deltaUsd} to this slice's bill`,
|
|
2937
3013
|
attribution: (callsBefore, callsAfter, outBefore, outAfter) =>
|
|
2938
|
-
`the world moved too: calls ${callsBefore} → ${callsAfter}, output/call ${outBefore} → ${outAfter} tokens
|
|
3014
|
+
`the world moved too: calls ${callsBefore} → ${callsAfter}, output/call ${outBefore} → ${outAfter} tokens, stated so the verdict is not read as the whole story`,
|
|
2939
3015
|
gateFailed: (failures, total) =>
|
|
2940
|
-
`GATE FAILED
|
|
3016
|
+
`GATE FAILED, ${failures} of ${total} actions did not produce what the plan promised, or stopped being measurable by the team's own log.`,
|
|
2941
3017
|
gateOk: () => 'Gate passed: every verifiable action arrived, and nothing became unverifiable.',
|
|
2942
3018
|
footer: () =>
|
|
2943
3019
|
'Arrived and did-not-arrive are measurements; cannot-be-told is the log refusing to guess. All three are the verification working, not failing.',
|
|
@@ -2955,26 +3031,26 @@ ${bold('EXAMPLES')}
|
|
|
2955
3031
|
picked: (label, model, candidate, usd, pct) =>
|
|
2956
3032
|
`${label} on ${model} → ${candidate}, worth ${usd} of this bill (${pct}).`,
|
|
2957
3033
|
willSpend: (calls, model, candidate) =>
|
|
2958
|
-
`This will make ${count(calls)} provider calls: two per case on ${model} to measure its own variance, one per case on ${candidate}. Nothing has been spent yet
|
|
3034
|
+
`This will make ${count(calls)} provider calls: two per case on ${model} to measure its own variance, one per case on ${candidate}. Nothing has been spent yet. Add --yes to run it.`,
|
|
2959
3035
|
dryRun: () => 'Nothing was called.',
|
|
2960
3036
|
running: (cases) => `Running ${count(cases)} cases...`,
|
|
2961
3037
|
agreement: (cross, self) =>
|
|
2962
|
-
`The cheaper model agrees with the original ${cross} of the time. The original agrees with itself ${self} of the time
|
|
3038
|
+
`The cheaper model agrees with the original ${cross} of the time. The original agrees with itself ${self} of the time: that is the yardstick, not 100%.`,
|
|
2963
3039
|
holds: (usd) =>
|
|
2964
|
-
`HOLDS
|
|
3040
|
+
`HOLDS: the difference is inside the original model's own noise. On this bill that route is worth ${usd}.`,
|
|
2965
3041
|
diverges: (usd) =>
|
|
2966
|
-
`DIVERGES
|
|
3042
|
+
`DIVERGES: the cheaper model gives materially different answers. The ${usd} is real and so is the change in behaviour; this one is not free money.`,
|
|
2967
3043
|
inconclusive: () =>
|
|
2968
|
-
'INCONCLUSIVE
|
|
3044
|
+
'INCONCLUSIVE: the original model was too inconsistent with itself on these cases to judge anything against. Add cases, or pick ones with less room for the model to wander.',
|
|
2969
3045
|
unlabelledSlice: () =>
|
|
2970
|
-
'These calls carry no label, so Trazum cannot tell whether they are all this prompt. If they are not, the figure above covers calls this measurement never touched
|
|
3046
|
+
'These calls carry no label, so Trazum cannot tell whether they are all this prompt. If they are not, the figure above covers calls this measurement never touched. Add "label" to the record and the slice becomes one workload, which is what makes the number attributable.',
|
|
2971
3047
|
yours: () =>
|
|
2972
|
-
'Agreement is not correctness. This measures whether the answers moved, not whether they were ever right
|
|
3048
|
+
'Agreement is not correctness. This measures whether the answers moved, not whether they were ever right: the decision is still yours.',
|
|
2973
3049
|
},
|
|
2974
3050
|
|
|
2975
3051
|
baseline: {
|
|
2976
3052
|
recorded: (path, files, tokens) =>
|
|
2977
|
-
`Recorded ${files} prompts, ${tokens} tokens, to ${path}. Commit it
|
|
3053
|
+
`Recorded ${files} prompts, ${tokens} tokens, to ${path}. Commit it: the gate compares the tree against what is committed.`,
|
|
2978
3054
|
recordedMoney: (monthly, model, calls) =>
|
|
2979
3055
|
`That is ${monthly} per month with ${model} at ${calls} calls. Reported, not gated on: thresholds are in tokens, so a repriced model never fails a build on its own.`,
|
|
2980
3056
|
heading: () => 'Against the baseline',
|
|
@@ -3000,7 +3076,7 @@ ${bold('EXAMPLES')}
|
|
|
3000
3076
|
slots: {
|
|
3001
3077
|
task: {
|
|
3002
3078
|
question: 'What should the model do, in one sentence?',
|
|
3003
|
-
unlocks: 'the whole prompt
|
|
3079
|
+
unlocks: 'the whole prompt, without it there is nothing to write',
|
|
3004
3080
|
},
|
|
3005
3081
|
role: {
|
|
3006
3082
|
question: 'Who is the model being while it does that?',
|
|
@@ -3008,7 +3084,7 @@ ${bold('EXAMPLES')}
|
|
|
3008
3084
|
},
|
|
3009
3085
|
inputs: {
|
|
3010
3086
|
question: 'What changes from one call to the next?',
|
|
3011
|
-
unlocks: 'the varying part
|
|
3087
|
+
unlocks: 'the varying part, without it the prompt hard-codes one case and is rewritten for the next',
|
|
3012
3088
|
},
|
|
3013
3089
|
'output-shape': {
|
|
3014
3090
|
question: 'What should come back: prose, json, list or table?',
|
|
@@ -3024,7 +3100,7 @@ ${bold('EXAMPLES')}
|
|
|
3024
3100
|
},
|
|
3025
3101
|
audience: {
|
|
3026
3102
|
question: 'Who reads the output?',
|
|
3027
|
-
unlocks: 'the register
|
|
3103
|
+
unlocks: 'the register: an answer for an engineer and one for a customer are different answers',
|
|
3028
3104
|
},
|
|
3029
3105
|
constraints: {
|
|
3030
3106
|
question: 'What must it never do?',
|
|
@@ -3040,7 +3116,7 @@ ${bold('EXAMPLES')}
|
|
|
3040
3116
|
},
|
|
3041
3117
|
'example-inputs': {
|
|
3042
3118
|
question: 'What input produced that example?',
|
|
3043
|
-
unlocks: 'the pairing
|
|
3119
|
+
unlocks: 'the pairing, an example answer with no input teaches the shape and not the mapping',
|
|
3044
3120
|
},
|
|
3045
3121
|
'failure-modes': {
|
|
3046
3122
|
question: 'What has gone wrong with this before?',
|
|
@@ -3048,11 +3124,11 @@ ${bold('EXAMPLES')}
|
|
|
3048
3124
|
},
|
|
3049
3125
|
model: {
|
|
3050
3126
|
question: 'Which model is this for?',
|
|
3051
|
-
unlocks: 'the cost estimate
|
|
3127
|
+
unlocks: 'the cost estimate, this changes the report and never the prompt',
|
|
3052
3128
|
},
|
|
3053
3129
|
budget: {
|
|
3054
3130
|
question: 'What is the monthly ceiling for this prompt?',
|
|
3055
|
-
unlocks: 'the budget check
|
|
3131
|
+
unlocks: 'the budget check, this changes the report and never the prompt',
|
|
3056
3132
|
},
|
|
3057
3133
|
},
|
|
3058
3134
|
missing: (count) =>
|
|
@@ -3068,7 +3144,7 @@ ${bold('EXAMPLES')}
|
|
|
3068
3144
|
: `"${id}" is not a slot. Did you mean "${nearest}"?`,
|
|
3069
3145
|
answerNotText: (id) => `"${id}" must be a string, or null to decline it.`,
|
|
3070
3146
|
tokens: (count) => `${count} tokens`,
|
|
3071
|
-
monthly: (usd) => `${usd} per month, estimated
|
|
3147
|
+
monthly: (usd) => `${usd} per month, estimated: nobody has sent this prompt yet`,
|
|
3072
3148
|
budget: (verdict, limit) =>
|
|
3073
3149
|
verdict === 'over' ? `over the budget of ${limit}` : `within the budget of ${limit}`,
|
|
3074
3150
|
noVerdict: (reason) =>
|
|
@@ -3079,7 +3155,7 @@ ${bold('EXAMPLES')}
|
|
|
3079
3155
|
: 'That model is not in the price catalogue, so it cannot be priced.',
|
|
3080
3156
|
clean: () => 'trazum optimize recovers nothing from this.',
|
|
3081
3157
|
notClean: (rules, tokens) =>
|
|
3082
|
-
`trazum optimize would still recover ${tokens} tokens here (${rules})
|
|
3158
|
+
`trazum optimize would still recover ${tokens} tokens here (${rules}), from your answers, not from the structure.`,
|
|
3083
3159
|
declined: (ids) => `Declined, and left out: ${ids}`,
|
|
3084
3160
|
},
|
|
3085
3161
|
};
|