@trazum/cli 1.50.7 → 1.50.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@trazum/cli",
3
- "version": "1.50.7",
3
+ "version": "1.50.9",
4
4
  "description": "Trazum CLI: find where your LLM bill goes, price every finding per month, and enforce token budgets in CI.",
5
5
  "license": "MIT",
6
6
  "author": "David Mu\u00f1oz Rey",
@@ -37,7 +37,7 @@
37
37
  "prepublishOnly": "npm run build && npm test"
38
38
  },
39
39
  "dependencies": {
40
- "@trazum/core": "1.50.7"
40
+ "@trazum/core": "1.50.9"
41
41
  },
42
42
  "devDependencies": {
43
43
  "@types/node": "^26.2.0",
package/src/i18n/en.ts CHANGED
@@ -59,6 +59,8 @@ ${bold('USAGE')}
59
59
  trazum rules
60
60
  trazum gateway <anthropic|openai> --on-cannot-tell <fail-open|fail-closed>
61
61
  trazum experiment <log> --a <label> --b <label> --min-outcomes <n>
62
+ trazum quality <log> --label <name> --at <iso> [--gate]
63
+ trazum semantic <prompt> [--yes]
62
64
  trazum ladder <log>
63
65
  trazum feedback
64
66
  trazum --version
@@ -132,6 +134,31 @@ ${bold('OPTIONS FOR experiment')}
132
134
  not separable — with the number of outcomes per arm that would settle it, so
133
135
  "run it longer" is an instruction rather than a shrug. Nothing is promoted.
134
136
 
137
+ ${bold('OPTIONS FOR quality')}
138
+ --label <name> The workload to judge. Required: a mixture would
139
+ average a regression away.
140
+ --at <iso> When the change landed. Required: without a
141
+ boundary there is nothing to compare across, and
142
+ picking one would be this tool choosing which
143
+ change to blame.
144
+ --gate Exit 1 on a measured drop, 2 on "cannot tell".
145
+ Three outcomes, never two.
146
+
147
+ A before-and-after rather than an experiment, so it reports "cannot tell"
148
+ whenever the model mix, the call volume or the outcome coverage moved across
149
+ the boundary — the prompt is not the only variable and it says so.
150
+
151
+ ${bold('OPTIONS FOR semantic')}
152
+ --yes Required to send anything. Without it the command
153
+ prints what the call would cost and stops.
154
+ --model <id> Which model to ask. Priced from the catalogue.
155
+
156
+ Finds what a dictionary cannot: the same rule taught twice in different
157
+ words, an instruction restated far away, a policy a later clause contradicts.
158
+ Every quoted passage is checked character for character against the prompt,
159
+ pairs the rules engine already catches are dropped, and every token figure is
160
+ counted rather than believed. Optional, and always will be.
161
+
135
162
  ${bold('OPTIONS FOR prune')}
136
163
  --cases <file> One input per line, or a JSON array. Required.
137
164
  --yes Actually spend the calls. Without it the estimate is
@@ -977,6 +1004,73 @@ ${bold('EXAMPLES')}
977
1004
  `${path} exists and could not be parsed, so nothing was written over it. Fix or move it first.`,
978
1005
  },
979
1006
 
1007
+ semantic: {
1008
+ heading: (path) => `Semantic pass on ${path}`,
1009
+ willCost: (usd, input, output, model) =>
1010
+ `This will send the prompt to ${model}: about ${input} tokens in and ${output} out, roughly ${usd}. Estimated, not measured \u2014 a tool that spends your money to tell you how to spend less should be the first thing audited by its own arithmetic. Pass --yes to run it.`,
1011
+ needsYes: () => 'Nothing was sent. Add --yes once you have read the price above.',
1012
+ finding: (kind, because) =>
1013
+ `${kind === 'contradiction' ? 'Contradiction' : kind === 'restated-instruction' ? 'Restated' : 'Same thing, different words'}: ${because}`,
1014
+ span: (line, text) => `line ${line}: ${text}`,
1015
+ ceiling: (tokens) =>
1016
+ `At most ${tokens} tokens \u2014 a ceiling, not a saving. Merging these two means writing one passage that does the work of both, and nobody knows yet how long that is.`,
1017
+ noCeiling: () =>
1018
+ 'No tokens attached. A contradiction is worth fixing because the prompt is wrong, not because it is long, and putting a figure on it would sell the wrong reason.',
1019
+ nothingFound: () => 'Nothing survived checking. That is a real answer.',
1020
+ rejected: (count) => `${count} proposals did not survive checking against the prompt.`,
1021
+ rejectedLine: (reason, span) =>
1022
+ reason === 'span-not-found'
1023
+ ? `paraphrased its own evidence: ${span}`
1024
+ : reason === 'already-detected'
1025
+ ? `already found without a model: ${span}`
1026
+ : reason === 'contradiction-of-a-copy'
1027
+ ? `called a near-copy a contradiction: ${span}`
1028
+ : reason === 'spans-identical' || reason === 'spans-overlap'
1029
+ ? `quoted the same passage twice: ${span}`
1030
+ : `covers ground an accepted finding covers: ${span}`,
1031
+ disposes: () =>
1032
+ 'The model proposes and the deterministic layer disposes: every quoted passage is checked character for character against the prompt, pairs the rules engine already catches are dropped, and every token figure is counted here rather than believed.',
1033
+ optIn: () =>
1034
+ 'This pass is optional and always will be. Trazum works with no key, no network and no model \u2014 that has been true since 0.1.0 and this does not change it.',
1035
+ },
1036
+
1037
+ quality: {
1038
+ heading: (label) => `Quality across the change: ${label}`,
1039
+ needsLabel: () => 'Name the workload with --label: this compares one label before and after a change, and a mixture of workloads would average a regression away.',
1040
+ needsAt: () =>
1041
+ '--at is required: give the moment the change landed, as an ISO timestamp. Without it there is no boundary to compare across, and picking one from the log would be this tool choosing which change to blame.',
1042
+ sides: (beforeRate, afterRate, before, after) =>
1043
+ `before ${beforeRate} (${before} outcomes) after ${afterRate} (${after} outcomes)`,
1044
+ dropped: (from, to, outcomes, cost) =>
1045
+ `The resolution rate moved from ${from} to ${to} on ${outcomes} measured outcomes, and this change ${cost}. Both halves are measured; neither is an estimate.`,
1046
+ held: (from, to, outcomes) =>
1047
+ `The resolution rate moved from ${from} to ${to} on ${outcomes} measured outcomes \u2014 up, measurably.`,
1048
+ cannotTell: (why, need) =>
1049
+ why === 'too-few-before'
1050
+ ? `Cannot tell: only ${need} outcomes before the change, and this gate needs 100 a side. It fails builds, so the threshold is not the one a rate uses elsewhere.`
1051
+ : why === 'too-few-after'
1052
+ ? `Cannot tell yet: only ${need} outcomes since the change, and this gate needs 100 a side. Run it again once the traffic has caught up.`
1053
+ : why === 'not-separable'
1054
+ ? 'Cannot tell: the rate did not measurably move. That is NOT the same as "it held" \u2014 a gate that spelled them the same way would pass a real regression it merely lacked the power to see.'
1055
+ : why === 'no-vocabulary'
1056
+ ? 'Cannot tell: "outcomes.success" declares nothing, so there is no resolution rate to compare.'
1057
+ : 'Cannot tell: something other than the prompt moved across the boundary. See below.',
1058
+ confoundersHeading: () => 'The prompt is not the only thing that changed',
1059
+ confounder: (kind, detail) =>
1060
+ kind === 'model-mix-moved'
1061
+ ? `The model mix moved by ${detail}. The drop may be entirely somebody else's migration, and this tool cannot separate the two.`
1062
+ : kind === 'volume-moved'
1063
+ ? `The call volume moved ${detail}. A workload whose traffic moved that much is usually a workload whose population changed \u2014 a new surface, a new customer, a campaign \u2014 and the questions being asked are not the questions from before.`
1064
+ : `Outcome coverage moved from ${detail}. The two rates describe different populations: a team that starts instrumenting its hard cases sees its measured rate fall without anything having got worse.`,
1065
+ notRandomised: () =>
1066
+ 'This is a before-and-after, not an experiment. It splits traffic by time rather than at random, so everything else that changed at the same time is in the difference too \u2014 which is why it says "cannot tell" far more readily than an A/B would.',
1067
+ cannotSee: () =>
1068
+ 'It cannot see anything else you deployed that day. A "dropped" verdict says the rate fell and the three things it can check did not move. That is a smaller claim than "the prompt did it", and it is the largest one the evidence supports.',
1069
+ gateFailed: () => 'Gate failed: a measured drop with nothing else to explain it.',
1070
+ gateHeldOpen: () =>
1071
+ 'Gate not passed and not failed. "Cannot tell" holds the claim open rather than exiting green \u2014 the posture verify has had since 1.39.',
1072
+ },
1073
+
980
1074
  experiment: {
981
1075
  heading: (a, b) => `Experiment: ${a} against ${b}`,
982
1076
  needsTwo: () =>
package/src/i18n/es.ts CHANGED
@@ -46,6 +46,8 @@ ${bold('USO')}
46
46
  trazum rules
47
47
  trazum gateway <anthropic|openai> --on-cannot-tell <fail-open|fail-closed>
48
48
  trazum experiment <log> --a <label> --b <label> --min-outcomes <n>
49
+ trazum quality <log> --label <name> --at <iso> [--gate]
50
+ trazum semantic <prompt> [--yes]
49
51
  trazum ladder <log>
50
52
  trazum feedback
51
53
  trazum --version
@@ -122,6 +124,31 @@ ${bold('OPCIONES DE experiment')}
122
124
  o no separables — con cuántos resultados por brazo lo zanjarían, para que
123
125
  "déjalo correr más" sea una instrucción y no un encogimiento de hombros.
124
126
 
127
+ ${bold('OPCIONES DE quality')}
128
+ --label <nombre> La carga a juzgar. Obligatorio: una mezcla
129
+ promediaría una regresión hasta hacerla
130
+ desaparecer.
131
+ --at <iso> Cuándo aterrizó el cambio. Obligatorio: sin
132
+ frontera no hay nada que comparar, y elegir una
133
+ sería que esta herramienta decida a qué culpar.
134
+ --gate Sale con 1 si hay caída medida, 2 si no se puede
135
+ decir. Tres resultados, nunca dos.
136
+
137
+ Un antes y después, no un experimento, así que dice "no se puede decir"
138
+ siempre que la mezcla de modelos, el volumen o la cobertura se movieran a
139
+ través de la frontera — el prompt no es la única variable y lo dice.
140
+
141
+ ${bold('OPCIONES DE semantic')}
142
+ --yes Obligatorio para enviar nada. Sin él, el comando
143
+ imprime lo que costaría la llamada y para.
144
+ --model <id> Qué modelo preguntar. Se tarifa del catálogo.
145
+
146
+ Encuentra lo que un diccionario no puede: la misma regla enseñada dos veces
147
+ con otras palabras, una instrucción repetida lejos, una política que una
148
+ cláusula posterior contradice. Cada pasaje citado se comprueba carácter a
149
+ carácter contra el prompt, los pares que el motor de reglas ya caza se
150
+ descartan, y toda cifra de tokens se cuenta en vez de creérsela.
151
+
125
152
  ${bold('OPCIONES DE prune')}
126
153
  --cases <fichero> Una entrada por línea, o un array JSON. Obligatorio.
127
154
  --yes Gasta las llamadas de verdad. Sin él se imprime la
@@ -1013,6 +1040,73 @@ ${bold('EJEMPLOS')}
1013
1040
  `${path} existe y no se pudo interpretar, así que no se escribió nada encima. Arréglalo o muévelo primero.`,
1014
1041
  },
1015
1042
 
1043
+ semantic: {
1044
+ heading: (path) => `Pase sem\u00e1ntico sobre ${path}`,
1045
+ willCost: (usd, input, output, model) =>
1046
+ `Esto enviar\u00e1 el prompt a ${model}: unos ${input} tokens de entrada y ${output} de salida, aproximadamente ${usd}. Estimado, no medido \u2014 una herramienta que gasta tu dinero para decirte c\u00f3mo gastar menos deber\u00eda ser lo primero que audite su propia aritm\u00e9tica. Pasa --yes para ejecutarlo.`,
1047
+ needsYes: () => 'No se envi\u00f3 nada. A\u00f1ade --yes cuando hayas le\u00eddo el precio de arriba.',
1048
+ finding: (kind, because) =>
1049
+ `${kind === 'contradiction' ? 'Contradicci\u00f3n' : kind === 'restated-instruction' ? 'Repetido' : 'Lo mismo con otras palabras'}: ${because}`,
1050
+ span: (line, text) => `l\u00ednea ${line}: ${text}`,
1051
+ ceiling: (tokens) =>
1052
+ `Como mucho ${tokens} tokens \u2014 un techo, no un ahorro. Fundir los dos significa escribir un pasaje que haga el trabajo de ambos, y nadie sabe todav\u00eda cu\u00e1nto ocupa.`,
1053
+ noCeiling: () =>
1054
+ 'Sin tokens asociados. Una contradicci\u00f3n merece arreglarse porque el prompt est\u00e1 mal, no porque sea largo, y ponerle una cifra vender\u00eda el motivo equivocado.',
1055
+ nothingFound: () => 'Nada sobrevivi\u00f3 a la comprobaci\u00f3n. Esa es una respuesta real.',
1056
+ rejected: (count) => `${count} propuestas no sobrevivieron a la comprobaci\u00f3n contra el prompt.`,
1057
+ rejectedLine: (reason, span) =>
1058
+ reason === 'span-not-found'
1059
+ ? `parafrase\u00f3 su propia evidencia: ${span}`
1060
+ : reason === 'already-detected'
1061
+ ? `ya se encuentra sin modelo: ${span}`
1062
+ : reason === 'contradiction-of-a-copy'
1063
+ ? `llam\u00f3 contradicci\u00f3n a una casi copia: ${span}`
1064
+ : reason === 'spans-identical' || reason === 'spans-overlap'
1065
+ ? `cit\u00f3 el mismo pasaje dos veces: ${span}`
1066
+ : `cubre terreno que ya cubre un hallazgo aceptado: ${span}`,
1067
+ disposes: () =>
1068
+ 'El modelo propone y la capa determinista dispone: cada pasaje citado se comprueba car\u00e1cter a car\u00e1cter contra el prompt, los pares que el motor de reglas ya caza se descartan, y toda cifra de tokens se cuenta aqu\u00ed en vez de cre\u00e9rsela.',
1069
+ optIn: () =>
1070
+ 'Este pase es opcional y siempre lo ser\u00e1. Trazum funciona sin clave, sin red y sin modelo \u2014 eso es cierto desde 0.1.0 y esto no lo cambia.',
1071
+ },
1072
+
1073
+ quality: {
1074
+ heading: (label) => `Calidad a trav\u00e9s del cambio: ${label}`,
1075
+ needsLabel: () => 'Nombra la carga con --label: esto compara una etiqueta antes y despu\u00e9s de un cambio, y una mezcla de cargas promediar\u00eda una regresi\u00f3n hasta hacerla desaparecer.',
1076
+ needsAt: () =>
1077
+ '--at es obligatorio: da el momento en que aterriz\u00f3 el cambio, como marca ISO. Sin \u00e9l no hay frontera que comparar, y elegir una del log ser\u00eda que esta herramienta decida a qu\u00e9 cambio culpar.',
1078
+ sides: (beforeRate, afterRate, before, after) =>
1079
+ `antes ${beforeRate} (${before} resultados) despu\u00e9s ${afterRate} (${after} resultados)`,
1080
+ dropped: (from, to, outcomes, cost) =>
1081
+ `La tasa de resoluci\u00f3n pas\u00f3 de ${from} a ${to} sobre ${outcomes} resultados medidos, y este cambio ${cost}. Las dos mitades son medidas; ninguna es una estimaci\u00f3n.`,
1082
+ held: (from, to, outcomes) =>
1083
+ `La tasa de resoluci\u00f3n pas\u00f3 de ${from} a ${to} sobre ${outcomes} resultados medidos \u2014 arriba, de forma medible.`,
1084
+ cannotTell: (why, need) =>
1085
+ why === 'too-few-before'
1086
+ ? `No se puede decir: solo ${need} resultados antes del cambio, y esta puerta necesita 100 por lado. Falla builds, as\u00ed que el umbral no es el que usa una tasa en otros sitios.`
1087
+ : why === 'too-few-after'
1088
+ ? `A\u00fan no se puede decir: solo ${need} resultados desde el cambio, y esta puerta necesita 100 por lado. Vuelve a correrlo cuando el tr\u00e1fico se ponga al d\u00eda.`
1089
+ : why === 'not-separable'
1090
+ ? 'No se puede decir: la tasa no se movi\u00f3 de forma medible. Eso NO es lo mismo que "se mantuvo" \u2014 una puerta que las escribiera igual dejar\u00eda pasar una regresi\u00f3n real que simplemente no tuvo potencia para ver.'
1091
+ : why === 'no-vocabulary'
1092
+ ? 'No se puede decir: "outcomes.success" no declara nada, as\u00ed que no hay tasa de resoluci\u00f3n que comparar.'
1093
+ : 'No se puede decir: algo que no es el prompt se movi\u00f3 a trav\u00e9s de la frontera. Ver abajo.',
1094
+ confoundersHeading: () => 'El prompt no es lo \u00fanico que cambi\u00f3',
1095
+ confounder: (kind, detail) =>
1096
+ kind === 'model-mix-moved'
1097
+ ? `La mezcla de modelos se movi\u00f3 un ${detail}. La ca\u00edda puede ser enteramente la migraci\u00f3n de otra persona, y esta herramienta no puede separarlas.`
1098
+ : kind === 'volume-moved'
1099
+ ? `El volumen de llamadas se movi\u00f3 ${detail}. Una carga cuyo tr\u00e1fico se mueve tanto suele ser una carga cuya poblaci\u00f3n cambi\u00f3 \u2014 una superficie nueva, un cliente nuevo, una campa\u00f1a \u2014 y las preguntas que se hacen no son las de antes.`
1100
+ : `La cobertura de resultados pas\u00f3 de ${detail}. Las dos tasas describen poblaciones distintas: un equipo que empieza a instrumentar sus casos dif\u00edciles ve caer su tasa medida sin que nada haya empeorado.`,
1101
+ notRandomised: () =>
1102
+ 'Esto es un antes y despu\u00e9s, no un experimento. Parte el tr\u00e1fico por tiempo y no al azar, as\u00ed que todo lo dem\u00e1s que cambi\u00f3 a la vez est\u00e1 tambi\u00e9n en la diferencia \u2014 por eso dice "no se puede decir" mucho m\u00e1s f\u00e1cilmente que un A/B.',
1103
+ cannotSee: () =>
1104
+ 'No puede ver nada m\u00e1s que desplegaras ese d\u00eda. Un veredicto de "ca\u00edda" dice que la tasa baj\u00f3 y que las tres cosas que puede comprobar no se movieron. Es una afirmaci\u00f3n m\u00e1s peque\u00f1a que "lo hizo el prompt", y es la mayor que sostiene la evidencia.',
1105
+ gateFailed: () => 'Puerta fallada: una ca\u00edda medida sin nada m\u00e1s que la explique.',
1106
+ gateHeldOpen: () =>
1107
+ 'Puerta ni pasada ni fallada. "No se puede decir" mantiene la afirmaci\u00f3n abierta en vez de salir en verde \u2014 la postura de verify desde 1.39.',
1108
+ },
1109
+
1016
1110
  experiment: {
1017
1111
  heading: (a, b) => `Experimento: ${a} contra ${b}`,
1018
1112
  needsTwo: () =>
package/src/i18n/types.ts CHANGED
@@ -268,6 +268,37 @@ export interface CliMessages {
268
268
  * The ladder. Every line here exists to stop somebody reading "we route to
269
269
  * the cheap model first" as a saving without the number that decides it.
270
270
  */
271
+ semantic: {
272
+ heading(path: string): string;
273
+ willCost(usd: string, input: string, output: string, model: string): string;
274
+ needsYes(): string;
275
+ finding(kind: string, because: string): string;
276
+ span(line: string, text: string): string;
277
+ ceiling(tokens: string): string;
278
+ noCeiling(): string;
279
+ nothingFound(): string;
280
+ rejected(count: string): string;
281
+ rejectedLine(reason: string, span: string): string;
282
+ disposes(): string;
283
+ optIn(): string;
284
+ };
285
+
286
+ quality: {
287
+ heading(label: string): string;
288
+ needsLabel(): string;
289
+ needsAt(): string;
290
+ sides(beforeRate: string, afterRate: string, before: string, after: string): string;
291
+ dropped(from: string, to: string, outcomes: string, cost: string): string;
292
+ held(from: string, to: string, outcomes: string): string;
293
+ cannotTell(why: string, need: string): string;
294
+ confounder(kind: string, detail: string): string;
295
+ confoundersHeading(): string;
296
+ notRandomised(): string;
297
+ cannotSee(): string;
298
+ gateFailed(): string;
299
+ gateHeldOpen(): string;
300
+ };
301
+
271
302
  experiment: {
272
303
  heading(a: string, b: string): string;
273
304
  needsTwo(): string;
package/src/index.ts CHANGED
@@ -41,6 +41,10 @@ import {
41
41
  conform,
42
42
  BREAK_EVEN_BAND,
43
43
  runExperiment,
44
+ qualityGate,
45
+ semanticPassCost,
46
+ verifySemanticProposals,
47
+ SEMANTIC_SYSTEM_PROMPT,
44
48
  ladderPosition,
45
49
  validateLadder,
46
50
  outcomeReport,
@@ -141,6 +145,8 @@ import type {
141
145
  BudgetReport,
142
146
  ContractName,
143
147
  ExperimentArm,
148
+ GateSide,
149
+ SemanticProposal,
144
150
  FailurePolicy,
145
151
  GatewayStanding,
146
152
  UsageProfileReport,
@@ -248,6 +254,7 @@ interface Args {
248
254
 
249
255
  const VALUE_FLAGS = new Set([
250
256
  'a',
257
+ 'at',
251
258
  'b',
252
259
  'min-outcomes',
253
260
  'against',
@@ -580,6 +587,8 @@ const COMMAND_FLAGS: Record<string, string[]> = {
580
587
  gateway: ['on-cannot-tell', 'port', 'socket', 'pricing', 'pricing-live'],
581
588
  ladder: ['pricing', 'pricing-live', 'since', 'until', 'label'],
582
589
  experiment: ['a', 'b', 'min-outcomes', 'pricing', 'pricing-live'],
590
+ quality: ['label', 'at', 'gate', 'pricing', 'pricing-live'],
591
+ semantic: ['yes', 'model', 'pricing', 'pricing-live'],
583
592
  where: [],
584
593
  rules: [],
585
594
  blame: ['limit', 'model', 'calls', 'output-tokens', 'batch', 'prompt', 'markdown-out'],
@@ -2459,6 +2468,272 @@ async function commandExperiment(
2459
2468
  console.log();
2460
2469
  }
2461
2470
 
2471
+ /**
2472
+ * `trazum quality <log> --label <name> --at <iso> [--gate]`
2473
+ *
2474
+ * The failure that actually matters: a prompt edit that quietly made the
2475
+ * product worse. CI has been able to fail a build for tokens since 1.4 and for
2476
+ * dollars since 1.21, and this has never been gateable — so every saving this
2477
+ * tool has ever recommended went into a repository with its most important
2478
+ * consequence unmeasured.
2479
+ *
2480
+ * **Named `quality` rather than `check --against-outcomes`, which is what the
2481
+ * plan called for.** `check` reads *prompt files* and gates on tokens; it has
2482
+ * never opened a usage log, and a command that takes either a prompt or a log
2483
+ * depending on a flag is two commands wearing one name. The split-by-time this
2484
+ * needs is also not a `check` idea — there is nothing in a prompt file with a
2485
+ * timestamp on it.
2486
+ */
2487
+ async function commandQuality(
2488
+ args: Args,
2489
+ config: TrazumConfig,
2490
+ pricing: PricingCatalogue,
2491
+ t: CliMessages,
2492
+ ): Promise<void> {
2493
+ const path = args.positional[0];
2494
+ if (path === undefined) throw new Error(t.errors.missingInputFile());
2495
+
2496
+ const label = stringFlag(args, 'label');
2497
+ if (label === undefined) throw new Error(t.quality.needsLabel());
2498
+
2499
+ const atRaw = stringFlag(args, 'at');
2500
+ const atMs = atRaw === undefined ? Number.NaN : Date.parse(atRaw);
2501
+ if (!Number.isFinite(atMs)) throw new Error(t.quality.needsAt());
2502
+
2503
+ /**
2504
+ * Two profiles over the same file, split at the boundary — rather than one
2505
+ * profile the caller has to slice.
2506
+ *
2507
+ * The alternative is asking somebody for two logs, which invites the mistake
2508
+ * this whole module exists to avoid: two files gathered under conditions
2509
+ * nobody wrote down.
2510
+ */
2511
+ const raw = await readUsageLog(path, t);
2512
+ const sideOf = (since: number | undefined, until: number | undefined): GateSide => {
2513
+ const report = profileUsage(raw, { catalogue: pricing, label, sinceMs: since, untilMs: until });
2514
+ const slice = report.outcomeTallyByLabel.find((entry) => entry.label === label);
2515
+ return {
2516
+ arm: {
2517
+ name: label,
2518
+ totalUsd: report.total.totalUsd,
2519
+ tally: slice?.tally ?? { byValue: [], recorded: 0, parsed: 0, unrecordedUsd: 0 },
2520
+ },
2521
+ calls: report.total.calls,
2522
+ usdByModel: report.byModel.map((entry) => ({ model: entry.model, usd: entry.breakdown.totalUsd })),
2523
+ };
2524
+ };
2525
+
2526
+ const result = qualityGate(sideOf(undefined, atMs), sideOf(atMs, undefined), config.outcomes ?? null);
2527
+ const pct = (value: number): string => `${(value * 100).toFixed(1)}%`;
2528
+ const n = (value: number): string => value.toLocaleString(t.numberLocale);
2529
+
2530
+ console.log();
2531
+ console.log(c.bold(t.quality.heading(label)));
2532
+ console.log(` ${c.dim(wrap(t.quality.notRandomised(), 74, ' '))}`);
2533
+ console.log();
2534
+ console.log(
2535
+ ` ${t.quality.sides(
2536
+ result.before.rate === null ? '—' : pct(result.before.rate),
2537
+ result.after.rate === null ? '—' : pct(result.after.rate),
2538
+ n(result.outcomes.before),
2539
+ n(result.outcomes.after),
2540
+ )}`,
2541
+ );
2542
+ console.log();
2543
+
2544
+ if (result.verdict === 'dropped') {
2545
+ const cost =
2546
+ result.cost === null
2547
+ ? ''
2548
+ : result.cost.deltaUsdPerCall < 0
2549
+ ? `saves ${formatUsd(-result.cost.deltaUsdPerCall)} a call`
2550
+ : `costs ${formatUsd(result.cost.deltaUsdPerCall)} a call more`;
2551
+ console.log(
2552
+ ` ${c.red('✗')} ${wrap(
2553
+ t.quality.dropped(
2554
+ pct(result.before.rate ?? 0),
2555
+ pct(result.after.rate ?? 0),
2556
+ n(result.outcomes.before + result.outcomes.after),
2557
+ cost,
2558
+ ),
2559
+ 74,
2560
+ ' ',
2561
+ )}`,
2562
+ );
2563
+ } else if (result.verdict === 'held') {
2564
+ console.log(
2565
+ ` ${c.green('✓')} ${wrap(
2566
+ t.quality.held(pct(result.before.rate ?? 0), pct(result.after.rate ?? 0), n(result.outcomes.before + result.outcomes.after)),
2567
+ 74,
2568
+ ' ',
2569
+ )}`,
2570
+ );
2571
+ } else {
2572
+ const need =
2573
+ result.unknown === 'too-few-before' ? n(result.outcomes.before) : n(result.outcomes.after);
2574
+ console.log(` ${c.yellow('?')} ${wrap(t.quality.cannotTell(result.unknown ?? '', need), 74, ' ')}`);
2575
+ }
2576
+
2577
+ /**
2578
+ * Confounders print on **every** verdict, not only on `cannot-tell`.
2579
+ *
2580
+ * A rate that held while the model changed underneath is not evidence that
2581
+ * the prompt is fine either, and hiding the confounder on a green result is
2582
+ * how a gate teaches people to trust it in exactly the case it should not be
2583
+ * trusted.
2584
+ */
2585
+ if (result.confounders.length > 0) {
2586
+ console.log();
2587
+ console.log(` ${c.bold(t.quality.confoundersHeading())}`);
2588
+ for (const confounder of result.confounders) {
2589
+ const detail =
2590
+ confounder.kind === 'model-mix-moved'
2591
+ ? `${pct(confounder.drift)} (${confounder.model})`
2592
+ : confounder.kind === 'volume-moved'
2593
+ ? `${n(confounder.beforeCalls)} → ${n(confounder.afterCalls)} calls`
2594
+ : `${pct(confounder.before)} → ${pct(confounder.after)}`;
2595
+ console.log(` ${c.yellow('!')} ${wrap(t.quality.confounder(confounder.kind, detail), 70, ' ')}`);
2596
+ }
2597
+ }
2598
+
2599
+ console.log();
2600
+ console.log(` ${c.dim(wrap(t.quality.cannotSee(), 74, ' '))}`);
2601
+
2602
+ if (boolFlag(args, 'gate')) {
2603
+ console.log();
2604
+ if (result.verdict === 'dropped') {
2605
+ console.log(` ${c.red(t.quality.gateFailed())}`);
2606
+ process.exitCode = 1;
2607
+ } else if (result.verdict === 'cannot-tell') {
2608
+ // Three outcomes, never two. `cannot tell` holds the claim open rather
2609
+ // than exiting green, the posture `verify --gate` has had since 1.39.
2610
+ console.log(` ${c.yellow(t.quality.gateHeldOpen())}`);
2611
+ process.exitCode = 2;
2612
+ }
2613
+ }
2614
+ console.log();
2615
+ }
2616
+
2617
+ /**
2618
+ * `trazum semantic <prompt> [--yes]` — the findings a dictionary cannot see.
2619
+ *
2620
+ * The rules engine has deferred these since 0.1.0 for one honest reason: a
2621
+ * dictionary cannot see meaning, and a model that hallucinates a finding is
2622
+ * worse than a rule that misses one.
2623
+ *
2624
+ * **The price is printed before anything is sent, and `--yes` is required.** A
2625
+ * tool that spends somebody's money to tell them how to spend less has to be
2626
+ * the first thing audited by its own arithmetic, and it has to ask.
2627
+ */
2628
+ async function commandSemantic(
2629
+ args: Args,
2630
+ config: TrazumConfig,
2631
+ pricing: PricingCatalogue,
2632
+ t: CliMessages,
2633
+ ): Promise<void> {
2634
+ const prompt = await readInput(args.positional[0], t);
2635
+ const modelId = stringFlag(args, 'model') ?? config.usage?.model ?? DEFAULT_USAGE.model;
2636
+ const model = pricing.byId.get(modelId) ?? getModel(DEFAULT_USAGE.model);
2637
+ const rates = { inputPerMTok: model.inputPerMTok, outputPerMTok: model.outputPerMTok };
2638
+ const cost = semanticPassCost(prompt, rates);
2639
+ const n = (value: number): string => value.toLocaleString(t.numberLocale);
2640
+
2641
+ console.log();
2642
+ console.log(c.bold(t.semantic.heading(args.positional[0] ?? '-')));
2643
+ console.log();
2644
+ console.log(
2645
+ ` ${wrap(
2646
+ t.semantic.willCost(formatUsd(cost.usd), n(cost.inputTokens), n(cost.outputTokens), model.displayName),
2647
+ 74,
2648
+ ' ',
2649
+ )}`,
2650
+ );
2651
+
2652
+ if (!boolFlag(args, 'yes')) {
2653
+ // Nothing has been sent at this point, and nothing will be. The price
2654
+ // above is the whole output of a run without --yes.
2655
+ console.log();
2656
+ console.log(` ${c.dim(t.semantic.needsYes())}`);
2657
+ console.log();
2658
+ return;
2659
+ }
2660
+
2661
+ const provider = providerFromEnv();
2662
+ if (!provider) throw new Error(t.errors.llmNotConfigured());
2663
+
2664
+ const answer = await provider.complete({ system: SEMANTIC_SYSTEM_PROMPT, user: prompt });
2665
+ let proposals: SemanticProposal[] = [];
2666
+ try {
2667
+ const parsed: unknown = JSON.parse(
2668
+ /^(?:```|~~~)[a-zA-Z]*\n([\s\S]*?)\n?(?:```|~~~)$/.exec(answer.trim())?.[1] ?? answer.trim(),
2669
+ );
2670
+ /**
2671
+ * A response that is not the shape asked for is **no proposals**, never a
2672
+ * crash and never a partial read. The model was told exactly what to
2673
+ * return; anything else is a response this layer cannot check, and an
2674
+ * unchecked finding is the one thing this whole module exists to prevent.
2675
+ */
2676
+ if (Array.isArray(parsed)) {
2677
+ proposals = parsed.filter(
2678
+ (entry): entry is SemanticProposal =>
2679
+ typeof entry === 'object' &&
2680
+ entry !== null &&
2681
+ Array.isArray((entry as SemanticProposal).spans) &&
2682
+ (entry as SemanticProposal).spans.length === 2 &&
2683
+ (entry as SemanticProposal).spans.every((span) => typeof span === 'string'),
2684
+ );
2685
+ }
2686
+ } catch {
2687
+ proposals = [];
2688
+ }
2689
+
2690
+ const result = verifySemanticProposals(prompt, proposals);
2691
+ const lineOf = (offset: number): number => prompt.slice(0, offset).split('\n').length;
2692
+
2693
+ console.log();
2694
+ if (result.findings.length === 0) {
2695
+ console.log(` ${c.dim(t.semantic.nothingFound())}`);
2696
+ }
2697
+ for (const finding of result.findings) {
2698
+ console.log(` ${c.bold(t.semantic.finding(finding.kind, finding.because))}`);
2699
+ finding.spans.forEach((span, index) => {
2700
+ const shown = span.length > 90 ? `${span.slice(0, 87)}…` : span;
2701
+ console.log(` ${c.dim(t.semantic.span(String(lineOf(finding.offsets[index] ?? 0)), shown))}`);
2702
+ });
2703
+ console.log(
2704
+ ` ${c.dim(
2705
+ wrap(
2706
+ finding.ceilingTokens > 0 ? t.semantic.ceiling(n(finding.ceilingTokens)) : t.semantic.noCeiling(),
2707
+ 70,
2708
+ ' ',
2709
+ ),
2710
+ )}`,
2711
+ );
2712
+ console.log();
2713
+ }
2714
+
2715
+ /**
2716
+ * What did **not** survive, counted and reasoned.
2717
+ *
2718
+ * A pass that showed only its accepted findings would hide its own hit
2719
+ * rate, and the hit rate is the most useful thing a reader can know about
2720
+ * whether to run it again.
2721
+ */
2722
+ if (result.rejected.length > 0) {
2723
+ console.log(` ${c.dim(t.semantic.rejected(n(result.rejected.length)))}`);
2724
+ for (const { proposal, reason } of result.rejected.slice(0, 5)) {
2725
+ const span = proposal.spans[0];
2726
+ const shown = span.length > 50 ? `${span.slice(0, 47)}…` : span;
2727
+ console.log(` ${c.dim(t.semantic.rejectedLine(reason, shown))}`);
2728
+ }
2729
+ console.log();
2730
+ }
2731
+
2732
+ console.log(` ${c.dim(wrap(t.semantic.disposes(), 74, ' '))}`);
2733
+ console.log(` ${c.dim(wrap(t.semantic.optIn(), 74, ' '))}`);
2734
+ console.log();
2735
+ }
2736
+
2462
2737
  function commandModels(t: CliMessages, pricing: PricingCatalogue): void {
2463
2738
  const n = (value: number): string => value.toLocaleString(t.numberLocale);
2464
2739
  const col = t.models.columns;
@@ -8659,6 +8934,12 @@ async function main(): Promise<void> {
8659
8934
  case 'models':
8660
8935
  commandModels(t, pricing);
8661
8936
  break;
8937
+ case 'semantic':
8938
+ await commandSemantic(args, config, pricing, t);
8939
+ break;
8940
+ case 'quality':
8941
+ await commandQuality(args, config, pricing, t);
8942
+ break;
8662
8943
  case 'experiment':
8663
8944
  await commandExperiment(args, config, pricing, t);
8664
8945
  break;