@trazum/cli 1.55.0 → 1.56.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +75 -1
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/es.d.ts.map +1 -1
- package/dist/i18n/es.js +76 -1
- package/dist/i18n/es.js.map +1 -1
- package/dist/i18n/types.d.ts +22 -0
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/index.js +225 -4
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
- package/src/i18n/en.ts +90 -1
- package/src/i18n/es.ts +92 -1
- package/src/i18n/types.ts +23 -0
- package/src/index.ts +245 -3
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@trazum/cli",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.56.1",
|
|
4
4
|
"description": "Trazum CLI: find where your LLM bill goes, price every finding per month, and enforce token budgets in CI.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "David Mu\u00f1oz Rey",
|
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
"prepublishOnly": "npm run build && npm test"
|
|
38
38
|
},
|
|
39
39
|
"dependencies": {
|
|
40
|
-
"@trazum/core": "1.
|
|
40
|
+
"@trazum/core": "1.56.1"
|
|
41
41
|
},
|
|
42
42
|
"devDependencies": {
|
|
43
43
|
"@types/node": "^26.2.0",
|
package/src/i18n/en.ts
CHANGED
|
@@ -57,8 +57,9 @@ ${bold('USAGE')}
|
|
|
57
57
|
trazum where [file]
|
|
58
58
|
trazum conform <file|-> [--contract <name>]
|
|
59
59
|
trazum rollup <document...|dir> [--json]
|
|
60
|
+
trazum pulse [--max-stale-hours <n>]
|
|
60
61
|
trazum models
|
|
61
|
-
trazum rules
|
|
62
|
+
trazum rules [--measure <dir>] [--level <safe|aggressive>]
|
|
62
63
|
trazum gateway <provider> --on-cannot-tell <fail-open|fail-closed>
|
|
63
64
|
trazum experiment <log> --a <label> --b <label> --min-outcomes <n>
|
|
64
65
|
trazum quality <log> --label <name> --at <iso> [--gate]
|
|
@@ -116,6 +117,44 @@ ${bold('OPTIONS FOR rollup')}
|
|
|
116
117
|
|
|
117
118
|
Exits 1 when a contribution was handed over and could not be merged.
|
|
118
119
|
|
|
120
|
+
${bold('OPTIONS FOR pulse')}
|
|
121
|
+
--max-stale-hours <n> Exit 1 when something that runs here has not run
|
|
122
|
+
in over n hours. Without it, the ages are
|
|
123
|
+
reported and nothing is judged.
|
|
124
|
+
--json The report as data.
|
|
125
|
+
|
|
126
|
+
Did the things that are supposed to run, run? \`watch --once\` is built for a
|
|
127
|
+
scheduler, and its state file is read by exactly one thing — the next cycle.
|
|
128
|
+
So nothing could tell you the watcher had stopped, because the thing that
|
|
129
|
+
would tell you was the thing that stopped.
|
|
130
|
+
|
|
131
|
+
It runs nothing and hosts nothing. Something has to notice, and the something
|
|
132
|
+
is your CI: a step running this on the schedule you already have turns a dead
|
|
133
|
+
cron into a red build, without Trazum holding anybody's metrics.
|
|
134
|
+
|
|
135
|
+
Never gates on a first run that never happened — that has no cadence to be
|
|
136
|
+
late against — and never on how far the measurements reach, which is a
|
|
137
|
+
provider reporting on its own schedule rather than a job that failed.
|
|
138
|
+
|
|
139
|
+
${bold('OPTIONS FOR rules')}
|
|
140
|
+
--measure <dir> Measure what each rule actually recovers over the
|
|
141
|
+
prompts in that directory, instead of listing what
|
|
142
|
+
each one does.
|
|
143
|
+
--level <safe|aggressive> Which rules to measure. Default: safe.
|
|
144
|
+
--json The measurement as data.
|
|
145
|
+
|
|
146
|
+
Without --measure it lists the rules and what each is for. With it, the
|
|
147
|
+
optimiser is run once per rule alone and once per rule removed, and both
|
|
148
|
+
figures are printed: where two rules find the same tokens they diverge, and
|
|
149
|
+
either one on its own would be wrong in a different direction.
|
|
150
|
+
|
|
151
|
+
The floor is separated out. The optimiser normalises whitespace whether or not
|
|
152
|
+
a rule is enabled, and crediting that to the rules is how a headline
|
|
153
|
+
percentage survives on a corpus where the rules recover nothing.
|
|
154
|
+
|
|
155
|
+
"Inert" is always said about the corpus. A rule that finds nothing in these
|
|
156
|
+
files has not been shown to find nothing anywhere.
|
|
157
|
+
|
|
119
158
|
${bold('OPTIONS FOR feedback')}
|
|
120
159
|
(none)
|
|
121
160
|
|
|
@@ -1461,6 +1500,31 @@ ${bold('EXAMPLES')}
|
|
|
1461
1500
|
},
|
|
1462
1501
|
},
|
|
1463
1502
|
|
|
1503
|
+
pulse: {
|
|
1504
|
+
heading: () => 'Did the things that are supposed to run, run?',
|
|
1505
|
+
kind: (kind) => {
|
|
1506
|
+
switch (kind) {
|
|
1507
|
+
case 'watch-cycle':
|
|
1508
|
+
return 'last watch cycle';
|
|
1509
|
+
case 'store-pull':
|
|
1510
|
+
return 'last pull into the store';
|
|
1511
|
+
case 'store-coverage':
|
|
1512
|
+
return 'measurements reach up to';
|
|
1513
|
+
default:
|
|
1514
|
+
return kind;
|
|
1515
|
+
}
|
|
1516
|
+
},
|
|
1517
|
+
neverRun: (name) => `${name}: never here. Not late — there is no cadence to be late against.`,
|
|
1518
|
+
age: (name, when, hours) => `${name}: ${when}Z, ${plural(hours, 'hour')} ago`,
|
|
1519
|
+
noThreshold: () =>
|
|
1520
|
+
'Nothing was judged. Pass --max-stale-hours <n> to make a run that stopped a failing exit code; how stale is too stale is a policy, and this tool does not write yours.',
|
|
1521
|
+
within: (hours) => `Everything that has run here ran within ${plural(hours, 'hour')}.`,
|
|
1522
|
+
stale: (hours) =>
|
|
1523
|
+
`Something that runs here has not run in over ${plural(hours, 'hour')}. Silence from a scheduled job and silence from a job with nothing to report look identical; this is which.`,
|
|
1524
|
+
notAService: () =>
|
|
1525
|
+
'This command runs nothing and hosts nothing. Something has to notice, and the something is your CI: a step that runs this on the schedule you already have turns a dead cron into a red build, without Trazum holding anybody\'s metrics. How far the measurements reach is reported and never judged — that is a provider reporting on its own schedule, not a job that failed.',
|
|
1526
|
+
},
|
|
1527
|
+
|
|
1464
1528
|
where: {
|
|
1465
1529
|
hostHeading: () => 'Running inside',
|
|
1466
1530
|
subscription: (host) =>
|
|
@@ -1567,6 +1631,24 @@ ${bold('EXAMPLES')}
|
|
|
1567
1631
|
rules: {
|
|
1568
1632
|
title: () => 'Available rules',
|
|
1569
1633
|
disableHint: () => ' Turn off the ones you do not want with --disable id1,id2',
|
|
1634
|
+
measureHeading: (root, prompts, level) =>
|
|
1635
|
+
`What each rule recovers in ${root} — ${plural(prompts, 'prompt')}, level ${level}`,
|
|
1636
|
+
measureTotals: (before, saved, floor) =>
|
|
1637
|
+
`${before} tokens before. The rules recover ${saved}; normalisation recovers ${floor} whether a rule is enabled or not, and that is not the rules' work.`,
|
|
1638
|
+
measureRow: (id, marginal, alone, prompts) =>
|
|
1639
|
+
`${id.padEnd(22)} lost if removed ${marginal.padStart(6)} alone ${alone.padStart(6)} ${plural(prompts, 'prompt')}`,
|
|
1640
|
+
measureOverlap: (sumOfAlone, saved) =>
|
|
1641
|
+
`The rules recover ${sumOfAlone} between them one at a time and ${saved} together. The gap is the overlap — two rules finding the same tokens — and it is stated rather than resolved into a total, because a single figure here is the one number that cannot be true.`,
|
|
1642
|
+
measureRedundant: (ids) =>
|
|
1643
|
+
`Every token these find, something else finds too, here: ${ids}. An overlap, not a defect — and not a reason to delete one without deciding which is the better to keep.`,
|
|
1644
|
+
measureFiredWithoutSaving: (ids) =>
|
|
1645
|
+
`These changed the prompt and recovered no tokens: ${ids}. Not the same as inert — they were exercised, and they are altering somebody's instruction for no measured benefit.`,
|
|
1646
|
+
measureInert: (ids) =>
|
|
1647
|
+
`These changed nothing in this corpus: ${ids}. That is a fact about these files, not about the rules: a rule that finds nothing here has not been shown to find nothing anywhere.`,
|
|
1648
|
+
measureBand: (source) =>
|
|
1649
|
+
source === 'heuristic'
|
|
1650
|
+
? 'Counted with the built-in estimator, so every figure above carries its documented band. A rule whose yield is a handful of tokens is inside the noise.'
|
|
1651
|
+
: 'Counted with an external tokenizer.',
|
|
1570
1652
|
},
|
|
1571
1653
|
|
|
1572
1654
|
|
|
@@ -2430,6 +2512,13 @@ ${bold('EXAMPLES')}
|
|
|
2430
2512
|
},
|
|
2431
2513
|
storeNoLabels: () =>
|
|
2432
2514
|
'This series comes from the store, and a usage API groups by model and workspace rather than by workload — so there is no label series here at all. Absent, not empty: nothing above says a workload did or did not move.',
|
|
2515
|
+
unmeasured: (days, from, to, after, before) =>
|
|
2516
|
+
`Nothing covers ${from} to ${to} — ${plural(days, 'day')} between ${after} and ${before}.`,
|
|
2517
|
+
unmeasuredTotal: (days) =>
|
|
2518
|
+
`${plural(days, 'day')} of this stretch are covered by no report. A series with a hole in it and a shorter series read identically; this is which.`,
|
|
2519
|
+
overlap: (a, b, days) =>
|
|
2520
|
+
`${a} and ${b} both cover ${plural(days, 'day')}. Named, never merged — which is the better measurement is not knowable from here, and adding both totals counts those days twice.`,
|
|
2521
|
+
runHole: (days) => ` — ${plural(days, 'day')} of this run are covered by no report`,
|
|
2433
2522
|
undated: (name) => `${name} carries no span, so it is on no timeline above — named, never silently absorbed.`,
|
|
2434
2523
|
unrecognized: (name) => `${name} is neither a stored report nor a saved plan, so it is in no series above.`,
|
|
2435
2524
|
footer: () =>
|
package/src/i18n/es.ts
CHANGED
|
@@ -44,8 +44,9 @@ ${bold('USO')}
|
|
|
44
44
|
trazum where [fichero]
|
|
45
45
|
trazum conform <fichero|-> [--contract <nombre>]
|
|
46
46
|
trazum rollup <documento...|dir> [--json]
|
|
47
|
+
trazum pulse [--max-stale-hours <n>]
|
|
47
48
|
trazum models
|
|
48
|
-
trazum rules
|
|
49
|
+
trazum rules [--measure <dir>] [--level <safe|aggressive>]
|
|
49
50
|
trazum gateway <provider> --on-cannot-tell <fail-open|fail-closed>
|
|
50
51
|
trazum experiment <log> --a <label> --b <label> --min-outcomes <n>
|
|
51
52
|
trazum quality <log> --label <name> --at <iso> [--gate]
|
|
@@ -104,6 +105,45 @@ ${bold('OPCIONES DE rollup')}
|
|
|
104
105
|
|
|
105
106
|
Sale con 1 cuando una contribución se entregó y no se pudo agregar.
|
|
106
107
|
|
|
108
|
+
${bold('OPCIONES DE pulse')}
|
|
109
|
+
--max-stale-hours <n> Sale con 1 cuando algo que corre aquí lleva más
|
|
110
|
+
de n horas sin correr. Sin esto, se informan las
|
|
111
|
+
edades y no se juzga nada.
|
|
112
|
+
--json El informe como datos.
|
|
113
|
+
|
|
114
|
+
¿Corrieron las cosas que tienen que correr? \`watch --once\` está hecho para un
|
|
115
|
+
planificador, y su fichero de estado lo lee exactamente una cosa: el siguiente
|
|
116
|
+
ciclo. Así que nada podía decirte que el vigilante se había parado, porque lo
|
|
117
|
+
que te lo diría era lo que se había parado.
|
|
118
|
+
|
|
119
|
+
No ejecuta nada y no aloja nada. Algo tiene que darse cuenta, y ese algo es tu
|
|
120
|
+
CI: un paso que ejecute esto con el horario que ya tienes convierte un cron
|
|
121
|
+
muerto en una build roja, sin que Trazum guarde las métricas de nadie.
|
|
122
|
+
|
|
123
|
+
Nunca falla por una primera ejecución que nunca ocurrió — no hay cadencia
|
|
124
|
+
contra la que ir tarde — ni por hasta dónde llegan las mediciones, que es un
|
|
125
|
+
proveedor informando a su ritmo y no un trabajo que falló.
|
|
126
|
+
|
|
127
|
+
${bold('OPCIONES DE rules')}
|
|
128
|
+
--measure <dir> Mide lo que recupera realmente cada regla sobre
|
|
129
|
+
los prompts de ese directorio, en vez de listar
|
|
130
|
+
qué hace cada una.
|
|
131
|
+
--level <safe|aggressive> Qué reglas medir. Por defecto: safe.
|
|
132
|
+
--json La medición como datos.
|
|
133
|
+
|
|
134
|
+
Sin --measure lista las reglas y para qué sirve cada una. Con él, el
|
|
135
|
+
optimizador se ejecuta una vez con cada regla sola y una vez con cada regla
|
|
136
|
+
quitada, y se imprimen ambas cifras: donde dos reglas encuentran los mismos
|
|
137
|
+
tokens divergen, y cualquiera de las dos por separado se equivocaría en una
|
|
138
|
+
dirección distinta.
|
|
139
|
+
|
|
140
|
+
El suelo se separa. El optimizador normaliza espacios haya reglas activas o
|
|
141
|
+
no, y atribuir eso a las reglas es como sobrevive un porcentaje de titular en
|
|
142
|
+
un corpus donde las reglas no recuperan nada.
|
|
143
|
+
|
|
144
|
+
"Inerte" se dice siempre sobre el corpus. Una regla que no encuentra nada en
|
|
145
|
+
estos ficheros no ha demostrado no encontrar nada en ninguna parte.
|
|
146
|
+
|
|
107
147
|
${bold('OPCIONES DE feedback')}
|
|
108
148
|
(ninguna)
|
|
109
149
|
|
|
@@ -1468,6 +1508,32 @@ ${bold('EJEMPLOS')}
|
|
|
1468
1508
|
},
|
|
1469
1509
|
},
|
|
1470
1510
|
|
|
1511
|
+
pulse: {
|
|
1512
|
+
heading: () => '¿Corrieron las cosas que tienen que correr?',
|
|
1513
|
+
kind: (kind) => {
|
|
1514
|
+
switch (kind) {
|
|
1515
|
+
case 'watch-cycle':
|
|
1516
|
+
return 'último ciclo de watch';
|
|
1517
|
+
case 'store-pull':
|
|
1518
|
+
return 'última descarga al almacén';
|
|
1519
|
+
case 'store-coverage':
|
|
1520
|
+
return 'las mediciones llegan hasta';
|
|
1521
|
+
default:
|
|
1522
|
+
return kind;
|
|
1523
|
+
}
|
|
1524
|
+
},
|
|
1525
|
+
neverRun: (name) =>
|
|
1526
|
+
`${name}: nunca aquí. No va tarde — no hay cadencia contra la que ir tarde.`,
|
|
1527
|
+
age: (name, when, hours) => `${name}: ${when}Z, hace ${plural(hours, 'hora')}`,
|
|
1528
|
+
noThreshold: () =>
|
|
1529
|
+
'No se juzgó nada. Pasa --max-stale-hours <n> para que algo que dejó de correr sea un código de salida en rojo; cuánto es demasiado es una política, y esta herramienta no escribe la tuya.',
|
|
1530
|
+
within: (hours) => `Todo lo que corre aquí corrió dentro de ${plural(hours, 'hora')}.`,
|
|
1531
|
+
stale: (hours) =>
|
|
1532
|
+
`Algo que corre aquí lleva más de ${plural(hours, 'hora')} sin correr. El silencio de un trabajo programado y el de un trabajo sin nada que decir son iguales; esto dice cuál es.`,
|
|
1533
|
+
notAService: () =>
|
|
1534
|
+
'Este comando no ejecuta nada y no aloja nada. Algo tiene que darse cuenta, y ese algo es tu CI: un paso que ejecute esto con el horario que ya tienes convierte un cron muerto en una build roja, sin que Trazum guarde las métricas de nadie. Hasta dónde llegan las mediciones se informa y nunca se juzga — eso es un proveedor informando a su ritmo, no un trabajo que falló.',
|
|
1535
|
+
},
|
|
1536
|
+
|
|
1471
1537
|
where: {
|
|
1472
1538
|
hostHeading: () => 'Ejecutándose dentro de',
|
|
1473
1539
|
subscription: (host) =>
|
|
@@ -1574,6 +1640,24 @@ ${bold('EJEMPLOS')}
|
|
|
1574
1640
|
rules: {
|
|
1575
1641
|
title: () => 'Reglas disponibles',
|
|
1576
1642
|
disableHint: () => ' Desactiva las que no quieras con --disable id1,id2',
|
|
1643
|
+
measureHeading: (root, prompts, level) =>
|
|
1644
|
+
`Qué recupera cada regla en ${root} — ${plural(prompts, 'prompt')}, nivel ${level}`,
|
|
1645
|
+
measureTotals: (before, saved, floor) =>
|
|
1646
|
+
`${before} tokens antes. Las reglas recuperan ${saved}; la normalización recupera ${floor} haya reglas activas o no, y eso no es obra de las reglas.`,
|
|
1647
|
+
measureRow: (id, marginal, alone, prompts) =>
|
|
1648
|
+
`${id.padEnd(22)} se pierde si se quita ${marginal.padStart(6)} sola ${alone.padStart(6)} ${plural(prompts, 'prompt')}`,
|
|
1649
|
+
measureOverlap: (sumOfAlone, saved) =>
|
|
1650
|
+
`Las reglas recuperan ${sumOfAlone} entre todas de una en una y ${saved} juntas. La diferencia es el solapamiento — dos reglas encontrando los mismos tokens — y se declara en vez de resolverse en un total, porque una sola cifra aquí es el único número que no puede ser cierto.`,
|
|
1651
|
+
measureRedundant: (ids) =>
|
|
1652
|
+
`Cada token que encuentran estas lo encuentra algo más, aquí: ${ids}. Un solapamiento, no un defecto — y no una razón para borrar una sin decidir cuál conviene conservar.`,
|
|
1653
|
+
measureFiredWithoutSaving: (ids) =>
|
|
1654
|
+
`Estas cambiaron el prompt y no recuperaron ningún token: ${ids}. No es lo mismo que inerte — sí se ejercitaron, y están alterando la instrucción de alguien sin beneficio medido.`,
|
|
1655
|
+
measureInert: (ids) =>
|
|
1656
|
+
`Estas no cambiaron nada en este corpus: ${ids}. Es un hecho sobre estos ficheros, no sobre las reglas: una regla que no encuentra nada aquí no ha demostrado no encontrar nada en ninguna parte.`,
|
|
1657
|
+
measureBand: (source) =>
|
|
1658
|
+
source === 'heuristic'
|
|
1659
|
+
? 'Contado con el estimador incorporado, así que toda cifra de arriba lleva su banda documentada. Una regla cuyo rendimiento son unos pocos tokens está dentro del ruido.'
|
|
1660
|
+
: 'Contado con un tokenizador externo.',
|
|
1577
1661
|
},
|
|
1578
1662
|
|
|
1579
1663
|
|
|
@@ -2434,6 +2518,13 @@ ${bold('EJEMPLOS')}
|
|
|
2434
2518
|
},
|
|
2435
2519
|
storeNoLabels: () =>
|
|
2436
2520
|
'Esta serie viene del almacén, y una API de uso agrupa por modelo y workspace, no por carga de trabajo — así que aquí no hay serie por label en absoluto. Ausente, no vacía: nada de lo de arriba dice que una carga se moviera o dejara de moverse.',
|
|
2521
|
+
unmeasured: (days, from, to, after, before) =>
|
|
2522
|
+
`Nada cubre del ${from} al ${to} — ${plural(days, 'día')} entre ${after} y ${before}.`,
|
|
2523
|
+
unmeasuredTotal: (days) =>
|
|
2524
|
+
`${plural(days, 'día')} de este tramo no los cubre ningún informe. Una serie con un agujero y una serie más corta se leen igual; esto dice cuál es.`,
|
|
2525
|
+
overlap: (a, b, days) =>
|
|
2526
|
+
`${a} y ${b} cubren ambos ${plural(days, 'día')}. Nombrado, nunca fusionado — cuál es la mejor medición no se sabe desde aquí, y sumar los dos totales cuenta esos días dos veces.`,
|
|
2527
|
+
runHole: (days) => ` — ${plural(days, 'día')} de esta racha no los cubre ningún informe`,
|
|
2437
2528
|
undated: (name) => `${name} no lleva período, así que no está en ninguna línea de tiempo de arriba — nombrado, nunca absorbido en silencio.`,
|
|
2438
2529
|
unrecognized: (name) => `${name} no es ni un informe guardado ni un plan guardado, así que no está en ninguna serie de arriba.`,
|
|
2439
2530
|
footer: () =>
|
package/src/i18n/types.ts
CHANGED
|
@@ -461,6 +461,17 @@ export interface CliMessages {
|
|
|
461
461
|
caveat(code: string): string;
|
|
462
462
|
};
|
|
463
463
|
|
|
464
|
+
pulse: {
|
|
465
|
+
heading(): string;
|
|
466
|
+
kind(kind: string): string;
|
|
467
|
+
neverRun(name: string): string;
|
|
468
|
+
age(name: string, when: string, hours: number): string;
|
|
469
|
+
noThreshold(): string;
|
|
470
|
+
within(hours: number): string;
|
|
471
|
+
stale(hours: number): string;
|
|
472
|
+
notAService(): string;
|
|
473
|
+
};
|
|
474
|
+
|
|
464
475
|
where: {
|
|
465
476
|
hostHeading(): string;
|
|
466
477
|
subscription(host: string): string;
|
|
@@ -505,6 +516,14 @@ export interface CliMessages {
|
|
|
505
516
|
rules: {
|
|
506
517
|
title(): string;
|
|
507
518
|
disableHint(): string;
|
|
519
|
+
measureHeading(root: string, prompts: number, level: string): string;
|
|
520
|
+
measureTotals(before: string, saved: string, floor: string): string;
|
|
521
|
+
measureRow(id: string, marginal: string, alone: string, prompts: number): string;
|
|
522
|
+
measureOverlap(sumOfAlone: string, saved: string): string;
|
|
523
|
+
measureRedundant(ids: string): string;
|
|
524
|
+
measureInert(ids: string): string;
|
|
525
|
+
measureFiredWithoutSaving(ids: string): string;
|
|
526
|
+
measureBand(source: string): string;
|
|
508
527
|
};
|
|
509
528
|
|
|
510
529
|
/** `trazum rank` — which prompts to fix first, and why. */
|
|
@@ -1509,6 +1528,10 @@ export interface CliMessages {
|
|
|
1509
1528
|
/** A store source carries no labels, so the label series is absent, not empty. */
|
|
1510
1529
|
storeNoLabels(): string;
|
|
1511
1530
|
undated(name: string): string;
|
|
1531
|
+
unmeasured(days: number, from: string, to: string, after: string, before: string): string;
|
|
1532
|
+
unmeasuredTotal(days: number): string;
|
|
1533
|
+
overlap(a: string, b: string, days: number): string;
|
|
1534
|
+
runHole(days: number): string;
|
|
1512
1535
|
unrecognized(name: string): string;
|
|
1513
1536
|
footer(): string;
|
|
1514
1537
|
/** The waiver record — what this team has been living with, and for how long. */
|
package/src/index.ts
CHANGED
|
@@ -70,6 +70,8 @@ import {
|
|
|
70
70
|
assignSources,
|
|
71
71
|
fleetRollup,
|
|
72
72
|
rollUp,
|
|
73
|
+
heartbeats,
|
|
74
|
+
ruleYield,
|
|
73
75
|
labelCoverage,
|
|
74
76
|
measuredUsage,
|
|
75
77
|
gateMargin,
|
|
@@ -297,6 +299,8 @@ const VALUE_FLAGS = new Set([
|
|
|
297
299
|
'max-usd',
|
|
298
300
|
'max-growth-usd',
|
|
299
301
|
'max-cache-loss-usd',
|
|
302
|
+
'max-stale-hours',
|
|
303
|
+
'measure',
|
|
300
304
|
'max-day-usd',
|
|
301
305
|
'max-session-usd',
|
|
302
306
|
'csv-out',
|
|
@@ -595,6 +599,7 @@ const COMMAND_FLAGS: Record<string, string[]> = {
|
|
|
595
599
|
init: ['dry-run', 'yes', 'json', 'pricing', 'pricing-live'],
|
|
596
600
|
conform: ['contract', 'json'],
|
|
597
601
|
rollup: ['json'],
|
|
602
|
+
pulse: ['json', 'max-stale-hours'],
|
|
598
603
|
feedback: [],
|
|
599
604
|
gateway: ['on-cannot-tell', 'port', 'socket', 'pricing', 'pricing-live'],
|
|
600
605
|
ladder: ['pricing', 'pricing-live', 'since', 'until', 'label'],
|
|
@@ -605,7 +610,7 @@ const COMMAND_FLAGS: Record<string, string[]> = {
|
|
|
605
610
|
commitment: ['floor', 'discount', 'months', 'pricing', 'pricing-live'],
|
|
606
611
|
report: ['year', 'json', 'pricing', 'pricing-live'],
|
|
607
612
|
where: [],
|
|
608
|
-
rules: [],
|
|
613
|
+
rules: ['measure', 'level', 'json'],
|
|
609
614
|
blame: ['limit', 'model', 'calls', 'output-tokens', 'batch', 'prompt', 'markdown-out'],
|
|
610
615
|
doctor: ['level', 'model', 'calls', 'output-tokens', 'batch', 'disable', 'prompt', 'otlp-out'],
|
|
611
616
|
};
|
|
@@ -2263,6 +2268,102 @@ async function commandRollup(args: Args, t: CliMessages): Promise<void> {
|
|
|
2263
2268
|
console.log();
|
|
2264
2269
|
}
|
|
2265
2270
|
|
|
2271
|
+
/**
|
|
2272
|
+
* `trazum pulse [--max-stale-hours <n>] [--json]` — did the things that are
|
|
2273
|
+
* supposed to run, run?
|
|
2274
|
+
*
|
|
2275
|
+
* `watch --once` is built for a scheduler: a cron entry is the whole daemon,
|
|
2276
|
+
* and its state file records each cycle precisely so a restart is honest about
|
|
2277
|
+
* the stretch it did not watch. That file is read by exactly one thing, and
|
|
2278
|
+
* that thing is the next cycle.
|
|
2279
|
+
*
|
|
2280
|
+
* **So nothing could tell you the watcher had stopped, because the thing that
|
|
2281
|
+
* would tell you was the thing that stopped.** A dead cron produces silence,
|
|
2282
|
+
* and a watcher with nothing to report produces silence too. This command is
|
|
2283
|
+
* the outside view: the age of the last watch cycle, the age of the last pull
|
|
2284
|
+
* into the store, and how far the stored measurements reach.
|
|
2285
|
+
*
|
|
2286
|
+
* **It is not a service and does not run itself.** Something has to notice,
|
|
2287
|
+
* and this product's answer is that the something is already in your CI: a
|
|
2288
|
+
* step that runs this with `--max-stale-hours` turns a dead cron into a red
|
|
2289
|
+
* build, on the schedule your CI already has, without Trazum holding anybody's
|
|
2290
|
+
* metrics. Where that answer runs out is written down in docs/ rather than
|
|
2291
|
+
* left to be discovered.
|
|
2292
|
+
*
|
|
2293
|
+
* Exits 1 only when a **run** that has happened before is past a **stated**
|
|
2294
|
+
* threshold. Never on a first run that never happened, and never on how far
|
|
2295
|
+
* the measurements reach — that is a provider reporting on its own schedule,
|
|
2296
|
+
* not a job that failed.
|
|
2297
|
+
*/
|
|
2298
|
+
async function commandPulse(args: Args, t: CliMessages): Promise<void> {
|
|
2299
|
+
const root = process.cwd();
|
|
2300
|
+
const maxStaleHours = numberFlag(args, 'max-stale-hours', Number.NaN, t);
|
|
2301
|
+
|
|
2302
|
+
const state = await readWatchState(root);
|
|
2303
|
+
const { resolved } = await readStore(root);
|
|
2304
|
+
|
|
2305
|
+
/**
|
|
2306
|
+
* The newest pull and the furthest reach, kept apart.
|
|
2307
|
+
*
|
|
2308
|
+
* One says a job ran; the other says how far its answers go. A store pulled
|
|
2309
|
+
* ten minutes ago whose newest record stops two days back is a healthy cron
|
|
2310
|
+
* in front of a provider that reports late, and a single figure would call
|
|
2311
|
+
* that either a failure or a success depending which half it took.
|
|
2312
|
+
*/
|
|
2313
|
+
let storePulledMs: number | null = null;
|
|
2314
|
+
let storeCoveredToMs: number | null = null;
|
|
2315
|
+
for (const record of resolved.records) {
|
|
2316
|
+
if (storePulledMs === null || record.pulledAtMs > storePulledMs) storePulledMs = record.pulledAtMs;
|
|
2317
|
+
if (storeCoveredToMs === null || record.toMs > storeCoveredToMs) storeCoveredToMs = record.toMs;
|
|
2318
|
+
}
|
|
2319
|
+
|
|
2320
|
+
const report = heartbeats(
|
|
2321
|
+
{ watchCycleMs: state?.lastCycleMs ?? null, storePulledMs, storeCoveredToMs },
|
|
2322
|
+
Number.isFinite(maxStaleHours)
|
|
2323
|
+
? { nowMs: Date.now(), maxStaleHours }
|
|
2324
|
+
: { nowMs: Date.now() },
|
|
2325
|
+
);
|
|
2326
|
+
|
|
2327
|
+
if (boolFlag(args, 'json')) {
|
|
2328
|
+
console.log(JSON.stringify(report, null, 2));
|
|
2329
|
+
if (report.stale) process.exitCode = 1;
|
|
2330
|
+
return;
|
|
2331
|
+
}
|
|
2332
|
+
|
|
2333
|
+
const when = (ms: number): string => new Date(ms).toISOString().replace('T', ' ').slice(0, 16);
|
|
2334
|
+
|
|
2335
|
+
console.log();
|
|
2336
|
+
console.log(c.bold(t.pulse.heading()));
|
|
2337
|
+
for (const heartbeat of report.beats) {
|
|
2338
|
+
const name = t.pulse.kind(heartbeat.kind);
|
|
2339
|
+
if (heartbeat.lastMs === null) {
|
|
2340
|
+
// Never run is not late. It is a thing nobody has started here, and a
|
|
2341
|
+
// line that shouted about it would be this tool nagging.
|
|
2342
|
+
console.log(` ${c.dim(t.pulse.neverRun(name))}`);
|
|
2343
|
+
continue;
|
|
2344
|
+
}
|
|
2345
|
+
const line = t.pulse.age(name, when(heartbeat.lastMs), heartbeat.ageHours ?? 0);
|
|
2346
|
+
if (heartbeat.verdict === 'stale') console.log(` ${c.red(`✗ ${line}`)}`);
|
|
2347
|
+
else if (heartbeat.verdict === 'within') console.log(` ${c.green(`✓ ${line}`)}`);
|
|
2348
|
+
else console.log(` ${line}`);
|
|
2349
|
+
}
|
|
2350
|
+
|
|
2351
|
+
console.log();
|
|
2352
|
+
if (report.maxStaleHours === null) {
|
|
2353
|
+
// Nothing was judged, and the exit code says so silently. Said out loud,
|
|
2354
|
+
// because a screen of green ticks with no threshold behind them is the
|
|
2355
|
+
// shape somebody reads as "checked".
|
|
2356
|
+
console.log(` ${c.dim(wrap(t.pulse.noThreshold(), 74, ' '))}`);
|
|
2357
|
+
} else if (report.stale) {
|
|
2358
|
+
process.exitCode = 1;
|
|
2359
|
+
console.log(` ${c.red(wrap(t.pulse.stale(report.maxStaleHours), 74, ' '))}`);
|
|
2360
|
+
} else {
|
|
2361
|
+
console.log(` ${c.dim(wrap(t.pulse.within(report.maxStaleHours), 74, ' '))}`);
|
|
2362
|
+
}
|
|
2363
|
+
console.log(` ${c.dim(wrap(t.pulse.notAService(), 74, ' '))}`);
|
|
2364
|
+
console.log();
|
|
2365
|
+
}
|
|
2366
|
+
|
|
2266
2367
|
/**
|
|
2267
2368
|
* `trazum feedback` — where to say it, and what to say.
|
|
2268
2369
|
*
|
|
@@ -3527,6 +3628,104 @@ function commandRules(t: CliMessages, locale: Locale): void {
|
|
|
3527
3628
|
}
|
|
3528
3629
|
}
|
|
3529
3630
|
|
|
3631
|
+
/**
|
|
3632
|
+
* `trazum rules --measure <dir> [--level <safe|aggressive>] [--json]` — and
|
|
3633
|
+
* what does each one actually recover here?
|
|
3634
|
+
*
|
|
3635
|
+
* The README says plainly that the deterministic rules recover about one per
|
|
3636
|
+
* cent, and that is the fair complaint about this tool. It is also an
|
|
3637
|
+
* **aggregate**, and an aggregate is where a distribution goes to hide: the
|
|
3638
|
+
* same one per cent is consistent with every rule pulling its weight and with
|
|
3639
|
+
* two rules doing all of it beside five that have never changed a byte.
|
|
3640
|
+
*
|
|
3641
|
+
* This measures which, over prompts the reader actually has, by running the
|
|
3642
|
+
* optimiser with each rule alone and then with each rule removed. **Both
|
|
3643
|
+
* figures are printed and neither is reconciled into the other**, because
|
|
3644
|
+
* where two rules find the same tokens they diverge, and either one on its own
|
|
3645
|
+
* would be wrong in a different direction.
|
|
3646
|
+
*
|
|
3647
|
+
* **The floor is separated out.** The optimiser normalises whitespace whether
|
|
3648
|
+
* or not any rule is enabled, and crediting that to the rules is how a
|
|
3649
|
+
* headline percentage survives on a corpus where the rules recover nothing.
|
|
3650
|
+
* The first version of the module did exactly that and reported every rule as
|
|
3651
|
+
* redundant.
|
|
3652
|
+
*
|
|
3653
|
+
* "Inert" is always said **about the corpus**. A rule that finds nothing in
|
|
3654
|
+
* these files has not been shown to find nothing anywhere, and the difference
|
|
3655
|
+
* is the whole distance between "delete this rule" and "measure it on
|
|
3656
|
+
* something else".
|
|
3657
|
+
*/
|
|
3658
|
+
async function commandRulesMeasure(
|
|
3659
|
+
args: Args,
|
|
3660
|
+
config: TrazumConfig,
|
|
3661
|
+
pricing: PricingCatalogue,
|
|
3662
|
+
t: CliMessages,
|
|
3663
|
+
): Promise<void> {
|
|
3664
|
+
const root = stringFlag(args, 'measure')!;
|
|
3665
|
+
const level = (stringFlag(args, 'level') ?? 'safe') as RuleLevel;
|
|
3666
|
+
if (level !== 'safe' && level !== 'aggressive') throw new Error(t.errors.badLevel(level));
|
|
3667
|
+
|
|
3668
|
+
const extensions = config.extensions ?? [...DEFAULT_EXTENSIONS, ...SOURCE_EXTENSIONS];
|
|
3669
|
+
const { files, truncated } = await walkPrompts(root, { extensions });
|
|
3670
|
+
if (files.length === 0) throw new Error(t.errors.noPromptsFound(root, extensions.join(' ')));
|
|
3671
|
+
|
|
3672
|
+
const prompts: string[] = [];
|
|
3673
|
+
for (const file of files) {
|
|
3674
|
+
// `walkPrompts` returns names relative to the root it was given, so the
|
|
3675
|
+
// root has to go back on. Reading them bare only worked from a run whose
|
|
3676
|
+
// cwd happened to be the root, which is the one case a first probe uses.
|
|
3677
|
+
const raw = await readFile(join(root, file), 'utf8');
|
|
3678
|
+
// A marked source file contributes its marked prompts; anything else
|
|
3679
|
+
// contributes itself. The same rule `check` and `doctor` already follow,
|
|
3680
|
+
// so the three commands measure the same text.
|
|
3681
|
+
if (hasMarker(raw)) prompts.push(...extractPrompts(raw).prompts.map((p) => p.text));
|
|
3682
|
+
else prompts.push(raw);
|
|
3683
|
+
}
|
|
3684
|
+
|
|
3685
|
+
const report = ruleYield(prompts, RULES.map((rule) => rule.id), { level, pricing });
|
|
3686
|
+
|
|
3687
|
+
if (boolFlag(args, 'json')) {
|
|
3688
|
+
console.log(JSON.stringify({ ...report, root, files: files.length, truncated }, null, 2));
|
|
3689
|
+
return;
|
|
3690
|
+
}
|
|
3691
|
+
|
|
3692
|
+
const n = (value: number): string => value.toLocaleString(t.numberLocale);
|
|
3693
|
+
|
|
3694
|
+
console.log();
|
|
3695
|
+
console.log(c.bold(t.rules.measureHeading(root, report.prompts, level)));
|
|
3696
|
+
console.log(` ${c.dim(t.rules.measureTotals(n(report.tokensBefore), n(report.tokensSaved), n(report.floor)))}`);
|
|
3697
|
+
console.log();
|
|
3698
|
+
|
|
3699
|
+
for (const rule of report.rules) {
|
|
3700
|
+
const line = t.rules.measureRow(rule.id, n(rule.marginal), n(rule.alone), rule.prompts);
|
|
3701
|
+
if (rule.marginal > 0) console.log(` ${line}`);
|
|
3702
|
+
else console.log(` ${c.dim(line)}`);
|
|
3703
|
+
}
|
|
3704
|
+
|
|
3705
|
+
console.log();
|
|
3706
|
+
// The overlap, stated as the gap rather than resolved into a total: a single
|
|
3707
|
+
// figure here is the one number that cannot be true.
|
|
3708
|
+
if (report.sumOfAlone !== report.tokensSaved) {
|
|
3709
|
+
console.log(` ${c.dim(wrap(t.rules.measureOverlap(n(report.sumOfAlone), n(report.tokensSaved)), 74, ' '))}`);
|
|
3710
|
+
}
|
|
3711
|
+
if (report.redundantHere.length > 0) {
|
|
3712
|
+
console.log(` ${c.dim(wrap(t.rules.measureRedundant(report.redundantHere.join(', ')), 74, ' '))}`);
|
|
3713
|
+
}
|
|
3714
|
+
// Fired-and-saved-nothing before never-fired: the first is a finding about
|
|
3715
|
+
// the rule, the second only about the corpus, and a reader who meets them the
|
|
3716
|
+
// other way round reads both as the same shrug.
|
|
3717
|
+
if (report.firedWithoutSavingHere.length > 0) {
|
|
3718
|
+
console.log(
|
|
3719
|
+
` ${c.yellow(wrap(t.rules.measureFiredWithoutSaving(report.firedWithoutSavingHere.join(', ')), 74, ' '))}`,
|
|
3720
|
+
);
|
|
3721
|
+
}
|
|
3722
|
+
if (report.inertHere.length > 0) {
|
|
3723
|
+
console.log(` ${c.dim(wrap(t.rules.measureInert(report.inertHere.join(', ')), 74, ' '))}`);
|
|
3724
|
+
}
|
|
3725
|
+
console.log(` ${c.dim(wrap(t.rules.measureBand(report.tokenSource), 74, ' '))}`);
|
|
3726
|
+
console.log();
|
|
3727
|
+
}
|
|
3728
|
+
|
|
3530
3729
|
async function readInput(source: string | undefined, t: CliMessages): Promise<string> {
|
|
3531
3730
|
if (!source) {
|
|
3532
3731
|
throw new Error(t.errors.missingInputFile());
|
|
@@ -5197,6 +5396,16 @@ async function commandHistory(
|
|
|
5197
5396
|
return t.history.runCache(n(run.periods), run.sinceName, pct(run.from), pct(run.to));
|
|
5198
5397
|
};
|
|
5199
5398
|
|
|
5399
|
+
/**
|
|
5400
|
+
* The hole under a run, on the run's own line.
|
|
5401
|
+
*
|
|
5402
|
+
* A run is consecutive *reports*, and reads as consecutive *time* unless
|
|
5403
|
+
* told otherwise. A caveat one section away is a caveat that arrives after
|
|
5404
|
+
* the reader has already formed the sentence.
|
|
5405
|
+
*/
|
|
5406
|
+
const runHole = (run: HistoryRun): string =>
|
|
5407
|
+
run.unmeasuredDays > 0 ? t.history.runHole(run.unmeasuredDays) : '';
|
|
5408
|
+
|
|
5200
5409
|
const lines = (md: boolean): string[] => {
|
|
5201
5410
|
const out: string[] = [];
|
|
5202
5411
|
const first = history.periods[0]!;
|
|
@@ -5214,7 +5423,8 @@ async function commandHistory(
|
|
|
5214
5423
|
}
|
|
5215
5424
|
if (history.runs.length > 0) out.push('');
|
|
5216
5425
|
for (const run of history.runs) {
|
|
5217
|
-
|
|
5426
|
+
const line = `${runLine(run)}${runHole(run)}`;
|
|
5427
|
+
out.push(md ? `- ${line}` : ` ! ${line}`);
|
|
5218
5428
|
}
|
|
5219
5429
|
if (history.repeatedPlanActions.length > 0) out.push('');
|
|
5220
5430
|
for (const repeat of history.repeatedPlanActions) {
|
|
@@ -5229,6 +5439,34 @@ async function commandHistory(
|
|
|
5229
5439
|
);
|
|
5230
5440
|
out.push(md ? `- ${row}` : ` ! ${row}`);
|
|
5231
5441
|
}
|
|
5442
|
+
/**
|
|
5443
|
+
* The holes in the timeline, before the per-file notes.
|
|
5444
|
+
*
|
|
5445
|
+
* A stretch nobody measured is a fact about the whole series rather than
|
|
5446
|
+
* about one file, and it is the finding this command could not make until
|
|
5447
|
+
* now: a scheduled run that stopped three weeks ago produces a series that
|
|
5448
|
+
* looks exactly like a shorter one.
|
|
5449
|
+
*/
|
|
5450
|
+
if (history.unmeasured.length > 0) {
|
|
5451
|
+
out.push('');
|
|
5452
|
+
const total = history.unmeasured.reduce((sum, hole) => sum + hole.days, 0);
|
|
5453
|
+
const summary = t.history.unmeasuredTotal(total);
|
|
5454
|
+
out.push(md ? `- **${summary}**` : ` ! ${summary}`);
|
|
5455
|
+
for (const hole of history.unmeasured) {
|
|
5456
|
+
const row = t.history.unmeasured(
|
|
5457
|
+
hole.days,
|
|
5458
|
+
day(hole.fromMs),
|
|
5459
|
+
day(hole.toMs),
|
|
5460
|
+
hole.afterName,
|
|
5461
|
+
hole.beforeName,
|
|
5462
|
+
);
|
|
5463
|
+
out.push(md ? `- ${row}` : ` ${row}`);
|
|
5464
|
+
}
|
|
5465
|
+
}
|
|
5466
|
+
for (const overlap of history.overlappingReports) {
|
|
5467
|
+
const row = t.history.overlap(overlap.a, overlap.b, overlap.days);
|
|
5468
|
+
out.push(md ? `- ${row}` : ` ! ${row}`);
|
|
5469
|
+
}
|
|
5232
5470
|
for (const name of history.undatedReports) {
|
|
5233
5471
|
out.push(md ? `- ${t.history.undated(name)}` : ` ${t.history.undated(name)}`);
|
|
5234
5472
|
}
|
|
@@ -9716,6 +9954,9 @@ async function main(): Promise<void> {
|
|
|
9716
9954
|
case 'rollup':
|
|
9717
9955
|
await commandRollup(args, t);
|
|
9718
9956
|
break;
|
|
9957
|
+
case 'pulse':
|
|
9958
|
+
await commandPulse(args, t);
|
|
9959
|
+
break;
|
|
9719
9960
|
case 'init':
|
|
9720
9961
|
await commandInit(args, config, pricing, t);
|
|
9721
9962
|
break;
|
|
@@ -9723,7 +9964,8 @@ async function main(): Promise<void> {
|
|
|
9723
9964
|
await commandWhere(args, config, pricing, t);
|
|
9724
9965
|
break;
|
|
9725
9966
|
case 'rules':
|
|
9726
|
-
|
|
9967
|
+
if (stringFlag(args, 'measure') !== undefined) await commandRulesMeasure(args, config, pricing, t);
|
|
9968
|
+
else commandRules(t, locale);
|
|
9727
9969
|
break;
|
|
9728
9970
|
case 'doctor':
|
|
9729
9971
|
await commandDoctor(args, config, pricing, t, locale);
|