@trazum/cli 1.62.0 → 1.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/git.d.ts +11 -0
- package/dist/git.d.ts.map +1 -1
- package/dist/git.js +34 -3
- package/dist/git.js.map +1 -1
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +50 -0
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/es.d.ts.map +1 -1
- package/dist/i18n/es.js +50 -0
- package/dist/i18n/es.js.map +1 -1
- package/dist/i18n/types.d.ts +23 -0
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/index.js +417 -22
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
- package/src/git.ts +35 -3
- package/src/i18n/en.ts +60 -0
- package/src/i18n/es.ts +60 -0
- package/src/i18n/types.ts +24 -0
- package/src/index.ts +475 -20
package/src/index.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { readFileSync } from 'node:fs';
|
|
3
|
-
import { open, readdir, readFile, stat, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { readdirSync, readFileSync } from 'node:fs';
|
|
3
|
+
import { mkdir, mkdtemp, open, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
|
4
|
+
import { cpus, tmpdir } from 'node:os';
|
|
4
5
|
import { createInterface } from 'node:readline/promises';
|
|
5
6
|
import { dirname, join, resolve as resolvePath } from 'node:path';
|
|
6
7
|
import { fileURLToPath } from 'node:url';
|
|
@@ -90,6 +91,7 @@ import {
|
|
|
90
91
|
listModels,
|
|
91
92
|
LOCALES,
|
|
92
93
|
MAX_BASELINE_BYTES,
|
|
94
|
+
MAX_INPUT_CHARS,
|
|
93
95
|
moneyIsComparable,
|
|
94
96
|
mostSpecificMatch,
|
|
95
97
|
assemble,
|
|
@@ -206,6 +208,7 @@ import {
|
|
|
206
208
|
pathInRepository,
|
|
207
209
|
repositoryRoot,
|
|
208
210
|
revisionsFor,
|
|
211
|
+
runSelf,
|
|
209
212
|
} from './git.js';
|
|
210
213
|
import type { Revision } from './git.js';
|
|
211
214
|
import { fetchProviderUsage, findCredential } from './connect.js';
|
|
@@ -312,6 +315,10 @@ const VALUE_FLAGS = new Set([
|
|
|
312
315
|
'max-cache-loss-usd',
|
|
313
316
|
'max-stale-hours',
|
|
314
317
|
'measure',
|
|
318
|
+
'workload',
|
|
319
|
+
'record',
|
|
320
|
+
'max-ratio',
|
|
321
|
+
'max-input',
|
|
315
322
|
'max-day-usd',
|
|
316
323
|
'max-session-usd',
|
|
317
324
|
'csv-out',
|
|
@@ -600,7 +607,7 @@ function disabledRules(args: Args, config: TrazumConfig): RuleId[] | undefined {
|
|
|
600
607
|
* a threshold is set — `--max-growh 5` would have been ignored and the build
|
|
601
608
|
* gone green. Silence is the wrong answer for a typo.
|
|
602
609
|
*/
|
|
603
|
-
const GLOBAL_FLAGS = ['help', 'h', 'version', 'v', 'locale', 'json', 'config', 'pricing', 'pricing-live'];
|
|
610
|
+
const GLOBAL_FLAGS = ['help', 'h', 'version', 'v', 'locale', 'json', 'config', 'pricing', 'pricing-live', 'max-input'];
|
|
604
611
|
const COMMAND_FLAGS: Record<string, string[]> = {
|
|
605
612
|
optimize: [
|
|
606
613
|
'level', 'model', 'calls', 'output-tokens', 'cache-hit-rate', 'batch',
|
|
@@ -628,6 +635,7 @@ const COMMAND_FLAGS: Record<string, string[]> = {
|
|
|
628
635
|
conform: ['contract', 'json'],
|
|
629
636
|
rollup: ['json'],
|
|
630
637
|
pulse: ['json', 'max-stale-hours'],
|
|
638
|
+
bench: ['workload', 'json', 'record', 'against', 'max-ratio'],
|
|
631
639
|
write: ['answers', 'json', 'out', 'o', 'calls', 'avg-output'],
|
|
632
640
|
feedback: [],
|
|
633
641
|
gateway: ['on-cannot-tell', 'port', 'socket', 'pricing', 'pricing-live'],
|
|
@@ -2054,7 +2062,8 @@ async function commandConform(args: Args, t: CliMessages): Promise<void> {
|
|
|
2054
2062
|
throw new Error(t.conform.badContract(named, CONTRACT_NAMES.join(', ')));
|
|
2055
2063
|
}
|
|
2056
2064
|
|
|
2057
|
-
|
|
2065
|
+
// A document or a log, not a prompt: deliberately uncapped (limit null).
|
|
2066
|
+
const text = target === '-' ? await readInput('-', t, null) : await readUsageLog(target, t);
|
|
2058
2067
|
const report = conform(text, named === undefined ? {} : { contract: named });
|
|
2059
2068
|
|
|
2060
2069
|
if (boolFlag(args, 'json')) {
|
|
@@ -2567,6 +2576,402 @@ async function commandPulse(args: Args, t: CliMessages): Promise<void> {
|
|
|
2567
2576
|
console.log();
|
|
2568
2577
|
}
|
|
2569
2578
|
|
|
2579
|
+
/**
|
|
2580
|
+
* `trazum bench` — measures this machine, honestly.
|
|
2581
|
+
*
|
|
2582
|
+
* The standard workloads, one shot each, wall time and peak RSS. **No
|
|
2583
|
+
* comparison and no judgement**: a number a person runs before and after a
|
|
2584
|
+
* change and reads side by side. The pathological cases were timed once, by
|
|
2585
|
+
* hand, during a stress session — 1MB of prose in about a second, a
|
|
2586
|
+
* 200,000-line log in about 1.3 — and nothing held them there. This command is
|
|
2587
|
+
* that measurement made repeatable; the gate that holds it belongs to a ratio
|
|
2588
|
+
* against in-process calibration, not to these wall clocks.
|
|
2589
|
+
*
|
|
2590
|
+
* Each workload runs in its own child process, because peak RSS is a fact
|
|
2591
|
+
* about a process: five workloads sharing one heap would each report the
|
|
2592
|
+
* high-water mark of whichever ran biggest before them. The child is this same
|
|
2593
|
+
* CLI with `--workload`, so what the bench measures is exactly what a user
|
|
2594
|
+
* runs.
|
|
2595
|
+
*
|
|
2596
|
+
* The workloads are **generated, deterministic and never committed** — the
|
|
2597
|
+
* fuzzer's own LCG, a fixed pricing date — so two runs on one machine differ
|
|
2598
|
+
* by the machine's weather, never by the input. Generation happens outside the
|
|
2599
|
+
* timed window: the bench times the product, not the bench.
|
|
2600
|
+
*
|
|
2601
|
+
* "Peak heap" in the plan is reported here as **peak RSS** —
|
|
2602
|
+
* `process.resourceUsage().maxRSS`, what the operating system actually billed
|
|
2603
|
+
* the process — because a true heap high-water mark is not observable from
|
|
2604
|
+
* inside a synchronous run without instrumentation that would itself move the
|
|
2605
|
+
* number. The field says what it is.
|
|
2606
|
+
*/
|
|
2607
|
+
|
|
2608
|
+
const BENCH_WORKLOADS = [
|
|
2609
|
+
'optimize-1mb-safe',
|
|
2610
|
+
'optimize-1mb-aggressive',
|
|
2611
|
+
'profile-200k',
|
|
2612
|
+
'walk-10k',
|
|
2613
|
+
'rollup-20k',
|
|
2614
|
+
] as const;
|
|
2615
|
+
type BenchWorkloadId = (typeof BENCH_WORKLOADS)[number];
|
|
2616
|
+
|
|
2617
|
+
interface BenchMeasurement {
|
|
2618
|
+
id: string;
|
|
2619
|
+
wallMs: number;
|
|
2620
|
+
/** The calibration loop's wall time, in this same process, right after the workload. */
|
|
2621
|
+
calibrationMs: number;
|
|
2622
|
+
/** wallMs over calibrationMs — the number a gate can hold, because the machine cancels out. */
|
|
2623
|
+
ratio: number;
|
|
2624
|
+
maxRssBytes: number;
|
|
2625
|
+
/** Input size in the unit the workload is named by; the others are null, never zero. */
|
|
2626
|
+
bytes: number | null;
|
|
2627
|
+
lines: number | null;
|
|
2628
|
+
files: number | null;
|
|
2629
|
+
}
|
|
2630
|
+
|
|
2631
|
+
/** The committed ratio baseline `--record` writes and `--against` reads. */
|
|
2632
|
+
interface BenchBaseline {
|
|
2633
|
+
schemaVersion: 1;
|
|
2634
|
+
workloads: { id: string; ratio: number }[];
|
|
2635
|
+
}
|
|
2636
|
+
|
|
2637
|
+
interface BenchDocument {
|
|
2638
|
+
schemaVersion: 1;
|
|
2639
|
+
node: string;
|
|
2640
|
+
platform: string;
|
|
2641
|
+
arch: string;
|
|
2642
|
+
cpus: number;
|
|
2643
|
+
cpuModel: string | null;
|
|
2644
|
+
workloads: BenchMeasurement[];
|
|
2645
|
+
}
|
|
2646
|
+
|
|
2647
|
+
/** The hostile-input suite's LCG: same seed, same workload, any machine. */
|
|
2648
|
+
function benchGenerator(seed: number): () => number {
|
|
2649
|
+
let state = seed;
|
|
2650
|
+
return () => (state = (state * 1103515245 + 12345) % 2147483648) / 2147483648;
|
|
2651
|
+
}
|
|
2652
|
+
|
|
2653
|
+
/** Prices resolve against a fixed date, so the workload cannot drift with the calendar. */
|
|
2654
|
+
const BENCH_DATE = new Date('2026-01-01T00:00:00Z');
|
|
2655
|
+
|
|
2656
|
+
/**
|
|
2657
|
+
* Enough integer work for the loop to take a stable fraction of a second on
|
|
2658
|
+
* anything that can run Node, and little enough that calibrating five
|
|
2659
|
+
* workloads is not itself the bench.
|
|
2660
|
+
*/
|
|
2661
|
+
const CALIBRATION_ITERATIONS = 1 << 24;
|
|
2662
|
+
|
|
2663
|
+
/**
|
|
2664
|
+
* The yardstick a ratio divides by: a fixed integer loop, timed in the same
|
|
2665
|
+
* process as the workload it calibrates.
|
|
2666
|
+
*
|
|
2667
|
+
* **Deliberately not the product's own code.** A calibration built on
|
|
2668
|
+
* `estimateTokens` would speed up when the tokenizer does, and every ratio in
|
|
2669
|
+
* every committed baseline would silently mean something new. This loop is
|
|
2670
|
+
* arithmetic that no release has a reason to touch — the same LCG the corpus
|
|
2671
|
+
* generators use, run hot — so a ratio moves only when the workload does.
|
|
2672
|
+
*
|
|
2673
|
+
* CI machines lie about wall time; they lie to the workload and the yardstick
|
|
2674
|
+
* by roughly the same amount, and the ratio is what is left when the lie
|
|
2675
|
+
* cancels out. That is the whole of chapter two's argument, and the reason a
|
|
2676
|
+
* gate on `ratio` can hold where a gate on `wallMs` would fail on weather.
|
|
2677
|
+
*/
|
|
2678
|
+
function benchCalibration(): number {
|
|
2679
|
+
const startedAt = performance.now();
|
|
2680
|
+
let state = 1;
|
|
2681
|
+
for (let i = 0; i < CALIBRATION_ITERATIONS; i += 1) {
|
|
2682
|
+
state = (state * 1103515245 + 12345) % 2147483648;
|
|
2683
|
+
}
|
|
2684
|
+
const elapsed = performance.now() - startedAt;
|
|
2685
|
+
// Reading the accumulator keeps the loop observable — a JIT that could prove
|
|
2686
|
+
// the result unused could also skip the work being timed.
|
|
2687
|
+
if (state < 0) throw new Error('unreachable: the LCG stays in [0, 2^31)');
|
|
2688
|
+
return elapsed;
|
|
2689
|
+
}
|
|
2690
|
+
|
|
2691
|
+
/**
|
|
2692
|
+
* Prose the rules have work in — verbose phrases, duplicate lines, emphasis,
|
|
2693
|
+
* spacing — with code fences and URLs mixed in so the segmenter and the masks
|
|
2694
|
+
* are part of what is timed, not skipped by an input too clean to exercise them.
|
|
2695
|
+
*/
|
|
2696
|
+
function benchPrompt(targetBytes: number): string {
|
|
2697
|
+
const rnd = benchGenerator(97);
|
|
2698
|
+
const sentences = [
|
|
2699
|
+
'Please kindly note that in order to get the best results you should always read the entire document.\n',
|
|
2700
|
+
'It is very very important that the answer is concise and complete.\n',
|
|
2701
|
+
'IMPORTANT: the sections below repeat their own headers.\n',
|
|
2702
|
+
'The quick summary follows the long summary, which follows the summary.\n',
|
|
2703
|
+
'See https://example.com/guide/section?step=3&mode=full for the walkthrough.\n',
|
|
2704
|
+
'```js\nconst total = items.reduce((sum, item) => sum + item.cost, 0);\n```\n',
|
|
2705
|
+
'- keep the tone neutral\n- keep the tone neutral\n- cite every claim\n',
|
|
2706
|
+
'In the event that the input is empty, respond with an empty list.\n',
|
|
2707
|
+
];
|
|
2708
|
+
let text = '';
|
|
2709
|
+
while (text.length < targetBytes) {
|
|
2710
|
+
text += sentences[Math.floor(rnd() * sentences.length)];
|
|
2711
|
+
if (rnd() < 0.15) text += '\n\n';
|
|
2712
|
+
}
|
|
2713
|
+
return text;
|
|
2714
|
+
}
|
|
2715
|
+
|
|
2716
|
+
/** A usage log in the documented shape: timestamps, labels, sessions, cache fields. */
|
|
2717
|
+
function benchLog(lines: number): string {
|
|
2718
|
+
const rnd = benchGenerator(53);
|
|
2719
|
+
const models = ['claude-opus-5', 'claude-sonnet-5', 'claude-haiku-4-5'];
|
|
2720
|
+
const labels = ['support-rag', 'classify', 'agent'];
|
|
2721
|
+
const startMs = Date.parse('2026-01-01T00:00:00Z');
|
|
2722
|
+
const out: string[] = [];
|
|
2723
|
+
for (let i = 0; i < lines; i += 1) {
|
|
2724
|
+
const record = {
|
|
2725
|
+
timestamp: new Date(startMs + i * 7000).toISOString(),
|
|
2726
|
+
model: models[Math.floor(rnd() * models.length)],
|
|
2727
|
+
label: labels[Math.floor(rnd() * labels.length)],
|
|
2728
|
+
session: `conv-${Math.floor(rnd() * 400)}`,
|
|
2729
|
+
stop_reason: rnd() < 0.05 ? 'max_tokens' : 'end_turn',
|
|
2730
|
+
usage: {
|
|
2731
|
+
input_tokens: 200 + Math.floor(rnd() * 4000),
|
|
2732
|
+
output_tokens: 50 + Math.floor(rnd() * 800),
|
|
2733
|
+
cache_read_input_tokens: rnd() < 0.6 ? Math.floor(rnd() * 16000) : 0,
|
|
2734
|
+
cache_creation_input_tokens: rnd() < 0.2 ? Math.floor(rnd() * 4000) : 0,
|
|
2735
|
+
},
|
|
2736
|
+
};
|
|
2737
|
+
out.push(JSON.stringify(record));
|
|
2738
|
+
}
|
|
2739
|
+
return out.join('\n');
|
|
2740
|
+
}
|
|
2741
|
+
|
|
2742
|
+
/** Runs one workload: generation outside the window, the product inside it. */
|
|
2743
|
+
async function benchRun(id: BenchWorkloadId): Promise<BenchMeasurement> {
|
|
2744
|
+
const measure = (work: () => unknown): number => {
|
|
2745
|
+
const startedAt = performance.now();
|
|
2746
|
+
work();
|
|
2747
|
+
return performance.now() - startedAt;
|
|
2748
|
+
};
|
|
2749
|
+
const done = (wallMs: number, sizes: Partial<Pick<BenchMeasurement, 'bytes' | 'lines' | 'files'>>): BenchMeasurement => {
|
|
2750
|
+
const calibrationMs = benchCalibration();
|
|
2751
|
+
return {
|
|
2752
|
+
id,
|
|
2753
|
+
wallMs,
|
|
2754
|
+
calibrationMs,
|
|
2755
|
+
ratio: wallMs / calibrationMs,
|
|
2756
|
+
// getrusage reports kilobytes; the field name promises bytes, so convert here.
|
|
2757
|
+
maxRssBytes: process.resourceUsage().maxRSS * 1024,
|
|
2758
|
+
bytes: sizes.bytes ?? null,
|
|
2759
|
+
lines: sizes.lines ?? null,
|
|
2760
|
+
files: sizes.files ?? null,
|
|
2761
|
+
};
|
|
2762
|
+
};
|
|
2763
|
+
|
|
2764
|
+
switch (id) {
|
|
2765
|
+
case 'optimize-1mb-safe':
|
|
2766
|
+
case 'optimize-1mb-aggressive': {
|
|
2767
|
+
const prompt = benchPrompt(1024 * 1024);
|
|
2768
|
+
const level = id === 'optimize-1mb-safe' ? 'safe' : 'aggressive';
|
|
2769
|
+
return done(measure(() => optimize(prompt, { level })), { bytes: Buffer.byteLength(prompt, 'utf8') });
|
|
2770
|
+
}
|
|
2771
|
+
case 'profile-200k': {
|
|
2772
|
+
const lines = 200_000;
|
|
2773
|
+
const text = benchLog(lines);
|
|
2774
|
+
return done(
|
|
2775
|
+
measure(() => profileUsage(text, { catalogue: BUNDLED_CATALOGUE, on: BENCH_DATE })),
|
|
2776
|
+
{ lines },
|
|
2777
|
+
);
|
|
2778
|
+
}
|
|
2779
|
+
case 'walk-10k': {
|
|
2780
|
+
/**
|
|
2781
|
+
* The discovery half of `rank` and directory `check`: find every file,
|
|
2782
|
+
* read it, estimate it. Generated under the system temp directory and
|
|
2783
|
+
* removed afterwards — ten thousand files are a workload, not a residue.
|
|
2784
|
+
*/
|
|
2785
|
+
const files = 10_000;
|
|
2786
|
+
const root = await mkdtemp(join(tmpdir(), 'trazum-bench-'));
|
|
2787
|
+
try {
|
|
2788
|
+
const rnd = benchGenerator(11);
|
|
2789
|
+
for (let dir = 0; dir < 100; dir += 1) {
|
|
2790
|
+
const dirPath = join(root, `d${dir}`);
|
|
2791
|
+
await mkdir(dirPath);
|
|
2792
|
+
for (let i = 0; i < files / 100; i += 1) {
|
|
2793
|
+
await writeFile(
|
|
2794
|
+
join(dirPath, `p${i}.txt`),
|
|
2795
|
+
`Summarise the report in ${3 + Math.floor(rnd() * 9)} bullet points.\nKeep every figure.\n`,
|
|
2796
|
+
);
|
|
2797
|
+
}
|
|
2798
|
+
}
|
|
2799
|
+
const startedAt = performance.now();
|
|
2800
|
+
let estimated = 0;
|
|
2801
|
+
const walk = (path: string): void => {
|
|
2802
|
+
for (const entry of readdirSync(path, { withFileTypes: true })) {
|
|
2803
|
+
const child = join(path, entry.name);
|
|
2804
|
+
if (entry.isDirectory()) walk(child);
|
|
2805
|
+
else estimated += estimateTokens(readFileSync(child, 'utf8'));
|
|
2806
|
+
}
|
|
2807
|
+
};
|
|
2808
|
+
walk(root);
|
|
2809
|
+
const wallMs = performance.now() - startedAt;
|
|
2810
|
+
if (estimated <= 0) throw new Error('bench walk read nothing — the workload is broken');
|
|
2811
|
+
return done(wallMs, { files });
|
|
2812
|
+
} finally {
|
|
2813
|
+
await rm(root, { recursive: true, force: true });
|
|
2814
|
+
}
|
|
2815
|
+
}
|
|
2816
|
+
case 'rollup-20k': {
|
|
2817
|
+
/**
|
|
2818
|
+
* Twenty contributors of a thousand lines each. Their profiles are built
|
|
2819
|
+
* outside the window — profiling is `profile-200k`'s measurement — so
|
|
2820
|
+
* what this times is the roll-up itself: twenty documents parsed,
|
|
2821
|
+
* checked and merged.
|
|
2822
|
+
*/
|
|
2823
|
+
const contributors = 20;
|
|
2824
|
+
const linesEach = 1000;
|
|
2825
|
+
const inputs: { name: string; text: string }[] = [];
|
|
2826
|
+
for (let i = 0; i < contributors; i += 1) {
|
|
2827
|
+
const report = profileUsage(benchLog(linesEach), { catalogue: BUNDLED_CATALOGUE, on: BENCH_DATE });
|
|
2828
|
+
inputs.push({ name: `team-${i}`, text: JSON.stringify(report) });
|
|
2829
|
+
}
|
|
2830
|
+
return done(measure(() => rollUp(inputs)), { lines: contributors * linesEach });
|
|
2831
|
+
}
|
|
2832
|
+
}
|
|
2833
|
+
}
|
|
2834
|
+
|
|
2835
|
+
function printBenchTable(measurements: BenchMeasurement[], t: CliMessages, machine?: BenchDocument): void {
|
|
2836
|
+
const n = (value: number): string => Math.round(value).toLocaleString(t.numberLocale);
|
|
2837
|
+
const mb = (bytes: number): string => `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
|
2838
|
+
const idWidth = Math.max(...measurements.map((m) => m.id.length), t.bench.colWorkload().length);
|
|
2839
|
+
|
|
2840
|
+
console.log();
|
|
2841
|
+
console.log(c.bold(t.bench.heading()));
|
|
2842
|
+
if (machine !== undefined) {
|
|
2843
|
+
console.log(` ${c.dim(t.bench.machine(machine.node, machine.platform, machine.cpus, machine.cpuModel))}`);
|
|
2844
|
+
}
|
|
2845
|
+
console.log();
|
|
2846
|
+
console.log(
|
|
2847
|
+
` ${c.dim(t.bench.colWorkload().padEnd(idWidth))} ${c.dim(t.bench.colWall().padStart(10))} ${c.dim(t.bench.colRatio().padStart(7))} ${c.dim(t.bench.colPeakRss().padStart(10))}`,
|
|
2848
|
+
);
|
|
2849
|
+
for (const m of measurements) {
|
|
2850
|
+
console.log(
|
|
2851
|
+
` ${m.id.padEnd(idWidth)} ${n(m.wallMs).padStart(10)} ${m.ratio.toFixed(2).padStart(7)} ${mb(m.maxRssBytes).padStart(10)}`,
|
|
2852
|
+
);
|
|
2853
|
+
}
|
|
2854
|
+
console.log();
|
|
2855
|
+
console.log(` ${c.dim(wrap(t.bench.note(), 74, ' '))}`);
|
|
2856
|
+
console.log();
|
|
2857
|
+
}
|
|
2858
|
+
|
|
2859
|
+
/** Reads and validates a committed ratio baseline, loudly on anything else. */
|
|
2860
|
+
async function readBenchBaseline(path: string, t: CliMessages): Promise<BenchBaseline> {
|
|
2861
|
+
let parsed: unknown;
|
|
2862
|
+
try {
|
|
2863
|
+
parsed = JSON.parse(await readFile(path, 'utf8'));
|
|
2864
|
+
} catch {
|
|
2865
|
+
throw new Error(t.bench.unreadableBaseline(path));
|
|
2866
|
+
}
|
|
2867
|
+
const candidate = parsed as BenchBaseline;
|
|
2868
|
+
// A version this Trazum does not know is a loud error naming the fix, never
|
|
2869
|
+
// a best-effort read: the file is committed, so it crosses upgrades, and a
|
|
2870
|
+
// gate on misread numbers is a gate on numbers somebody invented.
|
|
2871
|
+
if (candidate?.schemaVersion !== 1 || !Array.isArray(candidate.workloads)) {
|
|
2872
|
+
throw new Error(t.bench.badBaseline(path, JSON.stringify((parsed as { schemaVersion?: unknown })?.schemaVersion)));
|
|
2873
|
+
}
|
|
2874
|
+
for (const entry of candidate.workloads) {
|
|
2875
|
+
if (typeof entry?.id !== 'string' || !Number.isFinite(entry?.ratio) || entry.ratio <= 0) {
|
|
2876
|
+
throw new Error(t.bench.badBaseline(path, 'workloads'));
|
|
2877
|
+
}
|
|
2878
|
+
}
|
|
2879
|
+
return candidate;
|
|
2880
|
+
}
|
|
2881
|
+
|
|
2882
|
+
async function commandBench(args: Args, t: CliMessages): Promise<void> {
|
|
2883
|
+
const asJson = boolFlag(args, 'json');
|
|
2884
|
+
const chosen = stringFlag(args, 'workload');
|
|
2885
|
+
const recordPath = stringFlag(args, 'record');
|
|
2886
|
+
const againstPath = stringFlag(args, 'against');
|
|
2887
|
+
const maxRatioRaw = args.flags.get('max-ratio');
|
|
2888
|
+
|
|
2889
|
+
if (recordPath !== undefined && againstPath !== undefined) {
|
|
2890
|
+
throw new Error(t.bench.recordAndAgainst());
|
|
2891
|
+
}
|
|
2892
|
+
let maxRatio: number | null = null;
|
|
2893
|
+
if (againstPath !== undefined) {
|
|
2894
|
+
// The factor is a policy, so it is stated by the caller rather than
|
|
2895
|
+
// defaulted here — the same rule as pulse's threshold.
|
|
2896
|
+
if (maxRatioRaw === undefined) throw new Error(t.bench.needsMaxRatio());
|
|
2897
|
+
const factor = Number(maxRatioRaw);
|
|
2898
|
+
if (!Number.isFinite(factor) || factor < 1) throw new Error(t.bench.badMaxRatio(String(maxRatioRaw)));
|
|
2899
|
+
maxRatio = factor;
|
|
2900
|
+
} else if (maxRatioRaw !== undefined) {
|
|
2901
|
+
throw new Error(t.bench.maxRatioNeedsAgainst());
|
|
2902
|
+
}
|
|
2903
|
+
|
|
2904
|
+
// Read before measuring, so a baseline this run cannot gate on refuses in
|
|
2905
|
+
// milliseconds instead of after the workloads have been paid for.
|
|
2906
|
+
const baseline = againstPath !== undefined ? await readBenchBaseline(againstPath, t) : null;
|
|
2907
|
+
|
|
2908
|
+
let measurements: BenchMeasurement[];
|
|
2909
|
+
let machine: BenchDocument | undefined;
|
|
2910
|
+
|
|
2911
|
+
if (chosen !== undefined) {
|
|
2912
|
+
if (!(BENCH_WORKLOADS as readonly string[]).includes(chosen)) {
|
|
2913
|
+
throw new Error(t.bench.unknownWorkload(chosen, BENCH_WORKLOADS.join(', ')));
|
|
2914
|
+
}
|
|
2915
|
+
measurements = [await benchRun(chosen as BenchWorkloadId)];
|
|
2916
|
+
} else {
|
|
2917
|
+
const script = fileURLToPath(import.meta.url);
|
|
2918
|
+
measurements = [];
|
|
2919
|
+
for (const id of BENCH_WORKLOADS) {
|
|
2920
|
+
const stdout = runSelf(script, ['bench', '--workload', id, '--json', '--locale', t.locale]);
|
|
2921
|
+
measurements.push(JSON.parse(stdout) as BenchMeasurement);
|
|
2922
|
+
}
|
|
2923
|
+
machine = {
|
|
2924
|
+
schemaVersion: 1,
|
|
2925
|
+
node: process.version,
|
|
2926
|
+
platform: process.platform,
|
|
2927
|
+
arch: process.arch,
|
|
2928
|
+
cpus: cpus().length,
|
|
2929
|
+
cpuModel: cpus()[0]?.model ?? null,
|
|
2930
|
+
workloads: measurements,
|
|
2931
|
+
};
|
|
2932
|
+
}
|
|
2933
|
+
|
|
2934
|
+
if (recordPath !== undefined) {
|
|
2935
|
+
const baseline: BenchBaseline = {
|
|
2936
|
+
schemaVersion: 1,
|
|
2937
|
+
workloads: measurements.map((m) => ({ id: m.id, ratio: m.ratio })),
|
|
2938
|
+
};
|
|
2939
|
+
await writeFile(recordPath, `${JSON.stringify(baseline, null, 2)}\n`);
|
|
2940
|
+
}
|
|
2941
|
+
|
|
2942
|
+
if (asJson) {
|
|
2943
|
+
// The JSON shape never changes with the gate flags: a gate verdict is the
|
|
2944
|
+
// exit code and the sentences on stderr, the way `check` has always gated.
|
|
2945
|
+
console.log(JSON.stringify(chosen !== undefined ? measurements[0] : machine, null, 2));
|
|
2946
|
+
} else {
|
|
2947
|
+
printBenchTable(measurements, t, machine);
|
|
2948
|
+
}
|
|
2949
|
+
if (recordPath !== undefined) {
|
|
2950
|
+
console.error(t.bench.recorded(recordPath));
|
|
2951
|
+
}
|
|
2952
|
+
|
|
2953
|
+
if (againstPath !== undefined && baseline !== null && maxRatio !== null) {
|
|
2954
|
+
const byId = new Map(baseline.workloads.map((entry) => [entry.id, entry.ratio]));
|
|
2955
|
+
let over = false;
|
|
2956
|
+
for (const m of measurements) {
|
|
2957
|
+
const recorded = byId.get(m.id);
|
|
2958
|
+
// Measured but never recorded is not a pass: a gate that silently skips
|
|
2959
|
+
// a workload reads as green coverage it does not have.
|
|
2960
|
+
if (recorded === undefined) throw new Error(t.bench.notInBaseline(m.id, againstPath));
|
|
2961
|
+
const allowed = recorded * maxRatio;
|
|
2962
|
+
if (m.ratio > allowed) {
|
|
2963
|
+
over = true;
|
|
2964
|
+
console.error(c.red(t.bench.gateOver(m.id, m.ratio.toFixed(2), allowed.toFixed(2))));
|
|
2965
|
+
}
|
|
2966
|
+
}
|
|
2967
|
+
if (over) {
|
|
2968
|
+
process.exitCode = 1;
|
|
2969
|
+
} else {
|
|
2970
|
+
console.error(c.dim(t.bench.gateWithin(String(maxRatio))));
|
|
2971
|
+
}
|
|
2972
|
+
}
|
|
2973
|
+
}
|
|
2974
|
+
|
|
2570
2975
|
/**
|
|
2571
2976
|
* `trazum feedback` — where to say it, and what to say.
|
|
2572
2977
|
*
|
|
@@ -3216,7 +3621,7 @@ async function commandSemantic(
|
|
|
3216
3621
|
pricing: PricingCatalogue,
|
|
3217
3622
|
t: CliMessages,
|
|
3218
3623
|
): Promise<void> {
|
|
3219
|
-
const prompt = await readInput(args.positional[0], t);
|
|
3624
|
+
const prompt = await readInput(args.positional[0], t, maxInputFlag(args, t));
|
|
3220
3625
|
const modelId = stringFlag(args, 'model') ?? config.usage?.model ?? DEFAULT_USAGE.model;
|
|
3221
3626
|
const model = pricing.byId.get(modelId) ?? getModel(DEFAULT_USAGE.model);
|
|
3222
3627
|
const rates = { inputPerMTok: model.inputPerMTok, outputPerMTok: model.outputPerMTok };
|
|
@@ -3877,7 +4282,7 @@ async function commandRulesMeasure(
|
|
|
3877
4282
|
// `walkPrompts` returns names relative to the root it was given, so the
|
|
3878
4283
|
// root has to go back on. Reading them bare only worked from a run whose
|
|
3879
4284
|
// cwd happened to be the root, which is the one case a first probe uses.
|
|
3880
|
-
const raw = await readFile(join(root, file), 'utf8');
|
|
4285
|
+
const raw = capInput(await readFile(join(root, file), 'utf8'), file, maxInputFlag(args, t), t);
|
|
3881
4286
|
// A marked source file contributes its marked prompts; anything else
|
|
3882
4287
|
// contributes itself. The same rule `check` and `doctor` already follow,
|
|
3883
4288
|
// so the three commands measure the same text.
|
|
@@ -3929,16 +4334,57 @@ async function commandRulesMeasure(
|
|
|
3929
4334
|
console.log();
|
|
3930
4335
|
}
|
|
3931
4336
|
|
|
3932
|
-
|
|
4337
|
+
/**
|
|
4338
|
+
* The refusal ceiling for a prompt door: the flag, or the number written once
|
|
4339
|
+
* in `@trazum/core`.
|
|
4340
|
+
*
|
|
4341
|
+
* Raising it is deliberate — a flag typed by the person paying the wall time —
|
|
4342
|
+
* and nothing raises it by accident. Lowering it is equally legitimate: a CI
|
|
4343
|
+
* job that knows its prompts are small can make anything bigger a red build.
|
|
4344
|
+
*/
|
|
4345
|
+
function maxInputFlag(args: Args, t: CliMessages): number {
|
|
4346
|
+
const raw = args.flags.get('max-input');
|
|
4347
|
+
if (raw === undefined || typeof raw === 'boolean') return MAX_INPUT_CHARS;
|
|
4348
|
+
const value = Number(raw);
|
|
4349
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
4350
|
+
throw new Error(t.errors.badMaxInput(String(raw)));
|
|
4351
|
+
}
|
|
4352
|
+
return Math.floor(value);
|
|
4353
|
+
}
|
|
4354
|
+
|
|
4355
|
+
/** Refuses a prompt past the ceiling — the size and the limit named, never a grind. */
|
|
4356
|
+
function capInput(text: string, source: string, limit: number, t: CliMessages): string {
|
|
4357
|
+
if (text.length > limit) {
|
|
4358
|
+
throw new Error(t.errors.inputTooLarge(source, text.length, limit));
|
|
4359
|
+
}
|
|
4360
|
+
return text;
|
|
4361
|
+
}
|
|
4362
|
+
|
|
4363
|
+
/**
|
|
4364
|
+
* Reads a prompt-sized input, held to the ceiling.
|
|
4365
|
+
*
|
|
4366
|
+
* `limit: null` is for the callers whose input is a *document or a log* — a
|
|
4367
|
+
* 200,000-line export is ordinary and its size is the product's subject, so
|
|
4368
|
+
* holding it to a prompt's ceiling would refuse the exact input `profile`
|
|
4369
|
+
* exists to read. The null is written at the call site, so a reader of the
|
|
4370
|
+
* caller sees the decision.
|
|
4371
|
+
*/
|
|
4372
|
+
async function readInput(
|
|
4373
|
+
source: string | undefined,
|
|
4374
|
+
t: CliMessages,
|
|
4375
|
+
limit: number | null = null,
|
|
4376
|
+
): Promise<string> {
|
|
3933
4377
|
if (!source) {
|
|
3934
4378
|
throw new Error(t.errors.missingInputFile());
|
|
3935
4379
|
}
|
|
3936
4380
|
if (source === '-') {
|
|
3937
4381
|
const chunks: Buffer[] = [];
|
|
3938
4382
|
for await (const chunk of process.stdin) chunks.push(Buffer.from(chunk));
|
|
3939
|
-
|
|
4383
|
+
const text = Buffer.concat(chunks).toString('utf8');
|
|
4384
|
+
return limit === null ? text : capInput(text, 'stdin', limit, t);
|
|
3940
4385
|
}
|
|
3941
|
-
|
|
4386
|
+
const text = await readFile(source, 'utf8');
|
|
4387
|
+
return limit === null ? text : capInput(text, source, limit, t);
|
|
3942
4388
|
}
|
|
3943
4389
|
|
|
3944
4390
|
async function commandOptimize(
|
|
@@ -3990,6 +4436,7 @@ async function commandOptimize(
|
|
|
3990
4436
|
unreadable.push({ label, path: promptPath });
|
|
3991
4437
|
continue;
|
|
3992
4438
|
}
|
|
4439
|
+
text = capInput(text, promptPath, maxInputFlag(args, t), t);
|
|
3993
4440
|
const r = optimize(text, { level, usage: m.profile, locale, pricing });
|
|
3994
4441
|
rows.push({
|
|
3995
4442
|
label,
|
|
@@ -4040,7 +4487,7 @@ async function commandOptimize(
|
|
|
4040
4487
|
}
|
|
4041
4488
|
|
|
4042
4489
|
const target = args.positional[0];
|
|
4043
|
-
const raw = await readInput(target, t);
|
|
4490
|
+
const raw = await readInput(target, t, maxInputFlag(args, t));
|
|
4044
4491
|
const level = levelFlag(args, config, t);
|
|
4045
4492
|
|
|
4046
4493
|
// A source file is not a prompt.
|
|
@@ -4448,7 +4895,7 @@ async function commandCheck(
|
|
|
4448
4895
|
return;
|
|
4449
4896
|
}
|
|
4450
4897
|
|
|
4451
|
-
const prompt = await readInput(target, t);
|
|
4898
|
+
const prompt = await readInput(target, t, maxInputFlag(args, t));
|
|
4452
4899
|
const n = (value: number): string => value.toLocaleString(t.numberLocale);
|
|
4453
4900
|
|
|
4454
4901
|
// A single file falls back to the config budget for its own path, so
|
|
@@ -8460,7 +8907,7 @@ async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessage
|
|
|
8460
8907
|
return;
|
|
8461
8908
|
}
|
|
8462
8909
|
|
|
8463
|
-
const prompt = await readFile(promptPath, 'utf8');
|
|
8910
|
+
const prompt = capInput(await readFile(promptPath, 'utf8'), promptPath, maxInputFlag(args, t), t);
|
|
8464
8911
|
const inputs = parseCases(await readFile(casesPath, 'utf8'));
|
|
8465
8912
|
if (inputs.length === 0) throw new Error(t.errors.evalNoCases(casesPath));
|
|
8466
8913
|
|
|
@@ -8753,7 +9200,12 @@ async function scanPrompts(
|
|
|
8753
9200
|
const declined: Array<{ path: string; line: number; detail: string }> = [];
|
|
8754
9201
|
|
|
8755
9202
|
for (const relativePath of files) {
|
|
8756
|
-
const text =
|
|
9203
|
+
const text = capInput(
|
|
9204
|
+
await readFile(join(root, relativePath), 'utf8'),
|
|
9205
|
+
relativePath,
|
|
9206
|
+
maxInputFlag(args, t),
|
|
9207
|
+
t,
|
|
9208
|
+
);
|
|
8757
9209
|
// Budgets are keyed on paths as written in the repository, so a pattern like
|
|
8758
9210
|
// `prompts/**` has to be matched against the path including the root the
|
|
8759
9211
|
// user passed — not against the name relative to it.
|
|
@@ -9050,7 +9502,7 @@ function joinPosix(root: string, relativePath: string): string {
|
|
|
9050
9502
|
* never "delete this", and nothing here edits the prompt.
|
|
9051
9503
|
*/
|
|
9052
9504
|
async function commandPrune(args: Args, t: CliMessages): Promise<void> {
|
|
9053
|
-
const prompt = await readInput(args.positional[0], t);
|
|
9505
|
+
const prompt = await readInput(args.positional[0], t, maxInputFlag(args, t));
|
|
9054
9506
|
|
|
9055
9507
|
const casesPath = stringFlag(args, 'cases');
|
|
9056
9508
|
if (!casesPath) throw new Error(t.errors.evalNeedsCases());
|
|
@@ -9138,7 +9590,7 @@ async function commandEval(
|
|
|
9138
9590
|
t: CliMessages,
|
|
9139
9591
|
locale: Locale,
|
|
9140
9592
|
): Promise<void> {
|
|
9141
|
-
const prompt = await readInput(args.positional[0], t);
|
|
9593
|
+
const prompt = await readInput(args.positional[0], t, maxInputFlag(args, t));
|
|
9142
9594
|
const level = levelFlag(args, config, t);
|
|
9143
9595
|
|
|
9144
9596
|
const casesPath = stringFlag(args, 'cases');
|
|
@@ -9325,8 +9777,8 @@ async function commandDiff(
|
|
|
9325
9777
|
}
|
|
9326
9778
|
|
|
9327
9779
|
const [before, after] = await Promise.all([
|
|
9328
|
-
readInput(beforePath, t),
|
|
9329
|
-
readInput(afterPath, t),
|
|
9780
|
+
readInput(beforePath, t, maxInputFlag(args, t)),
|
|
9781
|
+
readInput(afterPath, t, maxInputFlag(args, t)),
|
|
9330
9782
|
]);
|
|
9331
9783
|
|
|
9332
9784
|
const comparison = comparePrompts(before, after, {
|
|
@@ -9801,7 +10253,7 @@ async function commandRank(
|
|
|
9801
10253
|
let skipped = 0;
|
|
9802
10254
|
|
|
9803
10255
|
for (const file of files) {
|
|
9804
|
-
const raw = await readFile(join(root, file), 'utf8');
|
|
10256
|
+
const raw = capInput(await readFile(join(root, file), 'utf8'), file, maxInputFlag(args, t), t);
|
|
9805
10257
|
|
|
9806
10258
|
// A source file contributes its marked prompt, or nothing. Ranking
|
|
9807
10259
|
// `src/prompts.ts` by the size of its imports would put the wrong file at
|
|
@@ -10168,6 +10620,9 @@ async function main(): Promise<void> {
|
|
|
10168
10620
|
case 'pulse':
|
|
10169
10621
|
await commandPulse(args, t);
|
|
10170
10622
|
break;
|
|
10623
|
+
case 'bench':
|
|
10624
|
+
await commandBench(args, t);
|
|
10625
|
+
break;
|
|
10171
10626
|
case 'write':
|
|
10172
10627
|
await commandWrite(args, t);
|
|
10173
10628
|
break;
|
|
@@ -10279,7 +10734,7 @@ async function commandDoctor(
|
|
|
10279
10734
|
let skipped = 0;
|
|
10280
10735
|
|
|
10281
10736
|
for (const file of files) {
|
|
10282
|
-
const raw = await readFile(join(root, file), 'utf8');
|
|
10737
|
+
const raw = capInput(await readFile(join(root, file), 'utf8'), file, maxInputFlag(args, t), t);
|
|
10283
10738
|
|
|
10284
10739
|
// Same contract as `rank`: a source file contributes its marked prompt or
|
|
10285
10740
|
// nothing, and an unmarked one is counted rather than allowed to abort a
|
|
@@ -10581,7 +11036,7 @@ async function diffDirectories(
|
|
|
10581
11036
|
|
|
10582
11037
|
/** The prompt at a path, or null when the file holds no marked prompt. */
|
|
10583
11038
|
const textAt = async (root: string, file: string): Promise<string | null> => {
|
|
10584
|
-
const raw = await readFile(join(root, file), 'utf8');
|
|
11039
|
+
const raw = capInput(await readFile(join(root, file), 'utf8'), file, maxInputFlag(args, t), t);
|
|
10585
11040
|
let source: { text: string; model?: string } | null;
|
|
10586
11041
|
try {
|
|
10587
11042
|
source = sourceFileOf(file, raw, pricing, stringFlag(args, 'prompt'));
|