@trazum/cli 1.61.2 → 1.62.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -1,6 +1,7 @@
1
1
  #!/usr/bin/env node
2
- import { readFileSync } from 'node:fs';
3
- import { open, readdir, readFile, stat, writeFile } from 'node:fs/promises';
2
+ import { readdirSync, readFileSync } from 'node:fs';
3
+ import { mkdir, mkdtemp, open, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
4
+ import { cpus, tmpdir } from 'node:os';
4
5
  import { createInterface } from 'node:readline/promises';
5
6
  import { dirname, join, resolve as resolvePath } from 'node:path';
6
7
  import { fileURLToPath } from 'node:url';
@@ -206,6 +207,7 @@ import {
206
207
  pathInRepository,
207
208
  repositoryRoot,
208
209
  revisionsFor,
210
+ runSelf,
209
211
  } from './git.js';
210
212
  import type { Revision } from './git.js';
211
213
  import { fetchProviderUsage, findCredential } from './connect.js';
@@ -312,6 +314,9 @@ const VALUE_FLAGS = new Set([
312
314
  'max-cache-loss-usd',
313
315
  'max-stale-hours',
314
316
  'measure',
317
+ 'workload',
318
+ 'record',
319
+ 'max-ratio',
315
320
  'max-day-usd',
316
321
  'max-session-usd',
317
322
  'csv-out',
@@ -628,6 +633,7 @@ const COMMAND_FLAGS: Record<string, string[]> = {
628
633
  conform: ['contract', 'json'],
629
634
  rollup: ['json'],
630
635
  pulse: ['json', 'max-stale-hours'],
636
+ bench: ['workload', 'json', 'record', 'against', 'max-ratio'],
631
637
  write: ['answers', 'json', 'out', 'o', 'calls', 'avg-output'],
632
638
  feedback: [],
633
639
  gateway: ['on-cannot-tell', 'port', 'socket', 'pricing', 'pricing-live'],
@@ -2567,6 +2573,402 @@ async function commandPulse(args: Args, t: CliMessages): Promise<void> {
2567
2573
  console.log();
2568
2574
  }
2569
2575
 
2576
+ /**
2577
+ * `trazum bench` — measures this machine, honestly.
2578
+ *
2579
+ * The standard workloads, one shot each, wall time and peak RSS. **No
2580
+ * comparison and no judgement**: a number a person runs before and after a
2581
+ * change and reads side by side. The pathological cases were timed once, by
2582
+ * hand, during a stress session — 1MB of prose in about a second, a
2583
+ * 200,000-line log in about 1.3 — and nothing held them there. This command is
2584
+ * that measurement made repeatable; the gate that holds it belongs to a ratio
2585
+ * against in-process calibration, not to these wall clocks.
2586
+ *
2587
+ * Each workload runs in its own child process, because peak RSS is a fact
2588
+ * about a process: five workloads sharing one heap would each report the
2589
+ * high-water mark of whichever ran biggest before them. The child is this same
2590
+ * CLI with `--workload`, so what the bench measures is exactly what a user
2591
+ * runs.
2592
+ *
2593
+ * The workloads are **generated, deterministic and never committed** — the
2594
+ * fuzzer's own LCG, a fixed pricing date — so two runs on one machine differ
2595
+ * by the machine's weather, never by the input. Generation happens outside the
2596
+ * timed window: the bench times the product, not the bench.
2597
+ *
2598
+ * "Peak heap" in the plan is reported here as **peak RSS** —
2599
+ * `process.resourceUsage().maxRSS`, what the operating system actually billed
2600
+ * the process — because a true heap high-water mark is not observable from
2601
+ * inside a synchronous run without instrumentation that would itself move the
2602
+ * number. The field says what it is.
2603
+ */
2604
+
2605
+ const BENCH_WORKLOADS = [
2606
+ 'optimize-1mb-safe',
2607
+ 'optimize-1mb-aggressive',
2608
+ 'profile-200k',
2609
+ 'walk-10k',
2610
+ 'rollup-20k',
2611
+ ] as const;
2612
+ type BenchWorkloadId = (typeof BENCH_WORKLOADS)[number];
2613
+
2614
+ interface BenchMeasurement {
2615
+ id: string;
2616
+ wallMs: number;
2617
+ /** The calibration loop's wall time, in this same process, right after the workload. */
2618
+ calibrationMs: number;
2619
+ /** wallMs over calibrationMs — the number a gate can hold, because the machine cancels out. */
2620
+ ratio: number;
2621
+ maxRssBytes: number;
2622
+ /** Input size in the unit the workload is named by; the others are null, never zero. */
2623
+ bytes: number | null;
2624
+ lines: number | null;
2625
+ files: number | null;
2626
+ }
2627
+
2628
+ /** The committed ratio baseline `--record` writes and `--against` reads. */
2629
+ interface BenchBaseline {
2630
+ schemaVersion: 1;
2631
+ workloads: { id: string; ratio: number }[];
2632
+ }
2633
+
2634
+ interface BenchDocument {
2635
+ schemaVersion: 1;
2636
+ node: string;
2637
+ platform: string;
2638
+ arch: string;
2639
+ cpus: number;
2640
+ cpuModel: string | null;
2641
+ workloads: BenchMeasurement[];
2642
+ }
2643
+
2644
+ /** The hostile-input suite's LCG: same seed, same workload, any machine. */
2645
+ function benchGenerator(seed: number): () => number {
2646
+ let state = seed;
2647
+ return () => (state = (state * 1103515245 + 12345) % 2147483648) / 2147483648;
2648
+ }
2649
+
2650
+ /** Prices resolve against a fixed date, so the workload cannot drift with the calendar. */
2651
+ const BENCH_DATE = new Date('2026-01-01T00:00:00Z');
2652
+
2653
+ /**
2654
+ * Enough integer work for the loop to take a stable fraction of a second on
2655
+ * anything that can run Node, and little enough that calibrating five
2656
+ * workloads is not itself the bench.
2657
+ */
2658
+ const CALIBRATION_ITERATIONS = 1 << 24;
2659
+
2660
+ /**
2661
+ * The yardstick a ratio divides by: a fixed integer loop, timed in the same
2662
+ * process as the workload it calibrates.
2663
+ *
2664
+ * **Deliberately not the product's own code.** A calibration built on
2665
+ * `estimateTokens` would speed up when the tokenizer does, and every ratio in
2666
+ * every committed baseline would silently mean something new. This loop is
2667
+ * arithmetic that no release has a reason to touch — the same LCG the corpus
2668
+ * generators use, run hot — so a ratio moves only when the workload does.
2669
+ *
2670
+ * CI machines lie about wall time; they lie to the workload and the yardstick
2671
+ * by roughly the same amount, and the ratio is what is left when the lie
2672
+ * cancels out. That is the whole of chapter two's argument, and the reason a
2673
+ * gate on `ratio` can hold where a gate on `wallMs` would fail on weather.
2674
+ */
2675
+ function benchCalibration(): number {
2676
+ const startedAt = performance.now();
2677
+ let state = 1;
2678
+ for (let i = 0; i < CALIBRATION_ITERATIONS; i += 1) {
2679
+ state = (state * 1103515245 + 12345) % 2147483648;
2680
+ }
2681
+ const elapsed = performance.now() - startedAt;
2682
+ // Reading the accumulator keeps the loop observable — a JIT that could prove
2683
+ // the result unused could also skip the work being timed.
2684
+ if (state < 0) throw new Error('unreachable: the LCG stays in [0, 2^31)');
2685
+ return elapsed;
2686
+ }
2687
+
2688
+ /**
2689
+ * Prose the rules have work in — verbose phrases, duplicate lines, emphasis,
2690
+ * spacing — with code fences and URLs mixed in so the segmenter and the masks
2691
+ * are part of what is timed, not skipped by an input too clean to exercise them.
2692
+ */
2693
+ function benchPrompt(targetBytes: number): string {
2694
+ const rnd = benchGenerator(97);
2695
+ const sentences = [
2696
+ 'Please kindly note that in order to get the best results you should always read the entire document.\n',
2697
+ 'It is very very important that the answer is concise and complete.\n',
2698
+ 'IMPORTANT: the sections below repeat their own headers.\n',
2699
+ 'The quick summary follows the long summary, which follows the summary.\n',
2700
+ 'See https://example.com/guide/section?step=3&mode=full for the walkthrough.\n',
2701
+ '```js\nconst total = items.reduce((sum, item) => sum + item.cost, 0);\n```\n',
2702
+ '- keep the tone neutral\n- keep the tone neutral\n- cite every claim\n',
2703
+ 'In the event that the input is empty, respond with an empty list.\n',
2704
+ ];
2705
+ let text = '';
2706
+ while (text.length < targetBytes) {
2707
+ text += sentences[Math.floor(rnd() * sentences.length)];
2708
+ if (rnd() < 0.15) text += '\n\n';
2709
+ }
2710
+ return text;
2711
+ }
2712
+
2713
+ /** A usage log in the documented shape: timestamps, labels, sessions, cache fields. */
2714
+ function benchLog(lines: number): string {
2715
+ const rnd = benchGenerator(53);
2716
+ const models = ['claude-opus-5', 'claude-sonnet-5', 'claude-haiku-4-5'];
2717
+ const labels = ['support-rag', 'classify', 'agent'];
2718
+ const startMs = Date.parse('2026-01-01T00:00:00Z');
2719
+ const out: string[] = [];
2720
+ for (let i = 0; i < lines; i += 1) {
2721
+ const record = {
2722
+ timestamp: new Date(startMs + i * 7000).toISOString(),
2723
+ model: models[Math.floor(rnd() * models.length)],
2724
+ label: labels[Math.floor(rnd() * labels.length)],
2725
+ session: `conv-${Math.floor(rnd() * 400)}`,
2726
+ stop_reason: rnd() < 0.05 ? 'max_tokens' : 'end_turn',
2727
+ usage: {
2728
+ input_tokens: 200 + Math.floor(rnd() * 4000),
2729
+ output_tokens: 50 + Math.floor(rnd() * 800),
2730
+ cache_read_input_tokens: rnd() < 0.6 ? Math.floor(rnd() * 16000) : 0,
2731
+ cache_creation_input_tokens: rnd() < 0.2 ? Math.floor(rnd() * 4000) : 0,
2732
+ },
2733
+ };
2734
+ out.push(JSON.stringify(record));
2735
+ }
2736
+ return out.join('\n');
2737
+ }
2738
+
2739
+ /** Runs one workload: generation outside the window, the product inside it. */
2740
+ async function benchRun(id: BenchWorkloadId): Promise<BenchMeasurement> {
2741
+ const measure = (work: () => unknown): number => {
2742
+ const startedAt = performance.now();
2743
+ work();
2744
+ return performance.now() - startedAt;
2745
+ };
2746
+ const done = (wallMs: number, sizes: Partial<Pick<BenchMeasurement, 'bytes' | 'lines' | 'files'>>): BenchMeasurement => {
2747
+ const calibrationMs = benchCalibration();
2748
+ return {
2749
+ id,
2750
+ wallMs,
2751
+ calibrationMs,
2752
+ ratio: wallMs / calibrationMs,
2753
+ // getrusage reports kilobytes; the field name promises bytes, so convert here.
2754
+ maxRssBytes: process.resourceUsage().maxRSS * 1024,
2755
+ bytes: sizes.bytes ?? null,
2756
+ lines: sizes.lines ?? null,
2757
+ files: sizes.files ?? null,
2758
+ };
2759
+ };
2760
+
2761
+ switch (id) {
2762
+ case 'optimize-1mb-safe':
2763
+ case 'optimize-1mb-aggressive': {
2764
+ const prompt = benchPrompt(1024 * 1024);
2765
+ const level = id === 'optimize-1mb-safe' ? 'safe' : 'aggressive';
2766
+ return done(measure(() => optimize(prompt, { level })), { bytes: Buffer.byteLength(prompt, 'utf8') });
2767
+ }
2768
+ case 'profile-200k': {
2769
+ const lines = 200_000;
2770
+ const text = benchLog(lines);
2771
+ return done(
2772
+ measure(() => profileUsage(text, { catalogue: BUNDLED_CATALOGUE, on: BENCH_DATE })),
2773
+ { lines },
2774
+ );
2775
+ }
2776
+ case 'walk-10k': {
2777
+ /**
2778
+ * The discovery half of `rank` and directory `check`: find every file,
2779
+ * read it, estimate it. Generated under the system temp directory and
2780
+ * removed afterwards — ten thousand files are a workload, not a residue.
2781
+ */
2782
+ const files = 10_000;
2783
+ const root = await mkdtemp(join(tmpdir(), 'trazum-bench-'));
2784
+ try {
2785
+ const rnd = benchGenerator(11);
2786
+ for (let dir = 0; dir < 100; dir += 1) {
2787
+ const dirPath = join(root, `d${dir}`);
2788
+ await mkdir(dirPath);
2789
+ for (let i = 0; i < files / 100; i += 1) {
2790
+ await writeFile(
2791
+ join(dirPath, `p${i}.txt`),
2792
+ `Summarise the report in ${3 + Math.floor(rnd() * 9)} bullet points.\nKeep every figure.\n`,
2793
+ );
2794
+ }
2795
+ }
2796
+ const startedAt = performance.now();
2797
+ let estimated = 0;
2798
+ const walk = (path: string): void => {
2799
+ for (const entry of readdirSync(path, { withFileTypes: true })) {
2800
+ const child = join(path, entry.name);
2801
+ if (entry.isDirectory()) walk(child);
2802
+ else estimated += estimateTokens(readFileSync(child, 'utf8'));
2803
+ }
2804
+ };
2805
+ walk(root);
2806
+ const wallMs = performance.now() - startedAt;
2807
+ if (estimated <= 0) throw new Error('bench walk read nothing — the workload is broken');
2808
+ return done(wallMs, { files });
2809
+ } finally {
2810
+ await rm(root, { recursive: true, force: true });
2811
+ }
2812
+ }
2813
+ case 'rollup-20k': {
2814
+ /**
2815
+ * Twenty contributors of a thousand lines each. Their profiles are built
2816
+ * outside the window — profiling is `profile-200k`'s measurement — so
2817
+ * what this times is the roll-up itself: twenty documents parsed,
2818
+ * checked and merged.
2819
+ */
2820
+ const contributors = 20;
2821
+ const linesEach = 1000;
2822
+ const inputs: { name: string; text: string }[] = [];
2823
+ for (let i = 0; i < contributors; i += 1) {
2824
+ const report = profileUsage(benchLog(linesEach), { catalogue: BUNDLED_CATALOGUE, on: BENCH_DATE });
2825
+ inputs.push({ name: `team-${i}`, text: JSON.stringify(report) });
2826
+ }
2827
+ return done(measure(() => rollUp(inputs)), { lines: contributors * linesEach });
2828
+ }
2829
+ }
2830
+ }
2831
+
2832
+ function printBenchTable(measurements: BenchMeasurement[], t: CliMessages, machine?: BenchDocument): void {
2833
+ const n = (value: number): string => Math.round(value).toLocaleString(t.numberLocale);
2834
+ const mb = (bytes: number): string => `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
2835
+ const idWidth = Math.max(...measurements.map((m) => m.id.length), t.bench.colWorkload().length);
2836
+
2837
+ console.log();
2838
+ console.log(c.bold(t.bench.heading()));
2839
+ if (machine !== undefined) {
2840
+ console.log(` ${c.dim(t.bench.machine(machine.node, machine.platform, machine.cpus, machine.cpuModel))}`);
2841
+ }
2842
+ console.log();
2843
+ console.log(
2844
+ ` ${c.dim(t.bench.colWorkload().padEnd(idWidth))} ${c.dim(t.bench.colWall().padStart(10))} ${c.dim(t.bench.colRatio().padStart(7))} ${c.dim(t.bench.colPeakRss().padStart(10))}`,
2845
+ );
2846
+ for (const m of measurements) {
2847
+ console.log(
2848
+ ` ${m.id.padEnd(idWidth)} ${n(m.wallMs).padStart(10)} ${m.ratio.toFixed(2).padStart(7)} ${mb(m.maxRssBytes).padStart(10)}`,
2849
+ );
2850
+ }
2851
+ console.log();
2852
+ console.log(` ${c.dim(wrap(t.bench.note(), 74, ' '))}`);
2853
+ console.log();
2854
+ }
2855
+
2856
+ /** Reads and validates a committed ratio baseline, loudly on anything else. */
2857
+ async function readBenchBaseline(path: string, t: CliMessages): Promise<BenchBaseline> {
2858
+ let parsed: unknown;
2859
+ try {
2860
+ parsed = JSON.parse(await readFile(path, 'utf8'));
2861
+ } catch {
2862
+ throw new Error(t.bench.unreadableBaseline(path));
2863
+ }
2864
+ const candidate = parsed as BenchBaseline;
2865
+ // A version this Trazum does not know is a loud error naming the fix, never
2866
+ // a best-effort read: the file is committed, so it crosses upgrades, and a
2867
+ // gate on misread numbers is a gate on numbers somebody invented.
2868
+ if (candidate?.schemaVersion !== 1 || !Array.isArray(candidate.workloads)) {
2869
+ throw new Error(t.bench.badBaseline(path, JSON.stringify((parsed as { schemaVersion?: unknown })?.schemaVersion)));
2870
+ }
2871
+ for (const entry of candidate.workloads) {
2872
+ if (typeof entry?.id !== 'string' || !Number.isFinite(entry?.ratio) || entry.ratio <= 0) {
2873
+ throw new Error(t.bench.badBaseline(path, 'workloads'));
2874
+ }
2875
+ }
2876
+ return candidate;
2877
+ }
2878
+
2879
+ async function commandBench(args: Args, t: CliMessages): Promise<void> {
2880
+ const asJson = boolFlag(args, 'json');
2881
+ const chosen = stringFlag(args, 'workload');
2882
+ const recordPath = stringFlag(args, 'record');
2883
+ const againstPath = stringFlag(args, 'against');
2884
+ const maxRatioRaw = args.flags.get('max-ratio');
2885
+
2886
+ if (recordPath !== undefined && againstPath !== undefined) {
2887
+ throw new Error(t.bench.recordAndAgainst());
2888
+ }
2889
+ let maxRatio: number | null = null;
2890
+ if (againstPath !== undefined) {
2891
+ // The factor is a policy, so it is stated by the caller rather than
2892
+ // defaulted here — the same rule as pulse's threshold.
2893
+ if (maxRatioRaw === undefined) throw new Error(t.bench.needsMaxRatio());
2894
+ const factor = Number(maxRatioRaw);
2895
+ if (!Number.isFinite(factor) || factor < 1) throw new Error(t.bench.badMaxRatio(String(maxRatioRaw)));
2896
+ maxRatio = factor;
2897
+ } else if (maxRatioRaw !== undefined) {
2898
+ throw new Error(t.bench.maxRatioNeedsAgainst());
2899
+ }
2900
+
2901
+ // Read before measuring, so a baseline this run cannot gate on refuses in
2902
+ // milliseconds instead of after the workloads have been paid for.
2903
+ const baseline = againstPath !== undefined ? await readBenchBaseline(againstPath, t) : null;
2904
+
2905
+ let measurements: BenchMeasurement[];
2906
+ let machine: BenchDocument | undefined;
2907
+
2908
+ if (chosen !== undefined) {
2909
+ if (!(BENCH_WORKLOADS as readonly string[]).includes(chosen)) {
2910
+ throw new Error(t.bench.unknownWorkload(chosen, BENCH_WORKLOADS.join(', ')));
2911
+ }
2912
+ measurements = [await benchRun(chosen as BenchWorkloadId)];
2913
+ } else {
2914
+ const script = fileURLToPath(import.meta.url);
2915
+ measurements = [];
2916
+ for (const id of BENCH_WORKLOADS) {
2917
+ const stdout = runSelf(script, ['bench', '--workload', id, '--json', '--locale', t.locale]);
2918
+ measurements.push(JSON.parse(stdout) as BenchMeasurement);
2919
+ }
2920
+ machine = {
2921
+ schemaVersion: 1,
2922
+ node: process.version,
2923
+ platform: process.platform,
2924
+ arch: process.arch,
2925
+ cpus: cpus().length,
2926
+ cpuModel: cpus()[0]?.model ?? null,
2927
+ workloads: measurements,
2928
+ };
2929
+ }
2930
+
2931
+ if (recordPath !== undefined) {
2932
+ const baseline: BenchBaseline = {
2933
+ schemaVersion: 1,
2934
+ workloads: measurements.map((m) => ({ id: m.id, ratio: m.ratio })),
2935
+ };
2936
+ await writeFile(recordPath, `${JSON.stringify(baseline, null, 2)}\n`);
2937
+ }
2938
+
2939
+ if (asJson) {
2940
+ // The JSON shape never changes with the gate flags: a gate verdict is the
2941
+ // exit code and the sentences on stderr, the way `check` has always gated.
2942
+ console.log(JSON.stringify(chosen !== undefined ? measurements[0] : machine, null, 2));
2943
+ } else {
2944
+ printBenchTable(measurements, t, machine);
2945
+ }
2946
+ if (recordPath !== undefined) {
2947
+ console.error(t.bench.recorded(recordPath));
2948
+ }
2949
+
2950
+ if (againstPath !== undefined && baseline !== null && maxRatio !== null) {
2951
+ const byId = new Map(baseline.workloads.map((entry) => [entry.id, entry.ratio]));
2952
+ let over = false;
2953
+ for (const m of measurements) {
2954
+ const recorded = byId.get(m.id);
2955
+ // Measured but never recorded is not a pass: a gate that silently skips
2956
+ // a workload reads as green coverage it does not have.
2957
+ if (recorded === undefined) throw new Error(t.bench.notInBaseline(m.id, againstPath));
2958
+ const allowed = recorded * maxRatio;
2959
+ if (m.ratio > allowed) {
2960
+ over = true;
2961
+ console.error(c.red(t.bench.gateOver(m.id, m.ratio.toFixed(2), allowed.toFixed(2))));
2962
+ }
2963
+ }
2964
+ if (over) {
2965
+ process.exitCode = 1;
2966
+ } else {
2967
+ console.error(c.dim(t.bench.gateWithin(String(maxRatio))));
2968
+ }
2969
+ }
2970
+ }
2971
+
2570
2972
  /**
2571
2973
  * `trazum feedback` — where to say it, and what to say.
2572
2974
  *
@@ -10168,6 +10570,9 @@ async function main(): Promise<void> {
10168
10570
  case 'pulse':
10169
10571
  await commandPulse(args, t);
10170
10572
  break;
10573
+ case 'bench':
10574
+ await commandBench(args, t);
10575
+ break;
10171
10576
  case 'write':
10172
10577
  await commandWrite(args, t);
10173
10578
  break;