@trazum/cli 1.62.0 → 1.62.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/git.d.ts +11 -0
- package/dist/git.d.ts.map +1 -1
- package/dist/git.js +34 -3
- package/dist/git.js.map +1 -1
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +44 -0
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/es.d.ts.map +1 -1
- package/dist/i18n/es.js +44 -0
- package/dist/i18n/es.js.map +1 -1
- package/dist/i18n/types.d.ts +20 -0
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/index.js +359 -3
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
- package/src/git.ts +35 -3
- package/src/i18n/en.ts +53 -0
- package/src/i18n/es.ts +53 -0
- package/src/i18n/types.ts +21 -0
- package/src/index.ts +407 -2
package/src/index.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { readFileSync } from 'node:fs';
|
|
3
|
-
import { open, readdir, readFile, stat, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { readdirSync, readFileSync } from 'node:fs';
|
|
3
|
+
import { mkdir, mkdtemp, open, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
|
4
|
+
import { cpus, tmpdir } from 'node:os';
|
|
4
5
|
import { createInterface } from 'node:readline/promises';
|
|
5
6
|
import { dirname, join, resolve as resolvePath } from 'node:path';
|
|
6
7
|
import { fileURLToPath } from 'node:url';
|
|
@@ -206,6 +207,7 @@ import {
|
|
|
206
207
|
pathInRepository,
|
|
207
208
|
repositoryRoot,
|
|
208
209
|
revisionsFor,
|
|
210
|
+
runSelf,
|
|
209
211
|
} from './git.js';
|
|
210
212
|
import type { Revision } from './git.js';
|
|
211
213
|
import { fetchProviderUsage, findCredential } from './connect.js';
|
|
@@ -312,6 +314,9 @@ const VALUE_FLAGS = new Set([
|
|
|
312
314
|
'max-cache-loss-usd',
|
|
313
315
|
'max-stale-hours',
|
|
314
316
|
'measure',
|
|
317
|
+
'workload',
|
|
318
|
+
'record',
|
|
319
|
+
'max-ratio',
|
|
315
320
|
'max-day-usd',
|
|
316
321
|
'max-session-usd',
|
|
317
322
|
'csv-out',
|
|
@@ -628,6 +633,7 @@ const COMMAND_FLAGS: Record<string, string[]> = {
|
|
|
628
633
|
conform: ['contract', 'json'],
|
|
629
634
|
rollup: ['json'],
|
|
630
635
|
pulse: ['json', 'max-stale-hours'],
|
|
636
|
+
bench: ['workload', 'json', 'record', 'against', 'max-ratio'],
|
|
631
637
|
write: ['answers', 'json', 'out', 'o', 'calls', 'avg-output'],
|
|
632
638
|
feedback: [],
|
|
633
639
|
gateway: ['on-cannot-tell', 'port', 'socket', 'pricing', 'pricing-live'],
|
|
@@ -2567,6 +2573,402 @@ async function commandPulse(args: Args, t: CliMessages): Promise<void> {
|
|
|
2567
2573
|
console.log();
|
|
2568
2574
|
}
|
|
2569
2575
|
|
|
2576
|
+
/**
|
|
2577
|
+
* `trazum bench` — measures this machine, honestly.
|
|
2578
|
+
*
|
|
2579
|
+
* The standard workloads, one shot each, wall time and peak RSS. **No
|
|
2580
|
+
* comparison and no judgement**: a number a person runs before and after a
|
|
2581
|
+
* change and reads side by side. The pathological cases were timed once, by
|
|
2582
|
+
* hand, during a stress session — 1MB of prose in about a second, a
|
|
2583
|
+
* 200,000-line log in about 1.3 — and nothing held them there. This command is
|
|
2584
|
+
* that measurement made repeatable; the gate that holds it belongs to a ratio
|
|
2585
|
+
* against in-process calibration, not to these wall clocks.
|
|
2586
|
+
*
|
|
2587
|
+
* Each workload runs in its own child process, because peak RSS is a fact
|
|
2588
|
+
* about a process: five workloads sharing one heap would each report the
|
|
2589
|
+
* high-water mark of whichever ran biggest before them. The child is this same
|
|
2590
|
+
* CLI with `--workload`, so what the bench measures is exactly what a user
|
|
2591
|
+
* runs.
|
|
2592
|
+
*
|
|
2593
|
+
* The workloads are **generated, deterministic and never committed** — the
|
|
2594
|
+
* fuzzer's own LCG, a fixed pricing date — so two runs on one machine differ
|
|
2595
|
+
* by the machine's weather, never by the input. Generation happens outside the
|
|
2596
|
+
* timed window: the bench times the product, not the bench.
|
|
2597
|
+
*
|
|
2598
|
+
* "Peak heap" in the plan is reported here as **peak RSS** —
|
|
2599
|
+
* `process.resourceUsage().maxRSS`, what the operating system actually billed
|
|
2600
|
+
* the process — because a true heap high-water mark is not observable from
|
|
2601
|
+
* inside a synchronous run without instrumentation that would itself move the
|
|
2602
|
+
* number. The field says what it is.
|
|
2603
|
+
*/
|
|
2604
|
+
|
|
2605
|
+
const BENCH_WORKLOADS = [
|
|
2606
|
+
'optimize-1mb-safe',
|
|
2607
|
+
'optimize-1mb-aggressive',
|
|
2608
|
+
'profile-200k',
|
|
2609
|
+
'walk-10k',
|
|
2610
|
+
'rollup-20k',
|
|
2611
|
+
] as const;
|
|
2612
|
+
type BenchWorkloadId = (typeof BENCH_WORKLOADS)[number];
|
|
2613
|
+
|
|
2614
|
+
interface BenchMeasurement {
|
|
2615
|
+
id: string;
|
|
2616
|
+
wallMs: number;
|
|
2617
|
+
/** The calibration loop's wall time, in this same process, right after the workload. */
|
|
2618
|
+
calibrationMs: number;
|
|
2619
|
+
/** wallMs over calibrationMs — the number a gate can hold, because the machine cancels out. */
|
|
2620
|
+
ratio: number;
|
|
2621
|
+
maxRssBytes: number;
|
|
2622
|
+
/** Input size in the unit the workload is named by; the others are null, never zero. */
|
|
2623
|
+
bytes: number | null;
|
|
2624
|
+
lines: number | null;
|
|
2625
|
+
files: number | null;
|
|
2626
|
+
}
|
|
2627
|
+
|
|
2628
|
+
/** The committed ratio baseline `--record` writes and `--against` reads. */
|
|
2629
|
+
interface BenchBaseline {
|
|
2630
|
+
schemaVersion: 1;
|
|
2631
|
+
workloads: { id: string; ratio: number }[];
|
|
2632
|
+
}
|
|
2633
|
+
|
|
2634
|
+
interface BenchDocument {
|
|
2635
|
+
schemaVersion: 1;
|
|
2636
|
+
node: string;
|
|
2637
|
+
platform: string;
|
|
2638
|
+
arch: string;
|
|
2639
|
+
cpus: number;
|
|
2640
|
+
cpuModel: string | null;
|
|
2641
|
+
workloads: BenchMeasurement[];
|
|
2642
|
+
}
|
|
2643
|
+
|
|
2644
|
+
/** The hostile-input suite's LCG: same seed, same workload, any machine. */
|
|
2645
|
+
function benchGenerator(seed: number): () => number {
|
|
2646
|
+
let state = seed;
|
|
2647
|
+
return () => (state = (state * 1103515245 + 12345) % 2147483648) / 2147483648;
|
|
2648
|
+
}
|
|
2649
|
+
|
|
2650
|
+
/** Prices resolve against a fixed date, so the workload cannot drift with the calendar. */
|
|
2651
|
+
const BENCH_DATE = new Date('2026-01-01T00:00:00Z');
|
|
2652
|
+
|
|
2653
|
+
/**
|
|
2654
|
+
* Enough integer work for the loop to take a stable fraction of a second on
|
|
2655
|
+
* anything that can run Node, and little enough that calibrating five
|
|
2656
|
+
* workloads is not itself the bench.
|
|
2657
|
+
*/
|
|
2658
|
+
const CALIBRATION_ITERATIONS = 1 << 24;
|
|
2659
|
+
|
|
2660
|
+
/**
|
|
2661
|
+
* The yardstick a ratio divides by: a fixed integer loop, timed in the same
|
|
2662
|
+
* process as the workload it calibrates.
|
|
2663
|
+
*
|
|
2664
|
+
* **Deliberately not the product's own code.** A calibration built on
|
|
2665
|
+
* `estimateTokens` would speed up when the tokenizer does, and every ratio in
|
|
2666
|
+
* every committed baseline would silently mean something new. This loop is
|
|
2667
|
+
* arithmetic that no release has a reason to touch — the same LCG the corpus
|
|
2668
|
+
* generators use, run hot — so a ratio moves only when the workload does.
|
|
2669
|
+
*
|
|
2670
|
+
* CI machines lie about wall time; they lie to the workload and the yardstick
|
|
2671
|
+
* by roughly the same amount, and the ratio is what is left when the lie
|
|
2672
|
+
* cancels out. That is the whole of chapter two's argument, and the reason a
|
|
2673
|
+
* gate on `ratio` can hold where a gate on `wallMs` would fail on weather.
|
|
2674
|
+
*/
|
|
2675
|
+
function benchCalibration(): number {
|
|
2676
|
+
const startedAt = performance.now();
|
|
2677
|
+
let state = 1;
|
|
2678
|
+
for (let i = 0; i < CALIBRATION_ITERATIONS; i += 1) {
|
|
2679
|
+
state = (state * 1103515245 + 12345) % 2147483648;
|
|
2680
|
+
}
|
|
2681
|
+
const elapsed = performance.now() - startedAt;
|
|
2682
|
+
// Reading the accumulator keeps the loop observable — a JIT that could prove
|
|
2683
|
+
// the result unused could also skip the work being timed.
|
|
2684
|
+
if (state < 0) throw new Error('unreachable: the LCG stays in [0, 2^31)');
|
|
2685
|
+
return elapsed;
|
|
2686
|
+
}
|
|
2687
|
+
|
|
2688
|
+
/**
|
|
2689
|
+
* Prose the rules have work in — verbose phrases, duplicate lines, emphasis,
|
|
2690
|
+
* spacing — with code fences and URLs mixed in so the segmenter and the masks
|
|
2691
|
+
* are part of what is timed, not skipped by an input too clean to exercise them.
|
|
2692
|
+
*/
|
|
2693
|
+
function benchPrompt(targetBytes: number): string {
|
|
2694
|
+
const rnd = benchGenerator(97);
|
|
2695
|
+
const sentences = [
|
|
2696
|
+
'Please kindly note that in order to get the best results you should always read the entire document.\n',
|
|
2697
|
+
'It is very very important that the answer is concise and complete.\n',
|
|
2698
|
+
'IMPORTANT: the sections below repeat their own headers.\n',
|
|
2699
|
+
'The quick summary follows the long summary, which follows the summary.\n',
|
|
2700
|
+
'See https://example.com/guide/section?step=3&mode=full for the walkthrough.\n',
|
|
2701
|
+
'```js\nconst total = items.reduce((sum, item) => sum + item.cost, 0);\n```\n',
|
|
2702
|
+
'- keep the tone neutral\n- keep the tone neutral\n- cite every claim\n',
|
|
2703
|
+
'In the event that the input is empty, respond with an empty list.\n',
|
|
2704
|
+
];
|
|
2705
|
+
let text = '';
|
|
2706
|
+
while (text.length < targetBytes) {
|
|
2707
|
+
text += sentences[Math.floor(rnd() * sentences.length)];
|
|
2708
|
+
if (rnd() < 0.15) text += '\n\n';
|
|
2709
|
+
}
|
|
2710
|
+
return text;
|
|
2711
|
+
}
|
|
2712
|
+
|
|
2713
|
+
/** A usage log in the documented shape: timestamps, labels, sessions, cache fields. */
|
|
2714
|
+
function benchLog(lines: number): string {
|
|
2715
|
+
const rnd = benchGenerator(53);
|
|
2716
|
+
const models = ['claude-opus-5', 'claude-sonnet-5', 'claude-haiku-4-5'];
|
|
2717
|
+
const labels = ['support-rag', 'classify', 'agent'];
|
|
2718
|
+
const startMs = Date.parse('2026-01-01T00:00:00Z');
|
|
2719
|
+
const out: string[] = [];
|
|
2720
|
+
for (let i = 0; i < lines; i += 1) {
|
|
2721
|
+
const record = {
|
|
2722
|
+
timestamp: new Date(startMs + i * 7000).toISOString(),
|
|
2723
|
+
model: models[Math.floor(rnd() * models.length)],
|
|
2724
|
+
label: labels[Math.floor(rnd() * labels.length)],
|
|
2725
|
+
session: `conv-${Math.floor(rnd() * 400)}`,
|
|
2726
|
+
stop_reason: rnd() < 0.05 ? 'max_tokens' : 'end_turn',
|
|
2727
|
+
usage: {
|
|
2728
|
+
input_tokens: 200 + Math.floor(rnd() * 4000),
|
|
2729
|
+
output_tokens: 50 + Math.floor(rnd() * 800),
|
|
2730
|
+
cache_read_input_tokens: rnd() < 0.6 ? Math.floor(rnd() * 16000) : 0,
|
|
2731
|
+
cache_creation_input_tokens: rnd() < 0.2 ? Math.floor(rnd() * 4000) : 0,
|
|
2732
|
+
},
|
|
2733
|
+
};
|
|
2734
|
+
out.push(JSON.stringify(record));
|
|
2735
|
+
}
|
|
2736
|
+
return out.join('\n');
|
|
2737
|
+
}
|
|
2738
|
+
|
|
2739
|
+
/** Runs one workload: generation outside the window, the product inside it. */
|
|
2740
|
+
async function benchRun(id: BenchWorkloadId): Promise<BenchMeasurement> {
|
|
2741
|
+
const measure = (work: () => unknown): number => {
|
|
2742
|
+
const startedAt = performance.now();
|
|
2743
|
+
work();
|
|
2744
|
+
return performance.now() - startedAt;
|
|
2745
|
+
};
|
|
2746
|
+
const done = (wallMs: number, sizes: Partial<Pick<BenchMeasurement, 'bytes' | 'lines' | 'files'>>): BenchMeasurement => {
|
|
2747
|
+
const calibrationMs = benchCalibration();
|
|
2748
|
+
return {
|
|
2749
|
+
id,
|
|
2750
|
+
wallMs,
|
|
2751
|
+
calibrationMs,
|
|
2752
|
+
ratio: wallMs / calibrationMs,
|
|
2753
|
+
// getrusage reports kilobytes; the field name promises bytes, so convert here.
|
|
2754
|
+
maxRssBytes: process.resourceUsage().maxRSS * 1024,
|
|
2755
|
+
bytes: sizes.bytes ?? null,
|
|
2756
|
+
lines: sizes.lines ?? null,
|
|
2757
|
+
files: sizes.files ?? null,
|
|
2758
|
+
};
|
|
2759
|
+
};
|
|
2760
|
+
|
|
2761
|
+
switch (id) {
|
|
2762
|
+
case 'optimize-1mb-safe':
|
|
2763
|
+
case 'optimize-1mb-aggressive': {
|
|
2764
|
+
const prompt = benchPrompt(1024 * 1024);
|
|
2765
|
+
const level = id === 'optimize-1mb-safe' ? 'safe' : 'aggressive';
|
|
2766
|
+
return done(measure(() => optimize(prompt, { level })), { bytes: Buffer.byteLength(prompt, 'utf8') });
|
|
2767
|
+
}
|
|
2768
|
+
case 'profile-200k': {
|
|
2769
|
+
const lines = 200_000;
|
|
2770
|
+
const text = benchLog(lines);
|
|
2771
|
+
return done(
|
|
2772
|
+
measure(() => profileUsage(text, { catalogue: BUNDLED_CATALOGUE, on: BENCH_DATE })),
|
|
2773
|
+
{ lines },
|
|
2774
|
+
);
|
|
2775
|
+
}
|
|
2776
|
+
case 'walk-10k': {
|
|
2777
|
+
/**
|
|
2778
|
+
* The discovery half of `rank` and directory `check`: find every file,
|
|
2779
|
+
* read it, estimate it. Generated under the system temp directory and
|
|
2780
|
+
* removed afterwards — ten thousand files are a workload, not a residue.
|
|
2781
|
+
*/
|
|
2782
|
+
const files = 10_000;
|
|
2783
|
+
const root = await mkdtemp(join(tmpdir(), 'trazum-bench-'));
|
|
2784
|
+
try {
|
|
2785
|
+
const rnd = benchGenerator(11);
|
|
2786
|
+
for (let dir = 0; dir < 100; dir += 1) {
|
|
2787
|
+
const dirPath = join(root, `d${dir}`);
|
|
2788
|
+
await mkdir(dirPath);
|
|
2789
|
+
for (let i = 0; i < files / 100; i += 1) {
|
|
2790
|
+
await writeFile(
|
|
2791
|
+
join(dirPath, `p${i}.txt`),
|
|
2792
|
+
`Summarise the report in ${3 + Math.floor(rnd() * 9)} bullet points.\nKeep every figure.\n`,
|
|
2793
|
+
);
|
|
2794
|
+
}
|
|
2795
|
+
}
|
|
2796
|
+
const startedAt = performance.now();
|
|
2797
|
+
let estimated = 0;
|
|
2798
|
+
const walk = (path: string): void => {
|
|
2799
|
+
for (const entry of readdirSync(path, { withFileTypes: true })) {
|
|
2800
|
+
const child = join(path, entry.name);
|
|
2801
|
+
if (entry.isDirectory()) walk(child);
|
|
2802
|
+
else estimated += estimateTokens(readFileSync(child, 'utf8'));
|
|
2803
|
+
}
|
|
2804
|
+
};
|
|
2805
|
+
walk(root);
|
|
2806
|
+
const wallMs = performance.now() - startedAt;
|
|
2807
|
+
if (estimated <= 0) throw new Error('bench walk read nothing — the workload is broken');
|
|
2808
|
+
return done(wallMs, { files });
|
|
2809
|
+
} finally {
|
|
2810
|
+
await rm(root, { recursive: true, force: true });
|
|
2811
|
+
}
|
|
2812
|
+
}
|
|
2813
|
+
case 'rollup-20k': {
|
|
2814
|
+
/**
|
|
2815
|
+
* Twenty contributors of a thousand lines each. Their profiles are built
|
|
2816
|
+
* outside the window — profiling is `profile-200k`'s measurement — so
|
|
2817
|
+
* what this times is the roll-up itself: twenty documents parsed,
|
|
2818
|
+
* checked and merged.
|
|
2819
|
+
*/
|
|
2820
|
+
const contributors = 20;
|
|
2821
|
+
const linesEach = 1000;
|
|
2822
|
+
const inputs: { name: string; text: string }[] = [];
|
|
2823
|
+
for (let i = 0; i < contributors; i += 1) {
|
|
2824
|
+
const report = profileUsage(benchLog(linesEach), { catalogue: BUNDLED_CATALOGUE, on: BENCH_DATE });
|
|
2825
|
+
inputs.push({ name: `team-${i}`, text: JSON.stringify(report) });
|
|
2826
|
+
}
|
|
2827
|
+
return done(measure(() => rollUp(inputs)), { lines: contributors * linesEach });
|
|
2828
|
+
}
|
|
2829
|
+
}
|
|
2830
|
+
}
|
|
2831
|
+
|
|
2832
|
+
function printBenchTable(measurements: BenchMeasurement[], t: CliMessages, machine?: BenchDocument): void {
|
|
2833
|
+
const n = (value: number): string => Math.round(value).toLocaleString(t.numberLocale);
|
|
2834
|
+
const mb = (bytes: number): string => `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
|
2835
|
+
const idWidth = Math.max(...measurements.map((m) => m.id.length), t.bench.colWorkload().length);
|
|
2836
|
+
|
|
2837
|
+
console.log();
|
|
2838
|
+
console.log(c.bold(t.bench.heading()));
|
|
2839
|
+
if (machine !== undefined) {
|
|
2840
|
+
console.log(` ${c.dim(t.bench.machine(machine.node, machine.platform, machine.cpus, machine.cpuModel))}`);
|
|
2841
|
+
}
|
|
2842
|
+
console.log();
|
|
2843
|
+
console.log(
|
|
2844
|
+
` ${c.dim(t.bench.colWorkload().padEnd(idWidth))} ${c.dim(t.bench.colWall().padStart(10))} ${c.dim(t.bench.colRatio().padStart(7))} ${c.dim(t.bench.colPeakRss().padStart(10))}`,
|
|
2845
|
+
);
|
|
2846
|
+
for (const m of measurements) {
|
|
2847
|
+
console.log(
|
|
2848
|
+
` ${m.id.padEnd(idWidth)} ${n(m.wallMs).padStart(10)} ${m.ratio.toFixed(2).padStart(7)} ${mb(m.maxRssBytes).padStart(10)}`,
|
|
2849
|
+
);
|
|
2850
|
+
}
|
|
2851
|
+
console.log();
|
|
2852
|
+
console.log(` ${c.dim(wrap(t.bench.note(), 74, ' '))}`);
|
|
2853
|
+
console.log();
|
|
2854
|
+
}
|
|
2855
|
+
|
|
2856
|
+
/** Reads and validates a committed ratio baseline, loudly on anything else. */
|
|
2857
|
+
async function readBenchBaseline(path: string, t: CliMessages): Promise<BenchBaseline> {
|
|
2858
|
+
let parsed: unknown;
|
|
2859
|
+
try {
|
|
2860
|
+
parsed = JSON.parse(await readFile(path, 'utf8'));
|
|
2861
|
+
} catch {
|
|
2862
|
+
throw new Error(t.bench.unreadableBaseline(path));
|
|
2863
|
+
}
|
|
2864
|
+
const candidate = parsed as BenchBaseline;
|
|
2865
|
+
// A version this Trazum does not know is a loud error naming the fix, never
|
|
2866
|
+
// a best-effort read: the file is committed, so it crosses upgrades, and a
|
|
2867
|
+
// gate on misread numbers is a gate on numbers somebody invented.
|
|
2868
|
+
if (candidate?.schemaVersion !== 1 || !Array.isArray(candidate.workloads)) {
|
|
2869
|
+
throw new Error(t.bench.badBaseline(path, JSON.stringify((parsed as { schemaVersion?: unknown })?.schemaVersion)));
|
|
2870
|
+
}
|
|
2871
|
+
for (const entry of candidate.workloads) {
|
|
2872
|
+
if (typeof entry?.id !== 'string' || !Number.isFinite(entry?.ratio) || entry.ratio <= 0) {
|
|
2873
|
+
throw new Error(t.bench.badBaseline(path, 'workloads'));
|
|
2874
|
+
}
|
|
2875
|
+
}
|
|
2876
|
+
return candidate;
|
|
2877
|
+
}
|
|
2878
|
+
|
|
2879
|
+
async function commandBench(args: Args, t: CliMessages): Promise<void> {
|
|
2880
|
+
const asJson = boolFlag(args, 'json');
|
|
2881
|
+
const chosen = stringFlag(args, 'workload');
|
|
2882
|
+
const recordPath = stringFlag(args, 'record');
|
|
2883
|
+
const againstPath = stringFlag(args, 'against');
|
|
2884
|
+
const maxRatioRaw = args.flags.get('max-ratio');
|
|
2885
|
+
|
|
2886
|
+
if (recordPath !== undefined && againstPath !== undefined) {
|
|
2887
|
+
throw new Error(t.bench.recordAndAgainst());
|
|
2888
|
+
}
|
|
2889
|
+
let maxRatio: number | null = null;
|
|
2890
|
+
if (againstPath !== undefined) {
|
|
2891
|
+
// The factor is a policy, so it is stated by the caller rather than
|
|
2892
|
+
// defaulted here — the same rule as pulse's threshold.
|
|
2893
|
+
if (maxRatioRaw === undefined) throw new Error(t.bench.needsMaxRatio());
|
|
2894
|
+
const factor = Number(maxRatioRaw);
|
|
2895
|
+
if (!Number.isFinite(factor) || factor < 1) throw new Error(t.bench.badMaxRatio(String(maxRatioRaw)));
|
|
2896
|
+
maxRatio = factor;
|
|
2897
|
+
} else if (maxRatioRaw !== undefined) {
|
|
2898
|
+
throw new Error(t.bench.maxRatioNeedsAgainst());
|
|
2899
|
+
}
|
|
2900
|
+
|
|
2901
|
+
// Read before measuring, so a baseline this run cannot gate on refuses in
|
|
2902
|
+
// milliseconds instead of after the workloads have been paid for.
|
|
2903
|
+
const baseline = againstPath !== undefined ? await readBenchBaseline(againstPath, t) : null;
|
|
2904
|
+
|
|
2905
|
+
let measurements: BenchMeasurement[];
|
|
2906
|
+
let machine: BenchDocument | undefined;
|
|
2907
|
+
|
|
2908
|
+
if (chosen !== undefined) {
|
|
2909
|
+
if (!(BENCH_WORKLOADS as readonly string[]).includes(chosen)) {
|
|
2910
|
+
throw new Error(t.bench.unknownWorkload(chosen, BENCH_WORKLOADS.join(', ')));
|
|
2911
|
+
}
|
|
2912
|
+
measurements = [await benchRun(chosen as BenchWorkloadId)];
|
|
2913
|
+
} else {
|
|
2914
|
+
const script = fileURLToPath(import.meta.url);
|
|
2915
|
+
measurements = [];
|
|
2916
|
+
for (const id of BENCH_WORKLOADS) {
|
|
2917
|
+
const stdout = runSelf(script, ['bench', '--workload', id, '--json', '--locale', t.locale]);
|
|
2918
|
+
measurements.push(JSON.parse(stdout) as BenchMeasurement);
|
|
2919
|
+
}
|
|
2920
|
+
machine = {
|
|
2921
|
+
schemaVersion: 1,
|
|
2922
|
+
node: process.version,
|
|
2923
|
+
platform: process.platform,
|
|
2924
|
+
arch: process.arch,
|
|
2925
|
+
cpus: cpus().length,
|
|
2926
|
+
cpuModel: cpus()[0]?.model ?? null,
|
|
2927
|
+
workloads: measurements,
|
|
2928
|
+
};
|
|
2929
|
+
}
|
|
2930
|
+
|
|
2931
|
+
if (recordPath !== undefined) {
|
|
2932
|
+
const baseline: BenchBaseline = {
|
|
2933
|
+
schemaVersion: 1,
|
|
2934
|
+
workloads: measurements.map((m) => ({ id: m.id, ratio: m.ratio })),
|
|
2935
|
+
};
|
|
2936
|
+
await writeFile(recordPath, `${JSON.stringify(baseline, null, 2)}\n`);
|
|
2937
|
+
}
|
|
2938
|
+
|
|
2939
|
+
if (asJson) {
|
|
2940
|
+
// The JSON shape never changes with the gate flags: a gate verdict is the
|
|
2941
|
+
// exit code and the sentences on stderr, the way `check` has always gated.
|
|
2942
|
+
console.log(JSON.stringify(chosen !== undefined ? measurements[0] : machine, null, 2));
|
|
2943
|
+
} else {
|
|
2944
|
+
printBenchTable(measurements, t, machine);
|
|
2945
|
+
}
|
|
2946
|
+
if (recordPath !== undefined) {
|
|
2947
|
+
console.error(t.bench.recorded(recordPath));
|
|
2948
|
+
}
|
|
2949
|
+
|
|
2950
|
+
if (againstPath !== undefined && baseline !== null && maxRatio !== null) {
|
|
2951
|
+
const byId = new Map(baseline.workloads.map((entry) => [entry.id, entry.ratio]));
|
|
2952
|
+
let over = false;
|
|
2953
|
+
for (const m of measurements) {
|
|
2954
|
+
const recorded = byId.get(m.id);
|
|
2955
|
+
// Measured but never recorded is not a pass: a gate that silently skips
|
|
2956
|
+
// a workload reads as green coverage it does not have.
|
|
2957
|
+
if (recorded === undefined) throw new Error(t.bench.notInBaseline(m.id, againstPath));
|
|
2958
|
+
const allowed = recorded * maxRatio;
|
|
2959
|
+
if (m.ratio > allowed) {
|
|
2960
|
+
over = true;
|
|
2961
|
+
console.error(c.red(t.bench.gateOver(m.id, m.ratio.toFixed(2), allowed.toFixed(2))));
|
|
2962
|
+
}
|
|
2963
|
+
}
|
|
2964
|
+
if (over) {
|
|
2965
|
+
process.exitCode = 1;
|
|
2966
|
+
} else {
|
|
2967
|
+
console.error(c.dim(t.bench.gateWithin(String(maxRatio))));
|
|
2968
|
+
}
|
|
2969
|
+
}
|
|
2970
|
+
}
|
|
2971
|
+
|
|
2570
2972
|
/**
|
|
2571
2973
|
* `trazum feedback` — where to say it, and what to say.
|
|
2572
2974
|
*
|
|
@@ -10168,6 +10570,9 @@ async function main(): Promise<void> {
|
|
|
10168
10570
|
case 'pulse':
|
|
10169
10571
|
await commandPulse(args, t);
|
|
10170
10572
|
break;
|
|
10573
|
+
case 'bench':
|
|
10574
|
+
await commandBench(args, t);
|
|
10575
|
+
break;
|
|
10171
10576
|
case 'write':
|
|
10172
10577
|
await commandWrite(args, t);
|
|
10173
10578
|
break;
|