@alis-build/harness-eval 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +92 -8
- package/dist/adapters/claude-code/index.d.ts +2 -2
- package/dist/adapters/claude-code/index.js +2 -1
- package/dist/adapters/codex/index.d.ts +68 -0
- package/dist/adapters/codex/index.js +3 -0
- package/dist/{claude-code-DZ4Vkgp6.js → claude-code-C_7hxC8z.js} +3 -245
- package/dist/claude-code-C_7hxC8z.js.map +1 -0
- package/dist/cli/bin.js +131 -151
- package/dist/cli/bin.js.map +1 -1
- package/dist/codex-0cHO2te9.js +496 -0
- package/dist/codex-0cHO2te9.js.map +1 -0
- package/dist/config/loader.d.ts +2 -2
- package/dist/config/loader.js +2 -2
- package/dist/{index-V22PrR0p.d.ts → index-DnvP1UBl.d.ts} +2 -2
- package/dist/index.d.ts +132 -6
- package/dist/index.js +6 -5
- package/dist/index.js.map +1 -1
- package/dist/loader-B1WmGGzf.d.ts +107 -0
- package/dist/{loader-DcI0KfRX.js → loader-DnQ6Jt0i.js} +472 -209
- package/dist/loader-DnQ6Jt0i.js.map +1 -0
- package/dist/{projections-BcX7w-f6.js → reporter-Biy-5-9M.js} +1335 -758
- package/dist/reporter-Biy-5-9M.js.map +1 -0
- package/dist/runner/suite.d.ts +1 -1
- package/dist/runner/suite.js +1 -1
- package/dist/{suite-DPJMIEbu.d.ts → suite-BEShV0by.d.ts} +2 -2
- package/dist/{suite-Dlzl-HI0.js → suite-BcP64nlb.js} +16 -2
- package/dist/{suite-Dlzl-HI0.js.map → suite-BcP64nlb.js.map} +1 -1
- package/dist/{types-CD3TwOtZ.d.ts → types-0QkNVyp9.d.ts} +2 -2
- package/dist/types-Bac8_Ixb.js +246 -0
- package/dist/types-Bac8_Ixb.js.map +1 -0
- package/dist/types-Bu8uOZZN.d.ts +77 -0
- package/dist/{types-B9H4IZtA.d.ts → types-C0gBkl0-.d.ts} +3 -2
- package/package.json +6 -2
- package/dist/claude-code-DZ4Vkgp6.js.map +0 -1
- package/dist/loader-C9yQHUPC.d.ts +0 -50
- package/dist/loader-DcI0KfRX.js.map +0 -1
- package/dist/projections-BcX7w-f6.js.map +0 -1
package/dist/cli/bin.js
CHANGED
|
@@ -1,152 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
3
|
-
import { t as runSuite, u as getAdapter } from "../suite-
|
|
4
|
-
import { i as loadGradingConfig, t as loadSuite } from "../loader-
|
|
2
|
+
import { F as loadSuiteReport, M as gradingReportPassed, N as resolveGradeOptions, P as gradeReport, a as envelopeCommand, c as getOptionInt, i as runPipeline, j as formatGradingConsole, l as hasOption, o as parseEnvelopeProjection, p as suiteDirectoryFromPath, r as trajectoryToOtlp, s as getOption, t as formatReport, u as parseArgs } from "../reporter-Biy-5-9M.js";
|
|
3
|
+
import { t as runSuite, u as getAdapter } from "../suite-BcP64nlb.js";
|
|
4
|
+
import { i as loadGradingConfig, o as loadSuiteDocument, t as loadSuite } from "../loader-DnQ6Jt0i.js";
|
|
5
5
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
6
|
-
import { dirname, join } from "node:path";
|
|
6
|
+
import { dirname, isAbsolute, join } from "node:path";
|
|
7
7
|
import { fileURLToPath } from "node:url";
|
|
8
|
-
//#region src/cli/args.ts
|
|
9
|
-
/** Parse process argv into command, positional args, and options. */
|
|
10
|
-
function parseArgs(argv) {
|
|
11
|
-
const positional = [];
|
|
12
|
-
const options = {};
|
|
13
|
-
let command;
|
|
14
|
-
const args = [...argv];
|
|
15
|
-
if (args.length > 0 && !args[0].startsWith("-")) command = args.shift();
|
|
16
|
-
for (let i = 0; i < args.length; i++) {
|
|
17
|
-
const arg = args[i];
|
|
18
|
-
if (arg === "--") {
|
|
19
|
-
positional.push(...args.slice(i + 1));
|
|
20
|
-
break;
|
|
21
|
-
}
|
|
22
|
-
if (arg.startsWith("--")) {
|
|
23
|
-
const key = arg.slice(2);
|
|
24
|
-
const next = args[i + 1];
|
|
25
|
-
if (next && !next.startsWith("-")) {
|
|
26
|
-
options[key] = next;
|
|
27
|
-
i++;
|
|
28
|
-
} else options[key] = true;
|
|
29
|
-
} else if (arg.startsWith("-") && arg.length === 2) {
|
|
30
|
-
const key = arg.slice(1);
|
|
31
|
-
const next = args[i + 1];
|
|
32
|
-
if (next && !next.startsWith("-")) {
|
|
33
|
-
options[key] = next;
|
|
34
|
-
i++;
|
|
35
|
-
} else options[key] = true;
|
|
36
|
-
} else positional.push(arg);
|
|
37
|
-
}
|
|
38
|
-
return {
|
|
39
|
-
command,
|
|
40
|
-
positional,
|
|
41
|
-
options
|
|
42
|
-
};
|
|
43
|
-
}
|
|
44
|
-
/** Return a string option value, or undefined when absent or boolean. */
|
|
45
|
-
function getOption(options, name) {
|
|
46
|
-
const v = options[name];
|
|
47
|
-
return typeof v === "string" ? v : void 0;
|
|
48
|
-
}
|
|
49
|
-
/** Parse an integer option with fallback when absent or non-numeric. */
|
|
50
|
-
function getOptionInt(options, name, defaultValue) {
|
|
51
|
-
const v = getOption(options, name);
|
|
52
|
-
if (v === void 0) return defaultValue;
|
|
53
|
-
const n = Number.parseInt(v, 10);
|
|
54
|
-
if (!Number.isFinite(n)) return defaultValue;
|
|
55
|
-
return n;
|
|
56
|
-
}
|
|
57
|
-
/** True when a boolean flag is set or explicitly `"true"`. */
|
|
58
|
-
function hasOption(options, name) {
|
|
59
|
-
const v = options[name];
|
|
60
|
-
return v === true || typeof v === "string" && v === "true";
|
|
61
|
-
}
|
|
62
|
-
//#endregion
|
|
63
|
-
//#region src/cli/commands/envelope.ts
|
|
64
|
-
/**
|
|
65
|
-
* `harness-eval envelope` — build EvalRunEnvelope and interchange projections.
|
|
66
|
-
*
|
|
67
|
-
* Reads a suite run report (and optional grading JSON), builds a versioned
|
|
68
|
-
* {@link EvalRunEnvelope}, and serializes one of three projections:
|
|
69
|
-
*
|
|
70
|
-
* - `envelope` — full nested JSON document (default)
|
|
71
|
-
* - `trajectory` — JSONL of {@link EvalDatasetRow} per repetition
|
|
72
|
-
* - `instances` — JSONL of {@link InstancesJsonlRow} for Vertex batch upload
|
|
73
|
-
*
|
|
74
|
-
* Exit code 0 when behavioral pass, 1 when any cell failed assertions.
|
|
75
|
-
*/
|
|
76
|
-
const PROJECTIONS = /* @__PURE__ */ new Set([
|
|
77
|
-
"envelope",
|
|
78
|
-
"trajectory",
|
|
79
|
-
"instances"
|
|
80
|
-
]);
|
|
81
|
-
/**
|
|
82
|
-
* Parse and validate `--projection` CLI flag.
|
|
83
|
-
*
|
|
84
|
-
* @returns `"envelope"` when omitted; `undefined` when value is invalid.
|
|
85
|
-
*/
|
|
86
|
-
function parseEnvelopeProjection(value) {
|
|
87
|
-
if (value === void 0) return "envelope";
|
|
88
|
-
if (PROJECTIONS.has(value)) return value;
|
|
89
|
-
}
|
|
90
|
-
/**
|
|
91
|
-
* Serialize an envelope to stdout/file string for the chosen projection.
|
|
92
|
-
*
|
|
93
|
-
* Trajectory and instances projections emit NDJSON (one JSON object per line).
|
|
94
|
-
*/
|
|
95
|
-
function serializeEnvelopeProjection(envelope, projection) {
|
|
96
|
-
switch (projection) {
|
|
97
|
-
case "trajectory": return `${toTrajectory(envelope).map((row) => JSON.stringify(row)).join("\n")}\n`;
|
|
98
|
-
case "instances": return `${toInstancesJsonl(envelope).map((row) => JSON.stringify(row)).join("\n")}\n`;
|
|
99
|
-
default: return `${JSON.stringify(envelope, null, 2)}\n`;
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
/** Read harness-eval package version for envelope harness.frameworkVersion. */
|
|
103
|
-
async function readFrameworkVersion() {
|
|
104
|
-
try {
|
|
105
|
-
const text = await readFile(join(dirname(fileURLToPath(import.meta.url)), "../../../package.json"), "utf8");
|
|
106
|
-
return JSON.parse(text).version;
|
|
107
|
-
} catch {
|
|
108
|
-
return;
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
/**
|
|
112
|
-
* CLI entry point for the `envelope` subcommand.
|
|
113
|
-
*
|
|
114
|
-
* @returns Process exit code: 0 on behavioral pass, 1 on failure, 2 on usage/error.
|
|
115
|
-
*/
|
|
116
|
-
async function envelopeCommand(args) {
|
|
117
|
-
const reportPath = args.positional[0];
|
|
118
|
-
if (!reportPath) {
|
|
119
|
-
console.error("usage: harness-eval envelope <report.json> [--output path] [--grading path] [--suite path] [--projection envelope|trajectory|instances] [--include-raw-stream-events] [--no-transcript]");
|
|
120
|
-
return 2;
|
|
121
|
-
}
|
|
122
|
-
const outputPath = getOption(args.options, "output");
|
|
123
|
-
const gradingPath = getOption(args.options, "grading");
|
|
124
|
-
const suitePath = getOption(args.options, "suite");
|
|
125
|
-
const projection = parseEnvelopeProjection(getOption(args.options, "projection"));
|
|
126
|
-
if (!projection) {
|
|
127
|
-
console.error("invalid --projection; expected envelope, trajectory, or instances");
|
|
128
|
-
return 2;
|
|
129
|
-
}
|
|
130
|
-
let envelope;
|
|
131
|
-
try {
|
|
132
|
-
const frameworkVersion = await readFrameworkVersion();
|
|
133
|
-
envelope = await buildEvalRunEnvelopeFromFiles(reportPath, {
|
|
134
|
-
gradingPath,
|
|
135
|
-
suitePath,
|
|
136
|
-
includeTranscript: !hasOption(args.options, "no-transcript"),
|
|
137
|
-
includeRawStreamEvents: hasOption(args.options, "include-raw-stream-events"),
|
|
138
|
-
harness: { frameworkVersion }
|
|
139
|
-
});
|
|
140
|
-
} catch (err) {
|
|
141
|
-
console.error(err instanceof Error ? err.message : String(err));
|
|
142
|
-
return 2;
|
|
143
|
-
}
|
|
144
|
-
const serialized = serializeEnvelopeProjection(envelope, projection);
|
|
145
|
-
if (outputPath) await writeFile(outputPath, serialized, "utf8");
|
|
146
|
-
else process.stdout.write(serialized);
|
|
147
|
-
return envelope.summary.behavioralPass ? 0 : 1;
|
|
148
|
-
}
|
|
149
|
-
//#endregion
|
|
150
8
|
//#region src/cli/commands/format.ts
|
|
151
9
|
/**
|
|
152
10
|
* `harness-eval format` command.
|
|
@@ -507,10 +365,11 @@ function optionalOptionInt(options, name) {
|
|
|
507
365
|
async function gradeCommand(args) {
|
|
508
366
|
const reportPath = args.positional[0];
|
|
509
367
|
if (!reportPath) {
|
|
510
|
-
console.error("usage: harness-eval grade <report.json> [--config grading.yaml] [--expectations path] [--output path] [--model id] [--timeout-ms N] [--max-concurrent N]");
|
|
368
|
+
console.error("usage: harness-eval grade <report.json> [--config grading.yaml] [--suite suite.yaml] [--expectations path] [--output path] [--model id] [--timeout-ms N] [--max-concurrent N]");
|
|
511
369
|
return 2;
|
|
512
370
|
}
|
|
513
371
|
const configPath = getOption(args.options, "config");
|
|
372
|
+
const suitePath = getOption(args.options, "suite");
|
|
514
373
|
const expectationsPath = getOption(args.options, "expectations");
|
|
515
374
|
const outputPath = getOption(args.options, "output");
|
|
516
375
|
const model = getOption(args.options, "model");
|
|
@@ -521,8 +380,13 @@ async function gradeCommand(args) {
|
|
|
521
380
|
const progressMode = resolveProgressMode(args.options);
|
|
522
381
|
const useProgressColor = progressMode !== "json" && resolveProgressColor(args.options);
|
|
523
382
|
let fileConfig;
|
|
524
|
-
|
|
525
|
-
|
|
383
|
+
const gradingConfigPath = configPath ?? suitePath;
|
|
384
|
+
if (configPath && suitePath) {
|
|
385
|
+
console.error("grade: use only one of --config or --suite");
|
|
386
|
+
return 2;
|
|
387
|
+
}
|
|
388
|
+
if (gradingConfigPath) try {
|
|
389
|
+
fileConfig = await loadGradingConfig(gradingConfigPath);
|
|
526
390
|
} catch (err) {
|
|
527
391
|
console.error(err instanceof Error ? err.message : String(err));
|
|
528
392
|
return 2;
|
|
@@ -543,7 +407,7 @@ async function gradeCommand(args) {
|
|
|
543
407
|
binary,
|
|
544
408
|
timeoutMs,
|
|
545
409
|
maxConcurrent
|
|
546
|
-
}, configPath);
|
|
410
|
+
}, configPath ?? suitePath);
|
|
547
411
|
} catch (err) {
|
|
548
412
|
console.error(err instanceof Error ? err.message : String(err));
|
|
549
413
|
return 2;
|
|
@@ -570,6 +434,120 @@ async function gradeCommand(args) {
|
|
|
570
434
|
return gradingReportPassed(grading) ? 0 : 1;
|
|
571
435
|
}
|
|
572
436
|
//#endregion
|
|
437
|
+
//#region src/cli/commands/pipeline.ts
|
|
438
|
+
/**
|
|
439
|
+
* `harness-eval pipeline` — orchestrate run → grade → envelope from suite.yaml.
|
|
440
|
+
*/
|
|
441
|
+
/** Read package version for envelope provenance (best-effort). */
|
|
442
|
+
async function readFrameworkVersion() {
|
|
443
|
+
try {
|
|
444
|
+
const text = await readFile(join(dirname(fileURLToPath(import.meta.url)), "../../../package.json"), "utf8");
|
|
445
|
+
return JSON.parse(text).version;
|
|
446
|
+
} catch {
|
|
447
|
+
return;
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
/** Resolve CLI path overrides relative to the suite directory unless absolute or `~/`. */
|
|
451
|
+
function resolveOverridePath(value, suiteDir) {
|
|
452
|
+
if (!value) return void 0;
|
|
453
|
+
return isAbsolute(value) || value.startsWith("~/") ? value : join(suiteDir, value);
|
|
454
|
+
}
|
|
455
|
+
/**
|
|
456
|
+
* Execute `harness-eval pipeline`.
|
|
457
|
+
*
|
|
458
|
+
* @returns Step exit code (0 pass, 1 eval fail, 2 usage/load error).
|
|
459
|
+
*/
|
|
460
|
+
async function pipelineCommand(args) {
|
|
461
|
+
const suitePath = args.positional[0];
|
|
462
|
+
if (!suitePath) {
|
|
463
|
+
console.error("usage: harness-eval pipeline <suite.yaml|dir> [--steps run,grade,envelope] [--output path] [--grading path] [--report path] [--max-concurrent N] [--progress default|quiet|verbose|json]");
|
|
464
|
+
return 2;
|
|
465
|
+
}
|
|
466
|
+
let doc;
|
|
467
|
+
try {
|
|
468
|
+
doc = await loadSuiteDocument(suitePath);
|
|
469
|
+
} catch (err) {
|
|
470
|
+
console.error(err instanceof Error ? err.message : String(err));
|
|
471
|
+
return 2;
|
|
472
|
+
}
|
|
473
|
+
if (!doc.pipeline) {
|
|
474
|
+
console.error("suite.yaml has no pipeline block; use run, grade, and envelope commands separately");
|
|
475
|
+
return 2;
|
|
476
|
+
}
|
|
477
|
+
const suiteDir = suiteDirectoryFromPath(doc.suitePath);
|
|
478
|
+
const steps = getOption(args.options, "steps");
|
|
479
|
+
const maxConcurrent = getOptionInt(args.options, "max-concurrent", 4);
|
|
480
|
+
const progressMode = resolveProgressMode(args.options);
|
|
481
|
+
const useProgressColor = progressMode !== "json" && resolveProgressColor(args.options);
|
|
482
|
+
const projection = parseEnvelopeProjection(getOption(args.options, "projection"));
|
|
483
|
+
if (getOption(args.options, "projection") && !projection) {
|
|
484
|
+
console.error("invalid --projection; expected envelope, trajectory, or instances");
|
|
485
|
+
return 2;
|
|
486
|
+
}
|
|
487
|
+
const overrides = {};
|
|
488
|
+
const runOutput = getOption(args.options, "output");
|
|
489
|
+
if (runOutput) overrides.run = {
|
|
490
|
+
output: resolveOverridePath(runOutput, suiteDir),
|
|
491
|
+
maxConcurrent
|
|
492
|
+
};
|
|
493
|
+
const reportOverride = getOption(args.options, "report");
|
|
494
|
+
if (reportOverride) {
|
|
495
|
+
overrides.grade = {
|
|
496
|
+
...overrides.grade,
|
|
497
|
+
input: resolveOverridePath(reportOverride, suiteDir)
|
|
498
|
+
};
|
|
499
|
+
overrides.envelope = {
|
|
500
|
+
...overrides.envelope,
|
|
501
|
+
report: resolveOverridePath(reportOverride, suiteDir)
|
|
502
|
+
};
|
|
503
|
+
}
|
|
504
|
+
const gradingOutput = getOption(args.options, "grading-output");
|
|
505
|
+
if (gradingOutput) overrides.grade = {
|
|
506
|
+
...overrides.grade,
|
|
507
|
+
output: resolveOverridePath(gradingOutput, suiteDir)
|
|
508
|
+
};
|
|
509
|
+
const gradingInput = getOption(args.options, "grading");
|
|
510
|
+
if (gradingInput) overrides.envelope = {
|
|
511
|
+
...overrides.envelope,
|
|
512
|
+
grading: resolveOverridePath(gradingInput, suiteDir)
|
|
513
|
+
};
|
|
514
|
+
const envelopeOutput = getOption(args.options, "envelope-output");
|
|
515
|
+
if (envelopeOutput) overrides.envelope = {
|
|
516
|
+
...overrides.envelope,
|
|
517
|
+
output: resolveOverridePath(envelopeOutput, suiteDir)
|
|
518
|
+
};
|
|
519
|
+
if (projection) overrides.envelope = {
|
|
520
|
+
...overrides.envelope,
|
|
521
|
+
projection
|
|
522
|
+
};
|
|
523
|
+
if (doc.pipeline.grade && !doc.judge) {
|
|
524
|
+
console.error("pipeline grade step requires inline judge: block in suite.yaml");
|
|
525
|
+
return 2;
|
|
526
|
+
}
|
|
527
|
+
const frameworkVersion = await readFrameworkVersion();
|
|
528
|
+
try {
|
|
529
|
+
return (await runPipeline(doc, {
|
|
530
|
+
steps,
|
|
531
|
+
maxConcurrent,
|
|
532
|
+
overrides,
|
|
533
|
+
frameworkVersion,
|
|
534
|
+
onRunProgress: createRunProgressHandler({
|
|
535
|
+
mode: progressMode,
|
|
536
|
+
maxConcurrent,
|
|
537
|
+
color: useProgressColor
|
|
538
|
+
}),
|
|
539
|
+
onGradeProgress: createGradeProgressHandler({
|
|
540
|
+
mode: progressMode,
|
|
541
|
+
maxConcurrent,
|
|
542
|
+
color: useProgressColor
|
|
543
|
+
})
|
|
544
|
+
})).exitCode;
|
|
545
|
+
} catch (err) {
|
|
546
|
+
console.error(err instanceof Error ? err.message : String(err));
|
|
547
|
+
return 2;
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
//#endregion
|
|
573
551
|
//#region src/cli/commands/otel-output.ts
|
|
574
552
|
/**
|
|
575
553
|
* Write OTLP JSON artifacts from a suite report.
|
|
@@ -670,8 +648,9 @@ const USAGE = `harness-eval — harness-level eval framework
|
|
|
670
648
|
|
|
671
649
|
Usage:
|
|
672
650
|
harness-eval run <suite.yaml> [--max-concurrent N] [--baseline path] [--output path] [--otel-output dir] [--format console|markdown|json] [--adapter id] [--quiet] [--verbose] [--progress default|quiet|verbose|json]
|
|
673
|
-
harness-eval grade <report.json> [--config grading.yaml] [--expectations path] [--output path] [--model id] [--timeout-ms N] [--max-concurrent N] [--format console|json] [--quiet] [--verbose] [--progress default|quiet|verbose|json]
|
|
651
|
+
harness-eval grade <report.json> [--config grading.yaml] [--suite suite.yaml] [--expectations path] [--output path] [--model id] [--timeout-ms N] [--max-concurrent N] [--format console|json] [--quiet] [--verbose] [--progress default|quiet|verbose|json]
|
|
674
652
|
harness-eval envelope <report.json> [--output path] [--grading path] [--suite path] [--projection envelope|trajectory|instances] [--include-raw-stream-events] [--no-transcript]
|
|
653
|
+
harness-eval pipeline <suite.yaml|dir> [--steps run,grade,envelope] [--output path] [--grading path] [--grading-output path] [--envelope-output path] [--report path] [--projection envelope|trajectory|instances] [--max-concurrent N] [--progress default|quiet|verbose|json]
|
|
675
654
|
harness-eval format <report.json> [--format console|markdown|json] [--baseline path]
|
|
676
655
|
harness-eval --help
|
|
677
656
|
|
|
@@ -698,6 +677,7 @@ async function main(argv) {
|
|
|
698
677
|
case "run": return await runCommand(parsed);
|
|
699
678
|
case "grade": return await gradeCommand(parsed);
|
|
700
679
|
case "envelope": return await envelopeCommand(parsed);
|
|
680
|
+
case "pipeline": return await pipelineCommand(parsed);
|
|
701
681
|
case "format": return await formatCommand(parsed);
|
|
702
682
|
case void 0:
|
|
703
683
|
console.error(USAGE);
|
package/dist/cli/bin.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bin.js","names":[],"sources":["../../src/cli/args.ts","../../src/cli/commands/envelope.ts","../../src/cli/commands/format.ts","../../src/cli/progress.ts","../../src/cli/commands/grade.ts","../../src/cli/commands/otel-output.ts","../../src/cli/commands/run.ts","../../src/cli/main.ts","../../src/cli/bin.ts"],"sourcesContent":["/**\n * Minimal argv parser — no external deps.\n *\n * Parses `command`, positional args, and `--long` / `-s` flags. Boolean\n * flags omit a value; the next token starting with `-` is not consumed as\n * a value. Use `--` to pass through remaining tokens as positional.\n */\n\n/** Parsed CLI argv: optional subcommand, positional args, and flag map. */\nexport interface ParsedArgs {\n command?: string;\n positional: string[];\n options: Record<string, string | boolean>;\n}\n\n/** Parse process argv into command, positional args, and options. */\nexport function parseArgs(argv: string[]): ParsedArgs {\n const positional: string[] = [];\n const options: Record<string, string | boolean> = {};\n let command: string | undefined;\n\n const args = [...argv];\n if (args.length > 0 && !args[0].startsWith(\"-\")) {\n command = args.shift();\n }\n\n for (let i = 0; i < args.length; i++) {\n const arg = args[i];\n if (arg === \"--\") {\n positional.push(...args.slice(i + 1));\n break;\n }\n if (arg.startsWith(\"--\")) {\n const key = arg.slice(2);\n const next = args[i + 1];\n if (next && !next.startsWith(\"-\")) {\n options[key] = next;\n i++;\n } else {\n options[key] = true;\n }\n } else if (arg.startsWith(\"-\") && arg.length === 2) {\n const key = arg.slice(1);\n const next = args[i + 1];\n if (next && !next.startsWith(\"-\")) {\n options[key] = next;\n i++;\n } else {\n options[key] = true;\n }\n } else {\n positional.push(arg);\n }\n }\n\n return { command, positional, options };\n}\n\n/** Return a string option value, or undefined when absent or boolean. */\nexport function getOption(\n options: Record<string, string | boolean>,\n name: string,\n): string | undefined {\n const v = options[name];\n return typeof v === \"string\" ? v : undefined;\n}\n\n/** Parse an integer option with fallback when absent or non-numeric. */\nexport function getOptionInt(\n options: Record<string, string | boolean>,\n name: string,\n defaultValue: number,\n): number {\n const v = getOption(options, name);\n if (v === undefined) return defaultValue;\n const n = Number.parseInt(v, 10);\n if (!Number.isFinite(n)) return defaultValue;\n return n;\n}\n\n/** True when a boolean flag is set or explicitly `\"true\"`. */\nexport function hasOption(\n options: Record<string, string | boolean>,\n name: string,\n): boolean {\n const v = options[name];\n return v === true || (typeof v === \"string\" && v === \"true\");\n}\n","/**\n * `harness-eval envelope` — build EvalRunEnvelope and interchange projections.\n *\n * Reads a suite run report (and optional grading JSON), builds a versioned\n * {@link EvalRunEnvelope}, and serializes one of three projections:\n *\n * - `envelope` — full nested JSON document (default)\n * - `trajectory` — JSONL of {@link EvalDatasetRow} per repetition\n * - `instances` — JSONL of {@link InstancesJsonlRow} for Vertex batch upload\n *\n * Exit code 0 when behavioral pass, 1 when any cell failed assertions.\n */\n\nimport { readFile, writeFile } from \"node:fs/promises\";\nimport { dirname, join } from \"node:path\";\nimport { fileURLToPath } from \"node:url\";\n\nimport { buildEvalRunEnvelopeFromFiles } from \"../../eval-record/build\";\nimport {\n toInstancesJsonl,\n toTrajectory,\n} from \"../../eval-interchange/projections\";\nimport type { EvalRunEnvelope } from \"../../types/eval-record\";\nimport { getOption, hasOption, type ParsedArgs } from \"../args\";\n\n/** Supported `--projection` values for envelope output. */\nexport type EnvelopeProjection =\n | \"envelope\"\n | \"trajectory\"\n | \"instances\";\n\nconst PROJECTIONS = new Set<EnvelopeProjection>([\n \"envelope\",\n \"trajectory\",\n \"instances\",\n]);\n\n/**\n * Parse and validate `--projection` CLI flag.\n *\n * @returns `\"envelope\"` when omitted; `undefined` when value is invalid.\n */\nexport function parseEnvelopeProjection(\n value: string | undefined,\n): EnvelopeProjection | undefined {\n if (value === undefined) return \"envelope\";\n if (PROJECTIONS.has(value as EnvelopeProjection)) {\n return value as EnvelopeProjection;\n }\n return undefined;\n}\n\n/**\n * Serialize an envelope to stdout/file string for the chosen projection.\n *\n * Trajectory and instances projections emit NDJSON (one JSON object per line).\n */\nexport function serializeEnvelopeProjection(\n envelope: EvalRunEnvelope,\n projection: EnvelopeProjection,\n): string {\n switch (projection) {\n case \"trajectory\":\n return `${toTrajectory(envelope).map((row) => JSON.stringify(row)).join(\"\\n\")}\\n`;\n case \"instances\":\n return `${toInstancesJsonl(envelope).map((row) => JSON.stringify(row)).join(\"\\n\")}\\n`;\n case \"envelope\":\n default:\n return `${JSON.stringify(envelope, null, 2)}\\n`;\n }\n}\n\n/** Read harness-eval package version for envelope harness.frameworkVersion. */\nasync function readFrameworkVersion(): Promise<string | undefined> {\n try {\n const packagePath = join(\n dirname(fileURLToPath(import.meta.url)),\n \"../../../package.json\",\n );\n const text = await readFile(packagePath, \"utf8\");\n const pkg = JSON.parse(text) as { version?: string };\n return pkg.version;\n } catch {\n return undefined;\n }\n}\n\n/**\n * CLI entry point for the `envelope` subcommand.\n *\n * @returns Process exit code: 0 on behavioral pass, 1 on failure, 2 on usage/error.\n */\nexport async function envelopeCommand(args: ParsedArgs): Promise<number> {\n const reportPath = args.positional[0];\n if (!reportPath) {\n console.error(\n \"usage: harness-eval envelope <report.json> [--output path] [--grading path] [--suite path] [--projection envelope|trajectory|instances] [--include-raw-stream-events] [--no-transcript]\",\n );\n return 2;\n }\n\n const outputPath = getOption(args.options, \"output\");\n const gradingPath = getOption(args.options, \"grading\");\n const suitePath = getOption(args.options, \"suite\");\n const projection = parseEnvelopeProjection(\n getOption(args.options, \"projection\"),\n );\n\n if (!projection) {\n console.error(\n \"invalid --projection; expected envelope, trajectory, or instances\",\n );\n return 2;\n }\n\n let envelope: EvalRunEnvelope;\n try {\n const frameworkVersion = await readFrameworkVersion();\n envelope = await buildEvalRunEnvelopeFromFiles(reportPath, {\n gradingPath,\n suitePath,\n includeTranscript: !hasOption(args.options, \"no-transcript\"),\n includeRawStreamEvents: hasOption(args.options, \"include-raw-stream-events\"),\n harness: { frameworkVersion },\n });\n } catch (err) {\n console.error(err instanceof Error ? err.message : String(err));\n return 2;\n }\n\n const serialized = serializeEnvelopeProjection(envelope, projection);\n\n if (outputPath) {\n await writeFile(outputPath, serialized, \"utf8\");\n } else {\n process.stdout.write(serialized);\n }\n\n return envelope.summary.behavioralPass ? 0 : 1;\n}\n","/**\n * `harness-eval format` command.\n */\n\nimport { readFile } from \"node:fs/promises\";\n\nimport { formatReport } from \"../../reporter/index\";\nimport type { SuiteReport } from \"../../runner/types\";\nimport { getOption, type ParsedArgs } from \"../args\";\n\n/**\n * Execute `harness-eval format`: re-render a saved report JSON.\n *\n * @returns 0 when all cells pass, 1 otherwise, 2 on load errors.\n */\nexport async function formatCommand(args: ParsedArgs): Promise<number> {\n const reportPath = args.positional[0];\n if (!reportPath) {\n console.error(\"usage: harness-eval format <report.json> [options]\");\n return 2;\n }\n\n const format = getOption(args.options, \"format\") ?? \"console\";\n const baselinePath = getOption(args.options, \"baseline\");\n\n let report: SuiteReport;\n try {\n report = JSON.parse(await readFile(reportPath, \"utf8\")) as SuiteReport;\n } catch (err) {\n console.error(err instanceof Error ? err.message : String(err));\n return 2;\n }\n\n let baseline: SuiteReport | undefined;\n if (baselinePath) {\n baseline = JSON.parse(await readFile(baselinePath, \"utf8\")) as SuiteReport;\n }\n\n const formatted = formatReport(report, {\n format:\n format === \"markdown\" || format === \"json\" ? format : \"console\",\n baseline,\n color: format === \"console\",\n });\n\n process.stdout.write(formatted);\n if (!formatted.endsWith(\"\\n\")) process.stdout.write(\"\\n\");\n return report.cells.every((c) => c.passed) ? 0 : 1;\n}\n","/**\n * CLI progress reporting for long-running harness and grade commands.\n *\n * Progress writes to stderr by default so stdout remains free for report\n * output. Supports human-readable modes and newline-delimited JSON events.\n */\n\nimport type { Writable } from \"node:stream\";\n\nimport { getOption, hasOption } from \"./args\";\nimport type { GradeProgressEvent as GraderGradeProgressEvent } from \"../grader/types\";\nimport type { AssertionResult } from \"../types/assertions\";\nimport type { CellReport, ProgressCallback } from \"../runner/types\";\n\n/** Progress display mode for run and grade commands. */\nexport type ProgressMode = \"default\" | \"quiet\" | \"verbose\" | \"json\";\n\n/** ANSI SGR codes for progress output. Disabled when {@link resolveProgressColor} returns false. */\nconst GREEN = \"\\x1b[32m\";\nconst RED = \"\\x1b[31m\";\nconst YELLOW = \"\\x1b[33m\";\nconst DIM = \"\\x1b[2m\";\nconst RESET = \"\\x1b[0m\";\n\n/** Options for {@link createRunProgressHandler}. */\nexport interface RunProgressOptions {\n mode: ProgressMode;\n maxConcurrent?: number;\n color?: boolean;\n stream?: Writable;\n}\n\n/** Options for {@link createGradeProgressHandler}. */\nexport interface GradeProgressOptions {\n mode: ProgressMode;\n maxConcurrent?: number;\n color?: boolean;\n stream?: Writable;\n}\n\n/**\n * Resolve progress mode from `--progress`, `--quiet`, or `--verbose` flags.\n *\n * Explicit `--progress` wins; otherwise `--quiet` / `--verbose` map to modes.\n */\nexport function resolveProgressMode(\n options: Record<string, string | boolean>,\n): ProgressMode {\n const progress = getOption(options, \"progress\");\n if (\n progress === \"json\" ||\n progress === \"quiet\" ||\n progress === \"verbose\" ||\n progress === \"default\"\n ) {\n return progress;\n }\n if (hasOption(options, \"quiet\")) return \"quiet\";\n if (hasOption(options, \"verbose\")) return \"verbose\";\n return \"default\";\n}\n\n/**\n * Whether to emit ANSI colors on the progress stream (stderr).\n *\n * Precedence: `--no-color` → off; `--color` → on; `NO_COLOR` env → off;\n * `FORCE_COLOR` (non-zero) → on; otherwise TTY detection on `stream`.\n */\nexport function resolveProgressColor(\n options: Record<string, string | boolean>,\n stream: Writable = process.stderr,\n): boolean {\n if (hasOption(options, \"no-color\")) return false;\n if (hasOption(options, \"color\")) return true;\n if (process.env.NO_COLOR !== undefined && process.env.NO_COLOR !== \"\") {\n return false;\n }\n if (process.env.FORCE_COLOR !== undefined && process.env.FORCE_COLOR !== \"0\") {\n return true;\n }\n return (\n \"isTTY\" in stream &&\n (stream as { isTTY?: boolean }).isTTY === true\n );\n}\n\n/** Green checkmark prefix for per-rep success lines. */\nfunction okMark(color: boolean): string {\n return color ? `${GREEN}✓${RESET}` : \"✓\";\n}\n\n/** Red cross prefix for per-rep failure lines. */\nfunction failMark(color: boolean): string {\n return color ? `${RED}✗${RESET}` : \"✗\";\n}\n\n/** Inline lowercase status word for repetition rows. */\nfunction okStatus(color: boolean): string {\n return color ? `${GREEN}ok${RESET}` : \"ok\";\n}\n\n/** Inline uppercase status word for repetition failures. */\nfunction failStatus(color: boolean): string {\n return color ? `${RED}FAIL${RESET}` : \"FAIL\";\n}\n\n/** Uppercase cell-level pass label in {@link formatCellSummary}. */\nfunction passLabel(color: boolean): string {\n return color ? `${GREEN}PASS${RESET}` : \"PASS\";\n}\n\n/** Uppercase cell-level fail label in {@link formatCellSummary}. */\nfunction failLabel(color: boolean): string {\n return color ? `${RED}FAIL${RESET}` : \"FAIL\";\n}\n\n/**\n * Build a {@link ProgressCallback} for suite runs.\n *\n * Writes to `options.stream` (default stderr). JSON mode emits one event per line.\n */\nexport function createRunProgressHandler(\n options: RunProgressOptions,\n): ProgressCallback {\n const stream = options.stream ?? process.stderr;\n const mode = options.mode;\n const color = options.color ?? false;\n\n let totalReps = 0;\n let completed = 0;\n let totalDurationMs = 0;\n\n return (event) => {\n switch (event.kind) {\n case \"suite-start\":\n totalReps = event.totalReps;\n completed = 0;\n totalDurationMs = 0;\n if (mode === \"quiet\") return;\n if (mode === \"json\") {\n writeJson(stream, {\n kind: \"suite-start\",\n totalReps: event.totalReps,\n maxConcurrent: options.maxConcurrent,\n });\n return;\n }\n const concurrent =\n options.maxConcurrent !== undefined\n ? ` (max-concurrent ${options.maxConcurrent})`\n : \"\";\n stream.write(`Running ${totalReps} repetitions${concurrent}...\\n\\n`);\n break;\n\n case \"rep-complete\":\n completed++;\n totalDurationMs += event.durationMs;\n if (mode === \"quiet\") {\n stream.write(event.ok ? (color ? `${GREEN}.${RESET}` : \".\") : (color ? `${RED}x${RESET}` : \"x\"));\n return;\n }\n if (mode === \"json\") {\n writeJson(stream, {\n kind: \"rep-complete\",\n index: completed,\n total: totalReps,\n caseId: event.caseId,\n cellLabel: event.cellLabel,\n repIndex: event.repIndex,\n ok: event.ok,\n durationMs: event.durationMs,\n toolCallCount: event.toolCallCount,\n errorMessage: event.errorMessage,\n });\n return;\n }\n\n const eta = formatEta(totalDurationMs, completed, totalReps);\n const icon = event.ok ? okMark(color) : failMark(color);\n const status = event.ok ? okStatus(color) : failStatus(color);\n let line = `${icon} [${completed}/${totalReps}] ${event.caseId} @ ${event.cellLabel} #${event.repIndex} ${status} ${formatDuration(event.durationMs)}`;\n if (eta) {\n line += color\n ? ` ${DIM}(${eta})${RESET}`\n : ` (${eta})`;\n }\n if (!event.ok && event.errorMessage) {\n line += color\n ? ` ${YELLOW}— ${truncate(event.errorMessage, 80)}${RESET}`\n : ` — ${truncate(event.errorMessage, 80)}`;\n }\n if (mode === \"verbose\") {\n if (event.toolCallCount !== undefined) {\n line += ` tools=${event.toolCallCount}`;\n }\n const summary = formatAssertionSummary(event.assertionResults, color);\n if (summary) line += ` ${summary}`;\n }\n stream.write(`${line}\\n`);\n break;\n\n case \"cell-complete\":\n if (mode === \"quiet\") return;\n if (mode === \"json\") {\n writeJson(stream, {\n kind: \"cell-complete\",\n caseId: event.report.caseId,\n cellLabel: event.report.cell.label,\n passed: event.report.passed,\n adapterErrors: event.report.adapterErrors,\n assertionStats: event.report.assertionStats.map((s) => ({\n description: s.description,\n passRate: s.passRate,\n meetsThreshold: s.meetsThreshold,\n })),\n });\n return;\n }\n stream.write(`${formatCellSummary(event.report, color)}\\n`);\n break;\n\n case \"suite-complete\":\n if (mode === \"quiet\") {\n stream.write(\"\\n\");\n return;\n }\n if (mode === \"json\") {\n writeJson(stream, {\n kind: \"suite-complete\",\n durationMs: event.report.durationMs,\n cellsTotal: event.report.cells.length,\n cellsPassed: event.report.cells.filter((c) => c.passed).length,\n });\n return;\n }\n const okReps = event.report.cells.reduce(\n (n, c) => n + c.repetitions.filter((r) => r.error === null).length,\n 0,\n );\n const totalRun = event.report.cells.reduce(\n (n, c) => n + c.repetitions.length,\n 0,\n );\n const adapterErrors = event.report.cells.reduce(\n (n, c) => n + c.adapterErrors,\n 0,\n );\n let footer = `\\nFinished in ${formatDuration(event.report.durationMs)} (${okReps}/${totalRun} reps ok`;\n if (adapterErrors > 0) {\n footer += `, ${adapterErrors} adapter error(s)`;\n }\n footer += \")\\n\\n\";\n stream.write(footer);\n break;\n\n default:\n break;\n }\n };\n}\n\n/** Build a progress handler for outcome grading ({@link GradeProgressEvent}). */\nexport function createGradeProgressHandler(\n options: GradeProgressOptions,\n): (event: GraderGradeProgressEvent) => void {\n const stream = options.stream ?? process.stderr;\n const mode = options.mode;\n const color = options.color ?? false;\n\n let total = 0;\n let completed = 0;\n let totalDurationMs = 0;\n\n return (event) => {\n switch (event.kind) {\n case \"grade-start\":\n total = event.total;\n completed = 0;\n totalDurationMs = 0;\n if (mode === \"quiet\" || total === 0) return;\n if (mode === \"json\") {\n writeJson(stream, {\n kind: \"grade-start\",\n total: event.total,\n maxConcurrent: options.maxConcurrent,\n });\n return;\n }\n const concurrent =\n options.maxConcurrent !== undefined\n ? ` (max-concurrent ${options.maxConcurrent})`\n : \"\";\n stream.write(\n `Grading ${total} repetition(s)${concurrent}...\\n\\n`,\n );\n break;\n\n case \"grade-complete\":\n completed++;\n totalDurationMs += event.durationMs;\n if (mode === \"quiet\") {\n const allPassed = event.failed === 0 && !event.graderError;\n stream.write(\n allPassed\n ? color ? `${GREEN}.${RESET}` : \".\"\n : color ? `${RED}x${RESET}` : \"x\",\n );\n return;\n }\n if (mode === \"json\") {\n writeJson(stream, {\n kind: \"grade-complete\",\n index: completed,\n total,\n caseId: event.caseId,\n cellLabel: event.cellLabel,\n repetitionIndex: event.repetitionIndex,\n passed: event.passed,\n failed: event.failed,\n durationMs: event.durationMs,\n graderError: event.graderError,\n });\n return;\n }\n\n const eta = formatEta(totalDurationMs, completed, total);\n const ok = event.failed === 0 && !event.graderError;\n const icon = ok ? okMark(color) : failMark(color);\n const status = ok ? okStatus(color) : failStatus(color);\n let line = `${icon} [${completed}/${total}] ${event.caseId} @ ${event.cellLabel} #${event.repetitionIndex} ${status} ${formatDuration(event.durationMs)}`;\n line += ` expectations ${event.passed}/${event.passed + event.failed}`;\n if (eta) {\n line += color ? ` ${DIM}(${eta})${RESET}` : ` (${eta})`;\n }\n if (event.graderError) {\n line += color\n ? ` ${YELLOW}— ${truncate(event.graderError, 80)}${RESET}`\n : ` — ${truncate(event.graderError, 80)}`;\n }\n if (mode === \"verbose\" && event.failed && event.failed > 0) {\n line += color ? ` ${YELLOW}see grading output${RESET}` : \" see grading output\";\n }\n stream.write(`${line}\\n`);\n break;\n\n case \"grade-done\":\n if (mode === \"quiet\") {\n stream.write(\"\\n\");\n return;\n }\n if (mode === \"json\") {\n writeJson(stream, {\n kind: \"grade-done\",\n durationMs: event.durationMs,\n totalExpectations: event.totalExpectations,\n passedExpectations: event.passedExpectations,\n });\n return;\n }\n if (total === 0) return;\n stream.write(\n `\\nGraded in ${formatDuration(event.durationMs)} (${event.passedExpectations}/${event.totalExpectations} expectations passed)\\n\\n`,\n );\n break;\n\n default:\n break;\n }\n };\n}\n\n/**\n * Write one NDJSON progress event line to the progress stream.\n *\n * JSON mode keeps stdout clean for machine-readable reports while still\n * exposing structured progress for CI log parsers.\n */\nfunction writeJson(stream: Writable, value: unknown): void {\n stream.write(`${JSON.stringify(value)}\\n`);\n}\n\n/** Format milliseconds as a human-readable duration string. */\nexport function formatDuration(ms: number): string {\n if (ms < 1000) return `${ms}ms`;\n const sec = ms / 1000;\n if (sec < 60) return `${sec.toFixed(1)}s`;\n const min = Math.floor(sec / 60);\n const remSec = Math.round(sec % 60);\n if (min < 60) return `${min}m ${remSec}s`;\n const hr = Math.floor(min / 60);\n const remMin = min % 60;\n return `${hr}h ${remMin}m`;\n}\n\n/**\n * Estimate remaining time from average completed rep duration.\n *\n * Uses a simple running mean — good enough for long suites without storing\n * per-rep history. Returns `undefined` at start and when all reps are done.\n */\nfunction formatEta(\n totalDurationMs: number,\n completed: number,\n total: number,\n): string | undefined {\n if (completed === 0 || completed >= total) return undefined;\n const avg = totalDurationMs / completed;\n const remaining = (total - completed) * avg;\n return `~${formatDuration(Math.round(remaining))} remaining`;\n}\n\n/** Truncate error text for single-line progress rows (Unicode ellipsis). */\nfunction truncate(text: string, max: number): string {\n if (text.length <= max) return text;\n return `${text.slice(0, max - 1)}…`;\n}\n\n/**\n * Compact per-assertion pass/fail summary for `--progress verbose` rep lines.\n *\n * @returns Comma-separated `✓ description` / `✗ description` fragments, or empty string.\n */\nfunction formatAssertionSummary(\n results?: AssertionResult[],\n color = false,\n): string {\n if (!results || results.length === 0) return \"\";\n return results\n .map((r) =>\n `${r.passed ? okMark(color) : failMark(color)} ${r.description}`,\n )\n .join(\", \");\n}\n\n/** One-line summary when a matrix cell finishes (used in default progress mode). */\nexport function formatCellSummary(cell: CellReport, color: boolean): string {\n const mark = cell.passed ? okMark(color) : failMark(color);\n const status = cell.passed ? passLabel(color) : failLabel(color);\n const parts = cell.assertionStats.map((s) => {\n const pct = (s.passRate * 100).toFixed(0);\n return `${s.description} ${s.passedCount}/${s.evaluatedCount} (${pct}%)`;\n });\n const crash =\n cell.adapterErrors > 0\n ? color\n ? ` ${YELLOW}[${cell.adapterErrors} adapter errors]${RESET}`\n : ` [${cell.adapterErrors} adapter errors]`\n : \"\";\n const stats = parts.length > 0 ? ` ${parts.join(\" · \")}` : \"\";\n return `${mark} ${cell.caseId} @ ${cell.cell.label} ${status}${crash}${stats}`;\n}\n","/**\n * `harness-eval grade` — LLM outcome grading on a suite report.\n */\n\nimport { writeFile } from \"node:fs/promises\";\n\nimport { loadGradingConfig } from \"../../config/grading-loader\";\nimport {\n formatGradingConsole,\n gradeReport,\n gradingReportPassed,\n loadSuiteReport,\n resolveGradeOptions,\n} from \"../../grader/index\";\nimport { getOption, getOptionInt, type ParsedArgs } from \"../args\";\nimport {\n createGradeProgressHandler,\n resolveProgressColor,\n resolveProgressMode,\n} from \"../progress\";\n\n/** Parse an optional integer CLI flag; returns undefined when absent or invalid. */\nfunction optionalOptionInt(\n options: Record<string, string | boolean>,\n name: string,\n): number | undefined {\n const raw = getOption(options, name);\n if (raw === undefined) return undefined;\n const n = Number.parseInt(raw, 10);\n return Number.isFinite(n) ? n : undefined;\n}\n\n/**\n * Execute `harness-eval grade`: LLM outcome grading on a suite report JSON.\n *\n * @returns 0 when all expectations pass, 1 on failure, 2 on usage/load errors or no reps graded.\n */\nexport async function gradeCommand(args: ParsedArgs): Promise<number> {\n const reportPath = args.positional[0];\n if (!reportPath) {\n console.error(\n \"usage: harness-eval grade <report.json> [--config grading.yaml] [--expectations path] [--output path] [--model id] [--timeout-ms N] [--max-concurrent N]\",\n );\n return 2;\n }\n\n const configPath = getOption(args.options, \"config\");\n const expectationsPath = getOption(args.options, \"expectations\");\n const outputPath = getOption(args.options, \"output\");\n const model = getOption(args.options, \"model\");\n const binary = getOption(args.options, \"binary\");\n const timeoutMs = optionalOptionInt(args.options, \"timeout-ms\");\n const maxConcurrentRaw = getOption(args.options, \"max-concurrent\");\n const maxConcurrent = maxConcurrentRaw\n ? getOptionInt(args.options, \"max-concurrent\", 2)\n : undefined;\n const format = getOption(args.options, \"format\") ?? \"console\";\n const progressMode = resolveProgressMode(args.options);\n const useProgressColor =\n progressMode !== \"json\" && resolveProgressColor(args.options);\n\n let fileConfig;\n if (configPath) {\n try {\n fileConfig = await loadGradingConfig(configPath);\n } catch (err) {\n console.error(err instanceof Error ? err.message : String(err));\n return 2;\n }\n }\n\n let report;\n try {\n report = await loadSuiteReport(reportPath);\n } catch (err) {\n console.error(err instanceof Error ? err.message : String(err));\n return 2;\n }\n\n let gradeOptions;\n try {\n gradeOptions = resolveGradeOptions(\n fileConfig,\n {\n sourceReport: reportPath,\n expectationsPath,\n model,\n binary,\n timeoutMs,\n maxConcurrent,\n },\n configPath,\n );\n } catch (err) {\n console.error(err instanceof Error ? err.message : String(err));\n return 2;\n }\n\n const onProgress = createGradeProgressHandler({\n mode: progressMode,\n maxConcurrent: gradeOptions.maxConcurrent ?? 2,\n color: useProgressColor,\n });\n\n const grading = await gradeReport(report, {\n ...gradeOptions,\n onProgress,\n });\n\n if (outputPath) {\n await writeFile(outputPath, JSON.stringify(grading, null, 2), \"utf8\");\n }\n\n if (format === \"json\") {\n process.stdout.write(JSON.stringify(grading, null, 2));\n process.stdout.write(\"\\n\");\n } else {\n const formatted = formatGradingConsole(grading, format === \"console\");\n process.stdout.write(formatted);\n if (!formatted.endsWith(\"\\n\")) process.stdout.write(\"\\n\");\n }\n\n if (grading.results.length === 0) {\n return 2;\n }\n\n return gradingReportPassed(grading) ? 0 : 1;\n}\n","/**\n * Write OTLP JSON artifacts from a suite report.\n */\n\nimport { mkdir, writeFile } from \"node:fs/promises\";\nimport { join } from \"node:path\";\n\nimport { trajectoryToOtlp } from \"../../otel/emitter\";\nimport type { SuiteReport, TestSuite } from \"../../runner/types\";\n\n/** Sanitize case/cell labels for use in OTLP artifact filenames. */\nfunction safeFilePart(value: string): string {\n return value.replace(/[^a-zA-Z0-9._-]+/g, \"_\");\n}\n\n/**\n * Write one OTLP JSON file per successful repetition.\n *\n * Files: `{caseId}__{cellLabel}__rep{N}.otlp.json`\n */\nexport async function writeOtelArtifacts(\n suite: TestSuite,\n report: SuiteReport,\n outputDir: string,\n): Promise<number> {\n await mkdir(outputDir, { recursive: true });\n\n let written = 0;\n for (const cellReport of report.cells) {\n const testCase = suite.cases.find((c) => c.id === cellReport.caseId);\n if (!testCase) continue;\n\n for (const rep of cellReport.repetitions) {\n if (!rep.adapterResult) continue;\n\n const otlp = trajectoryToOtlp(rep.adapterResult.view, {\n prompt: testCase.prompt,\n });\n\n const filename = `${safeFilePart(cellReport.caseId)}__${safeFilePart(\n cellReport.cell.label,\n )}__rep${rep.repetitionIndex}.otlp.json`;\n\n await writeFile(\n join(outputDir, filename),\n JSON.stringify(otlp, null, 2),\n \"utf8\",\n );\n written++;\n }\n }\n\n return written;\n}\n","/**\n * `harness-eval run` command.\n */\n\nimport { writeFile } from \"node:fs/promises\";\n\nimport { getAdapter } from \"../../adapters/registry\";\nimport { loadSuite } from \"../../config/loader\";\nimport { formatReport } from \"../../reporter/index\";\nimport { runSuite } from \"../../runner/suite\";\nimport type { SuiteReport } from \"../../runner/types\";\nimport { getOption, getOptionInt, type ParsedArgs } from \"../args\";\nimport {\n createRunProgressHandler,\n resolveProgressColor,\n resolveProgressMode,\n} from \"../progress\";\nimport { writeOtelArtifacts } from \"./otel-output\";\n\n/**\n * Execute `harness-eval run`: load suite, run repetitions, format report.\n *\n * @returns 0 when all cells pass thresholds, 1 on assertion failure, 2 on usage/load errors.\n */\nexport async function runCommand(args: ParsedArgs): Promise<number> {\n const suitePath = args.positional[0];\n if (!suitePath) {\n console.error(\"usage: harness-eval run <suite.yaml> [options]\");\n return 2;\n }\n\n const format = getOption(args.options, \"format\") ?? \"console\";\n const outputPath = getOption(args.options, \"output\");\n const otelOutputDir = getOption(args.options, \"otel-output\");\n const baselinePath = getOption(args.options, \"baseline\");\n const maxConcurrent = getOptionInt(args.options, \"max-concurrent\", 4);\n const adapterId = getOption(args.options, \"adapter\");\n const progressMode = resolveProgressMode(args.options);\n const useProgressColor =\n progressMode !== \"json\" && resolveProgressColor(args.options);\n\n let suite;\n try {\n suite = await loadSuite(suitePath);\n } catch (err) {\n console.error(err instanceof Error ? err.message : String(err));\n return 2;\n }\n\n const adapter = getAdapter(adapterId ?? suite.adapter ?? \"claude-code\");\n\n const onProgress = createRunProgressHandler({\n mode: progressMode,\n maxConcurrent,\n color: useProgressColor,\n });\n\n const report = await runSuite(suite, {\n adapter,\n maxConcurrent,\n onProgress,\n });\n\n if (outputPath) {\n await writeFile(outputPath, JSON.stringify(report, null, 2), \"utf8\");\n }\n\n if (otelOutputDir) {\n const count = await writeOtelArtifacts(suite, report, otelOutputDir);\n process.stderr.write(`otel: wrote ${count} trace file(s) to ${otelOutputDir}\\n`);\n }\n\n let baseline: SuiteReport | undefined;\n if (baselinePath) {\n const { readFile } = await import(\"node:fs/promises\");\n baseline = JSON.parse(await readFile(baselinePath, \"utf8\")) as SuiteReport;\n }\n\n const formatted = formatReport(report, {\n format:\n format === \"markdown\" || format === \"json\" ? format : \"console\",\n baseline,\n color: format === \"console\",\n });\n\n process.stdout.write(formatted);\n if (!formatted.endsWith(\"\\n\")) process.stdout.write(\"\\n\");\n\n return report.cells.every((c) => c.passed) ? 0 : 1;\n}\n","/**\n * CLI entry point — dispatches subcommands and prints usage on `--help`.\n *\n * Exit codes: 0 success, 1 eval/grade failure, 2 usage or load errors.\n */\n\nimport { envelopeCommand } from \"./commands/envelope\";\nimport { formatCommand } from \"./commands/format\";\nimport { gradeCommand } from \"./commands/grade\";\nimport { runCommand } from \"./commands/run\";\nimport { parseArgs } from \"./args\";\n\nconst USAGE = `harness-eval — harness-level eval framework\n\nUsage:\n harness-eval run <suite.yaml> [--max-concurrent N] [--baseline path] [--output path] [--otel-output dir] [--format console|markdown|json] [--adapter id] [--quiet] [--verbose] [--progress default|quiet|verbose|json]\n harness-eval grade <report.json> [--config grading.yaml] [--expectations path] [--output path] [--model id] [--timeout-ms N] [--max-concurrent N] [--format console|json] [--quiet] [--verbose] [--progress default|quiet|verbose|json]\n harness-eval envelope <report.json> [--output path] [--grading path] [--suite path] [--projection envelope|trajectory|instances] [--include-raw-stream-events] [--no-transcript]\n harness-eval format <report.json> [--format console|markdown|json] [--baseline path]\n harness-eval --help\n\n Progress (run & grade):\n default one line per repetition + per-cell summary (default)\n --quiet colored dots (. = ok, x = fail)\n --verbose per-rep details (tool counts, assertion summary)\n --progress json newline-delimited JSON events on stderr\n --no-color disable ANSI colors on progress output\n --color force ANSI colors on progress output\n`;\n\n/**\n * Route argv to the appropriate subcommand handler.\n *\n * @returns Process exit code (0 = success, 1 = eval failure, 2 = usage error).\n */\nexport async function main(argv: string[]): Promise<number> {\n const parsed = parseArgs(argv);\n\n if (parsed.options.help || parsed.command === \"help\" || parsed.options.h) {\n process.stdout.write(USAGE);\n return 0;\n }\n\n switch (parsed.command) {\n case \"run\":\n return await runCommand(parsed);\n case \"grade\":\n return await gradeCommand(parsed);\n case \"envelope\":\n return await envelopeCommand(parsed);\n case \"format\":\n return await formatCommand(parsed);\n case undefined:\n console.error(USAGE);\n return 2;\n default:\n console.error(`unknown command: ${parsed.command}\\n\\n${USAGE}`);\n return 2;\n }\n}\n","#!/usr/bin/env node\n/**\n * CLI executable entry point — delegates to {@link main} and exits with its code.\n */\n\nimport { main } from \"./main\";\n\nconst code = await main(process.argv.slice(2));\nprocess.exit(code);\n"],"mappings":";;;;;;;;;AAgBA,SAAgB,UAAU,MAA4B;CACpD,MAAM,aAAuB,CAAC;CAC9B,MAAM,UAA4C,CAAC;CACnD,IAAI;CAEJ,MAAM,OAAO,CAAC,GAAG,IAAI;CACrB,IAAI,KAAK,SAAS,KAAK,CAAC,KAAK,EAAE,CAAC,WAAW,GAAG,GAC5C,UAAU,KAAK,MAAM;CAGvB,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,MAAM,KAAK;EACjB,IAAI,QAAQ,MAAM;GAChB,WAAW,KAAK,GAAG,KAAK,MAAM,IAAI,CAAC,CAAC;GACpC;EACF;EACA,IAAI,IAAI,WAAW,IAAI,GAAG;GACxB,MAAM,MAAM,IAAI,MAAM,CAAC;GACvB,MAAM,OAAO,KAAK,IAAI;GACtB,IAAI,QAAQ,CAAC,KAAK,WAAW,GAAG,GAAG;IACjC,QAAQ,OAAO;IACf;GACF,OACE,QAAQ,OAAO;EAEnB,OAAO,IAAI,IAAI,WAAW,GAAG,KAAK,IAAI,WAAW,GAAG;GAClD,MAAM,MAAM,IAAI,MAAM,CAAC;GACvB,MAAM,OAAO,KAAK,IAAI;GACtB,IAAI,QAAQ,CAAC,KAAK,WAAW,GAAG,GAAG;IACjC,QAAQ,OAAO;IACf;GACF,OACE,QAAQ,OAAO;EAEnB,OACE,WAAW,KAAK,GAAG;CAEvB;CAEA,OAAO;EAAE;EAAS;EAAY;CAAQ;AACxC;;AAGA,SAAgB,UACd,SACA,MACoB;CACpB,MAAM,IAAI,QAAQ;CAClB,OAAO,OAAO,MAAM,WAAW,IAAI,KAAA;AACrC;;AAGA,SAAgB,aACd,SACA,MACA,cACQ;CACR,MAAM,IAAI,UAAU,SAAS,IAAI;CACjC,IAAI,MAAM,KAAA,GAAW,OAAO;CAC5B,MAAM,IAAI,OAAO,SAAS,GAAG,EAAE;CAC/B,IAAI,CAAC,OAAO,SAAS,CAAC,GAAG,OAAO;CAChC,OAAO;AACT;;AAGA,SAAgB,UACd,SACA,MACS;CACT,MAAM,IAAI,QAAQ;CAClB,OAAO,MAAM,QAAS,OAAO,MAAM,YAAY,MAAM;AACvD;;;;;;;;;;;;;;;ACxDA,MAAM,8BAAc,IAAI,IAAwB;CAC9C;CACA;CACA;AACF,CAAC;;;;;;AAOD,SAAgB,wBACd,OACgC;CAChC,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,IAAI,YAAY,IAAI,KAA2B,GAC7C,OAAO;AAGX;;;;;;AAOA,SAAgB,4BACd,UACA,YACQ;CACR,QAAQ,YAAR;EACE,KAAK,cACH,OAAO,GAAG,aAAa,QAAQ,CAAC,CAAC,KAAK,QAAQ,KAAK,UAAU,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE;EAChF,KAAK,aACH,OAAO,GAAG,iBAAiB,QAAQ,CAAC,CAAC,KAAK,QAAQ,KAAK,UAAU,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE;EAEpF,SACE,OAAO,GAAG,KAAK,UAAU,UAAU,MAAM,CAAC,EAAE;CAChD;AACF;;AAGA,eAAe,uBAAoD;CACjE,IAAI;EAKF,MAAM,OAAO,MAAM,SAJC,KAClB,QAAQ,cAAc,OAAO,KAAK,GAAG,CAAC,GACtC,uBAEoC,GAAG,MAAM;EAE/C,OADY,KAAK,MAAM,IACd,CAAC,CAAC;CACb,QAAQ;EACN;CACF;AACF;;;;;;AAOA,eAAsB,gBAAgB,MAAmC;CACvE,MAAM,aAAa,KAAK,WAAW;CACnC,IAAI,CAAC,YAAY;EACf,QAAQ,MACN,yLACF;EACA,OAAO;CACT;CAEA,MAAM,aAAa,UAAU,KAAK,SAAS,QAAQ;CACnD,MAAM,cAAc,UAAU,KAAK,SAAS,SAAS;CACrD,MAAM,YAAY,UAAU,KAAK,SAAS,OAAO;CACjD,MAAM,aAAa,wBACjB,UAAU,KAAK,SAAS,YAAY,CACtC;CAEA,IAAI,CAAC,YAAY;EACf,QAAQ,MACN,mEACF;EACA,OAAO;CACT;CAEA,IAAI;CACJ,IAAI;EACF,MAAM,mBAAmB,MAAM,qBAAqB;EACpD,WAAW,MAAM,8BAA8B,YAAY;GACzD;GACA;GACA,mBAAmB,CAAC,UAAU,KAAK,SAAS,eAAe;GAC3D,wBAAwB,UAAU,KAAK,SAAS,2BAA2B;GAC3E,SAAS,EAAE,iBAAiB;EAC9B,CAAC;CACH,SAAS,KAAK;EACZ,QAAQ,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;EAC9D,OAAO;CACT;CAEA,MAAM,aAAa,4BAA4B,UAAU,UAAU;CAEnE,IAAI,YACF,MAAM,UAAU,YAAY,YAAY,MAAM;MAE9C,QAAQ,OAAO,MAAM,UAAU;CAGjC,OAAO,SAAS,QAAQ,iBAAiB,IAAI;AAC/C;;;;;;;;;;;AC5HA,eAAsB,cAAc,MAAmC;CACrE,MAAM,aAAa,KAAK,WAAW;CACnC,IAAI,CAAC,YAAY;EACf,QAAQ,MAAM,oDAAoD;EAClE,OAAO;CACT;CAEA,MAAM,SAAS,UAAU,KAAK,SAAS,QAAQ,KAAK;CACpD,MAAM,eAAe,UAAU,KAAK,SAAS,UAAU;CAEvD,IAAI;CACJ,IAAI;EACF,SAAS,KAAK,MAAM,MAAM,SAAS,YAAY,MAAM,CAAC;CACxD,SAAS,KAAK;EACZ,QAAQ,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;EAC9D,OAAO;CACT;CAEA,IAAI;CACJ,IAAI,cACF,WAAW,KAAK,MAAM,MAAM,SAAS,cAAc,MAAM,CAAC;CAG5D,MAAM,YAAY,aAAa,QAAQ;EACrC,QACE,WAAW,cAAc,WAAW,SAAS,SAAS;EACxD;EACA,OAAO,WAAW;CACpB,CAAC;CAED,QAAQ,OAAO,MAAM,SAAS;CAC9B,IAAI,CAAC,UAAU,SAAS,IAAI,GAAG,QAAQ,OAAO,MAAM,IAAI;CACxD,OAAO,OAAO,MAAM,OAAO,MAAM,EAAE,MAAM,IAAI,IAAI;AACnD;;;;AC9BA,MAAM,QAAQ;AACd,MAAM,MAAM;AACZ,MAAM,SAAS;AACf,MAAM,MAAM;AACZ,MAAM,QAAQ;;;;;;AAuBd,SAAgB,oBACd,SACc;CACd,MAAM,WAAW,UAAU,SAAS,UAAU;CAC9C,IACE,aAAa,UACb,aAAa,WACb,aAAa,aACb,aAAa,WAEb,OAAO;CAET,IAAI,UAAU,SAAS,OAAO,GAAG,OAAO;CACxC,IAAI,UAAU,SAAS,SAAS,GAAG,OAAO;CAC1C,OAAO;AACT;;;;;;;AAQA,SAAgB,qBACd,SACA,SAAmB,QAAQ,QAClB;CACT,IAAI,UAAU,SAAS,UAAU,GAAG,OAAO;CAC3C,IAAI,UAAU,SAAS,OAAO,GAAG,OAAO;CACxC,IAAI,QAAQ,IAAI,aAAa,KAAA,KAAa,QAAQ,IAAI,aAAa,IACjE,OAAO;CAET,IAAI,QAAQ,IAAI,gBAAgB,KAAA,KAAa,QAAQ,IAAI,gBAAgB,KACvE,OAAO;CAET,OACE,WAAW,UACV,OAA+B,UAAU;AAE9C;;AAGA,SAAS,OAAO,OAAwB;CACtC,OAAO,QAAQ,GAAG,MAAM,GAAG,UAAU;AACvC;;AAGA,SAAS,SAAS,OAAwB;CACxC,OAAO,QAAQ,GAAG,IAAI,GAAG,UAAU;AACrC;;AAGA,SAAS,SAAS,OAAwB;CACxC,OAAO,QAAQ,GAAG,MAAM,IAAI,UAAU;AACxC;;AAGA,SAAS,WAAW,OAAwB;CAC1C,OAAO,QAAQ,GAAG,IAAI,MAAM,UAAU;AACxC;;AAGA,SAAS,UAAU,OAAwB;CACzC,OAAO,QAAQ,GAAG,MAAM,MAAM,UAAU;AAC1C;;AAGA,SAAS,UAAU,OAAwB;CACzC,OAAO,QAAQ,GAAG,IAAI,MAAM,UAAU;AACxC;;;;;;AAOA,SAAgB,yBACd,SACkB;CAClB,MAAM,SAAS,QAAQ,UAAU,QAAQ;CACzC,MAAM,OAAO,QAAQ;CACrB,MAAM,QAAQ,QAAQ,SAAS;CAE/B,IAAI,YAAY;CAChB,IAAI,YAAY;CAChB,IAAI,kBAAkB;CAEtB,QAAQ,UAAU;EAChB,QAAQ,MAAM,MAAd;GACE,KAAK;IACH,YAAY,MAAM;IAClB,YAAY;IACZ,kBAAkB;IAClB,IAAI,SAAS,SAAS;IACtB,IAAI,SAAS,QAAQ;KACnB,UAAU,QAAQ;MAChB,MAAM;MACN,WAAW,MAAM;MACjB,eAAe,QAAQ;KACzB,CAAC;KACD;IACF;IACA,MAAM,aACJ,QAAQ,kBAAkB,KAAA,IACtB,oBAAoB,QAAQ,cAAc,KAC1C;IACN,OAAO,MAAM,WAAW,UAAU,cAAc,WAAW,QAAQ;IACnE;GAEF,KAAK;IACH;IACA,mBAAmB,MAAM;IACzB,IAAI,SAAS,SAAS;KACpB,OAAO,MAAM,MAAM,KAAM,QAAQ,GAAG,MAAM,GAAG,UAAU,MAAQ,QAAQ,GAAG,IAAI,GAAG,UAAU,GAAI;KAC/F;IACF;IACA,IAAI,SAAS,QAAQ;KACnB,UAAU,QAAQ;MAChB,MAAM;MACN,OAAO;MACP,OAAO;MACP,QAAQ,MAAM;MACd,WAAW,MAAM;MACjB,UAAU,MAAM;MAChB,IAAI,MAAM;MACV,YAAY,MAAM;MAClB,eAAe,MAAM;MACrB,cAAc,MAAM;KACtB,CAAC;KACD;IACF;IAEA,MAAM,MAAM,UAAU,iBAAiB,WAAW,SAAS;IAC3D,MAAM,OAAO,MAAM,KAAK,OAAO,KAAK,IAAI,SAAS,KAAK;IACtD,MAAM,SAAS,MAAM,KAAK,SAAS,KAAK,IAAI,WAAW,KAAK;IAC5D,IAAI,OAAO,GAAG,KAAK,IAAI,UAAU,GAAG,UAAU,IAAI,MAAM,OAAO,KAAK,MAAM,UAAU,IAAI,MAAM,SAAS,IAAI,OAAO,IAAI,eAAe,MAAM,UAAU;IACrJ,IAAI,KACF,QAAQ,QACJ,KAAK,IAAI,GAAG,IAAI,GAAG,UACnB,MAAM,IAAI;IAEhB,IAAI,CAAC,MAAM,MAAM,MAAM,cACrB,QAAQ,QACJ,KAAK,OAAO,IAAI,SAAS,MAAM,cAAc,EAAE,IAAI,UACnD,OAAO,SAAS,MAAM,cAAc,EAAE;IAE5C,IAAI,SAAS,WAAW;KACtB,IAAI,MAAM,kBAAkB,KAAA,GAC1B,QAAQ,WAAW,MAAM;KAE3B,MAAM,UAAU,uBAAuB,MAAM,kBAAkB,KAAK;KACpE,IAAI,SAAS,QAAQ,KAAK;IAC5B;IACA,OAAO,MAAM,GAAG,KAAK,GAAG;IACxB;GAEF,KAAK;IACH,IAAI,SAAS,SAAS;IACtB,IAAI,SAAS,QAAQ;KACnB,UAAU,QAAQ;MAChB,MAAM;MACN,QAAQ,MAAM,OAAO;MACrB,WAAW,MAAM,OAAO,KAAK;MAC7B,QAAQ,MAAM,OAAO;MACrB,eAAe,MAAM,OAAO;MAC5B,gBAAgB,MAAM,OAAO,eAAe,KAAK,OAAO;OACtD,aAAa,EAAE;OACf,UAAU,EAAE;OACZ,gBAAgB,EAAE;MACpB,EAAE;KACJ,CAAC;KACD;IACF;IACA,OAAO,MAAM,GAAG,kBAAkB,MAAM,QAAQ,KAAK,EAAE,GAAG;IAC1D;GAEF,KAAK;IACH,IAAI,SAAS,SAAS;KACpB,OAAO,MAAM,IAAI;KACjB;IACF;IACA,IAAI,SAAS,QAAQ;KACnB,UAAU,QAAQ;MAChB,MAAM;MACN,YAAY,MAAM,OAAO;MACzB,YAAY,MAAM,OAAO,MAAM;MAC/B,aAAa,MAAM,OAAO,MAAM,QAAQ,MAAM,EAAE,MAAM,CAAC,CAAC;KAC1D,CAAC;KACD;IACF;IACA,MAAM,SAAS,MAAM,OAAO,MAAM,QAC/B,GAAG,MAAM,IAAI,EAAE,YAAY,QAAQ,MAAM,EAAE,UAAU,IAAI,CAAC,CAAC,QAC5D,CACF;IACA,MAAM,WAAW,MAAM,OAAO,MAAM,QACjC,GAAG,MAAM,IAAI,EAAE,YAAY,QAC5B,CACF;IACA,MAAM,gBAAgB,MAAM,OAAO,MAAM,QACtC,GAAG,MAAM,IAAI,EAAE,eAChB,CACF;IACA,IAAI,SAAS,iBAAiB,eAAe,MAAM,OAAO,UAAU,EAAE,IAAI,OAAO,GAAG,SAAS;IAC7F,IAAI,gBAAgB,GAClB,UAAU,KAAK,cAAc;IAE/B,UAAU;IACV,OAAO,MAAM,MAAM;IACnB;GAEF,SACE;EACJ;CACF;AACF;;AAGA,SAAgB,2BACd,SAC2C;CAC3C,MAAM,SAAS,QAAQ,UAAU,QAAQ;CACzC,MAAM,OAAO,QAAQ;CACrB,MAAM,QAAQ,QAAQ,SAAS;CAE/B,IAAI,QAAQ;CACZ,IAAI,YAAY;CAChB,IAAI,kBAAkB;CAEtB,QAAQ,UAAU;EAChB,QAAQ,MAAM,MAAd;GACE,KAAK;IACH,QAAQ,MAAM;IACd,YAAY;IACZ,kBAAkB;IAClB,IAAI,SAAS,WAAW,UAAU,GAAG;IACrC,IAAI,SAAS,QAAQ;KACnB,UAAU,QAAQ;MAChB,MAAM;MACN,OAAO,MAAM;MACb,eAAe,QAAQ;KACzB,CAAC;KACD;IACF;IACA,MAAM,aACJ,QAAQ,kBAAkB,KAAA,IACtB,oBAAoB,QAAQ,cAAc,KAC1C;IACN,OAAO,MACL,WAAW,MAAM,gBAAgB,WAAW,QAC9C;IACA;GAEF,KAAK;IACH;IACA,mBAAmB,MAAM;IACzB,IAAI,SAAS,SAAS;KACpB,MAAM,YAAY,MAAM,WAAW,KAAK,CAAC,MAAM;KAC/C,OAAO,MACL,YACI,QAAQ,GAAG,MAAM,GAAG,UAAU,MAC9B,QAAQ,GAAG,IAAI,GAAG,UAAU,GAClC;KACA;IACF;IACA,IAAI,SAAS,QAAQ;KACnB,UAAU,QAAQ;MAChB,MAAM;MACN,OAAO;MACP;MACA,QAAQ,MAAM;MACd,WAAW,MAAM;MACjB,iBAAiB,MAAM;MACvB,QAAQ,MAAM;MACd,QAAQ,MAAM;MACd,YAAY,MAAM;MAClB,aAAa,MAAM;KACrB,CAAC;KACD;IACF;IAEA,MAAM,MAAM,UAAU,iBAAiB,WAAW,KAAK;IACvD,MAAM,KAAK,MAAM,WAAW,KAAK,CAAC,MAAM;IACxC,MAAM,OAAO,KAAK,OAAO,KAAK,IAAI,SAAS,KAAK;IAChD,MAAM,SAAS,KAAK,SAAS,KAAK,IAAI,WAAW,KAAK;IACtD,IAAI,OAAO,GAAG,KAAK,IAAI,UAAU,GAAG,MAAM,IAAI,MAAM,OAAO,KAAK,MAAM,UAAU,IAAI,MAAM,gBAAgB,IAAI,OAAO,IAAI,eAAe,MAAM,UAAU;IACxJ,QAAQ,kBAAkB,MAAM,OAAO,GAAG,MAAM,SAAS,MAAM;IAC/D,IAAI,KACF,QAAQ,QAAQ,KAAK,IAAI,GAAG,IAAI,GAAG,UAAU,MAAM,IAAI;IAEzD,IAAI,MAAM,aACR,QAAQ,QACJ,KAAK,OAAO,IAAI,SAAS,MAAM,aAAa,EAAE,IAAI,UAClD,OAAO,SAAS,MAAM,aAAa,EAAE;IAE3C,IAAI,SAAS,aAAa,MAAM,UAAU,MAAM,SAAS,GACvD,QAAQ,QAAQ,KAAK,OAAO,oBAAoB,UAAU;IAE5D,OAAO,MAAM,GAAG,KAAK,GAAG;IACxB;GAEF,KAAK;IACH,IAAI,SAAS,SAAS;KACpB,OAAO,MAAM,IAAI;KACjB;IACF;IACA,IAAI,SAAS,QAAQ;KACnB,UAAU,QAAQ;MAChB,MAAM;MACN,YAAY,MAAM;MAClB,mBAAmB,MAAM;MACzB,oBAAoB,MAAM;KAC5B,CAAC;KACD;IACF;IACA,IAAI,UAAU,GAAG;IACjB,OAAO,MACL,eAAe,eAAe,MAAM,UAAU,EAAE,IAAI,MAAM,mBAAmB,GAAG,MAAM,kBAAkB,0BAC1G;IACA;GAEF,SACE;EACJ;CACF;AACF;;;;;;;AAQA,SAAS,UAAU,QAAkB,OAAsB;CACzD,OAAO,MAAM,GAAG,KAAK,UAAU,KAAK,EAAE,GAAG;AAC3C;;AAGA,SAAgB,eAAe,IAAoB;CACjD,IAAI,KAAK,KAAM,OAAO,GAAG,GAAG;CAC5B,MAAM,MAAM,KAAK;CACjB,IAAI,MAAM,IAAI,OAAO,GAAG,IAAI,QAAQ,CAAC,EAAE;CACvC,MAAM,MAAM,KAAK,MAAM,MAAM,EAAE;CAC/B,MAAM,SAAS,KAAK,MAAM,MAAM,EAAE;CAClC,IAAI,MAAM,IAAI,OAAO,GAAG,IAAI,IAAI,OAAO;CAGvC,OAAO,GAFI,KAAK,MAAM,MAAM,EAEjB,EAAE,IADE,MAAM,GACG;AAC1B;;;;;;;AAQA,SAAS,UACP,iBACA,WACA,OACoB;CACpB,IAAI,cAAc,KAAK,aAAa,OAAO,OAAO,KAAA;CAClD,MAAM,MAAM,kBAAkB;CAC9B,MAAM,aAAa,QAAQ,aAAa;CACxC,OAAO,IAAI,eAAe,KAAK,MAAM,SAAS,CAAC,EAAE;AACnD;;AAGA,SAAS,SAAS,MAAc,KAAqB;CACnD,IAAI,KAAK,UAAU,KAAK,OAAO;CAC/B,OAAO,GAAG,KAAK,MAAM,GAAG,MAAM,CAAC,EAAE;AACnC;;;;;;AAOA,SAAS,uBACP,SACA,QAAQ,OACA;CACR,IAAI,CAAC,WAAW,QAAQ,WAAW,GAAG,OAAO;CAC7C,OAAO,QACJ,KAAK,MACJ,GAAG,EAAE,SAAS,OAAO,KAAK,IAAI,SAAS,KAAK,EAAE,GAAG,EAAE,aACrD,CAAC,CACA,KAAK,IAAI;AACd;;AAGA,SAAgB,kBAAkB,MAAkB,OAAwB;CAC1E,MAAM,OAAO,KAAK,SAAS,OAAO,KAAK,IAAI,SAAS,KAAK;CACzD,MAAM,SAAS,KAAK,SAAS,UAAU,KAAK,IAAI,UAAU,KAAK;CAC/D,MAAM,QAAQ,KAAK,eAAe,KAAK,MAAM;EAC3C,MAAM,OAAO,EAAE,WAAW,IAAA,CAAK,QAAQ,CAAC;EACxC,OAAO,GAAG,EAAE,YAAY,GAAG,EAAE,YAAY,GAAG,EAAE,eAAe,IAAI,IAAI;CACvE,CAAC;CACD,MAAM,QACJ,KAAK,gBAAgB,IACjB,QACE,IAAI,OAAO,GAAG,KAAK,cAAc,kBAAkB,UACnD,KAAK,KAAK,cAAc,oBAC1B;CACN,MAAM,QAAQ,MAAM,SAAS,IAAI,KAAK,MAAM,KAAK,KAAK,MAAM;CAC5D,OAAO,GAAG,KAAK,GAAG,KAAK,OAAO,KAAK,KAAK,KAAK,MAAM,IAAI,SAAS,QAAQ;AAC1E;;;;;;;AC5aA,SAAS,kBACP,SACA,MACoB;CACpB,MAAM,MAAM,UAAU,SAAS,IAAI;CACnC,IAAI,QAAQ,KAAA,GAAW,OAAO,KAAA;CAC9B,MAAM,IAAI,OAAO,SAAS,KAAK,EAAE;CACjC,OAAO,OAAO,SAAS,CAAC,IAAI,IAAI,KAAA;AAClC;;;;;;AAOA,eAAsB,aAAa,MAAmC;CACpE,MAAM,aAAa,KAAK,WAAW;CACnC,IAAI,CAAC,YAAY;EACf,QAAQ,MACN,0JACF;EACA,OAAO;CACT;CAEA,MAAM,aAAa,UAAU,KAAK,SAAS,QAAQ;CACnD,MAAM,mBAAmB,UAAU,KAAK,SAAS,cAAc;CAC/D,MAAM,aAAa,UAAU,KAAK,SAAS,QAAQ;CACnD,MAAM,QAAQ,UAAU,KAAK,SAAS,OAAO;CAC7C,MAAM,SAAS,UAAU,KAAK,SAAS,QAAQ;CAC/C,MAAM,YAAY,kBAAkB,KAAK,SAAS,YAAY;CAE9D,MAAM,gBADmB,UAAU,KAAK,SAAS,gBACZ,IACjC,aAAa,KAAK,SAAS,kBAAkB,CAAC,IAC9C,KAAA;CACJ,MAAM,SAAS,UAAU,KAAK,SAAS,QAAQ,KAAK;CACpD,MAAM,eAAe,oBAAoB,KAAK,OAAO;CACrD,MAAM,mBACJ,iBAAiB,UAAU,qBAAqB,KAAK,OAAO;CAE9D,IAAI;CACJ,IAAI,YACF,IAAI;EACF,aAAa,MAAM,kBAAkB,UAAU;CACjD,SAAS,KAAK;EACZ,QAAQ,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;EAC9D,OAAO;CACT;CAGF,IAAI;CACJ,IAAI;EACF,SAAS,MAAM,gBAAgB,UAAU;CAC3C,SAAS,KAAK;EACZ,QAAQ,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;EAC9D,OAAO;CACT;CAEA,IAAI;CACJ,IAAI;EACF,eAAe,oBACb,YACA;GACE,cAAc;GACd;GACA;GACA;GACA;GACA;EACF,GACA,UACF;CACF,SAAS,KAAK;EACZ,QAAQ,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;EAC9D,OAAO;CACT;CAEA,MAAM,aAAa,2BAA2B;EAC5C,MAAM;EACN,eAAe,aAAa,iBAAiB;EAC7C,OAAO;CACT,CAAC;CAED,MAAM,UAAU,MAAM,YAAY,QAAQ;EACxC,GAAG;EACH;CACF,CAAC;CAED,IAAI,YACF,MAAM,UAAU,YAAY,KAAK,UAAU,SAAS,MAAM,CAAC,GAAG,MAAM;CAGtE,IAAI,WAAW,QAAQ;EACrB,QAAQ,OAAO,MAAM,KAAK,UAAU,SAAS,MAAM,CAAC,CAAC;EACrD,QAAQ,OAAO,MAAM,IAAI;CAC3B,OAAO;EACL,MAAM,YAAY,qBAAqB,SAAS,WAAW,SAAS;EACpE,QAAQ,OAAO,MAAM,SAAS;EAC9B,IAAI,CAAC,UAAU,SAAS,IAAI,GAAG,QAAQ,OAAO,MAAM,IAAI;CAC1D;CAEA,IAAI,QAAQ,QAAQ,WAAW,GAC7B,OAAO;CAGT,OAAO,oBAAoB,OAAO,IAAI,IAAI;AAC5C;;;;;;;ACpHA,SAAS,aAAa,OAAuB;CAC3C,OAAO,MAAM,QAAQ,qBAAqB,GAAG;AAC/C;;;;;;AAOA,eAAsB,mBACpB,OACA,QACA,WACiB;CACjB,MAAM,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;CAE1C,IAAI,UAAU;CACd,KAAK,MAAM,cAAc,OAAO,OAAO;EACrC,MAAM,WAAW,MAAM,MAAM,MAAM,MAAM,EAAE,OAAO,WAAW,MAAM;EACnE,IAAI,CAAC,UAAU;EAEf,KAAK,MAAM,OAAO,WAAW,aAAa;GACxC,IAAI,CAAC,IAAI,eAAe;GAExB,MAAM,OAAO,iBAAiB,IAAI,cAAc,MAAM,EACpD,QAAQ,SAAS,OACnB,CAAC;GAMD,MAAM,UACJ,KAAK,WAAW,GALE,aAAa,WAAW,MAAM,EAAE,IAAI,aACtD,WAAW,KAAK,KAClB,EAAE,OAAO,IAAI,gBAAgB,WAGH,GACxB,KAAK,UAAU,MAAM,MAAM,CAAC,GAC5B,MACF;GACA;EACF;CACF;CAEA,OAAO;AACT;;;;;;;;;;;AC7BA,eAAsB,WAAW,MAAmC;CAClE,MAAM,YAAY,KAAK,WAAW;CAClC,IAAI,CAAC,WAAW;EACd,QAAQ,MAAM,gDAAgD;EAC9D,OAAO;CACT;CAEA,MAAM,SAAS,UAAU,KAAK,SAAS,QAAQ,KAAK;CACpD,MAAM,aAAa,UAAU,KAAK,SAAS,QAAQ;CACnD,MAAM,gBAAgB,UAAU,KAAK,SAAS,aAAa;CAC3D,MAAM,eAAe,UAAU,KAAK,SAAS,UAAU;CACvD,MAAM,gBAAgB,aAAa,KAAK,SAAS,kBAAkB,CAAC;CACpE,MAAM,YAAY,UAAU,KAAK,SAAS,SAAS;CACnD,MAAM,eAAe,oBAAoB,KAAK,OAAO;CACrD,MAAM,mBACJ,iBAAiB,UAAU,qBAAqB,KAAK,OAAO;CAE9D,IAAI;CACJ,IAAI;EACF,QAAQ,MAAM,UAAU,SAAS;CACnC,SAAS,KAAK;EACZ,QAAQ,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;EAC9D,OAAO;CACT;CAEA,MAAM,UAAU,WAAW,aAAa,MAAM,WAAW,aAAa;CAEtE,MAAM,aAAa,yBAAyB;EAC1C,MAAM;EACN;EACA,OAAO;CACT,CAAC;CAED,MAAM,SAAS,MAAM,SAAS,OAAO;EACnC;EACA;EACA;CACF,CAAC;CAED,IAAI,YACF,MAAM,UAAU,YAAY,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,MAAM;CAGrE,IAAI,eAAe;EACjB,MAAM,QAAQ,MAAM,mBAAmB,OAAO,QAAQ,aAAa;EACnE,QAAQ,OAAO,MAAM,eAAe,MAAM,oBAAoB,cAAc,GAAG;CACjF;CAEA,IAAI;CACJ,IAAI,cAAc;EAChB,MAAM,EAAE,aAAa,MAAM,OAAO;EAClC,WAAW,KAAK,MAAM,MAAM,SAAS,cAAc,MAAM,CAAC;CAC5D;CAEA,MAAM,YAAY,aAAa,QAAQ;EACrC,QACE,WAAW,cAAc,WAAW,SAAS,SAAS;EACxD;EACA,OAAO,WAAW;CACpB,CAAC;CAED,QAAQ,OAAO,MAAM,SAAS;CAC9B,IAAI,CAAC,UAAU,SAAS,IAAI,GAAG,QAAQ,OAAO,MAAM,IAAI;CAExD,OAAO,OAAO,MAAM,OAAO,MAAM,EAAE,MAAM,IAAI,IAAI;AACnD;;;;;;;;AC7EA,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;AAuBd,eAAsB,KAAK,MAAiC;CAC1D,MAAM,SAAS,UAAU,IAAI;CAE7B,IAAI,OAAO,QAAQ,QAAQ,OAAO,YAAY,UAAU,OAAO,QAAQ,GAAG;EACxE,QAAQ,OAAO,MAAM,KAAK;EAC1B,OAAO;CACT;CAEA,QAAQ,OAAO,SAAf;EACE,KAAK,OACH,OAAO,MAAM,WAAW,MAAM;EAChC,KAAK,SACH,OAAO,MAAM,aAAa,MAAM;EAClC,KAAK,YACH,OAAO,MAAM,gBAAgB,MAAM;EACrC,KAAK,UACH,OAAO,MAAM,cAAc,MAAM;EACnC,KAAK,KAAA;GACH,QAAQ,MAAM,KAAK;GACnB,OAAO;EACT;GACE,QAAQ,MAAM,oBAAoB,OAAO,QAAQ,MAAM,OAAO;GAC9D,OAAO;CACX;AACF;;;;;;ACpDA,MAAM,OAAO,MAAM,KAAK,QAAQ,KAAK,MAAM,CAAC,CAAC;AAC7C,QAAQ,KAAK,IAAI"}
|
|
1
|
+
{"version":3,"file":"bin.js","names":[],"sources":["../../src/cli/commands/format.ts","../../src/cli/progress.ts","../../src/cli/commands/grade.ts","../../src/cli/commands/pipeline.ts","../../src/cli/commands/otel-output.ts","../../src/cli/commands/run.ts","../../src/cli/main.ts","../../src/cli/bin.ts"],"sourcesContent":["/**\n * `harness-eval format` command.\n */\n\nimport { readFile } from \"node:fs/promises\";\n\nimport { formatReport } from \"../../reporter/index\";\nimport type { SuiteReport } from \"../../runner/types\";\nimport { getOption, type ParsedArgs } from \"../args\";\n\n/**\n * Execute `harness-eval format`: re-render a saved report JSON.\n *\n * @returns 0 when all cells pass, 1 otherwise, 2 on load errors.\n */\nexport async function formatCommand(args: ParsedArgs): Promise<number> {\n const reportPath = args.positional[0];\n if (!reportPath) {\n console.error(\"usage: harness-eval format <report.json> [options]\");\n return 2;\n }\n\n const format = getOption(args.options, \"format\") ?? \"console\";\n const baselinePath = getOption(args.options, \"baseline\");\n\n let report: SuiteReport;\n try {\n report = JSON.parse(await readFile(reportPath, \"utf8\")) as SuiteReport;\n } catch (err) {\n console.error(err instanceof Error ? err.message : String(err));\n return 2;\n }\n\n let baseline: SuiteReport | undefined;\n if (baselinePath) {\n baseline = JSON.parse(await readFile(baselinePath, \"utf8\")) as SuiteReport;\n }\n\n const formatted = formatReport(report, {\n format:\n format === \"markdown\" || format === \"json\" ? format : \"console\",\n baseline,\n color: format === \"console\",\n });\n\n process.stdout.write(formatted);\n if (!formatted.endsWith(\"\\n\")) process.stdout.write(\"\\n\");\n return report.cells.every((c) => c.passed) ? 0 : 1;\n}\n","/**\n * CLI progress reporting for long-running harness and grade commands.\n *\n * Progress writes to stderr by default so stdout remains free for report\n * output. Supports human-readable modes and newline-delimited JSON events.\n */\n\nimport type { Writable } from \"node:stream\";\n\nimport { getOption, hasOption } from \"./args\";\nimport type { GradeProgressEvent as GraderGradeProgressEvent } from \"../grader/types\";\nimport type { AssertionResult } from \"../types/assertions\";\nimport type { CellReport, ProgressCallback } from \"../runner/types\";\n\n/** Progress display mode for run and grade commands. */\nexport type ProgressMode = \"default\" | \"quiet\" | \"verbose\" | \"json\";\n\n/** ANSI SGR codes for progress output. Disabled when {@link resolveProgressColor} returns false. */\nconst GREEN = \"\\x1b[32m\";\nconst RED = \"\\x1b[31m\";\nconst YELLOW = \"\\x1b[33m\";\nconst DIM = \"\\x1b[2m\";\nconst RESET = \"\\x1b[0m\";\n\n/** Options for {@link createRunProgressHandler}. */\nexport interface RunProgressOptions {\n mode: ProgressMode;\n maxConcurrent?: number;\n color?: boolean;\n stream?: Writable;\n}\n\n/** Options for {@link createGradeProgressHandler}. */\nexport interface GradeProgressOptions {\n mode: ProgressMode;\n maxConcurrent?: number;\n color?: boolean;\n stream?: Writable;\n}\n\n/**\n * Resolve progress mode from `--progress`, `--quiet`, or `--verbose` flags.\n *\n * Explicit `--progress` wins; otherwise `--quiet` / `--verbose` map to modes.\n */\nexport function resolveProgressMode(\n options: Record<string, string | boolean>,\n): ProgressMode {\n const progress = getOption(options, \"progress\");\n if (\n progress === \"json\" ||\n progress === \"quiet\" ||\n progress === \"verbose\" ||\n progress === \"default\"\n ) {\n return progress;\n }\n if (hasOption(options, \"quiet\")) return \"quiet\";\n if (hasOption(options, \"verbose\")) return \"verbose\";\n return \"default\";\n}\n\n/**\n * Whether to emit ANSI colors on the progress stream (stderr).\n *\n * Precedence: `--no-color` → off; `--color` → on; `NO_COLOR` env → off;\n * `FORCE_COLOR` (non-zero) → on; otherwise TTY detection on `stream`.\n */\nexport function resolveProgressColor(\n options: Record<string, string | boolean>,\n stream: Writable = process.stderr,\n): boolean {\n if (hasOption(options, \"no-color\")) return false;\n if (hasOption(options, \"color\")) return true;\n if (process.env.NO_COLOR !== undefined && process.env.NO_COLOR !== \"\") {\n return false;\n }\n if (process.env.FORCE_COLOR !== undefined && process.env.FORCE_COLOR !== \"0\") {\n return true;\n }\n return (\n \"isTTY\" in stream &&\n (stream as { isTTY?: boolean }).isTTY === true\n );\n}\n\n/** Green checkmark prefix for per-rep success lines. */\nfunction okMark(color: boolean): string {\n return color ? `${GREEN}✓${RESET}` : \"✓\";\n}\n\n/** Red cross prefix for per-rep failure lines. */\nfunction failMark(color: boolean): string {\n return color ? `${RED}✗${RESET}` : \"✗\";\n}\n\n/** Inline lowercase status word for repetition rows. */\nfunction okStatus(color: boolean): string {\n return color ? `${GREEN}ok${RESET}` : \"ok\";\n}\n\n/** Inline uppercase status word for repetition failures. */\nfunction failStatus(color: boolean): string {\n return color ? `${RED}FAIL${RESET}` : \"FAIL\";\n}\n\n/** Uppercase cell-level pass label in {@link formatCellSummary}. */\nfunction passLabel(color: boolean): string {\n return color ? `${GREEN}PASS${RESET}` : \"PASS\";\n}\n\n/** Uppercase cell-level fail label in {@link formatCellSummary}. */\nfunction failLabel(color: boolean): string {\n return color ? `${RED}FAIL${RESET}` : \"FAIL\";\n}\n\n/**\n * Build a {@link ProgressCallback} for suite runs.\n *\n * Writes to `options.stream` (default stderr). JSON mode emits one event per line.\n */\nexport function createRunProgressHandler(\n options: RunProgressOptions,\n): ProgressCallback {\n const stream = options.stream ?? process.stderr;\n const mode = options.mode;\n const color = options.color ?? false;\n\n let totalReps = 0;\n let completed = 0;\n let totalDurationMs = 0;\n\n return (event) => {\n switch (event.kind) {\n case \"suite-start\":\n totalReps = event.totalReps;\n completed = 0;\n totalDurationMs = 0;\n if (mode === \"quiet\") return;\n if (mode === \"json\") {\n writeJson(stream, {\n kind: \"suite-start\",\n totalReps: event.totalReps,\n maxConcurrent: options.maxConcurrent,\n });\n return;\n }\n const concurrent =\n options.maxConcurrent !== undefined\n ? ` (max-concurrent ${options.maxConcurrent})`\n : \"\";\n stream.write(`Running ${totalReps} repetitions${concurrent}...\\n\\n`);\n break;\n\n case \"rep-complete\":\n completed++;\n totalDurationMs += event.durationMs;\n if (mode === \"quiet\") {\n stream.write(event.ok ? (color ? `${GREEN}.${RESET}` : \".\") : (color ? `${RED}x${RESET}` : \"x\"));\n return;\n }\n if (mode === \"json\") {\n writeJson(stream, {\n kind: \"rep-complete\",\n index: completed,\n total: totalReps,\n caseId: event.caseId,\n cellLabel: event.cellLabel,\n repIndex: event.repIndex,\n ok: event.ok,\n durationMs: event.durationMs,\n toolCallCount: event.toolCallCount,\n errorMessage: event.errorMessage,\n });\n return;\n }\n\n const eta = formatEta(totalDurationMs, completed, totalReps);\n const icon = event.ok ? okMark(color) : failMark(color);\n const status = event.ok ? okStatus(color) : failStatus(color);\n let line = `${icon} [${completed}/${totalReps}] ${event.caseId} @ ${event.cellLabel} #${event.repIndex} ${status} ${formatDuration(event.durationMs)}`;\n if (eta) {\n line += color\n ? ` ${DIM}(${eta})${RESET}`\n : ` (${eta})`;\n }\n if (!event.ok && event.errorMessage) {\n line += color\n ? ` ${YELLOW}— ${truncate(event.errorMessage, 80)}${RESET}`\n : ` — ${truncate(event.errorMessage, 80)}`;\n }\n if (mode === \"verbose\") {\n if (event.toolCallCount !== undefined) {\n line += ` tools=${event.toolCallCount}`;\n }\n const summary = formatAssertionSummary(event.assertionResults, color);\n if (summary) line += ` ${summary}`;\n }\n stream.write(`${line}\\n`);\n break;\n\n case \"cell-complete\":\n if (mode === \"quiet\") return;\n if (mode === \"json\") {\n writeJson(stream, {\n kind: \"cell-complete\",\n caseId: event.report.caseId,\n cellLabel: event.report.cell.label,\n passed: event.report.passed,\n adapterErrors: event.report.adapterErrors,\n assertionStats: event.report.assertionStats.map((s) => ({\n description: s.description,\n passRate: s.passRate,\n meetsThreshold: s.meetsThreshold,\n })),\n });\n return;\n }\n stream.write(`${formatCellSummary(event.report, color)}\\n`);\n break;\n\n case \"suite-complete\":\n if (mode === \"quiet\") {\n stream.write(\"\\n\");\n return;\n }\n if (mode === \"json\") {\n writeJson(stream, {\n kind: \"suite-complete\",\n durationMs: event.report.durationMs,\n cellsTotal: event.report.cells.length,\n cellsPassed: event.report.cells.filter((c) => c.passed).length,\n });\n return;\n }\n const okReps = event.report.cells.reduce(\n (n, c) => n + c.repetitions.filter((r) => r.error === null).length,\n 0,\n );\n const totalRun = event.report.cells.reduce(\n (n, c) => n + c.repetitions.length,\n 0,\n );\n const adapterErrors = event.report.cells.reduce(\n (n, c) => n + c.adapterErrors,\n 0,\n );\n let footer = `\\nFinished in ${formatDuration(event.report.durationMs)} (${okReps}/${totalRun} reps ok`;\n if (adapterErrors > 0) {\n footer += `, ${adapterErrors} adapter error(s)`;\n }\n footer += \")\\n\\n\";\n stream.write(footer);\n break;\n\n default:\n break;\n }\n };\n}\n\n/** Build a progress handler for outcome grading ({@link GradeProgressEvent}). */\nexport function createGradeProgressHandler(\n options: GradeProgressOptions,\n): (event: GraderGradeProgressEvent) => void {\n const stream = options.stream ?? process.stderr;\n const mode = options.mode;\n const color = options.color ?? false;\n\n let total = 0;\n let completed = 0;\n let totalDurationMs = 0;\n\n return (event) => {\n switch (event.kind) {\n case \"grade-start\":\n total = event.total;\n completed = 0;\n totalDurationMs = 0;\n if (mode === \"quiet\" || total === 0) return;\n if (mode === \"json\") {\n writeJson(stream, {\n kind: \"grade-start\",\n total: event.total,\n maxConcurrent: options.maxConcurrent,\n });\n return;\n }\n const concurrent =\n options.maxConcurrent !== undefined\n ? ` (max-concurrent ${options.maxConcurrent})`\n : \"\";\n stream.write(\n `Grading ${total} repetition(s)${concurrent}...\\n\\n`,\n );\n break;\n\n case \"grade-complete\":\n completed++;\n totalDurationMs += event.durationMs;\n if (mode === \"quiet\") {\n const allPassed = event.failed === 0 && !event.graderError;\n stream.write(\n allPassed\n ? color ? `${GREEN}.${RESET}` : \".\"\n : color ? `${RED}x${RESET}` : \"x\",\n );\n return;\n }\n if (mode === \"json\") {\n writeJson(stream, {\n kind: \"grade-complete\",\n index: completed,\n total,\n caseId: event.caseId,\n cellLabel: event.cellLabel,\n repetitionIndex: event.repetitionIndex,\n passed: event.passed,\n failed: event.failed,\n durationMs: event.durationMs,\n graderError: event.graderError,\n });\n return;\n }\n\n const eta = formatEta(totalDurationMs, completed, total);\n const ok = event.failed === 0 && !event.graderError;\n const icon = ok ? okMark(color) : failMark(color);\n const status = ok ? okStatus(color) : failStatus(color);\n let line = `${icon} [${completed}/${total}] ${event.caseId} @ ${event.cellLabel} #${event.repetitionIndex} ${status} ${formatDuration(event.durationMs)}`;\n line += ` expectations ${event.passed}/${event.passed + event.failed}`;\n if (eta) {\n line += color ? ` ${DIM}(${eta})${RESET}` : ` (${eta})`;\n }\n if (event.graderError) {\n line += color\n ? ` ${YELLOW}— ${truncate(event.graderError, 80)}${RESET}`\n : ` — ${truncate(event.graderError, 80)}`;\n }\n if (mode === \"verbose\" && event.failed && event.failed > 0) {\n line += color ? ` ${YELLOW}see grading output${RESET}` : \" see grading output\";\n }\n stream.write(`${line}\\n`);\n break;\n\n case \"grade-done\":\n if (mode === \"quiet\") {\n stream.write(\"\\n\");\n return;\n }\n if (mode === \"json\") {\n writeJson(stream, {\n kind: \"grade-done\",\n durationMs: event.durationMs,\n totalExpectations: event.totalExpectations,\n passedExpectations: event.passedExpectations,\n });\n return;\n }\n if (total === 0) return;\n stream.write(\n `\\nGraded in ${formatDuration(event.durationMs)} (${event.passedExpectations}/${event.totalExpectations} expectations passed)\\n\\n`,\n );\n break;\n\n default:\n break;\n }\n };\n}\n\n/**\n * Write one NDJSON progress event line to the progress stream.\n *\n * JSON mode keeps stdout clean for machine-readable reports while still\n * exposing structured progress for CI log parsers.\n */\nfunction writeJson(stream: Writable, value: unknown): void {\n stream.write(`${JSON.stringify(value)}\\n`);\n}\n\n/** Format milliseconds as a human-readable duration string. */\nexport function formatDuration(ms: number): string {\n if (ms < 1000) return `${ms}ms`;\n const sec = ms / 1000;\n if (sec < 60) return `${sec.toFixed(1)}s`;\n const min = Math.floor(sec / 60);\n const remSec = Math.round(sec % 60);\n if (min < 60) return `${min}m ${remSec}s`;\n const hr = Math.floor(min / 60);\n const remMin = min % 60;\n return `${hr}h ${remMin}m`;\n}\n\n/**\n * Estimate remaining time from average completed rep duration.\n *\n * Uses a simple running mean — good enough for long suites without storing\n * per-rep history. Returns `undefined` at start and when all reps are done.\n */\nfunction formatEta(\n totalDurationMs: number,\n completed: number,\n total: number,\n): string | undefined {\n if (completed === 0 || completed >= total) return undefined;\n const avg = totalDurationMs / completed;\n const remaining = (total - completed) * avg;\n return `~${formatDuration(Math.round(remaining))} remaining`;\n}\n\n/** Truncate error text for single-line progress rows (Unicode ellipsis). */\nfunction truncate(text: string, max: number): string {\n if (text.length <= max) return text;\n return `${text.slice(0, max - 1)}…`;\n}\n\n/**\n * Compact per-assertion pass/fail summary for `--progress verbose` rep lines.\n *\n * @returns Comma-separated `✓ description` / `✗ description` fragments, or empty string.\n */\nfunction formatAssertionSummary(\n results?: AssertionResult[],\n color = false,\n): string {\n if (!results || results.length === 0) return \"\";\n return results\n .map((r) =>\n `${r.passed ? okMark(color) : failMark(color)} ${r.description}`,\n )\n .join(\", \");\n}\n\n/** One-line summary when a matrix cell finishes (used in default progress mode). */\nexport function formatCellSummary(cell: CellReport, color: boolean): string {\n const mark = cell.passed ? okMark(color) : failMark(color);\n const status = cell.passed ? passLabel(color) : failLabel(color);\n const parts = cell.assertionStats.map((s) => {\n const pct = (s.passRate * 100).toFixed(0);\n return `${s.description} ${s.passedCount}/${s.evaluatedCount} (${pct}%)`;\n });\n const crash =\n cell.adapterErrors > 0\n ? color\n ? ` ${YELLOW}[${cell.adapterErrors} adapter errors]${RESET}`\n : ` [${cell.adapterErrors} adapter errors]`\n : \"\";\n const stats = parts.length > 0 ? ` ${parts.join(\" · \")}` : \"\";\n return `${mark} ${cell.caseId} @ ${cell.cell.label} ${status}${crash}${stats}`;\n}\n","/**\n * `harness-eval grade` — LLM outcome grading on a suite report.\n */\n\nimport { writeFile } from \"node:fs/promises\";\n\nimport { loadGradingConfig } from \"../../config/grading-loader\";\nimport {\n formatGradingConsole,\n gradeReport,\n gradingReportPassed,\n loadSuiteReport,\n resolveGradeOptions,\n} from \"../../grader/index\";\nimport { getOption, getOptionInt, type ParsedArgs } from \"../args\";\nimport {\n createGradeProgressHandler,\n resolveProgressColor,\n resolveProgressMode,\n} from \"../progress\";\n\n/** Parse an optional integer CLI flag; returns undefined when absent or invalid. */\nfunction optionalOptionInt(\n options: Record<string, string | boolean>,\n name: string,\n): number | undefined {\n const raw = getOption(options, name);\n if (raw === undefined) return undefined;\n const n = Number.parseInt(raw, 10);\n return Number.isFinite(n) ? n : undefined;\n}\n\n/**\n * Execute `harness-eval grade`: LLM outcome grading on a suite report JSON.\n *\n * @returns 0 when all expectations pass, 1 on failure, 2 on usage/load errors or no reps graded.\n */\nexport async function gradeCommand(args: ParsedArgs): Promise<number> {\n const reportPath = args.positional[0];\n if (!reportPath) {\n console.error(\n \"usage: harness-eval grade <report.json> [--config grading.yaml] [--suite suite.yaml] [--expectations path] [--output path] [--model id] [--timeout-ms N] [--max-concurrent N]\",\n );\n return 2;\n }\n\n const configPath = getOption(args.options, \"config\");\n const suitePath = getOption(args.options, \"suite\");\n const expectationsPath = getOption(args.options, \"expectations\");\n const outputPath = getOption(args.options, \"output\");\n const model = getOption(args.options, \"model\");\n const binary = getOption(args.options, \"binary\");\n const timeoutMs = optionalOptionInt(args.options, \"timeout-ms\");\n const maxConcurrentRaw = getOption(args.options, \"max-concurrent\");\n const maxConcurrent = maxConcurrentRaw\n ? getOptionInt(args.options, \"max-concurrent\", 2)\n : undefined;\n const format = getOption(args.options, \"format\") ?? \"console\";\n const progressMode = resolveProgressMode(args.options);\n const useProgressColor =\n progressMode !== \"json\" && resolveProgressColor(args.options);\n\n let fileConfig;\n const gradingConfigPath = configPath ?? suitePath;\n if (configPath && suitePath) {\n console.error(\"grade: use only one of --config or --suite\");\n return 2;\n }\n if (gradingConfigPath) {\n try {\n fileConfig = await loadGradingConfig(gradingConfigPath);\n } catch (err) {\n console.error(err instanceof Error ? err.message : String(err));\n return 2;\n }\n }\n\n let report;\n try {\n report = await loadSuiteReport(reportPath);\n } catch (err) {\n console.error(err instanceof Error ? err.message : String(err));\n return 2;\n }\n\n let gradeOptions;\n try {\n gradeOptions = resolveGradeOptions(\n fileConfig,\n {\n sourceReport: reportPath,\n expectationsPath,\n model,\n binary,\n timeoutMs,\n maxConcurrent,\n },\n configPath ?? suitePath,\n );\n } catch (err) {\n console.error(err instanceof Error ? err.message : String(err));\n return 2;\n }\n\n const onProgress = createGradeProgressHandler({\n mode: progressMode,\n maxConcurrent: gradeOptions.maxConcurrent ?? 2,\n color: useProgressColor,\n });\n\n const grading = await gradeReport(report, {\n ...gradeOptions,\n onProgress,\n });\n\n if (outputPath) {\n await writeFile(outputPath, JSON.stringify(grading, null, 2), \"utf8\");\n }\n\n if (format === \"json\") {\n process.stdout.write(JSON.stringify(grading, null, 2));\n process.stdout.write(\"\\n\");\n } else {\n const formatted = formatGradingConsole(grading, format === \"console\");\n process.stdout.write(formatted);\n if (!formatted.endsWith(\"\\n\")) process.stdout.write(\"\\n\");\n }\n\n if (grading.results.length === 0) {\n return 2;\n }\n\n return gradingReportPassed(grading) ? 0 : 1;\n}\n","/**\n * `harness-eval pipeline` — orchestrate run → grade → envelope from suite.yaml.\n */\n\nimport { dirname, isAbsolute, join } from \"node:path\";\nimport { fileURLToPath } from \"node:url\";\nimport { readFile } from \"node:fs/promises\";\n\nimport { loadSuiteDocument } from \"../../config/loader\";\nimport { runPipeline } from \"../../pipeline/run-pipeline\";\nimport type { PipelineCliOverrides } from \"../../pipeline/resolve-inputs\";\nimport { suiteDirectoryFromPath } from \"../../pipeline/resolve-inputs\";\nimport {\n createGradeProgressHandler,\n createRunProgressHandler,\n resolveProgressColor,\n resolveProgressMode,\n} from \"../progress\";\nimport { getOption, getOptionInt, type ParsedArgs } from \"../args\";\nimport { parseEnvelopeProjection } from \"./envelope\";\n\n/** Read package version for envelope provenance (best-effort). */\nasync function readFrameworkVersion(): Promise<string | undefined> {\n try {\n const packagePath = join(\n dirname(fileURLToPath(import.meta.url)),\n \"../../../package.json\",\n );\n const text = await readFile(packagePath, \"utf8\");\n const pkg = JSON.parse(text) as { version?: string };\n return pkg.version;\n } catch {\n return undefined;\n }\n}\n\n/** Resolve CLI path overrides relative to the suite directory unless absolute or `~/`. */\nfunction resolveOverridePath(\n value: string | undefined,\n suiteDir: string,\n): string | undefined {\n if (!value) return undefined;\n return isAbsolute(value) || value.startsWith(\"~/\")\n ? value\n : join(suiteDir, value);\n}\n\n/**\n * Execute `harness-eval pipeline`.\n *\n * @returns Step exit code (0 pass, 1 eval fail, 2 usage/load error).\n */\nexport async function pipelineCommand(args: ParsedArgs): Promise<number> {\n const suitePath = args.positional[0];\n if (!suitePath) {\n console.error(\n \"usage: harness-eval pipeline <suite.yaml|dir> [--steps run,grade,envelope] [--output path] [--grading path] [--report path] [--max-concurrent N] [--progress default|quiet|verbose|json]\",\n );\n return 2;\n }\n\n let doc;\n try {\n doc = await loadSuiteDocument(suitePath);\n } catch (err) {\n console.error(err instanceof Error ? err.message : String(err));\n return 2;\n }\n\n if (!doc.pipeline) {\n console.error(\n \"suite.yaml has no pipeline block; use run, grade, and envelope commands separately\",\n );\n return 2;\n }\n\n const suiteDir = suiteDirectoryFromPath(doc.suitePath);\n const steps = getOption(args.options, \"steps\");\n const maxConcurrent = getOptionInt(args.options, \"max-concurrent\", 4);\n const progressMode = resolveProgressMode(args.options);\n const useProgressColor =\n progressMode !== \"json\" && resolveProgressColor(args.options);\n\n const projection = parseEnvelopeProjection(\n getOption(args.options, \"projection\"),\n );\n if (getOption(args.options, \"projection\") && !projection) {\n console.error(\n \"invalid --projection; expected envelope, trajectory, or instances\",\n );\n return 2;\n }\n\n const overrides: PipelineCliOverrides = {};\n const runOutput = getOption(args.options, \"output\");\n if (runOutput) {\n overrides.run = {\n output: resolveOverridePath(runOutput, suiteDir),\n maxConcurrent,\n };\n }\n const reportOverride = getOption(args.options, \"report\");\n if (reportOverride) {\n overrides.grade = {\n ...overrides.grade,\n input: resolveOverridePath(reportOverride, suiteDir),\n };\n overrides.envelope = {\n ...overrides.envelope,\n report: resolveOverridePath(reportOverride, suiteDir),\n };\n }\n const gradingOutput = getOption(args.options, \"grading-output\");\n if (gradingOutput) {\n overrides.grade = {\n ...overrides.grade,\n output: resolveOverridePath(gradingOutput, suiteDir),\n };\n }\n const gradingInput = getOption(args.options, \"grading\");\n if (gradingInput) {\n overrides.envelope = {\n ...overrides.envelope,\n grading: resolveOverridePath(gradingInput, suiteDir),\n };\n }\n const envelopeOutput = getOption(args.options, \"envelope-output\");\n if (envelopeOutput) {\n overrides.envelope = {\n ...overrides.envelope,\n output: resolveOverridePath(envelopeOutput, suiteDir),\n };\n }\n if (projection) {\n overrides.envelope = {\n ...overrides.envelope,\n projection,\n };\n }\n\n if (doc.pipeline.grade && !doc.judge) {\n console.error(\"pipeline grade step requires inline judge: block in suite.yaml\");\n return 2;\n }\n\n // Envelope provenance includes the harness-eval package version when available.\n const frameworkVersion = await readFrameworkVersion();\n\n try {\n const result = await runPipeline(doc, {\n steps,\n maxConcurrent,\n overrides,\n frameworkVersion,\n onRunProgress: createRunProgressHandler({\n mode: progressMode,\n maxConcurrent,\n color: useProgressColor,\n }),\n onGradeProgress: createGradeProgressHandler({\n mode: progressMode,\n maxConcurrent,\n color: useProgressColor,\n }),\n });\n return result.exitCode;\n } catch (err) {\n console.error(err instanceof Error ? err.message : String(err));\n return 2;\n }\n}\n","/**\n * Write OTLP JSON artifacts from a suite report.\n */\n\nimport { mkdir, writeFile } from \"node:fs/promises\";\nimport { join } from \"node:path\";\n\nimport { trajectoryToOtlp } from \"../../otel/emitter\";\nimport type { SuiteReport, TestSuite } from \"../../runner/types\";\n\n/** Sanitize case/cell labels for use in OTLP artifact filenames. */\nfunction safeFilePart(value: string): string {\n return value.replace(/[^a-zA-Z0-9._-]+/g, \"_\");\n}\n\n/**\n * Write one OTLP JSON file per successful repetition.\n *\n * Files: `{caseId}__{cellLabel}__rep{N}.otlp.json`\n */\nexport async function writeOtelArtifacts(\n suite: TestSuite,\n report: SuiteReport,\n outputDir: string,\n): Promise<number> {\n await mkdir(outputDir, { recursive: true });\n\n let written = 0;\n for (const cellReport of report.cells) {\n const testCase = suite.cases.find((c) => c.id === cellReport.caseId);\n if (!testCase) continue;\n\n for (const rep of cellReport.repetitions) {\n if (!rep.adapterResult) continue;\n\n const otlp = trajectoryToOtlp(rep.adapterResult.view, {\n prompt: testCase.prompt,\n });\n\n const filename = `${safeFilePart(cellReport.caseId)}__${safeFilePart(\n cellReport.cell.label,\n )}__rep${rep.repetitionIndex}.otlp.json`;\n\n await writeFile(\n join(outputDir, filename),\n JSON.stringify(otlp, null, 2),\n \"utf8\",\n );\n written++;\n }\n }\n\n return written;\n}\n","/**\n * `harness-eval run` command.\n */\n\nimport { writeFile } from \"node:fs/promises\";\n\nimport { getAdapter } from \"../../adapters/registry\";\nimport { loadSuite } from \"../../config/loader\";\nimport { formatReport } from \"../../reporter/index\";\nimport { runSuite } from \"../../runner/suite\";\nimport type { SuiteReport } from \"../../runner/types\";\nimport { getOption, getOptionInt, type ParsedArgs } from \"../args\";\nimport {\n createRunProgressHandler,\n resolveProgressColor,\n resolveProgressMode,\n} from \"../progress\";\nimport { writeOtelArtifacts } from \"./otel-output\";\n\n/**\n * Execute `harness-eval run`: load suite, run repetitions, format report.\n *\n * @returns 0 when all cells pass thresholds, 1 on assertion failure, 2 on usage/load errors.\n */\nexport async function runCommand(args: ParsedArgs): Promise<number> {\n const suitePath = args.positional[0];\n if (!suitePath) {\n console.error(\"usage: harness-eval run <suite.yaml> [options]\");\n return 2;\n }\n\n const format = getOption(args.options, \"format\") ?? \"console\";\n const outputPath = getOption(args.options, \"output\");\n const otelOutputDir = getOption(args.options, \"otel-output\");\n const baselinePath = getOption(args.options, \"baseline\");\n const maxConcurrent = getOptionInt(args.options, \"max-concurrent\", 4);\n const adapterId = getOption(args.options, \"adapter\");\n const progressMode = resolveProgressMode(args.options);\n const useProgressColor =\n progressMode !== \"json\" && resolveProgressColor(args.options);\n\n let suite;\n try {\n suite = await loadSuite(suitePath);\n } catch (err) {\n console.error(err instanceof Error ? err.message : String(err));\n return 2;\n }\n\n const adapter = getAdapter(adapterId ?? suite.adapter ?? \"claude-code\");\n\n const onProgress = createRunProgressHandler({\n mode: progressMode,\n maxConcurrent,\n color: useProgressColor,\n });\n\n const report = await runSuite(suite, {\n adapter,\n maxConcurrent,\n onProgress,\n });\n\n if (outputPath) {\n await writeFile(outputPath, JSON.stringify(report, null, 2), \"utf8\");\n }\n\n if (otelOutputDir) {\n const count = await writeOtelArtifacts(suite, report, otelOutputDir);\n process.stderr.write(`otel: wrote ${count} trace file(s) to ${otelOutputDir}\\n`);\n }\n\n let baseline: SuiteReport | undefined;\n if (baselinePath) {\n const { readFile } = await import(\"node:fs/promises\");\n baseline = JSON.parse(await readFile(baselinePath, \"utf8\")) as SuiteReport;\n }\n\n const formatted = formatReport(report, {\n format:\n format === \"markdown\" || format === \"json\" ? format : \"console\",\n baseline,\n color: format === \"console\",\n });\n\n process.stdout.write(formatted);\n if (!formatted.endsWith(\"\\n\")) process.stdout.write(\"\\n\");\n\n return report.cells.every((c) => c.passed) ? 0 : 1;\n}\n","/**\n * CLI entry point — dispatches subcommands and prints usage on `--help`.\n *\n * Exit codes: 0 success, 1 eval/grade failure, 2 usage or load errors.\n */\n\nimport { envelopeCommand } from \"./commands/envelope\";\nimport { formatCommand } from \"./commands/format\";\nimport { gradeCommand } from \"./commands/grade\";\nimport { pipelineCommand } from \"./commands/pipeline\";\nimport { runCommand } from \"./commands/run\";\nimport { parseArgs } from \"./args\";\n\nconst USAGE = `harness-eval — harness-level eval framework\n\nUsage:\n harness-eval run <suite.yaml> [--max-concurrent N] [--baseline path] [--output path] [--otel-output dir] [--format console|markdown|json] [--adapter id] [--quiet] [--verbose] [--progress default|quiet|verbose|json]\n harness-eval grade <report.json> [--config grading.yaml] [--suite suite.yaml] [--expectations path] [--output path] [--model id] [--timeout-ms N] [--max-concurrent N] [--format console|json] [--quiet] [--verbose] [--progress default|quiet|verbose|json]\n harness-eval envelope <report.json> [--output path] [--grading path] [--suite path] [--projection envelope|trajectory|instances] [--include-raw-stream-events] [--no-transcript]\n harness-eval pipeline <suite.yaml|dir> [--steps run,grade,envelope] [--output path] [--grading path] [--grading-output path] [--envelope-output path] [--report path] [--projection envelope|trajectory|instances] [--max-concurrent N] [--progress default|quiet|verbose|json]\n harness-eval format <report.json> [--format console|markdown|json] [--baseline path]\n harness-eval --help\n\n Progress (run & grade):\n default one line per repetition + per-cell summary (default)\n --quiet colored dots (. = ok, x = fail)\n --verbose per-rep details (tool counts, assertion summary)\n --progress json newline-delimited JSON events on stderr\n --no-color disable ANSI colors on progress output\n --color force ANSI colors on progress output\n`;\n\n/**\n * Route argv to the appropriate subcommand handler.\n *\n * @returns Process exit code (0 = success, 1 = eval failure, 2 = usage error).\n */\nexport async function main(argv: string[]): Promise<number> {\n const parsed = parseArgs(argv);\n\n if (parsed.options.help || parsed.command === \"help\" || parsed.options.h) {\n process.stdout.write(USAGE);\n return 0;\n }\n\n switch (parsed.command) {\n case \"run\":\n return await runCommand(parsed);\n case \"grade\":\n return await gradeCommand(parsed);\n case \"envelope\":\n return await envelopeCommand(parsed);\n case \"pipeline\":\n return await pipelineCommand(parsed);\n case \"format\":\n return await formatCommand(parsed);\n case undefined:\n console.error(USAGE);\n return 2;\n default:\n console.error(`unknown command: ${parsed.command}\\n\\n${USAGE}`);\n return 2;\n }\n}\n","#!/usr/bin/env node\n/**\n * CLI executable entry point — delegates to {@link main} and exits with its code.\n */\n\nimport { main } from \"./main\";\n\nconst code = await main(process.argv.slice(2));\nprocess.exit(code);\n"],"mappings":";;;;;;;;;;;;;;;;AAeA,eAAsB,cAAc,MAAmC;CACrE,MAAM,aAAa,KAAK,WAAW;CACnC,IAAI,CAAC,YAAY;EACf,QAAQ,MAAM,oDAAoD;EAClE,OAAO;CACT;CAEA,MAAM,SAAS,UAAU,KAAK,SAAS,QAAQ,KAAK;CACpD,MAAM,eAAe,UAAU,KAAK,SAAS,UAAU;CAEvD,IAAI;CACJ,IAAI;EACF,SAAS,KAAK,MAAM,MAAM,SAAS,YAAY,MAAM,CAAC;CACxD,SAAS,KAAK;EACZ,QAAQ,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;EAC9D,OAAO;CACT;CAEA,IAAI;CACJ,IAAI,cACF,WAAW,KAAK,MAAM,MAAM,SAAS,cAAc,MAAM,CAAC;CAG5D,MAAM,YAAY,aAAa,QAAQ;EACrC,QACE,WAAW,cAAc,WAAW,SAAS,SAAS;EACxD;EACA,OAAO,WAAW;CACpB,CAAC;CAED,QAAQ,OAAO,MAAM,SAAS;CAC9B,IAAI,CAAC,UAAU,SAAS,IAAI,GAAG,QAAQ,OAAO,MAAM,IAAI;CACxD,OAAO,OAAO,MAAM,OAAO,MAAM,EAAE,MAAM,IAAI,IAAI;AACnD;;;;AC9BA,MAAM,QAAQ;AACd,MAAM,MAAM;AACZ,MAAM,SAAS;AACf,MAAM,MAAM;AACZ,MAAM,QAAQ;;;;;;AAuBd,SAAgB,oBACd,SACc;CACd,MAAM,WAAW,UAAU,SAAS,UAAU;CAC9C,IACE,aAAa,UACb,aAAa,WACb,aAAa,aACb,aAAa,WAEb,OAAO;CAET,IAAI,UAAU,SAAS,OAAO,GAAG,OAAO;CACxC,IAAI,UAAU,SAAS,SAAS,GAAG,OAAO;CAC1C,OAAO;AACT;;;;;;;AAQA,SAAgB,qBACd,SACA,SAAmB,QAAQ,QAClB;CACT,IAAI,UAAU,SAAS,UAAU,GAAG,OAAO;CAC3C,IAAI,UAAU,SAAS,OAAO,GAAG,OAAO;CACxC,IAAI,QAAQ,IAAI,aAAa,KAAA,KAAa,QAAQ,IAAI,aAAa,IACjE,OAAO;CAET,IAAI,QAAQ,IAAI,gBAAgB,KAAA,KAAa,QAAQ,IAAI,gBAAgB,KACvE,OAAO;CAET,OACE,WAAW,UACV,OAA+B,UAAU;AAE9C;;AAGA,SAAS,OAAO,OAAwB;CACtC,OAAO,QAAQ,GAAG,MAAM,GAAG,UAAU;AACvC;;AAGA,SAAS,SAAS,OAAwB;CACxC,OAAO,QAAQ,GAAG,IAAI,GAAG,UAAU;AACrC;;AAGA,SAAS,SAAS,OAAwB;CACxC,OAAO,QAAQ,GAAG,MAAM,IAAI,UAAU;AACxC;;AAGA,SAAS,WAAW,OAAwB;CAC1C,OAAO,QAAQ,GAAG,IAAI,MAAM,UAAU;AACxC;;AAGA,SAAS,UAAU,OAAwB;CACzC,OAAO,QAAQ,GAAG,MAAM,MAAM,UAAU;AAC1C;;AAGA,SAAS,UAAU,OAAwB;CACzC,OAAO,QAAQ,GAAG,IAAI,MAAM,UAAU;AACxC;;;;;;AAOA,SAAgB,yBACd,SACkB;CAClB,MAAM,SAAS,QAAQ,UAAU,QAAQ;CACzC,MAAM,OAAO,QAAQ;CACrB,MAAM,QAAQ,QAAQ,SAAS;CAE/B,IAAI,YAAY;CAChB,IAAI,YAAY;CAChB,IAAI,kBAAkB;CAEtB,QAAQ,UAAU;EAChB,QAAQ,MAAM,MAAd;GACE,KAAK;IACH,YAAY,MAAM;IAClB,YAAY;IACZ,kBAAkB;IAClB,IAAI,SAAS,SAAS;IACtB,IAAI,SAAS,QAAQ;KACnB,UAAU,QAAQ;MAChB,MAAM;MACN,WAAW,MAAM;MACjB,eAAe,QAAQ;KACzB,CAAC;KACD;IACF;IACA,MAAM,aACJ,QAAQ,kBAAkB,KAAA,IACtB,oBAAoB,QAAQ,cAAc,KAC1C;IACN,OAAO,MAAM,WAAW,UAAU,cAAc,WAAW,QAAQ;IACnE;GAEF,KAAK;IACH;IACA,mBAAmB,MAAM;IACzB,IAAI,SAAS,SAAS;KACpB,OAAO,MAAM,MAAM,KAAM,QAAQ,GAAG,MAAM,GAAG,UAAU,MAAQ,QAAQ,GAAG,IAAI,GAAG,UAAU,GAAI;KAC/F;IACF;IACA,IAAI,SAAS,QAAQ;KACnB,UAAU,QAAQ;MAChB,MAAM;MACN,OAAO;MACP,OAAO;MACP,QAAQ,MAAM;MACd,WAAW,MAAM;MACjB,UAAU,MAAM;MAChB,IAAI,MAAM;MACV,YAAY,MAAM;MAClB,eAAe,MAAM;MACrB,cAAc,MAAM;KACtB,CAAC;KACD;IACF;IAEA,MAAM,MAAM,UAAU,iBAAiB,WAAW,SAAS;IAC3D,MAAM,OAAO,MAAM,KAAK,OAAO,KAAK,IAAI,SAAS,KAAK;IACtD,MAAM,SAAS,MAAM,KAAK,SAAS,KAAK,IAAI,WAAW,KAAK;IAC5D,IAAI,OAAO,GAAG,KAAK,IAAI,UAAU,GAAG,UAAU,IAAI,MAAM,OAAO,KAAK,MAAM,UAAU,IAAI,MAAM,SAAS,IAAI,OAAO,IAAI,eAAe,MAAM,UAAU;IACrJ,IAAI,KACF,QAAQ,QACJ,KAAK,IAAI,GAAG,IAAI,GAAG,UACnB,MAAM,IAAI;IAEhB,IAAI,CAAC,MAAM,MAAM,MAAM,cACrB,QAAQ,QACJ,KAAK,OAAO,IAAI,SAAS,MAAM,cAAc,EAAE,IAAI,UACnD,OAAO,SAAS,MAAM,cAAc,EAAE;IAE5C,IAAI,SAAS,WAAW;KACtB,IAAI,MAAM,kBAAkB,KAAA,GAC1B,QAAQ,WAAW,MAAM;KAE3B,MAAM,UAAU,uBAAuB,MAAM,kBAAkB,KAAK;KACpE,IAAI,SAAS,QAAQ,KAAK;IAC5B;IACA,OAAO,MAAM,GAAG,KAAK,GAAG;IACxB;GAEF,KAAK;IACH,IAAI,SAAS,SAAS;IACtB,IAAI,SAAS,QAAQ;KACnB,UAAU,QAAQ;MAChB,MAAM;MACN,QAAQ,MAAM,OAAO;MACrB,WAAW,MAAM,OAAO,KAAK;MAC7B,QAAQ,MAAM,OAAO;MACrB,eAAe,MAAM,OAAO;MAC5B,gBAAgB,MAAM,OAAO,eAAe,KAAK,OAAO;OACtD,aAAa,EAAE;OACf,UAAU,EAAE;OACZ,gBAAgB,EAAE;MACpB,EAAE;KACJ,CAAC;KACD;IACF;IACA,OAAO,MAAM,GAAG,kBAAkB,MAAM,QAAQ,KAAK,EAAE,GAAG;IAC1D;GAEF,KAAK;IACH,IAAI,SAAS,SAAS;KACpB,OAAO,MAAM,IAAI;KACjB;IACF;IACA,IAAI,SAAS,QAAQ;KACnB,UAAU,QAAQ;MAChB,MAAM;MACN,YAAY,MAAM,OAAO;MACzB,YAAY,MAAM,OAAO,MAAM;MAC/B,aAAa,MAAM,OAAO,MAAM,QAAQ,MAAM,EAAE,MAAM,CAAC,CAAC;KAC1D,CAAC;KACD;IACF;IACA,MAAM,SAAS,MAAM,OAAO,MAAM,QAC/B,GAAG,MAAM,IAAI,EAAE,YAAY,QAAQ,MAAM,EAAE,UAAU,IAAI,CAAC,CAAC,QAC5D,CACF;IACA,MAAM,WAAW,MAAM,OAAO,MAAM,QACjC,GAAG,MAAM,IAAI,EAAE,YAAY,QAC5B,CACF;IACA,MAAM,gBAAgB,MAAM,OAAO,MAAM,QACtC,GAAG,MAAM,IAAI,EAAE,eAChB,CACF;IACA,IAAI,SAAS,iBAAiB,eAAe,MAAM,OAAO,UAAU,EAAE,IAAI,OAAO,GAAG,SAAS;IAC7F,IAAI,gBAAgB,GAClB,UAAU,KAAK,cAAc;IAE/B,UAAU;IACV,OAAO,MAAM,MAAM;IACnB;GAEF,SACE;EACJ;CACF;AACF;;AAGA,SAAgB,2BACd,SAC2C;CAC3C,MAAM,SAAS,QAAQ,UAAU,QAAQ;CACzC,MAAM,OAAO,QAAQ;CACrB,MAAM,QAAQ,QAAQ,SAAS;CAE/B,IAAI,QAAQ;CACZ,IAAI,YAAY;CAChB,IAAI,kBAAkB;CAEtB,QAAQ,UAAU;EAChB,QAAQ,MAAM,MAAd;GACE,KAAK;IACH,QAAQ,MAAM;IACd,YAAY;IACZ,kBAAkB;IAClB,IAAI,SAAS,WAAW,UAAU,GAAG;IACrC,IAAI,SAAS,QAAQ;KACnB,UAAU,QAAQ;MAChB,MAAM;MACN,OAAO,MAAM;MACb,eAAe,QAAQ;KACzB,CAAC;KACD;IACF;IACA,MAAM,aACJ,QAAQ,kBAAkB,KAAA,IACtB,oBAAoB,QAAQ,cAAc,KAC1C;IACN,OAAO,MACL,WAAW,MAAM,gBAAgB,WAAW,QAC9C;IACA;GAEF,KAAK;IACH;IACA,mBAAmB,MAAM;IACzB,IAAI,SAAS,SAAS;KACpB,MAAM,YAAY,MAAM,WAAW,KAAK,CAAC,MAAM;KAC/C,OAAO,MACL,YACI,QAAQ,GAAG,MAAM,GAAG,UAAU,MAC9B,QAAQ,GAAG,IAAI,GAAG,UAAU,GAClC;KACA;IACF;IACA,IAAI,SAAS,QAAQ;KACnB,UAAU,QAAQ;MAChB,MAAM;MACN,OAAO;MACP;MACA,QAAQ,MAAM;MACd,WAAW,MAAM;MACjB,iBAAiB,MAAM;MACvB,QAAQ,MAAM;MACd,QAAQ,MAAM;MACd,YAAY,MAAM;MAClB,aAAa,MAAM;KACrB,CAAC;KACD;IACF;IAEA,MAAM,MAAM,UAAU,iBAAiB,WAAW,KAAK;IACvD,MAAM,KAAK,MAAM,WAAW,KAAK,CAAC,MAAM;IACxC,MAAM,OAAO,KAAK,OAAO,KAAK,IAAI,SAAS,KAAK;IAChD,MAAM,SAAS,KAAK,SAAS,KAAK,IAAI,WAAW,KAAK;IACtD,IAAI,OAAO,GAAG,KAAK,IAAI,UAAU,GAAG,MAAM,IAAI,MAAM,OAAO,KAAK,MAAM,UAAU,IAAI,MAAM,gBAAgB,IAAI,OAAO,IAAI,eAAe,MAAM,UAAU;IACxJ,QAAQ,kBAAkB,MAAM,OAAO,GAAG,MAAM,SAAS,MAAM;IAC/D,IAAI,KACF,QAAQ,QAAQ,KAAK,IAAI,GAAG,IAAI,GAAG,UAAU,MAAM,IAAI;IAEzD,IAAI,MAAM,aACR,QAAQ,QACJ,KAAK,OAAO,IAAI,SAAS,MAAM,aAAa,EAAE,IAAI,UAClD,OAAO,SAAS,MAAM,aAAa,EAAE;IAE3C,IAAI,SAAS,aAAa,MAAM,UAAU,MAAM,SAAS,GACvD,QAAQ,QAAQ,KAAK,OAAO,oBAAoB,UAAU;IAE5D,OAAO,MAAM,GAAG,KAAK,GAAG;IACxB;GAEF,KAAK;IACH,IAAI,SAAS,SAAS;KACpB,OAAO,MAAM,IAAI;KACjB;IACF;IACA,IAAI,SAAS,QAAQ;KACnB,UAAU,QAAQ;MAChB,MAAM;MACN,YAAY,MAAM;MAClB,mBAAmB,MAAM;MACzB,oBAAoB,MAAM;KAC5B,CAAC;KACD;IACF;IACA,IAAI,UAAU,GAAG;IACjB,OAAO,MACL,eAAe,eAAe,MAAM,UAAU,EAAE,IAAI,MAAM,mBAAmB,GAAG,MAAM,kBAAkB,0BAC1G;IACA;GAEF,SACE;EACJ;CACF;AACF;;;;;;;AAQA,SAAS,UAAU,QAAkB,OAAsB;CACzD,OAAO,MAAM,GAAG,KAAK,UAAU,KAAK,EAAE,GAAG;AAC3C;;AAGA,SAAgB,eAAe,IAAoB;CACjD,IAAI,KAAK,KAAM,OAAO,GAAG,GAAG;CAC5B,MAAM,MAAM,KAAK;CACjB,IAAI,MAAM,IAAI,OAAO,GAAG,IAAI,QAAQ,CAAC,EAAE;CACvC,MAAM,MAAM,KAAK,MAAM,MAAM,EAAE;CAC/B,MAAM,SAAS,KAAK,MAAM,MAAM,EAAE;CAClC,IAAI,MAAM,IAAI,OAAO,GAAG,IAAI,IAAI,OAAO;CAGvC,OAAO,GAFI,KAAK,MAAM,MAAM,EAEjB,EAAE,IADE,MAAM,GACG;AAC1B;;;;;;;AAQA,SAAS,UACP,iBACA,WACA,OACoB;CACpB,IAAI,cAAc,KAAK,aAAa,OAAO,OAAO,KAAA;CAClD,MAAM,MAAM,kBAAkB;CAC9B,MAAM,aAAa,QAAQ,aAAa;CACxC,OAAO,IAAI,eAAe,KAAK,MAAM,SAAS,CAAC,EAAE;AACnD;;AAGA,SAAS,SAAS,MAAc,KAAqB;CACnD,IAAI,KAAK,UAAU,KAAK,OAAO;CAC/B,OAAO,GAAG,KAAK,MAAM,GAAG,MAAM,CAAC,EAAE;AACnC;;;;;;AAOA,SAAS,uBACP,SACA,QAAQ,OACA;CACR,IAAI,CAAC,WAAW,QAAQ,WAAW,GAAG,OAAO;CAC7C,OAAO,QACJ,KAAK,MACJ,GAAG,EAAE,SAAS,OAAO,KAAK,IAAI,SAAS,KAAK,EAAE,GAAG,EAAE,aACrD,CAAC,CACA,KAAK,IAAI;AACd;;AAGA,SAAgB,kBAAkB,MAAkB,OAAwB;CAC1E,MAAM,OAAO,KAAK,SAAS,OAAO,KAAK,IAAI,SAAS,KAAK;CACzD,MAAM,SAAS,KAAK,SAAS,UAAU,KAAK,IAAI,UAAU,KAAK;CAC/D,MAAM,QAAQ,KAAK,eAAe,KAAK,MAAM;EAC3C,MAAM,OAAO,EAAE,WAAW,IAAA,CAAK,QAAQ,CAAC;EACxC,OAAO,GAAG,EAAE,YAAY,GAAG,EAAE,YAAY,GAAG,EAAE,eAAe,IAAI,IAAI;CACvE,CAAC;CACD,MAAM,QACJ,KAAK,gBAAgB,IACjB,QACE,IAAI,OAAO,GAAG,KAAK,cAAc,kBAAkB,UACnD,KAAK,KAAK,cAAc,oBAC1B;CACN,MAAM,QAAQ,MAAM,SAAS,IAAI,KAAK,MAAM,KAAK,KAAK,MAAM;CAC5D,OAAO,GAAG,KAAK,GAAG,KAAK,OAAO,KAAK,KAAK,KAAK,MAAM,IAAI,SAAS,QAAQ;AAC1E;;;;;;;AC5aA,SAAS,kBACP,SACA,MACoB;CACpB,MAAM,MAAM,UAAU,SAAS,IAAI;CACnC,IAAI,QAAQ,KAAA,GAAW,OAAO,KAAA;CAC9B,MAAM,IAAI,OAAO,SAAS,KAAK,EAAE;CACjC,OAAO,OAAO,SAAS,CAAC,IAAI,IAAI,KAAA;AAClC;;;;;;AAOA,eAAsB,aAAa,MAAmC;CACpE,MAAM,aAAa,KAAK,WAAW;CACnC,IAAI,CAAC,YAAY;EACf,QAAQ,MACN,+KACF;EACA,OAAO;CACT;CAEA,MAAM,aAAa,UAAU,KAAK,SAAS,QAAQ;CACnD,MAAM,YAAY,UAAU,KAAK,SAAS,OAAO;CACjD,MAAM,mBAAmB,UAAU,KAAK,SAAS,cAAc;CAC/D,MAAM,aAAa,UAAU,KAAK,SAAS,QAAQ;CACnD,MAAM,QAAQ,UAAU,KAAK,SAAS,OAAO;CAC7C,MAAM,SAAS,UAAU,KAAK,SAAS,QAAQ;CAC/C,MAAM,YAAY,kBAAkB,KAAK,SAAS,YAAY;CAE9D,MAAM,gBADmB,UAAU,KAAK,SAAS,gBACZ,IACjC,aAAa,KAAK,SAAS,kBAAkB,CAAC,IAC9C,KAAA;CACJ,MAAM,SAAS,UAAU,KAAK,SAAS,QAAQ,KAAK;CACpD,MAAM,eAAe,oBAAoB,KAAK,OAAO;CACrD,MAAM,mBACJ,iBAAiB,UAAU,qBAAqB,KAAK,OAAO;CAE9D,IAAI;CACJ,MAAM,oBAAoB,cAAc;CACxC,IAAI,cAAc,WAAW;EAC3B,QAAQ,MAAM,4CAA4C;EAC1D,OAAO;CACT;CACA,IAAI,mBACF,IAAI;EACF,aAAa,MAAM,kBAAkB,iBAAiB;CACxD,SAAS,KAAK;EACZ,QAAQ,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;EAC9D,OAAO;CACT;CAGF,IAAI;CACJ,IAAI;EACF,SAAS,MAAM,gBAAgB,UAAU;CAC3C,SAAS,KAAK;EACZ,QAAQ,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;EAC9D,OAAO;CACT;CAEA,IAAI;CACJ,IAAI;EACF,eAAe,oBACb,YACA;GACE,cAAc;GACd;GACA;GACA;GACA;GACA;EACF,GACA,cAAc,SAChB;CACF,SAAS,KAAK;EACZ,QAAQ,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;EAC9D,OAAO;CACT;CAEA,MAAM,aAAa,2BAA2B;EAC5C,MAAM;EACN,eAAe,aAAa,iBAAiB;EAC7C,OAAO;CACT,CAAC;CAED,MAAM,UAAU,MAAM,YAAY,QAAQ;EACxC,GAAG;EACH;CACF,CAAC;CAED,IAAI,YACF,MAAM,UAAU,YAAY,KAAK,UAAU,SAAS,MAAM,CAAC,GAAG,MAAM;CAGtE,IAAI,WAAW,QAAQ;EACrB,QAAQ,OAAO,MAAM,KAAK,UAAU,SAAS,MAAM,CAAC,CAAC;EACrD,QAAQ,OAAO,MAAM,IAAI;CAC3B,OAAO;EACL,MAAM,YAAY,qBAAqB,SAAS,WAAW,SAAS;EACpE,QAAQ,OAAO,MAAM,SAAS;EAC9B,IAAI,CAAC,UAAU,SAAS,IAAI,GAAG,QAAQ,OAAO,MAAM,IAAI;CAC1D;CAEA,IAAI,QAAQ,QAAQ,WAAW,GAC7B,OAAO;CAGT,OAAO,oBAAoB,OAAO,IAAI,IAAI;AAC5C;;;;;;;AC/GA,eAAe,uBAAoD;CACjE,IAAI;EAKF,MAAM,OAAO,MAAM,SAJC,KAClB,QAAQ,cAAc,OAAO,KAAK,GAAG,CAAC,GACtC,uBAEoC,GAAG,MAAM;EAE/C,OADY,KAAK,MAAM,IACd,CAAC,CAAC;CACb,QAAQ;EACN;CACF;AACF;;AAGA,SAAS,oBACP,OACA,UACoB;CACpB,IAAI,CAAC,OAAO,OAAO,KAAA;CACnB,OAAO,WAAW,KAAK,KAAK,MAAM,WAAW,IAAI,IAC7C,QACA,KAAK,UAAU,KAAK;AAC1B;;;;;;AAOA,eAAsB,gBAAgB,MAAmC;CACvE,MAAM,YAAY,KAAK,WAAW;CAClC,IAAI,CAAC,WAAW;EACd,QAAQ,MACN,0LACF;EACA,OAAO;CACT;CAEA,IAAI;CACJ,IAAI;EACF,MAAM,MAAM,kBAAkB,SAAS;CACzC,SAAS,KAAK;EACZ,QAAQ,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;EAC9D,OAAO;CACT;CAEA,IAAI,CAAC,IAAI,UAAU;EACjB,QAAQ,MACN,oFACF;EACA,OAAO;CACT;CAEA,MAAM,WAAW,uBAAuB,IAAI,SAAS;CACrD,MAAM,QAAQ,UAAU,KAAK,SAAS,OAAO;CAC7C,MAAM,gBAAgB,aAAa,KAAK,SAAS,kBAAkB,CAAC;CACpE,MAAM,eAAe,oBAAoB,KAAK,OAAO;CACrD,MAAM,mBACJ,iBAAiB,UAAU,qBAAqB,KAAK,OAAO;CAE9D,MAAM,aAAa,wBACjB,UAAU,KAAK,SAAS,YAAY,CACtC;CACA,IAAI,UAAU,KAAK,SAAS,YAAY,KAAK,CAAC,YAAY;EACxD,QAAQ,MACN,mEACF;EACA,OAAO;CACT;CAEA,MAAM,YAAkC,CAAC;CACzC,MAAM,YAAY,UAAU,KAAK,SAAS,QAAQ;CAClD,IAAI,WACF,UAAU,MAAM;EACd,QAAQ,oBAAoB,WAAW,QAAQ;EAC/C;CACF;CAEF,MAAM,iBAAiB,UAAU,KAAK,SAAS,QAAQ;CACvD,IAAI,gBAAgB;EAClB,UAAU,QAAQ;GAChB,GAAG,UAAU;GACb,OAAO,oBAAoB,gBAAgB,QAAQ;EACrD;EACA,UAAU,WAAW;GACnB,GAAG,UAAU;GACb,QAAQ,oBAAoB,gBAAgB,QAAQ;EACtD;CACF;CACA,MAAM,gBAAgB,UAAU,KAAK,SAAS,gBAAgB;CAC9D,IAAI,eACF,UAAU,QAAQ;EAChB,GAAG,UAAU;EACb,QAAQ,oBAAoB,eAAe,QAAQ;CACrD;CAEF,MAAM,eAAe,UAAU,KAAK,SAAS,SAAS;CACtD,IAAI,cACF,UAAU,WAAW;EACnB,GAAG,UAAU;EACb,SAAS,oBAAoB,cAAc,QAAQ;CACrD;CAEF,MAAM,iBAAiB,UAAU,KAAK,SAAS,iBAAiB;CAChE,IAAI,gBACF,UAAU,WAAW;EACnB,GAAG,UAAU;EACb,QAAQ,oBAAoB,gBAAgB,QAAQ;CACtD;CAEF,IAAI,YACF,UAAU,WAAW;EACnB,GAAG,UAAU;EACb;CACF;CAGF,IAAI,IAAI,SAAS,SAAS,CAAC,IAAI,OAAO;EACpC,QAAQ,MAAM,gEAAgE;EAC9E,OAAO;CACT;CAGA,MAAM,mBAAmB,MAAM,qBAAqB;CAEpD,IAAI;EAiBF,QAAO,MAhBc,YAAY,KAAK;GACpC;GACA;GACA;GACA;GACA,eAAe,yBAAyB;IACtC,MAAM;IACN;IACA,OAAO;GACT,CAAC;GACD,iBAAiB,2BAA2B;IAC1C,MAAM;IACN;IACA,OAAO;GACT,CAAC;EACH,CAAC,EAAA,CACa;CAChB,SAAS,KAAK;EACZ,QAAQ,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;EAC9D,OAAO;CACT;AACF;;;;;;;AC/JA,SAAS,aAAa,OAAuB;CAC3C,OAAO,MAAM,QAAQ,qBAAqB,GAAG;AAC/C;;;;;;AAOA,eAAsB,mBACpB,OACA,QACA,WACiB;CACjB,MAAM,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;CAE1C,IAAI,UAAU;CACd,KAAK,MAAM,cAAc,OAAO,OAAO;EACrC,MAAM,WAAW,MAAM,MAAM,MAAM,MAAM,EAAE,OAAO,WAAW,MAAM;EACnE,IAAI,CAAC,UAAU;EAEf,KAAK,MAAM,OAAO,WAAW,aAAa;GACxC,IAAI,CAAC,IAAI,eAAe;GAExB,MAAM,OAAO,iBAAiB,IAAI,cAAc,MAAM,EACpD,QAAQ,SAAS,OACnB,CAAC;GAMD,MAAM,UACJ,KAAK,WAAW,GALE,aAAa,WAAW,MAAM,EAAE,IAAI,aACtD,WAAW,KAAK,KAClB,EAAE,OAAO,IAAI,gBAAgB,WAGH,GACxB,KAAK,UAAU,MAAM,MAAM,CAAC,GAC5B,MACF;GACA;EACF;CACF;CAEA,OAAO;AACT;;;;;;;;;;;AC7BA,eAAsB,WAAW,MAAmC;CAClE,MAAM,YAAY,KAAK,WAAW;CAClC,IAAI,CAAC,WAAW;EACd,QAAQ,MAAM,gDAAgD;EAC9D,OAAO;CACT;CAEA,MAAM,SAAS,UAAU,KAAK,SAAS,QAAQ,KAAK;CACpD,MAAM,aAAa,UAAU,KAAK,SAAS,QAAQ;CACnD,MAAM,gBAAgB,UAAU,KAAK,SAAS,aAAa;CAC3D,MAAM,eAAe,UAAU,KAAK,SAAS,UAAU;CACvD,MAAM,gBAAgB,aAAa,KAAK,SAAS,kBAAkB,CAAC;CACpE,MAAM,YAAY,UAAU,KAAK,SAAS,SAAS;CACnD,MAAM,eAAe,oBAAoB,KAAK,OAAO;CACrD,MAAM,mBACJ,iBAAiB,UAAU,qBAAqB,KAAK,OAAO;CAE9D,IAAI;CACJ,IAAI;EACF,QAAQ,MAAM,UAAU,SAAS;CACnC,SAAS,KAAK;EACZ,QAAQ,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;EAC9D,OAAO;CACT;CAEA,MAAM,UAAU,WAAW,aAAa,MAAM,WAAW,aAAa;CAEtE,MAAM,aAAa,yBAAyB;EAC1C,MAAM;EACN;EACA,OAAO;CACT,CAAC;CAED,MAAM,SAAS,MAAM,SAAS,OAAO;EACnC;EACA;EACA;CACF,CAAC;CAED,IAAI,YACF,MAAM,UAAU,YAAY,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,MAAM;CAGrE,IAAI,eAAe;EACjB,MAAM,QAAQ,MAAM,mBAAmB,OAAO,QAAQ,aAAa;EACnE,QAAQ,OAAO,MAAM,eAAe,MAAM,oBAAoB,cAAc,GAAG;CACjF;CAEA,IAAI;CACJ,IAAI,cAAc;EAChB,MAAM,EAAE,aAAa,MAAM,OAAO;EAClC,WAAW,KAAK,MAAM,MAAM,SAAS,cAAc,MAAM,CAAC;CAC5D;CAEA,MAAM,YAAY,aAAa,QAAQ;EACrC,QACE,WAAW,cAAc,WAAW,SAAS,SAAS;EACxD;EACA,OAAO,WAAW;CACpB,CAAC;CAED,QAAQ,OAAO,MAAM,SAAS;CAC9B,IAAI,CAAC,UAAU,SAAS,IAAI,GAAG,QAAQ,OAAO,MAAM,IAAI;CAExD,OAAO,OAAO,MAAM,OAAO,MAAM,EAAE,MAAM,IAAI,IAAI;AACnD;;;;;;;;AC5EA,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;AAwBd,eAAsB,KAAK,MAAiC;CAC1D,MAAM,SAAS,UAAU,IAAI;CAE7B,IAAI,OAAO,QAAQ,QAAQ,OAAO,YAAY,UAAU,OAAO,QAAQ,GAAG;EACxE,QAAQ,OAAO,MAAM,KAAK;EAC1B,OAAO;CACT;CAEA,QAAQ,OAAO,SAAf;EACE,KAAK,OACH,OAAO,MAAM,WAAW,MAAM;EAChC,KAAK,SACH,OAAO,MAAM,aAAa,MAAM;EAClC,KAAK,YACH,OAAO,MAAM,gBAAgB,MAAM;EACrC,KAAK,YACH,OAAO,MAAM,gBAAgB,MAAM;EACrC,KAAK,UACH,OAAO,MAAM,cAAc,MAAM;EACnC,KAAK,KAAA;GACH,QAAQ,MAAM,KAAK;GACnB,OAAO;EACT;GACE,QAAQ,MAAM,oBAAoB,OAAO,QAAQ,MAAM,OAAO;GAC9D,OAAO;CACX;AACF;;;;;;ACxDA,MAAM,OAAO,MAAM,KAAK,QAAQ,KAAK,MAAM,CAAC,CAAC;AAC7C,QAAQ,KAAK,IAAI"}
|