@skill-harness/core 0.1.2 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/types.d.ts +6 -0
- package/dist/discover.d.ts +5 -1
- package/dist/discover.js +9 -1
- package/dist/grade.d.ts +12 -1
- package/dist/grade.js +48 -10
- package/dist/index.d.ts +4 -0
- package/dist/index.js +4 -0
- package/dist/journal.d.ts +14 -0
- package/dist/lift.d.ts +69 -0
- package/dist/lift.js +163 -0
- package/dist/lint.d.ts +1 -1
- package/dist/lint.js +100 -3
- package/dist/regrade.d.ts +6 -0
- package/dist/regrade.js +25 -3
- package/dist/report.d.ts +14 -0
- package/dist/report.js +16 -1
- package/dist/rescore.d.ts +34 -0
- package/dist/rescore.js +66 -0
- package/dist/results.d.ts +9 -0
- package/dist/results.js +4 -1
- package/dist/run.d.ts +25 -1
- package/dist/run.js +126 -21
- package/dist/scaffold.d.ts +28 -0
- package/dist/scaffold.js +188 -0
- package/dist/score.d.ts +5 -1
- package/dist/seeded.js +2 -1
- package/dist/spec.d.ts +2 -0
- package/dist/spec.js +32 -0
- package/dist/util/env.d.ts +32 -0
- package/dist/util/env.js +67 -0
- package/dist/workspace.d.ts +5 -1
- package/dist/workspace.js +82 -6
- package/package.json +1 -1
package/dist/report.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type ShipBar } from "./spec.js";
|
|
2
2
|
import { type ResultsFile } from "./results.js";
|
|
3
|
+
import { type Lift } from "./lift.js";
|
|
3
4
|
export interface RunColumn {
|
|
4
5
|
index: number;
|
|
5
6
|
label: string;
|
|
@@ -20,6 +21,17 @@ export interface RunColumn {
|
|
|
20
21
|
override: string | null;
|
|
21
22
|
note: string;
|
|
22
23
|
}>;
|
|
24
|
+
/**
|
|
25
|
+
* Red-vs-green lift for this model, when the tag has both a red baseline and a
|
|
26
|
+
* green run. Undefined means "never measured" — which is not the same claim as
|
|
27
|
+
* a zero lift, so the report must not render a 0 for it.
|
|
28
|
+
*
|
|
29
|
+
* Only set when THIS column is the green run the lift was computed from (see
|
|
30
|
+
* collectReport): the review UI recomputes lift from the column's live cells,
|
|
31
|
+
* which is only valid if those cells are the green side of the comparison.
|
|
32
|
+
*/
|
|
33
|
+
lift?: Lift;
|
|
34
|
+
liftHeadline?: string;
|
|
23
35
|
}
|
|
24
36
|
export interface ReportData {
|
|
25
37
|
skill: string;
|
|
@@ -48,6 +60,8 @@ export declare function publicView(data: ReportData): {
|
|
|
48
60
|
critical: boolean;
|
|
49
61
|
}[];
|
|
50
62
|
columns: {
|
|
63
|
+
lift?: Lift | undefined;
|
|
64
|
+
liftHeadline?: string | undefined;
|
|
51
65
|
index: number;
|
|
52
66
|
label: string;
|
|
53
67
|
tag: string;
|
package/dist/report.js
CHANGED
|
@@ -2,6 +2,7 @@ import { existsSync, readdirSync, statSync } from "node:fs";
|
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { loadSpec } from "./spec.js";
|
|
4
4
|
import { readResults } from "./results.js";
|
|
5
|
+
import { collectLift, liftHeadline } from "./lift.js";
|
|
5
6
|
/** Most-recent run dir (by name, which is an ISO-ish slug) under a model-tag dir. */
|
|
6
7
|
function latestRunDir(tagDir) {
|
|
7
8
|
if (!statSync(tagDir).isDirectory())
|
|
@@ -21,6 +22,8 @@ export function collectReport(skillDir) {
|
|
|
21
22
|
const spec = loadSpec(specPath);
|
|
22
23
|
const scenarios = spec.scenarios.map((s) => ({ id: s.id, title: s.title, critical: s.critical }));
|
|
23
24
|
const resultsRoot = join(skillDir, "tests", "results");
|
|
25
|
+
// Lift is keyed by model tag, the same key columns are built from.
|
|
26
|
+
const liftByTag = new Map(collectLift(skillDir).map((l) => [l.tag, l]));
|
|
24
27
|
const columns = [];
|
|
25
28
|
if (existsSync(resultsRoot)) {
|
|
26
29
|
const tags = readdirSync(resultsRoot)
|
|
@@ -46,16 +49,27 @@ export function collectReport(skillDir) {
|
|
|
46
49
|
note: s.note,
|
|
47
50
|
};
|
|
48
51
|
}
|
|
52
|
+
const tag = tagDir.split("/").pop();
|
|
53
|
+
// A column is the tag's LATEST run, which is not necessarily the green one —
|
|
54
|
+
// record a red baseline after a green run and the newest run in the tag is
|
|
55
|
+
// red. The review UI recomputes lift from `cells` (so author overrides move
|
|
56
|
+
// it live), so attaching a lift to a column whose cells are the RED run
|
|
57
|
+
// would have it compare red against red and report "no effect" for a skill
|
|
58
|
+
// that in fact gained every scenario. Attach only when this column IS the
|
|
59
|
+
// green side of the comparison.
|
|
60
|
+
const tagLift = liftByTag.get(tag);
|
|
61
|
+
const lift = tagLift && tagLift.greenTimestamp === r.timestamp ? tagLift : undefined;
|
|
49
62
|
columns.push({
|
|
50
63
|
index: columns.length,
|
|
51
64
|
label: r.model,
|
|
52
|
-
tag
|
|
65
|
+
tag,
|
|
53
66
|
runDir,
|
|
54
67
|
timestamp: r.timestamp,
|
|
55
68
|
mode: r.mode,
|
|
56
69
|
grade: r.effective_grade,
|
|
57
70
|
judge: r.judge,
|
|
58
71
|
cells,
|
|
72
|
+
...(lift ? { lift, liftHeadline: liftHeadline(lift) } : {}),
|
|
59
73
|
});
|
|
60
74
|
}
|
|
61
75
|
}
|
|
@@ -77,6 +91,7 @@ export function publicView(data) {
|
|
|
77
91
|
grade: c.grade,
|
|
78
92
|
judge: c.judge,
|
|
79
93
|
cells: c.cells,
|
|
94
|
+
...(c.lift ? { lift: c.lift, liftHeadline: c.liftHeadline } : {}),
|
|
80
95
|
})),
|
|
81
96
|
};
|
|
82
97
|
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { Spec } from "./spec.js";
|
|
2
|
+
import { type ResultsFile, type ScenarioResult } from "./results.js";
|
|
3
|
+
export interface RescoreOptions {
|
|
4
|
+
runDir: string;
|
|
5
|
+
spec: Spec;
|
|
6
|
+
now?: () => string;
|
|
7
|
+
}
|
|
8
|
+
export interface RescoreChange {
|
|
9
|
+
id: string;
|
|
10
|
+
from: ScenarioResult["judge_verdict"];
|
|
11
|
+
to: ScenarioResult["judge_verdict"];
|
|
12
|
+
passes: number;
|
|
13
|
+
clean: number;
|
|
14
|
+
fromThreshold: number;
|
|
15
|
+
toThreshold: number;
|
|
16
|
+
}
|
|
17
|
+
export interface RescoreResult {
|
|
18
|
+
results: ResultsFile;
|
|
19
|
+
changes: RescoreChange[];
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Re-score a run against the CURRENT spec's pass thresholds — no model calls, no judge
|
|
23
|
+
* calls. Reps are the raw measurement (`passes` of `clean`); a threshold is *policy*.
|
|
24
|
+
* When the policy changes, the honest move is to recompute the old measurements under it
|
|
25
|
+
* rather than reconcile two numbers in prose — and to record what moved.
|
|
26
|
+
*
|
|
27
|
+
* Only reps-bearing scenarios can be re-scored: a single-rep verdict has no rate to
|
|
28
|
+
* re-apply a threshold to, and ERROR/JUDGE-AMBIGUOUS carry no trustworthy rate at all —
|
|
29
|
+
* both are carried verbatim. Overrides, notes, and suspect flags are preserved: this
|
|
30
|
+
* changes the collapse rule, nothing about what the judge said.
|
|
31
|
+
*/
|
|
32
|
+
export declare function rescoreRun(opts: RescoreOptions): RescoreResult;
|
|
33
|
+
/** Locate the spec for a run dir: results/<tag>/<ts> → ../../../specification.yaml */
|
|
34
|
+
export declare function specPathForRunDir(runDir: string): string;
|
package/dist/rescore.js
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { readResults, writeResults } from "./results.js";
|
|
4
|
+
import { appendJournal } from "./journal.js";
|
|
5
|
+
/**
|
|
6
|
+
* Re-score a run against the CURRENT spec's pass thresholds — no model calls, no judge
|
|
7
|
+
* calls. Reps are the raw measurement (`passes` of `clean`); a threshold is *policy*.
|
|
8
|
+
* When the policy changes, the honest move is to recompute the old measurements under it
|
|
9
|
+
* rather than reconcile two numbers in prose — and to record what moved.
|
|
10
|
+
*
|
|
11
|
+
* Only reps-bearing scenarios can be re-scored: a single-rep verdict has no rate to
|
|
12
|
+
* re-apply a threshold to, and ERROR/JUDGE-AMBIGUOUS carry no trustworthy rate at all —
|
|
13
|
+
* both are carried verbatim. Overrides, notes, and suspect flags are preserved: this
|
|
14
|
+
* changes the collapse rule, nothing about what the judge said.
|
|
15
|
+
*/
|
|
16
|
+
export function rescoreRun(opts) {
|
|
17
|
+
const now = opts.now ?? (() => new Date().toISOString());
|
|
18
|
+
const prev = readResults(opts.runDir);
|
|
19
|
+
if (!prev)
|
|
20
|
+
throw new Error(`no results.yaml in ${opts.runDir}`);
|
|
21
|
+
const specById = new Map(opts.spec.scenarios.map((s) => [s.id, s]));
|
|
22
|
+
const changes = [];
|
|
23
|
+
const scenarios = prev.scenarios.map((s) => {
|
|
24
|
+
const scenario = specById.get(s.id);
|
|
25
|
+
// no rate to re-apply, or an untrustworthy verdict → carry verbatim
|
|
26
|
+
if (!scenario || s.reps === undefined || s.clean === undefined || s.passes === undefined)
|
|
27
|
+
return s;
|
|
28
|
+
if (s.judge_verdict === "ERROR" || s.judge_verdict === "JUDGE-AMBIGUOUS")
|
|
29
|
+
return s;
|
|
30
|
+
if (s.clean === 0)
|
|
31
|
+
return s;
|
|
32
|
+
const toThreshold = scenario.passThreshold ?? 0.5;
|
|
33
|
+
const fromThreshold = s.pass_threshold ?? toThreshold;
|
|
34
|
+
if (toThreshold === fromThreshold)
|
|
35
|
+
return s;
|
|
36
|
+
const rate = s.passes / s.clean;
|
|
37
|
+
const verdict = rate >= toThreshold ? "PASS" : "FAIL";
|
|
38
|
+
if (verdict !== s.judge_verdict) {
|
|
39
|
+
changes.push({ id: s.id, from: s.judge_verdict, to: verdict, passes: s.passes, clean: s.clean, fromThreshold, toThreshold });
|
|
40
|
+
}
|
|
41
|
+
return { ...s, judge_verdict: verdict, pass_threshold: toThreshold };
|
|
42
|
+
});
|
|
43
|
+
const ctx = prev.mode === "green" && !prev.partial
|
|
44
|
+
? { shipBar: opts.spec.ship_bar, critical: opts.spec.critical }
|
|
45
|
+
: null;
|
|
46
|
+
const results = writeResults(opts.runDir, {
|
|
47
|
+
skill: prev.skill, harness: prev.harness, model: prev.model, judge: prev.judge,
|
|
48
|
+
timestamp: prev.timestamp, label: prev.label, mode: prev.mode,
|
|
49
|
+
partial: prev.partial, source_hashes: prev.source_hashes, scenarios,
|
|
50
|
+
}, ctx);
|
|
51
|
+
appendJournal(opts.runDir, {
|
|
52
|
+
event: "rescore", ts: now(),
|
|
53
|
+
changed: changes.map((c) => `${c.id}: ${c.from}->${c.to} (${c.passes}/${c.clean} @ ${c.toThreshold})`),
|
|
54
|
+
passed: results.effective_grade.passed, total: results.effective_grade.total,
|
|
55
|
+
pct: results.effective_grade.pct, ship: results.effective_grade.ship,
|
|
56
|
+
});
|
|
57
|
+
return { results, changes };
|
|
58
|
+
}
|
|
59
|
+
/** Locate the spec for a run dir: results/<tag>/<ts> → ../../../specification.yaml */
|
|
60
|
+
export function specPathForRunDir(runDir) {
|
|
61
|
+
const p = join(runDir, "..", "..", "..", "specification.yaml");
|
|
62
|
+
if (!existsSync(p))
|
|
63
|
+
throw new Error(`no specification.yaml above ${runDir}`);
|
|
64
|
+
return p;
|
|
65
|
+
}
|
|
66
|
+
//# sourceMappingURL=rescore.js.map
|
package/dist/results.d.ts
CHANGED
|
@@ -35,6 +35,15 @@ export interface ResultsFile {
|
|
|
35
35
|
timestamp: string;
|
|
36
36
|
label: string | null;
|
|
37
37
|
mode: string;
|
|
38
|
+
/** True for an `--only`-filtered run: a scenario subset, never ship-graded, never a release run. */
|
|
39
|
+
partial?: boolean;
|
|
40
|
+
/**
|
|
41
|
+
* sha256 of every source file this run measured: SKILL.md plus each distinct
|
|
42
|
+
* system_prompt_file. Lint compares the newest run's hashes against the current
|
|
43
|
+
* files — a mismatch means the published result describes text that no longer
|
|
44
|
+
* exists (the stale-scorecard class this field exists to kill).
|
|
45
|
+
*/
|
|
46
|
+
source_hashes?: Record<string, string>;
|
|
38
47
|
effective_grade: GradeSummary;
|
|
39
48
|
scenarios: ScenarioResult[];
|
|
40
49
|
}
|
package/dist/results.js
CHANGED
|
@@ -46,7 +46,8 @@ export function finalizeResults(draft, ctx) {
|
|
|
46
46
|
effective_grade = { passed: s.passed, total: s.total, pct: s.pct, letter: s.letter, ship: s.ship, note: s.note };
|
|
47
47
|
}
|
|
48
48
|
else {
|
|
49
|
-
|
|
49
|
+
const why = draft.partial ? "partial run (--only) — not scored" : `mode=${draft.mode} (not scored)`;
|
|
50
|
+
effective_grade = { passed: 0, total: 0, pct: 0, letter: "-", ship: false, note: why };
|
|
50
51
|
}
|
|
51
52
|
return {
|
|
52
53
|
schema: 2,
|
|
@@ -57,6 +58,8 @@ export function finalizeResults(draft, ctx) {
|
|
|
57
58
|
timestamp: draft.timestamp,
|
|
58
59
|
label: draft.label,
|
|
59
60
|
mode: draft.mode,
|
|
61
|
+
...(draft.partial ? { partial: true } : {}),
|
|
62
|
+
...(draft.source_hashes ? { source_hashes: draft.source_hashes } : {}),
|
|
60
63
|
effective_grade,
|
|
61
64
|
scenarios: draft.scenarios,
|
|
62
65
|
};
|
package/dist/run.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { Spec } from "./spec.js";
|
|
2
2
|
import type { HarnessAdapter, ModelRef, RunMode } from "./adapters/types.js";
|
|
3
3
|
import { type ResultsFile } from "./results.js";
|
|
4
|
+
import { type Lift } from "./lift.js";
|
|
4
5
|
export interface RunOptions {
|
|
5
6
|
spec: Spec;
|
|
6
7
|
skillDir: string;
|
|
@@ -17,6 +18,12 @@ export interface RunOptions {
|
|
|
17
18
|
concurrency?: number;
|
|
18
19
|
reps?: number;
|
|
19
20
|
passThreshold?: number;
|
|
21
|
+
/**
|
|
22
|
+
* Run only these scenario ids — the iteration tool (re-testing 2 D-scenarios must not
|
|
23
|
+
* cost an 18-scenario run). The result is marked `partial: true` and is NEVER
|
|
24
|
+
* ship-graded: a subset passing says nothing about the ship bar.
|
|
25
|
+
*/
|
|
26
|
+
only?: string[];
|
|
20
27
|
}
|
|
21
28
|
export interface RunSummary {
|
|
22
29
|
runDir: string;
|
|
@@ -24,5 +31,22 @@ export interface RunSummary {
|
|
|
24
31
|
}
|
|
25
32
|
/** Run one skill against one model: run scenarios, grade, score, persist. */
|
|
26
33
|
export declare function runSkillModel(opts: RunOptions): Promise<RunSummary>;
|
|
34
|
+
/**
|
|
35
|
+
* True when any assistant turn in a transcript is blank — the shape a harness timeout
|
|
36
|
+
* leaves behind. Such a transcript must never reach the judge: grading an empty reply
|
|
37
|
+
* produces a confident FAIL about behavior that never happened (round 9 lost two
|
|
38
|
+
* scenarios this way). Sections are delimited by the adapters' shared transcript
|
|
39
|
+
* convention (">>> USER"/"<<< ASSISTANT:"); seeded gate output ("=== SEEDED GATES ===")
|
|
40
|
+
* ends the last assistant section.
|
|
41
|
+
*/
|
|
42
|
+
export declare function hasEmptyAssistantTurn(transcript: string): boolean;
|
|
27
43
|
/** A compact terminal scorecard for one run. */
|
|
28
|
-
|
|
44
|
+
/**
|
|
45
|
+
* The human-facing scorecard for one run.
|
|
46
|
+
*
|
|
47
|
+
* `lift` is the red-vs-green comparison for this model when a red baseline
|
|
48
|
+
* exists. Passing it for a green run turns the scorecard from "the skill scored
|
|
49
|
+
* B" into "the skill *did* this much" — without a baseline the grade alone can't
|
|
50
|
+
* distinguish a skill that works from a model that never needed it.
|
|
51
|
+
*/
|
|
52
|
+
export declare function formatScorecard(summary: RunSummary, lift?: Lift): string;
|
package/dist/run.js
CHANGED
|
@@ -1,18 +1,62 @@
|
|
|
1
|
-
import { mkdirSync, writeFileSync } from "node:fs";
|
|
2
|
-
import {
|
|
1
|
+
import { mkdirSync, writeFileSync, readFileSync } from "node:fs";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { dirname, resolve } from "node:path";
|
|
3
4
|
import { judgeResemblesSubject } from "./grade.js";
|
|
4
5
|
import { runDirFor, transcriptPath, writeResults, ensureResultsGitignore, } from "./results.js";
|
|
5
6
|
import { appendJournal } from "./journal.js";
|
|
7
|
+
import { liftHeadline } from "./lift.js";
|
|
6
8
|
import { runSeeded } from "./seeded.js";
|
|
7
9
|
import { createWorkspace } from "./workspace.js";
|
|
8
10
|
import { runPool } from "./scheduler.js";
|
|
9
11
|
import { outcomesToResult } from "./reps.js";
|
|
10
12
|
import { judgeOneRep } from "./regrade.js";
|
|
13
|
+
/** sha256 of a file, or null when it doesn't exist — missing sources are lint's problem, not run's. */
|
|
14
|
+
function sha256(path) {
|
|
15
|
+
try {
|
|
16
|
+
return createHash("sha256").update(readFileSync(path)).digest("hex");
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
return null;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Hash every source file this run measures: SKILL.md + each distinct
|
|
24
|
+
* system_prompt_file (agents/<name>.md). Recorded in results.yaml so lint can prove
|
|
25
|
+
* a published result still describes the current text.
|
|
26
|
+
*/
|
|
27
|
+
function sourceHashes(skillDir, specPath, scenarios) {
|
|
28
|
+
const hashes = {};
|
|
29
|
+
const skillMd = sha256(resolve(skillDir, "SKILL.md"));
|
|
30
|
+
if (skillMd)
|
|
31
|
+
hashes["SKILL.md"] = skillMd;
|
|
32
|
+
for (const s of scenarios) {
|
|
33
|
+
if (s.systemPromptFile && !(s.systemPromptFile in hashes)) {
|
|
34
|
+
const h = sha256(resolve(dirname(specPath), s.systemPromptFile));
|
|
35
|
+
if (h)
|
|
36
|
+
hashes[s.systemPromptFile] = h;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return hashes;
|
|
40
|
+
}
|
|
11
41
|
/** Run one skill against one model: run scenarios, grade, score, persist. */
|
|
12
42
|
export async function runSkillModel(opts) {
|
|
13
43
|
const { spec, skillDir, adapter, model, judge, mode, timestamp } = opts;
|
|
14
44
|
const log = opts.onProgress ?? (() => { });
|
|
15
45
|
const now = opts.now ?? (() => new Date().toISOString());
|
|
46
|
+
// --only: validate against the spec BEFORE spending anything — a typo'd id must not
|
|
47
|
+
// silently run zero scenarios and report success.
|
|
48
|
+
let scenarios = spec.scenarios;
|
|
49
|
+
const partial = Boolean(opts.only && opts.only.length > 0);
|
|
50
|
+
if (partial) {
|
|
51
|
+
const known = new Set(spec.scenarios.map((s) => s.id));
|
|
52
|
+
const unknown = opts.only.filter((id) => !known.has(id));
|
|
53
|
+
if (unknown.length > 0) {
|
|
54
|
+
throw new Error(`--only names unknown scenario id(s) ${unknown.join(", ")} — spec has: ${[...known].join(", ")}`);
|
|
55
|
+
}
|
|
56
|
+
const wanted = new Set(opts.only);
|
|
57
|
+
scenarios = spec.scenarios.filter((s) => wanted.has(s.id));
|
|
58
|
+
log(` --only ${opts.only.join(",")} — partial run, will not be ship-graded`);
|
|
59
|
+
}
|
|
16
60
|
if (judgeResemblesSubject(judge, model)) {
|
|
17
61
|
log(` ⚠ judge (${judge.provider}:${judge.model}) resembles the model under test ` +
|
|
18
62
|
`(${model.provider}:${model.model}) — verdicts may be inflated. Use a distinct judge.`);
|
|
@@ -27,10 +71,10 @@ export async function runSkillModel(opts) {
|
|
|
27
71
|
mode, label: opts.label ?? null,
|
|
28
72
|
});
|
|
29
73
|
// scenario × rep tasks; runPool preserves input order so we can slice per scenario.
|
|
30
|
-
const repCounts =
|
|
74
|
+
const repCounts = scenarios.map((s) => s.reps ?? opts.reps ?? 1);
|
|
31
75
|
const owners = [];
|
|
32
76
|
const tasks = [];
|
|
33
|
-
|
|
77
|
+
scenarios.forEach((scenario, si) => {
|
|
34
78
|
for (let k = 0; k < repCounts[si]; k++) {
|
|
35
79
|
const rep = k;
|
|
36
80
|
const total = repCounts[si];
|
|
@@ -39,13 +83,13 @@ export async function runSkillModel(opts) {
|
|
|
39
83
|
}
|
|
40
84
|
});
|
|
41
85
|
const flat = await runPool(tasks, opts.concurrency ?? 1);
|
|
42
|
-
const grouped =
|
|
86
|
+
const grouped = scenarios.map(() => []);
|
|
43
87
|
flat.forEach((outcome, i) => grouped[owners[i]].push(outcome));
|
|
44
|
-
const scenarioResults =
|
|
88
|
+
const scenarioResults = scenarios.map((scenario, si) => {
|
|
45
89
|
const threshold = scenario.passThreshold ?? opts.passThreshold ?? 0.5;
|
|
46
90
|
return outcomesToResult(scenario.id, grouped[si], repCounts[si], threshold);
|
|
47
91
|
});
|
|
48
|
-
const ctx = mode === "green" ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
|
|
92
|
+
const ctx = mode === "green" && !partial ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
|
|
49
93
|
const results = writeResults(runDir, {
|
|
50
94
|
skill: spec.skill,
|
|
51
95
|
harness: adapter.name,
|
|
@@ -54,6 +98,8 @@ export async function runSkillModel(opts) {
|
|
|
54
98
|
timestamp,
|
|
55
99
|
label: opts.label ?? null,
|
|
56
100
|
mode,
|
|
101
|
+
...(partial ? { partial: true } : {}),
|
|
102
|
+
source_hashes: sourceHashes(skillDir, opts.specPath, scenarios),
|
|
57
103
|
scenarios: scenarioResults,
|
|
58
104
|
}, ctx);
|
|
59
105
|
if (ctx) {
|
|
@@ -62,6 +108,23 @@ export async function runSkillModel(opts) {
|
|
|
62
108
|
}
|
|
63
109
|
return { runDir, results };
|
|
64
110
|
}
|
|
111
|
+
/**
|
|
112
|
+
* True when any assistant turn in a transcript is blank — the shape a harness timeout
|
|
113
|
+
* leaves behind. Such a transcript must never reach the judge: grading an empty reply
|
|
114
|
+
* produces a confident FAIL about behavior that never happened (round 9 lost two
|
|
115
|
+
* scenarios this way). Sections are delimited by the adapters' shared transcript
|
|
116
|
+
* convention (">>> USER"/"<<< ASSISTANT:"); seeded gate output ("=== SEEDED GATES ===")
|
|
117
|
+
* ends the last assistant section.
|
|
118
|
+
*/
|
|
119
|
+
export function hasEmptyAssistantTurn(transcript) {
|
|
120
|
+
const sections = transcript.split(/^<<< ASSISTANT:\s*$/m).slice(1);
|
|
121
|
+
if (sections.length === 0)
|
|
122
|
+
return false;
|
|
123
|
+
return sections.some((sec) => {
|
|
124
|
+
const body = sec.split(/^(?:>>> |=== SEEDED GATES ===|\[pi exited )/m)[0];
|
|
125
|
+
return body.trim() === "";
|
|
126
|
+
});
|
|
127
|
+
}
|
|
65
128
|
/** Run ONE rep of a scenario in its own isolated workspace. */
|
|
66
129
|
async function runRep(scenario, rep, repCount, ctx) {
|
|
67
130
|
const { spec, judge, mode, runDir, now, log } = ctx;
|
|
@@ -75,25 +138,44 @@ async function runRep(scenario, rep, repCount, ctx) {
|
|
|
75
138
|
let gatePrefix = null;
|
|
76
139
|
try {
|
|
77
140
|
try {
|
|
78
|
-
ws = createWorkspace(scenario.workspace, { specDir: dirname(ctx.specPath) });
|
|
141
|
+
ws = createWorkspace(scenario.workspace, { specDir: dirname(ctx.specPath), remote: scenario.remote });
|
|
79
142
|
}
|
|
80
143
|
catch (e) {
|
|
81
144
|
// A setup failure (e.g. missing fixture) is an objective FAIL, not an infra abort.
|
|
82
145
|
gatePrefix = e instanceof Error ? e.message : String(e);
|
|
83
146
|
transcript = `[workspace setup failed] ${gatePrefix}`;
|
|
84
147
|
}
|
|
148
|
+
let noResponse = false;
|
|
85
149
|
if (ws) {
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
150
|
+
// A blank assistant turn is a harness timeout, not model behavior: retry ONCE in a
|
|
151
|
+
// fresh workspace (the first attempt may have half-mutated a seeded repo), and if
|
|
152
|
+
// it happens again the verdict is ERROR — never a judged FAIL on an empty reply.
|
|
153
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
154
|
+
if (attempt > 0) {
|
|
155
|
+
appendJournal(runDir, { event: "empty-response-retry", ts: now(), id: scenario.id, attempt, ...repField });
|
|
156
|
+
log(` ${scenario.id}${repCount > 1 ? `#${rep}` : ""} empty response — retrying once`);
|
|
157
|
+
ws.cleanup();
|
|
158
|
+
ws = createWorkspace(scenario.workspace, { specDir: dirname(ctx.specPath), remote: scenario.remote });
|
|
159
|
+
}
|
|
160
|
+
if (scenario.mode === "seeded") {
|
|
161
|
+
const r = await runSeeded(scenario, {
|
|
162
|
+
skillDir: ctx.skillDir, adapter: ctx.adapter, model: ctx.model, mode, cwd: ws.cwd,
|
|
163
|
+
});
|
|
164
|
+
transcript = r.transcript;
|
|
165
|
+
gatePrefix = r.gateFailure;
|
|
166
|
+
}
|
|
167
|
+
else {
|
|
168
|
+
transcript = await ctx.adapter.run({
|
|
169
|
+
skillDir: ctx.skillDir, model: ctx.model, mode, turns: scenario.turns, cwd: ws.cwd,
|
|
170
|
+
// resolved like fixtures: relative to the spec's dir
|
|
171
|
+
systemPromptFile: scenario.systemPromptFile
|
|
172
|
+
? resolve(dirname(ctx.specPath), scenario.systemPromptFile)
|
|
173
|
+
: undefined,
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
noResponse = hasEmptyAssistantTurn(transcript);
|
|
177
|
+
if (!noResponse)
|
|
178
|
+
break;
|
|
97
179
|
}
|
|
98
180
|
}
|
|
99
181
|
writeFileSync(transcriptPath(runDir, scenario.id, mode, repCount > 1 ? rep : undefined), transcript, "utf8");
|
|
@@ -103,7 +185,12 @@ async function runRep(scenario, rep, repCount, ctx) {
|
|
|
103
185
|
let verdict;
|
|
104
186
|
let reason;
|
|
105
187
|
let suspect = false;
|
|
106
|
-
if (
|
|
188
|
+
if (noResponse) {
|
|
189
|
+
verdict = "ERROR";
|
|
190
|
+
reason = "model produced no response after a retry (harness timeout?) — infra, not skill behavior";
|
|
191
|
+
appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict, reason, suspect, ...repField });
|
|
192
|
+
}
|
|
193
|
+
else if (gatePrefix) {
|
|
107
194
|
verdict = "FAIL";
|
|
108
195
|
reason = gatePrefix;
|
|
109
196
|
// gate failures don't invoke the judge, but still record a judge-verdict event (as before)
|
|
@@ -126,7 +213,15 @@ async function runRep(scenario, rep, repCount, ctx) {
|
|
|
126
213
|
}
|
|
127
214
|
}
|
|
128
215
|
/** A compact terminal scorecard for one run. */
|
|
129
|
-
|
|
216
|
+
/**
|
|
217
|
+
* The human-facing scorecard for one run.
|
|
218
|
+
*
|
|
219
|
+
* `lift` is the red-vs-green comparison for this model when a red baseline
|
|
220
|
+
* exists. Passing it for a green run turns the scorecard from "the skill scored
|
|
221
|
+
* B" into "the skill *did* this much" — without a baseline the grade alone can't
|
|
222
|
+
* distinguish a skill that works from a model that never needed it.
|
|
223
|
+
*/
|
|
224
|
+
export function formatScorecard(summary, lift) {
|
|
130
225
|
const { results } = summary;
|
|
131
226
|
const g = results.effective_grade;
|
|
132
227
|
const lines = [];
|
|
@@ -143,6 +238,16 @@ export function formatScorecard(summary) {
|
|
|
143
238
|
const ship = g.ship ? "SHIP" : "NOT READY";
|
|
144
239
|
const note = g.note ? ` (${g.note})` : "";
|
|
145
240
|
lines.push(` GRADE: ${g.letter} (${g.pct}%) — ${g.passed}/${g.total} — ${ship}${note}`);
|
|
241
|
+
// Lift is a statement about a green run. On a red run the caller may still have
|
|
242
|
+
// a lift in hand (a green run exists in the same tag), but printing it under a
|
|
243
|
+
// baseline scorecard reads as if the baseline itself gained something.
|
|
244
|
+
if (lift && results.mode === "green") {
|
|
245
|
+
lines.push(` LIFT: ${liftHeadline(lift)} (vs red baseline ${lift.redTimestamp})`);
|
|
246
|
+
}
|
|
247
|
+
else if (results.mode === "green") {
|
|
248
|
+
// The grade alone can't answer "does this skill do anything?", so say how.
|
|
249
|
+
lines.push(` LIFT: no red baseline — run with --mode red to measure what the skill adds`);
|
|
250
|
+
}
|
|
146
251
|
return lines.join("\n");
|
|
147
252
|
}
|
|
148
253
|
//# sourceMappingURL=run.js.map
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/** Marker written into an `init` template's first comment. Its presence tells
|
|
2
|
+
* `suggest` the file is an unadopted template it may overwrite without --force. */
|
|
3
|
+
export declare const TEMPLATE_SENTINEL = "skill-harness: generated template";
|
|
4
|
+
/** Render a commented, empty-but-valid specification.yaml for a skill. */
|
|
5
|
+
export declare function renderTemplateSpec(skillName: string): string;
|
|
6
|
+
/** True if the text still carries the template sentinel (i.e. an unadopted template). */
|
|
7
|
+
export declare function isTemplateSpec(text: string): boolean;
|
|
8
|
+
export interface DraftScenario {
|
|
9
|
+
id: string;
|
|
10
|
+
title: string;
|
|
11
|
+
turns: string[];
|
|
12
|
+
checklist: string[];
|
|
13
|
+
}
|
|
14
|
+
export interface SuggestDraft {
|
|
15
|
+
judge_persona: string;
|
|
16
|
+
ship_bar: {
|
|
17
|
+
total: number;
|
|
18
|
+
min_pass: number;
|
|
19
|
+
no_critical_fail: boolean;
|
|
20
|
+
};
|
|
21
|
+
proposed_critical: string[];
|
|
22
|
+
scenarios: DraftScenario[];
|
|
23
|
+
}
|
|
24
|
+
/** Render a populated spec from an LLM draft. Strings are JSON-encoded (valid YAML
|
|
25
|
+
* flow scalars) so colons/quotes never break the file. Carries no sentinel. */
|
|
26
|
+
export declare function renderDraftSpec(skillName: string, draft: SuggestDraft): string;
|
|
27
|
+
export declare function buildSuggestPrompt(skillName: string, skillMd: string): string;
|
|
28
|
+
export declare function parseSuggestDraft(raw: string): SuggestDraft;
|