@skill-harness/core 0.1.2 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/report.d.ts CHANGED
@@ -1,5 +1,6 @@
1
1
  import { type ShipBar } from "./spec.js";
2
2
  import { type ResultsFile } from "./results.js";
3
+ import { type Lift } from "./lift.js";
3
4
  export interface RunColumn {
4
5
  index: number;
5
6
  label: string;
@@ -20,6 +21,17 @@ export interface RunColumn {
20
21
  override: string | null;
21
22
  note: string;
22
23
  }>;
24
+ /**
25
+ * Red-vs-green lift for this model, when the tag has both a red baseline and a
26
+ * green run. Undefined means "never measured" — which is not the same claim as
27
+ * a zero lift, so the report must not render a 0 for it.
28
+ *
29
+ * Only set when THIS column is the green run the lift was computed from (see
30
+ * collectReport): the review UI recomputes lift from the column's live cells,
31
+ * which is only valid if those cells are the green side of the comparison.
32
+ */
33
+ lift?: Lift;
34
+ liftHeadline?: string;
23
35
  }
24
36
  export interface ReportData {
25
37
  skill: string;
@@ -48,6 +60,8 @@ export declare function publicView(data: ReportData): {
48
60
  critical: boolean;
49
61
  }[];
50
62
  columns: {
63
+ lift?: Lift | undefined;
64
+ liftHeadline?: string | undefined;
51
65
  index: number;
52
66
  label: string;
53
67
  tag: string;
package/dist/report.js CHANGED
@@ -2,6 +2,7 @@ import { existsSync, readdirSync, statSync } from "node:fs";
2
2
  import { join } from "node:path";
3
3
  import { loadSpec } from "./spec.js";
4
4
  import { readResults } from "./results.js";
5
+ import { collectLift, liftHeadline } from "./lift.js";
5
6
  /** Most-recent run dir (by name, which is an ISO-ish slug) under a model-tag dir. */
6
7
  function latestRunDir(tagDir) {
7
8
  if (!statSync(tagDir).isDirectory())
@@ -21,6 +22,8 @@ export function collectReport(skillDir) {
21
22
  const spec = loadSpec(specPath);
22
23
  const scenarios = spec.scenarios.map((s) => ({ id: s.id, title: s.title, critical: s.critical }));
23
24
  const resultsRoot = join(skillDir, "tests", "results");
25
+ // Lift is keyed by model tag, the same key columns are built from.
26
+ const liftByTag = new Map(collectLift(skillDir).map((l) => [l.tag, l]));
24
27
  const columns = [];
25
28
  if (existsSync(resultsRoot)) {
26
29
  const tags = readdirSync(resultsRoot)
@@ -46,16 +49,27 @@ export function collectReport(skillDir) {
46
49
  note: s.note,
47
50
  };
48
51
  }
52
+ const tag = tagDir.split("/").pop();
53
+ // A column is the tag's LATEST run, which is not necessarily the green one —
54
+ // record a red baseline after a green run and the newest run in the tag is
55
+ // red. The review UI recomputes lift from `cells` (so author overrides move
56
+ // it live), so attaching a lift to a column whose cells are the RED run
57
+ // would have it compare red against red and report "no effect" for a skill
58
+ // that in fact gained every scenario. Attach only when this column IS the
59
+ // green side of the comparison.
60
+ const tagLift = liftByTag.get(tag);
61
+ const lift = tagLift && tagLift.greenTimestamp === r.timestamp ? tagLift : undefined;
49
62
  columns.push({
50
63
  index: columns.length,
51
64
  label: r.model,
52
- tag: tagDir.split("/").pop(),
65
+ tag,
53
66
  runDir,
54
67
  timestamp: r.timestamp,
55
68
  mode: r.mode,
56
69
  grade: r.effective_grade,
57
70
  judge: r.judge,
58
71
  cells,
72
+ ...(lift ? { lift, liftHeadline: liftHeadline(lift) } : {}),
59
73
  });
60
74
  }
61
75
  }
@@ -77,6 +91,7 @@ export function publicView(data) {
77
91
  grade: c.grade,
78
92
  judge: c.judge,
79
93
  cells: c.cells,
94
+ ...(c.lift ? { lift: c.lift, liftHeadline: c.liftHeadline } : {}),
80
95
  })),
81
96
  };
82
97
  }
@@ -0,0 +1,34 @@
1
+ import type { Spec } from "./spec.js";
2
+ import { type ResultsFile, type ScenarioResult } from "./results.js";
3
+ export interface RescoreOptions {
4
+ runDir: string;
5
+ spec: Spec;
6
+ now?: () => string;
7
+ }
8
+ export interface RescoreChange {
9
+ id: string;
10
+ from: ScenarioResult["judge_verdict"];
11
+ to: ScenarioResult["judge_verdict"];
12
+ passes: number;
13
+ clean: number;
14
+ fromThreshold: number;
15
+ toThreshold: number;
16
+ }
17
+ export interface RescoreResult {
18
+ results: ResultsFile;
19
+ changes: RescoreChange[];
20
+ }
21
+ /**
22
+ * Re-score a run against the CURRENT spec's pass thresholds — no model calls, no judge
23
+ * calls. Reps are the raw measurement (`passes` of `clean`); a threshold is *policy*.
24
+ * When the policy changes, the honest move is to recompute the old measurements under it
25
+ * rather than reconcile two numbers in prose — and to record what moved.
26
+ *
27
+ * Only reps-bearing scenarios can be re-scored: a single-rep verdict has no rate to
28
+ * re-apply a threshold to, and ERROR/JUDGE-AMBIGUOUS carry no trustworthy rate at all —
29
+ * both are carried verbatim. Overrides, notes, and suspect flags are preserved: this
30
+ * changes the collapse rule, nothing about what the judge said.
31
+ */
32
+ export declare function rescoreRun(opts: RescoreOptions): RescoreResult;
33
+ /** Locate the spec for a run dir: results/<tag>/<ts> → ../../../specification.yaml */
34
+ export declare function specPathForRunDir(runDir: string): string;
@@ -0,0 +1,66 @@
1
+ import { existsSync } from "node:fs";
2
+ import { join } from "node:path";
3
+ import { readResults, writeResults } from "./results.js";
4
+ import { appendJournal } from "./journal.js";
5
+ /**
6
+ * Re-score a run against the CURRENT spec's pass thresholds — no model calls, no judge
7
+ * calls. Reps are the raw measurement (`passes` of `clean`); a threshold is *policy*.
8
+ * When the policy changes, the honest move is to recompute the old measurements under it
9
+ * rather than reconcile two numbers in prose — and to record what moved.
10
+ *
11
+ * Only reps-bearing scenarios can be re-scored: a single-rep verdict has no rate to
12
+ * re-apply a threshold to, and ERROR/JUDGE-AMBIGUOUS carry no trustworthy rate at all —
13
+ * both are carried verbatim. Overrides, notes, and suspect flags are preserved: this
14
+ * changes the collapse rule, nothing about what the judge said.
15
+ */
16
+ export function rescoreRun(opts) {
17
+ const now = opts.now ?? (() => new Date().toISOString());
18
+ const prev = readResults(opts.runDir);
19
+ if (!prev)
20
+ throw new Error(`no results.yaml in ${opts.runDir}`);
21
+ const specById = new Map(opts.spec.scenarios.map((s) => [s.id, s]));
22
+ const changes = [];
23
+ const scenarios = prev.scenarios.map((s) => {
24
+ const scenario = specById.get(s.id);
25
+ // no rate to re-apply, or an untrustworthy verdict → carry verbatim
26
+ if (!scenario || s.reps === undefined || s.clean === undefined || s.passes === undefined)
27
+ return s;
28
+ if (s.judge_verdict === "ERROR" || s.judge_verdict === "JUDGE-AMBIGUOUS")
29
+ return s;
30
+ if (s.clean === 0)
31
+ return s;
32
+ const toThreshold = scenario.passThreshold ?? 0.5;
33
+ const fromThreshold = s.pass_threshold ?? toThreshold;
34
+ if (toThreshold === fromThreshold)
35
+ return s;
36
+ const rate = s.passes / s.clean;
37
+ const verdict = rate >= toThreshold ? "PASS" : "FAIL";
38
+ if (verdict !== s.judge_verdict) {
39
+ changes.push({ id: s.id, from: s.judge_verdict, to: verdict, passes: s.passes, clean: s.clean, fromThreshold, toThreshold });
40
+ }
41
+ return { ...s, judge_verdict: verdict, pass_threshold: toThreshold };
42
+ });
43
+ const ctx = prev.mode === "green" && !prev.partial
44
+ ? { shipBar: opts.spec.ship_bar, critical: opts.spec.critical }
45
+ : null;
46
+ const results = writeResults(opts.runDir, {
47
+ skill: prev.skill, harness: prev.harness, model: prev.model, judge: prev.judge,
48
+ timestamp: prev.timestamp, label: prev.label, mode: prev.mode,
49
+ partial: prev.partial, source_hashes: prev.source_hashes, scenarios,
50
+ }, ctx);
51
+ appendJournal(opts.runDir, {
52
+ event: "rescore", ts: now(),
53
+ changed: changes.map((c) => `${c.id}: ${c.from}->${c.to} (${c.passes}/${c.clean} @ ${c.toThreshold})`),
54
+ passed: results.effective_grade.passed, total: results.effective_grade.total,
55
+ pct: results.effective_grade.pct, ship: results.effective_grade.ship,
56
+ });
57
+ return { results, changes };
58
+ }
59
+ /** Locate the spec for a run dir: results/<tag>/<ts> → ../../../specification.yaml */
60
+ export function specPathForRunDir(runDir) {
61
+ const p = join(runDir, "..", "..", "..", "specification.yaml");
62
+ if (!existsSync(p))
63
+ throw new Error(`no specification.yaml above ${runDir}`);
64
+ return p;
65
+ }
66
+ //# sourceMappingURL=rescore.js.map
package/dist/results.d.ts CHANGED
@@ -35,6 +35,15 @@ export interface ResultsFile {
35
35
  timestamp: string;
36
36
  label: string | null;
37
37
  mode: string;
38
+ /** True for an `--only`-filtered run: a scenario subset, never ship-graded, never a release run. */
39
+ partial?: boolean;
40
+ /**
41
+ * sha256 of every source file this run measured: SKILL.md plus each distinct
42
+ * system_prompt_file. Lint compares the newest run's hashes against the current
43
+ * files — a mismatch means the published result describes text that no longer
44
+ * exists (the stale-scorecard class this field exists to kill).
45
+ */
46
+ source_hashes?: Record<string, string>;
38
47
  effective_grade: GradeSummary;
39
48
  scenarios: ScenarioResult[];
40
49
  }
package/dist/results.js CHANGED
@@ -46,7 +46,8 @@ export function finalizeResults(draft, ctx) {
46
46
  effective_grade = { passed: s.passed, total: s.total, pct: s.pct, letter: s.letter, ship: s.ship, note: s.note };
47
47
  }
48
48
  else {
49
- effective_grade = { passed: 0, total: 0, pct: 0, letter: "-", ship: false, note: `mode=${draft.mode} (not scored)` };
49
+ const why = draft.partial ? "partial run (--only) not scored" : `mode=${draft.mode} (not scored)`;
50
+ effective_grade = { passed: 0, total: 0, pct: 0, letter: "-", ship: false, note: why };
50
51
  }
51
52
  return {
52
53
  schema: 2,
@@ -57,6 +58,8 @@ export function finalizeResults(draft, ctx) {
57
58
  timestamp: draft.timestamp,
58
59
  label: draft.label,
59
60
  mode: draft.mode,
61
+ ...(draft.partial ? { partial: true } : {}),
62
+ ...(draft.source_hashes ? { source_hashes: draft.source_hashes } : {}),
60
63
  effective_grade,
61
64
  scenarios: draft.scenarios,
62
65
  };
package/dist/run.d.ts CHANGED
@@ -1,6 +1,7 @@
1
1
  import type { Spec } from "./spec.js";
2
2
  import type { HarnessAdapter, ModelRef, RunMode } from "./adapters/types.js";
3
3
  import { type ResultsFile } from "./results.js";
4
+ import { type Lift } from "./lift.js";
4
5
  export interface RunOptions {
5
6
  spec: Spec;
6
7
  skillDir: string;
@@ -17,6 +18,12 @@ export interface RunOptions {
17
18
  concurrency?: number;
18
19
  reps?: number;
19
20
  passThreshold?: number;
21
+ /**
22
+ * Run only these scenario ids — the iteration tool (re-testing 2 D-scenarios must not
23
+ * cost an 18-scenario run). The result is marked `partial: true` and is NEVER
24
+ * ship-graded: a subset passing says nothing about the ship bar.
25
+ */
26
+ only?: string[];
20
27
  }
21
28
  export interface RunSummary {
22
29
  runDir: string;
@@ -24,5 +31,22 @@ export interface RunSummary {
24
31
  }
25
32
  /** Run one skill against one model: run scenarios, grade, score, persist. */
26
33
  export declare function runSkillModel(opts: RunOptions): Promise<RunSummary>;
34
+ /**
35
+ * True when any assistant turn in a transcript is blank — the shape a harness timeout
36
+ * leaves behind. Such a transcript must never reach the judge: grading an empty reply
37
+ * produces a confident FAIL about behavior that never happened (round 9 lost two
38
+ * scenarios this way). Sections are delimited by the adapters' shared transcript
39
+ * convention (">>> USER"/"<<< ASSISTANT:"); seeded gate output ("=== SEEDED GATES ===")
40
+ * ends the last assistant section.
41
+ */
42
+ export declare function hasEmptyAssistantTurn(transcript: string): boolean;
27
43
  /** A compact terminal scorecard for one run. */
28
- export declare function formatScorecard(summary: RunSummary): string;
44
+ /**
45
+ * The human-facing scorecard for one run.
46
+ *
47
+ * `lift` is the red-vs-green comparison for this model when a red baseline
48
+ * exists. Passing it for a green run turns the scorecard from "the skill scored
49
+ * B" into "the skill *did* this much" — without a baseline the grade alone can't
50
+ * distinguish a skill that works from a model that never needed it.
51
+ */
52
+ export declare function formatScorecard(summary: RunSummary, lift?: Lift): string;
package/dist/run.js CHANGED
@@ -1,18 +1,62 @@
1
- import { mkdirSync, writeFileSync } from "node:fs";
2
- import { dirname } from "node:path";
1
+ import { mkdirSync, writeFileSync, readFileSync } from "node:fs";
2
+ import { createHash } from "node:crypto";
3
+ import { dirname, resolve } from "node:path";
3
4
  import { judgeResemblesSubject } from "./grade.js";
4
5
  import { runDirFor, transcriptPath, writeResults, ensureResultsGitignore, } from "./results.js";
5
6
  import { appendJournal } from "./journal.js";
7
+ import { liftHeadline } from "./lift.js";
6
8
  import { runSeeded } from "./seeded.js";
7
9
  import { createWorkspace } from "./workspace.js";
8
10
  import { runPool } from "./scheduler.js";
9
11
  import { outcomesToResult } from "./reps.js";
10
12
  import { judgeOneRep } from "./regrade.js";
13
+ /** sha256 of a file, or null when it doesn't exist — missing sources are lint's problem, not run's. */
14
+ function sha256(path) {
15
+ try {
16
+ return createHash("sha256").update(readFileSync(path)).digest("hex");
17
+ }
18
+ catch {
19
+ return null;
20
+ }
21
+ }
22
+ /**
23
+ * Hash every source file this run measures: SKILL.md + each distinct
24
+ * system_prompt_file (agents/<name>.md). Recorded in results.yaml so lint can prove
25
+ * a published result still describes the current text.
26
+ */
27
+ function sourceHashes(skillDir, specPath, scenarios) {
28
+ const hashes = {};
29
+ const skillMd = sha256(resolve(skillDir, "SKILL.md"));
30
+ if (skillMd)
31
+ hashes["SKILL.md"] = skillMd;
32
+ for (const s of scenarios) {
33
+ if (s.systemPromptFile && !(s.systemPromptFile in hashes)) {
34
+ const h = sha256(resolve(dirname(specPath), s.systemPromptFile));
35
+ if (h)
36
+ hashes[s.systemPromptFile] = h;
37
+ }
38
+ }
39
+ return hashes;
40
+ }
11
41
  /** Run one skill against one model: run scenarios, grade, score, persist. */
12
42
  export async function runSkillModel(opts) {
13
43
  const { spec, skillDir, adapter, model, judge, mode, timestamp } = opts;
14
44
  const log = opts.onProgress ?? (() => { });
15
45
  const now = opts.now ?? (() => new Date().toISOString());
46
+ // --only: validate against the spec BEFORE spending anything — a typo'd id must not
47
+ // silently run zero scenarios and report success.
48
+ let scenarios = spec.scenarios;
49
+ const partial = Boolean(opts.only && opts.only.length > 0);
50
+ if (partial) {
51
+ const known = new Set(spec.scenarios.map((s) => s.id));
52
+ const unknown = opts.only.filter((id) => !known.has(id));
53
+ if (unknown.length > 0) {
54
+ throw new Error(`--only names unknown scenario id(s) ${unknown.join(", ")} — spec has: ${[...known].join(", ")}`);
55
+ }
56
+ const wanted = new Set(opts.only);
57
+ scenarios = spec.scenarios.filter((s) => wanted.has(s.id));
58
+ log(` --only ${opts.only.join(",")} — partial run, will not be ship-graded`);
59
+ }
16
60
  if (judgeResemblesSubject(judge, model)) {
17
61
  log(` ⚠ judge (${judge.provider}:${judge.model}) resembles the model under test ` +
18
62
  `(${model.provider}:${model.model}) — verdicts may be inflated. Use a distinct judge.`);
@@ -27,10 +71,10 @@ export async function runSkillModel(opts) {
27
71
  mode, label: opts.label ?? null,
28
72
  });
29
73
  // scenario × rep tasks; runPool preserves input order so we can slice per scenario.
30
- const repCounts = spec.scenarios.map((s) => s.reps ?? opts.reps ?? 1);
74
+ const repCounts = scenarios.map((s) => s.reps ?? opts.reps ?? 1);
31
75
  const owners = [];
32
76
  const tasks = [];
33
- spec.scenarios.forEach((scenario, si) => {
77
+ scenarios.forEach((scenario, si) => {
34
78
  for (let k = 0; k < repCounts[si]; k++) {
35
79
  const rep = k;
36
80
  const total = repCounts[si];
@@ -39,13 +83,13 @@ export async function runSkillModel(opts) {
39
83
  }
40
84
  });
41
85
  const flat = await runPool(tasks, opts.concurrency ?? 1);
42
- const grouped = spec.scenarios.map(() => []);
86
+ const grouped = scenarios.map(() => []);
43
87
  flat.forEach((outcome, i) => grouped[owners[i]].push(outcome));
44
- const scenarioResults = spec.scenarios.map((scenario, si) => {
88
+ const scenarioResults = scenarios.map((scenario, si) => {
45
89
  const threshold = scenario.passThreshold ?? opts.passThreshold ?? 0.5;
46
90
  return outcomesToResult(scenario.id, grouped[si], repCounts[si], threshold);
47
91
  });
48
- const ctx = mode === "green" ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
92
+ const ctx = mode === "green" && !partial ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
49
93
  const results = writeResults(runDir, {
50
94
  skill: spec.skill,
51
95
  harness: adapter.name,
@@ -54,6 +98,8 @@ export async function runSkillModel(opts) {
54
98
  timestamp,
55
99
  label: opts.label ?? null,
56
100
  mode,
101
+ ...(partial ? { partial: true } : {}),
102
+ source_hashes: sourceHashes(skillDir, opts.specPath, scenarios),
57
103
  scenarios: scenarioResults,
58
104
  }, ctx);
59
105
  if (ctx) {
@@ -62,6 +108,23 @@ export async function runSkillModel(opts) {
62
108
  }
63
109
  return { runDir, results };
64
110
  }
111
+ /**
112
+ * True when any assistant turn in a transcript is blank — the shape a harness timeout
113
+ * leaves behind. Such a transcript must never reach the judge: grading an empty reply
114
+ * produces a confident FAIL about behavior that never happened (round 9 lost two
115
+ * scenarios this way). Sections are delimited by the adapters' shared transcript
116
+ * convention (">>> USER"/"<<< ASSISTANT:"); seeded gate output ("=== SEEDED GATES ===")
117
+ * ends the last assistant section.
118
+ */
119
+ export function hasEmptyAssistantTurn(transcript) {
120
+ const sections = transcript.split(/^<<< ASSISTANT:\s*$/m).slice(1);
121
+ if (sections.length === 0)
122
+ return false;
123
+ return sections.some((sec) => {
124
+ const body = sec.split(/^(?:>>> |=== SEEDED GATES ===|\[pi exited )/m)[0];
125
+ return body.trim() === "";
126
+ });
127
+ }
65
128
  /** Run ONE rep of a scenario in its own isolated workspace. */
66
129
  async function runRep(scenario, rep, repCount, ctx) {
67
130
  const { spec, judge, mode, runDir, now, log } = ctx;
@@ -75,25 +138,44 @@ async function runRep(scenario, rep, repCount, ctx) {
75
138
  let gatePrefix = null;
76
139
  try {
77
140
  try {
78
- ws = createWorkspace(scenario.workspace, { specDir: dirname(ctx.specPath) });
141
+ ws = createWorkspace(scenario.workspace, { specDir: dirname(ctx.specPath), remote: scenario.remote });
79
142
  }
80
143
  catch (e) {
81
144
  // A setup failure (e.g. missing fixture) is an objective FAIL, not an infra abort.
82
145
  gatePrefix = e instanceof Error ? e.message : String(e);
83
146
  transcript = `[workspace setup failed] ${gatePrefix}`;
84
147
  }
148
+ let noResponse = false;
85
149
  if (ws) {
86
- if (scenario.mode === "seeded") {
87
- const r = await runSeeded(scenario, {
88
- skillDir: ctx.skillDir, adapter: ctx.adapter, model: ctx.model, mode, cwd: ws.cwd,
89
- });
90
- transcript = r.transcript;
91
- gatePrefix = r.gateFailure;
92
- }
93
- else {
94
- transcript = await ctx.adapter.run({
95
- skillDir: ctx.skillDir, model: ctx.model, mode, turns: scenario.turns, cwd: ws.cwd,
96
- });
150
+ // A blank assistant turn is a harness timeout, not model behavior: retry ONCE in a
151
+ // fresh workspace (the first attempt may have half-mutated a seeded repo), and if
152
+ // it happens again the verdict is ERROR — never a judged FAIL on an empty reply.
153
+ for (let attempt = 0; attempt < 2; attempt++) {
154
+ if (attempt > 0) {
155
+ appendJournal(runDir, { event: "empty-response-retry", ts: now(), id: scenario.id, attempt, ...repField });
156
+ log(` ${scenario.id}${repCount > 1 ? `#${rep}` : ""} empty response — retrying once`);
157
+ ws.cleanup();
158
+ ws = createWorkspace(scenario.workspace, { specDir: dirname(ctx.specPath), remote: scenario.remote });
159
+ }
160
+ if (scenario.mode === "seeded") {
161
+ const r = await runSeeded(scenario, {
162
+ skillDir: ctx.skillDir, adapter: ctx.adapter, model: ctx.model, mode, cwd: ws.cwd,
163
+ });
164
+ transcript = r.transcript;
165
+ gatePrefix = r.gateFailure;
166
+ }
167
+ else {
168
+ transcript = await ctx.adapter.run({
169
+ skillDir: ctx.skillDir, model: ctx.model, mode, turns: scenario.turns, cwd: ws.cwd,
170
+ // resolved like fixtures: relative to the spec's dir
171
+ systemPromptFile: scenario.systemPromptFile
172
+ ? resolve(dirname(ctx.specPath), scenario.systemPromptFile)
173
+ : undefined,
174
+ });
175
+ }
176
+ noResponse = hasEmptyAssistantTurn(transcript);
177
+ if (!noResponse)
178
+ break;
97
179
  }
98
180
  }
99
181
  writeFileSync(transcriptPath(runDir, scenario.id, mode, repCount > 1 ? rep : undefined), transcript, "utf8");
@@ -103,7 +185,12 @@ async function runRep(scenario, rep, repCount, ctx) {
103
185
  let verdict;
104
186
  let reason;
105
187
  let suspect = false;
106
- if (gatePrefix) {
188
+ if (noResponse) {
189
+ verdict = "ERROR";
190
+ reason = "model produced no response after a retry (harness timeout?) — infra, not skill behavior";
191
+ appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict, reason, suspect, ...repField });
192
+ }
193
+ else if (gatePrefix) {
107
194
  verdict = "FAIL";
108
195
  reason = gatePrefix;
109
196
  // gate failures don't invoke the judge, but still record a judge-verdict event (as before)
@@ -126,7 +213,15 @@ async function runRep(scenario, rep, repCount, ctx) {
126
213
  }
127
214
  }
128
215
  /** A compact terminal scorecard for one run. */
129
- export function formatScorecard(summary) {
216
+ /**
217
+ * The human-facing scorecard for one run.
218
+ *
219
+ * `lift` is the red-vs-green comparison for this model when a red baseline
220
+ * exists. Passing it for a green run turns the scorecard from "the skill scored
221
+ * B" into "the skill *did* this much" — without a baseline the grade alone can't
222
+ * distinguish a skill that works from a model that never needed it.
223
+ */
224
+ export function formatScorecard(summary, lift) {
130
225
  const { results } = summary;
131
226
  const g = results.effective_grade;
132
227
  const lines = [];
@@ -143,6 +238,16 @@ export function formatScorecard(summary) {
143
238
  const ship = g.ship ? "SHIP" : "NOT READY";
144
239
  const note = g.note ? ` (${g.note})` : "";
145
240
  lines.push(` GRADE: ${g.letter} (${g.pct}%) — ${g.passed}/${g.total} — ${ship}${note}`);
241
+ // Lift is a statement about a green run. On a red run the caller may still have
242
+ // a lift in hand (a green run exists in the same tag), but printing it under a
243
+ // baseline scorecard reads as if the baseline itself gained something.
244
+ if (lift && results.mode === "green") {
245
+ lines.push(` LIFT: ${liftHeadline(lift)} (vs red baseline ${lift.redTimestamp})`);
246
+ }
247
+ else if (results.mode === "green") {
248
+ // The grade alone can't answer "does this skill do anything?", so say how.
249
+ lines.push(` LIFT: no red baseline — run with --mode red to measure what the skill adds`);
250
+ }
146
251
  return lines.join("\n");
147
252
  }
148
253
  //# sourceMappingURL=run.js.map
@@ -0,0 +1,28 @@
1
+ /** Marker written into an `init` template's first comment. Its presence tells
2
+ * `suggest` the file is an unadopted template it may overwrite without --force. */
3
+ export declare const TEMPLATE_SENTINEL = "skill-harness: generated template";
4
+ /** Render a commented, empty-but-valid specification.yaml for a skill. */
5
+ export declare function renderTemplateSpec(skillName: string): string;
6
+ /** True if the text still carries the template sentinel (i.e. an unadopted template). */
7
+ export declare function isTemplateSpec(text: string): boolean;
8
+ export interface DraftScenario {
9
+ id: string;
10
+ title: string;
11
+ turns: string[];
12
+ checklist: string[];
13
+ }
14
+ export interface SuggestDraft {
15
+ judge_persona: string;
16
+ ship_bar: {
17
+ total: number;
18
+ min_pass: number;
19
+ no_critical_fail: boolean;
20
+ };
21
+ proposed_critical: string[];
22
+ scenarios: DraftScenario[];
23
+ }
24
+ /** Render a populated spec from an LLM draft. Strings are JSON-encoded (valid YAML
25
+ * flow scalars) so colons/quotes never break the file. Carries no sentinel. */
26
+ export declare function renderDraftSpec(skillName: string, draft: SuggestDraft): string;
27
+ export declare function buildSuggestPrompt(skillName: string, skillMd: string): string;
28
+ export declare function parseSuggestDraft(raw: string): SuggestDraft;