@skill-harness/core 0.1.2 → 0.3.0

package/dist/results.js

@@ -46,7 +46,8 @@ export function finalizeResults(draft, ctx) {
         effective_grade = { passed: s.passed, total: s.total, pct: s.pct, letter: s.letter, ship: s.ship, note: s.note };
     }
     else {
-        effective_grade = { passed: 0, total: 0, pct: 0, letter: "-", ship: false, note: `mode=${draft.mode} (not scored)` };
+        const why = draft.partial ? "partial run (--only) — not scored" : `mode=${draft.mode} (not scored)`;
+        effective_grade = { passed: 0, total: 0, pct: 0, letter: "-", ship: false, note: why };
     }
     return {
         schema: 2,
@@ -57,6 +58,8 @@ export function finalizeResults(draft, ctx) {
         timestamp: draft.timestamp,
         label: draft.label,
         mode: draft.mode,
+        ...(draft.partial ? { partial: true } : {}),
+        ...(draft.source_hashes ? { source_hashes: draft.source_hashes } : {}),
         effective_grade,
         scenarios: draft.scenarios,
     };
@@ -147,9 +150,10 @@ export function ensureResultsGitignore(resultsRoot) {
         .filter((l) => l.startsWith("!") && l.trim() !== "!results.yaml");
     writeFileSync(giPath, GITIGNORE_BODY + preserved.map((l) => l + "\n").join(""), "utf8");
 }
-// Matches both transcript (`.rep<k>.txt`) and judge-raw (`.rep<k>.judge.txt`) rep suffixes.
-const REP_SUFFIX_RE = /\.rep(\d+)\.(?:judge\.)?txt$/;
-/** The rep index embedded in a transcript/judge-raw filename (`.rep<k>.`), or null for a plain (non-rep) file. */
+// Matches transcript (`.rep<k>.txt`), judge-raw (`.rep<k>.judge.txt`) and
+// staged-diff (`.rep<k>.diff.txt`) rep suffixes.
+const REP_SUFFIX_RE = /\.rep(\d+)\.(?:judge\.|diff\.)?txt$/;
+/** The rep index embedded in a transcript / judge-raw / staged-diff filename (`.rep<k>.`), or null for a plain (non-rep) file. */
 export function repIndexOf(filename) {
     const m = REP_SUFFIX_RE.exec(filename);
     return m ? Number(m[1]) : null;
@@ -178,7 +182,10 @@ function sortByRep(files) {
  * `<id>.<mode>.rep<k>.txt`) — e.g. to detect a green-only condition without
  * false positives from a red/force transcript of the same scenario. Omitted,
  * behavior is unchanged: any `<id>.*.txt` regardless of mode, excluding this
- * scenario's judge-raw artifacts (`<id>.*.judge.txt` — see judgeRawPath).
+ * scenario's sibling artifacts, which share the `.txt` extension deliberately
+ * (so `results/.gitignore`'s `*.txt` covers them all): judge-raw output
+ * (`<id>.*.judge.txt` — see judgeRawPath) and the staged diff
+ * (`<id>.*.diff.txt` — see diffPath).
  */
 export function findTranscriptFiles(runDir, scenarioId, mode) {
     if (!existsSync(runDir))
@@ -187,7 +194,12 @@ export function findTranscriptFiles(runDir, scenarioId, mode) {
     const matcher = mode !== undefined
         ? new RegExp(`^${escapedId}\\.${mode}(\\.rep\\d+)?\\.txt$`)
         : null;
-    const files = readdirSync(runDir).filter((f) => matcher ? matcher.test(f) : f.startsWith(`${scenarioId}.`) && f.endsWith(".txt") && !f.endsWith(".judge.txt"));
+    const files = readdirSync(runDir).filter((f) => matcher
+        ? matcher.test(f)
+        : f.startsWith(`${scenarioId}.`) &&
+            f.endsWith(".txt") &&
+            !f.endsWith(".judge.txt") &&
+            !f.endsWith(".diff.txt"));
     return sortByRep(files);
 }
 /** Path of a scenario's raw judge-output artifact within a run dir (rep-suffixed for reps). */
@@ -205,22 +217,54 @@ export function findJudgeRawFiles(runDir, scenarioId, mode) {
         : new RegExp(`^${esc}\\.${mode}(\\.rep\\d+)?\\.judge\\.txt$`);
     return sortByRep(readdirSync(runDir).filter((f) => re.test(f)));
 }
+/**
+ * Path of a seeded scenario's staged-diff artifact within a run dir (rep-suffixed
+ * for reps).
+ *
+ * The diff is the only record of what the model actually *did* — the workspace is
+ * torn down after every rep, so without this a seeded verdict cannot be audited
+ * after the fact. Named `<id>.<mode>[.rep<k>].diff.txt` so it sorts beside its
+ * transcript and is covered by the `*.txt` rule in results/.gitignore: diffs are
+ * generated evidence, ignored like transcripts, not committed like results.yaml.
+ */
+export function diffPath(runDir, scenarioId, mode, rep) {
+    const base = rep === undefined ? `${scenarioId}.${mode}` : `${scenarioId}.${mode}.rep${rep}`;
+    return join(runDir, `${base}.diff.txt`);
+}
+/** A scenario's staged-diff files, sorted (plain first, then numeric rep). Mode-scoped when given. */
+export function findDiffFiles(runDir, scenarioId, mode) {
+    if (!existsSync(runDir))
+        return [];
+    const esc = scenarioId.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const re = mode === undefined
+        ? new RegExp(`^${esc}\\..*\\.diff\\.txt$`)
+        : new RegExp(`^${esc}\\.${mode}(\\.rep\\d+)?\\.diff\\.txt$`);
+    return sortByRep(readdirSync(runDir).filter((f) => re.test(f)));
+}
 /** A single representative transcript file for a scenario in a run dir. Null if none. */
 export function findTranscriptFile(runDir, scenarioId) {
     return findTranscriptFiles(runDir, scenarioId)[0] ?? null;
 }
 /**
- * Un-gitignore ALL of a scenario's transcript AND judge-raw artifact files
- * (audit trail for an override — a --reps run has one transcript (and one
- * judge-raw file) per rep, and every rep that drove the verdict must survive
- * a commit, not just an arbitrary one).
+ * Un-gitignore ALL of a scenario's transcript, judge-raw AND staged-diff
+ * artifact files (audit trail for an override — a --reps run has one of each
+ * per rep, and every rep that drove the verdict must survive a commit, not just
+ * an arbitrary one).
  * Appends `!<tag>/<ts>/<id>.<mode>[.rep<k>].txt` (and the matching
- * `.judge.txt`) to results/.gitignore for each, once. The path uses POSIX
- * separators so the negation matches on Windows too (git ignore patterns are
- * always forward-slashed).
+ * `.judge.txt` / `.diff.txt`) to results/.gitignore for each, once. The path
+ * uses POSIX separators so the negation matches on Windows too (git ignore
+ * patterns are always forward-slashed).
+ *
+ * The diff belongs here for the same reason the judge-raw output does: an
+ * override says the judge got it wrong, and on a seeded scenario the evidence
+ * for that claim is the code the model wrote.
  */
 export function preserveTranscript(resultsRoot, runDir, scenarioId) {
-    const files = [...findTranscriptFiles(runDir, scenarioId), ...findJudgeRawFiles(runDir, scenarioId)];
+    const files = [
+        ...findTranscriptFiles(runDir, scenarioId),
+        ...findJudgeRawFiles(runDir, scenarioId),
+        ...findDiffFiles(runDir, scenarioId),
+    ];
     if (files.length === 0)
         return;
     ensureResultsGitignore(resultsRoot);

package/dist/run.d.ts

@@ -1,6 +1,7 @@
 import type { Spec } from "./spec.js";
 import type { HarnessAdapter, ModelRef, RunMode } from "./adapters/types.js";
 import { type ResultsFile } from "./results.js";
+import { type Lift } from "./lift.js";
 export interface RunOptions {
     spec: Spec;
     skillDir: string;
@@ -17,6 +18,12 @@ export interface RunOptions {
     concurrency?: number;
     reps?: number;
     passThreshold?: number;
+    /**
+     * Run only these scenario ids — the iteration tool (re-testing 2 D-scenarios must not
+     * cost an 18-scenario run). The result is marked `partial: true` and is NEVER
+     * ship-graded: a subset passing says nothing about the ship bar.
+     */
+    only?: string[];
 }
 export interface RunSummary {
     runDir: string;
@@ -24,5 +31,22 @@ export interface RunSummary {
 }
 /** Run one skill against one model: run scenarios, grade, score, persist. */
 export declare function runSkillModel(opts: RunOptions): Promise<RunSummary>;
+/**
+ * True when any assistant turn in a transcript is blank — the shape a harness timeout
+ * leaves behind. Such a transcript must never reach the judge: grading an empty reply
+ * produces a confident FAIL about behavior that never happened (round 9 lost two
+ * scenarios this way). Sections are delimited by the adapters' shared transcript
+ * convention (">>> USER"/"<<< ASSISTANT:"); seeded gate output ("=== SEEDED GATES ===")
+ * ends the last assistant section.
+ */
+export declare function hasEmptyAssistantTurn(transcript: string): boolean;
 /** A compact terminal scorecard for one run. */
-export declare function formatScorecard(summary: RunSummary): string;
+/**
+ * The human-facing scorecard for one run.
+ *
+ * `lift` is the red-vs-green comparison for this model when a red baseline
+ * exists. Passing it for a green run turns the scorecard from "the skill scored
+ * B" into "the skill *did* this much" — without a baseline the grade alone can't
+ * distinguish a skill that works from a model that never needed it.
+ */
+export declare function formatScorecard(summary: RunSummary, lift?: Lift): string;

package/dist/run.js

@@ -1,8 +1,10 @@
 import { mkdirSync, writeFileSync } from "node:fs";
-import { dirname } from "node:path";
+import { dirname, resolve } from "node:path";
+import { sourceHashes } from "./sources.js";
 import { judgeResemblesSubject } from "./grade.js";
-import { runDirFor, transcriptPath, writeResults, ensureResultsGitignore, } from "./results.js";
+import { runDirFor, transcriptPath, diffPath, writeResults, ensureResultsGitignore, } from "./results.js";
 import { appendJournal } from "./journal.js";
+import { liftHeadline } from "./lift.js";
 import { runSeeded } from "./seeded.js";
 import { createWorkspace } from "./workspace.js";
 import { runPool } from "./scheduler.js";
@@ -13,6 +15,20 @@ export async function runSkillModel(opts) {
     const { spec, skillDir, adapter, model, judge, mode, timestamp } = opts;
     const log = opts.onProgress ?? (() => { });
     const now = opts.now ?? (() => new Date().toISOString());
+    // --only: validate against the spec BEFORE spending anything — a typo'd id must not
+    // silently run zero scenarios and report success.
+    let scenarios = spec.scenarios;
+    const partial = Boolean(opts.only && opts.only.length > 0);
+    if (partial) {
+        const known = new Set(spec.scenarios.map((s) => s.id));
+        const unknown = opts.only.filter((id) => !known.has(id));
+        if (unknown.length > 0) {
+            throw new Error(`--only names unknown scenario id(s) ${unknown.join(", ")} — spec has: ${[...known].join(", ")}`);
+        }
+        const wanted = new Set(opts.only);
+        scenarios = spec.scenarios.filter((s) => wanted.has(s.id));
+        log(`  --only ${opts.only.join(",")} — partial run, will not be ship-graded`);
+    }
     if (judgeResemblesSubject(judge, model)) {
         log(`  ⚠ judge (${judge.provider}:${judge.model}) resembles the model under test ` +
             `(${model.provider}:${model.model}) — verdicts may be inflated. Use a distinct judge.`);
@@ -27,10 +43,10 @@ export async function runSkillModel(opts) {
         mode, label: opts.label ?? null,
     });
     // scenario × rep tasks; runPool preserves input order so we can slice per scenario.
-    const repCounts = spec.scenarios.map((s) => s.reps ?? opts.reps ?? 1);
+    const repCounts = scenarios.map((s) => s.reps ?? opts.reps ?? 1);
     const owners = [];
     const tasks = [];
-    spec.scenarios.forEach((scenario, si) => {
+    scenarios.forEach((scenario, si) => {
         for (let k = 0; k < repCounts[si]; k++) {
             const rep = k;
             const total = repCounts[si];
@@ -39,13 +55,13 @@ export async function runSkillModel(opts) {
         }
     });
     const flat = await runPool(tasks, opts.concurrency ?? 1);
-    const grouped = spec.scenarios.map(() => []);
+    const grouped = scenarios.map(() => []);
     flat.forEach((outcome, i) => grouped[owners[i]].push(outcome));
-    const scenarioResults = spec.scenarios.map((scenario, si) => {
+    const scenarioResults = scenarios.map((scenario, si) => {
         const threshold = scenario.passThreshold ?? opts.passThreshold ?? 0.5;
         return outcomesToResult(scenario.id, grouped[si], repCounts[si], threshold);
     });
-    const ctx = mode === "green" ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
+    const ctx = mode === "green" && !partial ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
     const results = writeResults(runDir, {
         skill: spec.skill,
         harness: adapter.name,
@@ -54,6 +70,10 @@ export async function runSkillModel(opts) {
         timestamp,
         label: opts.label ?? null,
         mode,
+        ...(partial ? { partial: true } : {}),
+        // Only the scenarios this run actually measured: a --only run must not claim
+        // coverage of scenarios it skipped.
+        source_hashes: sourceHashes({ skillDir, specDir: dirname(opts.specPath), scenarios }),
         scenarios: scenarioResults,
     }, ctx);
     if (ctx) {
@@ -62,6 +82,23 @@ export async function runSkillModel(opts) {
     }
     return { runDir, results };
 }
+/**
+ * True when any assistant turn in a transcript is blank — the shape a harness timeout
+ * leaves behind. Such a transcript must never reach the judge: grading an empty reply
+ * produces a confident FAIL about behavior that never happened (round 9 lost two
+ * scenarios this way). Sections are delimited by the adapters' shared transcript
+ * convention (">>> USER"/"<<< ASSISTANT:"); seeded gate output ("=== SEEDED GATES ===")
+ * ends the last assistant section.
+ */
+export function hasEmptyAssistantTurn(transcript) {
+    const sections = transcript.split(/^<<< ASSISTANT:\s*$/m).slice(1);
+    if (sections.length === 0)
+        return false;
+    return sections.some((sec) => {
+        const body = sec.split(/^(?:>>> |=== SEEDED GATES ===|\[pi exited )/m)[0];
+        return body.trim() === "";
+    });
+}
 /** Run ONE rep of a scenario in its own isolated workspace. */
 async function runRep(scenario, rep, repCount, ctx) {
     const { spec, judge, mode, runDir, now, log } = ctx;
@@ -73,37 +110,75 @@ async function runRep(scenario, rep, repCount, ctx) {
     let ws = null;
     let transcript = "";
     let gatePrefix = null;
+    // Null until a seeded rep actually reaches its gates: a workspace-setup failure
+    // produces no diff, and writing an empty artifact there would misreport "the
+    // model changed nothing" for a rep that never ran.
+    let stagedDiff = null;
     try {
         try {
-            ws = createWorkspace(scenario.workspace, { specDir: dirname(ctx.specPath) });
+            ws = createWorkspace(scenario.workspace, { specDir: dirname(ctx.specPath), remote: scenario.remote });
         }
         catch (e) {
             // A setup failure (e.g. missing fixture) is an objective FAIL, not an infra abort.
             gatePrefix = e instanceof Error ? e.message : String(e);
             transcript = `[workspace setup failed] ${gatePrefix}`;
         }
+        let noResponse = false;
         if (ws) {
-            if (scenario.mode === "seeded") {
-                const r = await runSeeded(scenario, {
-                    skillDir: ctx.skillDir, adapter: ctx.adapter, model: ctx.model, mode, cwd: ws.cwd,
-                });
-                transcript = r.transcript;
-                gatePrefix = r.gateFailure;
-            }
-            else {
-                transcript = await ctx.adapter.run({
-                    skillDir: ctx.skillDir, model: ctx.model, mode, turns: scenario.turns, cwd: ws.cwd,
-                });
+            // A blank assistant turn is a harness timeout, not model behavior: retry ONCE in a
+            // fresh workspace (the first attempt may have half-mutated a seeded repo), and if
+            // it happens again the verdict is ERROR — never a judged FAIL on an empty reply.
+            for (let attempt = 0; attempt < 2; attempt++) {
+                if (attempt > 0) {
+                    appendJournal(runDir, { event: "empty-response-retry", ts: now(), id: scenario.id, attempt, ...repField });
+                    log(`  ${scenario.id}${repCount > 1 ? `#${rep}` : ""} empty response — retrying once`);
+                    ws.cleanup();
+                    ws = createWorkspace(scenario.workspace, { specDir: dirname(ctx.specPath), remote: scenario.remote });
+                }
+                if (scenario.mode === "seeded") {
+                    const r = await runSeeded(scenario, {
+                        skillDir: ctx.skillDir, adapter: ctx.adapter, model: ctx.model, mode, cwd: ws.cwd,
+                        specDir: dirname(ctx.specPath), // assert.post_test resolves like a fixture
+                    });
+                    transcript = r.transcript;
+                    gatePrefix = r.gateFailure;
+                    stagedDiff = r.diff; // a retry replaces the aborted attempt's diff, as it should
+                }
+                else {
+                    transcript = await ctx.adapter.run({
+                        skillDir: ctx.skillDir, model: ctx.model, mode, turns: scenario.turns, cwd: ws.cwd,
+                        // resolved like fixtures: relative to the spec's dir
+                        systemPromptFile: scenario.systemPromptFile
+                            ? resolve(dirname(ctx.specPath), scenario.systemPromptFile)
+                            : undefined,
+                    });
+                }
+                noResponse = hasEmptyAssistantTurn(transcript);
+                if (!noResponse)
+                    break;
             }
         }
-        writeFileSync(transcriptPath(runDir, scenario.id, mode, repCount > 1 ? rep : undefined), transcript, "utf8");
+        const repSuffix = repCount > 1 ? rep : undefined;
+        writeFileSync(transcriptPath(runDir, scenario.id, mode, repSuffix), transcript, "utf8");
         if (scenario.mode === "seeded") {
+            // The workspace is torn down in the `finally` below, so this is the only
+            // chance to keep what the model actually wrote. Persisted uncapped (the
+            // transcript's copy is capped for the judge) and for every rep, pass or
+            // fail — a gate failure is exactly when you want to read the diff.
+            if (stagedDiff !== null) {
+                writeFileSync(diffPath(runDir, scenario.id, mode, repSuffix), stagedDiff, "utf8");
+            }
             appendJournal(runDir, { event: "gate-result", ts: now(), id: scenario.id, ok: !gatePrefix, detail: gatePrefix ?? "", ...repField });
         }
         let verdict;
         let reason;
         let suspect = false;
-        if (gatePrefix) {
+        if (noResponse) {
+            verdict = "ERROR";
+            reason = "model produced no response after a retry (harness timeout?) — infra, not skill behavior";
+            appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict, reason, suspect, ...repField });
+        }
+        else if (gatePrefix) {
             verdict = "FAIL";
             reason = gatePrefix;
             // gate failures don't invoke the judge, but still record a judge-verdict event (as before)
@@ -126,7 +201,15 @@ async function runRep(scenario, rep, repCount, ctx) {
     }
 }
 /** A compact terminal scorecard for one run. */
-export function formatScorecard(summary) {
+/**
+ * The human-facing scorecard for one run.
+ *
+ * `lift` is the red-vs-green comparison for this model when a red baseline
+ * exists. Passing it for a green run turns the scorecard from "the skill scored
+ * B" into "the skill *did* this much" — without a baseline the grade alone can't
+ * distinguish a skill that works from a model that never needed it.
+ */
+export function formatScorecard(summary, lift) {
     const { results } = summary;
     const g = results.effective_grade;
     const lines = [];
@@ -143,6 +226,16 @@ export function formatScorecard(summary) {
     const ship = g.ship ? "SHIP" : "NOT READY";
     const note = g.note ? ` (${g.note})` : "";
     lines.push(`  GRADE: ${g.letter} (${g.pct}%) — ${g.passed}/${g.total} — ${ship}${note}`);
+    // Lift is a statement about a green run. On a red run the caller may still have
+    // a lift in hand (a green run exists in the same tag), but printing it under a
+    // baseline scorecard reads as if the baseline itself gained something.
+    if (lift && results.mode === "green") {
+        lines.push(`  LIFT:  ${liftHeadline(lift)}  (vs red baseline ${lift.redTimestamp})`);
+    }
+    else if (results.mode === "green") {
+        // The grade alone can't answer "does this skill do anything?", so say how.
+        lines.push(`  LIFT:  no red baseline — run with --mode red to measure what the skill adds`);
+    }
     return lines.join("\n");
 }
 //# sourceMappingURL=run.js.map

package/dist/scaffold.d.ts

@@ -0,0 +1,28 @@
+/** Marker written into an `init` template's first comment. Its presence tells
+ *  `suggest` the file is an unadopted template it may overwrite without --force. */
+export declare const TEMPLATE_SENTINEL = "skill-harness: generated template";
+/** Render a commented, empty-but-valid specification.yaml for a skill. */
+export declare function renderTemplateSpec(skillName: string): string;
+/** True if the text still carries the template sentinel (i.e. an unadopted template). */
+export declare function isTemplateSpec(text: string): boolean;
+export interface DraftScenario {
+    id: string;
+    title: string;
+    turns: string[];
+    checklist: string[];
+}
+export interface SuggestDraft {
+    judge_persona: string;
+    ship_bar: {
+        total: number;
+        min_pass: number;
+        no_critical_fail: boolean;
+    };
+    proposed_critical: string[];
+    scenarios: DraftScenario[];
+}
+/** Render a populated spec from an LLM draft. Strings are JSON-encoded (valid YAML
+ *  flow scalars) so colons/quotes never break the file. Carries no sentinel. */
+export declare function renderDraftSpec(skillName: string, draft: SuggestDraft): string;
+export declare function buildSuggestPrompt(skillName: string, skillMd: string): string;
+export declare function parseSuggestDraft(raw: string): SuggestDraft;

package/dist/scaffold.js

@@ -0,0 +1,188 @@
+/** Marker written into an `init` template's first comment. Its presence tells
+ *  `suggest` the file is an unadopted template it may overwrite without --force. */
+export const TEMPLATE_SENTINEL = "skill-harness: generated template";
+/** Render a commented, empty-but-valid specification.yaml for a skill. */
+export function renderTemplateSpec(skillName) {
+    return `# ${TEMPLATE_SENTINEL} — \`suggest\` will overwrite this file while
+# this line is present; delete it once you start editing by hand.
+skill: ${skillName}
+
+# How the LLM judge should role-play when grading transcripts.
+judge_persona: >
+  a careful, fair reviewer.
+
+# The ship bar: what it takes to SHIP.
+#   total    = scenarios counted toward the bar
+#   min_pass = minimum passes required
+#   no_critical_fail = a critical-id fail blocks SHIP even if min_pass is met
+ship_bar:
+  total: 1
+  min_pass: 1
+  no_critical_fail: true
+
+# Scenario ids that block the ship if they fail (or set \`critical: true\` per scenario).
+critical: []
+
+scenarios:
+  # A* = baseline capability · B* = under-pressure / adversarial
+  - id: A1
+    title: describe what this scenario checks
+    # critical: true            # uncomment to gate the ship on this scenario
+    turns:
+      - "the user's first message"
+      # - "a follow-up message for a multi-turn scenario"
+    checklist:
+      - "an observable thing the response must do"
+`;
+}
+/** True if the text still carries the template sentinel (i.e. an unadopted template). */
+export function isTemplateSpec(text) {
+    return text.includes(TEMPLATE_SENTINEL);
+}
+/** Render a populated spec from an LLM draft. Strings are JSON-encoded (valid YAML
+ *  flow scalars) so colons/quotes never break the file. Carries no sentinel. */
+export function renderDraftSpec(skillName, draft) {
+    const scenarioBlocks = draft.scenarios
+        .map((s) => {
+        const turns = s.turns.map((t) => `      - ${JSON.stringify(t)}`).join("\n");
+        const checks = s.checklist.map((c) => `      - ${JSON.stringify(c)}`).join("\n");
+        return `  - id: ${s.id}\n    title: ${JSON.stringify(s.title)}\n    turns:\n${turns}\n    checklist:\n${checks}`;
+    })
+        .join("\n");
+    const proposed = draft.proposed_critical.length
+        ? `# proposed critical: [${draft.proposed_critical.join(", ")}] — move ids into \`critical: []\` below after review.`
+        : `# proposed critical: (none) — mark any ship-gating scenarios in \`critical: []\` below.`;
+    return `skill: ${skillName}
+
+# REVIEW: does this judge persona fit the skill? Edit freely.
+judge_persona: ${JSON.stringify(draft.judge_persona)}
+
+# REVIEW: tune the ship bar before your first run.
+ship_bar:
+  total: ${draft.ship_bar.total}
+  min_pass: ${draft.ship_bar.min_pass}
+  no_critical_fail: ${draft.ship_bar.no_critical_fail}
+
+${proposed}
+critical: []
+
+scenarios:
+${scenarioBlocks}
+`;
+}
+export function buildSuggestPrompt(skillName, skillMd) {
+    return `You are drafting a test specification for an agent skill named "${skillName}".
+Below is its SKILL.md. Propose scenarios that check whether an agent following this
+skill behaves correctly, including at least one adversarial / under-pressure case.
+
+Return ONLY a JSON object (no prose, no markdown fences) with exactly this shape:
+{
+  "judge_persona": "<how a judge should role-play when grading transcripts>",
+  "ship_bar": { "total": <int>, "min_pass": <int>, "no_critical_fail": true },
+  "proposed_critical": ["<scenario id you think should gate the ship>", ...],
+  "scenarios": [
+    { "id": "A1", "title": "<short title>",
+      "turns": ["<the user's message>", "<optional follow-up turns>"],
+      "checklist": ["<an observable thing the response must do>", ...] }
+  ]
+}
+Use ids A1, A2, ... for baseline scenarios and B1, B2, ... for adversarial ones.
+Every scenario needs at least one turn and one checklist item.
+
+--- SKILL.md ---
+${skillMd}`;
+}
+/** Ids are interpolated raw into YAML (see renderDraftSpec); restrict the character
+ *  set so a crafted id can never inject extra YAML keys (e.g. `critical: true`). */
+const SAFE_ID = /^[A-Za-z0-9_-]+$/;
+function asStringArray(v, ctx) {
+    if (!Array.isArray(v) || v.length === 0 || v.some((x) => typeof x !== "string")) {
+        throw new Error(`${ctx} must be a non-empty array of strings`);
+    }
+    return v;
+}
+/** Extract the first complete top-level JSON object from a model reply, tolerating
+ *  surrounding prose or ```json fences — including trailing text that itself
+ *  contains braces (e.g. "…} Does {this} work?"). Scans brace depth while skipping
+ *  string contents, so the object ends at its own matching `}`, not the last `}`
+ *  anywhere in the reply. */
+function extractJsonObject(raw) {
+    const start = raw.indexOf("{");
+    if (start < 0)
+        throw new Error("no JSON object in model output");
+    let depth = 0;
+    let inStr = false;
+    let escaped = false;
+    for (let i = start; i < raw.length; i++) {
+        const ch = raw[i];
+        if (inStr) {
+            if (escaped)
+                escaped = false;
+            else if (ch === "\\")
+                escaped = true;
+            else if (ch === '"')
+                inStr = false;
+            continue;
+        }
+        if (ch === '"')
+            inStr = true;
+        else if (ch === "{")
+            depth++;
+        else if (ch === "}" && --depth === 0)
+            return raw.slice(start, i + 1);
+    }
+    throw new Error("no complete JSON object in model output");
+}
+export function parseSuggestDraft(raw) {
+    let obj;
+    try {
+        obj = JSON.parse(extractJsonObject(raw));
+    }
+    catch (e) {
+        if (e instanceof Error && e.message.includes("JSON object in model output"))
+            throw e;
+        throw new Error(`model output is not valid JSON — ${e.message}`);
+    }
+    if (typeof obj.judge_persona !== "string" || !obj.judge_persona.trim()) {
+        throw new Error("judge_persona must be a non-empty string");
+    }
+    const sb = obj.ship_bar;
+    if (!sb || typeof sb.total !== "number" || typeof sb.min_pass !== "number") {
+        throw new Error("ship_bar must have numeric total and min_pass");
+    }
+    if (sb.min_pass > sb.total) {
+        throw new Error(`ship_bar.min_pass (${sb.min_pass}) cannot exceed total (${sb.total})`);
+    }
+    const proposed = Array.isArray(obj.proposed_critical)
+        ? obj.proposed_critical.filter((x) => typeof x === "string" && SAFE_ID.test(x))
+        : [];
+    if (!Array.isArray(obj.scenarios) || obj.scenarios.length === 0) {
+        throw new Error("scenarios must be a non-empty array");
+    }
+    const seen = new Set();
+    const scenarios = obj.scenarios.map((raw2, i) => {
+        const s = raw2;
+        if (typeof s.id !== "string" || !s.id.trim())
+            throw new Error(`scenario #${i + 1} needs a string id`);
+        if (!SAFE_ID.test(s.id))
+            throw new Error(`scenario id \`${s.id}\` must be alphanumeric (A-Z a-z 0-9 _ -)`);
+        if (seen.has(s.id))
+            throw new Error(`duplicate scenario id \`${s.id}\``);
+        seen.add(s.id);
+        if (typeof s.title !== "string" || !s.title.trim())
+            throw new Error(`scenario ${s.id} needs a title`);
+        return {
+            id: s.id,
+            title: s.title,
+            turns: asStringArray(s.turns, `scenario ${s.id} turns`),
+            checklist: asStringArray(s.checklist, `scenario ${s.id} checklist`),
+        };
+    });
+    return {
+        judge_persona: obj.judge_persona,
+        ship_bar: { total: sb.total, min_pass: sb.min_pass, no_critical_fail: sb.no_critical_fail !== false },
+        proposed_critical: proposed,
+        scenarios,
+    };
+}
+//# sourceMappingURL=scaffold.js.map

package/dist/score.d.ts

@@ -1,5 +1,9 @@
 import type { ShipBar } from "./spec.js";
-export type Verdict = "PASS" | "FAIL" | "ERROR";
+/**
+ * "JUDGE-AMBIGUOUS": the judge emitted conflicting verdicts for one transcript. It is
+ * never a pass and never silently resolved — it marks the run for a rejudge.
+ */
+export type Verdict = "PASS" | "FAIL" | "ERROR" | "JUDGE-AMBIGUOUS";
 export interface ScenarioVerdict {
     id: string;
     verdict: Verdict;