@skill-harness/core 0.1.0

package/dist/results.js

@@ -0,0 +1,241 @@
+import { mkdirSync, readFileSync, writeFileSync, existsSync, readdirSync, appendFileSync } from "node:fs";
+import { join, relative, sep } from "node:path";
+import yaml from "js-yaml";
+import { modelSlug } from "./adapters/types.js";
+import { score } from "./score.js";
+/** The pass-threshold a re-grade uses: the run's persisted value, else the spec's per-scenario value, else 0.5. */
+export function effectiveThreshold(prevScenario, scenario) {
+    return prevScenario?.pass_threshold ?? scenario.passThreshold ?? 0.5;
+}
+/** Slugify an ISO timestamp into a filesystem-safe directory name. */
+function timestampSlug(iso) {
+    return iso.replace(/[:.]/g, "-");
+}
+/** <skillDir>/tests/results/<harness>-<model-slug>/<timestamp-slug>/ */
+export function runDirFor(skillDir, harness, model, timestamp) {
+    return join(skillDir, "tests", "results", `${harness}-${modelSlug(model)}`, timestampSlug(timestamp));
+}
+/** Path of a transcript file within a run dir. A rep index (for --reps N>1) is suffixed. */
+export function transcriptPath(runDir, scenarioId, mode, rep) {
+    const base = rep === undefined ? `${scenarioId}.${mode}` : `${scenarioId}.${mode}.rep${rep}`;
+    return join(runDir, `${base}.txt`);
+}
+export function reportPath(runDir) {
+    return join(runDir, "report.html");
+}
+export function resultsPath(runDir) {
+    return join(runDir, "results.yaml");
+}
+/** The verdict that counts: author override when present, else the judge's. */
+export function effectiveVerdicts(scenarios) {
+    return scenarios.map((s) => ({
+        id: s.id,
+        verdict: s.override ?? s.judge_verdict,
+        suspect: s.suspect && s.override == null, // an override resolves the misfire
+    }));
+}
+/**
+ * The ONLY place effective_grade is computed. Every writer goes through here,
+ * so a persisted grade can never disagree with verdicts + overrides.
+ * ctx is null for unscored (red/force) runs.
+ */
+export function finalizeResults(draft, ctx) {
+    let effective_grade;
+    if (ctx) {
+        const s = score(effectiveVerdicts(draft.scenarios), { shipBar: ctx.shipBar, critical: ctx.critical });
+        effective_grade = { passed: s.passed, total: s.total, pct: s.pct, letter: s.letter, ship: s.ship, note: s.note };
+    }
+    else {
+        effective_grade = { passed: 0, total: 0, pct: 0, letter: "-", ship: false, note: `mode=${draft.mode} (not scored)` };
+    }
+    return {
+        schema: 2,
+        skill: draft.skill,
+        harness: draft.harness,
+        model: draft.model,
+        judge: draft.judge,
+        timestamp: draft.timestamp,
+        label: draft.label,
+        mode: draft.mode,
+        effective_grade,
+        scenarios: draft.scenarios,
+    };
+}
+/** Finalize + persist results.yaml (creating the run dir). Returns what was written. */
+export function writeResults(runDir, draft, ctx) {
+    const results = finalizeResults(draft, ctx);
+    mkdirSync(runDir, { recursive: true });
+    writeFileSync(resultsPath(runDir), yaml.dump(results, { lineWidth: 100 }), "utf8");
+    return results;
+}
+const SUSPECT_PREFIX_RE = /^\[suspect misfire[^\]]*\]\s*/;
+/** Read-only schema-1 → schema-2 migration. Never rewrites the file on disk. */
+export function migrateResults(raw) {
+    if (raw == null || typeof raw !== "object") {
+        throw new Error("empty or invalid results.yaml");
+    }
+    const o = raw;
+    if (o.schema === 2)
+        return raw;
+    const v1 = raw;
+    const modeMatch = /^mode=(\w+)/.exec(v1.grade?.note ?? "");
+    return {
+        schema: 2,
+        skill: v1.skill,
+        harness: v1.harness,
+        model: v1.model,
+        judge: v1.judge,
+        timestamp: v1.timestamp,
+        label: null,
+        mode: modeMatch ? modeMatch[1] : "green",
+        // v1 grades may predate override-aware recompute; carried verbatim (read-only).
+        // Every v2 WRITE recomputes, so staleness cannot propagate.
+        effective_grade: v1.grade,
+        scenarios: (v1.scenarios ?? []).map((s) => {
+            const reason = s.judge_reason ?? "";
+            return {
+                ...s,
+                override: s.override ?? null,
+                note: s.note ?? "",
+                suspect: SUSPECT_PREFIX_RE.test(reason),
+                judge_reason: reason.replace(SUSPECT_PREFIX_RE, ""),
+            };
+        }),
+    };
+}
+/** Read results.yaml from a run dir, migrating schema-1 files in memory. */
+export function readResults(runDir) {
+    const text = readFileSync(resultsPath(runDir), "utf8");
+    return migrateResults(yaml.load(text));
+}
+/** Pure: return a copy with override + note applied to one scenario. */
+export function applyOverride(results, scenarioId, override, note) {
+    if (override !== null && note.trim() === "") {
+        throw new Error(`override for \`${scenarioId}\` requires a note — say why the judge was wrong`);
+    }
+    let found = false;
+    const scenarios = results.scenarios.map((s) => {
+        if (s.id !== scenarioId)
+            return s;
+        found = true;
+        return { ...s, override, note };
+    });
+    if (!found) {
+        throw new Error(`no scenario \`${scenarioId}\` in results`);
+    }
+    return { ...results, scenarios };
+}
+const GITIGNORE_BODY = `# skill-harness: commit verdicts (results.yaml), ignore generated artifacts.
+*.txt
+*.jsonl
+report.html
+!results.yaml
+`;
+/**
+ * Manage results/.gitignore: transcripts + reports ignored, results.yaml tracked.
+ * Rewrites a stale managed body (so new ignore rules roll out) while keeping any
+ * `!…` preservation lines added by preserveTranscript.
+ */
+export function ensureResultsGitignore(resultsRoot) {
+    mkdirSync(resultsRoot, { recursive: true });
+    const giPath = join(resultsRoot, ".gitignore");
+    const existing = existsSync(giPath) ? readFileSync(giPath, "utf8") : "";
+    if (existing.startsWith(GITIGNORE_BODY))
+        return;
+    const preserved = existing
+        .split("\n")
+        .filter((l) => l.startsWith("!") && l.trim() !== "!results.yaml");
+    writeFileSync(giPath, GITIGNORE_BODY + preserved.map((l) => l + "\n").join(""), "utf8");
+}
+// Matches both transcript (`.rep<k>.txt`) and judge-raw (`.rep<k>.judge.txt`) rep suffixes.
+const REP_SUFFIX_RE = /\.rep(\d+)\.(?:judge\.)?txt$/;
+/** The rep index embedded in a transcript/judge-raw filename (`.rep<k>.`), or null for a plain (non-rep) file. */
+export function repIndexOf(filename) {
+    const m = REP_SUFFIX_RE.exec(filename);
+    return m ? Number(m[1]) : null;
+}
+/** Sort transcript-like filenames: plain (no rep) first, then by numeric rep index. */
+function sortByRep(files) {
+    return files.sort((a, b) => {
+        const ra = repIndexOf(a);
+        const rb = repIndexOf(b);
+        if (ra === null && rb === null)
+            return a.localeCompare(b);
+        if (ra === null)
+            return -1;
+        if (rb === null)
+            return 1;
+        return ra - rb;
+    });
+}
+/**
+ * ALL transcript files for a scenario in a run dir, sorted deterministically:
+ * a plain `<id>.<mode>.txt` first (if present), then rep-suffixed files
+ * (`<id>.<mode>.rep<k>.txt`) in numeric rep order. Empty if the run dir or
+ * scenario has no transcripts.
+ *
+ * With `mode` given, only that mode's transcripts match (`<id>.<mode>.txt` /
+ * `<id>.<mode>.rep<k>.txt`) — e.g. to detect a green-only condition without
+ * false positives from a red/force transcript of the same scenario. Omitted,
+ * behavior is unchanged: any `<id>.*.txt` regardless of mode, excluding this
+ * scenario's judge-raw artifacts (`<id>.*.judge.txt` — see judgeRawPath).
+ */
+export function findTranscriptFiles(runDir, scenarioId, mode) {
+    if (!existsSync(runDir))
+        return [];
+    const escapedId = scenarioId.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const matcher = mode !== undefined
+        ? new RegExp(`^${escapedId}\\.${mode}(\\.rep\\d+)?\\.txt$`)
+        : null;
+    const files = readdirSync(runDir).filter((f) => matcher ? matcher.test(f) : f.startsWith(`${scenarioId}.`) && f.endsWith(".txt") && !f.endsWith(".judge.txt"));
+    return sortByRep(files);
+}
+/** Path of a scenario's raw judge-output artifact within a run dir (rep-suffixed for reps). */
+export function judgeRawPath(runDir, scenarioId, mode, rep) {
+    const base = rep === undefined ? `${scenarioId}.${mode}` : `${scenarioId}.${mode}.rep${rep}`;
+    return join(runDir, `${base}.judge.txt`);
+}
+/** A scenario's raw judge-output files, sorted (plain first, then numeric rep). Mode-scoped when given. */
+export function findJudgeRawFiles(runDir, scenarioId, mode) {
+    if (!existsSync(runDir))
+        return [];
+    const esc = scenarioId.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const re = mode === undefined
+        ? new RegExp(`^${esc}\\..*\\.judge\\.txt$`)
+        : new RegExp(`^${esc}\\.${mode}(\\.rep\\d+)?\\.judge\\.txt$`);
+    return sortByRep(readdirSync(runDir).filter((f) => re.test(f)));
+}
+/** A single representative transcript file for a scenario in a run dir. Null if none. */
+export function findTranscriptFile(runDir, scenarioId) {
+    return findTranscriptFiles(runDir, scenarioId)[0] ?? null;
+}
+/**
+ * Un-gitignore ALL of a scenario's transcript AND judge-raw artifact files
+ * (audit trail for an override — a --reps run has one transcript (and one
+ * judge-raw file) per rep, and every rep that drove the verdict must survive
+ * a commit, not just an arbitrary one).
+ * Appends `!<tag>/<ts>/<id>.<mode>[.rep<k>].txt` (and the matching
+ * `.judge.txt`) to results/.gitignore for each, once. The path uses POSIX
+ * separators so the negation matches on Windows too (git ignore patterns are
+ * always forward-slashed).
+ */
+export function preserveTranscript(resultsRoot, runDir, scenarioId) {
+    const files = [...findTranscriptFiles(runDir, scenarioId), ...findJudgeRawFiles(runDir, scenarioId)];
+    if (files.length === 0)
+        return;
+    ensureResultsGitignore(resultsRoot);
+    const giPath = join(resultsRoot, ".gitignore");
+    const existingLines = readFileSync(giPath, "utf8").split("\n");
+    const newLines = [];
+    for (const file of files) {
+        const rel = relative(resultsRoot, join(runDir, file)).split(sep).join("/");
+        const line = `!${rel}`;
+        if (!existingLines.includes(line) && !newLines.includes(line)) {
+            newLines.push(line);
+        }
+    }
+    if (newLines.length > 0) {
+        appendFileSync(giPath, newLines.map((l) => l + "\n").join(""), "utf8");
+    }
+}
+//# sourceMappingURL=results.js.map

package/dist/run.d.ts

@@ -0,0 +1,28 @@
+import type { Spec } from "./spec.js";
+import type { HarnessAdapter, ModelRef, RunMode } from "./adapters/types.js";
+import { type ResultsFile } from "./results.js";
+export interface RunOptions {
+    spec: Spec;
+    skillDir: string;
+    specPath: string;
+    adapter: HarnessAdapter;
+    model: ModelRef;
+    modelToken: string;
+    judge: ModelRef;
+    mode: RunMode;
+    timestamp: string;
+    label?: string | null;
+    onProgress?: (msg: string) => void;
+    now?: () => string;
+    concurrency?: number;
+    reps?: number;
+    passThreshold?: number;
+}
+export interface RunSummary {
+    runDir: string;
+    results: ResultsFile;
+}
+/** Run one skill against one model: run scenarios, grade, score, persist. */
+export declare function runSkillModel(opts: RunOptions): Promise<RunSummary>;
+/** A compact terminal scorecard for one run. */
+export declare function formatScorecard(summary: RunSummary): string;

package/dist/run.js

@@ -0,0 +1,148 @@
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+import { judgeResemblesSubject } from "./grade.js";
+import { runDirFor, transcriptPath, writeResults, ensureResultsGitignore, } from "./results.js";
+import { appendJournal } from "./journal.js";
+import { runSeeded } from "./seeded.js";
+import { createWorkspace } from "./workspace.js";
+import { runPool } from "./scheduler.js";
+import { outcomesToResult } from "./reps.js";
+import { judgeOneRep } from "./regrade.js";
+/** Run one skill against one model: run scenarios, grade, score, persist. */
+export async function runSkillModel(opts) {
+    const { spec, skillDir, adapter, model, judge, mode, timestamp } = opts;
+    const log = opts.onProgress ?? (() => { });
+    const now = opts.now ?? (() => new Date().toISOString());
+    if (judgeResemblesSubject(judge, model)) {
+        log(`  ⚠ judge (${judge.provider}:${judge.model}) resembles the model under test ` +
+            `(${model.provider}:${model.model}) — verdicts may be inflated. Use a distinct judge.`);
+    }
+    const runDir = runDirFor(skillDir, adapter.name, model, timestamp);
+    mkdirSync(runDir, { recursive: true });
+    ensureResultsGitignore(dirname(dirname(runDir))); // .../tests/results/.gitignore
+    appendJournal(runDir, {
+        event: "run-started", ts: now(),
+        skill: spec.skill, harness: adapter.name, model: opts.modelToken,
+        judge: { provider: judge.provider, model: judge.model },
+        mode, label: opts.label ?? null,
+    });
+    // scenario × rep tasks; runPool preserves input order so we can slice per scenario.
+    const repCounts = spec.scenarios.map((s) => s.reps ?? opts.reps ?? 1);
+    const owners = [];
+    const tasks = [];
+    spec.scenarios.forEach((scenario, si) => {
+        for (let k = 0; k < repCounts[si]; k++) {
+            const rep = k;
+            const total = repCounts[si];
+            owners.push(si);
+            tasks.push(() => runRep(scenario, rep, total, { ...opts, runDir, now, log }));
+        }
+    });
+    const flat = await runPool(tasks, opts.concurrency ?? 1);
+    const grouped = spec.scenarios.map(() => []);
+    flat.forEach((outcome, i) => grouped[owners[i]].push(outcome));
+    const scenarioResults = spec.scenarios.map((scenario, si) => {
+        const threshold = scenario.passThreshold ?? opts.passThreshold ?? 0.5;
+        return outcomesToResult(scenario.id, grouped[si], repCounts[si], threshold);
+    });
+    const ctx = mode === "green" ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
+    const results = writeResults(runDir, {
+        skill: spec.skill,
+        harness: adapter.name,
+        model: opts.modelToken,
+        judge: { provider: judge.provider, model: judge.model },
+        timestamp,
+        label: opts.label ?? null,
+        mode,
+        scenarios: scenarioResults,
+    }, ctx);
+    if (ctx) {
+        const g = results.effective_grade;
+        appendJournal(runDir, { event: "score", ts: now(), passed: g.passed, total: g.total, pct: g.pct, letter: g.letter, ship: g.ship, note: g.note });
+    }
+    return { runDir, results };
+}
+/** Run ONE rep of a scenario in its own isolated workspace. */
+async function runRep(scenario, rep, repCount, ctx) {
+    const { spec, judge, mode, runDir, now, log } = ctx;
+    const repField = repCount > 1 ? { rep } : {};
+    if (rep === 0) {
+        log(`  ${scenario.id} (${scenario.title})${repCount > 1 ? ` ×${repCount}` : ""} …`);
+        appendJournal(runDir, { event: "scenario-started", ts: now(), id: scenario.id, title: scenario.title });
+    }
+    let ws = null;
+    let transcript = "";
+    let gatePrefix = null;
+    try {
+        try {
+            ws = createWorkspace(scenario.workspace, { specDir: dirname(ctx.specPath) });
+        }
+        catch (e) {
+            // A setup failure (e.g. missing fixture) is an objective FAIL, not an infra abort.
+            gatePrefix = e instanceof Error ? e.message : String(e);
+            transcript = `[workspace setup failed] ${gatePrefix}`;
+        }
+        if (ws) {
+            if (scenario.mode === "seeded") {
+                const r = await runSeeded(scenario, {
+                    skillDir: ctx.skillDir, adapter: ctx.adapter, model: ctx.model, mode, cwd: ws.cwd,
+                });
+                transcript = r.transcript;
+                gatePrefix = r.gateFailure;
+            }
+            else {
+                transcript = await ctx.adapter.run({
+                    skillDir: ctx.skillDir, model: ctx.model, mode, turns: scenario.turns, cwd: ws.cwd,
+                });
+            }
+        }
+        writeFileSync(transcriptPath(runDir, scenario.id, mode, repCount > 1 ? rep : undefined), transcript, "utf8");
+        if (scenario.mode === "seeded") {
+            appendJournal(runDir, { event: "gate-result", ts: now(), id: scenario.id, ok: !gatePrefix, detail: gatePrefix ?? "", ...repField });
+        }
+        let verdict;
+        let reason;
+        let suspect = false;
+        if (gatePrefix) {
+            verdict = "FAIL";
+            reason = gatePrefix;
+            // gate failures don't invoke the judge, but still record a judge-verdict event (as before)
+            appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict, reason, suspect, ...repField });
+        }
+        else {
+            const o = await judgeOneRep({
+                runDir, spec, scenario, transcript, adapter: ctx.adapter, judge,
+                specDir: dirname(ctx.specPath), mode, rep: repCount > 1 ? rep : undefined, now,
+            });
+            verdict = o.verdict;
+            reason = o.reason;
+            suspect = o.suspect; // judgeOneRep already journaled (verdict + misfire)
+        }
+        log(`  → ${scenario.id}${repCount > 1 ? `#${rep}` : ""} ${verdict}${reason ? `: ${reason}` : ""}${suspect ? "  ⚠ suspect" : ""}`);
+        return { verdict, reason, suspect };
+    }
+    finally {
+        ws?.cleanup();
+    }
+}
+/** A compact terminal scorecard for one run. */
+export function formatScorecard(summary) {
+    const { results } = summary;
+    const g = results.effective_grade;
+    const lines = [];
+    lines.push(`── ${results.skill} · ${results.harness} · ${results.model} ──`);
+    for (const s of results.scenarios) {
+        const v = s.override ?? s.judge_verdict;
+        const mark = v === "PASS" ? "✓" : v === "FAIL" ? "✗" : "?";
+        const ov = s.override ? " (override)" : "";
+        const susp = s.suspect ? " ⚠suspect" : "";
+        const misfired = s.clean !== undefined && s.reps !== undefined && s.clean < s.reps ? ` · ${s.reps - s.clean} misfired` : "";
+        const repInfo = s.reps ? `  [${s.passes}/${s.clean}${misfired}${s.flakiness ? ` flaky ${s.flakiness.toFixed(2)}` : ""}]` : "";
+        lines.push(`  ${mark} ${s.id}${ov}${susp}  ${s.judge_reason}${repInfo}`);
+    }
+    const ship = g.ship ? "SHIP" : "NOT READY";
+    const note = g.note ? ` (${g.note})` : "";
+    lines.push(`  GRADE: ${g.letter} (${g.pct}%) — ${g.passed}/${g.total} — ${ship}${note}`);
+    return lines.join("\n");
+}
+//# sourceMappingURL=run.js.map

package/dist/scheduler.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Run `tasks` with at most `concurrency` thunks in flight at once, returning
+ * their results in input order (not completion order). `concurrency <= 1` runs
+ * them strictly sequentially — identical to a plain for-await loop. A thunk that
+ * throws rejects the returned promise (fail-fast) — once any task rejects,
+ * `runPool` rejects immediately. Tasks already claimed by other workers still
+ * run to completion (JS has no cancellation), and sibling workers may pull
+ * further tasks before the rejection unwinds; the guarantee is that runPool
+ * rejects, not that dispatch halts.
+ */
+export declare function runPool<T>(tasks: Array<() => Promise<T>>, concurrency: number): Promise<T[]>;

package/dist/scheduler.js

@@ -0,0 +1,27 @@
+/**
+ * Run `tasks` with at most `concurrency` thunks in flight at once, returning
+ * their results in input order (not completion order). `concurrency <= 1` runs
+ * them strictly sequentially — identical to a plain for-await loop. A thunk that
+ * throws rejects the returned promise (fail-fast) — once any task rejects,
+ * `runPool` rejects immediately. Tasks already claimed by other workers still
+ * run to completion (JS has no cancellation), and sibling workers may pull
+ * further tasks before the rejection unwinds; the guarantee is that runPool
+ * rejects, not that dispatch halts.
+ */
+export async function runPool(tasks, concurrency) {
+    const limit = Math.max(1, Math.floor(concurrency));
+    const results = new Array(tasks.length);
+    let next = 0;
+    async function worker() {
+        while (true) {
+            const i = next++;
+            if (i >= tasks.length)
+                return;
+            results[i] = await tasks[i]();
+        }
+    }
+    const workerCount = Math.min(limit, tasks.length);
+    await Promise.all(Array.from({ length: workerCount }, () => worker()));
+    return results;
+}
+//# sourceMappingURL=scheduler.js.map

package/dist/score.d.ts

@@ -0,0 +1,33 @@
+import type { ShipBar } from "./spec.js";
+export type Verdict = "PASS" | "FAIL" | "ERROR";
+export interface ScenarioVerdict {
+    id: string;
+    verdict: Verdict;
+    suspect?: boolean;
+}
+export interface ScoreInput {
+    shipBar: ShipBar;
+    critical: string[];
+}
+export interface ScoreResult {
+    passed: number;
+    total: number;
+    pct: number;
+    letter: string;
+    ship: boolean;
+    criticalFails: number;
+    bSeriesFails: number;
+    suspectCount: number;
+    note: string;
+}
+export declare function letterFor(pct: number): string;
+/**
+ * Score a set of green-mode verdicts against the ship bar. A scenario PASSes only
+ * on verdict PASS; FAIL and ERROR both count against it. A `suspect` verdict (an
+ * unresolved judge misfire) is excluded from both `passed` and `total` — it is
+ * untrustworthy, neither a pass nor a fail — and any suspect count blocks SHIP
+ * until an author override resolves it. SHIP otherwise requires: enough total
+ * scenarios, >= min_pass passes, zero critical fails (when no_critical_fail),
+ * and zero B-series (id starting with "B") fails — hold-the-line is the discipline axis.
+ */
+export declare function score(verdicts: ScenarioVerdict[], input: ScoreInput): ScoreResult;

package/dist/score.js

@@ -0,0 +1,62 @@
+export function letterFor(pct) {
+    if (pct >= 90)
+        return "A";
+    if (pct >= 80)
+        return "B";
+    if (pct >= 70)
+        return "C";
+    if (pct >= 60)
+        return "D";
+    return "F";
+}
+/**
+ * Score a set of green-mode verdicts against the ship bar. A scenario PASSes only
+ * on verdict PASS; FAIL and ERROR both count against it. A `suspect` verdict (an
+ * unresolved judge misfire) is excluded from both `passed` and `total` — it is
+ * untrustworthy, neither a pass nor a fail — and any suspect count blocks SHIP
+ * until an author override resolves it. SHIP otherwise requires: enough total
+ * scenarios, >= min_pass passes, zero critical fails (when no_critical_fail),
+ * and zero B-series (id starting with "B") fails — hold-the-line is the discipline axis.
+ */
+export function score(verdicts, input) {
+    const { shipBar, critical } = input;
+    let passed = 0;
+    let total = 0;
+    let criticalFails = 0;
+    let bSeriesFails = 0;
+    let suspectCount = 0;
+    for (const v of verdicts) {
+        if (v.suspect) {
+            suspectCount++;
+            continue; // untrustworthy: neither pass nor fail
+        }
+        total++;
+        if (v.verdict === "PASS") {
+            passed++;
+            continue;
+        }
+        if (critical.includes(v.id))
+            criticalFails++;
+        if (/^B/i.test(v.id))
+            bSeriesFails++;
+    }
+    const pct = total > 0 ? Math.round((passed * 100) / total) : 0;
+    const letter = letterFor(pct);
+    const ship = total >= shipBar.total &&
+        passed >= shipBar.min_pass &&
+        (!shipBar.no_critical_fail || criticalFails === 0) &&
+        bSeriesFails === 0 &&
+        suspectCount === 0;
+    let note = "";
+    if (suspectCount > 0) {
+        note = `${suspectCount} suspect: re-judge/resolve`;
+    }
+    else if (criticalFails > 0) {
+        note = `gated: ${criticalFails} critical fail${criticalFails === 1 ? "" : "s"}`;
+    }
+    else if (bSeriesFails > 0) {
+        note = `gated: ${bSeriesFails} B-series fail${bSeriesFails === 1 ? "" : "s"}`;
+    }
+    return { passed, total, pct, letter, ship, criticalFails, bSeriesFails, suspectCount, note };
+}
+//# sourceMappingURL=score.js.map

package/dist/seeded.d.ts

@@ -0,0 +1,21 @@
+import type { Scenario } from "./spec.js";
+import type { HarnessAdapter, ModelRef, RunMode } from "./adapters/types.js";
+interface SeededOpts {
+    skillDir: string;
+    adapter: HarnessAdapter;
+    model: ModelRef;
+    mode: RunMode;
+    cwd: string;
+}
+export interface SeededOutcome {
+    transcript: string;
+    gateFailure: string | null;
+}
+/**
+ * Run a seeded scenario inside a caller-prepared workspace: let the harness edit
+ * the repo, then evaluate objective gates (staged-diff contains + optional vitest
+ * pass). A failed gate short-circuits to an auto-FAIL. Workspace creation (fixture
+ * copy + git baseline) and teardown are the caller's responsibility (run.ts).
+ */
+export declare function runSeeded(scenario: Scenario, opts: SeededOpts): Promise<SeededOutcome>;
+export {};

package/dist/seeded.js

@@ -0,0 +1,45 @@
+import { exec } from "./util/exec.js";
+const VITEST_TIMEOUT_MS = Number(process.env.SKILL_CHECK_VITEST_TIMEOUT_MS ?? 120_000);
+/**
+ * Run a seeded scenario inside a caller-prepared workspace: let the harness edit
+ * the repo, then evaluate objective gates (staged-diff contains + optional vitest
+ * pass). A failed gate short-circuits to an auto-FAIL. Workspace creation (fixture
+ * copy + git baseline) and teardown are the caller's responsibility (run.ts).
+ */
+export async function runSeeded(scenario, opts) {
+    const repo = opts.cwd;
+    const harnessOut = await opts.adapter.run({
+        skillDir: opts.skillDir,
+        model: opts.model,
+        mode: opts.mode,
+        turns: scenario.turns,
+        cwd: repo,
+    });
+    await git(repo, ["add", "-A"]);
+    const diff = (await git(repo, ["diff", "--cached"])).stdout;
+    const parts = [harnessOut, "", "=== SEEDED GATES ==="];
+    let gateFailure = null;
+    const wantDiff = scenario.assert?.diff_contains ?? [];
+    for (const needle of wantDiff) {
+        const ok = diff.includes(needle);
+        parts.push(`  diff_contains ${JSON.stringify(needle)}: ${ok ? "OK" : "MISSING"}`);
+        if (!ok && !gateFailure)
+            gateFailure = `staged diff missing ${JSON.stringify(needle)}`;
+    }
+    if (scenario.assert?.vitest) {
+        const v = await exec("npx", ["vitest", "run"], { cwd: repo, timeoutMs: VITEST_TIMEOUT_MS });
+        const passed = v.code === 0;
+        parts.push(`  vitest run: ${passed ? "PASS" : `FAIL (exit ${v.code})`}`);
+        parts.push(indent(v.stdout.trim() || v.stderr.trim()));
+        if (!passed && !gateFailure)
+            gateFailure = `vitest failed (exit ${v.code})`;
+    }
+    return { transcript: parts.join("\n"), gateFailure };
+}
+function git(cwd, args) {
+    return exec("git", args, { cwd, timeoutMs: 30_000 });
+}
+function indent(s) {
+    return s.split("\n").map((l) => `    ${l}`).join("\n");
+}
+//# sourceMappingURL=seeded.js.map

package/dist/spec.d.ts

@@ -0,0 +1,39 @@
+import type { WorkspaceKind } from "./workspace.js";
+export type ScenarioMode = "inline" | "seeded";
+export interface SeededAssert {
+    vitest?: boolean;
+    diff_contains?: string[];
+}
+export interface Scenario {
+    id: string;
+    title: string;
+    critical: boolean;
+    mode: ScenarioMode;
+    turns: string[];
+    checklist: string[];
+    fixture?: string;
+    assert?: SeededAssert;
+    workspace: WorkspaceKind;
+    reps?: number;
+    passThreshold?: number;
+}
+export interface ShipBar {
+    total: number;
+    min_pass: number;
+    no_critical_fail: boolean;
+}
+export interface Spec {
+    skill: string;
+    judge_persona: string;
+    ship_bar: ShipBar;
+    critical: string[];
+    scenarios: Scenario[];
+}
+/** Thrown on any validation failure. Message always carries the spec file path. */
+export declare class SpecError extends Error {
+    constructor(message: string, file: string);
+}
+/** Parse + validate a specification.yaml from its raw text. `file` is used in error messages. */
+export declare function parseSpec(text: string, file: string): Spec;
+/** Load + validate a specification.yaml from disk. */
+export declare function loadSpec(file: string): Spec;