@skill-harness/core 0.1.0

package/dist/regrade.js

@@ -0,0 +1,107 @@
+import { readFileSync, writeFileSync, existsSync } from "node:fs";
+import { join } from "node:path";
+import { buildJudgePrompt, judgeInWorkspace } from "./grade.js";
+import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResults, effectiveThreshold, } from "./results.js";
+import { outcomesToResult } from "./reps.js";
+import { appendJournal } from "./journal.js";
+/** Judge one saved transcript: writes the judge-raw artifact, emits a `judge-verdict` journal event (plus `misfire-flag` when the verdict is suspect), and returns the outcome. */
+export async function judgeOneRep(opts) {
+    const { runDir, spec, scenario, transcript, adapter, judge, specDir, mode, rep, now } = opts;
+    const prompt = buildJudgePrompt({ skill: spec.skill, persona: spec.judge_persona, scenario, transcript });
+    const g = await judgeInWorkspace(adapter, judge, prompt, specDir);
+    writeFileSync(judgeRawPath(runDir, scenario.id, mode, rep), g.raw, "utf8");
+    const repField = rep === undefined ? {} : { rep };
+    appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict: g.verdict, reason: g.reason, suspect: g.suspect, ...repField });
+    if (g.suspect)
+        appendJournal(runDir, { event: "misfire-flag", ts: now(), id: scenario.id, reason: g.reason, ...repField });
+    return { verdict: g.verdict, reason: g.reason, suspect: g.suspect };
+}
+/**
+ * Re-judge a scenario's saved GREEN transcript(s) with `judge` — no harness
+ * re-run. Rewrites the judge-raw artifact per rep, emits per-rep judge-verdict
+ * (+ misfire-flag) journal events, and returns the aggregated ScenarioResult
+ * (override/note empty; the caller merges any prior override + persists).
+ */
+export async function regradeScenario(opts) {
+    const now = opts.now ?? (() => new Date().toISOString());
+    const files = findTranscriptFiles(opts.runDir, opts.scenario.id, "green");
+    if (files.length === 0)
+        throw new Error(`no green transcripts for ${opts.scenario.id} in ${opts.runDir}`);
+    const repCount = files.length;
+    const outcomes = [];
+    for (const file of files) {
+        const rep = repIndexOf(file) ?? undefined;
+        const transcript = readFileSync(join(opts.runDir, file), "utf8");
+        outcomes.push(await judgeOneRep({
+            runDir: opts.runDir, spec: opts.spec, scenario: opts.scenario, transcript,
+            adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir, mode: "green", rep, now,
+        }));
+    }
+    return outcomesToResult(opts.scenario.id, outcomes, repCount, opts.threshold);
+}
+/**
+ * Re-judge every green-transcript scenario in a run dir with `judge` — no
+ * harness re-run. Targets are the run's RECORDED scenarios (falling back to
+ * the spec for a run with no prior results.yaml), so re-grading rewrites the
+ * whole results.yaml consistently with what the run actually recorded. Each
+ * target must still exist in the spec (for its checklist) AND have a green
+ * transcript on disk; anything missing fails fast before spending any judge
+ * calls. Preserves each prior scenario's override/note, rewrites
+ * results.yaml, emits the `score` journal event, and returns the new
+ * ResultsFile. Shared by `cmdGrade` and the pi-extension's `judge` command.
+ */
+export async function regradeRun(opts) {
+    const { runDir, spec, adapter, judge, specDir } = opts;
+    const now = opts.now ?? (() => new Date().toISOString());
+    const prev = existsSync(join(runDir, "results.yaml")) ? readResults(runDir) : null;
+    const overrides = new Map((prev?.scenarios ?? []).map((s) => [s.id, { override: s.override, note: s.note }]));
+    const mode = prev?.mode ?? "green";
+    // Re-grading rewrites the WHOLE results.yaml, so re-judge exactly the
+    // scenarios the run recorded (falling back to the spec for a run with no
+    // prior results). The guard and the loop iterate the SAME `targets` set, so
+    // they can't diverge: each target must still exist in the spec (for its
+    // checklist) AND have a transcript on disk — only overridden transcripts
+    // survive a commit (audit-trail design). Anything missing would silently drop
+    // a recorded verdict or shrink the grade denominator. Fail fast, before
+    // spending any judge calls.
+    const specById = new Map(spec.scenarios.map((s) => [s.id, s]));
+    const targets = (prev?.scenarios ?? spec.scenarios).map((s) => s.id);
+    const missing = targets.filter((id) => !specById.has(id) || findTranscriptFiles(runDir, id, "green").length === 0);
+    if (missing.length === targets.length) {
+        throw new Error(`no green transcripts in ${runDir} — nothing to re-grade`);
+    }
+    if (missing.length > 0) {
+        throw new Error(`cannot re-grade ${missing.join(", ")} in ${runDir} (transcript missing or scenario no longer in the spec) — re-run instead of grading`);
+    }
+    const scenarioResults = [];
+    for (const id of targets) {
+        const scenario = specById.get(id); // guaranteed present by the guard above
+        const prevScenario = prev?.scenarios.find((s) => s.id === id);
+        const threshold = effectiveThreshold(prevScenario, scenario);
+        const rr = await regradeScenario({
+            runDir, spec, scenario, adapter, judge, specDir, threshold, now,
+        });
+        const carry = overrides.get(id);
+        scenarioResults.push({ ...rr, override: carry?.override ?? null, note: carry?.note ?? "" });
+    }
+    const ctx = mode === "green" ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
+    const results = writeResults(runDir, {
+        skill: spec.skill,
+        harness: prev?.harness ?? "pi",
+        model: prev?.model ?? "unknown",
+        judge: { provider: judge.provider, model: judge.model },
+        timestamp: prev?.timestamp ?? now(),
+        label: prev?.label ?? null,
+        mode,
+        scenarios: scenarioResults,
+    }, ctx);
+    const g = results.effective_grade;
+    if (ctx) {
+        appendJournal(runDir, {
+            event: "score", ts: now(),
+            passed: g.passed, total: g.total, pct: g.pct, letter: g.letter, ship: g.ship, note: g.note,
+        });
+    }
+    return results;
+}
+//# sourceMappingURL=regrade.js.map

package/dist/report.d.ts

@@ -0,0 +1,80 @@
+import { type ShipBar } from "./spec.js";
+import { type ResultsFile } from "./results.js";
+export interface RunColumn {
+    index: number;
+    label: string;
+    tag: string;
+    runDir: string;
+    timestamp: string;
+    mode: string;
+    grade: ResultsFile["effective_grade"];
+    judge: ResultsFile["judge"];
+    cells: Record<string, {
+        judge_verdict: string;
+        judge_reason: string;
+        suspect: boolean;
+        reps?: number;
+        passes?: number;
+        clean?: number;
+        flakiness?: number;
+        override: string | null;
+        note: string;
+    }>;
+}
+export interface ReportData {
+    skill: string;
+    shipBar: ShipBar;
+    critical: string[];
+    scenarios: {
+        id: string;
+        title: string;
+        critical: boolean;
+    }[];
+    columns: RunColumn[];
+}
+/**
+ * Collect the latest run per model-tag under <skillDir>/tests/results/, plus the
+ * scenario list from the spec (for titles + order). One column per model.
+ */
+export declare function collectReport(skillDir: string): ReportData;
+/** Client-facing view (no absolute paths leaked). */
+export declare function publicView(data: ReportData): {
+    skill: string;
+    shipBar: ShipBar;
+    critical: string[];
+    scenarios: {
+        id: string;
+        title: string;
+        critical: boolean;
+    }[];
+    columns: {
+        index: number;
+        label: string;
+        tag: string;
+        timestamp: string;
+        mode: string;
+        grade: import("./results.js").GradeSummary;
+        judge: {
+            provider: string;
+            model: string;
+        };
+        cells: Record<string, {
+            judge_verdict: string;
+            judge_reason: string;
+            suspect: boolean;
+            reps?: number;
+            passes?: number;
+            clean?: number;
+            flakiness?: number;
+            override: string | null;
+            note: string;
+        }>;
+    }[];
+};
+/**
+ * Inject the run JSON and the client-scorer module into the template, at the
+ * __DATA__ and __GRADE__ placeholders respectively. `gradeScript` is the raw
+ * contents of assets/report.grade.js (sibling of the template) — the single,
+ * score.ts-parity-tested copy of the client grading logic.
+ */
+export declare function renderReport(template: string, data: ReportData, gradeScript: string): string;

package/dist/report.js

@@ -0,0 +1,106 @@
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { loadSpec } from "./spec.js";
+import { readResults } from "./results.js";
+/** Most-recent run dir (by name, which is an ISO-ish slug) under a model-tag dir. */
+function latestRunDir(tagDir) {
+    if (!statSync(tagDir).isDirectory())
+        return null;
+    const runs = readdirSync(tagDir)
+        .map((n) => join(tagDir, n))
+        .filter((p) => statSync(p).isDirectory() && existsSync(join(p, "results.yaml")))
+        .sort();
+    return runs.length ? runs[runs.length - 1] : null;
+}
+/**
+ * Collect the latest run per model-tag under <skillDir>/tests/results/, plus the
+ * scenario list from the spec (for titles + order). One column per model.
+ */
+export function collectReport(skillDir) {
+    const specPath = join(skillDir, "tests", "specification.yaml");
+    const spec = loadSpec(specPath);
+    const scenarios = spec.scenarios.map((s) => ({ id: s.id, title: s.title, critical: s.critical }));
+    const resultsRoot = join(skillDir, "tests", "results");
+    const columns = [];
+    if (existsSync(resultsRoot)) {
+        const tags = readdirSync(resultsRoot)
+            .map((n) => join(resultsRoot, n))
+            .filter((p) => statSync(p).isDirectory())
+            .sort();
+        for (const tagDir of tags) {
+            const runDir = latestRunDir(tagDir);
+            if (!runDir)
+                continue;
+            const r = readResults(runDir);
+            const cells = {};
+            for (const s of r.scenarios) {
+                cells[s.id] = {
+                    judge_verdict: s.judge_verdict,
+                    judge_reason: s.judge_reason,
+                    suspect: s.suspect ?? false, // suspect defaults false for older results that predate the field
+                    reps: s.reps,
+                    passes: s.passes,
+                    clean: s.clean,
+                    flakiness: s.flakiness,
+                    override: s.override,
+                    note: s.note,
+                };
+            }
+            columns.push({
+                index: columns.length,
+                label: r.model,
+                tag: tagDir.split("/").pop(),
+                runDir,
+                timestamp: r.timestamp,
+                mode: r.mode,
+                grade: r.effective_grade,
+                judge: r.judge,
+                cells,
+            });
+        }
+    }
+    return { skill: spec.skill, shipBar: spec.ship_bar, critical: spec.critical, scenarios, columns };
+}
+/** Client-facing view (no absolute paths leaked). */
+export function publicView(data) {
+    return {
+        skill: data.skill,
+        shipBar: data.shipBar,
+        critical: data.critical,
+        scenarios: data.scenarios,
+        columns: data.columns.map((c) => ({
+            index: c.index,
+            label: c.label,
+            tag: c.tag,
+            timestamp: c.timestamp,
+            mode: c.mode,
+            grade: c.grade,
+            judge: c.judge,
+            cells: c.cells,
+        })),
+    };
+}
+/**
+ * A bare inline <script> (no type="module") can't contain an `export`
+ * statement, but assets/report.grade.js is written as real ESM so it can also
+ * be imported directly (by Node, in the parity test). Strip the `export `
+ * keyword off each exported declaration so the leftover plain function
+ * declarations splice cleanly into the template's script scope.
+ */
+function stripExports(js) {
+    return js.replace(/^export\s+/gm, "");
+}
+/**
+ * Inject the run JSON and the client-scorer module into the template, at the
+ * __DATA__ and __GRADE__ placeholders respectively. `gradeScript` is the raw
+ * contents of assets/report.grade.js (sibling of the template) — the single,
+ * score.ts-parity-tested copy of the client grading logic.
+ */
+export function renderReport(template, data, gradeScript) {
+    const json = JSON.stringify(publicView(data));
+    return template
+        .replace("/*__DATA__*/null", json)
+        .replace("/*__GRADE__*/", stripExports(gradeScript))
+        .replace("__SKILL__", data.skill);
+}
+//# sourceMappingURL=report.js.map

package/dist/reps.d.ts

@@ -0,0 +1,34 @@
+import type { Verdict } from "./score.js";
+import type { ScenarioResult } from "./results.js";
+/** One rep's outcome (subject run + judge). */
+export interface RepOutcome {
+    verdict: Verdict;
+    reason: string;
+    suspect: boolean;
+}
+/** A scenario's aggregated result over N reps. */
+export interface RepAggregate {
+    verdict: Verdict;
+    reason: string;
+    passes: number;
+    reps: number;
+    clean: number;
+    flakiness: number;
+    suspect: boolean;
+}
+/**
+ * Collapse N rep outcomes into one scenario verdict. A rep is "clean" when its
+ * judge did not misfire. If fewer than half the reps are clean the scenario is
+ * `suspect` (its verdict is untrustworthy). Otherwise the pass-rate is computed
+ * over the clean reps and the scenario PASSes at `pass_rate >= threshold`
+ * (default caller threshold 0.5, ties pass). Flakiness = 1 - |2·pass_rate - 1|.
+ */
+export declare function aggregateReps(outcomes: RepOutcome[], threshold: number): RepAggregate;
+/**
+ * Collapse a scenario's rep outcomes into a ScenarioResult. N=1 preserves the
+ * single judge's verdict/reason with no reps fields (byte-identical to a plain
+ * run); N>1 aggregates and persists the effective threshold (so a later
+ * re-judge reproduces the same pass-rate). override/note are left empty for the
+ * caller to merge.
+ */
+export declare function outcomesToResult(id: string, outcomes: RepOutcome[], repCount: number, threshold: number): ScenarioResult;

package/dist/reps.js

@@ -0,0 +1,45 @@
+/**
+ * Collapse N rep outcomes into one scenario verdict. A rep is "clean" when its
+ * judge did not misfire. If fewer than half the reps are clean the scenario is
+ * `suspect` (its verdict is untrustworthy). Otherwise the pass-rate is computed
+ * over the clean reps and the scenario PASSes at `pass_rate >= threshold`
+ * (default caller threshold 0.5, ties pass). Flakiness = 1 - |2·pass_rate - 1|.
+ */
+export function aggregateReps(outcomes, threshold) {
+    const reps = outcomes.length;
+    const clean = outcomes.filter((o) => !o.suspect);
+    const passes = clean.filter((o) => o.verdict === "PASS").length;
+    if (clean.length * 2 < reps) {
+        // majority of reps misfired → untrustworthy
+        return { verdict: "FAIL", reason: `${reps - clean.length}/${reps} reps misfired — re-judge`, passes, reps, clean: clean.length, flakiness: 0, suspect: true };
+    }
+    const errored = clean.filter((o) => o.verdict === "ERROR").length;
+    if (clean.length > 0 && errored === clean.length) {
+        return { verdict: "ERROR", reason: `${errored}/${reps} reps errored`, passes: 0, reps, clean: clean.length, flakiness: 0, suspect: false };
+    }
+    const passRate = passes / clean.length;
+    const verdict = passRate >= threshold ? "PASS" : "FAIL";
+    const flakiness = 1 - Math.abs(2 * passRate - 1);
+    const reason = reps === 1 ? outcomes[0].reason : `${passes}/${clean.length} reps passed (flaky ${flakiness.toFixed(2)})`;
+    return { verdict, reason, passes, reps, clean: clean.length, flakiness, suspect: false };
+}
+/**
+ * Collapse a scenario's rep outcomes into a ScenarioResult. N=1 preserves the
+ * single judge's verdict/reason with no reps fields (byte-identical to a plain
+ * run); N>1 aggregates and persists the effective threshold (so a later
+ * re-judge reproduces the same pass-rate). override/note are left empty for the
+ * caller to merge.
+ */
+export function outcomesToResult(id, outcomes, repCount, threshold) {
+    if (repCount === 1) {
+        const o = outcomes[0];
+        return { id, judge_verdict: o.verdict, judge_reason: o.reason, suspect: o.suspect, override: null, note: "" };
+    }
+    const agg = aggregateReps(outcomes, threshold);
+    return {
+        id, judge_verdict: agg.verdict, judge_reason: agg.reason, suspect: agg.suspect,
+        reps: agg.reps, passes: agg.passes, clean: agg.clean, flakiness: agg.flakiness,
+        pass_threshold: threshold, override: null, note: "",
+    };
+}
+//# sourceMappingURL=reps.js.map

package/dist/results.d.ts

@@ -0,0 +1,108 @@
+import { type ModelRef } from "./adapters/types.js";
+import { type ScenarioVerdict } from "./score.js";
+import type { Verdict } from "./score.js";
+import type { ShipBar, Scenario } from "./spec.js";
+export interface ScenarioResult {
+    id: string;
+    judge_verdict: Verdict;
+    judge_reason: string;
+    suspect: boolean;
+    override: Verdict | null;
+    note: string;
+    reps?: number;
+    passes?: number;
+    clean?: number;
+    flakiness?: number;
+    pass_threshold?: number;
+}
+export interface GradeSummary {
+    passed: number;
+    total: number;
+    pct: number;
+    letter: string;
+    ship: boolean;
+    note: string;
+}
+export interface ResultsFile {
+    schema: 2;
+    skill: string;
+    harness: string;
+    model: string;
+    judge: {
+        provider: string;
+        model: string;
+    };
+    timestamp: string;
+    label: string | null;
+    mode: string;
+    effective_grade: GradeSummary;
+    scenarios: ScenarioResult[];
+}
+/** The pass-threshold a re-grade uses: the run's persisted value, else the spec's per-scenario value, else 0.5. */
+export declare function effectiveThreshold(prevScenario: ScenarioResult | undefined, scenario: Scenario): number;
+/** Everything a caller may set. The grade is computed, never supplied. */
+export type ResultsDraft = Omit<ResultsFile, "schema" | "effective_grade">;
+export interface ScoreContext {
+    shipBar: ShipBar;
+    critical: string[];
+}
+/** <skillDir>/tests/results/<harness>-<model-slug>/<timestamp-slug>/ */
+export declare function runDirFor(skillDir: string, harness: string, model: ModelRef, timestamp: string): string;
+/** Path of a transcript file within a run dir. A rep index (for --reps N>1) is suffixed. */
+export declare function transcriptPath(runDir: string, scenarioId: string, mode: string, rep?: number): string;
+export declare function reportPath(runDir: string): string;
+export declare function resultsPath(runDir: string): string;
+/** The verdict that counts: author override when present, else the judge's. */
+export declare function effectiveVerdicts(scenarios: ScenarioResult[]): ScenarioVerdict[];
+/**
+ * The ONLY place effective_grade is computed. Every writer goes through here,
+ * so a persisted grade can never disagree with verdicts + overrides.
+ * ctx is null for unscored (red/force) runs.
+ */
+export declare function finalizeResults(draft: ResultsDraft, ctx: ScoreContext | null): ResultsFile;
+/** Finalize + persist results.yaml (creating the run dir). Returns what was written. */
+export declare function writeResults(runDir: string, draft: ResultsDraft, ctx: ScoreContext | null): ResultsFile;
+/** Read-only schema-1 → schema-2 migration. Never rewrites the file on disk. */
+export declare function migrateResults(raw: unknown): ResultsFile;
+/** Read results.yaml from a run dir, migrating schema-1 files in memory. */
+export declare function readResults(runDir: string): ResultsFile;
+/** Pure: return a copy with override + note applied to one scenario. */
+export declare function applyOverride(results: ResultsFile, scenarioId: string, override: Verdict | null, note: string): ResultsFile;
+/**
+ * Manage results/.gitignore: transcripts + reports ignored, results.yaml tracked.
+ * Rewrites a stale managed body (so new ignore rules roll out) while keeping any
+ * `!…` preservation lines added by preserveTranscript.
+ */
+export declare function ensureResultsGitignore(resultsRoot: string): void;
+/** The rep index embedded in a transcript/judge-raw filename (`.rep<k>.`), or null for a plain (non-rep) file. */
+export declare function repIndexOf(filename: string): number | null;
+/**
+ * ALL transcript files for a scenario in a run dir, sorted deterministically:
+ * a plain `<id>.<mode>.txt` first (if present), then rep-suffixed files
+ * (`<id>.<mode>.rep<k>.txt`) in numeric rep order. Empty if the run dir or
+ * scenario has no transcripts.
+ *
+ * With `mode` given, only that mode's transcripts match (`<id>.<mode>.txt` /
+ * `<id>.<mode>.rep<k>.txt`) — e.g. to detect a green-only condition without
+ * false positives from a red/force transcript of the same scenario. Omitted,
+ * behavior is unchanged: any `<id>.*.txt` regardless of mode, excluding this
+ * scenario's judge-raw artifacts (`<id>.*.judge.txt` — see judgeRawPath).
+ */
+export declare function findTranscriptFiles(runDir: string, scenarioId: string, mode?: string): string[];
+/** Path of a scenario's raw judge-output artifact within a run dir (rep-suffixed for reps). */
+export declare function judgeRawPath(runDir: string, scenarioId: string, mode: string, rep?: number): string;
+/** A scenario's raw judge-output files, sorted (plain first, then numeric rep). Mode-scoped when given. */
+export declare function findJudgeRawFiles(runDir: string, scenarioId: string, mode?: string): string[];
+/** A single representative transcript file for a scenario in a run dir. Null if none. */
+export declare function findTranscriptFile(runDir: string, scenarioId: string): string | null;
+/**
+ * Un-gitignore ALL of a scenario's transcript AND judge-raw artifact files
+ * (audit trail for an override — a --reps run has one transcript (and one
+ * judge-raw file) per rep, and every rep that drove the verdict must survive
+ * a commit, not just an arbitrary one).
+ * Appends `!<tag>/<ts>/<id>.<mode>[.rep<k>].txt` (and the matching
+ * `.judge.txt`) to results/.gitignore for each, once. The path uses POSIX
+ * separators so the negation matches on Windows too (git ignore patterns are
+ * always forward-slashed).
+ */
+export declare function preserveTranscript(resultsRoot: string, runDir: string, scenarioId: string): void;