@skill-harness/core 0.1.2 → 0.3.0

package/dist/adapters/types.d.ts

@@ -14,6 +14,12 @@ export interface RunReq {
     mode: RunMode;
     turns: string[];
     cwd: string;
+    /**
+     * Abs path to a markdown file to use AS the system prompt, instead of activating
+     * skillDir as a skill. Used to test a subagent definition (agents/<name>.md) in the
+     * single-shot shape it actually runs in; overrides `mode`'s skill flags.
+     */
+    systemPromptFile?: string;
 }
 /** A judge request: single prompt, no skills, no session. */
 export interface JudgeReq {

package/dist/discover.d.ts

@@ -10,5 +10,9 @@ export interface DiscoveredSkill {
  * Returns skills sorted by name (testable or not).
  */
 export declare function discover(root: string): DiscoveredSkill[];
-/** Resolve a single skill by name; throws a helpful error if absent or specless. */
+/**
+ * Resolve a single skill by name; throws a helpful error if absent or specless.
+ * A directory that exists but lacks a SKILL.md gets a specific error (rather than
+ * the generic "no skill") so callers don't reimplement the SKILL.md existence check.
+ */
 export declare function resolveSkill(root: string, name: string): DiscoveredSkill;

package/dist/discover.js

@@ -24,10 +24,18 @@ export function discover(root) {
     skills.sort((a, b) => a.name.localeCompare(b.name));
     return skills;
 }
-/** Resolve a single skill by name; throws a helpful error if absent or specless. */
+/**
+ * Resolve a single skill by name; throws a helpful error if absent or specless.
+ * A directory that exists but lacks a SKILL.md gets a specific error (rather than
+ * the generic "no skill") so callers don't reimplement the SKILL.md existence check.
+ */
 export function resolveSkill(root, name) {
     const skill = discover(root).find((s) => s.name === name);
     if (!skill) {
+        const dir = join(root, name);
+        if (existsSync(dir) && statSync(dir).isDirectory() && !existsSync(join(dir, "SKILL.md"))) {
+            throw new Error(`skill \`${name}\` has no SKILL.md (looked in ${dir})`);
+        }
         throw new Error(`no skill \`${name}\` under ${root}`);
     }
     return skill;

package/dist/grade.d.ts

@@ -7,13 +7,26 @@ export interface JudgePromptInput {
     scenario: Scenario;
     transcript: string;
 }
+/** The heading runSeeded writes above the staged diff. Gating on this exact string is what keeps the guidance honest — see below. */
+export declare const STAGED_DIFF_HEADING = "=== STAGED DIFF ===";
 /** Build the LLM-judge prompt for one transcript (ported from the old grade.sh). */
 export declare function buildJudgePrompt(input: JudgePromptInput): string;
 export interface ParsedVerdict {
     verdict: Verdict;
     reason: string;
 }
-/** Parse a judge's raw output into a verdict + reason. Unparseable → ERROR. */
+/**
+ * Parse a judge's raw output into a verdict + reason.
+ *
+ * Judges sometimes emit MORE than one verdict block (a first pass, then a restated
+ * conclusion). Every block is read, never just the first:
+ *   - all blocks agree  → that verdict, with the reason from the LAST block (the
+ *     judge's final word) in full
+ *   - blocks disagree   → JUDGE-AMBIGUOUS, which counts as a non-pass and carries both
+ *     verdicts in the reason so a rejudge can be queued. Silently taking either one
+ *     would be inventing a grade the judge did not give.
+ * Unparseable → ERROR.
+ */
 export declare function parseVerdict(out: string): ParsedVerdict;
 /**
  * Judge-≠-subject de-confound guard. True when the judge resembles the model

package/dist/grade.js

@@ -1,8 +1,33 @@
 import { createWorkspace } from "./workspace.js";
+/** The heading runSeeded writes above the staged diff. Gating on this exact string is what keeps the guidance honest — see below. */
+export const STAGED_DIFF_HEADING = "=== STAGED DIFF ===";
+/**
+ * Addendum pointing the judge at the code, added only when the code is actually there.
+ *
+ * A seeded transcript ends with the staged diff, and without this the judge
+ * weighs the model's prose about its work equally with the work itself — which
+ * is how six reps that all passed the objective gates split PASS/FAIL purely on
+ * whether the model wrote "rejects overdrafts" or "subtracts amount".
+ *
+ * Gated on the transcript CONTAINING the diff section, not on `scenario.mode`.
+ * The first sentence is a factual claim about the transcript, and every seeded
+ * transcript saved before this feature existed lacks that section — so keying
+ * off the mode would tell the judge its primary evidence is at the end of a
+ * transcript that has none, while also telling it the gate lines prove nothing.
+ * That is a sweep of FAILs justified by absent evidence, and it would land
+ * squarely on `grade`, the command AGENTS.md rule 4 recommends as the *cheap*
+ * de-confounding step before re-running. Inline scenarios have no diff either,
+ * so their prompt stays byte-identical and every published inline verdict
+ * remains comparable.
+ */
+const SEEDED_DIFF_GUIDANCE = `
+This transcript ends with a "=== STAGED DIFF ===" section: the actual code the assistant wrote, as \`git diff --cached\`. It is the primary evidence. Grade what the diff shows the code DOES, not what the assistant's prose claims it does — a confident description of behavior the diff does not implement is a FAIL, and behavior the diff plainly implements passes even if the assistant described it poorly or not at all. The "=== SEEDED GATES ===" lines above it are keyword and test-run checks only; they do not establish that the required behavior exists. If the diff is marked truncated, judge only what you can see and never infer that cut-off code is missing.
+`;
 /** Build the LLM-judge prompt for one transcript (ported from the old grade.sh). */
 export function buildJudgePrompt(input) {
     const { skill, persona, scenario, transcript } = input;
     const numbered = scenario.checklist.map((c, i) => `${i + 1}. ${c}`).join("\n");
+    const diffGuidance = scenario.mode === "seeded" && transcript.includes(STAGED_DIFF_HEADING) ? SEEDED_DIFF_GUIDANCE : "";
     return `You are grading ONE response from an AI assistant using a "${skill}" skill — ${persona} Judge it ONLY against the checklist below — do not add requirements beyond it.
 
 CHECKLIST (every numbered item must hold for a PASS):
@@ -10,23 +35,44 @@ ${numbered}
 
 TRANSCRIPT (the assistant is the model under test):
 ${transcript}
-
+${diffGuidance}
 Grade each checklist item PASS or FAIL with a <=12-word justification quoting the transcript. Be skeptical: if an item is not clearly satisfied, mark it FAIL. Then output exactly these two lines:
 VERDICT: PASS      (only if EVERY item passed)   — or —   VERDICT: FAIL
 REASON: <15 words or fewer>`;
 }
-const VERDICT_RE = /VERDICT\**\s*:?\s*\**\s*(PASS|FAIL)/i;
-const REASON_RE = /REASON\**\s*:?\s*\**\s*(.*)$/im;
-/** Parse a judge's raw output into a verdict + reason. Unparseable → ERROR. */
+// Both anchor to the start of a line and REQUIRE the colon. Without those anchors the
+// reason pattern matched any word containing "reason" — git-ops GLM C1 stored
+// "able given no repo present.", a fragment of "Reasonable" in the judge's prose, which
+// then read as a FAIL-verdict-with-passing-reason misfire that never happened.
+const VERDICT_RE = /^\s*\**\s*VERDICT\**\s*:\s*\**\s*(PASS|FAIL)/gim;
+const REASON_RE = /^\s*\**\s*REASON\**\s*:\s*\**\s*(.*)$/gim;
+/**
+ * Parse a judge's raw output into a verdict + reason.
+ *
+ * Judges sometimes emit MORE than one verdict block (a first pass, then a restated
+ * conclusion). Every block is read, never just the first:
+ *   - all blocks agree  → that verdict, with the reason from the LAST block (the
+ *     judge's final word) in full
+ *   - blocks disagree   → JUDGE-AMBIGUOUS, which counts as a non-pass and carries both
+ *     verdicts in the reason so a rejudge can be queued. Silently taking either one
+ *     would be inventing a grade the judge did not give.
+ * Unparseable → ERROR.
+ */
 export function parseVerdict(out) {
-    const vm = out.match(VERDICT_RE);
-    if (!vm) {
+    const verdicts = [...out.matchAll(VERDICT_RE)].map((m) => m[1].toUpperCase());
+    if (verdicts.length === 0) {
         return { verdict: "ERROR", reason: "judge produced no parseable verdict" };
     }
-    const verdict = vm[1].toUpperCase();
-    const rm = out.match(REASON_RE);
-    const reason = rm ? rm[1].trim() : "";
-    return { verdict, reason };
+    const reasons = [...out.matchAll(REASON_RE)].map((m) => m[1].trim());
+    const reason = reasons.length > 0 ? reasons[reasons.length - 1] : "";
+    const unique = [...new Set(verdicts)];
+    if (unique.length > 1) {
+        return {
+            verdict: "JUDGE-AMBIGUOUS",
+            reason: `judge emitted conflicting verdicts (${verdicts.join(", ")}) — needs rejudge; last reason: ${reason}`,
+        };
+    }
+    return { verdict: unique[0], reason };
 }
 /**
  * Judge-≠-subject de-confound guard. True when the judge resembles the model
@@ -51,13 +97,30 @@ const ITEM_RE = /^\s*\d+[.)]\s*\**\s*(PASS|FAIL)\b/gim;
 export function detectMisfire(raw, verdict) {
     if (verdict === "ERROR")
         return false;
+    // Conflicting verdicts are suspect by construction — there is no consistent grade.
+    if (verdict === "JUDGE-AMBIGUOUS")
+        return true;
     const items = [...raw.matchAll(ITEM_RE)].map((m) => m[1].toUpperCase() === "PASS");
-    if (items.length === 0)
+    if (items.length === 0) {
+        // No item lines to cross-check, so fall back to the verdict-vs-reason shape: a FAIL
+        // whose reason says everything passed is the misfire class from REVIEW-FINDINGS
+        // finding 2. Deliberately narrow — an earlier version of this tripwire fired on
+        // terse genuine FAILs, so it requires an explicitly total claim ("all items ...
+        // pass", "every item ... satisfied") and no negation anywhere in the reason.
+        if (verdict === "FAIL") {
+            const reason = (raw.match(REASON_LINE_RE)?.[1] ?? "").trim();
+            const totalPass = /\b(all|every)\b[^.]*\b(pass(es|ed)?|satisf(y|ies|ied)|hold(s)?|met)\b/i.test(reason);
+            const negated = /\b(not|no|n't|fails?|failed|missing|except|but|however)\b/i.test(reason);
+            return totalPass && !negated;
+        }
         return false; // fail-open
+    }
     const andItems = items.every((ok) => ok);
     const verdictBool = verdict === "PASS";
     return verdictBool !== andItems;
 }
+// Non-global twin of REASON_RE: matchAll needs /g, a single .match() must not have it.
+const REASON_LINE_RE = /^\s*\**\s*REASON\**\s*:\s*\**\s*(.*)$/im;
 /** Drive the judge for one transcript and parse the result. */
 export async function gradeTranscript(adapter, judge, prompt, cwd) {
     const raw = await adapter.judge({ model: judge, prompt, cwd });

package/dist/index.d.ts

@@ -8,10 +8,15 @@ export * from "./journal.js";
 export * from "./scheduler.js";
 export * from "./reps.js";
 export * from "./regrade.js";
+export * from "./rescore.js";
 export * from "./workspace.js";
 export * from "./seeded.js";
 export * from "./report.js";
 export * from "./trends.js";
 export * from "./lint.js";
+export * from "./sources.js";
+export * from "./lift.js";
 export * from "./adapters/types.js";
 export * from "./util/exec.js";
+export * from "./util/env.js";
+export * from "./scaffold.js";

package/dist/index.js

@@ -8,11 +8,16 @@ export * from "./journal.js";
 export * from "./scheduler.js";
 export * from "./reps.js";
 export * from "./regrade.js";
+export * from "./rescore.js";
 export * from "./workspace.js";
 export * from "./seeded.js";
 export * from "./report.js";
 export * from "./trends.js";
 export * from "./lint.js";
+export * from "./sources.js";
+export * from "./lift.js";
 export * from "./adapters/types.js";
 export * from "./util/exec.js";
+export * from "./util/env.js";
+export * from "./scaffold.js";
 //# sourceMappingURL=index.js.map

package/dist/journal.d.ts

@@ -54,6 +54,20 @@ export type JournalEvent = {
     id: string;
     reason: string;
     rep?: number;
+} | {
+    event: "empty-response-retry";
+    ts: string;
+    id: string;
+    attempt: number;
+    rep?: number;
+} | {
+    event: "rescore";
+    ts: string;
+    changed: string[];
+    passed: number;
+    total: number;
+    pct: number;
+    ship: boolean;
 } | {
     event: "score";
     ts: string;

package/dist/lift.d.ts

@@ -0,0 +1,69 @@
+import { type ResultsFile } from "./results.js";
+import type { Verdict } from "./score.js";
+/**
+ * What a skill did to one scenario, red (skill off) vs green (skill active).
+ *
+ * `inconclusive` is load-bearing: it is what stops lift from becoming a number
+ * that only ever goes up. See `classify`.
+ */
+export type LiftClass = "gained" | "regressed" | "kept" | "both-fail" | "inconclusive";
+export interface LiftCell {
+    red: Verdict;
+    /**
+     * Whether the red side's verdict was an unresolved misfire. Carried (not just
+     * folded into `class`) because the review UI recomputes the class live as the
+     * author overrides green verdicts — it needs the red evidence, not a frozen
+     * conclusion. See assets/report.grade.js's liftClass.
+     */
+    redSuspect: boolean;
+    green: Verdict;
+    class: LiftClass;
+}
+export interface Lift {
+    tag: string;
+    model: string;
+    redTimestamp: string;
+    greenTimestamp: string;
+    /** Scenario ids present in both runs — the only ones a lift can speak to. */
+    compared: number;
+    gained: number;
+    regressed: number;
+    kept: number;
+    bothFail: number;
+    inconclusive: number;
+    redPassed: number;
+    greenPassed: number;
+    delta: number;
+    /** Ids the green run covered that the red baseline did not (and vice versa). */
+    greenOnly: string[];
+    redOnly: string[];
+    /** True when either side was an `--only` run, so coverage is a subset by construction. */
+    partial: boolean;
+    cells: Record<string, LiftCell>;
+}
+/**
+ * Compare a red (baseline, skill off) run against a green (skill active) run of
+ * the same model: the "does this skill actually do anything?" measurement.
+ *
+ * Both sides go through `effectiveVerdicts`, so an author override is what
+ * counts and an override resolves a misfire — the same rule scoring uses. Only
+ * the intersection of scenario ids is compared; a lift cannot speak to a
+ * scenario one side never ran.
+ */
+export declare function computeLift(red: ResultsFile, green: ResultsFile): Lift;
+/** One line for a human: what the skill did, and what it cost. */
+export declare function liftHeadline(lift: Lift): string;
+/**
+ * Per model-tag under <skillDir>/tests/results/, pair the most recent red run
+ * with the most recent green run and compute the lift.
+ *
+ * Deliberately derived on read rather than persisted into results.yaml: a lift
+ * is a fact about a *pair* of runs, so caching it inside one run's file would go
+ * stale the moment a new baseline lands — the stale-scorecard failure mode
+ * `source_hashes` exists to prevent. Deriving also means lift works
+ * retroactively on results already committed by 0.1.x/0.2.0 users.
+ *
+ * A tag with no red baseline is omitted entirely rather than reported as a zero
+ * lift: "not measured" and "measured no effect" are different claims.
+ */
+export declare function collectLift(skillDir: string): Lift[];

package/dist/lift.js

@@ -0,0 +1,163 @@
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { readResults, effectiveVerdicts } from "./results.js";
+/** A verdict that carries real evidence about the task, rather than about the harness or the judge. */
+function conclusive(verdict, suspect) {
+    // ERROR is a harness failure (timeout, empty reply) — it says nothing about
+    // whether the agent could do the task. JUDGE-AMBIGUOUS and an unresolved
+    // misfire are judge failures. Treating any of them as a FAIL would make
+    // "red FAIL -> green PASS" fire on infrastructure noise, and lift would
+    // measure flakiness while looking like skill value.
+    return !suspect && verdict !== "ERROR" && verdict !== "JUDGE-AMBIGUOUS";
+}
+function classify(red, green) {
+    if (!conclusive(red.verdict, red.suspect) || !conclusive(green.verdict, green.suspect))
+        return "inconclusive";
+    const redPass = red.verdict === "PASS";
+    const greenPass = green.verdict === "PASS";
+    if (redPass && greenPass)
+        return "kept";
+    if (!redPass && greenPass)
+        return "gained";
+    if (redPass && !greenPass)
+        return "regressed";
+    return "both-fail";
+}
+/**
+ * Compare a red (baseline, skill off) run against a green (skill active) run of
+ * the same model: the "does this skill actually do anything?" measurement.
+ *
+ * Both sides go through `effectiveVerdicts`, so an author override is what
+ * counts and an override resolves a misfire — the same rule scoring uses. Only
+ * the intersection of scenario ids is compared; a lift cannot speak to a
+ * scenario one side never ran.
+ */
+export function computeLift(red, green) {
+    const redV = new Map(effectiveVerdicts(red.scenarios).map((v) => [v.id, { verdict: v.verdict, suspect: v.suspect ?? false }]));
+    const greenV = new Map(effectiveVerdicts(green.scenarios).map((v) => [v.id, { verdict: v.verdict, suspect: v.suspect ?? false }]));
+    const cells = {};
+    const counts = { gained: 0, regressed: 0, kept: 0, "both-fail": 0, inconclusive: 0 };
+    let redPassed = 0;
+    let greenPassed = 0;
+    // Green order drives display order (it is the run the author is looking at),
+    // restricted to ids the red baseline also covered.
+    for (const [id, g] of greenV) {
+        const r = redV.get(id);
+        if (!r)
+            continue;
+        const cls = classify(r, g);
+        cells[id] = { red: r.verdict, redSuspect: r.suspect, green: g.verdict, class: cls };
+        counts[cls]++;
+        if (cls !== "inconclusive") {
+            if (r.verdict === "PASS")
+                redPassed++;
+            if (g.verdict === "PASS")
+                greenPassed++;
+        }
+    }
+    return {
+        tag: "",
+        model: green.model,
+        redTimestamp: red.timestamp,
+        greenTimestamp: green.timestamp,
+        compared: Object.keys(cells).length,
+        gained: counts.gained,
+        regressed: counts.regressed,
+        kept: counts.kept,
+        bothFail: counts["both-fail"],
+        inconclusive: counts.inconclusive,
+        redPassed,
+        greenPassed,
+        delta: greenPassed - redPassed,
+        greenOnly: [...greenV.keys()].filter((id) => !redV.has(id)),
+        redOnly: [...redV.keys()].filter((id) => !greenV.has(id)),
+        partial: Boolean(red.partial || green.partial),
+        cells,
+    };
+}
+/** One line for a human: what the skill did, and what it cost. */
+export function liftHeadline(lift) {
+    if (lift.compared === 0)
+        return "no shared scenarios to compare";
+    // Everything inconclusive is NOT "no effect" — it is no measurement. Saying
+    // "no measured effect" here would be the same not-measured/measured-no-effect
+    // conflation this module refuses to make when a red baseline is missing.
+    const conclusive = lift.compared - lift.inconclusive;
+    if (conclusive === 0) {
+        return `nothing conclusive to compare (${lift.inconclusive} inconclusive — fix the harness/judge, then re-run)`;
+    }
+    const segments = [];
+    if (lift.gained === 0 && lift.regressed === 0) {
+        segments.push(lift.kept > 0
+            ? `no measured effect (${lift.kept} passed without the skill too)`
+            : "no measured effect");
+    }
+    else {
+        const sign = lift.delta > 0 ? `+${lift.delta}` : String(lift.delta);
+        segments.push(`${sign} net (${lift.gained} gained, ${lift.regressed} regressed)`);
+    }
+    if (lift.inconclusive > 0)
+        segments.push(`${lift.inconclusive} inconclusive`);
+    if (lift.partial)
+        segments.push("partial run");
+    return segments.join(" · ");
+}
+/** A directory that exists right now; false (never throws) if it vanished concurrently. */
+function isDir(p) {
+    try {
+        return statSync(p).isDirectory();
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Per model-tag under <skillDir>/tests/results/, pair the most recent red run
+ * with the most recent green run and compute the lift.
+ *
+ * Deliberately derived on read rather than persisted into results.yaml: a lift
+ * is a fact about a *pair* of runs, so caching it inside one run's file would go
+ * stale the moment a new baseline lands — the stale-scorecard failure mode
+ * `source_hashes` exists to prevent. Deriving also means lift works
+ * retroactively on results already committed by 0.1.x/0.2.0 users.
+ *
+ * A tag with no red baseline is omitted entirely rather than reported as a zero
+ * lift: "not measured" and "measured no effect" are different claims.
+ */
+export function collectLift(skillDir) {
+    const resultsRoot = join(skillDir, "tests", "results");
+    if (!existsSync(resultsRoot))
+        return [];
+    const lifts = [];
+    for (const tag of readdirSync(resultsRoot).filter((n) => isDir(join(resultsRoot, n))).sort()) {
+        const tagDir = join(resultsRoot, tag);
+        const runDirs = readdirSync(tagDir)
+            .map((n) => join(tagDir, n))
+            .filter((p) => isDir(p) && existsSync(join(p, "results.yaml")))
+            .sort(); // timestamp-slug names ⇒ chronological ascending
+        // Mode is only knowable after reading results.yaml, so every run in the tag
+        // is read; last-wins gives the most recent of each mode.
+        let red;
+        let green;
+        for (const rd of runDirs) {
+            let r;
+            try {
+                r = readResults(rd);
+            }
+            catch (e) {
+                // A corrupt/truncated results.yaml must not take down the whole view.
+                console.warn(`skill-harness lift: skipping unreadable run ${rd}: ${e instanceof Error ? e.message : e}`);
+                continue;
+            }
+            if (r.mode === "red")
+                red = r;
+            else if (r.mode === "green")
+                green = r;
+        }
+        if (!red || !green)
+            continue;
+        lifts.push({ ...computeLift(red, green), tag });
+    }
+    return lifts;
+}
+//# sourceMappingURL=lift.js.map

package/dist/lint.d.ts

@@ -1,4 +1,4 @@
-export type LintCode = "spec" | "ship_bar" | "critical" | "fixture" | "consistency" | "lint-error";
+export type LintCode = "spec" | "ship_bar" | "critical" | "fixture" | "fixture-marker" | "consistency" | "stale" | "lint-error";
 export interface LintFinding {
     readonly skill: string;
     readonly scenario?: string;