@skill-harness/core 0.22.0 → 0.22.3

package/dist/grade.d.ts

@@ -38,6 +38,8 @@ export declare function judgeResemblesSubject(judge: ModelRef, subject: ModelRef
 export interface GradeResult extends ParsedVerdict {
     raw: string;
     criteria: CriterionVote[];
+    /** One identical-input retry after missing criterion votes. Absent on the first attempt. */
+    judgeRetries?: 1;
     /** Judge misfire: the overall verdict disagrees with AND(per-item grades). Recorded, never auto-passed; blocks SHIP until re-judged or overridden. */
     suspect: boolean;
 }
@@ -50,10 +52,10 @@ export interface GradeResult extends ParsedVerdict {
  */
 export declare function detectMisfire(raw: string, verdict: Verdict): boolean;
 /** Drive the judge for one transcript and parse the result. */
-export declare function gradeTranscript(adapter: HarnessAdapter, judge: ModelRef, prompt: string, cwd: string): Promise<GradeResult>;
+export declare function gradeTranscript(adapter: HarnessAdapter, judge: ModelRef, prompt: string, cwd: string, expectedCriteria?: number): Promise<GradeResult>;
 /**
  * Grade a transcript in a fresh, isolated, throwaway workspace — never the
  * subject's scenario dir — so the judge can't ingest repo context the subject
  * left behind (matters for provider CLIs that read cwd, including Pi).
  */
-export declare function judgeInWorkspace(adapter: HarnessAdapter, judge: ModelRef, prompt: string, specDir: string): Promise<GradeResult>;
+export declare function judgeInWorkspace(adapter: HarnessAdapter, judge: ModelRef, prompt: string, specDir: string, expectedCriteria?: number): Promise<GradeResult>;

package/dist/grade.js

@@ -1,5 +1,5 @@
 import { createWorkspace } from "./workspace.js";
-import { parseCriterionVotes } from "./results.js";
+import { parseCriterionVotes, completeCriterionVotes } from "./results.js";
 /** The heading runSeeded writes above the staged diff. Gating on this exact string is what keeps the guidance honest — see below. */
 export const STAGED_DIFF_HEADING = "=== STAGED DIFF ===";
 /**
@@ -123,8 +123,16 @@ export function detectMisfire(raw, verdict) {
 // Non-global twin of REASON_RE: matchAll needs /g, a single .match() must not have it.
 const REASON_LINE_RE = /^\s*\**\s*REASON\**\s*:\s*\**\s*(.*)$/im;
 /** Drive the judge for one transcript and parse the result. */
-export async function gradeTranscript(adapter, judge, prompt, cwd) {
-    const raw = await adapter.judge({ model: judge, prompt, cwd });
+export async function gradeTranscript(adapter, judge, prompt, cwd, expectedCriteria) {
+    const request = { model: judge, prompt, cwd };
+    let raw = await adapter.judge(request);
+    let criteria = parseCriterionVotes(raw);
+    let judgeRetries;
+    if (expectedCriteria !== undefined && completeCriterionVotes(criteria, expectedCriteria).some(vote => vote.verdict === "ERROR")) {
+        raw = await adapter.judge(request);
+        criteria = parseCriterionVotes(raw);
+        judgeRetries = 1;
+    }
     const parsed = parseVerdict(raw);
     // On a parse failure, surface what the judge actually emitted (e.g. a provider
     // error) rather than a generic message — otherwise the cause is invisible.
@@ -134,17 +142,17 @@ export async function gradeTranscript(adapter, judge, prompt, cwd) {
             parsed.reason = `judge unparseable: ${snippet}`;
     }
     const suspect = detectMisfire(raw, parsed.verdict);
-    return { ...parsed, raw, suspect, criteria: parseCriterionVotes(raw) };
+    return { ...parsed, raw, suspect, criteria, ...(judgeRetries ? { judgeRetries } : {}) };
 }
 /**
  * Grade a transcript in a fresh, isolated, throwaway workspace — never the
  * subject's scenario dir — so the judge can't ingest repo context the subject
  * left behind (matters for provider CLIs that read cwd, including Pi).
  */
-export async function judgeInWorkspace(adapter, judge, prompt, specDir) {
+export async function judgeInWorkspace(adapter, judge, prompt, specDir, expectedCriteria) {
     const ws = createWorkspace("none", { specDir });
     try {
-        return await gradeTranscript(adapter, judge, prompt, ws.cwd);
+        return await gradeTranscript(adapter, judge, prompt, ws.cwd, expectedCriteria);
     }
     finally {
         ws.cleanup();

package/dist/regate.js

@@ -290,12 +290,13 @@ export async function regateRun(opts) {
             }
             // The gate blocked this rep before, so no judgement of it exists anywhere.
             const transcript = readFileSync(tPath, "utf8");
-            outcomes.push({ ...(await judgeOneRep({
-                    runDir: opts.runDir, spec: opts.spec, scenario, transcript,
-                    adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir,
-                    mode, rep, now, rejudge: true,
-                })), objective });
-            judgeCalls++;
+            const judged = await judgeOneRep({
+                runDir: opts.runDir, spec: opts.spec, scenario, transcript,
+                adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir,
+                mode, rep, now, rejudge: true,
+            });
+            outcomes.push({ ...judged, objective });
+            judgeCalls += judged.metrics?.judge_calls ?? 0;
             judgedHere++;
         }
         const threshold = effectiveThreshold(rec, scenario);

package/dist/regrade.d.ts

@@ -34,6 +34,9 @@ export interface RegradeOptions {
     mode?: string;
     /** Recorded repetition count; when supplied, missing/extra rep artifacts fail before judge calls. */
     expectedReps?: number;
+    /** Select only repetitions with retained ERROR criterion votes. */
+    onlyUnparsed?: boolean;
+    prior?: ScenarioResult;
     now?: () => string;
 }
 /** Judge one saved transcript: writes the judge-raw artifact, emits a `judge-verdict` journal event (plus `misfire-flag` when the verdict is suspect), and returns the outcome. */
@@ -72,6 +75,8 @@ export interface RegradeRunOptions {
      * cleanly, so spending judge calls on them buys nothing.
      */
     onlySuspect?: boolean;
+    /** Rejudge only retained repetitions containing ERROR criterion votes, with the recorded judge. */
+    onlyUnparsed?: boolean;
 }
 /**
  * Re-judge every scenario in a run dir that has a transcript for the run's own

package/dist/regrade.js

@@ -56,18 +56,18 @@ export async function judgeOneRep(opts) {
         };
     }
     const prompt = buildJudgePrompt({ skill: spec.skill, persona: spec.judge_persona, scenario, transcript });
-    const g = await judgeInWorkspace(adapter, judge, prompt, specDir);
+    const g = await judgeInWorkspace(adapter, judge, prompt, specDir, scenario.checklist.length);
     writeFileSync(judgeRawPath(runDir, scenario.id, mode, rep), g.raw, "utf8");
     appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict: g.verdict, reason: g.reason, suspect: g.suspect, ...repField });
     if (g.suspect)
         appendJournal(runDir, { event: "misfire-flag", ts: now(), id: scenario.id, reason: g.reason, ...repField });
     return {
         verdict: g.verdict, reason: g.reason, suspect: g.suspect,
-        judgment: { ordinal: 1, judge: { ...judge }, verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, scenario.checklist.length) },
+        judgment: { ordinal: 1, judge: { ...judge }, verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, scenario.checklist.length), ...(g.judgeRetries ? { judgeRetries: g.judgeRetries } : {}) },
         metrics: {
             wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)),
-            judge_calls: 1,
-            judge_rejudge_calls: opts.rejudge ? 1 : 0,
+            judge_calls: 1 + (g.judgeRetries ?? 0),
+            judge_rejudge_calls: opts.rejudge ? 1 + (g.judgeRetries ?? 0) : 0,
         },
     };
 }
@@ -81,18 +81,30 @@ export async function judgeOneRep(opts) {
 export async function regradeScenario(opts) {
     const now = opts.now ?? (() => new Date().toISOString());
     const mode = opts.mode ?? "green";
-    const files = findTranscriptFiles(opts.runDir, opts.scenario.id, mode);
+    let files = findTranscriptFiles(opts.runDir, opts.scenario.id, mode);
+    if (opts.onlyUnparsed)
+        files = files.filter(file => hasUnparsedVotes(opts.prior?.rep_judgments?.find(panel => panel.repetition === (repIndexOf(file) ?? 0))));
     if (files.length === 0)
         throw new Error(`no ${mode} transcripts for ${opts.scenario.id} in ${opts.runDir}`);
     const expected = opts.expectedReps ?? files.length;
-    const expectedIndices = expected === 1 ? [null] : Array.from({ length: expected }, (_, index) => index);
+    const expectedIndices = opts.onlyUnparsed
+        ? opts.prior.rep_judgments.filter(hasUnparsedVotes).map(panel => expected === 1 ? null : panel.repetition).sort((a, b) => (a ?? -1) - (b ?? -1))
+        : expected === 1 ? [null] : Array.from({ length: expected }, (_, index) => index);
     const actualIndices = files.map((file) => repIndexOf(file)).sort((a, b) => (a ?? -1) - (b ?? -1));
-    if (files.length !== expected || JSON.stringify(actualIndices) !== JSON.stringify(expectedIndices)) {
+    if (files.length !== expectedIndices.length || JSON.stringify(actualIndices) !== JSON.stringify(expectedIndices)) {
         throw new Error(`${opts.scenario.id}: transcript artifacts are incomplete for ${expected} recorded rep(s) — re-run instead of grading a smaller repetition set`);
     }
     const repCount = expected;
     const outcomes = [];
-    for (const file of files) {
+    for (let repetition = 0; repetition < expected; repetition++) {
+        const panel = opts.prior?.rep_judgments?.find(panel => panel.repetition === repetition);
+        if (opts.onlyUnparsed && !hasUnparsedVotes(panel)) {
+            if (!panel)
+                throw new Error(`missing retained judgments for ${opts.scenario.id}#${repetition}`);
+            outcomes.push({ verdict: panel.recorded_verdict, reason: panel.judgments[0]?.reason ?? "", suspect: panel.judgments.length > 0 && !panel.judgments.some(judgment => !judgment.suspect && (judgment.verdict === "PASS" || judgment.verdict === "FAIL")), judgment: panel.judgments[0], objective: panel.objective });
+            continue;
+        }
+        const file = files.find(file => (repIndexOf(file) ?? 0) === repetition);
         const rep = repIndexOf(file) ?? undefined;
         const transcript = readFileSync(join(opts.runDir, file), "utf8");
         outcomes.push(await judgeOneRep({
@@ -100,7 +112,16 @@ export async function regradeScenario(opts) {
             adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir, mode, rep, now, rejudge: true,
         }));
     }
-    return outcomesToResult(opts.scenario.id, outcomes, repCount, opts.threshold);
+    const result = outcomesToResult(opts.scenario.id, outcomes, repCount, opts.threshold);
+    if (opts.onlyUnparsed)
+        result.rep_judgments = result.rep_judgments?.map(panel => {
+            const prior = opts.prior.rep_judgments.find(prior => prior.repetition === panel.repetition);
+            return hasUnparsedVotes(prior) ? panel : prior;
+        });
+    return result;
+}
+function hasUnparsedVotes(panel) {
+    return panel?.judgments.some(judgment => judgment.criteria?.some(vote => vote.verdict === "ERROR")) ?? false;
 }
 /**
  * Re-judge every scenario in a run dir that has a transcript for the run's own
@@ -114,17 +135,21 @@ export async function regradeScenario(opts) {
  * ResultsFile. Shared by `cmdGrade` and the pi-extension's `judge` command.
  */
 export async function regradeRun(opts) {
-    const { runDir, spec, adapter, judge, specDir } = opts;
+    const { runDir, spec, adapter, specDir } = opts;
     const now = opts.now ?? (() => new Date().toISOString());
     const prev = existsSync(join(runDir, "results.yaml")) ? readResults(runDir) : null;
+    if (opts.onlyUnparsed && !prev)
+        throw new Error(`--unparsed-only needs a prior results.yaml in ${runDir}`);
+    if (opts.onlyUnparsed && opts.onlySuspect)
+        throw new Error("--unparsed-only cannot be combined with --suspect-only");
+    const judge = opts.onlyUnparsed ? prev.judge : opts.judge;
     // Carried across a re-judge, per field, because the right answer differs:
     //  - `override`/`note`  — the author's, never the judge's to discard.
     //  - `objective`        — a re-judge does NOT re-evaluate trace gates (that is
     //    `regate`), so the recorded evidence still describes this run. Dropping it
     //    silently downgraded a gated scenario to "no assertions declared".
-    // `adjudication` is deliberately NOT carried: it describes the judgments this
-    // re-grade just replaced, and a stale panel beside a fresh verdict is worse
-    // than none.
+    // `adjudication` is dropped when its repetition is rejudged. A selective
+    // repair of a different repetition carries that untouched evidence instead.
     // The whole prior result per id — `rebuildScenarioResult` decides, field by
     // field, what survives. Passing a hand-picked subset here is how fields got
     // dropped before.
@@ -148,6 +173,8 @@ export async function regradeRun(opts) {
             .filter((s) => s.suspect || s.judge_verdict === "JUDGE-AMBIGUOUS")
             .map((s) => s.id);
     }
+    if (opts.onlyUnparsed)
+        targets = recorded.filter(s => s.rep_judgments?.some(hasUnparsedVotes)).map(s => s.id);
     // Schema-3 delivery is a prerequisite for behavioral judging. A later `grade`
     // cannot turn an undelivered or unauthenticated transcript into product evidence.
     if (prev?.schema === 3) {
@@ -161,6 +188,9 @@ export async function regradeRun(opts) {
     const completeTranscripts = (record) => {
         const expected = record.reps ?? 1;
         const files = findTranscriptFiles(runDir, record.id, mode);
+        if (opts.onlyUnparsed) {
+            return record.rep_judgments?.length === expected && record.rep_judgments.every(panel => !hasUnparsedVotes(panel) || files.some(file => (repIndexOf(file) ?? 0) === panel.repetition)) || false;
+        }
         const expectedIndices = expected === 1 ? [null] : Array.from({ length: expected }, (_, index) => index);
         const actualIndices = files.map((file) => repIndexOf(file)).sort((a, b) => (a ?? -1) - (b ?? -1));
         return files.length === expected && JSON.stringify(actualIndices) === JSON.stringify(expectedIndices);
@@ -173,6 +203,14 @@ export async function regradeRun(opts) {
     if (missing.length > 0) {
         throw new Error(`cannot re-grade ${missing.join(", ")} in ${runDir} (transcript missing or scenario no longer in the spec) — re-run instead of grading`);
     }
+    if (opts.onlyUnparsed) {
+        const drifted = targets.filter(id => {
+            const count = recordedById.get(id).criterion_count;
+            return count !== undefined && count !== specById.get(id).checklist.length;
+        });
+        if (drifted.length)
+            throw new Error(`criterion count changed for ${drifted.join(", ")} — use full grade instead of --unparsed-only`);
+    }
     const targetSet = new Set(targets);
     const scenarioResults = [];
     for (const rec of recorded) {
@@ -186,17 +224,17 @@ export async function regradeRun(opts) {
         const threshold = effectiveThreshold(prevScenario, scenario);
         const rr = await regradeScenario({
             runDir, spec, scenario, adapter, judge, specDir, threshold, mode, now,
-            expectedReps: prevScenario?.reps ?? 1,
+            expectedReps: prevScenario?.reps ?? 1, onlyUnparsed: opts.onlyUnparsed, prior: prevScenario,
         });
         const carry = overrides.get(id);
         if (prev?.schema === 3)
             rr.criterion_count = scenario.checklist.length;
         rr.metrics = mergeScenarioMetrics(carry?.metrics, rr.metrics);
         rr.rep_judgments = carryRepObjectives(rr.rep_judgments, carry?.rep_judgments);
-        // `grade` re-judges the saved transcript. It does not re-evaluate trace gates
-        // (that is `regate`), so `objective` still describes this run; and it replaced
-        // the judgments a prior adjudication described, so that panel must go.
-        scenarioResults.push(rebuildScenarioResult(rr, carry, { objective: "carry", adjudication: "drop" }));
+        // Gates still describe the original subject execution. An adjudication of
+        // an untouched repetition also survives, bounded by rebuild's conflict guards.
+        const carryAdjudication = opts.onlyUnparsed && carry?.adjudication && !hasUnparsedVotes(carry.rep_judgments?.find(panel => panel.repetition === (carry.adjudication.repetition ?? 0)));
+        scenarioResults.push(rebuildScenarioResult(rr, carry, { objective: "carry", adjudication: carryAdjudication ? "carry" : "drop" }));
     }
     const ctx = scoreContextFor({ mode, partial: prev?.partial }, spec);
     const results = writeResults(runDir, {
@@ -229,7 +267,7 @@ export async function regradeRun(opts) {
         // say so. Doctrine narrowed 0.4.0, from "recorded hashes stay" to "recorded
         // *stimulus* hashes stay" — see refreshRubricHashes.
         partial: prev?.partial,
-        source_hashes: refreshRubricHashes(prev?.source_hashes, spec, targets),
+        source_hashes: opts.onlyUnparsed ? prev?.source_hashes : refreshRubricHashes(prev?.source_hashes, spec, targets),
         source_hash_roots: prev?.source_hash_roots,
         scenarios: scenarioResults,
     }, ctx);

package/dist/results.d.ts

@@ -99,6 +99,8 @@ export interface Judgment {
     suspect: boolean;
     /** Every numbered rubric-item vote parsed from this exact judge response. Required in schema v3. */
     criteria?: CriterionVote[];
+    /** One retry with identical judge input after unparseable criterion votes. */
+    judgeRetries?: 1;
 }
 export interface AdjudicationResult {
     /** The saved transcript repetition judged by every panel member (first rep by policy). */

package/dist/results.js

@@ -613,7 +613,7 @@ export function rebuildScenarioResult(fresh, prior, policy) {
     const objective = pick(policy.objective, freshObjective, prior?.objective);
     const pickedAdjudication = pick(policy.adjudication, freshAdjudication, prior?.adjudication);
     const conflictsWithFreshEvidence = Boolean(pickedAdjudication?.verdict && (objective?.status === "FAIL" || objective?.status === "ERROR" || objective?.status === "NOT-MEASURED" ||
-        (pass_threshold === 1 && (reps ?? 1) > 1 && judge_verdict !== "PASS")));
+        (pickedAdjudication.verdict === "PASS" && pass_threshold === 1 && (reps ?? 1) > 1 && judge_verdict !== "PASS")));
     // A settled cell-level panel cannot overrule newly evaluated objective gates,
     // nor can one transcript erase a failed repetition under an all-clean policy.
     const adjudication = conflictsWithFreshEvidence && pickedAdjudication

package/dist/run.js

@@ -521,7 +521,7 @@ async function runRep(scenario, rep, repCount, ctx) {
             reason = o.reason;
             suspect = o.suspect;
             judgment = o.judgment; // judgeOneRep already journaled (verdict + misfire)
-            judgeCalls = 1;
+            judgeCalls = o.metrics?.judge_calls ?? 0;
         }
         log(`  → ${scenario.id}${repCount > 1 ? `#${rep}` : ""} ${verdict}${reason ? `: ${reason}` : ""}${suspect ? "  ⚠ suspect" : ""}`);
         const subject = mergeTraces(traces)?.metrics;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.22.0",
+    "version": "0.22.3",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",

package/schemas/results-v2.schema.json

@@ -232,6 +232,7 @@
         "suspect"
       ],
       "properties": {
+        "judgeRetries": { "type": "integer", "const": 1 },
         "ordinal": {
           "type": "integer",
           "minimum": 1,

package/schemas/results-v3.schema.json

@@ -244,6 +244,7 @@
         "criteria"
       ],
       "properties": {
+        "judgeRetries": { "type": "integer", "const": 1 },
         "ordinal": {
           "type": "integer",
           "minimum": 1,