@skill-harness/core 0.22.0 → 0.22.4

package/dist/grade.d.ts

@@ -38,6 +38,8 @@ export declare function judgeResemblesSubject(judge: ModelRef, subject: ModelRef
 export interface GradeResult extends ParsedVerdict {
     raw: string;
     criteria: CriterionVote[];
+    /** One identical-input retry after missing criterion votes. Absent on the first attempt. */
+    judgeRetries?: 1;
     /** Judge misfire: the overall verdict disagrees with AND(per-item grades). Recorded, never auto-passed; blocks SHIP until re-judged or overridden. */
     suspect: boolean;
 }
@@ -50,10 +52,10 @@ export interface GradeResult extends ParsedVerdict {
  */
 export declare function detectMisfire(raw: string, verdict: Verdict): boolean;
 /** Drive the judge for one transcript and parse the result. */
-export declare function gradeTranscript(adapter: HarnessAdapter, judge: ModelRef, prompt: string, cwd: string): Promise<GradeResult>;
+export declare function gradeTranscript(adapter: HarnessAdapter, judge: ModelRef, prompt: string, cwd: string, expectedCriteria?: number): Promise<GradeResult>;
 /**
  * Grade a transcript in a fresh, isolated, throwaway workspace — never the
  * subject's scenario dir — so the judge can't ingest repo context the subject
  * left behind (matters for provider CLIs that read cwd, including Pi).
  */
-export declare function judgeInWorkspace(adapter: HarnessAdapter, judge: ModelRef, prompt: string, specDir: string): Promise<GradeResult>;
+export declare function judgeInWorkspace(adapter: HarnessAdapter, judge: ModelRef, prompt: string, specDir: string, expectedCriteria?: number): Promise<GradeResult>;

package/dist/grade.js

@@ -1,5 +1,5 @@
 import { createWorkspace } from "./workspace.js";
-import { parseCriterionVotes } from "./results.js";
+import { parseCriterionVotes, completeCriterionVotes } from "./results.js";
 /** The heading runSeeded writes above the staged diff. Gating on this exact string is what keeps the guidance honest — see below. */
 export const STAGED_DIFF_HEADING = "=== STAGED DIFF ===";
 /**
@@ -123,8 +123,16 @@ export function detectMisfire(raw, verdict) {
 // Non-global twin of REASON_RE: matchAll needs /g, a single .match() must not have it.
 const REASON_LINE_RE = /^\s*\**\s*REASON\**\s*:\s*\**\s*(.*)$/im;
 /** Drive the judge for one transcript and parse the result. */
-export async function gradeTranscript(adapter, judge, prompt, cwd) {
-    const raw = await adapter.judge({ model: judge, prompt, cwd });
+export async function gradeTranscript(adapter, judge, prompt, cwd, expectedCriteria) {
+    const request = { model: judge, prompt, cwd };
+    let raw = await adapter.judge(request);
+    let criteria = parseCriterionVotes(raw);
+    let judgeRetries;
+    if (expectedCriteria !== undefined && completeCriterionVotes(criteria, expectedCriteria).some(vote => vote.verdict === "ERROR")) {
+        raw = await adapter.judge(request);
+        criteria = parseCriterionVotes(raw);
+        judgeRetries = 1;
+    }
     const parsed = parseVerdict(raw);
     // On a parse failure, surface what the judge actually emitted (e.g. a provider
     // error) rather than a generic message — otherwise the cause is invisible.
@@ -134,17 +142,17 @@ export async function gradeTranscript(adapter, judge, prompt, cwd) {
             parsed.reason = `judge unparseable: ${snippet}`;
     }
     const suspect = detectMisfire(raw, parsed.verdict);
-    return { ...parsed, raw, suspect, criteria: parseCriterionVotes(raw) };
+    return { ...parsed, raw, suspect, criteria, ...(judgeRetries ? { judgeRetries } : {}) };
 }
 /**
  * Grade a transcript in a fresh, isolated, throwaway workspace — never the
  * subject's scenario dir — so the judge can't ingest repo context the subject
  * left behind (matters for provider CLIs that read cwd, including Pi).
  */
-export async function judgeInWorkspace(adapter, judge, prompt, specDir) {
+export async function judgeInWorkspace(adapter, judge, prompt, specDir, expectedCriteria) {
     const ws = createWorkspace("none", { specDir });
     try {
-        return await gradeTranscript(adapter, judge, prompt, ws.cwd);
+        return await gradeTranscript(adapter, judge, prompt, ws.cwd, expectedCriteria);
     }
     finally {
         ws.cleanup();

package/dist/lift.js

@@ -29,7 +29,7 @@ function conclusive(verdict, suspect) {
     // misfire are judge failures. Treating any of them as a FAIL would make
     // "red FAIL -> green PASS" fire on infrastructure noise, and lift would
     // measure flakiness while looking like skill value.
-    return !suspect && verdict !== "ERROR" && verdict !== "NOT-MEASURED" && verdict !== "JUDGE-AMBIGUOUS";
+    return !suspect && verdict !== "ERROR" && verdict !== "NOT-MEASURED" && verdict !== "JUDGE-AMBIGUOUS" && verdict !== "UNGRADED";
 }
 function classify(red, green) {
     if (!conclusive(red.verdict, red.suspect) || !conclusive(green.verdict, green.suspect))

package/dist/regate.js

@@ -290,12 +290,13 @@ export async function regateRun(opts) {
             }
             // The gate blocked this rep before, so no judgement of it exists anywhere.
             const transcript = readFileSync(tPath, "utf8");
-            outcomes.push({ ...(await judgeOneRep({
-                    runDir: opts.runDir, spec: opts.spec, scenario, transcript,
-                    adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir,
-                    mode, rep, now, rejudge: true,
-                })), objective });
-            judgeCalls++;
+            const judged = await judgeOneRep({
+                runDir: opts.runDir, spec: opts.spec, scenario, transcript,
+                adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir,
+                mode, rep, now, rejudge: true,
+            });
+            outcomes.push({ ...judged, objective });
+            judgeCalls += judged.metrics?.judge_calls ?? 0;
             judgedHere++;
         }
         const threshold = effectiveThreshold(rec, scenario);

package/dist/regrade.d.ts

@@ -34,6 +34,9 @@ export interface RegradeOptions {
     mode?: string;
     /** Recorded repetition count; when supplied, missing/extra rep artifacts fail before judge calls. */
     expectedReps?: number;
+    /** Select only repetitions with retained ERROR criterion votes. */
+    onlyUnparsed?: boolean;
+    prior?: ScenarioResult;
     now?: () => string;
 }
 /** Judge one saved transcript: writes the judge-raw artifact, emits a `judge-verdict` journal event (plus `misfire-flag` when the verdict is suspect), and returns the outcome. */
@@ -50,6 +53,8 @@ export declare function judgeOneRep(opts: {
     now: () => string;
     /** True when this call revisits a saved subject transcript. */
     rejudge?: boolean;
+    /** Retained objective evidence must precede missing-vote normalization. */
+    objective?: RepOutcome["objective"];
 }): Promise<RepOutcome>;
 /**
  * Re-judge a scenario's saved transcript(s) for the run's mode with `judge` — no
@@ -72,6 +77,8 @@ export interface RegradeRunOptions {
      * cleanly, so spending judge calls on them buys nothing.
      */
     onlySuspect?: boolean;
+    /** Rejudge only retained repetitions containing ERROR criterion votes, with the recorded judge. */
+    onlyUnparsed?: boolean;
 }
 /**
  * Re-judge every scenario in a run dir that has a transcript for the run's own

package/dist/regrade.js

@@ -2,7 +2,7 @@ import { readFileSync, writeFileSync, existsSync } from "node:fs";
 import { join } from "node:path";
 import { buildJudgePrompt, judgeInWorkspace } from "./grade.js";
 import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResults, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, carryRepObjectives, completeCriterionVotes, } from "./results.js";
-import { outcomesToResult } from "./reps.js";
+import { outcomesToResult, normalizeRepOutcome } from "./reps.js";
 import { appendJournal } from "./journal.js";
 import { rubricDigest, personaDigest, RUBRIC_PREFIX, PERSONA_KEY } from "./sources.js";
 import { providerFailureFromTranscript } from "./provider-failure.js";
@@ -56,20 +56,21 @@ export async function judgeOneRep(opts) {
         };
     }
     const prompt = buildJudgePrompt({ skill: spec.skill, persona: spec.judge_persona, scenario, transcript });
-    const g = await judgeInWorkspace(adapter, judge, prompt, specDir);
+    const g = await judgeInWorkspace(adapter, judge, prompt, specDir, scenario.checklist.length);
     writeFileSync(judgeRawPath(runDir, scenario.id, mode, rep), g.raw, "utf8");
-    appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict: g.verdict, reason: g.reason, suspect: g.suspect, ...repField });
-    if (g.suspect)
-        appendJournal(runDir, { event: "misfire-flag", ts: now(), id: scenario.id, reason: g.reason, ...repField });
-    return {
-        verdict: g.verdict, reason: g.reason, suspect: g.suspect,
-        judgment: { ordinal: 1, judge: { ...judge }, verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, scenario.checklist.length) },
+    const outcome = normalizeRepOutcome({
+        verdict: g.verdict, reason: g.reason, suspect: g.suspect, objective: opts.objective,
+        judgment: { ordinal: 1, judge: { ...judge }, verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, scenario.checklist.length), ...(g.judgeRetries ? { judgeRetries: g.judgeRetries } : {}) },
         metrics: {
             wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)),
-            judge_calls: 1,
-            judge_rejudge_calls: opts.rejudge ? 1 : 0,
+            judge_calls: 1 + (g.judgeRetries ?? 0),
+            judge_rejudge_calls: opts.rejudge ? 1 + (g.judgeRetries ?? 0) : 0,
         },
-    };
+    });
+    appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict: outcome.verdict, reason: outcome.reason, suspect: outcome.suspect, ...repField });
+    if (outcome.suspect)
+        appendJournal(runDir, { event: "misfire-flag", ts: now(), id: scenario.id, reason: outcome.reason, ...repField });
+    return outcome;
 }
 /**
  * Re-judge a scenario's saved transcript(s) for the run's mode with `judge` — no
@@ -81,26 +82,48 @@ export async function judgeOneRep(opts) {
 export async function regradeScenario(opts) {
     const now = opts.now ?? (() => new Date().toISOString());
     const mode = opts.mode ?? "green";
-    const files = findTranscriptFiles(opts.runDir, opts.scenario.id, mode);
+    let files = findTranscriptFiles(opts.runDir, opts.scenario.id, mode);
+    if (opts.onlyUnparsed)
+        files = files.filter(file => hasUnparsedVotes(opts.prior?.rep_judgments?.find(panel => panel.repetition === (repIndexOf(file) ?? 0))));
     if (files.length === 0)
         throw new Error(`no ${mode} transcripts for ${opts.scenario.id} in ${opts.runDir}`);
     const expected = opts.expectedReps ?? files.length;
-    const expectedIndices = expected === 1 ? [null] : Array.from({ length: expected }, (_, index) => index);
+    const expectedIndices = opts.onlyUnparsed
+        ? opts.prior.rep_judgments.filter(hasUnparsedVotes).map(panel => expected === 1 ? null : panel.repetition).sort((a, b) => (a ?? -1) - (b ?? -1))
+        : expected === 1 ? [null] : Array.from({ length: expected }, (_, index) => index);
     const actualIndices = files.map((file) => repIndexOf(file)).sort((a, b) => (a ?? -1) - (b ?? -1));
-    if (files.length !== expected || JSON.stringify(actualIndices) !== JSON.stringify(expectedIndices)) {
+    if (files.length !== expectedIndices.length || JSON.stringify(actualIndices) !== JSON.stringify(expectedIndices)) {
         throw new Error(`${opts.scenario.id}: transcript artifacts are incomplete for ${expected} recorded rep(s) — re-run instead of grading a smaller repetition set`);
     }
     const repCount = expected;
     const outcomes = [];
-    for (const file of files) {
+    for (let repetition = 0; repetition < expected; repetition++) {
+        const panel = opts.prior?.rep_judgments?.find(panel => panel.repetition === repetition);
+        if (opts.onlyUnparsed && !hasUnparsedVotes(panel)) {
+            if (!panel)
+                throw new Error(`missing retained judgments for ${opts.scenario.id}#${repetition}`);
+            outcomes.push({ verdict: panel.recorded_verdict, reason: panel.judgments[0]?.reason ?? "", suspect: panel.judgments.length > 0 && !panel.judgments.some(judgment => !judgment.suspect && (judgment.verdict === "PASS" || judgment.verdict === "FAIL")), judgment: panel.judgments[0], objective: panel.objective });
+            continue;
+        }
+        const file = files.find(file => (repIndexOf(file) ?? 0) === repetition);
         const rep = repIndexOf(file) ?? undefined;
         const transcript = readFileSync(join(opts.runDir, file), "utf8");
         outcomes.push(await judgeOneRep({
             runDir: opts.runDir, spec: opts.spec, scenario: opts.scenario, transcript,
             adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir, mode, rep, now, rejudge: true,
+            objective: panel?.objective ?? (expected === 1 ? opts.prior?.objective : undefined),
         }));
     }
-    return outcomesToResult(opts.scenario.id, outcomes, repCount, opts.threshold);
+    const result = outcomesToResult(opts.scenario.id, outcomes, repCount, opts.threshold);
+    if (opts.onlyUnparsed)
+        result.rep_judgments = result.rep_judgments?.map(panel => {
+            const prior = opts.prior.rep_judgments.find(prior => prior.repetition === panel.repetition);
+            return hasUnparsedVotes(prior) ? panel : prior;
+        });
+    return result;
+}
+function hasUnparsedVotes(panel) {
+    return panel?.judgments.some(judgment => judgment.criteria?.some(vote => vote.verdict === "ERROR")) ?? false;
 }
 /**
  * Re-judge every scenario in a run dir that has a transcript for the run's own
@@ -114,17 +137,21 @@ export async function regradeScenario(opts) {
  * ResultsFile. Shared by `cmdGrade` and the pi-extension's `judge` command.
  */
 export async function regradeRun(opts) {
-    const { runDir, spec, adapter, judge, specDir } = opts;
+    const { runDir, spec, adapter, specDir } = opts;
     const now = opts.now ?? (() => new Date().toISOString());
     const prev = existsSync(join(runDir, "results.yaml")) ? readResults(runDir) : null;
+    if (opts.onlyUnparsed && !prev)
+        throw new Error(`--unparsed-only needs a prior results.yaml in ${runDir}`);
+    if (opts.onlyUnparsed && opts.onlySuspect)
+        throw new Error("--unparsed-only cannot be combined with --suspect-only");
+    const judge = opts.onlyUnparsed ? prev.judge : opts.judge;
     // Carried across a re-judge, per field, because the right answer differs:
     //  - `override`/`note`  — the author's, never the judge's to discard.
     //  - `objective`        — a re-judge does NOT re-evaluate trace gates (that is
     //    `regate`), so the recorded evidence still describes this run. Dropping it
     //    silently downgraded a gated scenario to "no assertions declared".
-    // `adjudication` is deliberately NOT carried: it describes the judgments this
-    // re-grade just replaced, and a stale panel beside a fresh verdict is worse
-    // than none.
+    // `adjudication` is dropped when its repetition is rejudged. A selective
+    // repair of a different repetition carries that untouched evidence instead.
     // The whole prior result per id — `rebuildScenarioResult` decides, field by
     // field, what survives. Passing a hand-picked subset here is how fields got
     // dropped before.
@@ -148,6 +175,8 @@ export async function regradeRun(opts) {
             .filter((s) => s.suspect || s.judge_verdict === "JUDGE-AMBIGUOUS")
             .map((s) => s.id);
     }
+    if (opts.onlyUnparsed)
+        targets = recorded.filter(s => s.rep_judgments?.some(hasUnparsedVotes)).map(s => s.id);
     // Schema-3 delivery is a prerequisite for behavioral judging. A later `grade`
     // cannot turn an undelivered or unauthenticated transcript into product evidence.
     if (prev?.schema === 3) {
@@ -161,6 +190,9 @@ export async function regradeRun(opts) {
     const completeTranscripts = (record) => {
         const expected = record.reps ?? 1;
         const files = findTranscriptFiles(runDir, record.id, mode);
+        if (opts.onlyUnparsed) {
+            return record.rep_judgments?.length === expected && record.rep_judgments.every(panel => !hasUnparsedVotes(panel) || files.some(file => (repIndexOf(file) ?? 0) === panel.repetition)) || false;
+        }
         const expectedIndices = expected === 1 ? [null] : Array.from({ length: expected }, (_, index) => index);
         const actualIndices = files.map((file) => repIndexOf(file)).sort((a, b) => (a ?? -1) - (b ?? -1));
         return files.length === expected && JSON.stringify(actualIndices) === JSON.stringify(expectedIndices);
@@ -173,6 +205,14 @@ export async function regradeRun(opts) {
     if (missing.length > 0) {
         throw new Error(`cannot re-grade ${missing.join(", ")} in ${runDir} (transcript missing or scenario no longer in the spec) — re-run instead of grading`);
     }
+    if (opts.onlyUnparsed) {
+        const drifted = targets.filter(id => {
+            const count = recordedById.get(id).criterion_count;
+            return count !== undefined && count !== specById.get(id).checklist.length;
+        });
+        if (drifted.length)
+            throw new Error(`criterion count changed for ${drifted.join(", ")} — use full grade instead of --unparsed-only`);
+    }
     const targetSet = new Set(targets);
     const scenarioResults = [];
     for (const rec of recorded) {
@@ -186,17 +226,17 @@ export async function regradeRun(opts) {
         const threshold = effectiveThreshold(prevScenario, scenario);
         const rr = await regradeScenario({
             runDir, spec, scenario, adapter, judge, specDir, threshold, mode, now,
-            expectedReps: prevScenario?.reps ?? 1,
+            expectedReps: prevScenario?.reps ?? 1, onlyUnparsed: opts.onlyUnparsed, prior: prevScenario,
         });
         const carry = overrides.get(id);
         if (prev?.schema === 3)
             rr.criterion_count = scenario.checklist.length;
         rr.metrics = mergeScenarioMetrics(carry?.metrics, rr.metrics);
         rr.rep_judgments = carryRepObjectives(rr.rep_judgments, carry?.rep_judgments);
-        // `grade` re-judges the saved transcript. It does not re-evaluate trace gates
-        // (that is `regate`), so `objective` still describes this run; and it replaced
-        // the judgments a prior adjudication described, so that panel must go.
-        scenarioResults.push(rebuildScenarioResult(rr, carry, { objective: "carry", adjudication: "drop" }));
+        // Gates still describe the original subject execution. An adjudication of
+        // an untouched repetition also survives, bounded by rebuild's conflict guards.
+        const carryAdjudication = opts.onlyUnparsed && carry?.adjudication && !hasUnparsedVotes(carry.rep_judgments?.find(panel => panel.repetition === (carry.adjudication.repetition ?? 0)));
+        scenarioResults.push(rebuildScenarioResult(rr, carry, { objective: "carry", adjudication: carryAdjudication ? "carry" : "drop" }));
     }
     const ctx = scoreContextFor({ mode, partial: prev?.partial }, spec);
     const results = writeResults(runDir, {
@@ -229,7 +269,7 @@ export async function regradeRun(opts) {
         // say so. Doctrine narrowed 0.4.0, from "recorded hashes stay" to "recorded
         // *stimulus* hashes stay" — see refreshRubricHashes.
         partial: prev?.partial,
-        source_hashes: refreshRubricHashes(prev?.source_hashes, spec, targets),
+        source_hashes: opts.onlyUnparsed ? prev?.source_hashes : refreshRubricHashes(prev?.source_hashes, spec, targets),
         source_hash_roots: prev?.source_hash_roots,
         scenarios: scenarioResults,
     }, ctx);

package/dist/report.d.ts

@@ -26,6 +26,7 @@ export interface RunColumn {
         passes?: number;
         clean?: number;
         flakiness?: number;
+        ungraded_reps?: number;
         metrics?: import("./results.js").ScenarioMetrics;
         override: string | null;
         note: string;
@@ -109,6 +110,7 @@ export declare function publicView(data: ReportData): {
             passes?: number;
             clean?: number;
             flakiness?: number;
+            ungraded_reps?: number;
             metrics?: import("./results.js").ScenarioMetrics;
             override: string | null;
             note: string;

package/dist/report.js

@@ -1,7 +1,7 @@
 import { existsSync, readdirSync, statSync } from "node:fs";
 import { join } from "node:path";
 import { loadSpec } from "./spec.js";
-import { readResults } from "./results.js";
+import { readResults, normalizeScenarioResult, finalizeResults, scoreContextFor } from "./results.js";
 import { collectLift, liftHeadline } from "./lift.js";
 import { boundaryCells, collectStability, stabilityNote } from "./stability.js";
 import { aggregateMetrics } from "./metrics.js";
@@ -40,9 +40,13 @@ export function collectReport(skillDir) {
             if (!runDir)
                 continue;
             const r = readResults(runDir);
+            const normalized = r.scenarios.map(normalizeScenarioResult);
+            const grade = normalized.some((s, i) => s !== r.scenarios[i])
+                ? finalizeResults({ ...r, scenarios: normalized }, scoreContextFor(r, spec)).effective_grade
+                : r.effective_grade;
             const tagName = tagDir.split("/").pop();
             const cells = {};
-            for (const s of r.scenarios) {
+            for (const s of normalized) {
                 const boundary = boundaryByCell.get(`${tagName}\u0000${r.mode}\u0000${s.id}`);
                 cells[s.id] = {
                     ...(boundary
@@ -72,6 +76,7 @@ export function collectReport(skillDir) {
                     passes: s.passes,
                     clean: s.clean,
                     flakiness: s.flakiness,
+                    ...(s.ungraded_reps ? { ungraded_reps: s.ungraded_reps } : {}),
                     metrics: s.metrics,
                     override: s.override,
                     note: s.note,
@@ -96,7 +101,7 @@ export function collectReport(skillDir) {
                 timestamp: r.timestamp,
                 mode: r.mode,
                 partial: r.partial === true,
-                grade: r.effective_grade,
+                grade,
                 judge: r.judge,
                 metrics: aggregateMetrics(r.scenarios),
                 cells,

package/dist/reps.d.ts

@@ -16,6 +16,8 @@ export interface RepOutcome {
         subject?: TraceMetrics;
     };
 }
+/** Missing criterion evidence is a non-pass, not a judge misfire or a judged FAIL. */
+export declare function normalizeRepOutcome(outcome: RepOutcome): RepOutcome;
 /**
  * Collapse per-rep objective results.
  *

package/dist/reps.js

@@ -1,3 +1,17 @@
+/** Missing criterion evidence is a non-pass, not a judge misfire or a judged FAIL. */
+export function normalizeRepOutcome(outcome) {
+    if (outcome.verdict === "ERROR")
+        return outcome;
+    if (outcome.objective && outcome.objective.status !== "PASS") {
+        return { ...outcome, verdict: outcome.objective.status, suspect: false };
+    }
+    if (outcome.verdict === "NOT-MEASURED")
+        return outcome;
+    if (outcome.verdict === "UNGRADED" || outcome.judgment?.criteria?.some(vote => vote.verdict === "ERROR")) {
+        return { ...outcome, verdict: "UNGRADED", reason: "UNGRADED: incomplete criterion votes after judge retry", suspect: false };
+    }
+    return outcome;
+}
 /**
  * Collapse per-rep objective results.
  *
@@ -31,7 +45,9 @@ export function aggregateObjective(outcomes) {
  * (default caller threshold 0.5, ties pass). Flakiness = 1 - |2·pass_rate - 1|.
  */
 export function aggregateReps(outcomes, threshold) {
+    outcomes = outcomes.map(normalizeRepOutcome);
     const reps = outcomes.length;
+    const ungraded = outcomes.filter(o => o.verdict === "UNGRADED").length;
     const clean = outcomes.filter((o) => !o.suspect);
     const passes = clean.filter((o) => o.verdict === "PASS").length;
     const errored = outcomes.filter((o) => o.verdict === "ERROR").length;
@@ -50,9 +66,9 @@ export function aggregateReps(outcomes, threshold) {
         return { verdict: "FAIL", reason: `${reps - clean.length}/${reps} reps misfired — re-judge`, passes, reps, clean: clean.length, flakiness: 0, suspect: true };
     }
     const passRate = passes / clean.length;
-    const verdict = passRate >= threshold ? "PASS" : "FAIL";
+    const verdict = passRate >= threshold && (ungraded === 0 || passes > 0) ? "PASS" : ungraded > 0 ? "UNGRADED" : "FAIL";
     const flakiness = 1 - Math.abs(2 * passRate - 1);
-    const reason = reps === 1 ? outcomes[0].reason : `${passes}/${clean.length} reps passed (flaky ${flakiness.toFixed(2)})`;
+    const reason = reps === 1 ? outcomes[0].reason : `${passes}/${clean.length} reps passed (flaky ${flakiness.toFixed(2)})${ungraded ? `; ${ungraded} UNGRADED` : ""}`;
     return { verdict, reason, passes, reps, clean: clean.length, flakiness, suspect: false };
 }
 /**
@@ -63,6 +79,9 @@ export function aggregateReps(outcomes, threshold) {
  * caller to merge.
  */
 export function outcomesToResult(id, outcomes, repCount, threshold) {
+    outcomes = outcomes.map(normalizeRepOutcome);
+    const ungraded = outcomes.filter(outcome => outcome.verdict === "UNGRADED").length;
+    const ungradedField = ungraded ? { ungraded_reps: ungraded } : {};
     // Spread rather than always-set: a scenario with no trace assertions must
     // produce a result byte-identical to one from before this field existed.
     const objective = aggregateObjective(outcomes);
@@ -88,13 +107,13 @@ export function outcomesToResult(id, outcomes, repCount, threshold) {
     const repJudgmentField = outcomes.some(outcome => outcome.judgment) ? { rep_judgments: repJudgments } : {};
     if (repCount === 1) {
         const o = outcomes[0];
-        return { id, judge_verdict: o.verdict, judge_reason: o.reason, suspect: o.suspect, ...metricsField, ...usageField, override: null, note: "", ...objectiveField, ...repJudgmentField };
+        return { id, judge_verdict: o.verdict, judge_reason: o.reason, suspect: o.suspect, ...ungradedField, ...metricsField, ...usageField, override: null, note: "", ...objectiveField, ...repJudgmentField };
     }
     const agg = aggregateReps(outcomes, threshold);
     return {
         id, judge_verdict: agg.verdict, judge_reason: agg.reason, suspect: agg.suspect,
         reps: agg.reps, passes: agg.passes, clean: agg.clean, flakiness: agg.flakiness,
-        pass_threshold: threshold, ...metricsField, ...usageField, override: null, note: "", ...objectiveField, ...repJudgmentField,
+        pass_threshold: threshold, ...ungradedField, ...metricsField, ...usageField, override: null, note: "", ...objectiveField, ...repJudgmentField,
     };
 }
 function aggregateMetrics(outcomes) {

package/dist/rescore.js

@@ -1,6 +1,6 @@
 import { existsSync } from "node:fs";
 import { join } from "node:path";
-import { readResults, writeResults, scoreContextFor, rebuildScenarioResult, effectiveThreshold } from "./results.js";
+import { readResults, writeResults, scoreContextFor, rebuildScenarioResult, normalizeScenarioResult, effectiveThreshold } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { policyDigest, POLICY_PREFIX } from "./sources.js";
 /**
@@ -47,7 +47,8 @@ export function rescoreRun(opts) {
         throw new Error(`no results.yaml in ${opts.runDir}`);
     const specById = new Map(opts.spec.scenarios.map((s) => [s.id, s]));
     const changes = [];
-    const scenarios = prev.scenarios.map((s) => {
+    const scenarios = prev.scenarios.map((recorded) => {
+        const s = normalizeScenarioResult(recorded);
         const scenario = specById.get(s.id);
         // no rate to re-apply, or an untrustworthy verdict → carry verbatim
         if (!scenario || s.reps === undefined || s.clean === undefined || s.passes === undefined)
@@ -61,7 +62,7 @@ export function rescoreRun(opts) {
         if (toThreshold === fromThreshold)
             return s;
         const rate = s.passes / s.clean;
-        const verdict = rate >= toThreshold ? "PASS" : "FAIL";
+        const verdict = rate >= toThreshold && (!s.ungraded_reps || s.passes > 0) ? "PASS" : s.ungraded_reps ? "UNGRADED" : "FAIL";
         if (verdict !== s.judge_verdict) {
             changes.push({ id: s.id, from: s.judge_verdict, to: verdict, passes: s.passes, clean: s.clean, fromThreshold, toThreshold });
         }
@@ -76,6 +77,14 @@ export function rescoreRun(opts) {
         // already measured. It re-measures nothing, so it may discard nothing.
         return rebuildScenarioResult({ ...s, judge_verdict: verdict, pass_threshold: toThreshold }, s, { objective: "carry", adjudication: "carry" });
     });
+    for (const s of scenarios) {
+        const recorded = prev.scenarios.find(prior => prior.id === s.id);
+        if (s.judge_verdict !== recorded.judge_verdict && !changes.some(change => change.id === s.id)) {
+            changes.push({ id: s.id, from: recorded.judge_verdict, to: s.judge_verdict,
+                passes: s.passes ?? (s.judge_verdict === "PASS" ? 1 : 0), clean: s.clean ?? 1,
+                fromThreshold: recorded.pass_threshold ?? 0.5, toThreshold: s.pass_threshold ?? 0.5 });
+        }
+    }
     const ctx = scoreContextFor(prev, opts.spec);
     const results = writeResults(opts.runDir, {
         schema: prev.schema, subject_invocations: prev.subject_invocations,

package/dist/results.d.ts

@@ -61,6 +61,8 @@ export interface ScenarioResult {
     passes?: number;
     clean?: number;
     flakiness?: number;
+    /** Reps with incomplete criterion votes; included as non-passes, never misfires. */
+    ungraded_reps?: number;
     pass_threshold?: number;
     /** Cost/latency counters. Separate from verdicts: cheaper failure is still failure. */
     metrics?: ScenarioMetrics;
@@ -99,6 +101,8 @@ export interface Judgment {
     suspect: boolean;
     /** Every numbered rubric-item vote parsed from this exact judge response. Required in schema v3. */
     criteria?: CriterionVote[];
+    /** One retry with identical judge input after unparseable criterion votes. */
+    judgeRetries?: 1;
 }
 export interface AdjudicationResult {
     /** The saved transcript repetition judged by every panel member (first rep by policy). */
@@ -347,6 +351,7 @@ export declare function resultsPath(runDir: string): string;
  * deterministic assertion is a deliberate, recorded human act — and the failure
  * this guards against was never a human deciding, it was nobody deciding.
  */
+export declare function normalizeScenarioResult(s: ScenarioResult): ScenarioResult;
 export declare function effectiveVerdicts(scenarios: ScenarioResult[]): ScenarioVerdict[];
 /**
  * The ONLY place effective_grade is computed. Every writer goes through here,

package/dist/results.js

@@ -5,6 +5,7 @@ import { modelSlug } from "./adapters/types.js";
 import { score } from "./score.js";
 import { HARNESS_VERSION } from "./version.js";
 import { collapseVotePanel } from "./vote-panel.js";
+import { outcomesToResult } from "./reps.js";
 /** Merge bookkeeping from a later judge/gate pass without duplicating subject usage. */
 export function mergeScenarioMetrics(prior, fresh) {
     if (!prior)
@@ -149,8 +150,28 @@ export function resultsPath(runDir) {
  * deterministic assertion is a deliberate, recorded human act — and the failure
  * this guards against was never a human deciding, it was nobody deciding.
  */
+export function normalizeScenarioResult(s) {
+    const panels = s.rep_judgments;
+    if (!panels?.some(panel => panel.recorded_verdict === "UNGRADED" || panel.judgments.some(judgment => judgment.criteria?.some(vote => vote.verdict === "ERROR"))))
+        return s;
+    const reps = s.reps ?? 1;
+    if (panels.length !== reps)
+        return { ...s, judge_verdict: "UNGRADED", suspect: false, ungraded_reps: panels.filter(panel => panel.judgments.some(judgment => judgment.criteria?.some(vote => vote.verdict === "ERROR"))).length };
+    const outcomes = [...panels].sort((a, b) => a.repetition - b.repetition).map(panel => ({
+        verdict: panel.recorded_verdict,
+        reason: panel.judgments[0]?.reason ?? s.judge_reason,
+        suspect: panel.recorded_verdict === "UNGRADED" ? false : panel.judgments.length > 0 && !panel.judgments.some(judgment => !judgment.suspect && (judgment.verdict === "PASS" || judgment.verdict === "FAIL")),
+        objective: panel.objective,
+        judgment: panel.judgments.find(judgment => judgment.criteria?.some(vote => vote.verdict === "ERROR")) ?? panel.judgments[0],
+    }));
+    const fresh = outcomesToResult(s.id, outcomes, reps, s.pass_threshold ?? 0.5);
+    fresh.metrics = s.metrics;
+    fresh.usage = s.usage;
+    fresh.rep_judgments = panels.map(panel => ({ ...panel, recorded_verdict: fresh.rep_judgments?.find(candidate => candidate.repetition === panel.repetition)?.recorded_verdict ?? panel.recorded_verdict }));
+    return rebuildScenarioResult(fresh, s, { objective: "carry", adjudication: "carry" });
+}
 export function effectiveVerdicts(scenarios) {
-    return scenarios.map((s) => ({
+    return scenarios.map(normalizeScenarioResult).map((s) => ({
         id: s.id,
         verdict: s.override ?? objectiveVerdict(s) ?? s.judge_verdict,
         suspect: s.suspect && s.override == null, // an override resolves the misfire
@@ -182,9 +203,10 @@ function objectiveVerdict(s) {
  * deployment and is scored (see SCORED_MODES directly above).
  */
 export function finalizeResults(draft, ctx) {
+    const scenarios = draft.scenarios.map(normalizeScenarioResult);
     let effective_grade;
     if (ctx) {
-        const s = score(effectiveVerdicts(draft.scenarios), { shipBar: ctx.shipBar, critical: ctx.critical });
+        const s = score(effectiveVerdicts(scenarios), { shipBar: ctx.shipBar, critical: ctx.critical });
         effective_grade = { passed: s.passed, total: s.total, pct: s.pct, letter: s.letter, ship: s.ship, note: s.note };
     }
     else {
@@ -214,7 +236,7 @@ export function finalizeResults(draft, ctx) {
         ...(draft.source_hashes ? { source_hashes: draft.source_hashes } : {}),
         ...(draft.source_hash_roots ? { source_hash_roots: draft.source_hash_roots } : {}),
         effective_grade,
-        scenarios: draft.scenarios,
+        scenarios,
         ...(draft.subject_invocations ? { subject_invocations: draft.subject_invocations } : {}),
     };
 }
@@ -304,7 +326,11 @@ export function recomputeRecordedPanels(results) {
         for (const panel of scenario.rep_judgments ?? []) {
             const clean = panel.judgments.filter(j => !j.suspect && (j.verdict === "PASS" || j.verdict === "FAIL"));
             let verdict = panel.recorded_verdict;
-            if (clean.length === 1)
+            if (panel.objective && panel.objective.status !== "PASS")
+                verdict = panel.objective.status;
+            else if (panel.recorded_verdict === "UNGRADED" && panel.judgments.some(judgment => judgment.criteria?.some(vote => vote.verdict === "ERROR")))
+                verdict = "UNGRADED";
+            else if (clean.length === 1)
                 verdict = clean[0].verdict;
             else if (clean.length >= 2)
                 verdict = collapseVotePanel(panel.judgments).verdict ?? "JUDGE-AMBIGUOUS";
@@ -417,7 +443,7 @@ export function validateResults(raw) {
             if (!Number.isInteger(scenario.adjudication.repetition) || scenario.adjudication.repetition < 0 || scenario.adjudication.repetition >= reps)
                 throw new Error(`schema v3 adjudication repetition missing or out of range for ${scenario.id}`);
             const collapsed = collapseVotePanel(scenario.adjudication.judgments);
-            const boundedCriticalAggregate = (scenario.reps ?? 1) > 1 && scenario.pass_threshold === 1 && scenario.judge_verdict !== "PASS" && collapsed.verdict === "PASS" && scenario.adjudication.state === "unresolved" && scenario.adjudication.verdict === undefined;
+            const boundedCriticalAggregate = ((scenario.reps ?? 1) > 1 && scenario.pass_threshold === 1 || scenario.judge_verdict === "UNGRADED") && scenario.judge_verdict !== "PASS" && collapsed.verdict === "PASS" && scenario.adjudication.state === "unresolved" && scenario.adjudication.verdict === undefined;
             if (!boundedCriticalAggregate && (scenario.adjudication.state !== collapsed.state || scenario.adjudication.verdict !== collapsed.verdict))
                 throw new Error(`recorded adjudication state/verdict diverges from recomputed votes for ${scenario.id}`);
             if (scenario.adjudication.state !== "unresolved")
@@ -426,13 +452,14 @@ export function validateResults(raw) {
         const panels = scenario.rep_judgments;
         const errorCount = panels.filter(panel => panel.recorded_verdict === "ERROR").length;
         const notMeasuredCount = panels.filter(panel => panel.recorded_verdict === "NOT-MEASURED").length;
-        const cleanPanels = panels.filter(panel => !(panel.judgments[0]?.suspect ?? false));
+        const cleanPanels = panels.filter(panel => panel.objective && panel.objective.status !== "PASS" || panel.recorded_verdict === "UNGRADED" || !(panel.judgments[0]?.suspect ?? false));
+        const ungradedCount = panels.filter(panel => panel.recorded_verdict === "UNGRADED").length;
         const passes = cleanPanels.filter(panel => panel.recorded_verdict === "PASS").length;
         let aggregateVerdict;
         let aggregateSuspect = false;
         if (panels.length === 1) {
             aggregateVerdict = panels[0].recorded_verdict;
-            aggregateSuspect = panels[0].judgments[0]?.suspect ?? false;
+            aggregateSuspect = panels[0].objective && panels[0].objective.status !== "PASS" || panels[0].recorded_verdict === "UNGRADED" ? false : panels[0].judgments[0]?.suspect ?? false;
         }
         else if (errorCount > 0)
             aggregateVerdict = "ERROR";
@@ -443,7 +470,7 @@ export function validateResults(raw) {
             aggregateSuspect = true;
         }
         else
-            aggregateVerdict = passes / cleanPanels.length >= (scenario.pass_threshold ?? 0.5) ? "PASS" : "FAIL";
+            aggregateVerdict = passes / cleanPanels.length >= (scenario.pass_threshold ?? 0.5) && (ungradedCount === 0 || passes > 0) ? "PASS" : ungradedCount > 0 ? "UNGRADED" : "FAIL";
         const expectedVerdict = adjudicatedVerdict ?? aggregateVerdict;
         const expectedSuspect = scenario.adjudication ? scenario.adjudication.state === "unresolved" : aggregateSuspect;
         if (scenario.judge_verdict !== expectedVerdict || scenario.suspect !== expectedSuspect)
@@ -600,7 +627,7 @@ export function diffPath(runDir, scenarioId, mode, rep) {
 export function rebuildScenarioResult(fresh, prior, policy) {
     // Exhaustive destructure. Do not replace with a spread: the spread is what
     // allowed a new field to pass through unconsidered in the first place.
-    const { id, criterion_count: freshCriterionCount, judge_verdict, judge_reason, suspect, override: _freshOverride, note: _freshNote, reps, passes, clean, flakiness, pass_threshold, metrics: freshMetrics, usage: freshUsage, objective: freshObjective, adjudication: freshAdjudication, rep_judgments: freshRepJudgments, ...rest } = fresh;
+    const { id, criterion_count: freshCriterionCount, judge_verdict, judge_reason, suspect, override: _freshOverride, note: _freshNote, reps, passes, clean, flakiness, pass_threshold, ungraded_reps, metrics: freshMetrics, usage: freshUsage, objective: freshObjective, adjudication: freshAdjudication, rep_judgments: freshRepJudgments, ...rest } = fresh;
     const _exhaustive = rest;
     void _exhaustive;
     void _freshOverride;
@@ -613,7 +640,7 @@ export function rebuildScenarioResult(fresh, prior, policy) {
     const objective = pick(policy.objective, freshObjective, prior?.objective);
     const pickedAdjudication = pick(policy.adjudication, freshAdjudication, prior?.adjudication);
     const conflictsWithFreshEvidence = Boolean(pickedAdjudication?.verdict && (objective?.status === "FAIL" || objective?.status === "ERROR" || objective?.status === "NOT-MEASURED" ||
-        (pass_threshold === 1 && (reps ?? 1) > 1 && judge_verdict !== "PASS")));
+        (pickedAdjudication.verdict === "PASS" && (judge_verdict === "UNGRADED" || pass_threshold === 1 && (reps ?? 1) > 1 && judge_verdict !== "PASS"))));
     // A settled cell-level panel cannot overrule newly evaluated objective gates,
     // nor can one transcript erase a failed repetition under an all-clean policy.
     const adjudication = conflictsWithFreshEvidence && pickedAdjudication
@@ -652,6 +679,7 @@ export function rebuildScenarioResult(fresh, prior, policy) {
         ...(clean === undefined ? {} : { clean }),
         ...(flakiness === undefined ? {} : { flakiness }),
         ...(pass_threshold === undefined ? {} : { pass_threshold }),
+        ...(ungraded_reps === undefined ? {} : { ungraded_reps }),
         ...((freshMetrics ?? prior?.metrics) ? { metrics: freshMetrics ?? prior.metrics } : {}),
         ...((freshUsage ?? prior?.usage) ? { usage: freshUsage ?? prior.usage } : {}),
         // The author owns the verdict; a re-measurement never discards their call.

package/dist/run.js

@@ -521,7 +521,7 @@ async function runRep(scenario, rep, repCount, ctx) {
             reason = o.reason;
             suspect = o.suspect;
             judgment = o.judgment; // judgeOneRep already journaled (verdict + misfire)
-            judgeCalls = 1;
+            judgeCalls = o.metrics?.judge_calls ?? 0;
         }
         log(`  → ${scenario.id}${repCount > 1 ? `#${rep}` : ""} ${verdict}${reason ? `: ${reason}` : ""}${suspect ? "  ⚠ suspect" : ""}`);
         const subject = mergeTraces(traces)?.metrics;
@@ -567,7 +567,8 @@ export function formatScorecard(summary, lift, stability) {
         const susp = s.suspect ? " ⚠suspect" : "";
         const misfired = s.clean !== undefined && s.reps !== undefined && s.clean < s.reps ? ` · ${s.reps - s.clean} misfired` : "";
         const repInfo = s.reps ? `  [${s.passes}/${s.clean}${misfired}${s.flakiness ? ` flaky ${s.flakiness.toFixed(2)}` : ""}]` : "";
-        lines.push(`  ${mark} ${s.id}${ov}${susp}  ${s.judge_reason}${repInfo}`);
+        const ungradedInfo = s.ungraded_reps ? ` · ${s.ungraded_reps} ungraded rep${s.ungraded_reps === 1 ? "" : "s"} (UNGRADED)` : "";
+        lines.push(`  ${mark} ${s.id}${ov}${susp}  ${s.judge_reason}${repInfo}${ungradedInfo}`);
     }
     const ship = g.ship ? "SHIP" : "NOT READY";
     const note = g.note ? ` (${g.note})` : "";

package/dist/score.d.ts

@@ -3,7 +3,7 @@ import type { ShipBar } from "./spec.js";
  * "JUDGE-AMBIGUOUS": the judge emitted conflicting verdicts for one transcript. It is
  * never a pass and never silently resolved — it marks the run for a rejudge.
  */
-export type Verdict = "PASS" | "FAIL" | "ERROR" | "NOT-MEASURED" | "JUDGE-AMBIGUOUS";
+export type Verdict = "PASS" | "FAIL" | "ERROR" | "NOT-MEASURED" | "JUDGE-AMBIGUOUS" | "UNGRADED";
 export interface ScenarioVerdict {
     id: string;
     verdict: Verdict;
@@ -24,6 +24,7 @@ export interface ScoreResult {
     suspectCount: number;
     errorCount: number;
     notMeasuredCount: number;
+    ungradedCount: number;
     note: string;
 }
 export declare function letterFor(pct: number): string;

package/dist/score.js

@@ -28,6 +28,7 @@ export function score(verdicts, input) {
     let suspectCount = 0;
     let errorCount = 0;
     let notMeasuredCount = 0;
+    let ungradedCount = 0;
     for (const v of verdicts) {
         if (v.suspect) {
             suspectCount++;
@@ -46,6 +47,8 @@ export function score(verdicts, input) {
             passed++;
             continue;
         }
+        if (v.verdict === "UNGRADED")
+            ungradedCount++;
         if (critical.includes(v.id))
             criticalFails++;
         if (/^B/i.test(v.id))
@@ -75,12 +78,15 @@ export function score(verdicts, input) {
     else if (notMeasuredCount > 0) {
         note = `${notMeasuredCount} not measured: skill delivery was not established`;
     }
+    else if (ungradedCount > 0) {
+        note = `${ungradedCount} UNGRADED: incomplete criterion votes (counted as non-passes)`;
+    }
     else if (criticalFails > 0) {
         note = `gated: ${criticalFails} critical fail${criticalFails === 1 ? "" : "s"}`;
     }
     else if (bSeriesFails > 0) {
         note = `gated: ${bSeriesFails} B-series fail${bSeriesFails === 1 ? "" : "s"}`;
     }
-    return { passed, total, pct, letter, ship, criticalFails, bSeriesFails, suspectCount, errorCount, notMeasuredCount, note };
+    return { passed, total, pct, letter, ship, criticalFails, bSeriesFails, suspectCount, errorCount, notMeasuredCount, ungradedCount, note };
 }
 //# sourceMappingURL=score.js.map

package/dist/stability.js

@@ -9,7 +9,7 @@ function conclusive(v) {
     // Same rule lift.ts applies, for the same reason: ERROR is a harness failure and an
     // unresolved misfire is a judge failure. Counting either as a side of a flip would
     // report infrastructure noise as behavioural instability.
-    return !v.suspect && v.verdict !== "ERROR" && v.verdict !== "NOT-MEASURED" && v.verdict !== "JUDGE-AMBIGUOUS";
+    return !v.suspect && v.verdict !== "ERROR" && v.verdict !== "NOT-MEASURED" && v.verdict !== "JUDGE-AMBIGUOUS" && v.verdict !== "UNGRADED";
 }
 function pointFor(r, s, verdict) {
     const reps = s.reps ?? 1;

package/dist/trends.js

@@ -1,7 +1,7 @@
 import { existsSync, readdirSync, statSync } from "node:fs";
 import { join } from "node:path";
 import { loadSpec } from "./spec.js";
-import { readResults, effectiveVerdicts, isScoredMode } from "./results.js";
+import { readResults, effectiveVerdicts, isScoredMode, normalizeScenarioResult, finalizeResults, scoreContextFor } from "./results.js";
 /** A directory that exists right now; false (never throws) if it vanished concurrently (e.g. ENOENT). */
 function isDir(p) {
     try {
@@ -112,12 +112,16 @@ export function collectTrends(skillDir, limit = 20) {
             // effectiveVerdicts is the single source of truth for the override-aware
             // verdict/suspect rule (suspect = s.suspect && s.override == null — an override
             // resolves the misfire); zip in flakiness from the matching ScenarioResult.
-            const verdicts = effectiveVerdicts(r.scenarios);
+            const normalized = r.scenarios.map(normalizeScenarioResult);
+            const grade = normalized.some((s, i) => s !== r.scenarios[i])
+                ? finalizeResults({ ...r, scenarios: normalized }, scoreContextFor(r, spec)).effective_grade
+                : r.effective_grade;
+            const verdicts = effectiveVerdicts(normalized);
             const cells = {};
-            r.scenarios.forEach((s, i) => {
+            normalized.forEach((s, i) => {
                 cells[s.id] = { verdict: verdicts[i].verdict, suspect: verdicts[i].suspect ?? false, flakiness: s.flakiness };
             });
-            runs.push({ timestamp: r.timestamp, label: r.label, grade: r.effective_grade, cells });
+            runs.push({ timestamp: r.timestamp, label: r.label, grade, cells });
         }
         models.push({ model: group.model, tag: group.tag, mode: group.mode, runs, truncated, skipped: group.skipped });
     }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.22.0",
+    "version": "0.22.4",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",

package/schemas/results-v2.schema.json

@@ -102,7 +102,8 @@
         "PASS",
         "FAIL",
         "ERROR",
-        "JUDGE-AMBIGUOUS"
+        "JUDGE-AMBIGUOUS",
+        "UNGRADED"
       ]
     },
     "grade": {
@@ -232,6 +233,7 @@
         "suspect"
       ],
       "properties": {
+        "judgeRetries": { "type": "integer", "const": 1 },
         "ordinal": {
           "type": "integer",
           "minimum": 1,
@@ -320,6 +322,10 @@
           "minimum": 0,
           "maximum": 1
         },
+        "ungraded_reps": {
+          "type": "integer",
+          "minimum": 1
+        },
         "pass_threshold": {
           "type": "number",
           "minimum": 0,

package/schemas/results-v3.schema.json

@@ -113,7 +113,8 @@
         "FAIL",
         "ERROR",
         "NOT-MEASURED",
-        "JUDGE-AMBIGUOUS"
+        "JUDGE-AMBIGUOUS",
+        "UNGRADED"
       ]
     },
     "grade": {
@@ -244,6 +245,7 @@
         "criteria"
       ],
       "properties": {
+        "judgeRetries": { "type": "integer", "const": 1 },
         "ordinal": {
           "type": "integer",
           "minimum": 1,
@@ -346,6 +348,10 @@
           "minimum": 0,
           "maximum": 1
         },
+        "ungraded_reps": {
+          "type": "integer",
+          "minimum": 1
+        },
         "pass_threshold": {
           "type": "number",
           "minimum": 0,