@skill-harness/core 0.22.3 → 0.22.4

package/dist/lift.js

@@ -29,7 +29,7 @@ function conclusive(verdict, suspect) {
     // misfire are judge failures. Treating any of them as a FAIL would make
     // "red FAIL -> green PASS" fire on infrastructure noise, and lift would
     // measure flakiness while looking like skill value.
-    return !suspect && verdict !== "ERROR" && verdict !== "NOT-MEASURED" && verdict !== "JUDGE-AMBIGUOUS";
+    return !suspect && verdict !== "ERROR" && verdict !== "NOT-MEASURED" && verdict !== "JUDGE-AMBIGUOUS" && verdict !== "UNGRADED";
 }
 function classify(red, green) {
     if (!conclusive(red.verdict, red.suspect) || !conclusive(green.verdict, green.suspect))

package/dist/regrade.d.ts

@@ -53,6 +53,8 @@ export declare function judgeOneRep(opts: {
     now: () => string;
     /** True when this call revisits a saved subject transcript. */
     rejudge?: boolean;
+    /** Retained objective evidence must precede missing-vote normalization. */
+    objective?: RepOutcome["objective"];
 }): Promise<RepOutcome>;
 /**
  * Re-judge a scenario's saved transcript(s) for the run's mode with `judge` — no

package/dist/regrade.js

@@ -2,7 +2,7 @@ import { readFileSync, writeFileSync, existsSync } from "node:fs";
 import { join } from "node:path";
 import { buildJudgePrompt, judgeInWorkspace } from "./grade.js";
 import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResults, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, carryRepObjectives, completeCriterionVotes, } from "./results.js";
-import { outcomesToResult } from "./reps.js";
+import { outcomesToResult, normalizeRepOutcome } from "./reps.js";
 import { appendJournal } from "./journal.js";
 import { rubricDigest, personaDigest, RUBRIC_PREFIX, PERSONA_KEY } from "./sources.js";
 import { providerFailureFromTranscript } from "./provider-failure.js";
@@ -58,18 +58,19 @@ export async function judgeOneRep(opts) {
     const prompt = buildJudgePrompt({ skill: spec.skill, persona: spec.judge_persona, scenario, transcript });
     const g = await judgeInWorkspace(adapter, judge, prompt, specDir, scenario.checklist.length);
     writeFileSync(judgeRawPath(runDir, scenario.id, mode, rep), g.raw, "utf8");
-    appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict: g.verdict, reason: g.reason, suspect: g.suspect, ...repField });
-    if (g.suspect)
-        appendJournal(runDir, { event: "misfire-flag", ts: now(), id: scenario.id, reason: g.reason, ...repField });
-    return {
-        verdict: g.verdict, reason: g.reason, suspect: g.suspect,
+    const outcome = normalizeRepOutcome({
+        verdict: g.verdict, reason: g.reason, suspect: g.suspect, objective: opts.objective,
         judgment: { ordinal: 1, judge: { ...judge }, verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, scenario.checklist.length), ...(g.judgeRetries ? { judgeRetries: g.judgeRetries } : {}) },
         metrics: {
             wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)),
             judge_calls: 1 + (g.judgeRetries ?? 0),
             judge_rejudge_calls: opts.rejudge ? 1 + (g.judgeRetries ?? 0) : 0,
         },
-    };
+    });
+    appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict: outcome.verdict, reason: outcome.reason, suspect: outcome.suspect, ...repField });
+    if (outcome.suspect)
+        appendJournal(runDir, { event: "misfire-flag", ts: now(), id: scenario.id, reason: outcome.reason, ...repField });
+    return outcome;
 }
 /**
  * Re-judge a scenario's saved transcript(s) for the run's mode with `judge` — no
@@ -110,6 +111,7 @@ export async function regradeScenario(opts) {
         outcomes.push(await judgeOneRep({
             runDir: opts.runDir, spec: opts.spec, scenario: opts.scenario, transcript,
             adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir, mode, rep, now, rejudge: true,
+            objective: panel?.objective ?? (expected === 1 ? opts.prior?.objective : undefined),
         }));
     }
     const result = outcomesToResult(opts.scenario.id, outcomes, repCount, opts.threshold);

package/dist/report.d.ts

@@ -26,6 +26,7 @@ export interface RunColumn {
         passes?: number;
         clean?: number;
         flakiness?: number;
+        ungraded_reps?: number;
         metrics?: import("./results.js").ScenarioMetrics;
         override: string | null;
         note: string;
@@ -109,6 +110,7 @@ export declare function publicView(data: ReportData): {
             passes?: number;
             clean?: number;
             flakiness?: number;
+            ungraded_reps?: number;
             metrics?: import("./results.js").ScenarioMetrics;
             override: string | null;
             note: string;

package/dist/report.js

@@ -1,7 +1,7 @@
 import { existsSync, readdirSync, statSync } from "node:fs";
 import { join } from "node:path";
 import { loadSpec } from "./spec.js";
-import { readResults } from "./results.js";
+import { readResults, normalizeScenarioResult, finalizeResults, scoreContextFor } from "./results.js";
 import { collectLift, liftHeadline } from "./lift.js";
 import { boundaryCells, collectStability, stabilityNote } from "./stability.js";
 import { aggregateMetrics } from "./metrics.js";
@@ -40,9 +40,13 @@ export function collectReport(skillDir) {
             if (!runDir)
                 continue;
             const r = readResults(runDir);
+            const normalized = r.scenarios.map(normalizeScenarioResult);
+            const grade = normalized.some((s, i) => s !== r.scenarios[i])
+                ? finalizeResults({ ...r, scenarios: normalized }, scoreContextFor(r, spec)).effective_grade
+                : r.effective_grade;
             const tagName = tagDir.split("/").pop();
             const cells = {};
-            for (const s of r.scenarios) {
+            for (const s of normalized) {
                 const boundary = boundaryByCell.get(`${tagName}\u0000${r.mode}\u0000${s.id}`);
                 cells[s.id] = {
                     ...(boundary
@@ -72,6 +76,7 @@ export function collectReport(skillDir) {
                     passes: s.passes,
                     clean: s.clean,
                     flakiness: s.flakiness,
+                    ...(s.ungraded_reps ? { ungraded_reps: s.ungraded_reps } : {}),
                     metrics: s.metrics,
                     override: s.override,
                     note: s.note,
@@ -96,7 +101,7 @@ export function collectReport(skillDir) {
                 timestamp: r.timestamp,
                 mode: r.mode,
                 partial: r.partial === true,
-                grade: r.effective_grade,
+                grade,
                 judge: r.judge,
                 metrics: aggregateMetrics(r.scenarios),
                 cells,

package/dist/reps.d.ts

@@ -16,6 +16,8 @@ export interface RepOutcome {
         subject?: TraceMetrics;
     };
 }
+/** Missing criterion evidence is a non-pass, not a judge misfire or a judged FAIL. */
+export declare function normalizeRepOutcome(outcome: RepOutcome): RepOutcome;
 /**
  * Collapse per-rep objective results.
  *

package/dist/reps.js

@@ -1,3 +1,17 @@
+/** Missing criterion evidence is a non-pass, not a judge misfire or a judged FAIL. */
+export function normalizeRepOutcome(outcome) {
+    if (outcome.verdict === "ERROR")
+        return outcome;
+    if (outcome.objective && outcome.objective.status !== "PASS") {
+        return { ...outcome, verdict: outcome.objective.status, suspect: false };
+    }
+    if (outcome.verdict === "NOT-MEASURED")
+        return outcome;
+    if (outcome.verdict === "UNGRADED" || outcome.judgment?.criteria?.some(vote => vote.verdict === "ERROR")) {
+        return { ...outcome, verdict: "UNGRADED", reason: "UNGRADED: incomplete criterion votes after judge retry", suspect: false };
+    }
+    return outcome;
+}
 /**
  * Collapse per-rep objective results.
  *
@@ -31,7 +45,9 @@ export function aggregateObjective(outcomes) {
  * (default caller threshold 0.5, ties pass). Flakiness = 1 - |2·pass_rate - 1|.
  */
 export function aggregateReps(outcomes, threshold) {
+    outcomes = outcomes.map(normalizeRepOutcome);
     const reps = outcomes.length;
+    const ungraded = outcomes.filter(o => o.verdict === "UNGRADED").length;
     const clean = outcomes.filter((o) => !o.suspect);
     const passes = clean.filter((o) => o.verdict === "PASS").length;
     const errored = outcomes.filter((o) => o.verdict === "ERROR").length;
@@ -50,9 +66,9 @@ export function aggregateReps(outcomes, threshold) {
         return { verdict: "FAIL", reason: `${reps - clean.length}/${reps} reps misfired — re-judge`, passes, reps, clean: clean.length, flakiness: 0, suspect: true };
     }
     const passRate = passes / clean.length;
-    const verdict = passRate >= threshold ? "PASS" : "FAIL";
+    const verdict = passRate >= threshold && (ungraded === 0 || passes > 0) ? "PASS" : ungraded > 0 ? "UNGRADED" : "FAIL";
     const flakiness = 1 - Math.abs(2 * passRate - 1);
-    const reason = reps === 1 ? outcomes[0].reason : `${passes}/${clean.length} reps passed (flaky ${flakiness.toFixed(2)})`;
+    const reason = reps === 1 ? outcomes[0].reason : `${passes}/${clean.length} reps passed (flaky ${flakiness.toFixed(2)})${ungraded ? `; ${ungraded} UNGRADED` : ""}`;
     return { verdict, reason, passes, reps, clean: clean.length, flakiness, suspect: false };
 }
 /**
@@ -63,6 +79,9 @@ export function aggregateReps(outcomes, threshold) {
  * caller to merge.
  */
 export function outcomesToResult(id, outcomes, repCount, threshold) {
+    outcomes = outcomes.map(normalizeRepOutcome);
+    const ungraded = outcomes.filter(outcome => outcome.verdict === "UNGRADED").length;
+    const ungradedField = ungraded ? { ungraded_reps: ungraded } : {};
     // Spread rather than always-set: a scenario with no trace assertions must
     // produce a result byte-identical to one from before this field existed.
     const objective = aggregateObjective(outcomes);
@@ -88,13 +107,13 @@ export function outcomesToResult(id, outcomes, repCount, threshold) {
     const repJudgmentField = outcomes.some(outcome => outcome.judgment) ? { rep_judgments: repJudgments } : {};
     if (repCount === 1) {
         const o = outcomes[0];
-        return { id, judge_verdict: o.verdict, judge_reason: o.reason, suspect: o.suspect, ...metricsField, ...usageField, override: null, note: "", ...objectiveField, ...repJudgmentField };
+        return { id, judge_verdict: o.verdict, judge_reason: o.reason, suspect: o.suspect, ...ungradedField, ...metricsField, ...usageField, override: null, note: "", ...objectiveField, ...repJudgmentField };
     }
     const agg = aggregateReps(outcomes, threshold);
     return {
         id, judge_verdict: agg.verdict, judge_reason: agg.reason, suspect: agg.suspect,
         reps: agg.reps, passes: agg.passes, clean: agg.clean, flakiness: agg.flakiness,
-        pass_threshold: threshold, ...metricsField, ...usageField, override: null, note: "", ...objectiveField, ...repJudgmentField,
+        pass_threshold: threshold, ...ungradedField, ...metricsField, ...usageField, override: null, note: "", ...objectiveField, ...repJudgmentField,
     };
 }
 function aggregateMetrics(outcomes) {

package/dist/rescore.js

@@ -1,6 +1,6 @@
 import { existsSync } from "node:fs";
 import { join } from "node:path";
-import { readResults, writeResults, scoreContextFor, rebuildScenarioResult, effectiveThreshold } from "./results.js";
+import { readResults, writeResults, scoreContextFor, rebuildScenarioResult, normalizeScenarioResult, effectiveThreshold } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { policyDigest, POLICY_PREFIX } from "./sources.js";
 /**
@@ -47,7 +47,8 @@ export function rescoreRun(opts) {
         throw new Error(`no results.yaml in ${opts.runDir}`);
     const specById = new Map(opts.spec.scenarios.map((s) => [s.id, s]));
     const changes = [];
-    const scenarios = prev.scenarios.map((s) => {
+    const scenarios = prev.scenarios.map((recorded) => {
+        const s = normalizeScenarioResult(recorded);
         const scenario = specById.get(s.id);
         // no rate to re-apply, or an untrustworthy verdict → carry verbatim
         if (!scenario || s.reps === undefined || s.clean === undefined || s.passes === undefined)
@@ -61,7 +62,7 @@ export function rescoreRun(opts) {
         if (toThreshold === fromThreshold)
             return s;
         const rate = s.passes / s.clean;
-        const verdict = rate >= toThreshold ? "PASS" : "FAIL";
+        const verdict = rate >= toThreshold && (!s.ungraded_reps || s.passes > 0) ? "PASS" : s.ungraded_reps ? "UNGRADED" : "FAIL";
         if (verdict !== s.judge_verdict) {
             changes.push({ id: s.id, from: s.judge_verdict, to: verdict, passes: s.passes, clean: s.clean, fromThreshold, toThreshold });
         }
@@ -76,6 +77,14 @@ export function rescoreRun(opts) {
         // already measured. It re-measures nothing, so it may discard nothing.
         return rebuildScenarioResult({ ...s, judge_verdict: verdict, pass_threshold: toThreshold }, s, { objective: "carry", adjudication: "carry" });
     });
+    for (const s of scenarios) {
+        const recorded = prev.scenarios.find(prior => prior.id === s.id);
+        if (s.judge_verdict !== recorded.judge_verdict && !changes.some(change => change.id === s.id)) {
+            changes.push({ id: s.id, from: recorded.judge_verdict, to: s.judge_verdict,
+                passes: s.passes ?? (s.judge_verdict === "PASS" ? 1 : 0), clean: s.clean ?? 1,
+                fromThreshold: recorded.pass_threshold ?? 0.5, toThreshold: s.pass_threshold ?? 0.5 });
+        }
+    }
     const ctx = scoreContextFor(prev, opts.spec);
     const results = writeResults(opts.runDir, {
         schema: prev.schema, subject_invocations: prev.subject_invocations,

package/dist/results.d.ts

@@ -61,6 +61,8 @@ export interface ScenarioResult {
     passes?: number;
     clean?: number;
     flakiness?: number;
+    /** Reps with incomplete criterion votes; included as non-passes, never misfires. */
+    ungraded_reps?: number;
     pass_threshold?: number;
     /** Cost/latency counters. Separate from verdicts: cheaper failure is still failure. */
     metrics?: ScenarioMetrics;
@@ -349,6 +351,7 @@ export declare function resultsPath(runDir: string): string;
  * deterministic assertion is a deliberate, recorded human act — and the failure
  * this guards against was never a human deciding, it was nobody deciding.
  */
+export declare function normalizeScenarioResult(s: ScenarioResult): ScenarioResult;
 export declare function effectiveVerdicts(scenarios: ScenarioResult[]): ScenarioVerdict[];
 /**
  * The ONLY place effective_grade is computed. Every writer goes through here,

package/dist/results.js

@@ -5,6 +5,7 @@ import { modelSlug } from "./adapters/types.js";
 import { score } from "./score.js";
 import { HARNESS_VERSION } from "./version.js";
 import { collapseVotePanel } from "./vote-panel.js";
+import { outcomesToResult } from "./reps.js";
 /** Merge bookkeeping from a later judge/gate pass without duplicating subject usage. */
 export function mergeScenarioMetrics(prior, fresh) {
     if (!prior)
@@ -149,8 +150,28 @@ export function resultsPath(runDir) {
  * deterministic assertion is a deliberate, recorded human act — and the failure
  * this guards against was never a human deciding, it was nobody deciding.
  */
+export function normalizeScenarioResult(s) {
+    const panels = s.rep_judgments;
+    if (!panels?.some(panel => panel.recorded_verdict === "UNGRADED" || panel.judgments.some(judgment => judgment.criteria?.some(vote => vote.verdict === "ERROR"))))
+        return s;
+    const reps = s.reps ?? 1;
+    if (panels.length !== reps)
+        return { ...s, judge_verdict: "UNGRADED", suspect: false, ungraded_reps: panels.filter(panel => panel.judgments.some(judgment => judgment.criteria?.some(vote => vote.verdict === "ERROR"))).length };
+    const outcomes = [...panels].sort((a, b) => a.repetition - b.repetition).map(panel => ({
+        verdict: panel.recorded_verdict,
+        reason: panel.judgments[0]?.reason ?? s.judge_reason,
+        suspect: panel.recorded_verdict === "UNGRADED" ? false : panel.judgments.length > 0 && !panel.judgments.some(judgment => !judgment.suspect && (judgment.verdict === "PASS" || judgment.verdict === "FAIL")),
+        objective: panel.objective,
+        judgment: panel.judgments.find(judgment => judgment.criteria?.some(vote => vote.verdict === "ERROR")) ?? panel.judgments[0],
+    }));
+    const fresh = outcomesToResult(s.id, outcomes, reps, s.pass_threshold ?? 0.5);
+    fresh.metrics = s.metrics;
+    fresh.usage = s.usage;
+    fresh.rep_judgments = panels.map(panel => ({ ...panel, recorded_verdict: fresh.rep_judgments?.find(candidate => candidate.repetition === panel.repetition)?.recorded_verdict ?? panel.recorded_verdict }));
+    return rebuildScenarioResult(fresh, s, { objective: "carry", adjudication: "carry" });
+}
 export function effectiveVerdicts(scenarios) {
-    return scenarios.map((s) => ({
+    return scenarios.map(normalizeScenarioResult).map((s) => ({
         id: s.id,
         verdict: s.override ?? objectiveVerdict(s) ?? s.judge_verdict,
         suspect: s.suspect && s.override == null, // an override resolves the misfire
@@ -182,9 +203,10 @@ function objectiveVerdict(s) {
  * deployment and is scored (see SCORED_MODES directly above).
  */
 export function finalizeResults(draft, ctx) {
+    const scenarios = draft.scenarios.map(normalizeScenarioResult);
     let effective_grade;
     if (ctx) {
-        const s = score(effectiveVerdicts(draft.scenarios), { shipBar: ctx.shipBar, critical: ctx.critical });
+        const s = score(effectiveVerdicts(scenarios), { shipBar: ctx.shipBar, critical: ctx.critical });
         effective_grade = { passed: s.passed, total: s.total, pct: s.pct, letter: s.letter, ship: s.ship, note: s.note };
     }
     else {
@@ -214,7 +236,7 @@ export function finalizeResults(draft, ctx) {
         ...(draft.source_hashes ? { source_hashes: draft.source_hashes } : {}),
         ...(draft.source_hash_roots ? { source_hash_roots: draft.source_hash_roots } : {}),
         effective_grade,
-        scenarios: draft.scenarios,
+        scenarios,
         ...(draft.subject_invocations ? { subject_invocations: draft.subject_invocations } : {}),
     };
 }
@@ -304,7 +326,11 @@ export function recomputeRecordedPanels(results) {
         for (const panel of scenario.rep_judgments ?? []) {
             const clean = panel.judgments.filter(j => !j.suspect && (j.verdict === "PASS" || j.verdict === "FAIL"));
             let verdict = panel.recorded_verdict;
-            if (clean.length === 1)
+            if (panel.objective && panel.objective.status !== "PASS")
+                verdict = panel.objective.status;
+            else if (panel.recorded_verdict === "UNGRADED" && panel.judgments.some(judgment => judgment.criteria?.some(vote => vote.verdict === "ERROR")))
+                verdict = "UNGRADED";
+            else if (clean.length === 1)
                 verdict = clean[0].verdict;
             else if (clean.length >= 2)
                 verdict = collapseVotePanel(panel.judgments).verdict ?? "JUDGE-AMBIGUOUS";
@@ -417,7 +443,7 @@ export function validateResults(raw) {
             if (!Number.isInteger(scenario.adjudication.repetition) || scenario.adjudication.repetition < 0 || scenario.adjudication.repetition >= reps)
                 throw new Error(`schema v3 adjudication repetition missing or out of range for ${scenario.id}`);
             const collapsed = collapseVotePanel(scenario.adjudication.judgments);
-            const boundedCriticalAggregate = (scenario.reps ?? 1) > 1 && scenario.pass_threshold === 1 && scenario.judge_verdict !== "PASS" && collapsed.verdict === "PASS" && scenario.adjudication.state === "unresolved" && scenario.adjudication.verdict === undefined;
+            const boundedCriticalAggregate = ((scenario.reps ?? 1) > 1 && scenario.pass_threshold === 1 || scenario.judge_verdict === "UNGRADED") && scenario.judge_verdict !== "PASS" && collapsed.verdict === "PASS" && scenario.adjudication.state === "unresolved" && scenario.adjudication.verdict === undefined;
             if (!boundedCriticalAggregate && (scenario.adjudication.state !== collapsed.state || scenario.adjudication.verdict !== collapsed.verdict))
                 throw new Error(`recorded adjudication state/verdict diverges from recomputed votes for ${scenario.id}`);
             if (scenario.adjudication.state !== "unresolved")
@@ -426,13 +452,14 @@ export function validateResults(raw) {
         const panels = scenario.rep_judgments;
         const errorCount = panels.filter(panel => panel.recorded_verdict === "ERROR").length;
         const notMeasuredCount = panels.filter(panel => panel.recorded_verdict === "NOT-MEASURED").length;
-        const cleanPanels = panels.filter(panel => !(panel.judgments[0]?.suspect ?? false));
+        const cleanPanels = panels.filter(panel => panel.objective && panel.objective.status !== "PASS" || panel.recorded_verdict === "UNGRADED" || !(panel.judgments[0]?.suspect ?? false));
+        const ungradedCount = panels.filter(panel => panel.recorded_verdict === "UNGRADED").length;
         const passes = cleanPanels.filter(panel => panel.recorded_verdict === "PASS").length;
         let aggregateVerdict;
         let aggregateSuspect = false;
         if (panels.length === 1) {
             aggregateVerdict = panels[0].recorded_verdict;
-            aggregateSuspect = panels[0].judgments[0]?.suspect ?? false;
+            aggregateSuspect = panels[0].objective && panels[0].objective.status !== "PASS" || panels[0].recorded_verdict === "UNGRADED" ? false : panels[0].judgments[0]?.suspect ?? false;
         }
         else if (errorCount > 0)
             aggregateVerdict = "ERROR";
@@ -443,7 +470,7 @@ export function validateResults(raw) {
             aggregateSuspect = true;
         }
         else
-            aggregateVerdict = passes / cleanPanels.length >= (scenario.pass_threshold ?? 0.5) ? "PASS" : "FAIL";
+            aggregateVerdict = passes / cleanPanels.length >= (scenario.pass_threshold ?? 0.5) && (ungradedCount === 0 || passes > 0) ? "PASS" : ungradedCount > 0 ? "UNGRADED" : "FAIL";
         const expectedVerdict = adjudicatedVerdict ?? aggregateVerdict;
         const expectedSuspect = scenario.adjudication ? scenario.adjudication.state === "unresolved" : aggregateSuspect;
         if (scenario.judge_verdict !== expectedVerdict || scenario.suspect !== expectedSuspect)
@@ -600,7 +627,7 @@ export function diffPath(runDir, scenarioId, mode, rep) {
 export function rebuildScenarioResult(fresh, prior, policy) {
     // Exhaustive destructure. Do not replace with a spread: the spread is what
     // allowed a new field to pass through unconsidered in the first place.
-    const { id, criterion_count: freshCriterionCount, judge_verdict, judge_reason, suspect, override: _freshOverride, note: _freshNote, reps, passes, clean, flakiness, pass_threshold, metrics: freshMetrics, usage: freshUsage, objective: freshObjective, adjudication: freshAdjudication, rep_judgments: freshRepJudgments, ...rest } = fresh;
+    const { id, criterion_count: freshCriterionCount, judge_verdict, judge_reason, suspect, override: _freshOverride, note: _freshNote, reps, passes, clean, flakiness, pass_threshold, ungraded_reps, metrics: freshMetrics, usage: freshUsage, objective: freshObjective, adjudication: freshAdjudication, rep_judgments: freshRepJudgments, ...rest } = fresh;
     const _exhaustive = rest;
     void _exhaustive;
     void _freshOverride;
@@ -613,7 +640,7 @@ export function rebuildScenarioResult(fresh, prior, policy) {
     const objective = pick(policy.objective, freshObjective, prior?.objective);
     const pickedAdjudication = pick(policy.adjudication, freshAdjudication, prior?.adjudication);
     const conflictsWithFreshEvidence = Boolean(pickedAdjudication?.verdict && (objective?.status === "FAIL" || objective?.status === "ERROR" || objective?.status === "NOT-MEASURED" ||
-        (pickedAdjudication.verdict === "PASS" && pass_threshold === 1 && (reps ?? 1) > 1 && judge_verdict !== "PASS")));
+        (pickedAdjudication.verdict === "PASS" && (judge_verdict === "UNGRADED" || pass_threshold === 1 && (reps ?? 1) > 1 && judge_verdict !== "PASS"))));
     // A settled cell-level panel cannot overrule newly evaluated objective gates,
     // nor can one transcript erase a failed repetition under an all-clean policy.
     const adjudication = conflictsWithFreshEvidence && pickedAdjudication
@@ -652,6 +679,7 @@ export function rebuildScenarioResult(fresh, prior, policy) {
         ...(clean === undefined ? {} : { clean }),
         ...(flakiness === undefined ? {} : { flakiness }),
         ...(pass_threshold === undefined ? {} : { pass_threshold }),
+        ...(ungraded_reps === undefined ? {} : { ungraded_reps }),
         ...((freshMetrics ?? prior?.metrics) ? { metrics: freshMetrics ?? prior.metrics } : {}),
         ...((freshUsage ?? prior?.usage) ? { usage: freshUsage ?? prior.usage } : {}),
         // The author owns the verdict; a re-measurement never discards their call.

package/dist/run.js

@@ -567,7 +567,8 @@ export function formatScorecard(summary, lift, stability) {
         const susp = s.suspect ? " ⚠suspect" : "";
         const misfired = s.clean !== undefined && s.reps !== undefined && s.clean < s.reps ? ` · ${s.reps - s.clean} misfired` : "";
         const repInfo = s.reps ? `  [${s.passes}/${s.clean}${misfired}${s.flakiness ? ` flaky ${s.flakiness.toFixed(2)}` : ""}]` : "";
-        lines.push(`  ${mark} ${s.id}${ov}${susp}  ${s.judge_reason}${repInfo}`);
+        const ungradedInfo = s.ungraded_reps ? ` · ${s.ungraded_reps} ungraded rep${s.ungraded_reps === 1 ? "" : "s"} (UNGRADED)` : "";
+        lines.push(`  ${mark} ${s.id}${ov}${susp}  ${s.judge_reason}${repInfo}${ungradedInfo}`);
     }
     const ship = g.ship ? "SHIP" : "NOT READY";
     const note = g.note ? ` (${g.note})` : "";

package/dist/score.d.ts

@@ -3,7 +3,7 @@ import type { ShipBar } from "./spec.js";
  * "JUDGE-AMBIGUOUS": the judge emitted conflicting verdicts for one transcript. It is
  * never a pass and never silently resolved — it marks the run for a rejudge.
  */
-export type Verdict = "PASS" | "FAIL" | "ERROR" | "NOT-MEASURED" | "JUDGE-AMBIGUOUS";
+export type Verdict = "PASS" | "FAIL" | "ERROR" | "NOT-MEASURED" | "JUDGE-AMBIGUOUS" | "UNGRADED";
 export interface ScenarioVerdict {
     id: string;
     verdict: Verdict;
@@ -24,6 +24,7 @@ export interface ScoreResult {
     suspectCount: number;
     errorCount: number;
     notMeasuredCount: number;
+    ungradedCount: number;
     note: string;
 }
 export declare function letterFor(pct: number): string;

package/dist/score.js

@@ -28,6 +28,7 @@ export function score(verdicts, input) {
     let suspectCount = 0;
     let errorCount = 0;
     let notMeasuredCount = 0;
+    let ungradedCount = 0;
     for (const v of verdicts) {
         if (v.suspect) {
             suspectCount++;
@@ -46,6 +47,8 @@ export function score(verdicts, input) {
             passed++;
             continue;
         }
+        if (v.verdict === "UNGRADED")
+            ungradedCount++;
         if (critical.includes(v.id))
             criticalFails++;
         if (/^B/i.test(v.id))
@@ -75,12 +78,15 @@ export function score(verdicts, input) {
     else if (notMeasuredCount > 0) {
         note = `${notMeasuredCount} not measured: skill delivery was not established`;
     }
+    else if (ungradedCount > 0) {
+        note = `${ungradedCount} UNGRADED: incomplete criterion votes (counted as non-passes)`;
+    }
     else if (criticalFails > 0) {
         note = `gated: ${criticalFails} critical fail${criticalFails === 1 ? "" : "s"}`;
     }
     else if (bSeriesFails > 0) {
         note = `gated: ${bSeriesFails} B-series fail${bSeriesFails === 1 ? "" : "s"}`;
     }
-    return { passed, total, pct, letter, ship, criticalFails, bSeriesFails, suspectCount, errorCount, notMeasuredCount, note };
+    return { passed, total, pct, letter, ship, criticalFails, bSeriesFails, suspectCount, errorCount, notMeasuredCount, ungradedCount, note };
 }
 //# sourceMappingURL=score.js.map

package/dist/stability.js

@@ -9,7 +9,7 @@ function conclusive(v) {
     // Same rule lift.ts applies, for the same reason: ERROR is a harness failure and an
     // unresolved misfire is a judge failure. Counting either as a side of a flip would
     // report infrastructure noise as behavioural instability.
-    return !v.suspect && v.verdict !== "ERROR" && v.verdict !== "NOT-MEASURED" && v.verdict !== "JUDGE-AMBIGUOUS";
+    return !v.suspect && v.verdict !== "ERROR" && v.verdict !== "NOT-MEASURED" && v.verdict !== "JUDGE-AMBIGUOUS" && v.verdict !== "UNGRADED";
 }
 function pointFor(r, s, verdict) {
     const reps = s.reps ?? 1;

package/dist/trends.js

@@ -1,7 +1,7 @@
 import { existsSync, readdirSync, statSync } from "node:fs";
 import { join } from "node:path";
 import { loadSpec } from "./spec.js";
-import { readResults, effectiveVerdicts, isScoredMode } from "./results.js";
+import { readResults, effectiveVerdicts, isScoredMode, normalizeScenarioResult, finalizeResults, scoreContextFor } from "./results.js";
 /** A directory that exists right now; false (never throws) if it vanished concurrently (e.g. ENOENT). */
 function isDir(p) {
     try {
@@ -112,12 +112,16 @@ export function collectTrends(skillDir, limit = 20) {
             // effectiveVerdicts is the single source of truth for the override-aware
             // verdict/suspect rule (suspect = s.suspect && s.override == null — an override
             // resolves the misfire); zip in flakiness from the matching ScenarioResult.
-            const verdicts = effectiveVerdicts(r.scenarios);
+            const normalized = r.scenarios.map(normalizeScenarioResult);
+            const grade = normalized.some((s, i) => s !== r.scenarios[i])
+                ? finalizeResults({ ...r, scenarios: normalized }, scoreContextFor(r, spec)).effective_grade
+                : r.effective_grade;
+            const verdicts = effectiveVerdicts(normalized);
             const cells = {};
-            r.scenarios.forEach((s, i) => {
+            normalized.forEach((s, i) => {
                 cells[s.id] = { verdict: verdicts[i].verdict, suspect: verdicts[i].suspect ?? false, flakiness: s.flakiness };
             });
-            runs.push({ timestamp: r.timestamp, label: r.label, grade: r.effective_grade, cells });
+            runs.push({ timestamp: r.timestamp, label: r.label, grade, cells });
         }
         models.push({ model: group.model, tag: group.tag, mode: group.mode, runs, truncated, skipped: group.skipped });
     }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.22.3",
+    "version": "0.22.4",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",

package/schemas/results-v2.schema.json

@@ -102,7 +102,8 @@
         "PASS",
         "FAIL",
         "ERROR",
-        "JUDGE-AMBIGUOUS"
+        "JUDGE-AMBIGUOUS",
+        "UNGRADED"
       ]
     },
     "grade": {
@@ -321,6 +322,10 @@
           "minimum": 0,
           "maximum": 1
         },
+        "ungraded_reps": {
+          "type": "integer",
+          "minimum": 1
+        },
         "pass_threshold": {
           "type": "number",
           "minimum": 0,

package/schemas/results-v3.schema.json

@@ -113,7 +113,8 @@
         "FAIL",
         "ERROR",
         "NOT-MEASURED",
-        "JUDGE-AMBIGUOUS"
+        "JUDGE-AMBIGUOUS",
+        "UNGRADED"
       ]
     },
     "grade": {
@@ -347,6 +348,10 @@
           "minimum": 0,
           "maximum": 1
         },
+        "ungraded_reps": {
+          "type": "integer",
+          "minimum": 1
+        },
         "pass_threshold": {
           "type": "number",
           "minimum": 0,