@pome-sh/cli 0.16.0 → 0.18.0

package/README.md

@@ -64,22 +64,30 @@ code is the verdict**. Gate CI on it directly.
 | Exit code | Meaning |
 | --- | --- |
 | `0` | pass (hosted/scored run), or trace captured (`--local`, not scored) |
-| `1` | ran and scored **below** the pass threshold |
+| `1` | ran and scored **below** the pass threshold, **or ran `INCOMPLETE`** |
 | `2` | twin / orchestration error (network, 5xx, twin spawn failed) |
 | `3` | auth error (401/403) — `pome login` again, or set `POME_API_KEY` in CI |
 | `4` | quota exceeded (402/429) |
 | `5` | usage error (bad flags, missing task file) |
 
-Two rules CI must honor:
+Three rules CI must honor:
 
 - **`--local` is not a verdict.** A `--local` run captures a raw trace and never
   scores, so its exit `0` means "trace captured," not "passed." Never gate CI on
   a `--local` exit code — score it later with `pome eval <run-dir>`.
+- **`INCOMPLETE` shares exit `1`, and it is not the agent's failure.** A run
+  whose criteria could not all be graded exits `1` rather than mapping its
+  partial score to a code — a run whose checks never ran is not a green CI
+  signal. The cost is stated rather than hidden: **`1` cannot tell "the agent
+  regressed" from "we could not grade it."** Read the verdict word printed
+  beside the score (`INCOMPLETE` vs a sub-threshold number) to separate them.
 - **Trial groups map as a whole.** `pome run -n k` (k>1) collapses the whole
   group to one code: `0` = at least one trial completed and every completed
-  trial passed; `1` = at least one completed trial failed its threshold; `2` =
-  no trial completed. Errored trials are excluded from the verdict fraction and
-  never drag a passing group below `0` on their own.
+  trial passed; `1` = at least one completed trial failed its threshold **or was
+  incomplete**; `2` = no trial completed. Errored and incomplete trials are
+  excluded from the verdict fraction (`3 of 4 passed · 1 incomplete`) so neither
+  is counted as a pass nor charged to the agent as a loss — but a group holding
+  one cannot exit `0`.
 
 ## Development
 

package/dist/build-info.json

@@ -1,6 +1,6 @@
 {
   "package": "pome-sh",
-  "version": "0.16.0",
-  "git_sha": "1d69d644c3acee14dc92d621968b1322c40a0bf0",
-  "build_time": "2026-07-31T00:23:39.731Z"
+  "version": "0.18.0",
+  "git_sha": "285d48dcd654088387b40e8d36a7a2bf8ffbb599",
+  "build_time": "2026-08-03T13:39:43.286Z"
 }

package/dist/src/cli/checks-add.d.ts

@@ -1,12 +1,7 @@
 import { type InteractiveSeams } from "./agent-resolver.js";
-export interface RemoteChecks {
+import { type RemoteVocabulary } from "./vocabulary-skew.js";
+export interface RemoteChecks extends RemoteVocabulary {
     twin: string;
-    digest: string;
-    checks: Array<{
-        id: string;
-        template: string;
-        substrate: string;
-    }>;
 }
 export type HandshakeResult = {
     kind: "match";

package/dist/src/cli/checks-add.js

@@ -8,7 +8,9 @@
  * the cloud resolves them from a different one. Before writing, ask the cloud
  * for the vocabulary it GRADES against and compare digests.
  *   equal        → write, silently
- *   different    → REFUSE, and name which check moved
+ *   different    → REFUSE, and name what moved (`vocabulary-skew.ts`, F-1137 —
+ *                  a check, a field of one, or the sdk that compiled it, but
+ *                  never an empty list)
  *   unreachable  → write from the local pin with a named note on stderr
  * Offline authoring is real, and the degraded path's worst case is bounded: a
  * stale sentence that keeps a template's literal segments fails at finalize as
@@ -22,6 +24,7 @@ import { resolveSeams } from "./agent-resolver.js";
 import { SUBSTRATE_HELP, checksFor, displaySentence, findCheck, localDigest, pinnedVersion, twinOf, } from "./checks.js";
 import { resolveCredentials } from "./credentials.js";
 import { auditCodeCriteria, formatBindingReport } from "./criterion-binding.js";
+import { explainSkew, formatSkewRefusal } from "./vocabulary-skew.js";
 export function parseArgFlags(pairs) {
     const args = {};
     for (const pair of pairs) {
@@ -47,29 +50,12 @@ export async function handshake(twin, fetchRemote) {
     }
     if (localDigest(twin) === remote.digest)
         return { kind: "match" };
-    const here = new Map(checksFor(twin).map((check) => [check.id, check.template]));
-    const there = new Map(remote.checks.map((check) => [check.id, check.template]));
-    const moved = [];
-    for (const [id, template] of here) {
-        if (!there.has(id))
-            moved.push(`${id} — this CLI has it, the cloud does not`);
-        else if (there.get(id) !== template) {
-            moved.push(`${id} — the sentence differs\n        here:  ${template}\n        cloud: ${there.get(id)}`);
-        }
-    }
-    for (const id of there.keys()) {
-        if (!here.has(id))
-            moved.push(`${id} — the cloud has it, this CLI does not`);
-    }
-    return {
-        kind: "skew",
-        message: `Refusing to write: this CLI and the cloud disagree about ${twin}'s vocabulary, so a ` +
-            `sentence written here might not be graded there.\n` +
-            moved.map((line) => `      - ${line}`).join("\n") +
-            `\n\n  local @pome-sh/twin-${twin} ${pin}` +
-            `\n  Update with \`npm i -g @pome-sh/cli@latest\`. If this CLI is already current, the ` +
-            `cloud is behind — that is a deploy, not something you can fix here.`,
-    };
+    // F-1137 — the taxonomy lives in `vocabulary-skew.ts` and is NON-EMPTY by
+    // construction, so this refusal cannot name the disagreement and then list
+    // nothing. Which is what it did whenever the skew was in `substrate` or in the
+    // compiled pattern: the two fields `checksDigest` hashes that the old
+    // id-and-template diff never looked at.
+    return { kind: "skew", message: formatSkewRefusal(twin, explainSkew(twin, remote)) };
 }
 async function liveFetch(creds, twin) {
     const res = await fetch(`${creds.apiBaseUrl}/v1/checks?twin=${encodeURIComponent(twin)}`, {

package/dist/src/cli/eval.js

@@ -296,7 +296,7 @@ export async function runEval(options) {
         return client.finalize(sid, {
             stopReason: "eval_upload",
             // /finalize's schema requires an integer exit_code, so `null` (agent
-            // timed out, or meta.json lacked the field) cannot pass through.
+            // timed out, or meta.json lacked the field) is not a legal value.
             // Send -1 as the explicit "unknown" sentinel — never a fabricated 0,
             // which would report a clean agent exit the trace can't vouch for.
             exitCode: artifacts.meta.exitCode ?? -1,
@@ -340,14 +340,16 @@ export async function runEval(options) {
     // never persisted next to the trace. Local artifacts stay trace-only (no
     // score.json), and the verdict lives in the cloud (see the dashboard URL).
     const score = scoreFromFinalizeResponse(finalized);
-    // Exit-code policy — DELIBERATE DIVERGENCE from hosted `pome run`
-    // (FDRS-618): `pome run` maps the raw cloud score (score >= threshold →
-    // 0), because pre-FDRS-618 cloud builds don't emit criteria_results and
-    // the cloud exit decision is documented as score-only. `pome eval` is a
-    // NEW command with no such compatibility surface, so it adopts the full
-    // FDRS-591/611 A5 guard up front: exit 0 ONLY when the run was evaluated,
-    // every criterion was judged (can_pass), AND the score clears the
-    // threshold. An UNEVAL verdict (e.g. all criteria skipped) exits 1.
+    // Exit-code policy — the full FDRS-591/611 A5 guard: exit 0 ONLY when the run
+    // was evaluated, every criterion was judged (can_pass), AND the score clears
+    // the threshold. An INCOMPLETE verdict (any criterion not evaluated) exits 1.
+    //
+    // F-925 retired the divergence that used to be documented here. `pome run`
+    // mapped the raw cloud score because "pre-FDRS-618 cloud builds don't emit
+    // criteria_results" — but `scoreFromFinalizeResponse` already handles that
+    // case (`hasCriteriaResults ? … : true`), so the guard degrades to score-only
+    // for exactly those builds on its own. The divergence was protecting a case
+    // its own helper already protected, and the two commands now agree.
     const exitCode = scoreStatus(score, EVAL_PASS_THRESHOLD) === "pass" ? 0 : 1;
     return {
         taskName,
@@ -403,7 +405,7 @@ export async function runEvalCommand(runDirArg, opts) {
         });
         // Same verdict shape as hosted `pome run`: LABEL, score line, cloud URL.
         const status = scoreStatus(result.score, EVAL_PASS_THRESHOLD);
-        const label = status === "pass" ? "PASS" : status === "fail" ? "FAIL" : "UNEVAL";
+        const label = status === "pass" ? "PASS" : status === "fail" ? "FAIL" : "INCOMPLETE";
         console.error(`${label} ${result.taskName}`);
         console.error(`  ${runScoreLine(result.score, EVAL_PASS_THRESHOLD, "cloud score")}`);
         if (result.score.results.length > 0) {

package/dist/src/cli/main.js

@@ -660,7 +660,7 @@ export function createProgram() {
                         agentVersion: options.agentVersion,
                     });
                     const status = scoreStatus(result.score, result.scenario.config.passThreshold);
-                    const label = status === "pass" ? "PASS" : status === "fail" ? "FAIL" : "UNEVAL";
+                    const label = status === "pass" ? "PASS" : status === "fail" ? "FAIL" : "INCOMPLETE";
                     console.error(`${label} ${result.scenario.title}`);
                     console.error(`  ${runScoreLine(result.score, result.scenario.config.passThreshold, "cloud score")}`);
                     console.error(`  local: ${result.artifacts.runDir}`);

package/dist/src/cli/vocabulary-skew.d.ts

@@ -0,0 +1,64 @@
+/** One check as `GET /v1/checks` publishes it. */
+export interface RemoteCheck {
+    id: string;
+    template: string;
+    substrate: string;
+    /** The compiled regex source. Optional because a control plane that does not
+     *  publish it must degrade to a named class, never to a crash or to silence. */
+    pattern?: string;
+    /** Ordered as the template names them — the compiled pattern wraps one group
+     *  per slot in that order, so the ORDER is part of what the digest sees. */
+    params?: Array<{
+        name: string;
+        pattern: string;
+    }>;
+}
+export interface RemoteVocabulary {
+    digest: string;
+    checks: RemoteCheck[];
+}
+export type SkewFinding = {
+    kind: "only_here";
+    check: string;
+} | {
+    kind: "only_there";
+    check: string;
+} | {
+    kind: "template";
+    check: string;
+    here: string;
+    there: string;
+} | {
+    kind: "substrate";
+    check: string;
+    here: string;
+    there: string;
+} | {
+    kind: "params";
+    check: string;
+    here: string;
+    there: string;
+} | {
+    kind: "pattern";
+    check: string;
+    here: string;
+    there: string;
+    paramsCompared: boolean;
+} | {
+    kind: "pattern_generation";
+    check: null;
+    here: string;
+    there: string;
+};
+/**
+ * Every way this CLI's vocabulary for `twin` differs from the one the cloud
+ * published, as at least one named finding.
+ *
+ * Ordered by check id, so the readout is a property of the difference rather
+ * than of either side's declaration order.
+ */
+export declare function explainSkew(twin: string, remote: RemoteVocabulary): [SkewFinding, ...SkewFinding[]];
+/** The refusal an author reads. Names the twin pin always, and the sdk pin only
+ *  when the sdk is what the findings implicate — a pin nobody needs is noise in a
+ *  message someone is reading because they are already blocked. */
+export declare function formatSkewRefusal(twin: string, findings: readonly SkewFinding[]): string;

package/dist/src/cli/vocabulary-skew.js

@@ -0,0 +1,187 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * F-1137 — WHY do this CLI and the cloud disagree about a twin's vocabulary?
+ *
+ * `checksDigest` hashes three fields per check: `id`, `substrate`, and the
+ * COMPILED pattern (`packages/sdk/src/checks.ts`). The refusal in `checks-add.ts`
+ * used to build its "which check moved" list from `id` and `template` — so a skew
+ * that moved only `substrate`, or only `buildPattern`'s output while every
+ * template was byte-identical, refused correctly and then named NOTHING. A named
+ * refusal that names nothing, in exactly the two cases the digest was widened to
+ * catch, during a situation that already blocks the author.
+ *
+ * The fix is a taxonomy with no silent branch: `explainSkew` returns a NON-EMPTY
+ * list by construction, and the type says so. The class names are pome-cloud's
+ * (`apps/control-plane/src/services/vocabulary-parity.ts`, F-1136) so the two
+ * surfaces that explain the same disagreement stay greppable against each other.
+ *
+ * This side can say more than the cloud-side monitor can. `GET /v1/checks`
+ * publishes the compiled `pattern` and the parameter patterns (F-1074 Phase 3),
+ * which the CLI's own `checks --json` does not — so a generator-only skew is
+ * localisable to a check HERE, and only falls back to the unlocalised
+ * `pattern_generation` class when the control plane published nothing to diff.
+ *
+ * `description` and `params[].example` are compared by nothing here on purpose:
+ * `checksDigest` does not hash them, so a difference in either cannot be what
+ * moved the digest, and comparing them could only manufacture false findings.
+ */
+import { checkPattern, templateSlots } from "@pome-sh/sdk/checks";
+import { checksFor, localDigest, pinnedVersion } from "./checks.js";
+function paramsHere(def) {
+    return templateSlots(def.template)
+        .params.map((name) => `${name}=${def.params[name].pattern}`)
+        .join(", ");
+}
+function paramsThere(check) {
+    return (check.params ?? []).map((p) => `${p.name}=${p.pattern}`).join(", ");
+}
+/** The fields BOTH sides always publish. Returns empty when the declarations
+ *  agree — which is the precondition for reading anything into the compiled
+ *  pattern, since a moved template moves the pattern too and reporting both
+ *  would bury the cause under its own consequence. */
+function declarationSkew(def, check) {
+    const findings = [];
+    if (def.template !== check.template) {
+        findings.push({ kind: "template", check: def.id, here: def.template, there: check.template });
+    }
+    if (def.substrate !== check.substrate) {
+        findings.push({
+            kind: "substrate",
+            check: def.id,
+            here: def.substrate,
+            there: check.substrate,
+        });
+    }
+    if (check.params !== undefined && paramsHere(def) !== paramsThere(check)) {
+        findings.push({
+            kind: "params",
+            check: def.id,
+            here: paramsHere(def),
+            there: paramsThere(check),
+        });
+    }
+    return findings;
+}
+/**
+ * Every way this CLI's vocabulary for `twin` differs from the one the cloud
+ * published, as at least one named finding.
+ *
+ * Ordered by check id, so the readout is a property of the difference rather
+ * than of either side's declaration order.
+ */
+export function explainSkew(twin, remote) {
+    const here = new Map(checksFor(twin).map((def) => [def.id, def]));
+    const there = new Map(remote.checks.map((check) => [check.id, check]));
+    const findings = [];
+    for (const id of [...new Set([...here.keys(), ...there.keys()])].sort()) {
+        const def = here.get(id);
+        const check = there.get(id);
+        if (!check) {
+            findings.push({ kind: "only_here", check: id });
+            continue;
+        }
+        if (!def) {
+            findings.push({ kind: "only_there", check: id });
+            continue;
+        }
+        const declaration = declarationSkew(def, check);
+        if (declaration.length > 0) {
+            findings.push(...declaration);
+            continue;
+        }
+        const compiled = checkPattern(def).source;
+        if (check.pattern !== undefined && check.pattern !== compiled) {
+            findings.push({
+                kind: "pattern",
+                check: id,
+                here: compiled,
+                there: check.pattern,
+                paramsCompared: check.params !== undefined,
+            });
+        }
+    }
+    const [first, ...rest] = findings;
+    if (!first) {
+        return [
+            {
+                kind: "pattern_generation",
+                check: null,
+                here: localDigest(twin),
+                there: remote.digest,
+            },
+        ];
+    }
+    return [first, ...rest];
+}
+function pair(here, there) {
+    return [`        here:  ${here}`, `        cloud: ${there}`];
+}
+// One bullet per class, and the compiler holds the line: the annotated return
+// type plus an exhaustive switch means a class added without a bullet is
+// `error TS2366`, not a line that renders `undefined`. Do NOT add a `default:` —
+// it would buy back the silence this module exists to remove.
+function bullet(finding) {
+    switch (finding.kind) {
+        case "only_here":
+            return [`      - ${finding.check} — this CLI has it, the cloud does not`];
+        case "only_there":
+            return [`      - ${finding.check} — the cloud has it, this CLI does not`];
+        case "template":
+            return [
+                `      - ${finding.check} — the sentence differs`,
+                ...pair(finding.here, finding.there),
+            ];
+        case "substrate":
+            return [
+                `      - ${finding.check} — the substrate differs, so the two sides grade it against`,
+                `        different evidence`,
+                ...pair(finding.here, finding.there),
+            ];
+        case "params":
+            return [
+                `      - ${finding.check} — the parameters differ`,
+                ...pair(`[${finding.here}]`, `[${finding.there}]`),
+            ];
+        case "pattern":
+            return [
+                ...(finding.paramsCompared
+                    ? [
+                        `      - ${finding.check} — same sentence and parameters, different compiled`,
+                        `        pattern: the two sides' @pome-sh/sdk compile one declaration differently,`,
+                        `        which is a buildPattern change and not a vocabulary change`,
+                    ]
+                    : [
+                        `      - ${finding.check} — same sentence, different compiled pattern, and no`,
+                        `        parameter patterns published: either a parameter type or @pome-sh/sdk`,
+                        `        buildPattern moved`,
+                    ]),
+                ...pair(finding.here, finding.there),
+            ];
+        case "pattern_generation":
+            return [
+                `      - the digests differ, but every field the cloud published matches this CLI's`,
+                `        declarations. That makes it an @pome-sh/sdk difference — buildPattern, or what`,
+                `        checksDigest hashes — rather than a vocabulary change`,
+                ...pair(finding.here, finding.there),
+            ];
+    }
+}
+/** The refusal an author reads. Names the twin pin always, and the sdk pin only
+ *  when the sdk is what the findings implicate — a pin nobody needs is noise in a
+ *  message someone is reading because they are already blocked. */
+export function formatSkewRefusal(twin, findings) {
+    const sdkImplicated = findings.some((f) => f.kind === "pattern" || f.kind === "pattern_generation");
+    return [
+        `Refusing to write: this CLI and the cloud disagree about ${twin}'s vocabulary, so a ` +
+            `sentence written here might not be graded there.`,
+        ...findings.flatMap(bullet),
+        "",
+        `  local @pome-sh/twin-${twin} ${pinnedVersion(`@pome-sh/twin-${twin}`)}` +
+            (sdkImplicated ? `, @pome-sh/sdk ${pinnedVersion("@pome-sh/sdk")}` : ""),
+        ...(sdkImplicated
+            ? [`  The cloud publishes no sdk version here, so that is the only pin this CLI can name.`]
+            : []),
+        `  Update with \`npm i -g @pome-sh/cli@latest\`. If this CLI is already current, the ` +
+            `cloud is behind — that is a deploy, not something you can fix here.`,
+    ].join("\n");
+}

package/dist/src/hosted/evalResultView.d.ts

@@ -29,7 +29,7 @@ export type Score = {
     judge_tokens_out: number | null;
 };
 export declare function outcomeOf(result: CriterionResult): CriterionOutcome;
-export type ScoreStatus = "pass" | "fail" | "unevaluated";
+export type ScoreStatus = "pass" | "fail" | "incomplete";
 export declare function scoreStatus(score: Score, passThreshold: number): ScoreStatus;
 export declare function taskPassed(score: Score, passThreshold: number): boolean;
 export declare function markerFor(outcome: CriterionOutcome): string;

package/dist/src/hosted/evalResultView.js

@@ -31,9 +31,20 @@ export function outcomeOf(result) {
 // Encodes the A5 guard: a run is only a PASS when it was evaluated, every
 // required criterion was evaluated (can_pass), AND satisfaction cleared the
 // threshold. PURE — no computation of the score itself.
+//
+// F-932 renamed the third state from `unevaluated` to `incomplete` and CHANGED
+// NOTHING ELSE HERE. The guard is the one place the CLI refuses to inflate a
+// partial run into a pass — the same refusal pome-cloud added server-side in
+// F-925 — so the rename must not become a loosening.
+//
+// One rule, two repos: `can_pass` is false for ANY abstention
+// (`uploadAndFinalize.ts`), and pome-cloud's `isRunIncomplete` says
+// `notEvaluated > 0` over the same `criteria_results`. Deliberately NOT read
+// from the wire's `all_skipped`, which is the narrower every-abstained
+// predicate and would loosen this guard.
 export function scoreStatus(score, passThreshold) {
     if (!score.evaluated || !score.can_pass)
-        return "unevaluated";
+        return "incomplete";
     return score.satisfaction >= passThreshold ? "pass" : "fail";
 }
 export function taskPassed(score, passThreshold) {
@@ -55,14 +66,14 @@ export function markerFor(outcome) {
 // Multi-twin (M3): the per-criterion bracket for terminal display —
 // `[code]` / `[model]`, plus the `:<twin>` suffix when the criterion attributes
 // to a specific twin (so a `[code:slack]`/`[model:github]` marker survives into the
-// UNEVAL / criteria list). A bare (primary-twin) criterion renders `[code]`
+// INCOMPLETE / criteria list). A bare (primary-twin) criterion renders `[code]`
 // unchanged.
 export function criterionMarkerLabel(criterion) {
     return criterion.twin ? `[${criterion.type}:${criterion.twin}]` : `[${criterion.type}]`;
 }
 // Multi-twin (M3): when the cloud could not evaluate a criterion for a
 // twin-related reason (a twin-tagged criterion, or a `no_matching_predicate`
-// skip), name the twin inline so the UNEVAL line explains WHICH twin's timeline
+// skip), name the twin inline so the INCOMPLETE line explains WHICH twin's timeline
 // came up empty. Returns "" when there's nothing twin-specific to add.
 export function twinSkipSuffix(result) {
     const twin = result.criterion.twin;
@@ -79,8 +90,14 @@ export function scoreCountsSummary(score) {
 }
 export function runScoreLine(score, passThreshold, unevaluatedNumericLabel) {
     const status = scoreStatus(score, passThreshold);
-    if (status === "unevaluated") {
-        return `score: un-evaluated (cannot pass) — ${scoreCountsSummary(score)}; ${unevaluatedNumericLabel}: ${score.satisfaction}/100`;
+    if (status === "incomplete") {
+        // Leads with the COUNT, which is the fact the reader needs and the same
+        // fact the cloud's own header now states. The old copy said "cannot pass",
+        // which is a verdict about the AGENT for a gap in the GRADER — the exact
+        // inversion F-925 exists to stop, one surface over.
+        const notEvaluated = score.skipped + score.errored;
+        const total = score.total_required + notEvaluated;
+        return `score: incomplete — ${notEvaluated} of ${total} criteria not evaluated; ${scoreCountsSummary(score)}; ${unevaluatedNumericLabel}: ${score.satisfaction}/100`;
     }
     return `score: ${score.satisfaction}/100`;
 }

package/dist/src/runner/groupRender.d.ts

@@ -1,9 +1,19 @@
+import type { ScoreStatus } from "../hosted/evalResultView.js";
 export type TrialRow = {
     kind: "completed";
     /** Cloud-authoritative satisfaction score, 0-100. */
     score: number;
-    /** Cleared the scenario's pass threshold. */
-    passed: boolean;
+    /**
+     * F-925 — three states, not a boolean. `incomplete` means the trial ran
+     * and finalized but at least one criterion never produced a verdict, so
+     * it is neither a pass nor the agent's failure. It was `passed: boolean`
+     * fed from `exitCode === 0`, which counted a 100/100 run with 3 of 4
+     * criteria skipped as a clean passing trial.
+     *
+     * Typed as the CLI's own `ScoreStatus` rather than a second enum, so the
+     * trial line and the single-run headline cannot drift apart.
+     */
+    verdict: ScoreStatus;
     seconds: number;
     /** Failing-criteria summary ("a · b"), absent when none were reported. */
     note?: string;

package/dist/src/runner/groupRender.js

@@ -45,20 +45,40 @@ export function trialRowLine(n, row) {
     if (row.kind === "errored") {
         return `trial ${n}  ⚠  ${"errored".padEnd(16)}${row.reason} — excluded`;
     }
-    const mark = row.passed ? "✓" : "✗";
+    // A dash for the ungradable trial: it ran, and it asserts nothing. Reusing
+    // ✗ would make a grader gap look like the agent's failure at a glance, which
+    // is the whole reading F-925 removes.
+    const mark = row.verdict === "pass" ? "✓" : row.verdict === "incomplete" ? "–" : "✗";
     const base = `trial ${n}  ${mark}  ${String(row.score).padEnd(9)}${row.seconds.toFixed(1)}s`;
     return row.note ? `${base}  ${row.note}` : base;
 }
 export function groupSummaryLines(input) {
     const completed = input.rows.filter((r) => r.kind === "completed");
-    const passed = completed.filter((r) => r.passed).length;
+    const passed = completed.filter((r) => r.verdict === "pass").length;
+    const incomplete = completed.filter((r) => r.verdict === "incomplete").length;
+    // F-925 — the fraction's denominator is the GRADED trials. A trial that
+    // finalized but could not be fully graded leaves both the numerator and the
+    // denominator, so a 5-trial set with one of them reads "3 of 4", never the
+    // "4 of 5" that counted it as a pass.
+    const graded = completed.length - incomplete;
     const errored = input.rows.length - completed.length;
     const lines = ["─────"];
-    // The fraction counts COMPLETED trials only; errored trials are named and
-    // excluded, never silently folded into the denominator.
-    let fraction = completed.length === 0
-        ? "no trials completed"
-        : `${passed} of ${completed.length} passed`;
+    // The fraction counts GRADED trials only; incomplete and errored trials are
+    // named and excluded, never silently folded into the denominator. They stay
+    // two clauses rather than one: "the trial died" is ours to retry, "the trial
+    // ran and could not be graded" is a grader gap, and a reader told the wrong
+    // one goes looking in the wrong place.
+    let fraction;
+    if (graded === 0) {
+        fraction =
+            completed.length === 0 ? "no trials completed" : "no trials could be graded";
+    }
+    else {
+        fraction = `${passed} of ${graded} passed`;
+    }
+    if (incomplete > 0) {
+        fraction += ` · ${incomplete} incomplete, excluded from the fraction`;
+    }
     if (errored > 0) {
         fraction += ` · ${errored} errored, excluded from the fraction`;
     }
@@ -85,7 +105,11 @@ export function groupExitCode(rows) {
     const completed = rows.filter((r) => r.kind === "completed");
     if (completed.length === 0)
         return 2;
-    return completed.every((r) => r.passed) ? 0 : 1;
+    // F-925 — an ungradable trial is not a pass, so a group holding one cannot
+    // exit 0: green here would tell CI the set was verified when part of it was
+    // never checked. It stays 1 rather than 2 even when EVERY trial was
+    // incomplete — `2` means nothing completed, and these completed.
+    return completed.every((r) => r.verdict === "pass") ? 0 : 1;
 }
 /** Modal failed-criterion text across the group's completed trials, as the
  *  short phrase the "start there" line renders. */

package/dist/src/runner/runTaskHosted.d.ts

@@ -1,7 +1,7 @@
 import { type HostedClient } from "../hosted/client.js";
 import type { CreateSessionResponse } from "../types/shared.js";
 import type { Task } from "../task/taskSchema.js";
-import type { Score } from "../hosted/evalResultView.js";
+import { type Score, type ScoreStatus } from "../hosted/evalResultView.js";
 import type { RunArtifacts } from "../recorder/artifacts.js";
 export interface RunTaskHostedOptions {
     taskPath: string;
@@ -44,6 +44,13 @@ export interface RunTaskHostedResult {
     cloudDashboardUrl: string;
     artifacts: RunArtifacts;
     score: Score;
+    /**
+     * F-925 — the run's own three-state verdict, computed ONCE here and carried
+     * out so a caller never re-derives it. `exitCode` cannot express it: 1 means
+     * both "failed" and "could not be graded", and a trial group needs to tell
+     * them apart to keep the ungradable one out of its fraction.
+     */
+    verdict: ScoreStatus;
     exitCode: number;
     /** Wall time from run start to post-agent state capture — the same value
      *  reported to /finalize as duration_ms. FDRS-636 renders it as the trial

package/dist/src/runner/runTaskHosted.js

@@ -13,6 +13,7 @@ import { ensureMcpSuffix } from "../cli/session.js";
 import { redactJsonl, scoreFromFinalizeResponse, uploadRunBlobs, } from "../hosted/uploadAndFinalize.js";
 import { resolveRunAgentIdentity } from "../cli/agent-identity.js";
 import { HostedOrchError, HostedTrialError } from "../hosted/errors.js";
+import { scoreStatus, } from "../hosted/evalResultView.js";
 /** Build the agent subprocess env for a hosted run.
  *
  * Two layers, in this order:
@@ -491,7 +492,15 @@ export async function runTaskHosted(options) {
         //     Pre-finalize agent failures (auth, quota, twin spawn, exec
         //     errors) take other code paths via thrown HostedAuthError /
         //     HostedQuotaError / HostedOrchError and never reach this line.
-        const exitCode = finalized.score >= scenario.config.passThreshold ? 0 : 1;
+        //     F-925 — the score alone is not the verdict. A 100/100 run with a
+        //     criterion that never ran is `incomplete`, and exiting 0 on it is how
+        //     "the fix passed" came to mean "the check never ran". `scoreStatus`
+        //     applies the A5 guard `pome eval` has always applied; the FDRS-618
+        //     compat this used to preserve is already preserved inside
+        //     `scoreFromFinalizeResponse`, which sets can_pass true when the
+        //     response carries no criteria_results at all.
+        const verdict = scoreStatus(score, scenario.config.passThreshold);
+        const exitCode = verdict === "pass" ? 0 : 1;
         // 11. FDRS-644 — cache the CLOUD verdict payload next to the raw trace
         //     (verdict.json, provenance-labeled `source: "cloud-finalize"`).
         //     Not a local score: evaluation stayed in the cloud; this records
@@ -528,6 +537,7 @@ export async function runTaskHosted(options) {
             cloudDashboardUrl: finalized.dashboard_url,
             artifacts,
             score,
+            verdict,
             exitCode,
             durationMs,
         };