@pome-sh/cli 0.17.0 → 0.18.0

package/README.md

@@ -64,22 +64,30 @@ code is the verdict**. Gate CI on it directly.
 | Exit code | Meaning |
 | --- | --- |
 | `0` | pass (hosted/scored run), or trace captured (`--local`, not scored) |
-| `1` | ran and scored **below** the pass threshold |
+| `1` | ran and scored **below** the pass threshold, **or ran `INCOMPLETE`** |
 | `2` | twin / orchestration error (network, 5xx, twin spawn failed) |
 | `3` | auth error (401/403) — `pome login` again, or set `POME_API_KEY` in CI |
 | `4` | quota exceeded (402/429) |
 | `5` | usage error (bad flags, missing task file) |
 
-Two rules CI must honor:
+Three rules CI must honor:
 
 - **`--local` is not a verdict.** A `--local` run captures a raw trace and never
   scores, so its exit `0` means "trace captured," not "passed." Never gate CI on
   a `--local` exit code — score it later with `pome eval <run-dir>`.
+- **`INCOMPLETE` shares exit `1`, and it is not the agent's failure.** A run
+  whose criteria could not all be graded exits `1` rather than mapping its
+  partial score to a code — a run whose checks never ran is not a green CI
+  signal. The cost is stated rather than hidden: **`1` cannot tell "the agent
+  regressed" from "we could not grade it."** Read the verdict word printed
+  beside the score (`INCOMPLETE` vs a sub-threshold number) to separate them.
 - **Trial groups map as a whole.** `pome run -n k` (k>1) collapses the whole
   group to one code: `0` = at least one trial completed and every completed
-  trial passed; `1` = at least one completed trial failed its threshold; `2` =
-  no trial completed. Errored trials are excluded from the verdict fraction and
-  never drag a passing group below `0` on their own.
+  trial passed; `1` = at least one completed trial failed its threshold **or was
+  incomplete**; `2` = no trial completed. Errored and incomplete trials are
+  excluded from the verdict fraction (`3 of 4 passed · 1 incomplete`) so neither
+  is counted as a pass nor charged to the agent as a loss — but a group holding
+  one cannot exit `0`.
 
 ## Development
 

package/dist/build-info.json

@@ -1,6 +1,6 @@
 {
   "package": "pome-sh",
-  "version": "0.17.0",
-  "git_sha": "46758262874f65435e8bfcff58f3881be4f731ca",
-  "build_time": "2026-08-03T08:56:27.021Z"
+  "version": "0.18.0",
+  "git_sha": "285d48dcd654088387b40e8d36a7a2bf8ffbb599",
+  "build_time": "2026-08-03T13:39:43.286Z"
 }

package/dist/src/cli/eval.js

@@ -296,7 +296,7 @@ export async function runEval(options) {
         return client.finalize(sid, {
             stopReason: "eval_upload",
             // /finalize's schema requires an integer exit_code, so `null` (agent
-            // timed out, or meta.json lacked the field) cannot pass through.
+            // timed out, or meta.json lacked the field) is not a legal value.
             // Send -1 as the explicit "unknown" sentinel — never a fabricated 0,
             // which would report a clean agent exit the trace can't vouch for.
             exitCode: artifacts.meta.exitCode ?? -1,
@@ -340,14 +340,16 @@ export async function runEval(options) {
     // never persisted next to the trace. Local artifacts stay trace-only (no
     // score.json), and the verdict lives in the cloud (see the dashboard URL).
     const score = scoreFromFinalizeResponse(finalized);
-    // Exit-code policy — DELIBERATE DIVERGENCE from hosted `pome run`
-    // (FDRS-618): `pome run` maps the raw cloud score (score >= threshold →
-    // 0), because pre-FDRS-618 cloud builds don't emit criteria_results and
-    // the cloud exit decision is documented as score-only. `pome eval` is a
-    // NEW command with no such compatibility surface, so it adopts the full
-    // FDRS-591/611 A5 guard up front: exit 0 ONLY when the run was evaluated,
-    // every criterion was judged (can_pass), AND the score clears the
-    // threshold. An UNEVAL verdict (e.g. all criteria skipped) exits 1.
+    // Exit-code policy — the full FDRS-591/611 A5 guard: exit 0 ONLY when the run
+    // was evaluated, every criterion was judged (can_pass), AND the score clears
+    // the threshold. An INCOMPLETE verdict (any criterion not evaluated) exits 1.
+    //
+    // F-925 retired the divergence that used to be documented here. `pome run`
+    // mapped the raw cloud score because "pre-FDRS-618 cloud builds don't emit
+    // criteria_results" — but `scoreFromFinalizeResponse` already handles that
+    // case (`hasCriteriaResults ? … : true`), so the guard degrades to score-only
+    // for exactly those builds on its own. The divergence was protecting a case
+    // its own helper already protected, and the two commands now agree.
     const exitCode = scoreStatus(score, EVAL_PASS_THRESHOLD) === "pass" ? 0 : 1;
     return {
         taskName,
@@ -403,7 +405,7 @@ export async function runEvalCommand(runDirArg, opts) {
         });
         // Same verdict shape as hosted `pome run`: LABEL, score line, cloud URL.
         const status = scoreStatus(result.score, EVAL_PASS_THRESHOLD);
-        const label = status === "pass" ? "PASS" : status === "fail" ? "FAIL" : "UNEVAL";
+        const label = status === "pass" ? "PASS" : status === "fail" ? "FAIL" : "INCOMPLETE";
         console.error(`${label} ${result.taskName}`);
         console.error(`  ${runScoreLine(result.score, EVAL_PASS_THRESHOLD, "cloud score")}`);
         if (result.score.results.length > 0) {

package/dist/src/cli/main.js

@@ -660,7 +660,7 @@ export function createProgram() {
                         agentVersion: options.agentVersion,
                     });
                     const status = scoreStatus(result.score, result.scenario.config.passThreshold);
-                    const label = status === "pass" ? "PASS" : status === "fail" ? "FAIL" : "UNEVAL";
+                    const label = status === "pass" ? "PASS" : status === "fail" ? "FAIL" : "INCOMPLETE";
                     console.error(`${label} ${result.scenario.title}`);
                     console.error(`  ${runScoreLine(result.score, result.scenario.config.passThreshold, "cloud score")}`);
                     console.error(`  local: ${result.artifacts.runDir}`);

package/dist/src/hosted/evalResultView.d.ts

@@ -29,7 +29,7 @@ export type Score = {
     judge_tokens_out: number | null;
 };
 export declare function outcomeOf(result: CriterionResult): CriterionOutcome;
-export type ScoreStatus = "pass" | "fail" | "unevaluated";
+export type ScoreStatus = "pass" | "fail" | "incomplete";
 export declare function scoreStatus(score: Score, passThreshold: number): ScoreStatus;
 export declare function taskPassed(score: Score, passThreshold: number): boolean;
 export declare function markerFor(outcome: CriterionOutcome): string;

package/dist/src/hosted/evalResultView.js

@@ -31,9 +31,20 @@ export function outcomeOf(result) {
 // Encodes the A5 guard: a run is only a PASS when it was evaluated, every
 // required criterion was evaluated (can_pass), AND satisfaction cleared the
 // threshold. PURE — no computation of the score itself.
+//
+// F-932 renamed the third state from `unevaluated` to `incomplete` and CHANGED
+// NOTHING ELSE HERE. The guard is the one place the CLI refuses to inflate a
+// partial run into a pass — the same refusal pome-cloud added server-side in
+// F-925 — so the rename must not become a loosening.
+//
+// One rule, two repos: `can_pass` is false for ANY abstention
+// (`uploadAndFinalize.ts`), and pome-cloud's `isRunIncomplete` says
+// `notEvaluated > 0` over the same `criteria_results`. Deliberately NOT read
+// from the wire's `all_skipped`, which is the narrower every-abstained
+// predicate and would loosen this guard.
 export function scoreStatus(score, passThreshold) {
     if (!score.evaluated || !score.can_pass)
-        return "unevaluated";
+        return "incomplete";
     return score.satisfaction >= passThreshold ? "pass" : "fail";
 }
 export function taskPassed(score, passThreshold) {
@@ -55,14 +66,14 @@ export function markerFor(outcome) {
 // Multi-twin (M3): the per-criterion bracket for terminal display —
 // `[code]` / `[model]`, plus the `:<twin>` suffix when the criterion attributes
 // to a specific twin (so a `[code:slack]`/`[model:github]` marker survives into the
-// UNEVAL / criteria list). A bare (primary-twin) criterion renders `[code]`
+// INCOMPLETE / criteria list). A bare (primary-twin) criterion renders `[code]`
 // unchanged.
 export function criterionMarkerLabel(criterion) {
     return criterion.twin ? `[${criterion.type}:${criterion.twin}]` : `[${criterion.type}]`;
 }
 // Multi-twin (M3): when the cloud could not evaluate a criterion for a
 // twin-related reason (a twin-tagged criterion, or a `no_matching_predicate`
-// skip), name the twin inline so the UNEVAL line explains WHICH twin's timeline
+// skip), name the twin inline so the INCOMPLETE line explains WHICH twin's timeline
 // came up empty. Returns "" when there's nothing twin-specific to add.
 export function twinSkipSuffix(result) {
     const twin = result.criterion.twin;
@@ -79,8 +90,14 @@ export function scoreCountsSummary(score) {
 }
 export function runScoreLine(score, passThreshold, unevaluatedNumericLabel) {
     const status = scoreStatus(score, passThreshold);
-    if (status === "unevaluated") {
-        return `score: un-evaluated (cannot pass) — ${scoreCountsSummary(score)}; ${unevaluatedNumericLabel}: ${score.satisfaction}/100`;
+    if (status === "incomplete") {
+        // Leads with the COUNT, which is the fact the reader needs and the same
+        // fact the cloud's own header now states. The old copy said "cannot pass",
+        // which is a verdict about the AGENT for a gap in the GRADER — the exact
+        // inversion F-925 exists to stop, one surface over.
+        const notEvaluated = score.skipped + score.errored;
+        const total = score.total_required + notEvaluated;
+        return `score: incomplete — ${notEvaluated} of ${total} criteria not evaluated; ${scoreCountsSummary(score)}; ${unevaluatedNumericLabel}: ${score.satisfaction}/100`;
     }
     return `score: ${score.satisfaction}/100`;
 }

package/dist/src/runner/groupRender.d.ts

@@ -1,9 +1,19 @@
+import type { ScoreStatus } from "../hosted/evalResultView.js";
 export type TrialRow = {
     kind: "completed";
     /** Cloud-authoritative satisfaction score, 0-100. */
     score: number;
-    /** Cleared the scenario's pass threshold. */
-    passed: boolean;
+    /**
+     * F-925 — three states, not a boolean. `incomplete` means the trial ran
+     * and finalized but at least one criterion never produced a verdict, so
+     * it is neither a pass nor the agent's failure. It was `passed: boolean`
+     * fed from `exitCode === 0`, which counted a 100/100 run with 3 of 4
+     * criteria skipped as a clean passing trial.
+     *
+     * Typed as the CLI's own `ScoreStatus` rather than a second enum, so the
+     * trial line and the single-run headline cannot drift apart.
+     */
+    verdict: ScoreStatus;
     seconds: number;
     /** Failing-criteria summary ("a · b"), absent when none were reported. */
     note?: string;

package/dist/src/runner/groupRender.js

@@ -45,20 +45,40 @@ export function trialRowLine(n, row) {
     if (row.kind === "errored") {
         return `trial ${n}  ⚠  ${"errored".padEnd(16)}${row.reason} — excluded`;
     }
-    const mark = row.passed ? "✓" : "✗";
+    // A dash for the ungradable trial: it ran, and it asserts nothing. Reusing
+    // ✗ would make a grader gap look like the agent's failure at a glance, which
+    // is the whole reading F-925 removes.
+    const mark = row.verdict === "pass" ? "✓" : row.verdict === "incomplete" ? "–" : "✗";
     const base = `trial ${n}  ${mark}  ${String(row.score).padEnd(9)}${row.seconds.toFixed(1)}s`;
     return row.note ? `${base}  ${row.note}` : base;
 }
 export function groupSummaryLines(input) {
     const completed = input.rows.filter((r) => r.kind === "completed");
-    const passed = completed.filter((r) => r.passed).length;
+    const passed = completed.filter((r) => r.verdict === "pass").length;
+    const incomplete = completed.filter((r) => r.verdict === "incomplete").length;
+    // F-925 — the fraction's denominator is the GRADED trials. A trial that
+    // finalized but could not be fully graded leaves both the numerator and the
+    // denominator, so a 5-trial set with one of them reads "3 of 4", never the
+    // "4 of 5" that counted it as a pass.
+    const graded = completed.length - incomplete;
     const errored = input.rows.length - completed.length;
     const lines = ["─────"];
-    // The fraction counts COMPLETED trials only; errored trials are named and
-    // excluded, never silently folded into the denominator.
-    let fraction = completed.length === 0
-        ? "no trials completed"
-        : `${passed} of ${completed.length} passed`;
+    // The fraction counts GRADED trials only; incomplete and errored trials are
+    // named and excluded, never silently folded into the denominator. They stay
+    // two clauses rather than one: "the trial died" is ours to retry, "the trial
+    // ran and could not be graded" is a grader gap, and a reader told the wrong
+    // one goes looking in the wrong place.
+    let fraction;
+    if (graded === 0) {
+        fraction =
+            completed.length === 0 ? "no trials completed" : "no trials could be graded";
+    }
+    else {
+        fraction = `${passed} of ${graded} passed`;
+    }
+    if (incomplete > 0) {
+        fraction += ` · ${incomplete} incomplete, excluded from the fraction`;
+    }
     if (errored > 0) {
         fraction += ` · ${errored} errored, excluded from the fraction`;
     }
@@ -85,7 +105,11 @@ export function groupExitCode(rows) {
     const completed = rows.filter((r) => r.kind === "completed");
     if (completed.length === 0)
         return 2;
-    return completed.every((r) => r.passed) ? 0 : 1;
+    // F-925 — an ungradable trial is not a pass, so a group holding one cannot
+    // exit 0: green here would tell CI the set was verified when part of it was
+    // never checked. It stays 1 rather than 2 even when EVERY trial was
+    // incomplete — `2` means nothing completed, and these completed.
+    return completed.every((r) => r.verdict === "pass") ? 0 : 1;
 }
 /** Modal failed-criterion text across the group's completed trials, as the
  *  short phrase the "start there" line renders. */

package/dist/src/runner/runTaskHosted.d.ts

@@ -1,7 +1,7 @@
 import { type HostedClient } from "../hosted/client.js";
 import type { CreateSessionResponse } from "../types/shared.js";
 import type { Task } from "../task/taskSchema.js";
-import type { Score } from "../hosted/evalResultView.js";
+import { type Score, type ScoreStatus } from "../hosted/evalResultView.js";
 import type { RunArtifacts } from "../recorder/artifacts.js";
 export interface RunTaskHostedOptions {
     taskPath: string;
@@ -44,6 +44,13 @@ export interface RunTaskHostedResult {
     cloudDashboardUrl: string;
     artifacts: RunArtifacts;
     score: Score;
+    /**
+     * F-925 — the run's own three-state verdict, computed ONCE here and carried
+     * out so a caller never re-derives it. `exitCode` cannot express it: 1 means
+     * both "failed" and "could not be graded", and a trial group needs to tell
+     * them apart to keep the ungradable one out of its fraction.
+     */
+    verdict: ScoreStatus;
     exitCode: number;
     /** Wall time from run start to post-agent state capture — the same value
      *  reported to /finalize as duration_ms. FDRS-636 renders it as the trial

package/dist/src/runner/runTaskHosted.js

@@ -13,6 +13,7 @@ import { ensureMcpSuffix } from "../cli/session.js";
 import { redactJsonl, scoreFromFinalizeResponse, uploadRunBlobs, } from "../hosted/uploadAndFinalize.js";
 import { resolveRunAgentIdentity } from "../cli/agent-identity.js";
 import { HostedOrchError, HostedTrialError } from "../hosted/errors.js";
+import { scoreStatus, } from "../hosted/evalResultView.js";
 /** Build the agent subprocess env for a hosted run.
  *
  * Two layers, in this order:
@@ -491,7 +492,15 @@ export async function runTaskHosted(options) {
         //     Pre-finalize agent failures (auth, quota, twin spawn, exec
         //     errors) take other code paths via thrown HostedAuthError /
         //     HostedQuotaError / HostedOrchError and never reach this line.
-        const exitCode = finalized.score >= scenario.config.passThreshold ? 0 : 1;
+        //     F-925 — the score alone is not the verdict. A 100/100 run with a
+        //     criterion that never ran is `incomplete`, and exiting 0 on it is how
+        //     "the fix passed" came to mean "the check never ran". `scoreStatus`
+        //     applies the A5 guard `pome eval` has always applied; the FDRS-618
+        //     compat this used to preserve is already preserved inside
+        //     `scoreFromFinalizeResponse`, which sets can_pass true when the
+        //     response carries no criteria_results at all.
+        const verdict = scoreStatus(score, scenario.config.passThreshold);
+        const exitCode = verdict === "pass" ? 0 : 1;
         // 11. FDRS-644 — cache the CLOUD verdict payload next to the raw trace
         //     (verdict.json, provenance-labeled `source: "cloud-finalize"`).
         //     Not a local score: evaluation stayed in the cloud; this records
@@ -528,6 +537,7 @@ export async function runTaskHosted(options) {
             cloudDashboardUrl: finalized.dashboard_url,
             artifacts,
             score,
+            verdict,
             exitCode,
             durationMs,
         };

package/dist/src/runner/runTrialGroup.js

@@ -179,7 +179,14 @@ export async function runTrialGroup(options) {
             row = {
                 kind: "completed",
                 score: result.score.satisfaction,
-                passed: result.exitCode === 0,
+                // F-925 — the trial's own verdict, carried out of the run rather than
+                // re-derived here. It was `result.exitCode === 0`, which cannot express
+                // the third state (1 means both "failed" and "could not be graded"), so
+                // a 100/100 trial with 3 of 4 criteria skipped counted as a clean pass.
+                // Reusing the run's own value rather than calling `scoreStatus` again
+                // on the same inputs is what keeps the trial line and the single-run
+                // headline from ever disagreeing.
+                verdict: result.verdict,
                 seconds: result.durationMs / 1000,
                 note: failing.length > 0
                     ? failing.map((r) => criterionPhrase(r.criterion.text)).join(" · ")
@@ -235,7 +242,13 @@ export async function runTrialGroup(options) {
     // 4. FDRS-644 — the fix & green handoff, only when a COMPLETED trial
     // failed. Errored trials are sandbox noise: the answer there is re-run,
     // not a code fix, so an errored-only group gets no handoff.
-    const failedCompleted = rows.filter((r) => r.kind === "completed" && !r.passed).length;
+    //
+    // F-925 — `fail`, NOT "anything that isn't a pass". This was `!r.passed`,
+    // which now includes the incomplete trial, and pointing someone at
+    // `pome fix-prompt` for a criterion that never ran tells them to fix an agent
+    // that may be blameless. An abstention is a grader gap; the handoff is for
+    // agent defects.
+    const failedCompleted = rows.filter((r) => r.kind === "completed" && r.verdict === "fail").length;
     if (failedCompleted > 0) {
         const artifactsDir = options.artifactsDir ?? "runs";
         const fixPromptCommand = artifactsDir === "runs"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pome-sh/cli",
-  "version": "0.17.0",
+  "version": "0.18.0",
   "description": "Digital-twin testing for AI agents — run tasks against resettable local or hosted twins and record tool-call traces for evaluation on pome.sh.",
   "keywords": [
     "ai",