@pome-sh/cli 0.23.4 → 0.23.5

package/README.md

@@ -99,6 +99,16 @@ Three rules CI must honor:
   excluded from the verdict fraction (`3 of 4 passed · 1 incomplete`) so neither
   is counted as a pass nor charged to the agent as a loss — but a group holding
   one cannot exit `0`.
+- **`pome fix-prompt` uses the same codes, and its `1` is only ever
+  INCOMPLETE.** Building a prompt for a failed run set exits `0` (the prompt is
+  on stdout, and stdout being non-empty is the signal that there was something
+  to fix); an all-green root exits `0` with nothing on stdout; a bad argument or
+  a root with no readable run sets exits `5`. `1` is reserved for the one case
+  where the newest non-passing set was never fully graded: no prompt is built,
+  because a run whose checks never ran is not evidence of an agent defect. This
+  matches `pome run`, where `1` also covers INCOMPLETE — the two commands do not
+  disagree about what an ungraded run exits, and `verdict.json`'s `state` stays
+  the field to read when a script needs the reason rather than the code.
 
 ## Development
 

package/dist/build-info.json

@@ -1,6 +1,6 @@
 {
   "package": "pome-sh",
-  "version": "0.23.4",
-  "git_sha": "33793129a877db31c9ef5c777f5dd45c2f6d1a8c",
-  "build_time": "2026-08-10T14:48:52.274Z"
+  "version": "0.23.5",
+  "git_sha": "4f55e6deab92cf0ff34dd769c73763fc9b480375",
+  "build_time": "2026-08-10T15:36:42.602Z"
 }

package/dist/{chunk-LRNQ3IDA.js → chunk-RSQY6UIL.js}

@@ -117,13 +117,15 @@ function groupRunSets(trials) {
       (a, b) => a.verdict.finalized_at.localeCompare(b.verdict.finalized_at)
     );
     const last = bucket[bucket.length - 1];
+    const hasFailed = bucket.some((t) => t.verdict.state === "fail");
+    const allPassed = bucket.every((t) => t.verdict.state === "pass");
     sets.push({
       groupId: bucket[0].verdict.group_id,
       taskName: bucket[0].verdict.task_name,
       taskPath: bucket[0].verdict.task_path,
       trials: bucket,
       latestFinalizedAt: last.verdict.finalized_at,
-      anyFailed: bucket.some((t) => !t.verdict.passed)
+      outcome: hasFailed ? "fail" : allPassed ? "pass" : "incomplete"
     });
   }
   sets.sort((a, b) => a.latestFinalizedAt.localeCompare(b.latestFinalizedAt));
@@ -131,14 +133,26 @@ function groupRunSets(trials) {
 }
 function latestFailedRunSet(sets) {
   for (let i = sets.length - 1; i >= 0; i -= 1) {
-    if (sets[i].anyFailed) return sets[i];
+    if (sets[i].outcome === "fail") return sets[i];
+  }
+  return null;
+}
+function latestIncompleteRunSet(sets) {
+  for (let i = sets.length - 1; i >= 0; i -= 1) {
+    if (sets[i].outcome === "incomplete") return sets[i];
   }
   return null;
 }
 async function discoverRunSet(target) {
   const anchorResult = await readVerdictArtifactDetailed(target);
   if (anchorResult.status === "stale-version") {
-    return { kind: "trial-dir", set: null, totalSets: 0, staleVersionCount: 1 };
+    return {
+      kind: "trial-dir",
+      set: null,
+      incompleteSet: null,
+      totalSets: 0,
+      staleVersionCount: 1
+    };
   }
   if (anchorResult.status === "ok") {
     const anchor = anchorResult.trial;
@@ -151,18 +165,27 @@ async function discoverRunSet(target) {
     return {
       kind: "trial-dir",
       set: own,
+      incompleteSet: null,
       totalSets: Math.max(sets2.length, 1),
       staleVersionCount: staleVersionDirs2.length
     };
   }
   if (!existsSync(target)) {
-    return { kind: "root", set: null, totalSets: 0, staleVersionCount: 0 };
+    return {
+      kind: "root",
+      set: null,
+      incompleteSet: null,
+      totalSets: 0,
+      staleVersionCount: 0
+    };
   }
   const { trials, staleVersionDirs } = await scanVerdictArtifactsDetailed(target);
   const sets = groupRunSets(trials);
+  const failedSet = latestFailedRunSet(sets);
   return {
     kind: "root",
-    set: latestFailedRunSet(sets),
+    set: failedSet,
+    incompleteSet: failedSet ? null : latestIncompleteRunSet(sets),
     totalSets: sets.length,
     staleVersionCount: staleVersionDirs.length
   };

package/dist/{runTrialGroup-AVNW3D3L.js → runTrialGroup-NIBA5KRF.js}

@@ -1,5 +1,5 @@
 import { newGroupId, criterionPhrase } from './chunk-RGZBC7NF.js';
-import { runTaskHosted, resolveRunAgentIdentity } from './chunk-LRNQ3IDA.js';
+import { runTaskHosted, resolveRunAgentIdentity } from './chunk-RSQY6UIL.js';
 import './chunk-XOWIA7NR.js';
 import { createHostedClient, parseTaskFile, outcomeOf } from './chunk-GFTFMA3T.js';
 import './chunk-NW7HGA2K.js';

package/dist/src/cli/main.js

@@ -1,10 +1,10 @@
 #!/usr/bin/env node
-import { DEFAULT_CONTROL_PLANE_URL, DEFAULT_DASHBOARD_URL, clearLocalCredentials, friendlyHostedError, runSessionCreate, runSessionList, runSessionStop, resolveCredentials, runTaskHosted, discoverRunSet, VERDICT_ARTIFACT_VERSION, loadTrialEvents, persistCredentialsAfterLogin, DEFAULT_DOCS_SITE_ORIGIN, resolveSeams, resolveCachedAgentId, readLinkCache, postAgentResolver, writeLinkCache, ensurePomeGitignored } from '../../chunk-LRNQ3IDA.js';
+import { DEFAULT_CONTROL_PLANE_URL, DEFAULT_DASHBOARD_URL, clearLocalCredentials, friendlyHostedError, runSessionCreate, runSessionList, runSessionStop, resolveCredentials, runTaskHosted, discoverRunSet, VERDICT_ARTIFACT_VERSION, loadTrialEvents, persistCredentialsAfterLogin, DEFAULT_DOCS_SITE_ORIGIN, resolveSeams, resolveCachedAgentId, readLinkCache, postAgentResolver, writeLinkCache, ensurePomeGitignored } from '../../chunk-RSQY6UIL.js';
 import { readManifest, writeManifest, MANIFEST_JSON, readRequiredManifest, normalizeManifestTwins } from '../../chunk-XOWIA7NR.js';
 import { assetPath, runTask, resolvePackageRoot, DEMO_TASK_NAME, demoTaskPath } from '../../chunk-YUKMLGYF.js';
 import '../../chunk-XDU6TD4O.js';
 import '../../chunk-CBFKZZBR.js';
-import { parseTaskFile, scoreStatus, runScoreLine, readLatestRun, readMetaSummary, readConfigTwins, scoreCountsSummary, markerFor, outcomeOf, criterionMarkerLabel, twinSkipSuffix, readCodeCriteria, createHostedClient, toTwinHttpEvent, redactJsonl, scoreFromFinalizeResponse, parseGitHubSeedState, uploadRunBlobs, isPreSatisfied } from '../../chunk-GFTFMA3T.js';
+import { parseTaskFile, scoreStatus, runScoreLine, readLatestRun, readMetaSummary, outcomeOf, readConfigTwins, scoreCountsSummary, markerFor, criterionMarkerLabel, twinSkipSuffix, readCodeCriteria, createHostedClient, toTwinHttpEvent, redactJsonl, scoreFromFinalizeResponse, parseGitHubSeedState, uploadRunBlobs, isPreSatisfied } from '../../chunk-GFTFMA3T.js';
 import '../../chunk-NW7HGA2K.js';
 import { MOUNTED_TWINS, deriveAgentSlug, exitCodeFor, HostedUsageError, HostedOrchError, HostedAuthError, HostedQuotaError } from '../../chunk-7AMIVWUB.js';
 import { TAPE_ASSERTABLE_TOOLS } from '../../chunk-FKZZWWYC.js';
@@ -4348,6 +4348,14 @@ function criterionMarker(c) {
 function failedResults(verdict) {
   return verdict.criteria_results.filter((r) => outcomeOf(r) === "failed");
 }
+function isGraded(t) {
+  return t.verdict.state !== "incomplete";
+}
+function ungradedCount(verdict) {
+  return verdict.criteria_results.filter(
+    (r) => !isPreSatisfied(r) && outcomeOf(r) !== "passed" && outcomeOf(r) !== "failed"
+  ).length;
+}
 function flattenLine(text, max = 300) {
   const flat = text.replace(/\s+/g, " ").trim();
   return flat.length > max ? `${flat.slice(0, max - 1)}\u2026` : flat;
@@ -4355,10 +4363,14 @@ function flattenLine(text, max = 300) {
 function renderGroupedSignatures(trials) {
   const byCriterion = /* @__PURE__ */ new Map();
   const outcomesSeen = /* @__PURE__ */ new Map();
+  const gradedFor = /* @__PURE__ */ new Map();
   for (const trial of trials) {
     for (const result of trial.verdict.criteria_results) {
       const key = result.criterion.text;
       const outcome = isPreSatisfied(result) ? "excluded" : outcomeOf(result);
+      if (outcome === "passed" || outcome === "failed") {
+        gradedFor.set(key, (gradedFor.get(key) ?? 0) + 1);
+      }
       if (outcome === "failed") {
         const entry = byCriterion.get(key) ?? {
           marker: criterionMarker(result.criterion),
@@ -4381,12 +4393,12 @@ function renderGroupedSignatures(trials) {
     else if (seen.size === 1 && seen.has("excluded")) preSatisfiedEverywhere.push(key);
     else notUniformlyEvaluated.push(key);
   }
-  const completed = trials.length;
   const blocks = [...byCriterion.entries()].sort((a, b) => b[1].hits.length - a[1].hits.length).map(([text, { marker, hits }], idx) => {
     const lines = hits.map(
       (h) => `   - ${h.label}: ${flattenLine(h.reason)}`
     );
-    return `${idx + 1}. ${marker} ${flattenLine(text)} \u2014 failed in ${hits.length} of ${completed} completed trials
+    const graded = gradedFor.get(text) ?? hits.length;
+    return `${idx + 1}. ${marker} ${flattenLine(text)} \u2014 failed in ${hits.length} of ${graded} trials that graded it
 ${lines.join("\n")}`;
   });
   if (blocks.length === 0 && passedEverywhere.length === 0 && preSatisfiedEverywhere.length === 0 && notUniformlyEvaluated.length === 0) {
@@ -4414,18 +4426,19 @@ ${lines.join("\n")}`;
   return [...blocks, ...notes].join("\n");
 }
 function representativeFailingTrial(trials) {
-  const failing = trials.filter((t) => !t.verdict.passed);
+  const failing = trials.filter((t) => t.verdict.state === "fail");
   if (failing.length === 0) return null;
   return failing.reduce(
     (worst, t) => failedResults(t.verdict).length > failedResults(worst.verdict).length ? t : worst
   );
 }
 function buildGroupFixUserPrompt(ctx) {
-  const completed = ctx.trials.length;
-  const passed = ctx.trials.filter((t) => t.verdict.passed).length;
+  const incomplete = ctx.trials.filter((t) => !isGraded(t));
+  const completed = ctx.trials.length - incomplete.length;
+  const passed = ctx.trials.filter((t) => t.verdict.state === "pass").length;
   const representative = representativeFailingTrial(ctx.trials);
   const otherFailing = ctx.trials.filter(
-    (t) => !t.verdict.passed && t !== representative
+    (t) => t.verdict.state === "fail" && t !== representative
   );
   const signatures = redactSecrets(
     renderGroupedSignatures(ctx.trials)
@@ -4439,9 +4452,11 @@ function buildGroupFixUserPrompt(ctx) {
     )
   );
   const promptBlock = ctx.task ? redactSecrets(ctx.task.prompt) : `(task file not found at ${ctx.trials[0]?.verdict.task_path ?? "?"} \u2014 criteria above come from the cloud verdicts)`;
+  const tally = completed === 0 ? "no trial in this set was graded end to end" : `${passed} of ${completed} completed trials passed`;
+  const gapNote = incomplete.length > 0 ? ` \xB7 ${incomplete.length} INCOMPLETE (counted in nothing below \u2014 see the last section)` : "";
   const sections = [];
   sections.push(`## Run set (cloud-judged)
-task ${redactSecrets(ctx.taskName)} \xB7 ${ctx.groupId ? `group ${ctx.groupId}` : "single run"} \xB7 ${passed} of ${completed} completed trials passed`);
+task ${redactSecrets(ctx.taskName)} \xB7 ${ctx.groupId ? `group ${ctx.groupId}` : "single run"} \xB7 ${tally}${gapNote}`);
   sections.push(`## Grouped failure signatures (from the cloud judge)
 ${escapeTagContent(signatures)}`);
   sections.push(`## Task prompt (what the agent was told to do)
@@ -4463,6 +4478,17 @@ ${escapeTagContent(trace)}
       return `- ${t.label} \u2014 failed: ${failed || "(see verdict)"} \u2014 trace at ${join(t.runDir, "events.jsonl")}`;
     });
     sections.push(`## Other failing trials (traces on disk)
+${escapeTagContent(redactSecrets(lines.join("\n")))}`);
+  }
+  if (incomplete.length > 0) {
+    const lines = incomplete.map(
+      (t) => `- ${t.label} \u2014 ${ungradedCount(t.verdict)} criterion(s) never graded \u2014 trace at ${join(t.runDir, "events.jsonl")}`
+    );
+    sections.push(`## Trials the grader never finished (INCOMPLETE)
+The grader never reached every criterion in these trials, so they are neither
+passes nor failures and are counted in no fraction above. Do NOT treat them as
+evidence for or against any fix: a criterion that never ran is a grader or seed
+gap, not something the agent did wrong.
 ${escapeTagContent(redactSecrets(lines.join("\n")))}`);
   }
   if (passed > 0 && passed < completed) {
@@ -4495,7 +4521,7 @@ var DEFAULT_AGENT_FILE = "examples/agents/scripted-triage-agent.ts";
 var DEFAULT_AGENT_COMMAND = `node ${DEFAULT_AGENT_FILE}`;
 var MANIFEST_SCHEMA_URL = "https://pome.sh/schemas/v1/pome.json";
 function readPackageVersion() {
-  if ("0.23.4".length > 0) return "0.23.4";
+  if ("0.23.5".length > 0) return "0.23.5";
   try {
     const here = dirname(fileURLToPath(import.meta.url));
     const candidates = [
@@ -4988,7 +5014,7 @@ function createProgram() {
               taskForRuns.config.runs
             );
             if (k > 1) {
-              const { runTrialGroup } = await import('../../runTrialGroup-AVNW3D3L.js');
+              const { runTrialGroup } = await import('../../runTrialGroup-NIBA5KRF.js');
               const fileForRerun = relative(process.cwd(), file);
               const rerunCommand = defaultTask ? options.trials !== void 0 ? `pome run -n ${k}` : "pome run" : `pome run ${fileForRerun && !fileForRerun.startsWith("..") ? fileForRerun : file} -n ${k}`;
               const groupResult = await runTrialGroup({
@@ -5202,6 +5228,31 @@ function createProgram() {
       return;
     }
     if (!discovery.set) {
+      const incomplete = discovery.incompleteSet;
+      if (incomplete) {
+        const ungradedTrials = incomplete.trials.filter(
+          (t) => t.verdict.state === "incomplete"
+        ).length;
+        const gradedFailures = incomplete.trials.reduce(
+          (n, t) => n + t.verdict.criteria_results.filter((r) => outcomeOf(r) === "failed").length,
+          0
+        );
+        const which = `task ${incomplete.taskName}${incomplete.groupId ? ` \xB7 group ${incomplete.groupId}` : ""}`;
+        console.error(
+          `Not routed to fix-prompt: no run set under ${root} failed outright. The most recent non-passing one (${which}) is INCOMPLETE \u2014 ${ungradedTrials} of ${incomplete.trials.length} trial(s) have criteria the grader never graded.`
+        );
+        if (gradedFailures > 0) {
+          console.error(
+            `${gradedFailures} criterion result(s) in that set WERE graded and did fail, so this is not only a grading gap \u2014 but no trial in it was graded end to end, and a fix prompt built from a partial grading would claim more than was checked. Re-run \`pome run ${incomplete.taskPath}\` to grade the rest, or point fix-prompt straight at one trial (\`pome fix-prompt ${incomplete.trials[0].runDir}\`) to build one from the partial grading anyway.`
+          );
+        } else {
+          console.error(
+            `Nothing in that set was graded and failed, so it is a grader/seed gap, not an agent defect, and fix-prompt will not hand it to your coding agent. Re-run \`pome run ${incomplete.taskPath}\` to grade those criteria; if they come back ungraded, the gap is in the task's checks or its seed, not in your prompt.`
+          );
+        }
+        process.exitCode = 1;
+        return;
+      }
       console.error(
         `Nothing to fix: the latest run sets under ${root} all passed.`
       );

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pome-sh/cli",
-  "version": "0.23.4",
+  "version": "0.23.5",
   "description": "Digital-twin testing for AI agents \u2014 run tasks against resettable local or hosted twins and record tool-call traces for evaluation on pome.sh.",
   "keywords": [
     "ai",