@pome-sh/cli 0.23.1 → 0.23.2

package/README.md

@@ -79,8 +79,19 @@ Three rules CI must honor:
   whose criteria could not all be graded exits `1` rather than mapping its
   partial score to a code — a run whose checks never ran is not a green CI
   signal. The cost is stated rather than hidden: **`1` cannot tell "the agent
-  regressed" from "we could not grade it."** Read the verdict word printed
-  beside the score (`INCOMPLETE` vs a sub-threshold number) to separate them.
+  regressed" from "we could not grade it."** To separate them programmatically,
+  do not compare `score` against `pass_threshold` yourself — a run with a third
+  of its criteria unevaluated can still read `score: 100, pass_threshold: 100`
+  with nothing in those two fields alone saying so. Read `state` in the
+  `verdict.json` a hosted `pome run` writes to
+  `<artifacts-dir>/<task-slug>/<session-id>/verdict.json`: `"pass"`, `"fail"`,
+  or `"incomplete"` — the same word the terminal prints beside the score, and
+  the field to gate on. The `evaluated` / `not_evaluated` / `pre_satisfied` /
+  `total` counts alongside it say how much of the task `score` covers:
+  **`score` is a percentage over `evaluated` alone**, so `not_evaluated > 0`
+  means `score` is silent about part of the run, and `evaluated: 0` means it
+  scored nothing at all (the cloud sends `0` there for want of a denominator
+  — "nothing was scored", not "nothing was correct").
 - **Trial groups map as a whole.** `pome run -n k` (k>1) collapses the whole
   group to one code: `0` = at least one trial completed and every completed
   trial passed; `1` = at least one completed trial failed its threshold **or was

package/dist/build-info.json

@@ -1,6 +1,6 @@
 {
   "package": "pome-sh",
-  "version": "0.23.1",
-  "git_sha": "f57142b584d0bbeaa6b1c3e6194daf2723ed6d3c",
-  "build_time": "2026-08-10T12:44:52.803Z"
+  "version": "0.23.2",
+  "git_sha": "9fe20d1ba7f0f9b40cfb7561e4ceab144c22c71d",
+  "build_time": "2026-08-10T13:45:05.230Z"
 }

package/dist/{chunk-54OLERUK.js → chunk-IQ6LOK4D.js}

@@ -1,6 +1,6 @@
 import { getAvailablePort } from './chunk-XDU6TD4O.js';
 import { buildEgressAllowlist, readBlockedEgress } from './chunk-CBFKZZBR.js';
-import { parseTaskFile, seedStateForTwin, runAgentCommand, writeRunArtifactsCore } from './chunk-Z32CZ45D.js';
+import { parseTaskFile, seedStateForTwin, runAgentCommand, writeRunArtifactsCore } from './chunk-N4UTEHX5.js';
 import { createRecorder, bootTwin } from './chunk-TG22SRUL.js';
 import { eventSchema } from './chunk-VBATFCWR.js';
 import { redactSecrets, redactEvent } from './chunk-SG6ZTIMT.js';

package/dist/{chunk-Z32CZ45D.js → chunk-N4UTEHX5.js}

@@ -1265,15 +1265,23 @@ function twinSkipSuffix(result) {
 function criteriaWord(n) {
   return n === 1 ? "criterion" : "criteria";
 }
+function evaluationCounts(score) {
+  const evaluated = score.total_required;
+  const notEvaluated = score.skipped + score.errored - score.preSatisfied;
+  return {
+    evaluated,
+    notEvaluated,
+    preSatisfied: score.preSatisfied,
+    total: evaluated + notEvaluated + score.preSatisfied
+  };
+}
 function scoreCountsSummary(score) {
   return `${score.passed ?? 0} passed, ${score.failed ?? 0} failed, ${score.skipped ?? 0} skipped, ${score.errored ?? 0} errored`;
 }
 function runScoreLine(score, passThreshold, unevaluatedNumericLabel) {
   const status = scoreStatus(score, passThreshold);
   if (status === "incomplete") {
-    const allExcluded = score.skipped + score.errored;
-    const unreached = allExcluded - score.preSatisfied;
-    const total = score.total_required + allExcluded;
+    const { notEvaluated: unreached, total } = evaluationCounts(score);
     if (score.total_required === 0 && unreached === 0 && score.preSatisfied > 0) {
       return `score: incomplete \u2014 nothing was at risk (${score.preSatisfied} ${criteriaWord(score.preSatisfied)} already true in the seed); ${scoreCountsSummary(score)}; ${unevaluatedNumericLabel}: ${score.satisfaction}/100`;
     }
@@ -1630,4 +1638,4 @@ function splitCommand(command) {
   return file ? { file, args } : null;
 }
 
-export { createHostedClient, criterionMarkerLabel, isPreSatisfied, markerFor, outcomeOf, parseGitHubSeedState, parseTaskFile, perTwinReturnedByCloud, readCodeCriteria, readConfigTwins, readLatestRun, readMetaSummary, redactJsonl, runAgentCommand, runScoreLine, scoreCountsSummary, scoreFromFinalizeResponse, scoreStatus, seedStateForTwin, toTwinHttpEvent, twinSkipSuffix, uploadRunBlobs, writeRunArtifactsCore };
+export { createHostedClient, criterionMarkerLabel, evaluationCounts, isPreSatisfied, markerFor, outcomeOf, parseGitHubSeedState, parseTaskFile, perTwinReturnedByCloud, readCodeCriteria, readConfigTwins, readLatestRun, readMetaSummary, redactJsonl, runAgentCommand, runScoreLine, scoreCountsSummary, scoreFromFinalizeResponse, scoreStatus, seedStateForTwin, toTwinHttpEvent, twinSkipSuffix, uploadRunBlobs, writeRunArtifactsCore };

package/dist/{chunk-BCI7KJRH.js → chunk-NRECWO3Y.js}

@@ -1,5 +1,5 @@
 import { readManifest, normalizeManifestTwins } from './chunk-KUVTL4NZ.js';
-import { createHostedClient, perTwinReturnedByCloud, parseTaskFile, runAgentCommand, writeRunArtifactsCore, toTwinHttpEvent, redactJsonl, uploadRunBlobs, scoreFromFinalizeResponse, scoreStatus } from './chunk-Z32CZ45D.js';
+import { createHostedClient, perTwinReturnedByCloud, parseTaskFile, runAgentCommand, writeRunArtifactsCore, toTwinHttpEvent, redactJsonl, uploadRunBlobs, scoreFromFinalizeResponse, scoreStatus, evaluationCounts } from './chunk-N4UTEHX5.js';
 import { MOUNTED_TWINS, HostedAuthError, HostedDiscardRefusedError, HostedQuotaError, HostedOrchError, HostedTrialError, agentResponseSchema } from './chunk-PQYIAA6K.js';
 import { redactSecrets, redactEvent } from './chunk-SG6ZTIMT.js';
 import { existsSync } from 'node:fs';
@@ -11,8 +11,13 @@ import { promisify } from 'node:util';
 import { createInterface } from 'node:readline';
 import { randomUUID, createHash } from 'node:crypto';
 
-var VERDICT_ARTIFACT_VERSION = 1;
+var VERDICT_ARTIFACT_VERSION = 2;
 var VERDICT_FILENAME = "verdict.json";
+var VALID_STATES = /* @__PURE__ */ new Set([
+  "pass",
+  "fail",
+  "incomplete"
+]);
 async function writeVerdictArtifact(runDir, verdict) {
   await writeFile(
     join(runDir, VERDICT_FILENAME),
@@ -21,7 +26,7 @@ async function writeVerdictArtifact(runDir, verdict) {
     "utf8"
   );
 }
-function isVerdictArtifact(parsed) {
+function looksLikeVerdictArtifactBase(parsed) {
   if (typeof parsed !== "object" || parsed === null) return false;
   const v = parsed;
   if (v.source !== "cloud-finalize") return false;
@@ -40,33 +45,46 @@ function isVerdictArtifact(parsed) {
     return typeof criterion === "object" && criterion !== null && typeof criterion.text === "string" && typeof result.reason === "string" && typeof result.passed === "boolean" && typeof result.skipped === "boolean";
   });
 }
-function normalizeVerdictArtifact(parsed) {
-  const { scenario_path: legacyPath, task_path: taskPath, ...rest } = parsed;
-  return { ...rest, task_path: taskPath ?? legacyPath };
+function isVerdictArtifact(parsed) {
+  if (!looksLikeVerdictArtifactBase(parsed)) return false;
+  const v = parsed;
+  if (v.version !== VERDICT_ARTIFACT_VERSION) return false;
+  if (typeof v.task_path !== "string") return false;
+  if (typeof v.state !== "string" || !VALID_STATES.has(v.state)) return false;
+  if (typeof v.evaluated !== "number") return false;
+  if (typeof v.not_evaluated !== "number") return false;
+  if (typeof v.pre_satisfied !== "number") return false;
+  if (typeof v.total !== "number") return false;
+  return true;
 }
-async function readVerdictArtifact(runDir) {
+async function readVerdictArtifactDetailed(runDir) {
   const path = join(runDir, VERDICT_FILENAME);
   let raw;
   try {
     raw = await readFile(path, "utf8");
   } catch {
-    return null;
+    return { status: "unreadable" };
   }
+  let parsed;
   try {
-    const parsed = JSON.parse(raw);
-    if (!isVerdictArtifact(parsed)) return null;
-    return { runDir, verdict: normalizeVerdictArtifact(parsed) };
+    parsed = JSON.parse(raw);
   } catch {
-    return null;
+    return { status: "unreadable" };
   }
+  if (!looksLikeVerdictArtifactBase(parsed)) return { status: "unreadable" };
+  const version = typeof parsed.version === "number" ? parsed.version : null;
+  if (version !== VERDICT_ARTIFACT_VERSION) return { status: "stale-version", version };
+  if (!isVerdictArtifact(parsed)) return { status: "unreadable" };
+  return { status: "ok", trial: { runDir, verdict: parsed } };
 }
-async function scanVerdictArtifacts(artifactsRoot) {
-  const found = [];
+async function scanVerdictArtifactsDetailed(artifactsRoot) {
+  const trials = [];
+  const staleVersionDirs = [];
   let slugs;
   try {
     slugs = await readdir(artifactsRoot);
   } catch {
-    return found;
+    return { trials, staleVersionDirs };
   }
   for (const slug of slugs) {
     const slugDir = join(artifactsRoot, slug);
@@ -77,11 +95,13 @@ async function scanVerdictArtifacts(artifactsRoot) {
       continue;
     }
     for (const runId of runIds) {
-      const trial = await readVerdictArtifact(join(slugDir, runId));
-      if (trial) found.push(trial);
+      const runDir = join(slugDir, runId);
+      const result = await readVerdictArtifactDetailed(runDir);
+      if (result.status === "ok") trials.push(result.trial);
+      else if (result.status === "stale-version") staleVersionDirs.push(runDir);
     }
   }
-  return found;
+  return { trials, staleVersionDirs };
 }
 function groupRunSets(trials) {
   const byKey = /* @__PURE__ */ new Map();
@@ -116,21 +136,35 @@ function latestFailedRunSet(sets) {
   return null;
 }
 async function discoverRunSet(target) {
-  const anchor = await readVerdictArtifact(target);
-  if (anchor) {
+  const anchorResult = await readVerdictArtifactDetailed(target);
+  if (anchorResult.status === "stale-version") {
+    return { kind: "trial-dir", set: null, totalSets: 0, staleVersionCount: 1 };
+  }
+  if (anchorResult.status === "ok") {
+    const anchor = anchorResult.trial;
     const root = join(target, "..", "..");
-    const sets2 = groupRunSets(await scanVerdictArtifacts(root));
+    const { trials: trials2, staleVersionDirs: staleVersionDirs2 } = await scanVerdictArtifactsDetailed(root);
+    const sets2 = groupRunSets(trials2);
     const own = sets2.find(
       (s) => anchor.verdict.group_id !== null && s.groupId === anchor.verdict.group_id || anchor.verdict.group_id === null && s.trials.length === 1 && s.trials[0].verdict.session_id === anchor.verdict.session_id
     ) ?? groupRunSets([anchor])[0];
-    return { kind: "trial-dir", set: own, totalSets: Math.max(sets2.length, 1) };
+    return {
+      kind: "trial-dir",
+      set: own,
+      totalSets: Math.max(sets2.length, 1),
+      staleVersionCount: staleVersionDirs2.length
+    };
+  }
+  if (!existsSync(target)) {
+    return { kind: "root", set: null, totalSets: 0, staleVersionCount: 0 };
   }
-  if (!existsSync(target)) return { kind: "root", set: null, totalSets: 0 };
-  const sets = groupRunSets(await scanVerdictArtifacts(target));
+  const { trials, staleVersionDirs } = await scanVerdictArtifactsDetailed(target);
+  const sets = groupRunSets(trials);
   return {
     kind: "root",
     set: latestFailedRunSet(sets),
-    totalSets: sets.length
+    totalSets: sets.length,
+    staleVersionCount: staleVersionDirs.length
   };
 }
 async function loadTrialEvents(runDir) {
@@ -1067,6 +1101,7 @@ async function runTaskHosted(options) {
     const verdict = scoreStatus(score, scenario.config.passThreshold);
     const exitCode = verdict === "pass" ? 0 : 1;
     try {
+      const counts = evaluationCounts(score);
       await writeVerdictArtifact(artifacts.runDir, {
         version: VERDICT_ARTIFACT_VERSION,
         source: "cloud-finalize",
@@ -1079,7 +1114,12 @@ async function runTaskHosted(options) {
         judge_model: score.judge_model,
         score: finalized.score,
         pass_threshold: scenario.config.passThreshold,
+        state: verdict,
         passed: exitCode === 0,
+        evaluated: counts.evaluated,
+        not_evaluated: counts.notEvaluated,
+        pre_satisfied: counts.preSatisfied,
+        total: counts.total,
         criteria_results: score.results,
         duration_ms: durationMs,
         finalized_at: (/* @__PURE__ */ new Date()).toISOString()
@@ -1176,4 +1216,4 @@ async function abandonBestEffort(client, sessionId, errorCode) {
   }
 }
 
-export { DEFAULT_CONTROL_PLANE_URL, DEFAULT_DASHBOARD_URL, DEFAULT_DOCS_SITE_ORIGIN, clearLocalCredentials, discoverRunSet, ensurePomeGitignored, friendlyHostedError, loadTrialEvents, persistCredentialsAfterLogin, postAgentResolver, readLinkCache, resolveCachedAgentId, resolveCredentials, resolveRunAgentIdentity, resolveSeams, runSessionCreate, runSessionList, runSessionStop, runTaskHosted, writeLinkCache };
+export { DEFAULT_CONTROL_PLANE_URL, DEFAULT_DASHBOARD_URL, DEFAULT_DOCS_SITE_ORIGIN, VERDICT_ARTIFACT_VERSION, clearLocalCredentials, discoverRunSet, ensurePomeGitignored, friendlyHostedError, loadTrialEvents, persistCredentialsAfterLogin, postAgentResolver, readLinkCache, resolveCachedAgentId, resolveCredentials, resolveRunAgentIdentity, resolveSeams, runSessionCreate, runSessionList, runSessionStop, runTaskHosted, writeLinkCache };

package/dist/{runDemo-Q3WATCYJ.js → runDemo-NPBMT4BF.js}

@@ -1,9 +1,9 @@
 import { newGroupId, reassuranceBox, twinReadyLine, trialsHeaderLine, trialLine, summaryLines, evaluatingLine, criterionPhrase } from './chunk-RGZBC7NF.js';
 import { DemoCapacityError, capacityLabel, parseCapacityMarker, capacityKindFrom } from './chunk-ZX4WNSZ5.js';
-import { runTask, demoTaskPath, DEMO_TASK_NAME, DEMO_REPO } from './chunk-54OLERUK.js';
+import { runTask, demoTaskPath, DEMO_TASK_NAME, DEMO_REPO } from './chunk-IQ6LOK4D.js';
 import { getAvailablePort } from './chunk-XDU6TD4O.js';
 import './chunk-CBFKZZBR.js';
-import { createHostedClient, parseTaskFile, uploadRunBlobs, scoreFromFinalizeResponse, scoreStatus, outcomeOf } from './chunk-Z32CZ45D.js';
+import { createHostedClient, parseTaskFile, uploadRunBlobs, scoreFromFinalizeResponse, scoreStatus, outcomeOf } from './chunk-N4UTEHX5.js';
 import './chunk-NW7HGA2K.js';
 import { HostedQuotaError, HostedOrchError } from './chunk-PQYIAA6K.js';
 import './chunk-SGDUD7KK.js';

package/dist/{runTrialGroup-O7DQIB7T.js → runTrialGroup-M4CP7WKY.js}

@@ -1,7 +1,7 @@
 import { newGroupId, criterionPhrase } from './chunk-RGZBC7NF.js';
-import { runTaskHosted, resolveRunAgentIdentity } from './chunk-BCI7KJRH.js';
+import { runTaskHosted, resolveRunAgentIdentity } from './chunk-NRECWO3Y.js';
 import './chunk-KUVTL4NZ.js';
-import { createHostedClient, parseTaskFile, outcomeOf } from './chunk-Z32CZ45D.js';
+import { createHostedClient, parseTaskFile, outcomeOf } from './chunk-N4UTEHX5.js';
 import './chunk-NW7HGA2K.js';
 import { HostedQuotaError, HostedTrialError } from './chunk-PQYIAA6K.js';
 import './chunk-SGDUD7KK.js';

package/dist/src/cli/main.js

@@ -1,10 +1,10 @@
 #!/usr/bin/env node
-import { DEFAULT_CONTROL_PLANE_URL, DEFAULT_DASHBOARD_URL, clearLocalCredentials, friendlyHostedError, runSessionCreate, runSessionList, runSessionStop, resolveCredentials, runTaskHosted, discoverRunSet, loadTrialEvents, persistCredentialsAfterLogin, DEFAULT_DOCS_SITE_ORIGIN, resolveSeams, resolveCachedAgentId, readLinkCache, postAgentResolver, writeLinkCache, ensurePomeGitignored } from '../../chunk-BCI7KJRH.js';
+import { DEFAULT_CONTROL_PLANE_URL, DEFAULT_DASHBOARD_URL, clearLocalCredentials, friendlyHostedError, runSessionCreate, runSessionList, runSessionStop, resolveCredentials, runTaskHosted, discoverRunSet, VERDICT_ARTIFACT_VERSION, loadTrialEvents, persistCredentialsAfterLogin, DEFAULT_DOCS_SITE_ORIGIN, resolveSeams, resolveCachedAgentId, readLinkCache, postAgentResolver, writeLinkCache, ensurePomeGitignored } from '../../chunk-NRECWO3Y.js';
 import { readManifest, writeManifest, MANIFEST_JSON, readRequiredManifest, normalizeManifestTwins } from '../../chunk-KUVTL4NZ.js';
-import { assetPath, runTask, resolvePackageRoot, DEMO_TASK_NAME, demoTaskPath } from '../../chunk-54OLERUK.js';
+import { assetPath, runTask, resolvePackageRoot, DEMO_TASK_NAME, demoTaskPath } from '../../chunk-IQ6LOK4D.js';
 import '../../chunk-XDU6TD4O.js';
 import '../../chunk-CBFKZZBR.js';
-import { parseTaskFile, scoreStatus, runScoreLine, readLatestRun, readMetaSummary, readConfigTwins, scoreCountsSummary, markerFor, outcomeOf, criterionMarkerLabel, twinSkipSuffix, readCodeCriteria, createHostedClient, toTwinHttpEvent, redactJsonl, scoreFromFinalizeResponse, parseGitHubSeedState, uploadRunBlobs, isPreSatisfied } from '../../chunk-Z32CZ45D.js';
+import { parseTaskFile, scoreStatus, runScoreLine, readLatestRun, readMetaSummary, readConfigTwins, scoreCountsSummary, markerFor, outcomeOf, criterionMarkerLabel, twinSkipSuffix, readCodeCriteria, createHostedClient, toTwinHttpEvent, redactJsonl, scoreFromFinalizeResponse, parseGitHubSeedState, uploadRunBlobs, isPreSatisfied } from '../../chunk-N4UTEHX5.js';
 import '../../chunk-NW7HGA2K.js';
 import { MOUNTED_TWINS, deriveAgentSlug, exitCodeFor, HostedUsageError, HostedOrchError, HostedAuthError, HostedQuotaError } from '../../chunk-PQYIAA6K.js';
 import { TAPE_ASSERTABLE_TOOLS } from '../../chunk-FKZZWWYC.js';
@@ -4497,7 +4497,7 @@ var DEFAULT_AGENT_FILE = "examples/agents/scripted-triage-agent.ts";
 var DEFAULT_AGENT_COMMAND = `node ${DEFAULT_AGENT_FILE}`;
 var MANIFEST_SCHEMA_URL = "https://pome.sh/schemas/v1/pome.json";
 function readPackageVersion() {
-  if ("0.23.1".length > 0) return "0.23.1";
+  if ("0.23.2".length > 0) return "0.23.2";
   try {
     const here = dirname(fileURLToPath(import.meta.url));
     const candidates = [
@@ -4990,7 +4990,7 @@ function createProgram() {
               taskForRuns.config.runs
             );
             if (k > 1) {
-              const { runTrialGroup } = await import('../../runTrialGroup-O7DQIB7T.js');
+              const { runTrialGroup } = await import('../../runTrialGroup-M4CP7WKY.js');
               const fileForRerun = relative(process.cwd(), file);
               const rerunCommand = defaultTask ? options.trials !== void 0 ? `pome run -n ${k}` : "pome run" : `pome run ${fileForRerun && !fileForRerun.startsWith("..") ? fileForRerun : file} -n ${k}`;
               const groupResult = await runTrialGroup({
@@ -5084,7 +5084,7 @@ function createProgram() {
         process.exitCode = 5;
         return;
       }
-      const { runDemo } = await import('../../runDemo-Q3WATCYJ.js');
+      const { runDemo } = await import('../../runDemo-NPBMT4BF.js');
       const result = await runDemo({
         apiBase: opts.apiUrl.replace(/\/$/, ""),
         dashboardBase: process.env.POME_DASHBOARD_URL ?? DEFAULT_DASHBOARD_URL,
@@ -5189,10 +5189,17 @@ function createProgram() {
     }
     const root = target ?? "runs";
     const discovery = await discoverRunSet(resolve(root));
-    if (discovery.totalSets === 0) {
+    if (discovery.staleVersionCount > 0) {
       console.error(
-        `No finalized run sets under ${root} \u2014 hosted \`pome run\` records a verdict.json per trial; run one first (or point fix-prompt at your artifacts dir).`
+        `${discovery.staleVersionCount} verdict.json file(s) under ${root} are not artifact version ${VERDICT_ARTIFACT_VERSION} (the only version this CLI reads) and were skipped \u2014 re-run \`pome run\` to record those trials again.`
       );
+    }
+    if (discovery.totalSets === 0) {
+      if (discovery.staleVersionCount === 0) {
+        console.error(
+          `No finalized run sets under ${root} \u2014 hosted \`pome run\` records a verdict.json per trial; run one first (or point fix-prompt at your artifacts dir).`
+        );
+      }
       process.exitCode = 5;
       return;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pome-sh/cli",
-  "version": "0.23.1",
+  "version": "0.23.2",
   "description": "Digital-twin testing for AI agents \u2014 run tasks against resettable local or hosted twins and record tool-call traces for evaluation on pome.sh.",
   "keywords": [
     "ai",