@plune-ai/cli 0.17.1 → 0.18.0

package/README.md

@@ -124,7 +124,7 @@ the last resort, used only when `PLUNE_TOKEN` is set nowhere else. Nothing write
 | `plune run start` | Open a platform run — or join the one already carrying `--key` — and print its id. Flags: `--key <externalKey>` (generated when absent), `--json` for one machine-readable line. |
 | `plune run finish <id>` | Close a platform run. This is what the reporter tells you to do for a run it had to leave open. Flags: `--terminate` to record it as cut short, `--reason <text>`. |
 | `plune run exec -- <command>` | Open a run, run the command inside it, close the run — and exit with whatever the command returned. Sets `PLUNE_SHARED_RUN` for you, so anything reporting inside joins that run. Flags: `--key <externalKey>`. |
-| `plune run delete <id>` | Delete a run and everything it produced — its results and the review-queue entries it raised. Approved test cases and the audit log stay. **Recoverable for six months** (ask Plune to put it back), then gone for good. No prompt: it is your data, and this command belongs in scripts. |
+| `plune run delete <id>` | Delete a run and everything it produced — its results and the review-queue entries it raised. Approved test cases and the audit log stay. **Recoverable for 30 days** (an owner of the project can put it back from Settings → Bin), then gone for good. No prompt: it is your data, and this command belongs in scripts. |
 | `plune run report` | Replay `.plune/pending-results.jsonl` — send what the reporter could not. The lines of one reporter run go back as one run, with the commit and branch the reporter recorded. Once every result has gone the file is renamed to `pending-results.<time>.sent.jsonl`; if anything did not go, it stays as it was. Flags: `--file <path>`. |
 | `plune ingest [dir]` | Record a [Cairn](https://github.com/plune-ai/cairn) run in Plune. Omit `[dir]` for the newest run under `./runs`, or name the directory holding `report.json`. Generated cases arrive as **review proposals** — nothing is created until a person approves it. |
 | `plune pull [file]` | Write the project's test cases as **one Markdown document** — Testomat's classical format, plus Plune's own columns — to `plune/cases.md` (or `[file]`). Refuses to overwrite a file git sees as modified; `--force` overrides, `--suite <id>` takes one suite or folder and what is under it. |
@@ -291,8 +291,8 @@ Two suites, or a sharded matrix, report into a single run when every job shares
 it is not the last one:
 
 ```bash
-PLUNE_SHARED_RUN=1 plune run import ./junit.xml     --key "$GITHUB_RUN_ID"
-PLUNE_SHARED_RUN=1 plune run import ./app/junit.xml --key "$GITHUB_RUN_ID"
+PLUNE_SHARED_RUN=1 plune run import ./junit.xml     --key "$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
+PLUNE_SHARED_RUN=1 plune run import ./app/junit.xml --key "$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
 plune run finish "$RUN_ID"                          # once, when they are all done
 ```
 

package/dist/cli.cjs

@@ -1621,6 +1621,76 @@ var init_concurrency = __esm({
   }
 });
 
+// src/types/test-case-schema.ts
+var import_zod2, TEST_CASE_SCHEMA_VERSION, externalKeySchema, manualStepSchema, assertionResultRecordSchema, manualCaseSchema, automatedCaseSchema, evalCaseSchema, testCaseBaseSchema, testCaseSchema;
+var init_test_case_schema = __esm({
+  "src/types/test-case-schema.ts"() {
+    "use strict";
+    init_cjs_shims();
+    import_zod2 = require("zod");
+    TEST_CASE_SCHEMA_VERSION = 2;
+    externalKeySchema = import_zod2.z.object({
+      kind: import_zod2.z.enum(["playwright-id", "path-title", "allure-history", "cairn-stable", "eval-id", "qase", "testrail"]),
+      value: import_zod2.z.string().min(1)
+    });
+    manualStepSchema = import_zod2.z.object({
+      action: import_zod2.z.string().min(1),
+      expected: import_zod2.z.string().min(1).optional()
+    });
+    assertionResultRecordSchema = import_zod2.z.object({
+      type: import_zod2.z.string().min(1),
+      passed: import_zod2.z.boolean(),
+      score: import_zod2.z.number().optional(),
+      reason: import_zod2.z.string().optional()
+    });
+    manualCaseSchema = import_zod2.z.object({
+      type: import_zod2.z.literal("manual"),
+      steps: import_zod2.z.array(manualStepSchema).min(1),
+      expected: import_zod2.z.string().min(1).optional()
+    });
+    automatedCaseSchema = import_zod2.z.object({
+      type: import_zod2.z.literal("automated"),
+      specRef: import_zod2.z.string().min(1)
+    });
+    evalCaseSchema = import_zod2.z.object({
+      type: import_zod2.z.literal("eval"),
+      prompt: import_zod2.z.string().min(1),
+      goldenSetId: import_zod2.z.string().min(1).optional(),
+      assertions: import_zod2.z.array(assertionResultRecordSchema).min(1)
+    });
+    testCaseBaseSchema = import_zod2.z.object({
+      schemaVersion: import_zod2.z.literal(TEST_CASE_SCHEMA_VERSION),
+      id: import_zod2.z.string().min(1),
+      externalKeys: import_zod2.z.array(externalKeySchema).optional(),
+      /** ADR 0045 / plune#1246 — optional, purely additive; schemaVersion stays 2. */
+      is_flaky: import_zod2.z.boolean().optional(),
+      /** ADR 0045 / plune#1246 — optional, purely additive; schemaVersion stays 2. */
+      is_muted: import_zod2.z.boolean().optional()
+    });
+    testCaseSchema = import_zod2.z.discriminatedUnion("type", [
+      testCaseBaseSchema.merge(manualCaseSchema),
+      testCaseBaseSchema.merge(automatedCaseSchema),
+      testCaseBaseSchema.merge(evalCaseSchema)
+    ]);
+  }
+});
+
+// src/types/results.ts
+function rowStatus(row) {
+  if (row.inconclusive !== void 0) return row.inconclusive;
+  if (row.error !== void 0) return "errored";
+  return row.assertions.every((a) => a.passed) ? "passed" : "failed";
+}
+var ROW_INCONCLUSIVE;
+var init_results = __esm({
+  "src/types/results.ts"() {
+    "use strict";
+    init_cjs_shims();
+    init_test_case_schema();
+    ROW_INCONCLUSIVE = ["blocked", "skipped"];
+  }
+});
+
 // src/orchestrator/run.ts
 function resolvePrompt(template, vars) {
   return template.replace(/\{\{\s*(\w+)\s*\}\}/g, (_match, key) => {
@@ -1754,10 +1824,25 @@ function resolveTemplate(ev, baseDir) {
   }
   throw new RunConfigError(`Eval "${ev.id}" has neither prompt nor prompt_file`);
 }
-function classify(row) {
-  if (row.error !== void 0) return "errored";
-  if (row.assertions.some((a) => !a.passed)) return "failed";
-  return "passed";
+function summarize(evals, durationMs) {
+  const summary = {
+    total: 0,
+    passed: 0,
+    failed: 0,
+    errored: 0,
+    blocked: 0,
+    skipped: 0,
+    cost_usd: 0,
+    duration_ms: durationMs
+  };
+  for (const ev of evals) {
+    for (const row of ev.rows) {
+      summary.total += 1;
+      summary[rowStatus(row)] += 1;
+      summary.cost_usd += row.usage?.cost_usd ?? 0;
+    }
+  }
+  return summary;
 }
 async function runOrchestration(config, options, deps) {
   const started = deps.now();
@@ -1792,26 +1877,12 @@ async function runOrchestration(config, options, deps) {
         noCache
       })
     );
-    const passed = rowResults.every((r) => classify(r) === "passed");
+    const passed = rowResults.every((r) => rowStatus(r) === "passed");
     evalResults.push({ id: ev.id, tags: ev.tags ?? [], rows: rowResults, passed });
     if (options.bail === true && !passed) break;
   }
   const finished = deps.now();
-  const summary = {
-    total: 0,
-    passed: 0,
-    failed: 0,
-    errored: 0,
-    cost_usd: 0,
-    duration_ms: finished - started
-  };
-  for (const ev of evalResults) {
-    for (const row of ev.rows) {
-      summary.total += 1;
-      summary[classify(row)] += 1;
-      summary.cost_usd += row.usage?.cost_usd ?? 0;
-    }
-  }
+  const summary = summarize(evalResults, finished - started);
   return {
     schemaVersion: 1,
     plune_version: PLUNE_VERSION,
@@ -1834,6 +1905,7 @@ var init_run = __esm({
     init_judge();
     init_concurrency();
     init_errors2();
+    init_results();
     DEFAULT_MAX_TOKENS2 = 1024;
     PLUNE_VERSION = "0.1.0";
     RunConfigError = class extends Error {
@@ -2027,8 +2099,10 @@ function renderConsole(result, opts = {}) {
   const passedTxt = s.green(`${summary.passed} passed`);
   const failedTxt = summary.failed > 0 ? s.red(`${summary.failed} failed`) : `${summary.failed} failed`;
   const erroredTxt = summary.errored > 0 ? s.yellow(`${summary.errored} errored`) : `${summary.errored} errored`;
+  const blockedTxt = (summary.blocked ?? 0) > 0 ? ` \xB7 ${s.yellow(`${summary.blocked} blocked`)}` : "";
+  const skippedTxt = (summary.skipped ?? 0) > 0 ? ` \xB7 ${s.dim(`${summary.skipped} skipped`)}` : "";
   lines.push(
-    `${passedTxt} \xB7 ${failedTxt} \xB7 ${erroredTxt} \xB7 ${summary.total} total \xB7 $${summary.cost_usd.toFixed(4)} \xB7 ${summary.duration_ms}ms`
+    `${passedTxt} \xB7 ${failedTxt} \xB7 ${erroredTxt}${blockedTxt}${skippedTxt} \xB7 ${summary.total} total \xB7 $${summary.cost_usd.toFixed(4)} \xB7 ${summary.duration_ms}ms`
   );
   lines.push("");
   for (const ev of result.evals) {
@@ -2037,17 +2111,21 @@ function renderConsole(result, opts = {}) {
     lines.push(`${mark} ${s.bold(ev.id)}${tags}`);
     let passedCount = 0;
     for (const row of ev.rows) {
+      const status = rowStatus(row);
+      if (status === "passed") {
+        passedCount += 1;
+        continue;
+      }
+      lines.push(`  ${s.dim("vars:")} ${JSON.stringify(row.vars)}`);
+      if (status === "blocked" || status === "skipped") {
+        lines.push(`  ${s.yellow(status)} (not judged)`);
+        continue;
+      }
       if (row.error !== void 0) {
-        lines.push(`  ${s.dim("vars:")} ${JSON.stringify(row.vars)}`);
         lines.push(`  ${s.red("error:")} ${row.error.message}`);
         continue;
       }
       const failed = row.assertions.filter((a) => !a.passed);
-      if (failed.length === 0) {
-        passedCount += 1;
-        continue;
-      }
-      lines.push(`  ${s.dim("vars:")} ${JSON.stringify(row.vars)}`);
       if (row.output !== null) {
         lines.push(`  ${s.dim("output:")} ${truncate(row.output, maxOut)}`);
       }
@@ -2068,6 +2146,7 @@ var init_console = __esm({
   "src/reporters/console.ts"() {
     "use strict";
     init_cjs_shims();
+    init_results();
     init_style();
     init_helpers();
     DEFAULT_MAX_OUTPUT = 500;
@@ -2086,11 +2165,6 @@ var init_json = __esm({
 });
 
 // src/reporters/markdown.ts
-function classify2(row) {
-  if (row.error !== void 0) return "errored";
-  if (row.assertions.some((a) => !a.passed)) return "failed";
-  return "passed";
-}
 function fence(content) {
   const runs = content.match(new RegExp(TICK + "+", "g"));
   const longest = runs ? runs.reduce((m, r) => Math.max(m, r.length), 0) : 0;
@@ -2111,18 +2185,22 @@ function renderMarkdown(result, opts = {}) {
   out.push(`| Passed | ${summary.passed} |`);
   out.push(`| Failed | ${summary.failed} |`);
   out.push(`| Errored | ${summary.errored} |`);
+  if ((summary.blocked ?? 0) > 0) out.push(`| Blocked | ${summary.blocked} |`);
+  if ((summary.skipped ?? 0) > 0) out.push(`| Skipped | ${summary.skipped} |`);
   out.push(`| Cost (USD) | ${summary.cost_usd.toFixed(4)} |`);
   out.push(`| Duration (ms) | ${summary.duration_ms} |`);
   out.push("");
   const failures = [];
+  let allPassed = true;
   for (const ev of result.evals) {
     for (const row of ev.rows) {
-      const state = classify2(row);
-      if (state !== "passed") failures.push({ id: ev.id, row, state });
+      const state = rowStatus(row);
+      if (state !== "passed") allPassed = false;
+      if (state === "failed" || state === "errored") failures.push({ id: ev.id, row, state });
     }
   }
   if (failures.length === 0) {
-    out.push("All evals passed.");
+    if (allPassed) out.push("All evals passed.");
     return out.join("\n");
   }
   out.push("## Failures");
@@ -2153,6 +2231,7 @@ var init_markdown = __esm({
   "src/reporters/markdown.ts"() {
     "use strict";
     init_cjs_shims();
+    init_results();
     init_helpers();
     DEFAULT_MAX_OUTPUT2 = 500;
     TICK = String.fromCharCode(96);
@@ -2298,7 +2377,7 @@ function retryAfterMs2(res) {
   const seconds = Number(raw.trim());
   return Number.isFinite(seconds) && seconds >= 0 ? seconds * 1e3 : null;
 }
-function classify3(status) {
+function classify(status) {
   if (status === 401 || status === 403) return "auth";
   if (status === 409) return "conflict";
   if (status === 400) return "invalid";
@@ -2341,13 +2420,13 @@ function createClient(opts) {
         }
       }
       const said = await detailOf(res);
-      const kind = classify3(res.status);
+      const kind = classify(res.status);
       last = { kind, status: res.status, detail: scrub(said.detail), ...said.code !== void 0 ? { code: said.code } : {} };
       if (kind !== "unavailable") break;
       if (res.status === 429 && retryAfterMs2(res) === null) {
         last.kind = "invalid";
         last.detail = "quota exceeded (no Retry-After)";
-        if (said.detail) last.detail += ` \u2014 ${said.detail}`;
+        if (said.detail) last.detail += ` \u2014 ${scrub(said.detail)}`;
         break;
       }
       if (attempt < RETRY_DELAYS_MS.length) {
@@ -3404,7 +3483,7 @@ async function handleRunDelete(id, options = {}) {
     );
   }
   write(`Run ${id} deleted, with its results and the review-queue entries it produced.`);
-  write("Recoverable for 30 days \u2014 ask Plune to put it back.");
+  write("Recoverable for 30 days \u2014 an owner of the project can put it back from Settings \u2192 Bin.");
 }
 async function handleRunExec(options) {
   const [command, ...args] = options.argv;
@@ -4544,18 +4623,13 @@ var init_report = __esm({
 });
 
 // src/diff/diff.ts
-function rowStatus(r) {
-  if (r.error !== void 0) return "errored";
-  if (r.assertions.some((a) => !a.passed)) return "failed";
-  return "passed";
-}
 function evalStatus(e) {
   const statuses = e.rows.map(rowStatus);
   if (statuses.includes("failed")) return "failed";
   if (statuses.includes("errored")) return "errored";
   return "passed";
 }
-function classify4(baseline, current) {
+function classify2(baseline, current) {
   if (current === "errored") return "errored";
   if (baseline === "absent") return current === "passed" ? "new-pass" : "new-fail";
   if (current === "absent") return "removed";
@@ -4569,7 +4643,7 @@ function diffRuns(baseline, current) {
   const evals = ids.map((id) => {
     const baselineP = baseMap.get(id) ?? "absent";
     const currentP = curMap.get(id) ?? "absent";
-    return { id, status: classify4(baselineP, currentP), baseline: baselineP, current: currentP };
+    return { id, status: classify2(baselineP, currentP), baseline: baselineP, current: currentP };
   });
   const count2 = (s) => evals.filter((e) => e.status === s).length;
   const summary = {
@@ -4589,6 +4663,7 @@ var init_diff = __esm({
   "src/diff/diff.ts"() {
     "use strict";
     init_cjs_shims();
+    init_results();
   }
 });
 
@@ -5007,71 +5082,6 @@ var init_logout = __esm({
   }
 });
 
-// src/types/test-case-schema.ts
-var import_zod2, TEST_CASE_SCHEMA_VERSION, externalKeySchema, manualStepSchema, assertionResultRecordSchema, manualCaseSchema, automatedCaseSchema, evalCaseSchema, testCaseBaseSchema, testCaseSchema;
-var init_test_case_schema = __esm({
-  "src/types/test-case-schema.ts"() {
-    "use strict";
-    init_cjs_shims();
-    import_zod2 = require("zod");
-    TEST_CASE_SCHEMA_VERSION = 2;
-    externalKeySchema = import_zod2.z.object({
-      kind: import_zod2.z.enum(["playwright-id", "path-title", "allure-history", "cairn-stable", "eval-id", "qase", "testrail"]),
-      value: import_zod2.z.string().min(1)
-    });
-    manualStepSchema = import_zod2.z.object({
-      action: import_zod2.z.string().min(1),
-      expected: import_zod2.z.string().min(1).optional()
-    });
-    assertionResultRecordSchema = import_zod2.z.object({
-      type: import_zod2.z.string().min(1),
-      passed: import_zod2.z.boolean(),
-      score: import_zod2.z.number().optional(),
-      reason: import_zod2.z.string().optional()
-    });
-    manualCaseSchema = import_zod2.z.object({
-      type: import_zod2.z.literal("manual"),
-      steps: import_zod2.z.array(manualStepSchema).min(1),
-      expected: import_zod2.z.string().min(1).optional()
-    });
-    automatedCaseSchema = import_zod2.z.object({
-      type: import_zod2.z.literal("automated"),
-      specRef: import_zod2.z.string().min(1)
-    });
-    evalCaseSchema = import_zod2.z.object({
-      type: import_zod2.z.literal("eval"),
-      prompt: import_zod2.z.string().min(1),
-      goldenSetId: import_zod2.z.string().min(1).optional(),
-      assertions: import_zod2.z.array(assertionResultRecordSchema).min(1)
-    });
-    testCaseBaseSchema = import_zod2.z.object({
-      schemaVersion: import_zod2.z.literal(TEST_CASE_SCHEMA_VERSION),
-      id: import_zod2.z.string().min(1),
-      externalKeys: import_zod2.z.array(externalKeySchema).optional(),
-      /** ADR 0045 / plune#1246 — optional, purely additive; schemaVersion stays 2. */
-      is_flaky: import_zod2.z.boolean().optional(),
-      /** ADR 0045 / plune#1246 — optional, purely additive; schemaVersion stays 2. */
-      is_muted: import_zod2.z.boolean().optional()
-    });
-    testCaseSchema = import_zod2.z.discriminatedUnion("type", [
-      testCaseBaseSchema.merge(manualCaseSchema),
-      testCaseBaseSchema.merge(automatedCaseSchema),
-      testCaseBaseSchema.merge(evalCaseSchema)
-    ]);
-  }
-});
-
-// src/types/results.ts
-var ROW_INCONCLUSIVE;
-var init_results = __esm({
-  "src/types/results.ts"() {
-    "use strict";
-    init_cjs_shims();
-    init_test_case_schema();
-    ROW_INCONCLUSIVE = ["blocked", "skipped"];
-  }
-});
-
 // src/types/run-result-schema.ts
 function parseRunResult(value2) {
   return runResultSchema.parse(value2);
@@ -5120,6 +5130,9 @@ var init_run_result_schema = __esm({
       passed: import_zod3.z.number(),
       failed: import_zod3.z.number(),
       errored: import_zod3.z.number(),
+      /** plune#1245. Optional and additive — a RunResult written before them validates unchanged. */
+      blocked: import_zod3.z.number().optional(),
+      skipped: import_zod3.z.number().optional(),
       cost_usd: import_zod3.z.number(),
       duration_ms: import_zod3.z.number()
     });
@@ -5934,7 +5947,7 @@ function createProgram() {
   runCommand.command("delete").argument(
     "<id>",
     "The run id \u2014 the one `plune run import` printed, or the one in the dashboard URL"
-  ).description("Delete a run and everything it produced \u2014 recoverable for six months").action(async (id, _options, command) => {
+  ).description("Delete a run and everything it produced \u2014 recoverable for 30 days").action(async (id, _options, command) => {
     try {
       const { handleRunDelete: handleRunDelete2 } = await Promise.resolve().then(() => (init_run_lifecycle(), run_lifecycle_exports));
       await handleRunDelete2(id);

package/dist/index.cjs

@@ -1272,6 +1272,64 @@ async function mapLimit(items, limit, fn) {
   return results;
 }
 
+// src/types/test-case-schema.ts
+var import_zod2 = require("zod");
+var TEST_CASE_SCHEMA_VERSION = 2;
+var externalKeySchema = import_zod2.z.object({
+  kind: import_zod2.z.enum(["playwright-id", "path-title", "allure-history", "cairn-stable", "eval-id", "qase", "testrail"]),
+  value: import_zod2.z.string().min(1)
+});
+var manualStepSchema = import_zod2.z.object({
+  action: import_zod2.z.string().min(1),
+  expected: import_zod2.z.string().min(1).optional()
+});
+var assertionResultRecordSchema = import_zod2.z.object({
+  type: import_zod2.z.string().min(1),
+  passed: import_zod2.z.boolean(),
+  score: import_zod2.z.number().optional(),
+  reason: import_zod2.z.string().optional()
+});
+var manualCaseSchema = import_zod2.z.object({
+  type: import_zod2.z.literal("manual"),
+  steps: import_zod2.z.array(manualStepSchema).min(1),
+  expected: import_zod2.z.string().min(1).optional()
+});
+var automatedCaseSchema = import_zod2.z.object({
+  type: import_zod2.z.literal("automated"),
+  specRef: import_zod2.z.string().min(1)
+});
+var evalCaseSchema = import_zod2.z.object({
+  type: import_zod2.z.literal("eval"),
+  prompt: import_zod2.z.string().min(1),
+  goldenSetId: import_zod2.z.string().min(1).optional(),
+  assertions: import_zod2.z.array(assertionResultRecordSchema).min(1)
+});
+var testCaseBaseSchema = import_zod2.z.object({
+  schemaVersion: import_zod2.z.literal(TEST_CASE_SCHEMA_VERSION),
+  id: import_zod2.z.string().min(1),
+  externalKeys: import_zod2.z.array(externalKeySchema).optional(),
+  /** ADR 0045 / plune#1246 — optional, purely additive; schemaVersion stays 2. */
+  is_flaky: import_zod2.z.boolean().optional(),
+  /** ADR 0045 / plune#1246 — optional, purely additive; schemaVersion stays 2. */
+  is_muted: import_zod2.z.boolean().optional()
+});
+var testCaseSchema = import_zod2.z.discriminatedUnion("type", [
+  testCaseBaseSchema.merge(manualCaseSchema),
+  testCaseBaseSchema.merge(automatedCaseSchema),
+  testCaseBaseSchema.merge(evalCaseSchema)
+]);
+function parseTestCase(value) {
+  return testCaseSchema.parse(value);
+}
+
+// src/types/results.ts
+var ROW_INCONCLUSIVE = ["blocked", "skipped"];
+function rowStatus(row) {
+  if (row.inconclusive !== void 0) return row.inconclusive;
+  if (row.error !== void 0) return "errored";
+  return row.assertions.every((a) => a.passed) ? "passed" : "failed";
+}
+
 // src/orchestrator/run.ts
 var DEFAULT_MAX_TOKENS2 = 1024;
 var PLUNE_VERSION = "0.1.0";
@@ -1414,10 +1472,25 @@ function resolveTemplate(ev, baseDir) {
   }
   throw new RunConfigError(`Eval "${ev.id}" has neither prompt nor prompt_file`);
 }
-function classify(row) {
-  if (row.error !== void 0) return "errored";
-  if (row.assertions.some((a) => !a.passed)) return "failed";
-  return "passed";
+function summarize(evals, durationMs) {
+  const summary = {
+    total: 0,
+    passed: 0,
+    failed: 0,
+    errored: 0,
+    blocked: 0,
+    skipped: 0,
+    cost_usd: 0,
+    duration_ms: durationMs
+  };
+  for (const ev of evals) {
+    for (const row of ev.rows) {
+      summary.total += 1;
+      summary[rowStatus(row)] += 1;
+      summary.cost_usd += row.usage?.cost_usd ?? 0;
+    }
+  }
+  return summary;
 }
 async function runOrchestration(config, options, deps) {
   const started = deps.now();
@@ -1452,26 +1525,12 @@ async function runOrchestration(config, options, deps) {
         noCache
       })
     );
-    const passed = rowResults.every((r) => classify(r) === "passed");
+    const passed = rowResults.every((r) => rowStatus(r) === "passed");
     evalResults.push({ id: ev.id, tags: ev.tags ?? [], rows: rowResults, passed });
     if (options.bail === true && !passed) break;
   }
   const finished = deps.now();
-  const summary = {
-    total: 0,
-    passed: 0,
-    failed: 0,
-    errored: 0,
-    cost_usd: 0,
-    duration_ms: finished - started
-  };
-  for (const ev of evalResults) {
-    for (const row of ev.rows) {
-      summary.total += 1;
-      summary[classify(row)] += 1;
-      summary.cost_usd += row.usage?.cost_usd ?? 0;
-    }
-  }
+  const summary = summarize(evalResults, finished - started);
   return {
     schemaVersion: 1,
     plune_version: PLUNE_VERSION,
@@ -1556,61 +1615,6 @@ var path5 = __toESM(require("path"), 1);
 
 // src/types/run-result-schema.ts
 var import_zod3 = require("zod");
-
-// src/types/test-case-schema.ts
-var import_zod2 = require("zod");
-var TEST_CASE_SCHEMA_VERSION = 2;
-var externalKeySchema = import_zod2.z.object({
-  kind: import_zod2.z.enum(["playwright-id", "path-title", "allure-history", "cairn-stable", "eval-id", "qase", "testrail"]),
-  value: import_zod2.z.string().min(1)
-});
-var manualStepSchema = import_zod2.z.object({
-  action: import_zod2.z.string().min(1),
-  expected: import_zod2.z.string().min(1).optional()
-});
-var assertionResultRecordSchema = import_zod2.z.object({
-  type: import_zod2.z.string().min(1),
-  passed: import_zod2.z.boolean(),
-  score: import_zod2.z.number().optional(),
-  reason: import_zod2.z.string().optional()
-});
-var manualCaseSchema = import_zod2.z.object({
-  type: import_zod2.z.literal("manual"),
-  steps: import_zod2.z.array(manualStepSchema).min(1),
-  expected: import_zod2.z.string().min(1).optional()
-});
-var automatedCaseSchema = import_zod2.z.object({
-  type: import_zod2.z.literal("automated"),
-  specRef: import_zod2.z.string().min(1)
-});
-var evalCaseSchema = import_zod2.z.object({
-  type: import_zod2.z.literal("eval"),
-  prompt: import_zod2.z.string().min(1),
-  goldenSetId: import_zod2.z.string().min(1).optional(),
-  assertions: import_zod2.z.array(assertionResultRecordSchema).min(1)
-});
-var testCaseBaseSchema = import_zod2.z.object({
-  schemaVersion: import_zod2.z.literal(TEST_CASE_SCHEMA_VERSION),
-  id: import_zod2.z.string().min(1),
-  externalKeys: import_zod2.z.array(externalKeySchema).optional(),
-  /** ADR 0045 / plune#1246 — optional, purely additive; schemaVersion stays 2. */
-  is_flaky: import_zod2.z.boolean().optional(),
-  /** ADR 0045 / plune#1246 — optional, purely additive; schemaVersion stays 2. */
-  is_muted: import_zod2.z.boolean().optional()
-});
-var testCaseSchema = import_zod2.z.discriminatedUnion("type", [
-  testCaseBaseSchema.merge(manualCaseSchema),
-  testCaseBaseSchema.merge(automatedCaseSchema),
-  testCaseBaseSchema.merge(evalCaseSchema)
-]);
-function parseTestCase(value) {
-  return testCaseSchema.parse(value);
-}
-
-// src/types/results.ts
-var ROW_INCONCLUSIVE = ["blocked", "skipped"];
-
-// src/types/run-result-schema.ts
 var RUN_RESULT_SCHEMA_VERSION = 1;
 var binaryVerdictSchema = import_zod3.z.object({
   question: import_zod3.z.string().min(1),
@@ -1648,6 +1652,9 @@ var summarySchema = import_zod3.z.object({
   passed: import_zod3.z.number(),
   failed: import_zod3.z.number(),
   errored: import_zod3.z.number(),
+  /** plune#1245. Optional and additive — a RunResult written before them validates unchanged. */
+  blocked: import_zod3.z.number().optional(),
+  skipped: import_zod3.z.number().optional(),
   cost_usd: import_zod3.z.number(),
   duration_ms: import_zod3.z.number()
 });

package/dist/index.d.cts

@@ -5,6 +5,14 @@ interface Summary {
     passed: number;
     failed: number;
     errored: number;
+    /**
+     * plune#1245. Rows the runner said were never judged (`RowResult.inconclusive`), counted apart from
+     * `failed`. The runner always writes both; they are optional so a RunResult written before them is
+     * still valid and `schemaVersion` stays 1. With them, `passed + failed + errored + blocked + skipped`
+     * is `total`.
+     */
+    blocked?: number;
+    skipped?: number;
     cost_usd: number;
     duration_ms: number;
 }
@@ -430,6 +438,8 @@ declare const runResultSchema: z.ZodObject<{
         passed: z.ZodNumber;
         failed: z.ZodNumber;
         errored: z.ZodNumber;
+        blocked: z.ZodOptional<z.ZodNumber>;
+        skipped: z.ZodOptional<z.ZodNumber>;
         cost_usd: z.ZodNumber;
         duration_ms: z.ZodNumber;
     }, z.core.$strip>;

package/dist/index.d.ts

@@ -5,6 +5,14 @@ interface Summary {
     passed: number;
     failed: number;
     errored: number;
+    /**
+     * plune#1245. Rows the runner said were never judged (`RowResult.inconclusive`), counted apart from
+     * `failed`. The runner always writes both; they are optional so a RunResult written before them is
+     * still valid and `schemaVersion` stays 1. With them, `passed + failed + errored + blocked + skipped`
+     * is `total`.
+     */
+    blocked?: number;
+    skipped?: number;
     cost_usd: number;
     duration_ms: number;
 }
@@ -430,6 +438,8 @@ declare const runResultSchema: z.ZodObject<{
         passed: z.ZodNumber;
         failed: z.ZodNumber;
         errored: z.ZodNumber;
+        blocked: z.ZodOptional<z.ZodNumber>;
+        skipped: z.ZodOptional<z.ZodNumber>;
         cost_usd: z.ZodNumber;
         duration_ms: z.ZodNumber;
     }, z.core.$strip>;

package/dist/index.js

@@ -1216,6 +1216,64 @@ async function mapLimit(items, limit, fn) {
   return results;
 }
 
+// src/types/test-case-schema.ts
+import { z as z2 } from "zod";
+var TEST_CASE_SCHEMA_VERSION = 2;
+var externalKeySchema = z2.object({
+  kind: z2.enum(["playwright-id", "path-title", "allure-history", "cairn-stable", "eval-id", "qase", "testrail"]),
+  value: z2.string().min(1)
+});
+var manualStepSchema = z2.object({
+  action: z2.string().min(1),
+  expected: z2.string().min(1).optional()
+});
+var assertionResultRecordSchema = z2.object({
+  type: z2.string().min(1),
+  passed: z2.boolean(),
+  score: z2.number().optional(),
+  reason: z2.string().optional()
+});
+var manualCaseSchema = z2.object({
+  type: z2.literal("manual"),
+  steps: z2.array(manualStepSchema).min(1),
+  expected: z2.string().min(1).optional()
+});
+var automatedCaseSchema = z2.object({
+  type: z2.literal("automated"),
+  specRef: z2.string().min(1)
+});
+var evalCaseSchema = z2.object({
+  type: z2.literal("eval"),
+  prompt: z2.string().min(1),
+  goldenSetId: z2.string().min(1).optional(),
+  assertions: z2.array(assertionResultRecordSchema).min(1)
+});
+var testCaseBaseSchema = z2.object({
+  schemaVersion: z2.literal(TEST_CASE_SCHEMA_VERSION),
+  id: z2.string().min(1),
+  externalKeys: z2.array(externalKeySchema).optional(),
+  /** ADR 0045 / plune#1246 — optional, purely additive; schemaVersion stays 2. */
+  is_flaky: z2.boolean().optional(),
+  /** ADR 0045 / plune#1246 — optional, purely additive; schemaVersion stays 2. */
+  is_muted: z2.boolean().optional()
+});
+var testCaseSchema = z2.discriminatedUnion("type", [
+  testCaseBaseSchema.merge(manualCaseSchema),
+  testCaseBaseSchema.merge(automatedCaseSchema),
+  testCaseBaseSchema.merge(evalCaseSchema)
+]);
+function parseTestCase(value) {
+  return testCaseSchema.parse(value);
+}
+
+// src/types/results.ts
+var ROW_INCONCLUSIVE = ["blocked", "skipped"];
+function rowStatus(row) {
+  if (row.inconclusive !== void 0) return row.inconclusive;
+  if (row.error !== void 0) return "errored";
+  return row.assertions.every((a) => a.passed) ? "passed" : "failed";
+}
+
 // src/orchestrator/run.ts
 var DEFAULT_MAX_TOKENS2 = 1024;
 var PLUNE_VERSION = "0.1.0";
@@ -1358,10 +1416,25 @@ function resolveTemplate(ev, baseDir) {
   }
   throw new RunConfigError(`Eval "${ev.id}" has neither prompt nor prompt_file`);
 }
-function classify(row) {
-  if (row.error !== void 0) return "errored";
-  if (row.assertions.some((a) => !a.passed)) return "failed";
-  return "passed";
+function summarize(evals, durationMs) {
+  const summary = {
+    total: 0,
+    passed: 0,
+    failed: 0,
+    errored: 0,
+    blocked: 0,
+    skipped: 0,
+    cost_usd: 0,
+    duration_ms: durationMs
+  };
+  for (const ev of evals) {
+    for (const row of ev.rows) {
+      summary.total += 1;
+      summary[rowStatus(row)] += 1;
+      summary.cost_usd += row.usage?.cost_usd ?? 0;
+    }
+  }
+  return summary;
 }
 async function runOrchestration(config, options, deps) {
   const started = deps.now();
@@ -1396,26 +1469,12 @@ async function runOrchestration(config, options, deps) {
         noCache
       })
     );
-    const passed = rowResults.every((r) => classify(r) === "passed");
+    const passed = rowResults.every((r) => rowStatus(r) === "passed");
     evalResults.push({ id: ev.id, tags: ev.tags ?? [], rows: rowResults, passed });
     if (options.bail === true && !passed) break;
   }
   const finished = deps.now();
-  const summary = {
-    total: 0,
-    passed: 0,
-    failed: 0,
-    errored: 0,
-    cost_usd: 0,
-    duration_ms: finished - started
-  };
-  for (const ev of evalResults) {
-    for (const row of ev.rows) {
-      summary.total += 1;
-      summary[classify(row)] += 1;
-      summary.cost_usd += row.usage?.cost_usd ?? 0;
-    }
-  }
+  const summary = summarize(evalResults, finished - started);
   return {
     schemaVersion: 1,
     plune_version: PLUNE_VERSION,
@@ -1500,61 +1559,6 @@ import * as path5 from "path";
 
 // src/types/run-result-schema.ts
 import { z as z3 } from "zod";
-
-// src/types/test-case-schema.ts
-import { z as z2 } from "zod";
-var TEST_CASE_SCHEMA_VERSION = 2;
-var externalKeySchema = z2.object({
-  kind: z2.enum(["playwright-id", "path-title", "allure-history", "cairn-stable", "eval-id", "qase", "testrail"]),
-  value: z2.string().min(1)
-});
-var manualStepSchema = z2.object({
-  action: z2.string().min(1),
-  expected: z2.string().min(1).optional()
-});
-var assertionResultRecordSchema = z2.object({
-  type: z2.string().min(1),
-  passed: z2.boolean(),
-  score: z2.number().optional(),
-  reason: z2.string().optional()
-});
-var manualCaseSchema = z2.object({
-  type: z2.literal("manual"),
-  steps: z2.array(manualStepSchema).min(1),
-  expected: z2.string().min(1).optional()
-});
-var automatedCaseSchema = z2.object({
-  type: z2.literal("automated"),
-  specRef: z2.string().min(1)
-});
-var evalCaseSchema = z2.object({
-  type: z2.literal("eval"),
-  prompt: z2.string().min(1),
-  goldenSetId: z2.string().min(1).optional(),
-  assertions: z2.array(assertionResultRecordSchema).min(1)
-});
-var testCaseBaseSchema = z2.object({
-  schemaVersion: z2.literal(TEST_CASE_SCHEMA_VERSION),
-  id: z2.string().min(1),
-  externalKeys: z2.array(externalKeySchema).optional(),
-  /** ADR 0045 / plune#1246 — optional, purely additive; schemaVersion stays 2. */
-  is_flaky: z2.boolean().optional(),
-  /** ADR 0045 / plune#1246 — optional, purely additive; schemaVersion stays 2. */
-  is_muted: z2.boolean().optional()
-});
-var testCaseSchema = z2.discriminatedUnion("type", [
-  testCaseBaseSchema.merge(manualCaseSchema),
-  testCaseBaseSchema.merge(automatedCaseSchema),
-  testCaseBaseSchema.merge(evalCaseSchema)
-]);
-function parseTestCase(value) {
-  return testCaseSchema.parse(value);
-}
-
-// src/types/results.ts
-var ROW_INCONCLUSIVE = ["blocked", "skipped"];
-
-// src/types/run-result-schema.ts
 var RUN_RESULT_SCHEMA_VERSION = 1;
 var binaryVerdictSchema = z3.object({
   question: z3.string().min(1),
@@ -1592,6 +1596,9 @@ var summarySchema = z3.object({
   passed: z3.number(),
   failed: z3.number(),
   errored: z3.number(),
+  /** plune#1245. Optional and additive — a RunResult written before them validates unchanged. */
+  blocked: z3.number().optional(),
+  skipped: z3.number().optional(),
   cost_usd: z3.number(),
   duration_ms: z3.number()
 });

package/dist/reporter-core.cjs

@@ -215,7 +215,7 @@ function createClient(opts) {
       if (res.status === 429 && retryAfterMs(res) === null) {
         last.kind = "invalid";
         last.detail = "quota exceeded (no Retry-After)";
-        if (said.detail) last.detail += ` \u2014 ${said.detail}`;
+        if (said.detail) last.detail += ` \u2014 ${scrub(said.detail)}`;
         break;
       }
       if (attempt < RETRY_DELAYS_MS.length) {

package/dist/reporter-core.js

@@ -166,7 +166,7 @@ function createClient(opts) {
       if (res.status === 429 && retryAfterMs(res) === null) {
         last.kind = "invalid";
         last.detail = "quota exceeded (no Retry-After)";
-        if (said.detail) last.detail += ` \u2014 ${said.detail}`;
+        if (said.detail) last.detail += ` \u2014 ${scrub(said.detail)}`;
         break;
       }
       if (attempt < RETRY_DELAYS_MS.length) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@plune-ai/cli",
-  "version": "0.17.1",
+  "version": "0.18.0",
   "description": "AI-powered assertion testing for LLM apps — write assertions, run them against any provider, get a pass/fail report (local, CI, or diff).",
   "keywords": [
     "ai",