@plune-ai/cli 0.34.0 → 0.34.1

package/README.md

@@ -117,7 +117,7 @@ the last resort, used only when `PLUNE_TOKEN` is set nowhere else. Nothing write
 | ------- | ------- |
 | `plune run` | Run the suite. Flags: `--dry-run` (price the run from the model's rates — no provider key, no network), `--only <id\|tag>` (repeatable), `--bail`, `--no-cache`, `--cache-strict`, `--cache-max-age <duration>`, `--concurrency <n>`, `--format console\|json\|markdown`, `-o, --output <file>`. |
 | `plune report` | Re-render the most recent run. Flags: `--format`, `-o`. |
-| `plune diff <baseline> <current>` | Compare two `plune run --format json` outputs and report pass→fail regressions. Flags: `--fail-on-regression`, `--format`, `-o`. |
+| `plune diff <baseline> <current>` | Compare two `plune run --format json` outputs and report pass→fail regressions. Flags: `--fail-on-regression` (exit `1` on a regression, `3` when none but an eval was not measured), `--format`, `-o`. |
 | `plune init` | Scaffold `plune.yaml`, a sample dataset, and `.env.example`. Flags: `--yes` (non-interactive), `--force`. |
 | `plune login` | Save a [Plune platform](https://plune.ai/platform) API token so `sync` and `ingest` can reach it. The token is **checked against the API before it is saved**, so a wrong one fails here rather than two commands later. Get one at `https://beta.plune.ai` → Account → API tokens. Flags: `--token <token>` (omit to paste it or pipe it via stdin), `--skip-verify` (save without checking, for offline setup). In CI, skip this step: every platform command reads `PLUNE_TOKEN` from the environment first, and the saved login only when it is not set. |
 | `plune logout` | Remove the saved token. |

package/dist/cli.cjs

@@ -2010,7 +2010,8 @@ async function runOrchestration(config, options, deps) {
       limit,
       (row) => runRow({
         template,
-        assertions: ev.assertions,
+        // `defaults.assertions` apply to every eval, ahead of its own (#2415).
+        assertions: [...config.defaults?.assertions ?? [], ...ev.assertions],
         row,
         providerConfig,
         provider,
@@ -2887,13 +2888,13 @@ var init_fallback = __esm({
 function failureOf(attempt, home = (0, import_node_os2.homedir)()) {
   const error = chosen(attempt);
   const line = error === void 0 ? void 0 : firstLine(machineless(error.text, attempt.repoRoot, home));
-  const headline = line !== void 0 && jsonBytes(line) <= HEADLINE_LIMIT ? line : void 0;
+  const headline2 = line !== void 0 && jsonBytes(line) <= HEADLINE_LIMIT ? line : void 0;
   const steps = chainOf(attempt.steps);
   const location = error === void 0 ? void 0 : fromRoot(placeOf(error, attempt.testFile), attempt.repoRoot);
   const artifacts = attempt.attachments.filter((a) => a.path !== void 0 && !a.name.startsWith("_") && a.contentType !== METADATA_TYPE).map(({ name, contentType }) => ({ name: fileNameOf(name), ...contentType ? { contentType } : {} })).filter((a) => a.name !== "");
   const ciUrl = webLink(attempt.buildHref);
   const failure = {
-    ...headline !== void 0 ? { headline } : {},
+    ...headline2 !== void 0 ? { headline: headline2 } : {},
     ...steps.length > 0 ? { steps } : {},
     ...location !== void 0 ? { location } : {},
     ...artifacts.length > 0 ? { artifacts } : {},
@@ -6241,7 +6242,8 @@ function classify2(baseline, current) {
   if (baseline === "absent") return current === "passed" ? "new-pass" : "new-fail";
   if (current === "absent") return "removed";
   if (baseline === "passed") return current === "passed" ? "stable-pass" : "regression";
-  return current === "passed" ? "improvement" : "pre-existing-fail";
+  if (current === "passed") return "improvement";
+  return baseline === "errored" ? "unmeasured" : "pre-existing-fail";
 }
 function diffRuns(baseline, current) {
   const baseMap = new Map(baseline.evals.map((e) => [e.id, evalStatus(e)]));
@@ -6262,6 +6264,7 @@ function diffRuns(baseline, current) {
     stablePasses: count2("stable-pass"),
     errored: count2("errored"),
     removed: count2("removed"),
+    unmeasured: count2("errored") + count2("unmeasured"),
     hasRegression: count2("regression") > 0
   };
   return { evals, summary };
@@ -6332,6 +6335,11 @@ __export(diff_exports2, {
   renderDiff: () => renderDiff,
   renderDiffMarkdown: () => renderDiffMarkdown
 });
+function headline(d) {
+  if (d.summary.hasRegression) return { mark: "\u274C", text: `${d.summary.regressions} regression(s)` };
+  if (d.summary.unmeasured > 0) return { mark: "\u26A0\uFE0F", text: `${d.summary.unmeasured} eval(s) not measured` };
+  return null;
+}
 function summaryLine(d) {
   const s = d.summary;
   return [
@@ -6341,14 +6349,14 @@ function summaryLine(d) {
     `${s.preExistingFails} pre-existing-fail`,
     `${s.errored} errored`,
     `${s.removed} removed`,
+    `${s.unmeasured} not measured`,
     `${s.stablePasses} stable`
   ].join(" \xB7 ");
 }
 function renderDiffMarkdown(d) {
   const out = [STICKY_MARKER, "## Plune eval diff", ""];
-  out.push(
-    d.summary.hasRegression ? `### \u274C ${d.summary.regressions} regression(s)` : "### \u2705 No regressions"
-  );
+  const head = headline(d);
+  out.push(head ? `### ${head.mark} ${head.text}` : "### \u2705 No regressions");
   out.push("");
   const notable = d.evals.filter((e) => e.status !== "stable-pass");
   if (notable.length > 0) {
@@ -6363,9 +6371,8 @@ function renderDiffMarkdown(d) {
   return out.join("\n");
 }
 function renderDiffConsole(d) {
-  const out = [
-    d.summary.hasRegression ? `Plune eval diff: ${d.summary.regressions} regression(s)` : "Plune eval diff: no regressions"
-  ];
+  const head = headline(d);
+  const out = [`Plune eval diff: ${head ? head.text : "no regressions"}`];
   for (const e of d.evals) {
     if (e.status === "stable-pass") continue;
     out.push(`  ${e.id}: ${e.baseline} -> ${e.current} [${e.status}]`);
@@ -6395,6 +6402,7 @@ var init_diff3 = __esm({
       "new-pass": "\u{1F7E2} new pass",
       "stable-pass": "\u2705 stable",
       errored: "\u26A0\uFE0F errored",
+      unmeasured: "\u26A0\uFE0F not measured (baseline errored)",
       removed: "\u2796 removed"
     };
   }
@@ -9014,7 +9022,11 @@ ${missed.map((m) => `  - ${m}`).join("\n")}
       failUnexpected(err, globals.verbose === true);
     }
   });
-  program.command("diff").description("Compare two `plune run --format json` outputs and report regressions").argument("<baseline>", "Path to the baseline run JSON (e.g. from main)").argument("<current>", "Path to the current run JSON (e.g. from the PR)").option("--format <fmt>", "console | json | markdown", "console").option("--fail-on-regression", "Exit 1 when a pass\u2192fail regression is detected", false).option("-o, --output <path>", "Write the diff to a file instead of stdout").action(
+  program.command("diff").description("Compare two `plune run --format json` outputs and report regressions").argument("<baseline>", "Path to the baseline run JSON (e.g. from main)").argument("<current>", "Path to the current run JSON (e.g. from the PR)").option("--format <fmt>", "console | json | markdown", "console").option(
+    "--fail-on-regression",
+    "Exit 1 when a pass\u2192fail regression is detected, 3 when none is but an eval was not measured",
+    false
+  ).option("-o, --output <path>", "Write the diff to a file instead of stdout").action(
     async (baseline, current, options, command) => {
       const verbose = command.optsWithGlobals().verbose === true;
       const [{ handleDiff: handleDiff2, DiffInputError: DiffInputError2 }, { renderDiff: renderDiff2 }] = await Promise.all([
@@ -9024,7 +9036,9 @@ ${missed.map((m) => `  - ${m}`).join("\n")}
       try {
         const diff = handleDiff2({ baselinePath: baseline, currentPath: current });
         writeReport(renderDiff2(diff, toFormat(options.format)), options.output);
-        process.exitCode = options.failOnRegression && diff.summary.hasRegression ? 1 : 0;
+        process.exitCode = 0;
+        if (options.failOnRegression && diff.summary.hasRegression) process.exitCode = 1;
+        else if (options.failOnRegression && diff.summary.unmeasured > 0) process.exitCode = 3;
       } catch (err) {
         if (err instanceof DiffInputError2) {
           process.stderr.write(err.message + "\n");

package/dist/index.cjs

@@ -1647,7 +1647,8 @@ async function runOrchestration(config, options, deps) {
       limit,
       (row) => runRow({
         template,
-        assertions: ev.assertions,
+        // `defaults.assertions` apply to every eval, ahead of its own (#2415).
+        assertions: [...config.defaults?.assertions ?? [], ...ev.assertions],
         row,
         providerConfig,
         provider,

package/dist/index.js

@@ -1587,7 +1587,8 @@ async function runOrchestration(config, options, deps) {
       limit,
       (row) => runRow({
         template,
-        assertions: ev.assertions,
+        // `defaults.assertions` apply to every eval, ahead of its own (#2415).
+        assertions: [...config.defaults?.assertions ?? [], ...ev.assertions],
         row,
         providerConfig,
         provider,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@plune-ai/cli",
-  "version": "0.34.0",
+  "version": "0.34.1",
   "description": "AI-powered assertion testing for LLM apps — write assertions, run them against any provider, get a pass/fail report (local, CI, or diff).",
   "keywords": [
     "ai",