@wayai/cli 0.3.129 → 0.3.130

package/dist/index.js

@@ -5004,6 +5004,15 @@ function collectTurnAttachmentHashes(turn) {
   }
   return hashes;
 }
+function evaluatedRunCount(counts) {
+  return counts.successful_runs + (counts.assertion_failure_runs ?? counts.failed_runs);
+}
+function unprovenRunCount(counts) {
+  return (counts.errored_runs ?? 0) + (counts.invalid_runs ?? 0);
+}
+function terminalRunCount(counts) {
+  return counts.successful_runs + counts.failed_runs;
+}
 function isSafeTemplatePath(path31) {
   if (path31.length === 0 || path31.length > 300) return false;
   if (path31.startsWith("/") || path31.includes("\\")) return false;
@@ -6869,7 +6878,17 @@ var init_contracts = __esm({
       total_evals: external_exports.number(),
       total_runs: external_exports.number(),
       successful_runs: external_exports.number(),
+      /**
+       * Every terminal run that did NOT pass, whatever the reason. Unchanged
+       * meaning — deliberately NOT narrowed to behavioral misses, so a released CLI
+       * that predates the split cannot read an infrastructure wipeout as a clean
+       * sweep. Use `assertion_failure_runs` for the narrow count.
+       */
       failed_runs: external_exports.number(),
+      /** The `failed_runs` breakdown by cause (see `EvalRunOutcomeKind`); sums to it. */
+      assertion_failure_runs: external_exports.number(),
+      errored_runs: external_exports.number(),
+      invalid_runs: external_exports.number(),
       created_by: external_exports.string(),
       /** Optional — sessions created pre-PR2 may not have this. */
       scenario_set_id: external_exports.string().nullable().optional(),
@@ -11900,7 +11919,17 @@ var init_dist = __esm({
       total_evals: external_exports.number(),
       total_runs: external_exports.number(),
       successful_runs: external_exports.number(),
+      /**
+       * Every terminal run that did NOT pass, whatever the reason. Unchanged
+       * meaning — deliberately NOT narrowed to behavioral misses, so a released CLI
+       * that predates the split cannot read an infrastructure wipeout as a clean
+       * sweep. Use `assertion_failure_runs` for the narrow count.
+       */
       failed_runs: external_exports.number(),
+      /** The `failed_runs` breakdown by cause (see `EvalRunOutcomeKind`); sums to it. */
+      assertion_failure_runs: external_exports.number(),
+      errored_runs: external_exports.number(),
+      invalid_runs: external_exports.number(),
       created_by: external_exports.string(),
       /** Optional — sessions created pre-PR2 may not have this. */
       scenario_set_id: external_exports.string().nullable().optional(),
@@ -18799,15 +18828,24 @@ var init_send_message = __esm({
 });
 
 // src/lib/eval-format.ts
+function runNotes(unproven, inProgress) {
+  return [
+    unproven > 0 ? `${unproven} unproven` : "",
+    inProgress > 0 ? `${inProgress} in progress` : ""
+  ].filter(Boolean);
+}
 function printResultsTable(results, mode) {
   let totalPassed = 0;
-  let totalTerminal = 0;
+  let totalEvaluated = 0;
+  let totalUnproven = 0;
   let totalInProgress = 0;
   for (const r of results) {
     const evalName = r.eval?.eval_name || r.eval_id.slice(0, 8);
-    const terminalRuns = r.successful_runs + r.failed_runs;
+    const evaluatedRuns = evaluatedRunCount(r);
+    const unproven = unprovenRunCount(r);
     const inProgress = (r.running_runs ?? 0) + (r.pending_runs ?? 0);
-    const passed = inProgress > 0 ? `${r.successful_runs}/${terminalRuns} passed (+${inProgress} in progress)` : `${r.successful_runs}/${terminalRuns} passed`;
+    const notes2 = runNotes(unproven, inProgress);
+    const passed = notes2.length > 0 ? `${r.successful_runs}/${evaluatedRuns} passed (${notes2.join(", ")})` : `${r.successful_runs}/${evaluatedRuns} passed`;
     const avgTime = r.avg_execution_time_ms != null ? `avg ${(r.avg_execution_time_ms / 1e3).toFixed(1)}s` : "";
     let scoresStr = "";
     if (r.aggregated_scores) {
@@ -18818,13 +18856,15 @@ function printResultsTable(results, mode) {
     }
     console.log(`  ${evalName.padEnd(20)}  ${passed.padEnd(14)}  ${avgTime}${scoresStr}`);
     totalPassed += r.successful_runs;
-    totalTerminal += terminalRuns;
+    totalEvaluated += evaluatedRuns;
+    totalUnproven += unproven;
     totalInProgress += inProgress;
   }
-  const pct = totalTerminal > 0 ? (totalPassed / totalTerminal * 100).toFixed(1) : "0.0";
-  const inProgressNote = totalInProgress > 0 ? ` \u2014 ${totalInProgress} in progress` : "";
+  const pct = totalEvaluated > 0 ? (totalPassed / totalEvaluated * 100).toFixed(1) : "0.0";
+  const notes = runNotes(totalUnproven, totalInProgress);
+  const note = notes.length > 0 ? ` \u2014 ${notes.join(", ")}` : "";
   console.log(`
-Overall: ${totalPassed}/${totalTerminal} passed (${pct}%)${inProgressNote}`);
+Overall: ${totalPassed}/${totalEvaluated} passed (${pct}%)${note}`);
 }
 function truncate(str, maxLen) {
   if (str.length <= maxLen) return str;
@@ -18846,6 +18886,17 @@ function isFailingRun(run) {
   if (run.run_status && !TERMINAL_RUN_STATUSES.has(run.run_status)) return false;
   return run.response_match !== true;
 }
+function failureTag(run) {
+  switch (run.outcome_kind) {
+    case "execution_error":
+      return "ERROR";
+    case "evaluation_invalid":
+      return "INVALID";
+    // `assertion_failure`, and any pre-taxonomy payload.
+    default:
+      return "FAIL";
+  }
+}
 function printFailingRuns(runs) {
   const failing = runs.filter(isFailingRun);
   if (failing.length === 0) return;
@@ -18854,7 +18905,7 @@ Failures (${failing.length}):`);
   let anyComment = false;
   for (const run of failing) {
     const evalName = run.eval?.eval_name || run.eval_id?.slice(0, 8) || "eval";
-    console.log(`  ${evalName} \u2014 Run #${run.run_number} [FAIL]`);
+    console.log(`  ${evalName} \u2014 Run #${run.run_number} [${failureTag(run)}]`);
     if (run.error_message) {
       console.log(`    Error: ${run.error_message}`);
     }
@@ -18876,6 +18927,7 @@ var NO_EVALUATOR_NOTES_HINT, TERMINAL_RUN_STATUSES;
 var init_eval_format = __esm({
   "src/lib/eval-format.ts"() {
     "use strict";
+    init_contracts();
     NO_EVALUATOR_NOTES_HINT = "No evaluator notes found \u2014 add a `notes` evaluation variable to the message_evaluator agent to surface its reasoning here.";
     TERMINAL_RUN_STATUSES = /* @__PURE__ */ new Set(["completed", "failed"]);
   }
@@ -20261,7 +20313,7 @@ Lost connection to the eval session: ${extractApiMessage(err)}`);
     const session = details.data.session;
     const results = details.data.results;
     const totalRuns = results.reduce((sum, r) => sum + r.total_runs, 0);
-    const completedRuns = results.reduce((sum, r) => sum + r.successful_runs + r.failed_runs, 0);
+    const completedRuns = results.reduce((sum, r) => sum + terminalRunCount(r), 0);
     if (TERMINAL_STATUSES.has(session.session_status)) {
       if (await signalController.interrupted()) return;
       console.log("");