@wayai/cli 0.3.128 → 0.3.130

package/dist/index.js

@@ -5004,6 +5004,15 @@ function collectTurnAttachmentHashes(turn) {
   }
   return hashes;
 }
+function evaluatedRunCount(counts) {
+  return counts.successful_runs + (counts.assertion_failure_runs ?? counts.failed_runs);
+}
+function unprovenRunCount(counts) {
+  return (counts.errored_runs ?? 0) + (counts.invalid_runs ?? 0);
+}
+function terminalRunCount(counts) {
+  return counts.successful_runs + counts.failed_runs;
+}
 function isSafeTemplatePath(path31) {
   if (path31.length === 0 || path31.length > 300) return false;
   if (path31.startsWith("/") || path31.includes("\\")) return false;
@@ -6869,7 +6878,17 @@ var init_contracts = __esm({
       total_evals: external_exports.number(),
       total_runs: external_exports.number(),
       successful_runs: external_exports.number(),
+      /**
+       * Every terminal run that did NOT pass, whatever the reason. Unchanged
+       * meaning — deliberately NOT narrowed to behavioral misses, so a released CLI
+       * that predates the split cannot read an infrastructure wipeout as a clean
+       * sweep. Use `assertion_failure_runs` for the narrow count.
+       */
       failed_runs: external_exports.number(),
+      /** The `failed_runs` breakdown by cause (see `EvalRunOutcomeKind`); sums to it. */
+      assertion_failure_runs: external_exports.number(),
+      errored_runs: external_exports.number(),
+      invalid_runs: external_exports.number(),
       created_by: external_exports.string(),
       /** Optional — sessions created pre-PR2 may not have this. */
       scenario_set_id: external_exports.string().nullable().optional(),
@@ -9388,6 +9407,16 @@ function extractApiErrorCode(err) {
     return null;
   }
 }
+function extractApiRetryAfterSeconds(err) {
+  if (!(err instanceof ApiError)) return null;
+  try {
+    const parsed = JSON.parse(err.body);
+    const seconds = parsed?.details?.retry_after_seconds;
+    return typeof seconds === "number" && Number.isFinite(seconds) && seconds >= 0 ? seconds : null;
+  } catch {
+    return null;
+  }
+}
 function friendlyHint(err) {
   if (isNetworkError(err)) {
     return "Couldn't reach WayAI \u2014 check your network connection and try again.";
@@ -11890,7 +11919,17 @@ var init_dist = __esm({
       total_evals: external_exports.number(),
       total_runs: external_exports.number(),
       successful_runs: external_exports.number(),
+      /**
+       * Every terminal run that did NOT pass, whatever the reason. Unchanged
+       * meaning — deliberately NOT narrowed to behavioral misses, so a released CLI
+       * that predates the split cannot read an infrastructure wipeout as a clean
+       * sweep. Use `assertion_failure_runs` for the narrow count.
+       */
       failed_runs: external_exports.number(),
+      /** The `failed_runs` breakdown by cause (see `EvalRunOutcomeKind`); sums to it. */
+      assertion_failure_runs: external_exports.number(),
+      errored_runs: external_exports.number(),
+      invalid_runs: external_exports.number(),
       created_by: external_exports.string(),
       /** Optional — sessions created pre-PR2 may not have this. */
       scenario_set_id: external_exports.string().nullable().optional(),
@@ -18789,15 +18828,24 @@ var init_send_message = __esm({
 });
 
 // src/lib/eval-format.ts
+function runNotes(unproven, inProgress) {
+  return [
+    unproven > 0 ? `${unproven} unproven` : "",
+    inProgress > 0 ? `${inProgress} in progress` : ""
+  ].filter(Boolean);
+}
 function printResultsTable(results, mode) {
   let totalPassed = 0;
-  let totalTerminal = 0;
+  let totalEvaluated = 0;
+  let totalUnproven = 0;
   let totalInProgress = 0;
   for (const r of results) {
     const evalName = r.eval?.eval_name || r.eval_id.slice(0, 8);
-    const terminalRuns = r.successful_runs + r.failed_runs;
+    const evaluatedRuns = evaluatedRunCount(r);
+    const unproven = unprovenRunCount(r);
     const inProgress = (r.running_runs ?? 0) + (r.pending_runs ?? 0);
-    const passed = inProgress > 0 ? `${r.successful_runs}/${terminalRuns} passed (+${inProgress} in progress)` : `${r.successful_runs}/${terminalRuns} passed`;
+    const notes2 = runNotes(unproven, inProgress);
+    const passed = notes2.length > 0 ? `${r.successful_runs}/${evaluatedRuns} passed (${notes2.join(", ")})` : `${r.successful_runs}/${evaluatedRuns} passed`;
     const avgTime = r.avg_execution_time_ms != null ? `avg ${(r.avg_execution_time_ms / 1e3).toFixed(1)}s` : "";
     let scoresStr = "";
     if (r.aggregated_scores) {
@@ -18808,13 +18856,15 @@ function printResultsTable(results, mode) {
     }
     console.log(`  ${evalName.padEnd(20)}  ${passed.padEnd(14)}  ${avgTime}${scoresStr}`);
     totalPassed += r.successful_runs;
-    totalTerminal += terminalRuns;
+    totalEvaluated += evaluatedRuns;
+    totalUnproven += unproven;
     totalInProgress += inProgress;
   }
-  const pct = totalTerminal > 0 ? (totalPassed / totalTerminal * 100).toFixed(1) : "0.0";
-  const inProgressNote = totalInProgress > 0 ? ` \u2014 ${totalInProgress} in progress` : "";
+  const pct = totalEvaluated > 0 ? (totalPassed / totalEvaluated * 100).toFixed(1) : "0.0";
+  const notes = runNotes(totalUnproven, totalInProgress);
+  const note = notes.length > 0 ? ` \u2014 ${notes.join(", ")}` : "";
   console.log(`
-Overall: ${totalPassed}/${totalTerminal} passed (${pct}%)${inProgressNote}`);
+Overall: ${totalPassed}/${totalEvaluated} passed (${pct}%)${note}`);
 }
 function truncate(str, maxLen) {
   if (str.length <= maxLen) return str;
@@ -18836,6 +18886,17 @@ function isFailingRun(run) {
   if (run.run_status && !TERMINAL_RUN_STATUSES.has(run.run_status)) return false;
   return run.response_match !== true;
 }
+function failureTag(run) {
+  switch (run.outcome_kind) {
+    case "execution_error":
+      return "ERROR";
+    case "evaluation_invalid":
+      return "INVALID";
+    // `assertion_failure`, and any pre-taxonomy payload.
+    default:
+      return "FAIL";
+  }
+}
 function printFailingRuns(runs) {
   const failing = runs.filter(isFailingRun);
   if (failing.length === 0) return;
@@ -18844,7 +18905,7 @@ Failures (${failing.length}):`);
   let anyComment = false;
   for (const run of failing) {
     const evalName = run.eval?.eval_name || run.eval_id?.slice(0, 8) || "eval";
-    console.log(`  ${evalName} \u2014 Run #${run.run_number} [FAIL]`);
+    console.log(`  ${evalName} \u2014 Run #${run.run_number} [${failureTag(run)}]`);
     if (run.error_message) {
       console.log(`    Error: ${run.error_message}`);
     }
@@ -18866,6 +18927,7 @@ var NO_EVALUATOR_NOTES_HINT, TERMINAL_RUN_STATUSES;
 var init_eval_format = __esm({
   "src/lib/eval-format.ts"() {
     "use strict";
+    init_contracts();
     NO_EVALUATOR_NOTES_HINT = "No evaluator notes found \u2014 add a `notes` evaluation variable to the message_evaluator agent to surface its reasoning here.";
     TERMINAL_RUN_STATUSES = /* @__PURE__ */ new Set(["completed", "failed"]);
   }
@@ -20002,17 +20064,22 @@ var init_eval_session_control = __esm({
 });
 
 // src/lib/eval-fixture-queue.ts
+function isQueueableLaunchError(err) {
+  const code = extractApiErrorCode(err);
+  return code !== null && QUEUEABLE_LAUNCH_ERRORS.has(code);
+}
 async function runWithFixtureQueue(opts) {
   let delay2 = FIXTURE_QUEUE_INITIAL_DELAY_MS;
   for (; ; ) {
     try {
       return await opts.attempt();
     } catch (err) {
-      if (extractApiErrorCode(err) !== QUEUEABLE_CONFLICT) throw err;
+      if (!isQueueableLaunchError(err)) throw err;
       if (opts.noQueue) throw err;
       const remaining = opts.deadlineMs - opts.now();
       if (remaining <= 0) throw err;
-      const waitMs = Math.min(delay2, remaining);
+      const retryAfterMs = (extractApiRetryAfterSeconds(err) ?? 0) * 1e3;
+      const waitMs = Math.min(Math.max(delay2, retryAfterMs), remaining);
       opts.onWait({ message: extractApiMessage(err), waitMs });
       await opts.sleep(waitMs);
       if (await opts.interrupted?.()) throw err;
@@ -20021,14 +20088,17 @@ async function runWithFixtureQueue(opts) {
     }
   }
 }
-var FIXTURE_QUEUE_INITIAL_DELAY_MS, FIXTURE_QUEUE_MAX_DELAY_MS, QUEUEABLE_CONFLICT;
+var FIXTURE_QUEUE_INITIAL_DELAY_MS, FIXTURE_QUEUE_MAX_DELAY_MS, QUEUEABLE_LAUNCH_ERRORS;
 var init_eval_fixture_queue = __esm({
   "src/lib/eval-fixture-queue.ts"() {
     "use strict";
     init_errors2();
     FIXTURE_QUEUE_INITIAL_DELAY_MS = 5e3;
     FIXTURE_QUEUE_MAX_DELAY_MS = 3e4;
-    QUEUEABLE_CONFLICT = "fixture_target_in_use";
+    QUEUEABLE_LAUNCH_ERRORS = /* @__PURE__ */ new Set([
+      "fixture_target_in_use",
+      "fixture_seed_unavailable"
+    ]);
   }
 });
 
@@ -20204,8 +20274,8 @@ async function pollEvalSession(input) {
     if (await signalController.interrupted()) return;
     console.error(`
 Failed to start session: ${extractApiMessage(err)}`);
-    if (extractApiErrorCode(err) === "fixture_target_in_use" && !noQueue) {
-      console.error(`Waited ${timeoutSeconds}s for the fixture. Raise --timeout, or use --no-queue to fail immediately.`);
+    if (isQueueableLaunchError(err) && !noQueue) {
+      console.error(`Waited ${timeoutSeconds}s for the fixture to become available. Raise --timeout, or use --no-queue to fail immediately.`);
     }
     console.error(`Session ID: ${sessionId}`);
     console.error(`Stop it if it started anyway: ${manualStopCommand}`);
@@ -20243,7 +20313,7 @@ Lost connection to the eval session: ${extractApiMessage(err)}`);
     const session = details.data.session;
     const results = details.data.results;
     const totalRuns = results.reduce((sum, r) => sum + r.total_runs, 0);
-    const completedRuns = results.reduce((sum, r) => sum + r.successful_runs + r.failed_runs, 0);
+    const completedRuns = results.reduce((sum, r) => sum + terminalRunCount(r), 0);
     if (TERMINAL_STATUSES.has(session.session_status)) {
       if (await signalController.interrupted()) return;
       console.log("");
@@ -25186,7 +25256,7 @@ Flags:
   --connection <name>    MCP connection to re-sync \u2014 display name or UUID (sync-mcp)
   --enabled/--disabled   Filter evals by status (evals)
   --no-wait              Don't wait for eval session to complete (run-eval)
-  --no-queue             Fail immediately if another session holds the fixture (run-eval; implied by --no-wait)
+  --no-queue             Fail immediately instead of waiting for the fixture to free up (run-eval; implied by --no-wait)
   --timeout <seconds>    Max wait time for eval session (default: 600)
   --pacing <preset|ms>   Run pacing: conservative | balanced | fast | <milliseconds> (run-eval; default balanced)
   --fixture <name>       Override the suite's declared seed fixture for this run (run-eval)