@wayai/cli 0.3.125 → 0.3.126

package/dist/index.js

@@ -8927,6 +8927,11 @@ var init_api_client = __esm({
         if (fixture) params.set("fixture", fixture);
         return this.request("POST", `/api/evals/sessions/${sessionId}/run?${params.toString()}`);
       }
+      /** Cancel a running eval session. A session that already finished is left unchanged. */
+      async stopEvalSession(hubId, sessionId) {
+        const params = new URLSearchParams({ hub_id: hubId });
+        return this.request("POST", `/api/evals/sessions/${sessionId}/stop?${params.toString()}`);
+      }
       async getEvalSessionDetails(hubId, sessionId) {
         const params = new URLSearchParams({ hub_id: hubId });
         return this.request("GET", `/api/evals/sessions/${sessionId}?${params.toString()}`);
@@ -9332,6 +9337,43 @@ var init_api_client = __esm({
   }
 });
 
+// src/lib/errors.ts
+function extractApiMessage(err) {
+  if (err instanceof ApiError) {
+    try {
+      const parsed = JSON.parse(err.body);
+      if (parsed.message) return parsed.message;
+      if (parsed.error) return parsed.error;
+    } catch {
+    }
+    return err.body || err.message;
+  }
+  return err instanceof Error ? err.message : String(err);
+}
+function friendlyHint(err) {
+  if (isNetworkError(err)) {
+    return "Couldn't reach WayAI \u2014 check your network connection and try again.";
+  }
+  if (!(err instanceof ApiError)) return null;
+  if (err.body.includes("FREE_PLAN_RESTRICTION")) {
+    return "Publishing to production requires a paid plan. Upgrade your organization to publish or sync.";
+  }
+  const { status } = err;
+  if (status === 401) return "Your session may have expired. Run `wayai login` to re-authenticate.";
+  if (status === 402) return "Quota exceeded for your plan. Review usage or upgrade your plan.";
+  if (status === 403) return "Access denied. Your token may lack permission for this hub or resource.";
+  if (status === 429) return "Rate limited. Wait a moment and try again.";
+  if (status >= 500) return "WayAI server error \u2014 likely temporary. Try again shortly.";
+  return null;
+}
+var init_errors2 = __esm({
+  "src/lib/errors.ts"() {
+    "use strict";
+    init_api_client();
+    init_network_error();
+  }
+});
+
 // ../../packages/core/dist/index.js
 function isPublicIdentifier(value) {
   return UUID_RE.test(value) || HEX_RE.test(value) || WORKOS_ID_RE.test(value);
@@ -19772,6 +19814,126 @@ var init_evals = __esm({
   }
 });
 
+// src/lib/eval-session-control.ts
+function formatEvalSessionStopCommand(sessionId, hubId) {
+  return `wayai eval session stop ${sessionId} --hub ${hubId}`;
+}
+function formatEvalResultsCommand(sessionId, hubId) {
+  return `wayai eval-results --session ${sessionId} --hub ${hubId}`;
+}
+function classifyStopOutcome(response) {
+  return response.data.stopped ? { kind: "cancelled" } : { kind: "already_finished", status: response.data.session.session_status };
+}
+function isTerminalStopFailure(err) {
+  return err instanceof ApiError && err.status >= 400 && err.status < 500 && err.status !== 408 && err.status !== 429;
+}
+async function stopEvalSessionWithRetries(stop, opts = {}) {
+  const attempts = opts.attempts ?? STOP_ATTEMPTS;
+  const attemptTimeoutMs = opts.attemptTimeoutMs ?? STOP_ATTEMPT_TIMEOUT_MS;
+  const backoffMs = opts.backoffMs ?? STOP_BACKOFF_MS;
+  const sleep2 = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+  let lastError = new Error("Stop was never attempted");
+  for (let attempt = 1; attempt <= attempts; attempt++) {
+    let timeout;
+    try {
+      return await Promise.race([
+        stop(),
+        new Promise((_resolve, reject) => {
+          timeout = setTimeout(
+            () => reject(new Error(`Stop attempt ${attempt} timed out after ${attemptTimeoutMs}ms`)),
+            attemptTimeoutMs
+          );
+        })
+      ]);
+    } catch (err) {
+      lastError = err;
+      if (isTerminalStopFailure(err)) break;
+    } finally {
+      if (timeout !== void 0) clearTimeout(timeout);
+    }
+    const delay2 = backoffMs[attempt - 1];
+    if (attempt < attempts && delay2) await sleep2(delay2);
+  }
+  throw lastError;
+}
+function installEvalSignalHandlers(input) {
+  const exit = input.exit ?? ((code) => process.exit(code));
+  const manualStopCommand = formatEvalSessionStopCommand(input.sessionId, input.hubId);
+  let signalCount = 0;
+  let disposed = false;
+  let settle;
+  const settled = new Promise((resolve6) => {
+    settle = resolve6;
+  });
+  const listeners = /* @__PURE__ */ new Map();
+  const dispose = () => {
+    if (disposed) return;
+    disposed = true;
+    for (const [signal, listener] of listeners) process.off(signal, listener);
+  };
+  const finish = (exitCode) => {
+    dispose();
+    settle();
+    exit(exitCode);
+  };
+  const handle = (signal) => {
+    signalCount += 1;
+    const signalExitCode = SIGNAL_EXIT_CODE[signal];
+    if (signalCount > 1) {
+      console.error("\nSecond signal received; exiting immediately. The remote eval session may still be running.");
+      console.error(`Run: ${manualStopCommand}`);
+      finish(signalExitCode);
+      return;
+    }
+    console.error(`
+${signal} received; stopping remote eval session ${input.sessionId}...`);
+    void stopEvalSessionWithRetries(input.stop).then((response) => {
+      if (signalCount > 1) return;
+      const outcome = classifyStopOutcome(response);
+      if (outcome.kind === "already_finished") {
+        console.error(`Session had already finished (${outcome.status}) \u2014 nothing was cancelled.`);
+        console.error(`See its results: ${formatEvalResultsCommand(input.sessionId, input.hubId)}`);
+      } else {
+        console.error("Remote session cancellation requested.");
+        console.error("A turn already in progress may still be finishing \u2014 wait for it before starting another run.");
+      }
+      finish(signalExitCode);
+    }).catch((err) => {
+      if (signalCount > 1) return;
+      console.error(`Failed to confirm remote cancellation: ${extractApiMessage(err)}`);
+      console.error(`The session may still be running. Run: ${manualStopCommand}`);
+      finish(1);
+    });
+  };
+  for (const signal of HANDLED_SIGNALS) {
+    const listener = () => handle(signal);
+    listeners.set(signal, listener);
+    process.on(signal, listener);
+  }
+  return {
+    manualStopCommand,
+    async interrupted() {
+      if (signalCount === 0) return false;
+      await settled;
+      return true;
+    },
+    dispose
+  };
+}
+var SIGNAL_EXIT_CODE, HANDLED_SIGNALS, STOP_ATTEMPTS, STOP_ATTEMPT_TIMEOUT_MS, STOP_BACKOFF_MS;
+var init_eval_session_control = __esm({
+  "src/lib/eval-session-control.ts"() {
+    "use strict";
+    init_api_client();
+    init_errors2();
+    SIGNAL_EXIT_CODE = { SIGINT: 130, SIGTERM: 143, SIGHUP: 129 };
+    HANDLED_SIGNALS = Object.keys(SIGNAL_EXIT_CODE);
+    STOP_ATTEMPTS = 3;
+    STOP_ATTEMPT_TIMEOUT_MS = 1e4;
+    STOP_BACKOFF_MS = [500, 1e3];
+  }
+});
+
 // src/commands/run-eval.ts
 var run_eval_exports = {};
 __export(run_eval_exports, {
@@ -19858,6 +20020,31 @@ async function runEvalCommand(args2) {
   const sessionName = `CLI run ${now.toISOString().slice(0, 16).replace("T", " ")}`;
   const createResult = await client.createEvalSession(hubId, sessionName, scenarioSetId, pacing);
   const sessionId = createResult.data.session.eval_session_id;
+  const signalController = installEvalSignalHandlers({
+    hubId,
+    sessionId,
+    stop: () => client.stopEvalSession(hubId, sessionId)
+  });
+  try {
+    await pollEvalSession({
+      client,
+      hubId,
+      sessionId,
+      sessionName,
+      pacing,
+      fixtureOverride,
+      noWait,
+      timeoutSeconds,
+      signal: signalController
+    });
+  } finally {
+    signalController.dispose();
+  }
+}
+async function pollEvalSession(input) {
+  const { client, hubId, sessionId, sessionName, pacing, fixtureOverride, noWait, timeoutSeconds } = input;
+  const signalController = input.signal;
+  const manualStopCommand = signalController.manualStopCommand;
   console.log(`Created session: ${sessionName} (${sessionId})`);
   if (pacing) {
     const desc = pacing.preset === "custom" ? `custom (${pacing.custom_interval_ms}ms/run)` : pacing.preset;
@@ -19870,21 +20057,27 @@ async function runEvalCommand(args2) {
       console.log(`Fixture: ${seed.fixture}${seed.target_base ? ` (base: ${seed.target_base})` : ""}`);
     }
   } catch (err) {
+    if (await signalController.interrupted()) return;
     console.error(`
 Failed to start session: ${extractApiMessage(err)}`);
+    console.error(`Session ID: ${sessionId}`);
+    console.error(`Stop it if it started anyway: ${manualStopCommand}`);
     process.exit(1);
   }
+  if (await signalController.interrupted()) return;
   console.log("Session started");
   if (noWait) {
     console.log(`
 Session ID: ${sessionId}`);
     console.log("Use `wayai eval-results --session " + sessionId + "` to check results.");
+    console.log(`Stop it early with: ${manualStopCommand}`);
     return;
   }
   const deadline = Date.now() + timeoutSeconds * 1e3;
   let consecutivePollErrors = 0;
   while (Date.now() < deadline) {
     await sleep(POLL_INTERVAL_MS);
+    if (await signalController.interrupted()) return;
     let details;
     try {
       details = await client.getEvalSessionDetails(hubId, sessionId);
@@ -19894,7 +20087,8 @@ Session ID: ${sessionId}`);
         console.error(`
 
 Lost connection to the eval session: ${extractApiMessage(err)}`);
-        console.error(`The session may still be running. Check results: wayai eval-results --session ${sessionId}`);
+        console.error(`The session may still be running. Stop it with: ${manualStopCommand}`);
+        console.error(`Check results: wayai eval-results --session ${sessionId}`);
         process.exit(1);
       }
       continue;
@@ -19904,6 +20098,7 @@ Lost connection to the eval session: ${extractApiMessage(err)}`);
     const totalRuns = results.reduce((sum, r) => sum + r.total_runs, 0);
     const completedRuns = results.reduce((sum, r) => sum + r.successful_runs + r.failed_runs, 0);
     if (TERMINAL_STATUSES.has(session.session_status)) {
+      if (await signalController.interrupted()) return;
       console.log("");
       console.log(`Session ${session.session_status}: ${session.session_name} (${session.eval_session_id})
 `);
@@ -19925,6 +20120,7 @@ Could not load failure details (${extractApiMessage(err)}).`);
   console.log(`
 
 Timeout after ${timeoutSeconds}s. Session is still running.`);
+  console.log(`Stop it with: ${manualStopCommand}`);
   console.log(`Check results: wayai eval-results --session ${sessionId}`);
 }
 function sleep(ms) {
@@ -19951,18 +20147,6 @@ function parsePacing(raw) {
   }
   return { preset: value };
 }
-function extractApiMessage(err) {
-  if (err instanceof ApiError) {
-    try {
-      const parsed = JSON.parse(err.body);
-      if (parsed.message) return parsed.message;
-      if (parsed.error) return parsed.error;
-    } catch {
-    }
-    return err.body || err.message;
-  }
-  return err instanceof Error ? err.message : String(err);
-}
 var TERMINAL_STATUSES, POLL_INTERVAL_MS, DEFAULT_TIMEOUT_S, MAX_CONSECUTIVE_POLL_ERRORS;
 var init_run_eval = __esm({
   "src/commands/run-eval.ts"() {
@@ -19971,9 +20155,11 @@ var init_run_eval = __esm({
     init_repo_config();
     init_workspace();
     init_api_client();
+    init_errors2();
     init_eval_format();
     init_dist();
     init_contracts();
+    init_eval_session_control();
     TERMINAL_STATUSES = /* @__PURE__ */ new Set(["completed", "failed", "cancelled", "awaiting_review"]);
     POLL_INTERVAL_MS = 3e3;
     DEFAULT_TIMEOUT_S = 600;
@@ -20352,6 +20538,54 @@ var init_journey_capture = __esm({
   }
 });
 
+// src/commands/eval-session-stop.ts
+var eval_session_stop_exports = {};
+__export(eval_session_stop_exports, {
+  evalSessionStopCommand: () => evalSessionStopCommand
+});
+async function evalSessionStopCommand(args2) {
+  const hubId = resolveActiveHubId(args2);
+  const { config, accessToken } = await requireAuth();
+  requireRepoConfig();
+  const sessionId = args2.find((arg) => !arg.startsWith("-"));
+  if (!sessionId) {
+    console.error("Usage: wayai eval session stop <session_id> [--hub <id>]");
+    process.exit(1);
+  }
+  if (!UUID_RE2.test(sessionId)) {
+    console.error(`Invalid session id: ${sessionId}`);
+    process.exit(1);
+  }
+  const client = new ApiClient({ apiUrl: config.api_url, accessToken });
+  try {
+    const outcome = classifyStopOutcome(await client.stopEvalSession(hubId, sessionId));
+    if (outcome.kind === "already_finished") {
+      console.log(`Eval session ${sessionId} had already finished (${outcome.status}) \u2014 nothing was cancelled.`);
+      console.log(`See its results: ${formatEvalResultsCommand(sessionId, hubId)}`);
+    } else {
+      console.log(`Cancellation requested for eval session ${sessionId}.`);
+      console.log("A turn already in progress may still be finishing \u2014 wait for it before starting another run.");
+    }
+  } catch (err) {
+    if (err instanceof ApiError && err.status === 404) {
+      console.error(`Eval session not found: ${sessionId}`);
+      process.exit(1);
+    }
+    throw err;
+  }
+}
+var init_eval_session_stop = __esm({
+  "src/commands/eval-session-stop.ts"() {
+    "use strict";
+    init_auth();
+    init_repo_config();
+    init_workspace();
+    init_api_client();
+    init_eval_session_control();
+    init_utils();
+  }
+});
+
 // src/commands/eval-session-delete.ts
 var eval_session_delete_exports = {};
 __export(eval_session_delete_exports, {
@@ -20457,6 +20691,11 @@ async function evalCommand(args2) {
     }
     case "session": {
       const [sessionSub, ...sessionRest] = rest;
+      if (sessionSub === "stop") {
+        const { evalSessionStopCommand: evalSessionStopCommand2 } = await Promise.resolve().then(() => (init_eval_session_stop(), eval_session_stop_exports));
+        await evalSessionStopCommand2(sessionRest);
+        return;
+      }
       if (sessionSub === "delete") {
         const { evalSessionDeleteCommand: evalSessionDeleteCommand2 } = await Promise.resolve().then(() => (init_eval_session_delete(), eval_session_delete_exports));
         await evalSessionDeleteCommand2(sessionRest);
@@ -20490,6 +20729,8 @@ Subcommands:
                                      exchange as a scenario YAML in evals/.
   journey capture <conversation_id>  Capture a real conversation's FULL
                                      transcript as a platform-managed journey.
+  session stop <session_id>          Cancel a running eval session (e.g. one
+                                     left behind by an interrupted run-eval).
   session delete <session_id>        Delete one eval session (and its
                                      conversations). Destructive, irreversible.
   session delete --all               Delete every eval session on the hub.
@@ -20511,6 +20752,7 @@ Examples:
   wayai eval capture 11111111-2222-3333-4444-555555555555
   wayai eval capture <conv_id> --set regression-suite --name "Cancel order"
   wayai eval journey capture <conv_id> --name "Refund happy path"
+  wayai eval session stop 11111111-2222-3333-4444-555555555555
   wayai eval session delete 11111111-2222-3333-4444-555555555555
   wayai eval session delete --all -y
 
@@ -24282,34 +24524,13 @@ var init_update = __esm({
 
 // src/index.ts
 init_sentry();
+init_errors2();
+init_mask_secrets();
+init_utils();
 import { readFileSync as readFileSync21 } from "fs";
 import { fileURLToPath as fileURLToPath2 } from "url";
 import { dirname as dirname9, join as join28 } from "path";
 
-// src/lib/errors.ts
-init_api_client();
-init_network_error();
-function friendlyHint(err) {
-  if (isNetworkError(err)) {
-    return "Couldn't reach WayAI \u2014 check your network connection and try again.";
-  }
-  if (!(err instanceof ApiError)) return null;
-  if (err.body.includes("FREE_PLAN_RESTRICTION")) {
-    return "Publishing to production requires a paid plan. Upgrade your organization to publish or sync.";
-  }
-  const { status } = err;
-  if (status === 401) return "Your session may have expired. Run `wayai login` to re-authenticate.";
-  if (status === 402) return "Quota exceeded for your plan. Review usage or upgrade your plan.";
-  if (status === 403) return "Access denied. Your token may lack permission for this hub or resource.";
-  if (status === 429) return "Rate limited. Wait a moment and try again.";
-  if (status >= 500) return "WayAI server error \u2014 likely temporary. Try again shortly.";
-  return null;
-}
-
-// src/index.ts
-init_mask_secrets();
-init_utils();
-
 // src/lib/version-refresh.ts
 init_version_cache();
 init_skill_version();
@@ -24718,6 +24939,7 @@ Commands:
   run-eval           Run all enabled evals and show results
   eval-results       View results from a completed eval session
   eval capture       Capture a real conversation as a scenario YAML
+  eval session stop  Cancel a running eval session (recovery for an interrupted run-eval)
   eval session delete  Delete one eval session (\`--all\` for every session on the hub)
   sync-skills        Sync skills to Anthropic/OpenAI provider connections
   sync-mcp           Re-discover an MCP connection's tools (refresh stale schemas)