@skaile/workspaces 3.7.2 → 3.8.0

package/dist/bridge/drivers/claude-sdk.js

@@ -2,7 +2,7 @@ import { scrubPoisonedTranscript, resolveClaudeConfigDir, SUPPORTED_CLOUDS, buil
 import { dispatchCapability } from '../../chunk-RRVQAE5D.js';
 import { fetchProviderModels } from '../../chunk-V34ZEXSE.js';
 import { AuthError, classifyClaudeSdkError, reclassifyAuthAsAccountUnavailable, AccountUnavailableError } from '../../chunk-GAH4YRCI.js';
-import { BUILTIN_DRIVER_CATALOG, AgentDriver, getBridgeLogger } from '../../chunk-32TJ55W5.js';
+import { BUILTIN_DRIVER_CATALOG, AgentDriver, getBridgeLogger } from '../../chunk-BGAAZMBK.js';
 import '../../chunk-6E6PKKAD.js';
 import '../../chunk-AZPXCHDK.js';
 import '../../chunk-F3HNW354.js';
@@ -159,7 +159,12 @@ if (typeof globalThis.require === "undefined") {
   }
 }
 function freshRetryBudget() {
-  return { staleResume: false, poison: false, auth: false, queryEnded: false };
+  return { staleResume: false, poison: false, auth: false, queryEnded: false, limit: false };
+}
+function isSeatCapacityFailure(detail, subtype) {
+  if (detail.category !== "rate_limit") return false;
+  if (subtype === "error_max_budget_usd") return false;
+  return detail.statusCode !== 529;
 }
 var QueryEndedBeforeTurnError = class extends Error {
   constructor(message) {
@@ -216,6 +221,7 @@ var PushableInputStream = class {
   }
 };
 var ABORT_GRACE_MS = 3e3;
+var QUESTION_SUPERSEDED_BY_PROMPT = "The user replied in chat instead of answering this dialog, so the question was cancelled. Their reply arrives as the next user turn \u2014 read it before you re-ask or assume an answer.";
 var claudeBinaryCache;
 var claudeBinaryResolved = false;
 var ClaudeSdkDriver = class extends AgentDriver {
@@ -291,6 +297,52 @@ var ClaudeSdkDriver = class extends AgentDriver {
    * reset elsewhere — the next call's set is the only legitimate transition.
    */
   authSelfHealUsed = false;
+  /**
+   * Mirror of the in-flight turn's seat-failover budget — same contract as
+   * {@link authSelfHealUsed}, for the `onLimitBlocked` path.
+   */
+  limitFailoverUsed = false;
+  /**
+   * Set when THIS attempt saw an upstream usage-limit rejection: either the
+   * SDK's `rate_limit_event` telemetry with `status: "rejected"`, or an error
+   * result the classifier calls `rate_limit`.
+   *
+   * The telemetry is the load-bearing half. A subscription ceiling arrives as
+   * `{"status":"rejected","type":"five_hour"}` followed by an error result
+   * whose text ("You've hit your session limit · resets 9:50am") matches none
+   * of the classifier's rate-limit keywords — so without this flag the
+   * rejection classifies as `unknown` and no failover is reachable at all.
+   *
+   * Reset at the top of every {@link runPrompt} attempt, so the replay after a
+   * failover observes its own outcome rather than inheriting the first one's.
+   */
+  limitRejectionSeen = false;
+  /**
+   * True when {@link handleResultErrorPayload} rejected this turn ONLY so the
+   * seat-failover branch could run — a turn that, before this existed, would
+   * have emitted its error and resolved. If the failover then declines, the
+   * rejection is undone so a limit block with no peer available looks exactly
+   * as it always has downstream (one error event, one `agent_end`, a resolved
+   * promise) rather than also drawing the runner's fatal "prompt failed".
+   */
+  limitTurnRejectedForFailover = false;
+  /**
+   * Whether this attempt already reported a provider rejection. One upstream
+   * failure routinely reaches the driver twice — `rejected` telemetry followed
+   * by the error result, or an `is_error` result followed by the iterator throw
+   * carrying the real text — and these observations feed counters, so counting
+   * one rejection twice would overstate the seat's pressure.
+   */
+  providerResponseReported = false;
+  /** Limit-window metadata from this attempt's rejection telemetry, if any. */
+  lastLimitInfo = null;
+  /**
+   * The `agent-event: error` {@link failTurn} withheld because a recovery
+   * (auth self-heal or seat failover) was about to run. The recovery emits it
+   * if it declines, and drops it if it retries — so a recovered turn shows the
+   * user nothing and a failed one shows exactly one error.
+   */
+  deferredTurnError = null;
   /** Tracks whether a session has been started (for continue: true). */
   hasSession = false;
   /** Session ID from the SDK — used for streamInput messages. */
@@ -404,7 +456,7 @@ var ClaudeSdkDriver = class extends AgentDriver {
    * occurs after a container crash that prevented the SDK from flushing its session file.
    */
   prompt(message) {
-    this.denyPendingQuestion("superseded by a new prompt");
+    this.denyPendingQuestion(QUESTION_SUPERSEDED_BY_PROMPT);
     const queuedPrompt = this.promptTail.then(() => this.runPrompt(message, freshRetryBudget()));
     this.promptTail = queuedPrompt.then(
       () => void 0,
@@ -421,8 +473,14 @@ var ClaudeSdkDriver = class extends AgentDriver {
   async runPrompt(message, _budget) {
     if (!this.sdk) await this.start();
     this.authSelfHealUsed = _budget.auth;
+    this.limitFailoverUsed = _budget.limit;
+    this.limitRejectionSeen = false;
+    this.limitTurnRejectedForFailover = false;
+    this.providerResponseReported = false;
+    this.lastLimitInfo = null;
+    this.deferredTurnError = null;
     let retrying = false;
-    const isFirstAttempt = !_budget.staleResume && !_budget.poison && !_budget.auth && !_budget.queryEnded;
+    const isFirstAttempt = !_budget.staleResume && !_budget.poison && !_budget.auth && !_budget.queryEnded && !_budget.limit;
     if (isFirstAttempt) {
       this._lastTokens = null;
       this._lastContextTokens = null;
@@ -475,12 +533,26 @@ var ClaudeSdkDriver = class extends AgentDriver {
           return this.runPrompt(message, { ..._budget, poison: true });
         }
       }
-      const isAuthError = err instanceof AuthError && !_budget.auth;
-      if (isAuthError && this.config.onAuthError) {
+      const authOwns = err instanceof AuthError && !_budget.auth && this.config.onAuthError != null;
+      if (authOwns) {
         if (await this.attemptAuthSelfHeal(err)) {
           retrying = true;
           return this.runPrompt(message, { ..._budget, auth: true });
         }
+      } else if (!_budget.limit && this.limitRejectionSeen && this.config.onLimitBlocked) {
+        if (await this.attemptLimitFailover(err)) {
+          if (this.abortController?.signal.aborted) {
+            this.log.info("aborted during the seat failover; not replaying the prompt");
+            this.emitAgentEnd();
+            return;
+          }
+          retrying = true;
+          return this.runPrompt(message, { ..._budget, limit: true });
+        }
+        if (this.limitTurnRejectedForFailover) {
+          this.emitAgentEnd();
+          return;
+        }
       }
       throw err;
     } finally {
@@ -489,6 +561,10 @@ var ClaudeSdkDriver = class extends AgentDriver {
         this.turnResolve = null;
         this.turnReject = null;
         this.running = false;
+        this.limitRejectionSeen = false;
+        this.limitTurnRejectedForFailover = false;
+        this.lastLimitInfo = null;
+        this.deferredTurnError = null;
       }
     }
   }
@@ -802,18 +878,60 @@ var ClaudeSdkDriver = class extends AgentDriver {
       this.running = false;
       return true;
     }
+    this.emitDeferredTurnError({ type: "error", error: err.message, detail: err.detail });
+    return false;
+  }
+  /**
+   * Side-effects of the seat-failover path: ask the host to re-resolve the AI
+   * provider config behind this session. On `switched: true` drop the query and
+   * return `true` (the caller recurses onto a fresh CLI); otherwise emit the
+   * deferred error and return `false`. Mirrors {@link attemptAuthSelfHeal} —
+   * this performs the state mutation, the caller owns the control flow.
+   *
+   * Dropping `this.query` is the whole point of the retry, not bookkeeping: a
+   * resident CLI caches "blocked until <reset>" and short-circuits the next
+   * prompt locally with no API attempt at all (skaile-ai/workspaces#566), so a
+   * rewritten credentials file alone would change nothing. Only a fresh spawn
+   * clears it. `sessionId` / `hasSession` are preserved so the new query
+   * resumes the same conversation.
+   */
+  async attemptLimitFailover(err) {
+    this.log.warn("turn blocked by an upstream usage limit; invoking onLimitBlocked", {
+      configId: this.config.aiProviderConfigId,
+      limitType: this.lastLimitInfo?.limitType,
+      resetsAt: this.lastLimitInfo?.resetsAt
+    });
+    let outcome;
     try {
-      this.emit("agent-event", {
-        type: "error",
-        error: err.message,
-        detail: err.detail
+      outcome = await this.config.onLimitBlocked({
+        configId: this.config.aiProviderConfigId ?? "",
+        ...this.lastLimitInfo ?? {}
       });
-    } catch (emitErr) {
-      this.log.warn(
-        "agent-event listener threw during deferred auth-error surfacing; suppressing",
-        { error: emitErr instanceof Error ? emitErr.message : String(emitErr) }
-      );
+    } catch (failoverErr) {
+      this.log.warn("onLimitBlocked callback threw; surfacing the original limit error", {
+        error: failoverErr instanceof Error ? failoverErr.message : String(failoverErr)
+      });
+    }
+    if (outcome?.switched === true) {
+      this.log.info("switched AI provider config; respawning the CLI and replaying the prompt", {
+        configId: outcome.configId
+      });
+      this.query = null;
+      this.endInputStream();
+      this.turnResolve = null;
+      this.turnReject = null;
+      this.running = false;
+      return true;
     }
+    this.log.info("no alternative AI provider config available; surfacing the limit error", {
+      reason: outcome?.reason
+    });
+    const detail = err?.detail;
+    this.emitDeferredTurnError({
+      type: "error",
+      error: err instanceof Error ? err.message : String(err),
+      ...detail ? { detail } : {}
+    });
     return false;
   }
   /**
@@ -1260,6 +1378,12 @@ var ClaudeSdkDriver = class extends AgentDriver {
       category: detail.category,
       retryable: detail.retryable
     });
+    if (detail.category === "auth") {
+      this.reportProviderResponse({ status: 401 });
+    } else if (isSeatCapacityFailure(detail)) {
+      this.limitRejectionSeen = true;
+      this.reportProviderResponse({ status: 429, ...this.lastLimitInfo ?? {} });
+    }
     const surfaced = detail.category === "auth" ? new AuthError(detail) : detail.category === "account" ? new AccountUnavailableError(detail) : error;
     this.failTurn(surfaced, detail);
   }
@@ -1280,10 +1404,14 @@ var ClaudeSdkDriver = class extends AgentDriver {
       this.turnResolve = null;
       this.turnReject = null;
     }
+    this.emitAgentEnd();
+  }
+  /** Emit the turn-end signal, swallowing a throwing listener. */
+  emitAgentEnd() {
     try {
       this.emit("agent-event", { type: "agent_end" });
     } catch (emitErr) {
-      this.log.warn("agent-event listener threw during completeTurn; suppressing", {
+      this.log.warn("agent-event listener threw while ending the turn; suppressing", {
         error: emitErr instanceof Error ? emitErr.message : String(emitErr)
       });
     }
@@ -1343,29 +1471,62 @@ var ClaudeSdkDriver = class extends AgentDriver {
    */
   failTurn(err, detail) {
     this.disarmTurnStallWatchdog();
+    const hadPendingTurn = this.turnReject != null;
     if (this.turnReject) {
       this.turnReject(err);
       this.turnResolve = null;
       this.turnReject = null;
     }
     this.turnCompleted = true;
+    const event = {
+      type: "error",
+      error: err.message,
+      ...detail ? { detail } : {}
+    };
     const willSelfHeal = err instanceof AuthError && !this.authSelfHealUsed && this.config.onAuthError != null;
-    if (willSelfHeal) {
+    if (hadPendingTurn && willSelfHeal) {
       this.log.info("deferring auth-error event; self-heal will attempt refresh");
+      this.deferredTurnError = event;
       return;
     }
+    if (hadPendingTurn && this.willFailoverOnLimit()) {
+      this.log.info("deferring limit error event; seat failover will be attempted");
+      this.deferredTurnError = event;
+      return;
+    }
+    this.emitTurnErrorEvent(event);
+  }
+  /**
+   * Whether a usage-limit block on the current attempt still has a seat
+   * failover available. Evaluated identically by {@link failTurn} (to decide
+   * whether to withhold the error event) and by {@link handleResultErrorPayload}
+   * (to decide whether to reject the turn at all) — one predicate so the two
+   * can never disagree and strand a deferred event.
+   */
+  willFailoverOnLimit() {
+    return this.limitRejectionSeen && !this.limitFailoverUsed && this.config.onLimitBlocked != null;
+  }
+  /** Emit one `agent-event: error`, swallowing a throwing listener. */
+  emitTurnErrorEvent(event) {
     try {
-      this.emit("agent-event", {
-        type: "error",
-        error: err.message,
-        ...detail ? { detail } : {}
-      });
+      this.emit("agent-event", event);
     } catch (emitErr) {
-      this.log.warn("agent-event listener threw during failTurn; suppressing", {
+      this.log.warn("agent-event listener threw while surfacing a turn error; suppressing", {
         error: emitErr instanceof Error ? emitErr.message : String(emitErr)
       });
     }
   }
+  /**
+   * Emit the error {@link failTurn} withheld, exactly once. Called by a
+   * recovery path that decided NOT to retry, so every failure produces one
+   * downstream error event and a recovered one produces none. `fallback` covers
+   * the paths that reject without going through `failTurn`.
+   */
+  emitDeferredTurnError(fallback) {
+    const event = this.deferredTurnError ?? fallback;
+    this.deferredTurnError = null;
+    this.emitTurnErrorEvent(event);
+  }
   // ---------------------------------------------------------------------------
   // Message handling
   // ---------------------------------------------------------------------------
@@ -1382,9 +1543,17 @@ var ClaudeSdkDriver = class extends AgentDriver {
     }
   }
   /**
-   * Surface subscription rate-limit telemetry as a structured warning log.
+   * Surface subscription rate-limit telemetry as a structured warning log, and
+   * — on `status: "rejected"` — arm the seat failover and report the rejection
+   * to the host for per-seat health attribution.
+   *
    * Returns `true` when the message was a rate-limit event (and fully handled),
    * so the caller can stop dispatching.
+   *
+   * This used to only log. That is what left a session pinned to a parked seat:
+   * the rejection was fully handled here and nothing propagated, so the
+   * platform's re-resolution — which was ready and working — was reachable only
+   * through a token refresh that happened to be due (skaile-ai/platform#2976).
    */
   handleRateLimitEvent(msg) {
     const maybeRateLimit = msg;
@@ -1396,10 +1565,39 @@ var ClaudeSdkDriver = class extends AgentDriver {
         utilization: info.utilization,
         resetsAt: info.resetsAt
       });
+      if (info.status === "rejected") {
+        this.limitRejectionSeen = true;
+        this.lastLimitInfo = {
+          ...info.rateLimitType ? { limitType: info.rateLimitType } : {},
+          ...typeof info.resetsAt === "number" ? { resetsAt: info.resetsAt } : {},
+          ...typeof info.utilization === "number" ? { utilization: info.utilization } : {}
+        };
+        this.reportProviderResponse({ status: 429, ...this.lastLimitInfo });
+      }
       return true;
     }
     return false;
   }
+  /**
+   * Report an observed 401 / 429 to the host so the owning AI provider config's
+   * live health counters can be bumped (skaile-ai/workspaces#368). Strictly
+   * fire-and-forget: a throwing host callback must never fail the turn that
+   * produced the observation.
+   */
+  reportProviderResponse(observation) {
+    if (!this.config.onProviderResponse || this.providerResponseReported) return;
+    this.providerResponseReported = true;
+    try {
+      this.config.onProviderResponse({
+        ...observation,
+        ...this.config.aiProviderConfigId ? { configId: this.config.aiProviderConfigId } : {}
+      });
+    } catch (err) {
+      this.log.warn("onProviderResponse threw; suppressing", {
+        error: err instanceof Error ? err.message : String(err)
+      });
+    }
+  }
   handleAssistantMessage(msg) {
     this.captureAssistantContext(msg);
     const agentMsg = this.mapAssistantMessage(msg);
@@ -1593,11 +1791,20 @@ var ClaudeSdkDriver = class extends AgentDriver {
       category: detail.category,
       retryable: detail.retryable
     });
+    if (detail.category === "auth") {
+      this.reportProviderResponse({ status: 401 });
+    } else if (isSeatCapacityFailure(detail, msg.subtype)) {
+      this.limitRejectionSeen = true;
+      this.reportProviderResponse({ status: 429, ...this.lastLimitInfo ?? {} });
+    }
     const errorText = errors.join("; ") || `Agent stopped: ${msg.subtype}`;
     if (detail.category === "auth") {
       this.failTurn(new AuthError({ ...detail, message: errorText }), detail);
     } else if (detail.category === "account") {
       this.failTurn(new AccountUnavailableError({ ...detail, message: errorText }), detail);
+    } else if (this.willFailoverOnLimit()) {
+      this.limitTurnRejectedForFailover = true;
+      this.failTurn(new Error(errorText), detail);
     } else if (errors.length > 0) {
       this.emit("agent-event", {
         type: "error",