cascade-ai 0.49.0 → 0.50.0

package/dist/cli.js

@@ -1740,7 +1740,7 @@ var CASCADE_VERSION, CASCADE_CONFIG_FILE, CASCADE_DB_FILE, CASCADE_DASHBOARD_SEC
 var init_constants = __esm({
   "src/constants.ts"() {
     init_pricing();
-    CASCADE_VERSION = "0.49.0";
+    CASCADE_VERSION = "0.50.0";
     CASCADE_CONFIG_FILE = ".cascade/config.json";
     CASCADE_DB_FILE = ".cascade/memory.db";
     CASCADE_DASHBOARD_SECRET_FILE = ".cascade/dashboard-secret";
@@ -7615,6 +7615,14 @@ var BaseTier = class extends EventEmitter {
     this.emit("tier:status", event);
   }
   /** Record the model serving this tier; future status events carry it. */
+  /**
+   * The model actually serving this tier ("provider:id"), once one has been
+   * chosen. The run breaker reads it to attribute a failure to a model rather
+   * than to the subtask that happened to hit it.
+   */
+  getServingModel() {
+    return this.servingModel;
+  }
   setServingModel(model) {
     this.servingModel = model || void 0;
   }
@@ -9351,6 +9359,7 @@ var T2Manager = class extends BaseTier {
   escalations = [];
   peerSyncBuffer = [];
   store;
+  runBreaker;
   t3PeerBus = new PeerBus();
   // ← T3↔T3 bus (local to this T2)
   t2PeerBus;
@@ -9381,6 +9390,15 @@ var T2Manager = class extends BaseTier {
     this.router = router;
     this.toolRegistry = toolRegistry;
   }
+  /**
+   * Share the run-wide circuit breaker. Every T2 in a run gets the SAME
+   * instance: three sections each failing once against a dead model is the
+   * same fact as one section failing three times, and a per-section breaker
+   * would never see it.
+   */
+  setRunBreaker(breaker) {
+    this.runBreaker = breaker;
+  }
   setStore(store) {
     this.store = store;
   }
@@ -9504,7 +9522,8 @@ Guidance (must be followed): ${decision.note}`
       const overallStatus = this.determineStatus(t3Results);
       const isOk = overallStatus === "COMPLETED" || overallStatus === "PARTIAL";
       this.setStatus(isOk ? "COMPLETED" : "FAILED", summary);
-      this.sendStatusUpdate({ progressPct: 100, currentAction: "Section complete", status: "IN_PROGRESS", output: summary });
+      const sectionAction = overallStatus === "COMPLETED" ? "Section complete" : overallStatus === "PARTIAL" ? "Section partially complete \u2014 some subtasks failed" : overallStatus === "ESCALATED" ? "Section escalated \u2014 needs a decision" : "Section failed \u2014 no subtask completed";
+      this.sendStatusUpdate({ progressPct: 100, currentAction: sectionAction, status: "IN_PROGRESS", output: summary });
       const result = {
         sectionId: assignment.sectionId,
         sectionTitle: assignment.sectionTitle,
@@ -9674,6 +9693,24 @@ HIERARCHY CONTEXT: ${this.hierarchyContext}` : ""),
     const reinforceCfg = this.router.getReinforcementsConfig?.() ?? { enabled: false, maxPerSection: 4 };
     let reinforcementsAdded = 0;
     while (remaining.size > 0) {
+      if (this.runBreaker?.isOpen()) {
+        const why = this.runBreaker.skipMessage();
+        this.log(`Run breaker open \u2014 skipping ${remaining.size} remaining subtask(s)`);
+        for (const id of remaining) {
+          this.t3PeerBus.publish(this.id, id, why, "FAILED");
+          resultMap.set(id, {
+            subtaskId: id,
+            status: "FAILED",
+            output: why,
+            testResults: { checksRun: [], passed: [], failed: [] },
+            issues: [why],
+            peerSyncsUsed: [],
+            correctionAttempts: 0
+          });
+        }
+        remaining.clear();
+        break;
+      }
       const runnableIds = [...remaining].filter((id) => (inDegree.get(id) ?? 0) === 0);
       if (runnableIds.length === 0) {
         const fallbackId = [...remaining].sort(
@@ -9776,8 +9813,32 @@ HIERARCHY CONTEXT: ${this.hierarchyContext}` : ""),
         remaining.delete(id);
         const r = waveResults[i];
         if (r.status === "rejected") {
-          this.log(`T3 worker ${id} failed: ${r.reason instanceof Error ? r.reason.message : String(r.reason)} \u2014 retrying once`);
           const assignment = sanitizedAssignments.find((a) => a.subtaskId === id);
+          const failedModel = workerMap.get(id)?.getServingModel();
+          const tripped = this.runBreaker?.record(r.reason, failedModel);
+          if (tripped) {
+            this.log(`Run breaker opened: ${tripped.failures} consecutive ${tripped.kind} failures on ${tripped.modelId} \u2014 stopping`);
+            this.emit("run:breaker", tripped);
+          }
+          if (this.runBreaker?.isOpen()) {
+            const why = this.runBreaker.skipMessage();
+            this.log(`T3 worker ${id} failed and the run breaker is open \u2014 not retrying`);
+            this.t3PeerBus.publish(this.id, id, why, "FAILED");
+            resultMap.set(id, {
+              subtaskId: id,
+              status: "FAILED",
+              output: why,
+              testResults: { checksRun: [], passed: [], failed: [] },
+              issues: [why],
+              peerSyncsUsed: [],
+              correctionAttempts: 0
+            });
+            for (const dependent of adj.get(id) ?? []) {
+              inDegree.set(dependent, Math.max(0, (inDegree.get(dependent) ?? 0) - 1));
+            }
+            continue;
+          }
+          this.log(`T3 worker ${id} failed: ${r.reason instanceof Error ? r.reason.message : String(r.reason)} \u2014 retrying once`);
           try {
             const retried = await this.retryT3(assignment, taskId);
             resultMap.set(id, retried);
@@ -10112,6 +10173,7 @@ var T1Administrator = class extends BaseTier {
   t2Managers = /* @__PURE__ */ new Map();
   escalations = [];
   store;
+  runBreaker;
   t2PeerBus = new PeerBus();
   permissionEscalator;
   toolCreator;
@@ -10126,6 +10188,10 @@ var T1Administrator = class extends BaseTier {
     this.toolRegistry = toolRegistry;
     this.config = config;
   }
+  /** Share the run-wide circuit breaker with every section manager. */
+  setRunBreaker(breaker) {
+    this.runBreaker = breaker;
+  }
   setStore(store) {
     this.store = store;
   }
@@ -10488,6 +10554,7 @@ SPEC RULES \u2014 each subtask is a self-contained spec slice (workers execute f
     };
     const managers = sections.map((section) => {
       const manager = new T2Manager(this.router, this.toolRegistry, this.id);
+      if (this.runBreaker) manager.setRunBreaker(this.runBreaker);
       manager.setHierarchyContext(`You are a T2 Manager for the section "${section.sectionTitle}". You are part of a COMPLEX task overseen by T1 Administrator.`);
       if (this.store) {
         manager.setStore(this.store);
@@ -10776,6 +10843,158 @@ Reply with exactly one word: YES, NO, or UNSURE.
 // src/core/cascade.ts
 init_cost();
 
+// src/core/router/provider-errors.ts
+var SYSTEMIC = /* @__PURE__ */ new Set([
+  "rate_limit",
+  "quota_exhausted",
+  "auth",
+  "model_unavailable"
+]);
+function isSystemicKind(kind) {
+  return SYSTEMIC.has(kind);
+}
+function statusOf(err) {
+  if (!err || typeof err !== "object") return void 0;
+  const e = err;
+  for (const key of ["status", "statusCode", "code"]) {
+    const v = e[key];
+    if (typeof v === "number" && v >= 100 && v < 600) return v;
+  }
+  const resp = e["response"];
+  if (resp && typeof resp["status"] === "number") return resp["status"];
+  const inner = e["error"];
+  if (inner && typeof inner["code"] === "number") return inner["code"];
+  return void 0;
+}
+function messageOf(err) {
+  if (err instanceof Error) return err.message;
+  if (typeof err === "string") return err;
+  if (err && typeof err === "object") {
+    const e = err;
+    if (typeof e["message"] === "string") return e["message"];
+    const inner = e["error"];
+    if (inner && typeof inner["message"] === "string") return inner["message"];
+    try {
+      return JSON.stringify(err);
+    } catch {
+    }
+  }
+  return String(err);
+}
+function classifyProviderError(err) {
+  const raw = messageOf(err).trim();
+  const status = statusOf(err);
+  const m = raw.toLowerCase();
+  const kind = (() => {
+    if (status === 429) {
+      return /quota|billing|credit|exceeded your current quota|insufficient/.test(m) ? "quota_exhausted" : "rate_limit";
+    }
+    if (status === 401 || status === 403) return "auth";
+    if (status === 404) return "model_unavailable";
+    if (/rate limit|too many requests|resource_exhausted/.test(m)) {
+      return /quota|billing|credit|insufficient/.test(m) ? "quota_exhausted" : "rate_limit";
+    }
+    if (/quota|billing|insufficient_quota|insufficient credit|payment required/.test(m)) return "quota_exhausted";
+    if (/api key|unauthorized|authentication|permission denied|invalid[_ ]api/.test(m)) return "auth";
+    if (/model .*(not found|does not exist|unavailable)|no such model|unknown model|not supported for/.test(m)) {
+      return "model_unavailable";
+    }
+    if (/context length|too many tokens|maximum context|prompt is too long|input too long/.test(m)) return "context_length";
+    if (/safety|content filter|blocked|recitation|prohibited/.test(m)) return "content_filter";
+    if (/econnreset|etimedout|enotfound|socket hang up|network|fetch failed|aborted/.test(m)) return "network";
+    return "unknown";
+  })();
+  return { kind, systemic: isSystemicKind(kind), status, raw };
+}
+function describeProviderError(c, modelId) {
+  const on = ` on ${modelId}` ;
+  const detail = c.raw ? ` Provider said: ${c.raw}` : "";
+  switch (c.kind) {
+    case "rate_limit":
+      return `Rate limit hit${on}. The provider is throttling requests \u2014 lower the number of parallel workers, or use a model with a higher requests-per-minute allowance.${detail}`;
+    case "quota_exhausted":
+      return `Quota or billing limit reached${on}. This will not recover on its own \u2014 check the provider's billing page, or switch this tier to a different provider.${detail}`;
+    case "auth":
+      return `Authentication failed${on}. The API key is missing, expired, or lacks access to this model \u2014 re-check the key in Settings > Providers.${detail}`;
+    case "model_unavailable":
+      return `Model unavailable${on}. The id may be wrong, retired, or not enabled for this API key \u2014 pick a different model for this tier.${detail}`;
+    case "context_length":
+      return `The prompt was too long${on}. Reduce the attached context, or use a model with a larger context window for this tier.${detail}`;
+    case "content_filter":
+      return `The provider's safety filter refused this request${on}.${detail}`;
+    case "network":
+      return `Network error reaching the provider${on}. This is usually transient.${detail}`;
+    default:
+      return `The model call failed${on}.${detail}`;
+  }
+}
+
+// src/core/run-breaker.ts
+var DEFAULT_FAILURE_THRESHOLD = 3;
+var RunBreaker = class {
+  threshold;
+  /** modelId → consecutive systemic failures of the SAME kind. */
+  counts = /* @__PURE__ */ new Map();
+  trip = null;
+  constructor(threshold = DEFAULT_FAILURE_THRESHOLD) {
+    this.threshold = Math.max(1, threshold);
+  }
+  /**
+   * Record a failed model call. Returns the trip if this failure is the one
+   * that opened the breaker, otherwise null.
+   *
+   * A non-systemic failure RESETS the model's counter: it is evidence the model
+   * is reachable and answering, which is exactly what a systemic streak claims
+   * it is not.
+   */
+  record(err, modelId) {
+    if (this.trip) return null;
+    const id = modelId?.trim() || "unknown-model";
+    const c = classifyProviderError(err);
+    if (!c.systemic) {
+      this.counts.delete(id);
+      return null;
+    }
+    const prev = this.counts.get(id);
+    const n = prev && prev.kind === c.kind ? prev.n + 1 : 1;
+    this.counts.set(id, { kind: c.kind, n, last: c });
+    if (n >= this.threshold) {
+      this.trip = {
+        modelId: id,
+        kind: c.kind,
+        failures: n,
+        message: describeProviderError(c, id)
+      };
+      return this.trip;
+    }
+    return null;
+  }
+  /** True once the run should stop launching work that would fail identically. */
+  isOpen() {
+    return this.trip !== null;
+  }
+  /** Why the breaker opened, or null while it is closed. */
+  reason() {
+    return this.trip;
+  }
+  /**
+   * One line for a worker/section that never ran because the breaker was
+   * already open. Says the work was skipped deliberately — not that it failed
+   * on its own merits, which is what an empty FAILED status would imply.
+   */
+  skipMessage() {
+    const t = this.trip;
+    if (!t) return "Skipped.";
+    return `Skipped \u2014 the run stopped after ${t.failures} consecutive failures on ${t.modelId}. ${t.message}`;
+  }
+  /** Test/diagnostic view of the per-model streaks. */
+  streaks() {
+    const out = {};
+    for (const [k, v] of this.counts) out[k] = { kind: v.kind, n: v.n };
+    return out;
+  }
+};
+
 // src/tools/registry.ts
 init_constants();
 
@@ -15908,6 +16127,7 @@ ${prompt}` : prompt;
     if (toolCreator) toolCreator.setPermissionEscalator(escalator);
     const rootPrompt = buildContextualPrompt(options.prompt, options.conversationHistory);
     let finalOutput = "";
+    const runBreaker = new RunBreaker();
     let t2Results = [];
     let runError = null;
     let identityPrompt = "";
@@ -15976,6 +16196,7 @@ ${prompt}` : prompt;
         this.emit("tier:status", { tierId: "t3-root", status: "COMPLETED", role: "T3" });
       } else if (complexity === "Moderate") {
         const t2 = new T2Manager(this.router, this.toolRegistry, "root");
+        t2.setRunBreaker(runBreaker);
         t2.setPresenter(true);
         t2.setHierarchyContext("You are the ROOT Manager for this task. There is no T1 Administrator involved in this run. You are responsible for decomposing the task and managing T3 workers directly.");
         if (identityPrompt) {
@@ -16036,6 +16257,7 @@ _(incomplete: ${reasons.join("; ")})_` : partial;
         }
       } else {
         const t1 = new T1Administrator(this.router, this.toolRegistry, this.config);
+        t1.setRunBreaker(runBreaker);
         t1.setPresenter(true);
         t1.setHierarchyContext("You are the top-level Administrator. You are responsible for the overall plan and supervising multiple T2 Managers.");
         if (identityPrompt) {
@@ -16060,6 +16282,20 @@ _(incomplete: ${reasons.join("; ")})_` : partial;
         finalOutput = result.output;
         t2Results = result.t2Results;
       }
+      const trip = runBreaker.reason();
+      if (trip) {
+        finalOutput = [
+          `\u26A0 **Run stopped early \u2014 ${trip.modelId} was failing every request.**`,
+          "",
+          trip.message,
+          "",
+          `Cascade stopped after ${trip.failures} consecutive failures rather than retrying every remaining subtask against the same model, which would have cost more and produced the same result.`,
+          finalOutput ? `
+---
+
+${finalOutput}` : ""
+        ].join("\n").trimEnd();
+      }
     } catch (err) {
       if (err instanceof CascadeCancelledError || err instanceof Error && err.name === "AbortError" || options.signal?.aborted) {
         this.emit("run:cancelled", {