cascade-ai 0.49.0 → 0.50.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -224247,7 +224247,7 @@ function reconcilePrice(model, live) {
224247
224247
  }
224248
224248
 
224249
224249
  // src/constants.ts
224250
- var CASCADE_VERSION = "0.49.0";
224250
+ var CASCADE_VERSION = "0.50.0";
224251
224251
  var CASCADE_CONFIG_DIR = ".cascade";
224252
224252
  var CASCADE_MD_FILE = "CASCADE.md";
224253
224253
  var CASCADE_IGNORE_FILE = ".cascadeignore";
@@ -253768,6 +253768,14 @@ var BaseTier = class extends EventEmitter__default.default {
253768
253768
  this.emit("tier:status", event);
253769
253769
  }
253770
253770
  /** Record the model serving this tier; future status events carry it. */
253771
+ /**
253772
+ * The model actually serving this tier ("provider:id"), once one has been
253773
+ * chosen. The run breaker reads it to attribute a failure to a model rather
253774
+ * than to the subtask that happened to hit it.
253775
+ */
253776
+ getServingModel() {
253777
+ return this.servingModel;
253778
+ }
253771
253779
  setServingModel(model) {
253772
253780
  this.servingModel = model || void 0;
253773
253781
  }
@@ -255503,6 +255511,7 @@ var T2Manager = class extends BaseTier {
255503
255511
  escalations = [];
255504
255512
  peerSyncBuffer = [];
255505
255513
  store;
255514
+ runBreaker;
255506
255515
  t3PeerBus = new PeerBus();
255507
255516
  // ← T3↔T3 bus (local to this T2)
255508
255517
  t2PeerBus;
@@ -255533,6 +255542,15 @@ var T2Manager = class extends BaseTier {
255533
255542
  this.router = router;
255534
255543
  this.toolRegistry = toolRegistry;
255535
255544
  }
255545
+ /**
255546
+ * Share the run-wide circuit breaker. Every T2 in a run gets the SAME
255547
+ * instance: three sections each failing once against a dead model is the
255548
+ * same fact as one section failing three times, and a per-section breaker
255549
+ * would never see it.
255550
+ */
255551
+ setRunBreaker(breaker) {
255552
+ this.runBreaker = breaker;
255553
+ }
255536
255554
  setStore(store) {
255537
255555
  this.store = store;
255538
255556
  }
@@ -255656,7 +255674,8 @@ Guidance (must be followed): ${decision.note}`
255656
255674
  const overallStatus = this.determineStatus(t3Results);
255657
255675
  const isOk = overallStatus === "COMPLETED" || overallStatus === "PARTIAL";
255658
255676
  this.setStatus(isOk ? "COMPLETED" : "FAILED", summary);
255659
- this.sendStatusUpdate({ progressPct: 100, currentAction: "Section complete", status: "IN_PROGRESS", output: summary });
255677
+ const sectionAction = overallStatus === "COMPLETED" ? "Section complete" : overallStatus === "PARTIAL" ? "Section partially complete \u2014 some subtasks failed" : overallStatus === "ESCALATED" ? "Section escalated \u2014 needs a decision" : "Section failed \u2014 no subtask completed";
255678
+ this.sendStatusUpdate({ progressPct: 100, currentAction: sectionAction, status: "IN_PROGRESS", output: summary });
255660
255679
  const result = {
255661
255680
  sectionId: assignment.sectionId,
255662
255681
  sectionTitle: assignment.sectionTitle,
@@ -255826,6 +255845,24 @@ HIERARCHY CONTEXT: ${this.hierarchyContext}` : ""),
255826
255845
  const reinforceCfg = this.router.getReinforcementsConfig?.() ?? { enabled: false, maxPerSection: 4 };
255827
255846
  let reinforcementsAdded = 0;
255828
255847
  while (remaining.size > 0) {
255848
+ if (this.runBreaker?.isOpen()) {
255849
+ const why = this.runBreaker.skipMessage();
255850
+ this.log(`Run breaker open \u2014 skipping ${remaining.size} remaining subtask(s)`);
255851
+ for (const id of remaining) {
255852
+ this.t3PeerBus.publish(this.id, id, why, "FAILED");
255853
+ resultMap.set(id, {
255854
+ subtaskId: id,
255855
+ status: "FAILED",
255856
+ output: why,
255857
+ testResults: { checksRun: [], passed: [], failed: [] },
255858
+ issues: [why],
255859
+ peerSyncsUsed: [],
255860
+ correctionAttempts: 0
255861
+ });
255862
+ }
255863
+ remaining.clear();
255864
+ break;
255865
+ }
255829
255866
  const runnableIds = [...remaining].filter((id) => (inDegree.get(id) ?? 0) === 0);
255830
255867
  if (runnableIds.length === 0) {
255831
255868
  const fallbackId = [...remaining].sort(
@@ -255928,8 +255965,32 @@ HIERARCHY CONTEXT: ${this.hierarchyContext}` : ""),
255928
255965
  remaining.delete(id);
255929
255966
  const r4 = waveResults[i4];
255930
255967
  if (r4.status === "rejected") {
255931
- this.log(`T3 worker ${id} failed: ${r4.reason instanceof Error ? r4.reason.message : String(r4.reason)} \u2014 retrying once`);
255932
255968
  const assignment = sanitizedAssignments.find((a2) => a2.subtaskId === id);
255969
+ const failedModel = workerMap.get(id)?.getServingModel();
255970
+ const tripped = this.runBreaker?.record(r4.reason, failedModel);
255971
+ if (tripped) {
255972
+ this.log(`Run breaker opened: ${tripped.failures} consecutive ${tripped.kind} failures on ${tripped.modelId} \u2014 stopping`);
255973
+ this.emit("run:breaker", tripped);
255974
+ }
255975
+ if (this.runBreaker?.isOpen()) {
255976
+ const why = this.runBreaker.skipMessage();
255977
+ this.log(`T3 worker ${id} failed and the run breaker is open \u2014 not retrying`);
255978
+ this.t3PeerBus.publish(this.id, id, why, "FAILED");
255979
+ resultMap.set(id, {
255980
+ subtaskId: id,
255981
+ status: "FAILED",
255982
+ output: why,
255983
+ testResults: { checksRun: [], passed: [], failed: [] },
255984
+ issues: [why],
255985
+ peerSyncsUsed: [],
255986
+ correctionAttempts: 0
255987
+ });
255988
+ for (const dependent of adj.get(id) ?? []) {
255989
+ inDegree.set(dependent, Math.max(0, (inDegree.get(dependent) ?? 0) - 1));
255990
+ }
255991
+ continue;
255992
+ }
255993
+ this.log(`T3 worker ${id} failed: ${r4.reason instanceof Error ? r4.reason.message : String(r4.reason)} \u2014 retrying once`);
255933
255994
  try {
255934
255995
  const retried = await this.retryT3(assignment, taskId);
255935
255996
  resultMap.set(id, retried);
@@ -256261,6 +256322,7 @@ var T1Administrator = class extends BaseTier {
256261
256322
  t2Managers = /* @__PURE__ */ new Map();
256262
256323
  escalations = [];
256263
256324
  store;
256325
+ runBreaker;
256264
256326
  t2PeerBus = new PeerBus();
256265
256327
  permissionEscalator;
256266
256328
  toolCreator;
@@ -256275,6 +256337,10 @@ var T1Administrator = class extends BaseTier {
256275
256337
  this.toolRegistry = toolRegistry;
256276
256338
  this.config = config2;
256277
256339
  }
256340
+ /** Share the run-wide circuit breaker with every section manager. */
256341
+ setRunBreaker(breaker) {
256342
+ this.runBreaker = breaker;
256343
+ }
256278
256344
  setStore(store) {
256279
256345
  this.store = store;
256280
256346
  }
@@ -256637,6 +256703,7 @@ SPEC RULES \u2014 each subtask is a self-contained spec slice (workers execute f
256637
256703
  };
256638
256704
  const managers = sections.map((section) => {
256639
256705
  const manager = new T2Manager(this.router, this.toolRegistry, this.id);
256706
+ if (this.runBreaker) manager.setRunBreaker(this.runBreaker);
256640
256707
  manager.setHierarchyContext(`You are a T2 Manager for the section "${section.sectionTitle}". You are part of a COMPLEX task overseen by T1 Administrator.`);
256641
256708
  if (this.store) {
256642
256709
  manager.setStore(this.store);
@@ -256922,6 +256989,158 @@ Reply with exactly one word: YES, NO, or UNSURE.
256922
256989
  }
256923
256990
  };
256924
256991
 
256992
+ // src/core/router/provider-errors.ts
256993
+ var SYSTEMIC = /* @__PURE__ */ new Set([
256994
+ "rate_limit",
256995
+ "quota_exhausted",
256996
+ "auth",
256997
+ "model_unavailable"
256998
+ ]);
256999
+ function isSystemicKind(kind2) {
257000
+ return SYSTEMIC.has(kind2);
257001
+ }
257002
+ function statusOf(err) {
257003
+ if (!err || typeof err !== "object") return void 0;
257004
+ const e3 = err;
257005
+ for (const key of ["status", "statusCode", "code"]) {
257006
+ const v6 = e3[key];
257007
+ if (typeof v6 === "number" && v6 >= 100 && v6 < 600) return v6;
257008
+ }
257009
+ const resp = e3["response"];
257010
+ if (resp && typeof resp["status"] === "number") return resp["status"];
257011
+ const inner = e3["error"];
257012
+ if (inner && typeof inner["code"] === "number") return inner["code"];
257013
+ return void 0;
257014
+ }
257015
+ function messageOf(err) {
257016
+ if (err instanceof Error) return err.message;
257017
+ if (typeof err === "string") return err;
257018
+ if (err && typeof err === "object") {
257019
+ const e3 = err;
257020
+ if (typeof e3["message"] === "string") return e3["message"];
257021
+ const inner = e3["error"];
257022
+ if (inner && typeof inner["message"] === "string") return inner["message"];
257023
+ try {
257024
+ return JSON.stringify(err);
257025
+ } catch {
257026
+ }
257027
+ }
257028
+ return String(err);
257029
+ }
257030
+ function classifyProviderError(err) {
257031
+ const raw = messageOf(err).trim();
257032
+ const status = statusOf(err);
257033
+ const m3 = raw.toLowerCase();
257034
+ const kind2 = (() => {
257035
+ if (status === 429) {
257036
+ return /quota|billing|credit|exceeded your current quota|insufficient/.test(m3) ? "quota_exhausted" : "rate_limit";
257037
+ }
257038
+ if (status === 401 || status === 403) return "auth";
257039
+ if (status === 404) return "model_unavailable";
257040
+ if (/rate limit|too many requests|resource_exhausted/.test(m3)) {
257041
+ return /quota|billing|credit|insufficient/.test(m3) ? "quota_exhausted" : "rate_limit";
257042
+ }
257043
+ if (/quota|billing|insufficient_quota|insufficient credit|payment required/.test(m3)) return "quota_exhausted";
257044
+ if (/api key|unauthorized|authentication|permission denied|invalid[_ ]api/.test(m3)) return "auth";
257045
+ if (/model .*(not found|does not exist|unavailable)|no such model|unknown model|not supported for/.test(m3)) {
257046
+ return "model_unavailable";
257047
+ }
257048
+ if (/context length|too many tokens|maximum context|prompt is too long|input too long/.test(m3)) return "context_length";
257049
+ if (/safety|content filter|blocked|recitation|prohibited/.test(m3)) return "content_filter";
257050
+ if (/econnreset|etimedout|enotfound|socket hang up|network|fetch failed|aborted/.test(m3)) return "network";
257051
+ return "unknown";
257052
+ })();
257053
+ return { kind: kind2, systemic: isSystemicKind(kind2), status, raw };
257054
+ }
257055
+ function describeProviderError(c4, modelId) {
257056
+ const on = ` on ${modelId}` ;
257057
+ const detail = c4.raw ? ` Provider said: ${c4.raw}` : "";
257058
+ switch (c4.kind) {
257059
+ case "rate_limit":
257060
+ return `Rate limit hit${on}. The provider is throttling requests \u2014 lower the number of parallel workers, or use a model with a higher requests-per-minute allowance.${detail}`;
257061
+ case "quota_exhausted":
257062
+ return `Quota or billing limit reached${on}. This will not recover on its own \u2014 check the provider's billing page, or switch this tier to a different provider.${detail}`;
257063
+ case "auth":
257064
+ return `Authentication failed${on}. The API key is missing, expired, or lacks access to this model \u2014 re-check the key in Settings > Providers.${detail}`;
257065
+ case "model_unavailable":
257066
+ return `Model unavailable${on}. The id may be wrong, retired, or not enabled for this API key \u2014 pick a different model for this tier.${detail}`;
257067
+ case "context_length":
257068
+ return `The prompt was too long${on}. Reduce the attached context, or use a model with a larger context window for this tier.${detail}`;
257069
+ case "content_filter":
257070
+ return `The provider's safety filter refused this request${on}.${detail}`;
257071
+ case "network":
257072
+ return `Network error reaching the provider${on}. This is usually transient.${detail}`;
257073
+ default:
257074
+ return `The model call failed${on}.${detail}`;
257075
+ }
257076
+ }
257077
+
257078
+ // src/core/run-breaker.ts
257079
+ var DEFAULT_FAILURE_THRESHOLD = 3;
257080
+ var RunBreaker = class {
257081
+ threshold;
257082
+ /** modelId → consecutive systemic failures of the SAME kind. */
257083
+ counts = /* @__PURE__ */ new Map();
257084
+ trip = null;
257085
+ constructor(threshold = DEFAULT_FAILURE_THRESHOLD) {
257086
+ this.threshold = Math.max(1, threshold);
257087
+ }
257088
+ /**
257089
+ * Record a failed model call. Returns the trip if this failure is the one
257090
+ * that opened the breaker, otherwise null.
257091
+ *
257092
+ * A non-systemic failure RESETS the model's counter: it is evidence the model
257093
+ * is reachable and answering, which is exactly what a systemic streak claims
257094
+ * it is not.
257095
+ */
257096
+ record(err, modelId) {
257097
+ if (this.trip) return null;
257098
+ const id = modelId?.trim() || "unknown-model";
257099
+ const c4 = classifyProviderError(err);
257100
+ if (!c4.systemic) {
257101
+ this.counts.delete(id);
257102
+ return null;
257103
+ }
257104
+ const prev = this.counts.get(id);
257105
+ const n8 = prev && prev.kind === c4.kind ? prev.n + 1 : 1;
257106
+ this.counts.set(id, { kind: c4.kind, n: n8, last: c4 });
257107
+ if (n8 >= this.threshold) {
257108
+ this.trip = {
257109
+ modelId: id,
257110
+ kind: c4.kind,
257111
+ failures: n8,
257112
+ message: describeProviderError(c4, id)
257113
+ };
257114
+ return this.trip;
257115
+ }
257116
+ return null;
257117
+ }
257118
+ /** True once the run should stop launching work that would fail identically. */
257119
+ isOpen() {
257120
+ return this.trip !== null;
257121
+ }
257122
+ /** Why the breaker opened, or null while it is closed. */
257123
+ reason() {
257124
+ return this.trip;
257125
+ }
257126
+ /**
257127
+ * One line for a worker/section that never ran because the breaker was
257128
+ * already open. Says the work was skipped deliberately — not that it failed
257129
+ * on its own merits, which is what an empty FAILED status would imply.
257130
+ */
257131
+ skipMessage() {
257132
+ const t4 = this.trip;
257133
+ if (!t4) return "Skipped.";
257134
+ return `Skipped \u2014 the run stopped after ${t4.failures} consecutive failures on ${t4.modelId}. ${t4.message}`;
257135
+ }
257136
+ /** Test/diagnostic view of the per-model streaks. */
257137
+ streaks() {
257138
+ const out = {};
257139
+ for (const [k4, v6] of this.counts) out[k4] = { kind: v6.kind, n: v6.n };
257140
+ return out;
257141
+ }
257142
+ };
257143
+
256925
257144
  // src/tools/registry.ts
256926
257145
  var import_ignore = __toESM(require_ignore(), 1);
256927
257146
 
@@ -302798,6 +303017,7 @@ ${prompt}` : prompt;
302798
303017
  if (toolCreator) toolCreator.setPermissionEscalator(escalator);
302799
303018
  const rootPrompt = buildContextualPrompt(options.prompt, options.conversationHistory);
302800
303019
  let finalOutput = "";
303020
+ const runBreaker = new RunBreaker();
302801
303021
  let t2Results = [];
302802
303022
  let runError = null;
302803
303023
  let identityPrompt = "";
@@ -302866,6 +303086,7 @@ ${prompt}` : prompt;
302866
303086
  this.emit("tier:status", { tierId: "t3-root", status: "COMPLETED", role: "T3" });
302867
303087
  } else if (complexity === "Moderate") {
302868
303088
  const t22 = new T2Manager(this.router, this.toolRegistry, "root");
303089
+ t22.setRunBreaker(runBreaker);
302869
303090
  t22.setPresenter(true);
302870
303091
  t22.setHierarchyContext("You are the ROOT Manager for this task. There is no T1 Administrator involved in this run. You are responsible for decomposing the task and managing T3 workers directly.");
302871
303092
  if (identityPrompt) {
@@ -302926,6 +303147,7 @@ _(incomplete: ${reasons.join("; ")})_` : partial2;
302926
303147
  }
302927
303148
  } else {
302928
303149
  const t1 = new T1Administrator(this.router, this.toolRegistry, this.config);
303150
+ t1.setRunBreaker(runBreaker);
302929
303151
  t1.setPresenter(true);
302930
303152
  t1.setHierarchyContext("You are the top-level Administrator. You are responsible for the overall plan and supervising multiple T2 Managers.");
302931
303153
  if (identityPrompt) {
@@ -302950,6 +303172,20 @@ _(incomplete: ${reasons.join("; ")})_` : partial2;
302950
303172
  finalOutput = result.output;
302951
303173
  t2Results = result.t2Results;
302952
303174
  }
303175
+ const trip = runBreaker.reason();
303176
+ if (trip) {
303177
+ finalOutput = [
303178
+ `\u26A0 **Run stopped early \u2014 ${trip.modelId} was failing every request.**`,
303179
+ "",
303180
+ trip.message,
303181
+ "",
303182
+ `Cascade stopped after ${trip.failures} consecutive failures rather than retrying every remaining subtask against the same model, which would have cost more and produced the same result.`,
303183
+ finalOutput ? `
303184
+ ---
303185
+
303186
+ ${finalOutput}` : ""
303187
+ ].join("\n").trimEnd();
303188
+ }
302953
303189
  } catch (err) {
302954
303190
  if (err instanceof CascadeCancelledError || err instanceof Error && err.name === "AbortError" || options.signal?.aborted) {
302955
303191
  this.emit("run:cancelled", {
package/dist/index.cjs CHANGED
@@ -1891,7 +1891,7 @@ function reconcilePrice(model, live) {
1891
1891
  }
1892
1892
 
1893
1893
  // src/constants.ts
1894
- var CASCADE_VERSION = "0.49.0";
1894
+ var CASCADE_VERSION = "0.50.0";
1895
1895
  var CASCADE_CONFIG_DIR = ".cascade";
1896
1896
  var CASCADE_MD_FILE = "CASCADE.md";
1897
1897
  var CASCADE_IGNORE_FILE = ".cascadeignore";
@@ -5507,6 +5507,14 @@ var BaseTier = class extends EventEmitter__default.default {
5507
5507
  this.emit("tier:status", event);
5508
5508
  }
5509
5509
  /** Record the model serving this tier; future status events carry it. */
5510
+ /**
5511
+ * The model actually serving this tier ("provider:id"), once one has been
5512
+ * chosen. The run breaker reads it to attribute a failure to a model rather
5513
+ * than to the subtask that happened to hit it.
5514
+ */
5515
+ getServingModel() {
5516
+ return this.servingModel;
5517
+ }
5510
5518
  setServingModel(model) {
5511
5519
  this.servingModel = model || void 0;
5512
5520
  }
@@ -7242,6 +7250,7 @@ var T2Manager = class extends BaseTier {
7242
7250
  escalations = [];
7243
7251
  peerSyncBuffer = [];
7244
7252
  store;
7253
+ runBreaker;
7245
7254
  t3PeerBus = new PeerBus();
7246
7255
  // ← T3↔T3 bus (local to this T2)
7247
7256
  t2PeerBus;
@@ -7272,6 +7281,15 @@ var T2Manager = class extends BaseTier {
7272
7281
  this.router = router;
7273
7282
  this.toolRegistry = toolRegistry;
7274
7283
  }
7284
+ /**
7285
+ * Share the run-wide circuit breaker. Every T2 in a run gets the SAME
7286
+ * instance: three sections each failing once against a dead model is the
7287
+ * same fact as one section failing three times, and a per-section breaker
7288
+ * would never see it.
7289
+ */
7290
+ setRunBreaker(breaker) {
7291
+ this.runBreaker = breaker;
7292
+ }
7275
7293
  setStore(store) {
7276
7294
  this.store = store;
7277
7295
  }
@@ -7395,7 +7413,8 @@ Guidance (must be followed): ${decision.note}`
7395
7413
  const overallStatus = this.determineStatus(t3Results);
7396
7414
  const isOk = overallStatus === "COMPLETED" || overallStatus === "PARTIAL";
7397
7415
  this.setStatus(isOk ? "COMPLETED" : "FAILED", summary);
7398
- this.sendStatusUpdate({ progressPct: 100, currentAction: "Section complete", status: "IN_PROGRESS", output: summary });
7416
+ const sectionAction = overallStatus === "COMPLETED" ? "Section complete" : overallStatus === "PARTIAL" ? "Section partially complete \u2014 some subtasks failed" : overallStatus === "ESCALATED" ? "Section escalated \u2014 needs a decision" : "Section failed \u2014 no subtask completed";
7417
+ this.sendStatusUpdate({ progressPct: 100, currentAction: sectionAction, status: "IN_PROGRESS", output: summary });
7399
7418
  const result = {
7400
7419
  sectionId: assignment.sectionId,
7401
7420
  sectionTitle: assignment.sectionTitle,
@@ -7565,6 +7584,24 @@ HIERARCHY CONTEXT: ${this.hierarchyContext}` : ""),
7565
7584
  const reinforceCfg = this.router.getReinforcementsConfig?.() ?? { enabled: false, maxPerSection: 4 };
7566
7585
  let reinforcementsAdded = 0;
7567
7586
  while (remaining.size > 0) {
7587
+ if (this.runBreaker?.isOpen()) {
7588
+ const why = this.runBreaker.skipMessage();
7589
+ this.log(`Run breaker open \u2014 skipping ${remaining.size} remaining subtask(s)`);
7590
+ for (const id of remaining) {
7591
+ this.t3PeerBus.publish(this.id, id, why, "FAILED");
7592
+ resultMap.set(id, {
7593
+ subtaskId: id,
7594
+ status: "FAILED",
7595
+ output: why,
7596
+ testResults: { checksRun: [], passed: [], failed: [] },
7597
+ issues: [why],
7598
+ peerSyncsUsed: [],
7599
+ correctionAttempts: 0
7600
+ });
7601
+ }
7602
+ remaining.clear();
7603
+ break;
7604
+ }
7568
7605
  const runnableIds = [...remaining].filter((id) => (inDegree.get(id) ?? 0) === 0);
7569
7606
  if (runnableIds.length === 0) {
7570
7607
  const fallbackId = [...remaining].sort(
@@ -7667,8 +7704,32 @@ HIERARCHY CONTEXT: ${this.hierarchyContext}` : ""),
7667
7704
  remaining.delete(id);
7668
7705
  const r = waveResults[i];
7669
7706
  if (r.status === "rejected") {
7670
- this.log(`T3 worker ${id} failed: ${r.reason instanceof Error ? r.reason.message : String(r.reason)} \u2014 retrying once`);
7671
7707
  const assignment = sanitizedAssignments.find((a) => a.subtaskId === id);
7708
+ const failedModel = workerMap.get(id)?.getServingModel();
7709
+ const tripped = this.runBreaker?.record(r.reason, failedModel);
7710
+ if (tripped) {
7711
+ this.log(`Run breaker opened: ${tripped.failures} consecutive ${tripped.kind} failures on ${tripped.modelId} \u2014 stopping`);
7712
+ this.emit("run:breaker", tripped);
7713
+ }
7714
+ if (this.runBreaker?.isOpen()) {
7715
+ const why = this.runBreaker.skipMessage();
7716
+ this.log(`T3 worker ${id} failed and the run breaker is open \u2014 not retrying`);
7717
+ this.t3PeerBus.publish(this.id, id, why, "FAILED");
7718
+ resultMap.set(id, {
7719
+ subtaskId: id,
7720
+ status: "FAILED",
7721
+ output: why,
7722
+ testResults: { checksRun: [], passed: [], failed: [] },
7723
+ issues: [why],
7724
+ peerSyncsUsed: [],
7725
+ correctionAttempts: 0
7726
+ });
7727
+ for (const dependent of adj.get(id) ?? []) {
7728
+ inDegree.set(dependent, Math.max(0, (inDegree.get(dependent) ?? 0) - 1));
7729
+ }
7730
+ continue;
7731
+ }
7732
+ this.log(`T3 worker ${id} failed: ${r.reason instanceof Error ? r.reason.message : String(r.reason)} \u2014 retrying once`);
7672
7733
  try {
7673
7734
  const retried = await this.retryT3(assignment, taskId);
7674
7735
  resultMap.set(id, retried);
@@ -8000,6 +8061,7 @@ var T1Administrator = class extends BaseTier {
8000
8061
  t2Managers = /* @__PURE__ */ new Map();
8001
8062
  escalations = [];
8002
8063
  store;
8064
+ runBreaker;
8003
8065
  t2PeerBus = new PeerBus();
8004
8066
  permissionEscalator;
8005
8067
  toolCreator;
@@ -8014,6 +8076,10 @@ var T1Administrator = class extends BaseTier {
8014
8076
  this.toolRegistry = toolRegistry;
8015
8077
  this.config = config;
8016
8078
  }
8079
+ /** Share the run-wide circuit breaker with every section manager. */
8080
+ setRunBreaker(breaker) {
8081
+ this.runBreaker = breaker;
8082
+ }
8017
8083
  setStore(store) {
8018
8084
  this.store = store;
8019
8085
  }
@@ -8376,6 +8442,7 @@ SPEC RULES \u2014 each subtask is a self-contained spec slice (workers execute f
8376
8442
  };
8377
8443
  const managers = sections.map((section) => {
8378
8444
  const manager = new T2Manager(this.router, this.toolRegistry, this.id);
8445
+ if (this.runBreaker) manager.setRunBreaker(this.runBreaker);
8379
8446
  manager.setHierarchyContext(`You are a T2 Manager for the section "${section.sectionTitle}". You are part of a COMPLEX task overseen by T1 Administrator.`);
8380
8447
  if (this.store) {
8381
8448
  manager.setStore(this.store);
@@ -8661,6 +8728,158 @@ Reply with exactly one word: YES, NO, or UNSURE.
8661
8728
  }
8662
8729
  };
8663
8730
 
8731
+ // src/core/router/provider-errors.ts
8732
+ var SYSTEMIC = /* @__PURE__ */ new Set([
8733
+ "rate_limit",
8734
+ "quota_exhausted",
8735
+ "auth",
8736
+ "model_unavailable"
8737
+ ]);
8738
+ function isSystemicKind(kind) {
8739
+ return SYSTEMIC.has(kind);
8740
+ }
8741
+ function statusOf(err) {
8742
+ if (!err || typeof err !== "object") return void 0;
8743
+ const e = err;
8744
+ for (const key of ["status", "statusCode", "code"]) {
8745
+ const v = e[key];
8746
+ if (typeof v === "number" && v >= 100 && v < 600) return v;
8747
+ }
8748
+ const resp = e["response"];
8749
+ if (resp && typeof resp["status"] === "number") return resp["status"];
8750
+ const inner = e["error"];
8751
+ if (inner && typeof inner["code"] === "number") return inner["code"];
8752
+ return void 0;
8753
+ }
8754
+ function messageOf(err) {
8755
+ if (err instanceof Error) return err.message;
8756
+ if (typeof err === "string") return err;
8757
+ if (err && typeof err === "object") {
8758
+ const e = err;
8759
+ if (typeof e["message"] === "string") return e["message"];
8760
+ const inner = e["error"];
8761
+ if (inner && typeof inner["message"] === "string") return inner["message"];
8762
+ try {
8763
+ return JSON.stringify(err);
8764
+ } catch {
8765
+ }
8766
+ }
8767
+ return String(err);
8768
+ }
8769
+ function classifyProviderError(err) {
8770
+ const raw = messageOf(err).trim();
8771
+ const status = statusOf(err);
8772
+ const m = raw.toLowerCase();
8773
+ const kind = (() => {
8774
+ if (status === 429) {
8775
+ return /quota|billing|credit|exceeded your current quota|insufficient/.test(m) ? "quota_exhausted" : "rate_limit";
8776
+ }
8777
+ if (status === 401 || status === 403) return "auth";
8778
+ if (status === 404) return "model_unavailable";
8779
+ if (/rate limit|too many requests|resource_exhausted/.test(m)) {
8780
+ return /quota|billing|credit|insufficient/.test(m) ? "quota_exhausted" : "rate_limit";
8781
+ }
8782
+ if (/quota|billing|insufficient_quota|insufficient credit|payment required/.test(m)) return "quota_exhausted";
8783
+ if (/api key|unauthorized|authentication|permission denied|invalid[_ ]api/.test(m)) return "auth";
8784
+ if (/model .*(not found|does not exist|unavailable)|no such model|unknown model|not supported for/.test(m)) {
8785
+ return "model_unavailable";
8786
+ }
8787
+ if (/context length|too many tokens|maximum context|prompt is too long|input too long/.test(m)) return "context_length";
8788
+ if (/safety|content filter|blocked|recitation|prohibited/.test(m)) return "content_filter";
8789
+ if (/econnreset|etimedout|enotfound|socket hang up|network|fetch failed|aborted/.test(m)) return "network";
8790
+ return "unknown";
8791
+ })();
8792
+ return { kind, systemic: isSystemicKind(kind), status, raw };
8793
+ }
8794
+ function describeProviderError(c, modelId) {
8795
+ const on = ` on ${modelId}` ;
8796
+ const detail = c.raw ? ` Provider said: ${c.raw}` : "";
8797
+ switch (c.kind) {
8798
+ case "rate_limit":
8799
+ return `Rate limit hit${on}. The provider is throttling requests \u2014 lower the number of parallel workers, or use a model with a higher requests-per-minute allowance.${detail}`;
8800
+ case "quota_exhausted":
8801
+ return `Quota or billing limit reached${on}. This will not recover on its own \u2014 check the provider's billing page, or switch this tier to a different provider.${detail}`;
8802
+ case "auth":
8803
+ return `Authentication failed${on}. The API key is missing, expired, or lacks access to this model \u2014 re-check the key in Settings > Providers.${detail}`;
8804
+ case "model_unavailable":
8805
+ return `Model unavailable${on}. The id may be wrong, retired, or not enabled for this API key \u2014 pick a different model for this tier.${detail}`;
8806
+ case "context_length":
8807
+ return `The prompt was too long${on}. Reduce the attached context, or use a model with a larger context window for this tier.${detail}`;
8808
+ case "content_filter":
8809
+ return `The provider's safety filter refused this request${on}.${detail}`;
8810
+ case "network":
8811
+ return `Network error reaching the provider${on}. This is usually transient.${detail}`;
8812
+ default:
8813
+ return `The model call failed${on}.${detail}`;
8814
+ }
8815
+ }
8816
+
8817
+ // src/core/run-breaker.ts
8818
+ var DEFAULT_FAILURE_THRESHOLD = 3;
8819
+ var RunBreaker = class {
8820
+ threshold;
8821
+ /** modelId → consecutive systemic failures of the SAME kind. */
8822
+ counts = /* @__PURE__ */ new Map();
8823
+ trip = null;
8824
+ constructor(threshold = DEFAULT_FAILURE_THRESHOLD) {
8825
+ this.threshold = Math.max(1, threshold);
8826
+ }
8827
+ /**
8828
+ * Record a failed model call. Returns the trip if this failure is the one
8829
+ * that opened the breaker, otherwise null.
8830
+ *
8831
+ * A non-systemic failure RESETS the model's counter: it is evidence the model
8832
+ * is reachable and answering, which is exactly what a systemic streak claims
8833
+ * it is not.
8834
+ */
8835
+ record(err, modelId) {
8836
+ if (this.trip) return null;
8837
+ const id = modelId?.trim() || "unknown-model";
8838
+ const c = classifyProviderError(err);
8839
+ if (!c.systemic) {
8840
+ this.counts.delete(id);
8841
+ return null;
8842
+ }
8843
+ const prev = this.counts.get(id);
8844
+ const n = prev && prev.kind === c.kind ? prev.n + 1 : 1;
8845
+ this.counts.set(id, { kind: c.kind, n, last: c });
8846
+ if (n >= this.threshold) {
8847
+ this.trip = {
8848
+ modelId: id,
8849
+ kind: c.kind,
8850
+ failures: n,
8851
+ message: describeProviderError(c, id)
8852
+ };
8853
+ return this.trip;
8854
+ }
8855
+ return null;
8856
+ }
8857
+ /** True once the run should stop launching work that would fail identically. */
8858
+ isOpen() {
8859
+ return this.trip !== null;
8860
+ }
8861
+ /** Why the breaker opened, or null while it is closed. */
8862
+ reason() {
8863
+ return this.trip;
8864
+ }
8865
+ /**
8866
+ * One line for a worker/section that never ran because the breaker was
8867
+ * already open. Says the work was skipped deliberately — not that it failed
8868
+ * on its own merits, which is what an empty FAILED status would imply.
8869
+ */
8870
+ skipMessage() {
8871
+ const t = this.trip;
8872
+ if (!t) return "Skipped.";
8873
+ return `Skipped \u2014 the run stopped after ${t.failures} consecutive failures on ${t.modelId}. ${t.message}`;
8874
+ }
8875
+ /** Test/diagnostic view of the per-model streaks. */
8876
+ streaks() {
8877
+ const out = {};
8878
+ for (const [k, v] of this.counts) out[k] = { kind: v.kind, n: v.n };
8879
+ return out;
8880
+ }
8881
+ };
8882
+
8664
8883
  // src/tools/base.ts
8665
8884
  var BaseTool = class {
8666
8885
  workspaceRoot = process.cwd();
@@ -14233,6 +14452,7 @@ ${prompt}` : prompt;
14233
14452
  if (toolCreator) toolCreator.setPermissionEscalator(escalator);
14234
14453
  const rootPrompt = buildContextualPrompt(options.prompt, options.conversationHistory);
14235
14454
  let finalOutput = "";
14455
+ const runBreaker = new RunBreaker();
14236
14456
  let t2Results = [];
14237
14457
  let runError = null;
14238
14458
  let identityPrompt = "";
@@ -14301,6 +14521,7 @@ ${prompt}` : prompt;
14301
14521
  this.emit("tier:status", { tierId: "t3-root", status: "COMPLETED", role: "T3" });
14302
14522
  } else if (complexity === "Moderate") {
14303
14523
  const t2 = new T2Manager(this.router, this.toolRegistry, "root");
14524
+ t2.setRunBreaker(runBreaker);
14304
14525
  t2.setPresenter(true);
14305
14526
  t2.setHierarchyContext("You are the ROOT Manager for this task. There is no T1 Administrator involved in this run. You are responsible for decomposing the task and managing T3 workers directly.");
14306
14527
  if (identityPrompt) {
@@ -14361,6 +14582,7 @@ _(incomplete: ${reasons.join("; ")})_` : partial;
14361
14582
  }
14362
14583
  } else {
14363
14584
  const t1 = new T1Administrator(this.router, this.toolRegistry, this.config);
14585
+ t1.setRunBreaker(runBreaker);
14364
14586
  t1.setPresenter(true);
14365
14587
  t1.setHierarchyContext("You are the top-level Administrator. You are responsible for the overall plan and supervising multiple T2 Managers.");
14366
14588
  if (identityPrompt) {
@@ -14385,6 +14607,20 @@ _(incomplete: ${reasons.join("; ")})_` : partial;
14385
14607
  finalOutput = result.output;
14386
14608
  t2Results = result.t2Results;
14387
14609
  }
14610
+ const trip = runBreaker.reason();
14611
+ if (trip) {
14612
+ finalOutput = [
14613
+ `\u26A0 **Run stopped early \u2014 ${trip.modelId} was failing every request.**`,
14614
+ "",
14615
+ trip.message,
14616
+ "",
14617
+ `Cascade stopped after ${trip.failures} consecutive failures rather than retrying every remaining subtask against the same model, which would have cost more and produced the same result.`,
14618
+ finalOutput ? `
14619
+ ---
14620
+
14621
+ ${finalOutput}` : ""
14622
+ ].join("\n").trimEnd();
14623
+ }
14388
14624
  } catch (err) {
14389
14625
  if (err instanceof CascadeCancelledError || err instanceof Error && err.name === "AbortError" || options.signal?.aborted) {
14390
14626
  this.emit("run:cancelled", {