cascade-ai 0.49.0 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.cjs +239 -3
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +239 -3
- package/dist/cli.js.map +1 -1
- package/dist/desktop-core.cjs +239 -3
- package/dist/index.cjs +239 -3
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +72 -0
- package/dist/index.d.ts +72 -0
- package/dist/index.js +239 -3
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/desktop-core.cjs
CHANGED
|
@@ -224247,7 +224247,7 @@ function reconcilePrice(model, live) {
|
|
|
224247
224247
|
}
|
|
224248
224248
|
|
|
224249
224249
|
// src/constants.ts
|
|
224250
|
-
var CASCADE_VERSION = "0.
|
|
224250
|
+
var CASCADE_VERSION = "0.50.0";
|
|
224251
224251
|
var CASCADE_CONFIG_DIR = ".cascade";
|
|
224252
224252
|
var CASCADE_MD_FILE = "CASCADE.md";
|
|
224253
224253
|
var CASCADE_IGNORE_FILE = ".cascadeignore";
|
|
@@ -253768,6 +253768,14 @@ var BaseTier = class extends EventEmitter__default.default {
|
|
|
253768
253768
|
this.emit("tier:status", event);
|
|
253769
253769
|
}
|
|
253770
253770
|
/** Record the model serving this tier; future status events carry it. */
|
|
253771
|
+
/**
|
|
253772
|
+
* The model actually serving this tier ("provider:id"), once one has been
|
|
253773
|
+
* chosen. The run breaker reads it to attribute a failure to a model rather
|
|
253774
|
+
* than to the subtask that happened to hit it.
|
|
253775
|
+
*/
|
|
253776
|
+
getServingModel() {
|
|
253777
|
+
return this.servingModel;
|
|
253778
|
+
}
|
|
253771
253779
|
setServingModel(model) {
|
|
253772
253780
|
this.servingModel = model || void 0;
|
|
253773
253781
|
}
|
|
@@ -255503,6 +255511,7 @@ var T2Manager = class extends BaseTier {
|
|
|
255503
255511
|
escalations = [];
|
|
255504
255512
|
peerSyncBuffer = [];
|
|
255505
255513
|
store;
|
|
255514
|
+
runBreaker;
|
|
255506
255515
|
t3PeerBus = new PeerBus();
|
|
255507
255516
|
// ← T3↔T3 bus (local to this T2)
|
|
255508
255517
|
t2PeerBus;
|
|
@@ -255533,6 +255542,15 @@ var T2Manager = class extends BaseTier {
|
|
|
255533
255542
|
this.router = router;
|
|
255534
255543
|
this.toolRegistry = toolRegistry;
|
|
255535
255544
|
}
|
|
255545
|
+
/**
|
|
255546
|
+
* Share the run-wide circuit breaker. Every T2 in a run gets the SAME
|
|
255547
|
+
* instance: three sections each failing once against a dead model is the
|
|
255548
|
+
* same fact as one section failing three times, and a per-section breaker
|
|
255549
|
+
* would never see it.
|
|
255550
|
+
*/
|
|
255551
|
+
setRunBreaker(breaker) {
|
|
255552
|
+
this.runBreaker = breaker;
|
|
255553
|
+
}
|
|
255536
255554
|
setStore(store) {
|
|
255537
255555
|
this.store = store;
|
|
255538
255556
|
}
|
|
@@ -255656,7 +255674,8 @@ Guidance (must be followed): ${decision.note}`
|
|
|
255656
255674
|
const overallStatus = this.determineStatus(t3Results);
|
|
255657
255675
|
const isOk = overallStatus === "COMPLETED" || overallStatus === "PARTIAL";
|
|
255658
255676
|
this.setStatus(isOk ? "COMPLETED" : "FAILED", summary);
|
|
255659
|
-
|
|
255677
|
+
const sectionAction = overallStatus === "COMPLETED" ? "Section complete" : overallStatus === "PARTIAL" ? "Section partially complete \u2014 some subtasks failed" : overallStatus === "ESCALATED" ? "Section escalated \u2014 needs a decision" : "Section failed \u2014 no subtask completed";
|
|
255678
|
+
this.sendStatusUpdate({ progressPct: 100, currentAction: sectionAction, status: "IN_PROGRESS", output: summary });
|
|
255660
255679
|
const result = {
|
|
255661
255680
|
sectionId: assignment.sectionId,
|
|
255662
255681
|
sectionTitle: assignment.sectionTitle,
|
|
@@ -255826,6 +255845,24 @@ HIERARCHY CONTEXT: ${this.hierarchyContext}` : ""),
|
|
|
255826
255845
|
const reinforceCfg = this.router.getReinforcementsConfig?.() ?? { enabled: false, maxPerSection: 4 };
|
|
255827
255846
|
let reinforcementsAdded = 0;
|
|
255828
255847
|
while (remaining.size > 0) {
|
|
255848
|
+
if (this.runBreaker?.isOpen()) {
|
|
255849
|
+
const why = this.runBreaker.skipMessage();
|
|
255850
|
+
this.log(`Run breaker open \u2014 skipping ${remaining.size} remaining subtask(s)`);
|
|
255851
|
+
for (const id of remaining) {
|
|
255852
|
+
this.t3PeerBus.publish(this.id, id, why, "FAILED");
|
|
255853
|
+
resultMap.set(id, {
|
|
255854
|
+
subtaskId: id,
|
|
255855
|
+
status: "FAILED",
|
|
255856
|
+
output: why,
|
|
255857
|
+
testResults: { checksRun: [], passed: [], failed: [] },
|
|
255858
|
+
issues: [why],
|
|
255859
|
+
peerSyncsUsed: [],
|
|
255860
|
+
correctionAttempts: 0
|
|
255861
|
+
});
|
|
255862
|
+
}
|
|
255863
|
+
remaining.clear();
|
|
255864
|
+
break;
|
|
255865
|
+
}
|
|
255829
255866
|
const runnableIds = [...remaining].filter((id) => (inDegree.get(id) ?? 0) === 0);
|
|
255830
255867
|
if (runnableIds.length === 0) {
|
|
255831
255868
|
const fallbackId = [...remaining].sort(
|
|
@@ -255928,8 +255965,32 @@ HIERARCHY CONTEXT: ${this.hierarchyContext}` : ""),
|
|
|
255928
255965
|
remaining.delete(id);
|
|
255929
255966
|
const r4 = waveResults[i4];
|
|
255930
255967
|
if (r4.status === "rejected") {
|
|
255931
|
-
this.log(`T3 worker ${id} failed: ${r4.reason instanceof Error ? r4.reason.message : String(r4.reason)} \u2014 retrying once`);
|
|
255932
255968
|
const assignment = sanitizedAssignments.find((a2) => a2.subtaskId === id);
|
|
255969
|
+
const failedModel = workerMap.get(id)?.getServingModel();
|
|
255970
|
+
const tripped = this.runBreaker?.record(r4.reason, failedModel);
|
|
255971
|
+
if (tripped) {
|
|
255972
|
+
this.log(`Run breaker opened: ${tripped.failures} consecutive ${tripped.kind} failures on ${tripped.modelId} \u2014 stopping`);
|
|
255973
|
+
this.emit("run:breaker", tripped);
|
|
255974
|
+
}
|
|
255975
|
+
if (this.runBreaker?.isOpen()) {
|
|
255976
|
+
const why = this.runBreaker.skipMessage();
|
|
255977
|
+
this.log(`T3 worker ${id} failed and the run breaker is open \u2014 not retrying`);
|
|
255978
|
+
this.t3PeerBus.publish(this.id, id, why, "FAILED");
|
|
255979
|
+
resultMap.set(id, {
|
|
255980
|
+
subtaskId: id,
|
|
255981
|
+
status: "FAILED",
|
|
255982
|
+
output: why,
|
|
255983
|
+
testResults: { checksRun: [], passed: [], failed: [] },
|
|
255984
|
+
issues: [why],
|
|
255985
|
+
peerSyncsUsed: [],
|
|
255986
|
+
correctionAttempts: 0
|
|
255987
|
+
});
|
|
255988
|
+
for (const dependent of adj.get(id) ?? []) {
|
|
255989
|
+
inDegree.set(dependent, Math.max(0, (inDegree.get(dependent) ?? 0) - 1));
|
|
255990
|
+
}
|
|
255991
|
+
continue;
|
|
255992
|
+
}
|
|
255993
|
+
this.log(`T3 worker ${id} failed: ${r4.reason instanceof Error ? r4.reason.message : String(r4.reason)} \u2014 retrying once`);
|
|
255933
255994
|
try {
|
|
255934
255995
|
const retried = await this.retryT3(assignment, taskId);
|
|
255935
255996
|
resultMap.set(id, retried);
|
|
@@ -256261,6 +256322,7 @@ var T1Administrator = class extends BaseTier {
|
|
|
256261
256322
|
t2Managers = /* @__PURE__ */ new Map();
|
|
256262
256323
|
escalations = [];
|
|
256263
256324
|
store;
|
|
256325
|
+
runBreaker;
|
|
256264
256326
|
t2PeerBus = new PeerBus();
|
|
256265
256327
|
permissionEscalator;
|
|
256266
256328
|
toolCreator;
|
|
@@ -256275,6 +256337,10 @@ var T1Administrator = class extends BaseTier {
|
|
|
256275
256337
|
this.toolRegistry = toolRegistry;
|
|
256276
256338
|
this.config = config2;
|
|
256277
256339
|
}
|
|
256340
|
+
/** Share the run-wide circuit breaker with every section manager. */
|
|
256341
|
+
setRunBreaker(breaker) {
|
|
256342
|
+
this.runBreaker = breaker;
|
|
256343
|
+
}
|
|
256278
256344
|
setStore(store) {
|
|
256279
256345
|
this.store = store;
|
|
256280
256346
|
}
|
|
@@ -256637,6 +256703,7 @@ SPEC RULES \u2014 each subtask is a self-contained spec slice (workers execute f
|
|
|
256637
256703
|
};
|
|
256638
256704
|
const managers = sections.map((section) => {
|
|
256639
256705
|
const manager = new T2Manager(this.router, this.toolRegistry, this.id);
|
|
256706
|
+
if (this.runBreaker) manager.setRunBreaker(this.runBreaker);
|
|
256640
256707
|
manager.setHierarchyContext(`You are a T2 Manager for the section "${section.sectionTitle}". You are part of a COMPLEX task overseen by T1 Administrator.`);
|
|
256641
256708
|
if (this.store) {
|
|
256642
256709
|
manager.setStore(this.store);
|
|
@@ -256922,6 +256989,158 @@ Reply with exactly one word: YES, NO, or UNSURE.
|
|
|
256922
256989
|
}
|
|
256923
256990
|
};
|
|
256924
256991
|
|
|
256992
|
+
// src/core/router/provider-errors.ts
|
|
256993
|
+
var SYSTEMIC = /* @__PURE__ */ new Set([
|
|
256994
|
+
"rate_limit",
|
|
256995
|
+
"quota_exhausted",
|
|
256996
|
+
"auth",
|
|
256997
|
+
"model_unavailable"
|
|
256998
|
+
]);
|
|
256999
|
+
function isSystemicKind(kind2) {
|
|
257000
|
+
return SYSTEMIC.has(kind2);
|
|
257001
|
+
}
|
|
257002
|
+
function statusOf(err) {
|
|
257003
|
+
if (!err || typeof err !== "object") return void 0;
|
|
257004
|
+
const e3 = err;
|
|
257005
|
+
for (const key of ["status", "statusCode", "code"]) {
|
|
257006
|
+
const v6 = e3[key];
|
|
257007
|
+
if (typeof v6 === "number" && v6 >= 100 && v6 < 600) return v6;
|
|
257008
|
+
}
|
|
257009
|
+
const resp = e3["response"];
|
|
257010
|
+
if (resp && typeof resp["status"] === "number") return resp["status"];
|
|
257011
|
+
const inner = e3["error"];
|
|
257012
|
+
if (inner && typeof inner["code"] === "number") return inner["code"];
|
|
257013
|
+
return void 0;
|
|
257014
|
+
}
|
|
257015
|
+
function messageOf(err) {
|
|
257016
|
+
if (err instanceof Error) return err.message;
|
|
257017
|
+
if (typeof err === "string") return err;
|
|
257018
|
+
if (err && typeof err === "object") {
|
|
257019
|
+
const e3 = err;
|
|
257020
|
+
if (typeof e3["message"] === "string") return e3["message"];
|
|
257021
|
+
const inner = e3["error"];
|
|
257022
|
+
if (inner && typeof inner["message"] === "string") return inner["message"];
|
|
257023
|
+
try {
|
|
257024
|
+
return JSON.stringify(err);
|
|
257025
|
+
} catch {
|
|
257026
|
+
}
|
|
257027
|
+
}
|
|
257028
|
+
return String(err);
|
|
257029
|
+
}
|
|
257030
|
+
function classifyProviderError(err) {
|
|
257031
|
+
const raw = messageOf(err).trim();
|
|
257032
|
+
const status = statusOf(err);
|
|
257033
|
+
const m3 = raw.toLowerCase();
|
|
257034
|
+
const kind2 = (() => {
|
|
257035
|
+
if (status === 429) {
|
|
257036
|
+
return /quota|billing|credit|exceeded your current quota|insufficient/.test(m3) ? "quota_exhausted" : "rate_limit";
|
|
257037
|
+
}
|
|
257038
|
+
if (status === 401 || status === 403) return "auth";
|
|
257039
|
+
if (status === 404) return "model_unavailable";
|
|
257040
|
+
if (/rate limit|too many requests|resource_exhausted/.test(m3)) {
|
|
257041
|
+
return /quota|billing|credit|insufficient/.test(m3) ? "quota_exhausted" : "rate_limit";
|
|
257042
|
+
}
|
|
257043
|
+
if (/quota|billing|insufficient_quota|insufficient credit|payment required/.test(m3)) return "quota_exhausted";
|
|
257044
|
+
if (/api key|unauthorized|authentication|permission denied|invalid[_ ]api/.test(m3)) return "auth";
|
|
257045
|
+
if (/model .*(not found|does not exist|unavailable)|no such model|unknown model|not supported for/.test(m3)) {
|
|
257046
|
+
return "model_unavailable";
|
|
257047
|
+
}
|
|
257048
|
+
if (/context length|too many tokens|maximum context|prompt is too long|input too long/.test(m3)) return "context_length";
|
|
257049
|
+
if (/safety|content filter|blocked|recitation|prohibited/.test(m3)) return "content_filter";
|
|
257050
|
+
if (/econnreset|etimedout|enotfound|socket hang up|network|fetch failed|aborted/.test(m3)) return "network";
|
|
257051
|
+
return "unknown";
|
|
257052
|
+
})();
|
|
257053
|
+
return { kind: kind2, systemic: isSystemicKind(kind2), status, raw };
|
|
257054
|
+
}
|
|
257055
|
+
function describeProviderError(c4, modelId) {
|
|
257056
|
+
const on = ` on ${modelId}` ;
|
|
257057
|
+
const detail = c4.raw ? ` Provider said: ${c4.raw}` : "";
|
|
257058
|
+
switch (c4.kind) {
|
|
257059
|
+
case "rate_limit":
|
|
257060
|
+
return `Rate limit hit${on}. The provider is throttling requests \u2014 lower the number of parallel workers, or use a model with a higher requests-per-minute allowance.${detail}`;
|
|
257061
|
+
case "quota_exhausted":
|
|
257062
|
+
return `Quota or billing limit reached${on}. This will not recover on its own \u2014 check the provider's billing page, or switch this tier to a different provider.${detail}`;
|
|
257063
|
+
case "auth":
|
|
257064
|
+
return `Authentication failed${on}. The API key is missing, expired, or lacks access to this model \u2014 re-check the key in Settings > Providers.${detail}`;
|
|
257065
|
+
case "model_unavailable":
|
|
257066
|
+
return `Model unavailable${on}. The id may be wrong, retired, or not enabled for this API key \u2014 pick a different model for this tier.${detail}`;
|
|
257067
|
+
case "context_length":
|
|
257068
|
+
return `The prompt was too long${on}. Reduce the attached context, or use a model with a larger context window for this tier.${detail}`;
|
|
257069
|
+
case "content_filter":
|
|
257070
|
+
return `The provider's safety filter refused this request${on}.${detail}`;
|
|
257071
|
+
case "network":
|
|
257072
|
+
return `Network error reaching the provider${on}. This is usually transient.${detail}`;
|
|
257073
|
+
default:
|
|
257074
|
+
return `The model call failed${on}.${detail}`;
|
|
257075
|
+
}
|
|
257076
|
+
}
|
|
257077
|
+
|
|
257078
|
+
// src/core/run-breaker.ts
|
|
257079
|
+
var DEFAULT_FAILURE_THRESHOLD = 3;
|
|
257080
|
+
var RunBreaker = class {
|
|
257081
|
+
threshold;
|
|
257082
|
+
/** modelId → consecutive systemic failures of the SAME kind. */
|
|
257083
|
+
counts = /* @__PURE__ */ new Map();
|
|
257084
|
+
trip = null;
|
|
257085
|
+
constructor(threshold = DEFAULT_FAILURE_THRESHOLD) {
|
|
257086
|
+
this.threshold = Math.max(1, threshold);
|
|
257087
|
+
}
|
|
257088
|
+
/**
|
|
257089
|
+
* Record a failed model call. Returns the trip if this failure is the one
|
|
257090
|
+
* that opened the breaker, otherwise null.
|
|
257091
|
+
*
|
|
257092
|
+
* A non-systemic failure RESETS the model's counter: it is evidence the model
|
|
257093
|
+
* is reachable and answering, which is exactly what a systemic streak claims
|
|
257094
|
+
* it is not.
|
|
257095
|
+
*/
|
|
257096
|
+
record(err, modelId) {
|
|
257097
|
+
if (this.trip) return null;
|
|
257098
|
+
const id = modelId?.trim() || "unknown-model";
|
|
257099
|
+
const c4 = classifyProviderError(err);
|
|
257100
|
+
if (!c4.systemic) {
|
|
257101
|
+
this.counts.delete(id);
|
|
257102
|
+
return null;
|
|
257103
|
+
}
|
|
257104
|
+
const prev = this.counts.get(id);
|
|
257105
|
+
const n8 = prev && prev.kind === c4.kind ? prev.n + 1 : 1;
|
|
257106
|
+
this.counts.set(id, { kind: c4.kind, n: n8, last: c4 });
|
|
257107
|
+
if (n8 >= this.threshold) {
|
|
257108
|
+
this.trip = {
|
|
257109
|
+
modelId: id,
|
|
257110
|
+
kind: c4.kind,
|
|
257111
|
+
failures: n8,
|
|
257112
|
+
message: describeProviderError(c4, id)
|
|
257113
|
+
};
|
|
257114
|
+
return this.trip;
|
|
257115
|
+
}
|
|
257116
|
+
return null;
|
|
257117
|
+
}
|
|
257118
|
+
/** True once the run should stop launching work that would fail identically. */
|
|
257119
|
+
isOpen() {
|
|
257120
|
+
return this.trip !== null;
|
|
257121
|
+
}
|
|
257122
|
+
/** Why the breaker opened, or null while it is closed. */
|
|
257123
|
+
reason() {
|
|
257124
|
+
return this.trip;
|
|
257125
|
+
}
|
|
257126
|
+
/**
|
|
257127
|
+
* One line for a worker/section that never ran because the breaker was
|
|
257128
|
+
* already open. Says the work was skipped deliberately — not that it failed
|
|
257129
|
+
* on its own merits, which is what an empty FAILED status would imply.
|
|
257130
|
+
*/
|
|
257131
|
+
skipMessage() {
|
|
257132
|
+
const t4 = this.trip;
|
|
257133
|
+
if (!t4) return "Skipped.";
|
|
257134
|
+
return `Skipped \u2014 the run stopped after ${t4.failures} consecutive failures on ${t4.modelId}. ${t4.message}`;
|
|
257135
|
+
}
|
|
257136
|
+
/** Test/diagnostic view of the per-model streaks. */
|
|
257137
|
+
streaks() {
|
|
257138
|
+
const out = {};
|
|
257139
|
+
for (const [k4, v6] of this.counts) out[k4] = { kind: v6.kind, n: v6.n };
|
|
257140
|
+
return out;
|
|
257141
|
+
}
|
|
257142
|
+
};
|
|
257143
|
+
|
|
256925
257144
|
// src/tools/registry.ts
|
|
256926
257145
|
var import_ignore = __toESM(require_ignore(), 1);
|
|
256927
257146
|
|
|
@@ -302798,6 +303017,7 @@ ${prompt}` : prompt;
|
|
|
302798
303017
|
if (toolCreator) toolCreator.setPermissionEscalator(escalator);
|
|
302799
303018
|
const rootPrompt = buildContextualPrompt(options.prompt, options.conversationHistory);
|
|
302800
303019
|
let finalOutput = "";
|
|
303020
|
+
const runBreaker = new RunBreaker();
|
|
302801
303021
|
let t2Results = [];
|
|
302802
303022
|
let runError = null;
|
|
302803
303023
|
let identityPrompt = "";
|
|
@@ -302866,6 +303086,7 @@ ${prompt}` : prompt;
|
|
|
302866
303086
|
this.emit("tier:status", { tierId: "t3-root", status: "COMPLETED", role: "T3" });
|
|
302867
303087
|
} else if (complexity === "Moderate") {
|
|
302868
303088
|
const t22 = new T2Manager(this.router, this.toolRegistry, "root");
|
|
303089
|
+
t22.setRunBreaker(runBreaker);
|
|
302869
303090
|
t22.setPresenter(true);
|
|
302870
303091
|
t22.setHierarchyContext("You are the ROOT Manager for this task. There is no T1 Administrator involved in this run. You are responsible for decomposing the task and managing T3 workers directly.");
|
|
302871
303092
|
if (identityPrompt) {
|
|
@@ -302926,6 +303147,7 @@ _(incomplete: ${reasons.join("; ")})_` : partial2;
|
|
|
302926
303147
|
}
|
|
302927
303148
|
} else {
|
|
302928
303149
|
const t1 = new T1Administrator(this.router, this.toolRegistry, this.config);
|
|
303150
|
+
t1.setRunBreaker(runBreaker);
|
|
302929
303151
|
t1.setPresenter(true);
|
|
302930
303152
|
t1.setHierarchyContext("You are the top-level Administrator. You are responsible for the overall plan and supervising multiple T2 Managers.");
|
|
302931
303153
|
if (identityPrompt) {
|
|
@@ -302950,6 +303172,20 @@ _(incomplete: ${reasons.join("; ")})_` : partial2;
|
|
|
302950
303172
|
finalOutput = result.output;
|
|
302951
303173
|
t2Results = result.t2Results;
|
|
302952
303174
|
}
|
|
303175
|
+
const trip = runBreaker.reason();
|
|
303176
|
+
if (trip) {
|
|
303177
|
+
finalOutput = [
|
|
303178
|
+
`\u26A0 **Run stopped early \u2014 ${trip.modelId} was failing every request.**`,
|
|
303179
|
+
"",
|
|
303180
|
+
trip.message,
|
|
303181
|
+
"",
|
|
303182
|
+
`Cascade stopped after ${trip.failures} consecutive failures rather than retrying every remaining subtask against the same model, which would have cost more and produced the same result.`,
|
|
303183
|
+
finalOutput ? `
|
|
303184
|
+
---
|
|
303185
|
+
|
|
303186
|
+
${finalOutput}` : ""
|
|
303187
|
+
].join("\n").trimEnd();
|
|
303188
|
+
}
|
|
302953
303189
|
} catch (err) {
|
|
302954
303190
|
if (err instanceof CascadeCancelledError || err instanceof Error && err.name === "AbortError" || options.signal?.aborted) {
|
|
302955
303191
|
this.emit("run:cancelled", {
|
package/dist/index.cjs
CHANGED
|
@@ -1891,7 +1891,7 @@ function reconcilePrice(model, live) {
|
|
|
1891
1891
|
}
|
|
1892
1892
|
|
|
1893
1893
|
// src/constants.ts
|
|
1894
|
-
var CASCADE_VERSION = "0.
|
|
1894
|
+
var CASCADE_VERSION = "0.50.0";
|
|
1895
1895
|
var CASCADE_CONFIG_DIR = ".cascade";
|
|
1896
1896
|
var CASCADE_MD_FILE = "CASCADE.md";
|
|
1897
1897
|
var CASCADE_IGNORE_FILE = ".cascadeignore";
|
|
@@ -5507,6 +5507,14 @@ var BaseTier = class extends EventEmitter__default.default {
|
|
|
5507
5507
|
this.emit("tier:status", event);
|
|
5508
5508
|
}
|
|
5509
5509
|
/** Record the model serving this tier; future status events carry it. */
|
|
5510
|
+
/**
|
|
5511
|
+
* The model actually serving this tier ("provider:id"), once one has been
|
|
5512
|
+
* chosen. The run breaker reads it to attribute a failure to a model rather
|
|
5513
|
+
* than to the subtask that happened to hit it.
|
|
5514
|
+
*/
|
|
5515
|
+
getServingModel() {
|
|
5516
|
+
return this.servingModel;
|
|
5517
|
+
}
|
|
5510
5518
|
setServingModel(model) {
|
|
5511
5519
|
this.servingModel = model || void 0;
|
|
5512
5520
|
}
|
|
@@ -7242,6 +7250,7 @@ var T2Manager = class extends BaseTier {
|
|
|
7242
7250
|
escalations = [];
|
|
7243
7251
|
peerSyncBuffer = [];
|
|
7244
7252
|
store;
|
|
7253
|
+
runBreaker;
|
|
7245
7254
|
t3PeerBus = new PeerBus();
|
|
7246
7255
|
// ← T3↔T3 bus (local to this T2)
|
|
7247
7256
|
t2PeerBus;
|
|
@@ -7272,6 +7281,15 @@ var T2Manager = class extends BaseTier {
|
|
|
7272
7281
|
this.router = router;
|
|
7273
7282
|
this.toolRegistry = toolRegistry;
|
|
7274
7283
|
}
|
|
7284
|
+
/**
|
|
7285
|
+
* Share the run-wide circuit breaker. Every T2 in a run gets the SAME
|
|
7286
|
+
* instance: three sections each failing once against a dead model is the
|
|
7287
|
+
* same fact as one section failing three times, and a per-section breaker
|
|
7288
|
+
* would never see it.
|
|
7289
|
+
*/
|
|
7290
|
+
setRunBreaker(breaker) {
|
|
7291
|
+
this.runBreaker = breaker;
|
|
7292
|
+
}
|
|
7275
7293
|
setStore(store) {
|
|
7276
7294
|
this.store = store;
|
|
7277
7295
|
}
|
|
@@ -7395,7 +7413,8 @@ Guidance (must be followed): ${decision.note}`
|
|
|
7395
7413
|
const overallStatus = this.determineStatus(t3Results);
|
|
7396
7414
|
const isOk = overallStatus === "COMPLETED" || overallStatus === "PARTIAL";
|
|
7397
7415
|
this.setStatus(isOk ? "COMPLETED" : "FAILED", summary);
|
|
7398
|
-
|
|
7416
|
+
const sectionAction = overallStatus === "COMPLETED" ? "Section complete" : overallStatus === "PARTIAL" ? "Section partially complete \u2014 some subtasks failed" : overallStatus === "ESCALATED" ? "Section escalated \u2014 needs a decision" : "Section failed \u2014 no subtask completed";
|
|
7417
|
+
this.sendStatusUpdate({ progressPct: 100, currentAction: sectionAction, status: "IN_PROGRESS", output: summary });
|
|
7399
7418
|
const result = {
|
|
7400
7419
|
sectionId: assignment.sectionId,
|
|
7401
7420
|
sectionTitle: assignment.sectionTitle,
|
|
@@ -7565,6 +7584,24 @@ HIERARCHY CONTEXT: ${this.hierarchyContext}` : ""),
|
|
|
7565
7584
|
const reinforceCfg = this.router.getReinforcementsConfig?.() ?? { enabled: false, maxPerSection: 4 };
|
|
7566
7585
|
let reinforcementsAdded = 0;
|
|
7567
7586
|
while (remaining.size > 0) {
|
|
7587
|
+
if (this.runBreaker?.isOpen()) {
|
|
7588
|
+
const why = this.runBreaker.skipMessage();
|
|
7589
|
+
this.log(`Run breaker open \u2014 skipping ${remaining.size} remaining subtask(s)`);
|
|
7590
|
+
for (const id of remaining) {
|
|
7591
|
+
this.t3PeerBus.publish(this.id, id, why, "FAILED");
|
|
7592
|
+
resultMap.set(id, {
|
|
7593
|
+
subtaskId: id,
|
|
7594
|
+
status: "FAILED",
|
|
7595
|
+
output: why,
|
|
7596
|
+
testResults: { checksRun: [], passed: [], failed: [] },
|
|
7597
|
+
issues: [why],
|
|
7598
|
+
peerSyncsUsed: [],
|
|
7599
|
+
correctionAttempts: 0
|
|
7600
|
+
});
|
|
7601
|
+
}
|
|
7602
|
+
remaining.clear();
|
|
7603
|
+
break;
|
|
7604
|
+
}
|
|
7568
7605
|
const runnableIds = [...remaining].filter((id) => (inDegree.get(id) ?? 0) === 0);
|
|
7569
7606
|
if (runnableIds.length === 0) {
|
|
7570
7607
|
const fallbackId = [...remaining].sort(
|
|
@@ -7667,8 +7704,32 @@ HIERARCHY CONTEXT: ${this.hierarchyContext}` : ""),
|
|
|
7667
7704
|
remaining.delete(id);
|
|
7668
7705
|
const r = waveResults[i];
|
|
7669
7706
|
if (r.status === "rejected") {
|
|
7670
|
-
this.log(`T3 worker ${id} failed: ${r.reason instanceof Error ? r.reason.message : String(r.reason)} \u2014 retrying once`);
|
|
7671
7707
|
const assignment = sanitizedAssignments.find((a) => a.subtaskId === id);
|
|
7708
|
+
const failedModel = workerMap.get(id)?.getServingModel();
|
|
7709
|
+
const tripped = this.runBreaker?.record(r.reason, failedModel);
|
|
7710
|
+
if (tripped) {
|
|
7711
|
+
this.log(`Run breaker opened: ${tripped.failures} consecutive ${tripped.kind} failures on ${tripped.modelId} \u2014 stopping`);
|
|
7712
|
+
this.emit("run:breaker", tripped);
|
|
7713
|
+
}
|
|
7714
|
+
if (this.runBreaker?.isOpen()) {
|
|
7715
|
+
const why = this.runBreaker.skipMessage();
|
|
7716
|
+
this.log(`T3 worker ${id} failed and the run breaker is open \u2014 not retrying`);
|
|
7717
|
+
this.t3PeerBus.publish(this.id, id, why, "FAILED");
|
|
7718
|
+
resultMap.set(id, {
|
|
7719
|
+
subtaskId: id,
|
|
7720
|
+
status: "FAILED",
|
|
7721
|
+
output: why,
|
|
7722
|
+
testResults: { checksRun: [], passed: [], failed: [] },
|
|
7723
|
+
issues: [why],
|
|
7724
|
+
peerSyncsUsed: [],
|
|
7725
|
+
correctionAttempts: 0
|
|
7726
|
+
});
|
|
7727
|
+
for (const dependent of adj.get(id) ?? []) {
|
|
7728
|
+
inDegree.set(dependent, Math.max(0, (inDegree.get(dependent) ?? 0) - 1));
|
|
7729
|
+
}
|
|
7730
|
+
continue;
|
|
7731
|
+
}
|
|
7732
|
+
this.log(`T3 worker ${id} failed: ${r.reason instanceof Error ? r.reason.message : String(r.reason)} \u2014 retrying once`);
|
|
7672
7733
|
try {
|
|
7673
7734
|
const retried = await this.retryT3(assignment, taskId);
|
|
7674
7735
|
resultMap.set(id, retried);
|
|
@@ -8000,6 +8061,7 @@ var T1Administrator = class extends BaseTier {
|
|
|
8000
8061
|
t2Managers = /* @__PURE__ */ new Map();
|
|
8001
8062
|
escalations = [];
|
|
8002
8063
|
store;
|
|
8064
|
+
runBreaker;
|
|
8003
8065
|
t2PeerBus = new PeerBus();
|
|
8004
8066
|
permissionEscalator;
|
|
8005
8067
|
toolCreator;
|
|
@@ -8014,6 +8076,10 @@ var T1Administrator = class extends BaseTier {
|
|
|
8014
8076
|
this.toolRegistry = toolRegistry;
|
|
8015
8077
|
this.config = config;
|
|
8016
8078
|
}
|
|
8079
|
+
/** Share the run-wide circuit breaker with every section manager. */
|
|
8080
|
+
setRunBreaker(breaker) {
|
|
8081
|
+
this.runBreaker = breaker;
|
|
8082
|
+
}
|
|
8017
8083
|
setStore(store) {
|
|
8018
8084
|
this.store = store;
|
|
8019
8085
|
}
|
|
@@ -8376,6 +8442,7 @@ SPEC RULES \u2014 each subtask is a self-contained spec slice (workers execute f
|
|
|
8376
8442
|
};
|
|
8377
8443
|
const managers = sections.map((section) => {
|
|
8378
8444
|
const manager = new T2Manager(this.router, this.toolRegistry, this.id);
|
|
8445
|
+
if (this.runBreaker) manager.setRunBreaker(this.runBreaker);
|
|
8379
8446
|
manager.setHierarchyContext(`You are a T2 Manager for the section "${section.sectionTitle}". You are part of a COMPLEX task overseen by T1 Administrator.`);
|
|
8380
8447
|
if (this.store) {
|
|
8381
8448
|
manager.setStore(this.store);
|
|
@@ -8661,6 +8728,158 @@ Reply with exactly one word: YES, NO, or UNSURE.
|
|
|
8661
8728
|
}
|
|
8662
8729
|
};
|
|
8663
8730
|
|
|
8731
|
+
// src/core/router/provider-errors.ts
|
|
8732
|
+
var SYSTEMIC = /* @__PURE__ */ new Set([
|
|
8733
|
+
"rate_limit",
|
|
8734
|
+
"quota_exhausted",
|
|
8735
|
+
"auth",
|
|
8736
|
+
"model_unavailable"
|
|
8737
|
+
]);
|
|
8738
|
+
function isSystemicKind(kind) {
|
|
8739
|
+
return SYSTEMIC.has(kind);
|
|
8740
|
+
}
|
|
8741
|
+
function statusOf(err) {
|
|
8742
|
+
if (!err || typeof err !== "object") return void 0;
|
|
8743
|
+
const e = err;
|
|
8744
|
+
for (const key of ["status", "statusCode", "code"]) {
|
|
8745
|
+
const v = e[key];
|
|
8746
|
+
if (typeof v === "number" && v >= 100 && v < 600) return v;
|
|
8747
|
+
}
|
|
8748
|
+
const resp = e["response"];
|
|
8749
|
+
if (resp && typeof resp["status"] === "number") return resp["status"];
|
|
8750
|
+
const inner = e["error"];
|
|
8751
|
+
if (inner && typeof inner["code"] === "number") return inner["code"];
|
|
8752
|
+
return void 0;
|
|
8753
|
+
}
|
|
8754
|
+
function messageOf(err) {
|
|
8755
|
+
if (err instanceof Error) return err.message;
|
|
8756
|
+
if (typeof err === "string") return err;
|
|
8757
|
+
if (err && typeof err === "object") {
|
|
8758
|
+
const e = err;
|
|
8759
|
+
if (typeof e["message"] === "string") return e["message"];
|
|
8760
|
+
const inner = e["error"];
|
|
8761
|
+
if (inner && typeof inner["message"] === "string") return inner["message"];
|
|
8762
|
+
try {
|
|
8763
|
+
return JSON.stringify(err);
|
|
8764
|
+
} catch {
|
|
8765
|
+
}
|
|
8766
|
+
}
|
|
8767
|
+
return String(err);
|
|
8768
|
+
}
|
|
8769
|
+
function classifyProviderError(err) {
|
|
8770
|
+
const raw = messageOf(err).trim();
|
|
8771
|
+
const status = statusOf(err);
|
|
8772
|
+
const m = raw.toLowerCase();
|
|
8773
|
+
const kind = (() => {
|
|
8774
|
+
if (status === 429) {
|
|
8775
|
+
return /quota|billing|credit|exceeded your current quota|insufficient/.test(m) ? "quota_exhausted" : "rate_limit";
|
|
8776
|
+
}
|
|
8777
|
+
if (status === 401 || status === 403) return "auth";
|
|
8778
|
+
if (status === 404) return "model_unavailable";
|
|
8779
|
+
if (/rate limit|too many requests|resource_exhausted/.test(m)) {
|
|
8780
|
+
return /quota|billing|credit|insufficient/.test(m) ? "quota_exhausted" : "rate_limit";
|
|
8781
|
+
}
|
|
8782
|
+
if (/quota|billing|insufficient_quota|insufficient credit|payment required/.test(m)) return "quota_exhausted";
|
|
8783
|
+
if (/api key|unauthorized|authentication|permission denied|invalid[_ ]api/.test(m)) return "auth";
|
|
8784
|
+
if (/model .*(not found|does not exist|unavailable)|no such model|unknown model|not supported for/.test(m)) {
|
|
8785
|
+
return "model_unavailable";
|
|
8786
|
+
}
|
|
8787
|
+
if (/context length|too many tokens|maximum context|prompt is too long|input too long/.test(m)) return "context_length";
|
|
8788
|
+
if (/safety|content filter|blocked|recitation|prohibited/.test(m)) return "content_filter";
|
|
8789
|
+
if (/econnreset|etimedout|enotfound|socket hang up|network|fetch failed|aborted/.test(m)) return "network";
|
|
8790
|
+
return "unknown";
|
|
8791
|
+
})();
|
|
8792
|
+
return { kind, systemic: isSystemicKind(kind), status, raw };
|
|
8793
|
+
}
|
|
8794
|
+
function describeProviderError(c, modelId) {
|
|
8795
|
+
const on = ` on ${modelId}` ;
|
|
8796
|
+
const detail = c.raw ? ` Provider said: ${c.raw}` : "";
|
|
8797
|
+
switch (c.kind) {
|
|
8798
|
+
case "rate_limit":
|
|
8799
|
+
return `Rate limit hit${on}. The provider is throttling requests \u2014 lower the number of parallel workers, or use a model with a higher requests-per-minute allowance.${detail}`;
|
|
8800
|
+
case "quota_exhausted":
|
|
8801
|
+
return `Quota or billing limit reached${on}. This will not recover on its own \u2014 check the provider's billing page, or switch this tier to a different provider.${detail}`;
|
|
8802
|
+
case "auth":
|
|
8803
|
+
return `Authentication failed${on}. The API key is missing, expired, or lacks access to this model \u2014 re-check the key in Settings > Providers.${detail}`;
|
|
8804
|
+
case "model_unavailable":
|
|
8805
|
+
return `Model unavailable${on}. The id may be wrong, retired, or not enabled for this API key \u2014 pick a different model for this tier.${detail}`;
|
|
8806
|
+
case "context_length":
|
|
8807
|
+
return `The prompt was too long${on}. Reduce the attached context, or use a model with a larger context window for this tier.${detail}`;
|
|
8808
|
+
case "content_filter":
|
|
8809
|
+
return `The provider's safety filter refused this request${on}.${detail}`;
|
|
8810
|
+
case "network":
|
|
8811
|
+
return `Network error reaching the provider${on}. This is usually transient.${detail}`;
|
|
8812
|
+
default:
|
|
8813
|
+
return `The model call failed${on}.${detail}`;
|
|
8814
|
+
}
|
|
8815
|
+
}
|
|
8816
|
+
|
|
8817
|
+
// src/core/run-breaker.ts
|
|
8818
|
+
var DEFAULT_FAILURE_THRESHOLD = 3;
|
|
8819
|
+
var RunBreaker = class {
|
|
8820
|
+
threshold;
|
|
8821
|
+
/** modelId → consecutive systemic failures of the SAME kind. */
|
|
8822
|
+
counts = /* @__PURE__ */ new Map();
|
|
8823
|
+
trip = null;
|
|
8824
|
+
constructor(threshold = DEFAULT_FAILURE_THRESHOLD) {
|
|
8825
|
+
this.threshold = Math.max(1, threshold);
|
|
8826
|
+
}
|
|
8827
|
+
/**
|
|
8828
|
+
* Record a failed model call. Returns the trip if this failure is the one
|
|
8829
|
+
* that opened the breaker, otherwise null.
|
|
8830
|
+
*
|
|
8831
|
+
* A non-systemic failure RESETS the model's counter: it is evidence the model
|
|
8832
|
+
* is reachable and answering, which is exactly what a systemic streak claims
|
|
8833
|
+
* it is not.
|
|
8834
|
+
*/
|
|
8835
|
+
record(err, modelId) {
|
|
8836
|
+
if (this.trip) return null;
|
|
8837
|
+
const id = modelId?.trim() || "unknown-model";
|
|
8838
|
+
const c = classifyProviderError(err);
|
|
8839
|
+
if (!c.systemic) {
|
|
8840
|
+
this.counts.delete(id);
|
|
8841
|
+
return null;
|
|
8842
|
+
}
|
|
8843
|
+
const prev = this.counts.get(id);
|
|
8844
|
+
const n = prev && prev.kind === c.kind ? prev.n + 1 : 1;
|
|
8845
|
+
this.counts.set(id, { kind: c.kind, n, last: c });
|
|
8846
|
+
if (n >= this.threshold) {
|
|
8847
|
+
this.trip = {
|
|
8848
|
+
modelId: id,
|
|
8849
|
+
kind: c.kind,
|
|
8850
|
+
failures: n,
|
|
8851
|
+
message: describeProviderError(c, id)
|
|
8852
|
+
};
|
|
8853
|
+
return this.trip;
|
|
8854
|
+
}
|
|
8855
|
+
return null;
|
|
8856
|
+
}
|
|
8857
|
+
/** True once the run should stop launching work that would fail identically. */
|
|
8858
|
+
isOpen() {
|
|
8859
|
+
return this.trip !== null;
|
|
8860
|
+
}
|
|
8861
|
+
/** Why the breaker opened, or null while it is closed. */
|
|
8862
|
+
reason() {
|
|
8863
|
+
return this.trip;
|
|
8864
|
+
}
|
|
8865
|
+
/**
|
|
8866
|
+
* One line for a worker/section that never ran because the breaker was
|
|
8867
|
+
* already open. Says the work was skipped deliberately — not that it failed
|
|
8868
|
+
* on its own merits, which is what an empty FAILED status would imply.
|
|
8869
|
+
*/
|
|
8870
|
+
skipMessage() {
|
|
8871
|
+
const t = this.trip;
|
|
8872
|
+
if (!t) return "Skipped.";
|
|
8873
|
+
return `Skipped \u2014 the run stopped after ${t.failures} consecutive failures on ${t.modelId}. ${t.message}`;
|
|
8874
|
+
}
|
|
8875
|
+
/** Test/diagnostic view of the per-model streaks. */
|
|
8876
|
+
streaks() {
|
|
8877
|
+
const out = {};
|
|
8878
|
+
for (const [k, v] of this.counts) out[k] = { kind: v.kind, n: v.n };
|
|
8879
|
+
return out;
|
|
8880
|
+
}
|
|
8881
|
+
};
|
|
8882
|
+
|
|
8664
8883
|
// src/tools/base.ts
|
|
8665
8884
|
var BaseTool = class {
|
|
8666
8885
|
workspaceRoot = process.cwd();
|
|
@@ -14233,6 +14452,7 @@ ${prompt}` : prompt;
|
|
|
14233
14452
|
if (toolCreator) toolCreator.setPermissionEscalator(escalator);
|
|
14234
14453
|
const rootPrompt = buildContextualPrompt(options.prompt, options.conversationHistory);
|
|
14235
14454
|
let finalOutput = "";
|
|
14455
|
+
const runBreaker = new RunBreaker();
|
|
14236
14456
|
let t2Results = [];
|
|
14237
14457
|
let runError = null;
|
|
14238
14458
|
let identityPrompt = "";
|
|
@@ -14301,6 +14521,7 @@ ${prompt}` : prompt;
|
|
|
14301
14521
|
this.emit("tier:status", { tierId: "t3-root", status: "COMPLETED", role: "T3" });
|
|
14302
14522
|
} else if (complexity === "Moderate") {
|
|
14303
14523
|
const t2 = new T2Manager(this.router, this.toolRegistry, "root");
|
|
14524
|
+
t2.setRunBreaker(runBreaker);
|
|
14304
14525
|
t2.setPresenter(true);
|
|
14305
14526
|
t2.setHierarchyContext("You are the ROOT Manager for this task. There is no T1 Administrator involved in this run. You are responsible for decomposing the task and managing T3 workers directly.");
|
|
14306
14527
|
if (identityPrompt) {
|
|
@@ -14361,6 +14582,7 @@ _(incomplete: ${reasons.join("; ")})_` : partial;
|
|
|
14361
14582
|
}
|
|
14362
14583
|
} else {
|
|
14363
14584
|
const t1 = new T1Administrator(this.router, this.toolRegistry, this.config);
|
|
14585
|
+
t1.setRunBreaker(runBreaker);
|
|
14364
14586
|
t1.setPresenter(true);
|
|
14365
14587
|
t1.setHierarchyContext("You are the top-level Administrator. You are responsible for the overall plan and supervising multiple T2 Managers.");
|
|
14366
14588
|
if (identityPrompt) {
|
|
@@ -14385,6 +14607,20 @@ _(incomplete: ${reasons.join("; ")})_` : partial;
|
|
|
14385
14607
|
finalOutput = result.output;
|
|
14386
14608
|
t2Results = result.t2Results;
|
|
14387
14609
|
}
|
|
14610
|
+
const trip = runBreaker.reason();
|
|
14611
|
+
if (trip) {
|
|
14612
|
+
finalOutput = [
|
|
14613
|
+
`\u26A0 **Run stopped early \u2014 ${trip.modelId} was failing every request.**`,
|
|
14614
|
+
"",
|
|
14615
|
+
trip.message,
|
|
14616
|
+
"",
|
|
14617
|
+
`Cascade stopped after ${trip.failures} consecutive failures rather than retrying every remaining subtask against the same model, which would have cost more and produced the same result.`,
|
|
14618
|
+
finalOutput ? `
|
|
14619
|
+
---
|
|
14620
|
+
|
|
14621
|
+
${finalOutput}` : ""
|
|
14622
|
+
].join("\n").trimEnd();
|
|
14623
|
+
}
|
|
14388
14624
|
} catch (err) {
|
|
14389
14625
|
if (err instanceof CascadeCancelledError || err instanceof Error && err.name === "AbortError" || options.signal?.aborted) {
|
|
14390
14626
|
this.emit("run:cancelled", {
|