@coool/opencode-adaptive 1.0.1 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/health/classifier/classify.js +22 -3
- package/dist/health/classifier/classify.js.map +1 -1
- package/dist/health/classifier/types.d.ts +2 -0
- package/dist/health/classifier/types.js.map +1 -1
- package/dist/supervisor/supervisor.js +110 -7
- package/dist/supervisor/supervisor.js.map +1 -1
- package/dist/ui/tui/controller.js +48 -18
- package/dist/ui/tui/controller.js.map +1 -1
- package/package.json +1 -1
|
@@ -4,6 +4,9 @@
|
|
|
4
4
|
* classification cost or smuggle patterns in late bytes (ADP-008).
|
|
5
5
|
*/
|
|
6
6
|
const TEXT_BOUND = 512;
|
|
7
|
+
// These messages describe the subscription's shared allowance or balance,
|
|
8
|
+
// unlike a generic quota error or a named model-family limit (ADP-007).
|
|
9
|
+
const SHARED_QUOTA_PATTERN = /\byou['’]ve hit your (?:session|usage|weekly) limit(?=\s*(?:$|[·.,;(:]|resets?\b))|\b(?:out of credits|insufficient credits|credit balance)\b/i;
|
|
7
10
|
/**
|
|
8
11
|
* Retry/failover policy per category (ADP-010, ADP-011).
|
|
9
12
|
*
|
|
@@ -54,7 +57,19 @@ const TEXTUAL_PATTERNS = [
|
|
|
54
57
|
category: "context_overflow",
|
|
55
58
|
pattern: /context (length|window)|maximum context|input length exceeds|prompt is too long|too many (input )?tokens|exceeds the maximum/i,
|
|
56
59
|
},
|
|
57
|
-
{
|
|
60
|
+
{
|
|
61
|
+
// Subscription limits ("You've hit your session/usage limit") are quota
|
|
62
|
+
// exhaustion with a provider-side reset window; the reset time in the
|
|
63
|
+
// text is not parsed (the classifier has no clock) — the health layer's
|
|
64
|
+
// default quota window bounds the quarantine instead.
|
|
65
|
+
// Sibling fallback plugins' issue trackers document the real-world
|
|
66
|
+
// phrasings that missed curated whitelists (oh-my-openagent #2393/#5815,
|
|
67
|
+
// lichter's 429 lesson): snake-cased limits, "usage exceeded" without
|
|
68
|
+
// "limit", credit exhaustion without "balance". Qualifiers stay required
|
|
69
|
+
// so "token limit reached" (context overflow) never matches quota.
|
|
70
|
+
category: "quota_exhausted",
|
|
71
|
+
pattern: /\bquota\b|credit balance|(?:session|usage|free usage|message|plan|monthly|weekly|daily)[ _-]?limit|usage exceeded|out of credits|insufficient credits/i,
|
|
72
|
+
},
|
|
58
73
|
{ category: "rate_limited", pattern: /rate ?limit/i },
|
|
59
74
|
{ category: "timeout", pattern: /\btimeout\b|timed? ?out|deadline exceeded|etimedout/i },
|
|
60
75
|
{
|
|
@@ -141,7 +156,7 @@ function retryAdvice(category, retryAfterMs) {
|
|
|
141
156
|
}
|
|
142
157
|
return { retryable: policy.retryable, retryAfterMs: bounded };
|
|
143
158
|
}
|
|
144
|
-
function classification(category, layer, definitive, retryAfterMs) {
|
|
159
|
+
function classification(category, layer, definitive, retryAfterMs, quotaScope) {
|
|
145
160
|
const policy = CATEGORY_POLICY[category];
|
|
146
161
|
return {
|
|
147
162
|
category,
|
|
@@ -149,6 +164,7 @@ function classification(category, layer, definitive, retryAfterMs) {
|
|
|
149
164
|
definitive,
|
|
150
165
|
retry: retryAdvice(category, retryAfterMs),
|
|
151
166
|
failoverEligible: policy.failoverEligible,
|
|
167
|
+
...(quotaScope !== undefined ? { quotaScope } : {}),
|
|
152
168
|
};
|
|
153
169
|
}
|
|
154
170
|
/**
|
|
@@ -237,7 +253,10 @@ export function classifyFailure(evidence) {
|
|
|
237
253
|
if (text !== undefined) {
|
|
238
254
|
const category = classifyText(text);
|
|
239
255
|
if (category !== undefined) {
|
|
240
|
-
|
|
256
|
+
const quotaScope = category === "quota_exhausted" && SHARED_QUOTA_PATTERN.test(text.slice(0, TEXT_BOUND))
|
|
257
|
+
? "provider"
|
|
258
|
+
: undefined;
|
|
259
|
+
return classification(category, "textual", false, structuredRetryAfterMs, quotaScope);
|
|
241
260
|
}
|
|
242
261
|
}
|
|
243
262
|
return classification("unknown", "none", false, undefined);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"classify.js","sourceRoot":"","sources":["../../../src/health/classifier/classify.ts"],"names":[],"mappings":"AAEA;;;;GAIG;AACH,MAAM,UAAU,GAAG,GAAG,CAAA;AAEtB;;;;;;;;;;;GAWG;AACH,MAAM,eAAe,GAEjB;IACF,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE;IACzD,eAAe,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,gBAAgB,EAAE,IAAI,EAAE;IAC7D,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE;IACnD,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE;IACrD,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE;IACpD,YAAY,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAE;IAC3D,aAAa,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAE;IAC5D,gBAAgB,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAE;IAC/D,cAAc,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAE;IAC7D,cAAc,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAE;IAC7D,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAE;CACvD,CAAA;AAED;;;;;;GAMG;AACH,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAA0B;IACxD,CAAC,mBAAmB,EAAE,cAAc,CAAC;IACrC,CAAC,gBAAgB,EAAE,cAAc,CAAC;IAClC,CAAC,oBAAoB,EAAE,eAAe,CAAC;IACvC,CAAC,uBAAuB,EAAE,eAAe,CAAC;IAC1C,CAAC,yBAAyB,EAAE,QAAQ,CAAC;CACtC,CAAC,CAAA;AAEF;;;;;GAKG;AACH,MAAM,gBAAgB,GAGhB;IACJ;QACE,QAAQ,EAAE,kBAAkB;QAC5B,OAAO,EACL,+HAA+H;KAClI;IACD,EAAE,QAAQ,EAAE,iBAAiB,EAAE,OAAO,EAAE,2BAA2B,EAAE;IACrE,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,cAAc,EAAE;IACrD,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,sDAAsD,EAAE;IACxF;QACE,QAAQ,EAAE,cAAc;QACxB,OAAO,EACL,mGAAmG;KACtG;IACD;QACE,QAAQ,EAAE,eAAe;QACzB,OAAO,EAAE,mFAAmF;KAC7F;IACD;QACE,QAAQ,EAAE,gBAAgB;QAC1B,OAAO,EACL,wFAAwF;KAC3F;IACD,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,uCAAuC,EAAE;IAC1E,EAAE,QAAQ,EAAE,gBAAgB,EAAE,OAAO,EAAE,kDAAkD,EAAE;CAC5F,CAAA;AAED,SAAS,YAAY,CAAC,MAA0B;IAC9C,OAAO,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,CAAA;AACjG,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,kBAAkB,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAE9C,SAAS,mBAAmB,CAAC,YAAgC;IAC3D,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;QAC/B,OAAO,SAAS,CAAA;IAClB,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAA;AACnD,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAyB;IACrD,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;AAC3E,CAAC;AAED,SAAS,iBAAiB,CACxB,OAAqD;IAErD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAA;IACzC,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,OAAO,OAAO,CAAA;IAChB,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,CAAA;IACxC,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED;;;;;GAKG;AACH,SAAS,kBAAkB,CAAC,OAAoC;IAC9D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAA;IACtC,IAAI,GAAG,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5C,OAAO,SAAS,CAAA;IAClB,CAAC;IACD,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAA;AAC3B,CAAC;AAED;;;;GAIG;AACH,SAAS,oBAAoB,CAAC,OAAoC;IAChE,IAAI,YAAY,GAAG,KAAK,CAAA;IACxB,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,IAAI,IAAI,CAAC,UAAU,CAAC,uBAAuB,CAAC,EAAE,CAAC;YAC7C,YAAY,GAAG,IAAI,CAAA;YACnB,IAAI,KAAK,KAAK,GAAG,EAAE,CAAC;gBAClB,OAAO,KAAK,CAAA;YACd,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,YAAY,CAAA;AACrB,CAAC;AAED,SAAS,WAAW,CAAC,QAAyB,EAAE,YAAgC;IAC9E,MAAM,MAAM,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAA;IACxC,MAAM,OAAO,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAA;IACjD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAA;IACxC,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,CAAA;AAC/D,CAAC;AAED,SAAS,cAAc,CACrB,QAAyB,EACzB,KAAqC,EACrC,UAAmB,EACnB,YAAgC;IAEhC,MAAM,MAAM,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAA;IACxC,OAAO;QACL,QAAQ;QACR,KAAK;QACL,UAAU;QACV,KAAK,EAAE,WAAW,CAAC,QAAQ,EAAE,YAAY,CAAC;QAC1C,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;KAC1C,CAAA;AACH,CAAC;AAED;;;;;GAKG;AACH,SAAS,cAAc,CACrB,MAAc,EACd,OAAoC;IAEpC,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QACnB,OAAO,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,cAAc,CAAA;IAC3E,CAAC;IACD,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QACrC,OAAO,cAAc,CAAA;IACvB,CAAC;IACD,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QACnB,OAAO,SAAS,CAAA;IAClB,CAAC;IACD,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QACnB,OAAO,kBAAkB,CAAA;IAC3B,CAAC;IACD,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QACvD,OAAO,QAAQ,CAAA;IACjB,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC3E,KAAK,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,gBAAgB,EAAE,CAAC;QACrD,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1B,OAAO,QAAQ,CAAA;QACjB,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,QAAuB;IACrD,wEAAwE;IACxE,6DAA6D;IAC7D,IAAI,QAAQ,CAAC,WAAW,KAAK,MAAM,EAAE,CAAC;QACpC,OAAO,cAAc,CAAC,gBAAgB,EAAE,cAAc,EAAE,IAAI,EAAE,SAAS,CAAC,CAAA;IAC1E,CAAC;IACD,IAAI,QAAQ,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACvC,OAAO,cAAc,CAAC,SAAS,EAAE,cAAc,EAAE,IAAI,EAAE,SAAS,CAAC,CAAA;IACnE,CAAC;IAED,yCAAyC;IACzC,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAA;IACtC,MAAM,sBAAsB,GAAG,oBAAoB,CAAC,UAAU,EAAE,YAAY,CAAC;QAC3E,CAAC,CAAC,UAAU,CAAC,YAAY;QACzB,CAAC,CAAC,SAAS,CAAA;IACb,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;QACpD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,cAAc,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,sBAAsB,CAAC,CAAA;QAC3E,CAAC;QACD,wEAAwE;IAC1E,CAAC;IAED,oEAAoE;IACpE,MAAM,OAAO,GAAG,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAA;IACnD,IAAI,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QACzD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,cAAc,CACnB,QAAQ,EACR,QAAQ,EACR,IAAI,EACJ,sBAAsB,IAAI,kBAAkB,CAAC,OAAO,CAAC,CACtD,CAAA;QACH,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAA;IACtD,IAAI,kBAAkB,KAAK,SAAS,EAAE,CAAC;QACrC,OAAO,cAAc,CAAC,cAAc,EAAE,SAAS,EAAE,KAAK,EAAE,kBAAkB,CAAC,CAAA;IAC7E,CAAC;IAED,wEAAwE;IACxE,oCAAoC;IACpC,IAAI,QAAQ,CAAC,gBAAgB,EAAE,cAAc,KAAK,IAAI,EAAE,CAAC;QACvD,OAAO,cAAc,CAAC,iBAAiB,EAAE,mBAAmB,EAAE,IAAI,EAAE,SAAS,CAAC,CAAA;IAChF,CAAC;IACD,IAAI,QAAQ,CAAC,gBAAgB,EAAE,UAAU,KAAK,IAAI,EAAE,CAAC;QACnD,OAAO,cAAc,CAAC,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,sBAAsB,CAAC,CAAA;IACtF,CAAC;IAED,kEAAkE;IAClE,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAA;IAC1B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;QACnC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,cAAc,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,sBAAsB,CAAC,CAAA;QAC3E,CAAC;IACH,CAAC;IAED,OAAO,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,CAAA;AAC5D,CAAC","sourcesContent":["import type { ErrorEvidence, FailureCategory, FailureClassification, RetryAdvice } from \"./types.js\"\n\n/**\n * Text fed to the textual last-resort layer is truncated to this bound before\n * matching, so adversarial or enormous error bodies cannot dominate\n * classification cost or smuggle patterns in late bytes (ADP-008).\n */\nconst TEXT_BOUND = 512\n\n/**\n * Retry/failover policy per category (ADP-010, ADP-011).\n *\n * - Infrastructure failures (rate limiting, quota, outage, overload, timeout)\n * are failover-eligible: horizontal recovery within the tier is exactly\n * their remedy.\n * - Permanent configuration failures (auth, invalid model, content policy)\n * and context overflow never retry and never failover: retrying the same\n * request cannot help and switching resources would reproduce the failure.\n * - User cancellation is final: no retry, no failover (ADP-010).\n * - Unknown failures defer to OpenCode's native handling.\n */\nconst CATEGORY_POLICY: Readonly<\n Record<FailureCategory, { retryable: boolean; failoverEligible: boolean }>\n> = {\n rate_limited: { retryable: true, failoverEligible: true },\n quota_exhausted: { retryable: false, failoverEligible: true },\n outage: { retryable: true, failoverEligible: true },\n overload: { retryable: true, failoverEligible: true },\n timeout: { retryable: true, failoverEligible: true },\n auth_failure: { retryable: false, failoverEligible: false },\n invalid_model: { retryable: false, failoverEligible: false },\n context_overflow: { retryable: false, failoverEligible: false },\n content_policy: { retryable: false, failoverEligible: false },\n user_cancelled: { retryable: false, failoverEligible: false },\n unknown: { retryable: false, failoverEligible: false },\n}\n\n/**\n * Error tags from the documented @opencode/protocol vocabulary that map\n * directly to a failure category. Tags absent here (`ProviderRequestError`,\n * `UnknownError`, anything unrecognized) are structured but not\n * category-specific: they never guess, and classification falls through to\n * the refining layers.\n */\nconst STRUCTURED_TYPES = new Map<string, FailureCategory>([\n [\"UnauthorizedError\", \"auth_failure\"],\n [\"ForbiddenError\", \"auth_failure\"],\n [\"ModelNotFoundError\", \"invalid_model\"],\n [\"UnsupportedModelError\", \"invalid_model\"],\n [\"ServiceUnavailableError\", \"outage\"],\n])\n\n/**\n * Bounded textual patterns, last resort only (ADP-008). Ordered most\n * specific first; the first match wins. These encode provider error-message\n * conventions documented by the research phase, not a general-purpose\n * keyword classifier.\n */\nconst TEXTUAL_PATTERNS: readonly {\n readonly category: FailureCategory\n readonly pattern: RegExp\n}[] = [\n {\n category: \"context_overflow\",\n pattern:\n /context (length|window)|maximum context|input length exceeds|prompt is too long|too many (input )?tokens|exceeds the maximum/i,\n },\n { category: \"quota_exhausted\", pattern: /\\bquota\\b|credit balance/i },\n { category: \"rate_limited\", pattern: /rate ?limit/i },\n { category: \"timeout\", pattern: /\\btimeout\\b|timed? ?out|deadline exceeded|etimedout/i },\n {\n category: \"auth_failure\",\n pattern:\n /unauthorized|invalid (api|access) key|api key (expired|invalid)|authentication|permission denied/i,\n },\n {\n category: \"invalid_model\",\n pattern: /model.{0,30}(does not exist|not found)|invalid model|unknown model|no such model/i,\n },\n {\n category: \"content_policy\",\n pattern:\n /content (management )?(policy|filter)|safety system|violates?.{0,32}policy|flagged as/i,\n },\n { category: \"overload\", pattern: /overloaded|over capacity|high demand/i },\n { category: \"user_cancelled\", pattern: /cancel+(l)?ed|aborted by (the )?user|user abort/i },\n]\n\nfunction isHttpStatus(status: number | undefined): status is number {\n return typeof status === \"number\" && Number.isInteger(status) && status >= 100 && status <= 599\n}\n\n/**\n * Upper bound on a provider-claimed retry delay. `Retry-After` and structured\n * retry metadata are untrusted provider data: a hostile or misconfigured\n * value (\"999999999\" seconds) must not convert one response into a\n * near-permanent resource blacklist. The host's own retry scheduling waits\n * out the raw header verbatim — this bound only concerns the health window\n * the supervisor derives from the classification (ADP-011 boundedness).\n */\nconst MAX_RETRY_AFTER_MS = 24 * 60 * 60 * 1000\n\nfunction boundedRetryAfterMs(retryAfterMs: number | undefined): number | undefined {\n if (retryAfterMs === undefined) {\n return undefined\n }\n return Math.min(retryAfterMs, MAX_RETRY_AFTER_MS)\n}\n\nfunction isNonNegativeInteger(value: number | undefined): value is number {\n return typeof value === \"number\" && Number.isInteger(value) && value >= 0\n}\n\nfunction lowercasedHeaders(\n headers: Readonly<Record<string, string>> | undefined,\n): ReadonlyMap<string, string> {\n const lowered = new Map<string, string>()\n if (headers === undefined) {\n return lowered\n }\n for (const [name, value] of Object.entries(headers)) {\n lowered.set(name.toLowerCase(), value)\n }\n return lowered\n}\n\n/**\n * Parses a `Retry-After` header into milliseconds. Only the integer-seconds\n * form is accepted: the HTTP-date form requires a clock, which the classifier\n * deliberately does not have (reset-time handling belongs to the capacity\n * boundary, T-05/T-15).\n */\nfunction retryAfterHeaderMs(headers: ReadonlyMap<string, string>): number | undefined {\n const raw = headers.get(\"retry-after\")\n if (raw === undefined || !/^\\d+$/.test(raw)) {\n return undefined\n }\n return Number(raw) * 1000\n}\n\n/**\n * Standard rate-limit headers (case-insensitive `x-ratelimit-remaining*`).\n * All-zero remaining capacity indicates a fixed-window quota exhaustion\n * rather than a short burst limit.\n */\nfunction fixedWindowExhausted(headers: ReadonlyMap<string, string>): boolean {\n let sawRemaining = false\n for (const [name, value] of headers) {\n if (name.startsWith(\"x-ratelimit-remaining\")) {\n sawRemaining = true\n if (value !== \"0\") {\n return false\n }\n }\n }\n return sawRemaining\n}\n\nfunction retryAdvice(category: FailureCategory, retryAfterMs: number | undefined): RetryAdvice {\n const policy = CATEGORY_POLICY[category]\n const bounded = boundedRetryAfterMs(retryAfterMs)\n if (bounded === undefined) {\n return { retryable: policy.retryable }\n }\n return { retryable: policy.retryable, retryAfterMs: bounded }\n}\n\nfunction classification(\n category: FailureCategory,\n layer: FailureClassification[\"layer\"],\n definitive: boolean,\n retryAfterMs: number | undefined,\n): FailureClassification {\n const policy = CATEGORY_POLICY[category]\n return {\n category,\n layer,\n definitive,\n retry: retryAdvice(category, retryAfterMs),\n failoverEligible: policy.failoverEligible,\n }\n}\n\n/**\n * Classifies an HTTP status. 429 consults the standard rate-limit headers in\n * the same step: status plus documented standard headers are one HTTP\n * semantics layer, and the quota refinement is more specific evidence than a\n * bare 429, so it cannot weaken the ADP-008 precedence.\n */\nfunction classifyStatus(\n status: number,\n headers: ReadonlyMap<string, string>,\n): FailureCategory | undefined {\n if (status === 429) {\n return fixedWindowExhausted(headers) ? \"quota_exhausted\" : \"rate_limited\"\n }\n if (status === 401 || status === 403) {\n return \"auth_failure\"\n }\n if (status === 408) {\n return \"timeout\"\n }\n if (status === 413) {\n return \"context_overflow\"\n }\n if (status === 502 || status === 503 || status === 504) {\n return \"outage\"\n }\n return undefined\n}\n\nfunction classifyText(text: string): FailureCategory | undefined {\n const bounded = text.length > TEXT_BOUND ? text.slice(0, TEXT_BOUND) : text\n for (const { category, pattern } of TEXTUAL_PATTERNS) {\n if (pattern.test(bounded)) {\n return category\n }\n }\n return undefined\n}\n\n/**\n * Total classification of layered failure evidence. Never throws and always\n * returns a category: any evidence shape outside the documented vocabularies\n * classifies as `unknown` via the `none` layer, deferring to OpenCode's\n * native failure handling (ADP-009, ADP-012).\n */\nexport function classifyFailure(evidence: ErrorEvidence): FailureClassification {\n // Layer 0: interruption evidence. Interruptions are host decisions, not\n // provider failures; a user cancellation is final (ADP-010).\n if (evidence.interrupted === \"user\") {\n return classification(\"user_cancelled\", \"interruption\", true, undefined)\n }\n if (evidence.interrupted !== undefined) {\n return classification(\"unknown\", \"interruption\", true, undefined)\n }\n\n // Layer 1: structured OpenCode metadata.\n const structured = evidence.structured\n const structuredRetryAfterMs = isNonNegativeInteger(structured?.retryAfterMs)\n ? structured.retryAfterMs\n : undefined\n if (structured !== undefined) {\n const mapped = STRUCTURED_TYPES.get(structured.type)\n if (mapped !== undefined) {\n return classification(mapped, \"structured\", true, structuredRetryAfterMs)\n }\n // Unmapped tag: fall through so HTTP semantics can refine the category.\n }\n\n // Layer 2: HTTP status (+ standard headers for the 429 refinement).\n const headers = lowercasedHeaders(evidence.headers)\n if (isHttpStatus(evidence.status)) {\n const category = classifyStatus(evidence.status, headers)\n if (category !== undefined) {\n return classification(\n category,\n \"status\",\n true,\n structuredRetryAfterMs ?? retryAfterHeaderMs(headers),\n )\n }\n }\n\n // Layer 3: standard headers alone (Retry-After without a decisive status).\n const headerRetryAfterMs = retryAfterHeaderMs(headers)\n if (headerRetryAfterMs !== undefined) {\n return classification(\"rate_limited\", \"headers\", false, headerRetryAfterMs)\n }\n\n // Layer 4: documented provider-specific metadata, normalized by our own\n // adapters after validation (T-15).\n if (evidence.providerMetadata?.quotaExhausted === true) {\n return classification(\"quota_exhausted\", \"provider-metadata\", true, undefined)\n }\n if (evidence.providerMetadata?.overloaded === true) {\n return classification(\"overload\", \"provider-metadata\", true, structuredRetryAfterMs)\n }\n\n // Layer 5: bounded textual classification, always non-definitive.\n const text = evidence.text\n if (text !== undefined) {\n const category = classifyText(text)\n if (category !== undefined) {\n return classification(category, \"textual\", false, structuredRetryAfterMs)\n }\n }\n\n return classification(\"unknown\", \"none\", false, undefined)\n}\n"]}
|
|
1
|
+
{"version":3,"file":"classify.js","sourceRoot":"","sources":["../../../src/health/classifier/classify.ts"],"names":[],"mappings":"AAEA;;;;GAIG;AACH,MAAM,UAAU,GAAG,GAAG,CAAA;AAEtB,0EAA0E;AAC1E,wEAAwE;AACxE,MAAM,oBAAoB,GACxB,gJAAgJ,CAAA;AAElJ;;;;;;;;;;;GAWG;AACH,MAAM,eAAe,GAEjB;IACF,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE;IACzD,eAAe,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,gBAAgB,EAAE,IAAI,EAAE;IAC7D,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE;IACnD,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE;IACrD,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE;IACpD,YAAY,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAE;IAC3D,aAAa,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAE;IAC5D,gBAAgB,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAE;IAC/D,cAAc,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAE;IAC7D,cAAc,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAE;IAC7D,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAE;CACvD,CAAA;AAED;;;;;;GAMG;AACH,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAA0B;IACxD,CAAC,mBAAmB,EAAE,cAAc,CAAC;IACrC,CAAC,gBAAgB,EAAE,cAAc,CAAC;IAClC,CAAC,oBAAoB,EAAE,eAAe,CAAC;IACvC,CAAC,uBAAuB,EAAE,eAAe,CAAC;IAC1C,CAAC,yBAAyB,EAAE,QAAQ,CAAC;CACtC,CAAC,CAAA;AAEF;;;;;GAKG;AACH,MAAM,gBAAgB,GAGhB;IACJ;QACE,QAAQ,EAAE,kBAAkB;QAC5B,OAAO,EACL,+HAA+H;KAClI;IACD;QACE,wEAAwE;QACxE,sEAAsE;QACtE,wEAAwE;QACxE,sDAAsD;QACtD,mEAAmE;QACnE,yEAAyE;QACzE,sEAAsE;QACtE,yEAAyE;QACzE,mEAAmE;QACnE,QAAQ,EAAE,iBAAiB;QAC3B,OAAO,EACL,wJAAwJ;KAC3J;IACD,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,cAAc,EAAE;IACrD,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,sDAAsD,EAAE;IACxF;QACE,QAAQ,EAAE,cAAc;QACxB,OAAO,EACL,mGAAmG;KACtG;IACD;QACE,QAAQ,EAAE,eAAe;QACzB,OAAO,EAAE,mFAAmF;KAC7F;IACD;QACE,QAAQ,EAAE,gBAAgB;QAC1B,OAAO,EACL,wFAAwF;KAC3F;IACD,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,uCAAuC,EAAE;IAC1E,EAAE,QAAQ,EAAE,gBAAgB,EAAE,OAAO,EAAE,kDAAkD,EAAE;CAC5F,CAAA;AAED,SAAS,YAAY,CAAC,MAA0B;IAC9C,OAAO,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,CAAA;AACjG,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,kBAAkB,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAE9C,SAAS,mBAAmB,CAAC,YAAgC;IAC3D,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;QAC/B,OAAO,SAAS,CAAA;IAClB,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAA;AACnD,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAyB;IACrD,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;AAC3E,CAAC;AAED,SAAS,iBAAiB,CACxB,OAAqD;IAErD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAA;IACzC,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,OAAO,OAAO,CAAA;IAChB,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,CAAA;IACxC,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED;;;;;GAKG;AACH,SAAS,kBAAkB,CAAC,OAAoC;IAC9D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAA;IACtC,IAAI,GAAG,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5C,OAAO,SAAS,CAAA;IAClB,CAAC;IACD,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAA;AAC3B,CAAC;AAED;;;;GAIG;AACH,SAAS,oBAAoB,CAAC,OAAoC;IAChE,IAAI,YAAY,GAAG,KAAK,CAAA;IACxB,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,IAAI,IAAI,CAAC,UAAU,CAAC,uBAAuB,CAAC,EAAE,CAAC;YAC7C,YAAY,GAAG,IAAI,CAAA;YACnB,IAAI,KAAK,KAAK,GAAG,EAAE,CAAC;gBAClB,OAAO,KAAK,CAAA;YACd,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,YAAY,CAAA;AACrB,CAAC;AAED,SAAS,WAAW,CAAC,QAAyB,EAAE,YAAgC;IAC9E,MAAM,MAAM,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAA;IACxC,MAAM,OAAO,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAA;IACjD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAA;IACxC,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,CAAA;AAC/D,CAAC;AAED,SAAS,cAAc,CACrB,QAAyB,EACzB,KAAqC,EACrC,UAAmB,EACnB,YAAgC,EAChC,UAAuB;IAEvB,MAAM,MAAM,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAA;IACxC,OAAO;QACL,QAAQ;QACR,KAAK;QACL,UAAU;QACV,KAAK,EAAE,WAAW,CAAC,QAAQ,EAAE,YAAY,CAAC;QAC1C,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;QACzC,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACpD,CAAA;AACH,CAAC;AAED;;;;;GAKG;AACH,SAAS,cAAc,CACrB,MAAc,EACd,OAAoC;IAEpC,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QACnB,OAAO,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,cAAc,CAAA;IAC3E,CAAC;IACD,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QACrC,OAAO,cAAc,CAAA;IACvB,CAAC;IACD,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QACnB,OAAO,SAAS,CAAA;IAClB,CAAC;IACD,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QACnB,OAAO,kBAAkB,CAAA;IAC3B,CAAC;IACD,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QACvD,OAAO,QAAQ,CAAA;IACjB,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC3E,KAAK,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,gBAAgB,EAAE,CAAC;QACrD,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1B,OAAO,QAAQ,CAAA;QACjB,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,QAAuB;IACrD,wEAAwE;IACxE,6DAA6D;IAC7D,IAAI,QAAQ,CAAC,WAAW,KAAK,MAAM,EAAE,CAAC;QACpC,OAAO,cAAc,CAAC,gBAAgB,EAAE,cAAc,EAAE,IAAI,EAAE,SAAS,CAAC,CAAA;IAC1E,CAAC;IACD,IAAI,QAAQ,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACvC,OAAO,cAAc,CAAC,SAAS,EAAE,cAAc,EAAE,IAAI,EAAE,SAAS,CAAC,CAAA;IACnE,CAAC;IAED,yCAAyC;IACzC,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAA;IACtC,MAAM,sBAAsB,GAAG,oBAAoB,CAAC,UAAU,EAAE,YAAY,CAAC;QAC3E,CAAC,CAAC,UAAU,CAAC,YAAY;QACzB,CAAC,CAAC,SAAS,CAAA;IACb,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;QACpD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,cAAc,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,sBAAsB,CAAC,CAAA;QAC3E,CAAC;QACD,wEAAwE;IAC1E,CAAC;IAED,oEAAoE;IACpE,MAAM,OAAO,GAAG,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAA;IACnD,IAAI,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QACzD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,cAAc,CACnB,QAAQ,EACR,QAAQ,EACR,IAAI,EACJ,sBAAsB,IAAI,kBAAkB,CAAC,OAAO,CAAC,CACtD,CAAA;QACH,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAA;IACtD,IAAI,kBAAkB,KAAK,SAAS,EAAE,CAAC;QACrC,OAAO,cAAc,CAAC,cAAc,EAAE,SAAS,EAAE,KAAK,EAAE,kBAAkB,CAAC,CAAA;IAC7E,CAAC;IAED,wEAAwE;IACxE,oCAAoC;IACpC,IAAI,QAAQ,CAAC,gBAAgB,EAAE,cAAc,KAAK,IAAI,EAAE,CAAC;QACvD,OAAO,cAAc,CAAC,iBAAiB,EAAE,mBAAmB,EAAE,IAAI,EAAE,SAAS,CAAC,CAAA;IAChF,CAAC;IACD,IAAI,QAAQ,CAAC,gBAAgB,EAAE,UAAU,KAAK,IAAI,EAAE,CAAC;QACnD,OAAO,cAAc,CAAC,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,sBAAsB,CAAC,CAAA;IACtF,CAAC;IAED,kEAAkE;IAClE,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAA;IAC1B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;QACnC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,MAAM,UAAU,GACd,QAAQ,KAAK,iBAAiB,IAAI,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;gBACpF,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,SAAS,CAAA;YACf,OAAO,cAAc,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,sBAAsB,EAAE,UAAU,CAAC,CAAA;QACvF,CAAC;IACH,CAAC;IAED,OAAO,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,CAAA;AAC5D,CAAC","sourcesContent":["import type { ErrorEvidence, FailureCategory, FailureClassification, RetryAdvice } from \"./types.js\"\n\n/**\n * Text fed to the textual last-resort layer is truncated to this bound before\n * matching, so adversarial or enormous error bodies cannot dominate\n * classification cost or smuggle patterns in late bytes (ADP-008).\n */\nconst TEXT_BOUND = 512\n\n// These messages describe the subscription's shared allowance or balance,\n// unlike a generic quota error or a named model-family limit (ADP-007).\nconst SHARED_QUOTA_PATTERN =\n /\\byou['’]ve hit your (?:session|usage|weekly) limit(?=\\s*(?:$|[·.,;(:]|resets?\\b))|\\b(?:out of credits|insufficient credits|credit balance)\\b/i\n\n/**\n * Retry/failover policy per category (ADP-010, ADP-011).\n *\n * - Infrastructure failures (rate limiting, quota, outage, overload, timeout)\n * are failover-eligible: horizontal recovery within the tier is exactly\n * their remedy.\n * - Permanent configuration failures (auth, invalid model, content policy)\n * and context overflow never retry and never failover: retrying the same\n * request cannot help and switching resources would reproduce the failure.\n * - User cancellation is final: no retry, no failover (ADP-010).\n * - Unknown failures defer to OpenCode's native handling.\n */\nconst CATEGORY_POLICY: Readonly<\n Record<FailureCategory, { retryable: boolean; failoverEligible: boolean }>\n> = {\n rate_limited: { retryable: true, failoverEligible: true },\n quota_exhausted: { retryable: false, failoverEligible: true },\n outage: { retryable: true, failoverEligible: true },\n overload: { retryable: true, failoverEligible: true },\n timeout: { retryable: true, failoverEligible: true },\n auth_failure: { retryable: false, failoverEligible: false },\n invalid_model: { retryable: false, failoverEligible: false },\n context_overflow: { retryable: false, failoverEligible: false },\n content_policy: { retryable: false, failoverEligible: false },\n user_cancelled: { retryable: false, failoverEligible: false },\n unknown: { retryable: false, failoverEligible: false },\n}\n\n/**\n * Error tags from the documented @opencode/protocol vocabulary that map\n * directly to a failure category. Tags absent here (`ProviderRequestError`,\n * `UnknownError`, anything unrecognized) are structured but not\n * category-specific: they never guess, and classification falls through to\n * the refining layers.\n */\nconst STRUCTURED_TYPES = new Map<string, FailureCategory>([\n [\"UnauthorizedError\", \"auth_failure\"],\n [\"ForbiddenError\", \"auth_failure\"],\n [\"ModelNotFoundError\", \"invalid_model\"],\n [\"UnsupportedModelError\", \"invalid_model\"],\n [\"ServiceUnavailableError\", \"outage\"],\n])\n\n/**\n * Bounded textual patterns, last resort only (ADP-008). Ordered most\n * specific first; the first match wins. These encode provider error-message\n * conventions documented by the research phase, not a general-purpose\n * keyword classifier.\n */\nconst TEXTUAL_PATTERNS: readonly {\n readonly category: FailureCategory\n readonly pattern: RegExp\n}[] = [\n {\n category: \"context_overflow\",\n pattern:\n /context (length|window)|maximum context|input length exceeds|prompt is too long|too many (input )?tokens|exceeds the maximum/i,\n },\n {\n // Subscription limits (\"You've hit your session/usage limit\") are quota\n // exhaustion with a provider-side reset window; the reset time in the\n // text is not parsed (the classifier has no clock) — the health layer's\n // default quota window bounds the quarantine instead.\n // Sibling fallback plugins' issue trackers document the real-world\n // phrasings that missed curated whitelists (oh-my-openagent #2393/#5815,\n // lichter's 429 lesson): snake-cased limits, \"usage exceeded\" without\n // \"limit\", credit exhaustion without \"balance\". Qualifiers stay required\n // so \"token limit reached\" (context overflow) never matches quota.\n category: \"quota_exhausted\",\n pattern:\n /\\bquota\\b|credit balance|(?:session|usage|free usage|message|plan|monthly|weekly|daily)[ _-]?limit|usage exceeded|out of credits|insufficient credits/i,\n },\n { category: \"rate_limited\", pattern: /rate ?limit/i },\n { category: \"timeout\", pattern: /\\btimeout\\b|timed? ?out|deadline exceeded|etimedout/i },\n {\n category: \"auth_failure\",\n pattern:\n /unauthorized|invalid (api|access) key|api key (expired|invalid)|authentication|permission denied/i,\n },\n {\n category: \"invalid_model\",\n pattern: /model.{0,30}(does not exist|not found)|invalid model|unknown model|no such model/i,\n },\n {\n category: \"content_policy\",\n pattern:\n /content (management )?(policy|filter)|safety system|violates?.{0,32}policy|flagged as/i,\n },\n { category: \"overload\", pattern: /overloaded|over capacity|high demand/i },\n { category: \"user_cancelled\", pattern: /cancel+(l)?ed|aborted by (the )?user|user abort/i },\n]\n\nfunction isHttpStatus(status: number | undefined): status is number {\n return typeof status === \"number\" && Number.isInteger(status) && status >= 100 && status <= 599\n}\n\n/**\n * Upper bound on a provider-claimed retry delay. `Retry-After` and structured\n * retry metadata are untrusted provider data: a hostile or misconfigured\n * value (\"999999999\" seconds) must not convert one response into a\n * near-permanent resource blacklist. The host's own retry scheduling waits\n * out the raw header verbatim — this bound only concerns the health window\n * the supervisor derives from the classification (ADP-011 boundedness).\n */\nconst MAX_RETRY_AFTER_MS = 24 * 60 * 60 * 1000\n\nfunction boundedRetryAfterMs(retryAfterMs: number | undefined): number | undefined {\n if (retryAfterMs === undefined) {\n return undefined\n }\n return Math.min(retryAfterMs, MAX_RETRY_AFTER_MS)\n}\n\nfunction isNonNegativeInteger(value: number | undefined): value is number {\n return typeof value === \"number\" && Number.isInteger(value) && value >= 0\n}\n\nfunction lowercasedHeaders(\n headers: Readonly<Record<string, string>> | undefined,\n): ReadonlyMap<string, string> {\n const lowered = new Map<string, string>()\n if (headers === undefined) {\n return lowered\n }\n for (const [name, value] of Object.entries(headers)) {\n lowered.set(name.toLowerCase(), value)\n }\n return lowered\n}\n\n/**\n * Parses a `Retry-After` header into milliseconds. Only the integer-seconds\n * form is accepted: the HTTP-date form requires a clock, which the classifier\n * deliberately does not have (reset-time handling belongs to the capacity\n * boundary, T-05/T-15).\n */\nfunction retryAfterHeaderMs(headers: ReadonlyMap<string, string>): number | undefined {\n const raw = headers.get(\"retry-after\")\n if (raw === undefined || !/^\\d+$/.test(raw)) {\n return undefined\n }\n return Number(raw) * 1000\n}\n\n/**\n * Standard rate-limit headers (case-insensitive `x-ratelimit-remaining*`).\n * All-zero remaining capacity indicates a fixed-window quota exhaustion\n * rather than a short burst limit.\n */\nfunction fixedWindowExhausted(headers: ReadonlyMap<string, string>): boolean {\n let sawRemaining = false\n for (const [name, value] of headers) {\n if (name.startsWith(\"x-ratelimit-remaining\")) {\n sawRemaining = true\n if (value !== \"0\") {\n return false\n }\n }\n }\n return sawRemaining\n}\n\nfunction retryAdvice(category: FailureCategory, retryAfterMs: number | undefined): RetryAdvice {\n const policy = CATEGORY_POLICY[category]\n const bounded = boundedRetryAfterMs(retryAfterMs)\n if (bounded === undefined) {\n return { retryable: policy.retryable }\n }\n return { retryable: policy.retryable, retryAfterMs: bounded }\n}\n\nfunction classification(\n category: FailureCategory,\n layer: FailureClassification[\"layer\"],\n definitive: boolean,\n retryAfterMs: number | undefined,\n quotaScope?: \"provider\",\n): FailureClassification {\n const policy = CATEGORY_POLICY[category]\n return {\n category,\n layer,\n definitive,\n retry: retryAdvice(category, retryAfterMs),\n failoverEligible: policy.failoverEligible,\n ...(quotaScope !== undefined ? { quotaScope } : {}),\n }\n}\n\n/**\n * Classifies an HTTP status. 429 consults the standard rate-limit headers in\n * the same step: status plus documented standard headers are one HTTP\n * semantics layer, and the quota refinement is more specific evidence than a\n * bare 429, so it cannot weaken the ADP-008 precedence.\n */\nfunction classifyStatus(\n status: number,\n headers: ReadonlyMap<string, string>,\n): FailureCategory | undefined {\n if (status === 429) {\n return fixedWindowExhausted(headers) ? \"quota_exhausted\" : \"rate_limited\"\n }\n if (status === 401 || status === 403) {\n return \"auth_failure\"\n }\n if (status === 408) {\n return \"timeout\"\n }\n if (status === 413) {\n return \"context_overflow\"\n }\n if (status === 502 || status === 503 || status === 504) {\n return \"outage\"\n }\n return undefined\n}\n\nfunction classifyText(text: string): FailureCategory | undefined {\n const bounded = text.length > TEXT_BOUND ? text.slice(0, TEXT_BOUND) : text\n for (const { category, pattern } of TEXTUAL_PATTERNS) {\n if (pattern.test(bounded)) {\n return category\n }\n }\n return undefined\n}\n\n/**\n * Total classification of layered failure evidence. Never throws and always\n * returns a category: any evidence shape outside the documented vocabularies\n * classifies as `unknown` via the `none` layer, deferring to OpenCode's\n * native failure handling (ADP-009, ADP-012).\n */\nexport function classifyFailure(evidence: ErrorEvidence): FailureClassification {\n // Layer 0: interruption evidence. Interruptions are host decisions, not\n // provider failures; a user cancellation is final (ADP-010).\n if (evidence.interrupted === \"user\") {\n return classification(\"user_cancelled\", \"interruption\", true, undefined)\n }\n if (evidence.interrupted !== undefined) {\n return classification(\"unknown\", \"interruption\", true, undefined)\n }\n\n // Layer 1: structured OpenCode metadata.\n const structured = evidence.structured\n const structuredRetryAfterMs = isNonNegativeInteger(structured?.retryAfterMs)\n ? structured.retryAfterMs\n : undefined\n if (structured !== undefined) {\n const mapped = STRUCTURED_TYPES.get(structured.type)\n if (mapped !== undefined) {\n return classification(mapped, \"structured\", true, structuredRetryAfterMs)\n }\n // Unmapped tag: fall through so HTTP semantics can refine the category.\n }\n\n // Layer 2: HTTP status (+ standard headers for the 429 refinement).\n const headers = lowercasedHeaders(evidence.headers)\n if (isHttpStatus(evidence.status)) {\n const category = classifyStatus(evidence.status, headers)\n if (category !== undefined) {\n return classification(\n category,\n \"status\",\n true,\n structuredRetryAfterMs ?? retryAfterHeaderMs(headers),\n )\n }\n }\n\n // Layer 3: standard headers alone (Retry-After without a decisive status).\n const headerRetryAfterMs = retryAfterHeaderMs(headers)\n if (headerRetryAfterMs !== undefined) {\n return classification(\"rate_limited\", \"headers\", false, headerRetryAfterMs)\n }\n\n // Layer 4: documented provider-specific metadata, normalized by our own\n // adapters after validation (T-15).\n if (evidence.providerMetadata?.quotaExhausted === true) {\n return classification(\"quota_exhausted\", \"provider-metadata\", true, undefined)\n }\n if (evidence.providerMetadata?.overloaded === true) {\n return classification(\"overload\", \"provider-metadata\", true, structuredRetryAfterMs)\n }\n\n // Layer 5: bounded textual classification, always non-definitive.\n const text = evidence.text\n if (text !== undefined) {\n const category = classifyText(text)\n if (category !== undefined) {\n const quotaScope =\n category === \"quota_exhausted\" && SHARED_QUOTA_PATTERN.test(text.slice(0, TEXT_BOUND))\n ? \"provider\"\n : undefined\n return classification(category, \"textual\", false, structuredRetryAfterMs, quotaScope)\n }\n }\n\n return classification(\"unknown\", \"none\", false, undefined)\n}\n"]}
|
|
@@ -59,6 +59,8 @@ export interface RetryAdvice {
|
|
|
59
59
|
export interface FailureClassification {
|
|
60
60
|
readonly category: FailureCategory;
|
|
61
61
|
readonly layer: EvidenceLayer;
|
|
62
|
+
/** Explicit shared subscription/balance evidence; absent means keep the model scope. */
|
|
63
|
+
readonly quotaScope?: "provider";
|
|
62
64
|
/**
|
|
63
65
|
* True when the evidence conclusively identifies the failure. Textual
|
|
64
66
|
* classifications are always non-definitive; scheduling treats
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/health/classifier/types.ts"],"names":[],"mappings":"AAUA,MAAM,CAAC,MAAM,kBAAkB,GAAG;IAChC,cAAc;IACd,iBAAiB;IACjB,QAAQ;IACR,UAAU;IACV,SAAS;IACT,cAAc;IACd,eAAe;IACf,kBAAkB;IAClB,gBAAgB;IAChB,gBAAgB;IAChB,SAAS;CAC2B,CAAA;AAItC;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,cAAc;IACd,YAAY;IACZ,QAAQ;IACR,SAAS;IACT,mBAAmB;IACnB,SAAS;IACT,MAAM;CAC8B,CAAA","sourcesContent":["/**\n * Failure classification types (ADP-008, ADP-009).\n *\n * The classifier consumes a layered evidence record assembled by the OpenCode\n * adapter. Evidence layers exist so classification can honor the ADP-008\n * precedence chain: structured metadata beats HTTP status beats standard\n * headers beats documented provider-specific metadata beats bounded text.\n */\nimport type { InterruptReason } from \"../../core/work-unit/interruption.js\"\n\nexport const FAILURE_CATEGORIES = [\n \"rate_limited\",\n \"quota_exhausted\",\n \"outage\",\n \"overload\",\n \"timeout\",\n \"auth_failure\",\n \"invalid_model\",\n \"context_overflow\",\n \"content_policy\",\n \"user_cancelled\",\n \"unknown\",\n] as const satisfies readonly string[]\n\nexport type FailureCategory = (typeof FAILURE_CATEGORIES)[number]\n\n/**\n * Where the winning classification came from. Exposed so receipts can show\n * why a decision was made and so tests can pin the precedence chain.\n */\nexport const EVIDENCE_LAYERS = [\n \"interruption\",\n \"structured\",\n \"status\",\n \"headers\",\n \"provider-metadata\",\n \"textual\",\n \"none\",\n] as const satisfies readonly string[]\n\nexport type EvidenceLayer = (typeof EVIDENCE_LAYERS)[number]\n\n/**\n * Structured OpenCode evidence (`Session.StructuredError`): a server-assigned\n * error tag plus, when the provider sent it, a retry delay OpenCode already\n * parsed from `Retry-After`. `type` is a free-form string on the wire, so the\n * classifier only interprets types from the documented @opencode/protocol\n * vocabulary and falls through to lower layers for anything unmapped instead\n * of guessing.\n */\nexport interface StructuredSignal {\n readonly type: string\n readonly retryAfterMs?: number | undefined\n}\n\n/**\n * Documented provider-specific metadata, already normalized by our own\n * adapters (e.g. the Codex capacity sensor validates `x-codex-*` headers\n * before producing these). The classifier never reads raw provider headers\n * itself.\n */\nexport interface ProviderMetadata {\n readonly quotaExhausted?: boolean | undefined\n readonly overloaded?: boolean | undefined\n}\n\n/**\n * Layered failure evidence. `status` is the merged HTTP status (the adapter\n * prefers the wire status seen by `http.response` over any status echoed in\n * structured metadata). `text` is the bounded concatenation of error message\n * and response body, bounded again defensively inside the classifier.\n */\nexport interface ErrorEvidence {\n readonly interrupted?: InterruptReason | undefined\n readonly structured?: StructuredSignal | undefined\n readonly status?: number | undefined\n readonly headers?: Readonly<Record<string, string>> | undefined\n readonly providerMetadata?: ProviderMetadata | undefined\n readonly text?: string | undefined\n}\n\n/** Advisory retry semantics; the retry policy (T-12) makes final decisions. */\nexport interface RetryAdvice {\n readonly retryable: boolean\n readonly retryAfterMs?: number\n}\n\nexport interface FailureClassification {\n readonly category: FailureCategory\n readonly layer: EvidenceLayer\n /**\n * True when the evidence conclusively identifies the failure. Textual\n * classifications are always non-definitive; scheduling treats\n * non-definitive evidence more conservatively (ADP-012).\n */\n readonly definitive: boolean\n readonly retry: RetryAdvice\n /** Horizontal recovery (switch resource within the tier) is allowed. */\n readonly failoverEligible: boolean\n}\n"]}
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/health/classifier/types.ts"],"names":[],"mappings":"AAUA,MAAM,CAAC,MAAM,kBAAkB,GAAG;IAChC,cAAc;IACd,iBAAiB;IACjB,QAAQ;IACR,UAAU;IACV,SAAS;IACT,cAAc;IACd,eAAe;IACf,kBAAkB;IAClB,gBAAgB;IAChB,gBAAgB;IAChB,SAAS;CAC2B,CAAA;AAItC;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,cAAc;IACd,YAAY;IACZ,QAAQ;IACR,SAAS;IACT,mBAAmB;IACnB,SAAS;IACT,MAAM;CAC8B,CAAA","sourcesContent":["/**\n * Failure classification types (ADP-008, ADP-009).\n *\n * The classifier consumes a layered evidence record assembled by the OpenCode\n * adapter. Evidence layers exist so classification can honor the ADP-008\n * precedence chain: structured metadata beats HTTP status beats standard\n * headers beats documented provider-specific metadata beats bounded text.\n */\nimport type { InterruptReason } from \"../../core/work-unit/interruption.js\"\n\nexport const FAILURE_CATEGORIES = [\n \"rate_limited\",\n \"quota_exhausted\",\n \"outage\",\n \"overload\",\n \"timeout\",\n \"auth_failure\",\n \"invalid_model\",\n \"context_overflow\",\n \"content_policy\",\n \"user_cancelled\",\n \"unknown\",\n] as const satisfies readonly string[]\n\nexport type FailureCategory = (typeof FAILURE_CATEGORIES)[number]\n\n/**\n * Where the winning classification came from. Exposed so receipts can show\n * why a decision was made and so tests can pin the precedence chain.\n */\nexport const EVIDENCE_LAYERS = [\n \"interruption\",\n \"structured\",\n \"status\",\n \"headers\",\n \"provider-metadata\",\n \"textual\",\n \"none\",\n] as const satisfies readonly string[]\n\nexport type EvidenceLayer = (typeof EVIDENCE_LAYERS)[number]\n\n/**\n * Structured OpenCode evidence (`Session.StructuredError`): a server-assigned\n * error tag plus, when the provider sent it, a retry delay OpenCode already\n * parsed from `Retry-After`. `type` is a free-form string on the wire, so the\n * classifier only interprets types from the documented @opencode/protocol\n * vocabulary and falls through to lower layers for anything unmapped instead\n * of guessing.\n */\nexport interface StructuredSignal {\n readonly type: string\n readonly retryAfterMs?: number | undefined\n}\n\n/**\n * Documented provider-specific metadata, already normalized by our own\n * adapters (e.g. the Codex capacity sensor validates `x-codex-*` headers\n * before producing these). The classifier never reads raw provider headers\n * itself.\n */\nexport interface ProviderMetadata {\n readonly quotaExhausted?: boolean | undefined\n readonly overloaded?: boolean | undefined\n}\n\n/**\n * Layered failure evidence. `status` is the merged HTTP status (the adapter\n * prefers the wire status seen by `http.response` over any status echoed in\n * structured metadata). `text` is the bounded concatenation of error message\n * and response body, bounded again defensively inside the classifier.\n */\nexport interface ErrorEvidence {\n readonly interrupted?: InterruptReason | undefined\n readonly structured?: StructuredSignal | undefined\n readonly status?: number | undefined\n readonly headers?: Readonly<Record<string, string>> | undefined\n readonly providerMetadata?: ProviderMetadata | undefined\n readonly text?: string | undefined\n}\n\n/** Advisory retry semantics; the retry policy (T-12) makes final decisions. */\nexport interface RetryAdvice {\n readonly retryable: boolean\n readonly retryAfterMs?: number\n}\n\nexport interface FailureClassification {\n readonly category: FailureCategory\n readonly layer: EvidenceLayer\n /** Explicit shared subscription/balance evidence; absent means keep the model scope. */\n readonly quotaScope?: \"provider\"\n /**\n * True when the evidence conclusively identifies the failure. Textual\n * classifications are always non-definitive; scheduling treats\n * non-definitive evidence more conservatively (ADP-012).\n */\n readonly definitive: boolean\n readonly retry: RetryAdvice\n /** Horizontal recovery (switch resource within the tier) is allowed. */\n readonly failoverEligible: boolean\n}\n"]}
|
|
@@ -152,6 +152,8 @@ export class Supervisor {
|
|
|
152
152
|
* the stale decision instead of acting after it.
|
|
153
153
|
*/
|
|
154
154
|
#recoveryPending = new Set();
|
|
155
|
+
/** Snapshot-sized budget shared by a task's terminal quota continuations. */
|
|
156
|
+
#quotaRecoveriesRemaining = new Map();
|
|
155
157
|
/** Interruptions observed while a recovery was pending on the session. */
|
|
156
158
|
#recoveryInterrupted = new Set();
|
|
157
159
|
/**
|
|
@@ -346,6 +348,24 @@ export class Supervisor {
|
|
|
346
348
|
#manuallySelected(sessionID, settings) {
|
|
347
349
|
return settings.pin === null && this.#manualSelections.has(sessionID);
|
|
348
350
|
}
|
|
351
|
+
/**
|
|
352
|
+
* The manual-pick suppression actually applied at an admission. It yields
|
|
353
|
+
* when the picked model is health-blocked: keeping the session on a
|
|
354
|
+
* quarantined resource does not honor the pick — the configured fallback
|
|
355
|
+
* takes over instead (the pick is re-honored only if the user picks again).
|
|
356
|
+
* Uses the non-mutating evaluation, never a probe admission.
|
|
357
|
+
*/
|
|
358
|
+
#manualSuppressionHolds(sessionID, settings) {
|
|
359
|
+
if (!this.#manuallySelected(sessionID, settings))
|
|
360
|
+
return false;
|
|
361
|
+
const current = this.#tracker.lastModel(sessionID)?.model;
|
|
362
|
+
if (current === undefined)
|
|
363
|
+
return true;
|
|
364
|
+
if (this.#health.evaluate(refKey(current)).kind !== "blocked")
|
|
365
|
+
return true;
|
|
366
|
+
this.#manualSelections.delete(sessionID);
|
|
367
|
+
return false;
|
|
368
|
+
}
|
|
349
369
|
/**
|
|
350
370
|
* Handle one `retry` hook invocation. On a veto this mutates
|
|
351
371
|
* `input.decision` to `{ retry: false }`; otherwise the decision is left
|
|
@@ -584,11 +604,17 @@ export class Supervisor {
|
|
|
584
604
|
this.#promptAdmission.set(sessionID, tier !== undefined ? { tier } : {});
|
|
585
605
|
return;
|
|
586
606
|
}
|
|
607
|
+
// Prompt hooks also admit steering messages into an active episode. They
|
|
608
|
+
// are not routing boundaries: switching here interrupts productive work
|
|
609
|
+
// and resets its recovery budget before the host consumes the steering.
|
|
610
|
+
if (this.#tracker.active(sessionID) !== undefined)
|
|
611
|
+
return;
|
|
587
612
|
// A fresh user prompt cancels any stale continuation intent and marker:
|
|
588
613
|
// this admission re-routes from scratch.
|
|
589
614
|
this.#continuationTiers.delete(sessionID);
|
|
590
615
|
this.#promptAdmission.delete(sessionID);
|
|
591
616
|
this.#escalations.delete(sessionID);
|
|
617
|
+
this.#quotaRecoveriesRemaining.delete(sessionID);
|
|
592
618
|
// A pending probe ticket from an earlier prompt admission (or a failed
|
|
593
619
|
// recovery) can only be carried by the unit the ORIGINAL prompt opens;
|
|
594
620
|
// a fresh admission means that episode never ran, so the ticket is
|
|
@@ -598,10 +624,11 @@ export class Supervisor {
|
|
|
598
624
|
this.#sessionProbeTickets.delete(sessionID);
|
|
599
625
|
this.#health.releaseProbe(stale);
|
|
600
626
|
}
|
|
601
|
-
if (this.#
|
|
602
|
-
// ADP-028: the user's own model pick stands
|
|
603
|
-
//
|
|
604
|
-
//
|
|
627
|
+
if (this.#manualSuppressionHolds(sessionID, settings)) {
|
|
628
|
+
// ADP-028: the user's own model pick stands (yielded to health if the
|
|
629
|
+
// picked resource is quarantined). Recording the defer (no switch, no
|
|
630
|
+
// marker) leaves OpenCode's choice — the picked model — in place;
|
|
631
|
+
// execution-started will skip routing the same way.
|
|
605
632
|
await this.#recordReceipt({
|
|
606
633
|
sessionID,
|
|
607
634
|
decision: { kind: "defer", reason: "routing-defer", detail: "manual-selection" },
|
|
@@ -778,8 +805,9 @@ export class Supervisor {
|
|
|
778
805
|
return { routing, decision, ...(probeTicket !== undefined ? { probeTicket } : {}) };
|
|
779
806
|
}
|
|
780
807
|
async #routeAdmission(unit, settings, mode) {
|
|
781
|
-
if (this.#
|
|
782
|
-
// ADP-028: the session runs on the user's own pick
|
|
808
|
+
if (this.#manualSuppressionHolds(unit.sessionID, settings)) {
|
|
809
|
+
// ADP-028: the session runs on the user's own pick (yielded to health
|
|
810
|
+
// if the picked resource is quarantined) — no re-route.
|
|
783
811
|
await this.#recordReceipt({
|
|
784
812
|
sessionID: unit.sessionID,
|
|
785
813
|
unitID: unit.id,
|
|
@@ -901,8 +929,71 @@ export class Supervisor {
|
|
|
901
929
|
this.#reportProbeOutcome(probeTicket, { success: false, category: classification.category });
|
|
902
930
|
}
|
|
903
931
|
}
|
|
932
|
+
if (unit !== undefined &&
|
|
933
|
+
model !== undefined &&
|
|
934
|
+
classification.category === "quota_exhausted") {
|
|
935
|
+
await this.#recoverTerminalQuota(unit, admissionTier, settings);
|
|
936
|
+
return;
|
|
937
|
+
}
|
|
904
938
|
await this.#escalateOnFailure(report, unitID, model, classification, admissionTier, escalationUsed, toolEvidence, settings);
|
|
905
939
|
}
|
|
940
|
+
async #recoverTerminalQuota(unit, requiredTier, settings) {
|
|
941
|
+
const sessionID = unit.sessionID;
|
|
942
|
+
// A retry veto already owns the continuation; its terminal echo must not
|
|
943
|
+
// start another. OBSERVE computes the same admission without applying it.
|
|
944
|
+
const mode = this.#modeFor(sessionID, settings);
|
|
945
|
+
if (mode === "off" ||
|
|
946
|
+
this.#recoveryPending.has(sessionID) ||
|
|
947
|
+
this.#continuationTiers.has(sessionID) ||
|
|
948
|
+
this.#continuationPending.has(sessionID))
|
|
949
|
+
return;
|
|
950
|
+
if (mode === "active")
|
|
951
|
+
this.#manualSuppressionHolds(sessionID, settings);
|
|
952
|
+
const { routing, decision, probeTicket } = await this.#admit(settings, sessionID, requiredTier);
|
|
953
|
+
const tier = routingTier(routing);
|
|
954
|
+
if (mode === "observe") {
|
|
955
|
+
if (probeTicket !== undefined)
|
|
956
|
+
this.#health.releaseProbe(probeTicket);
|
|
957
|
+
await this.#recordReceipt({ sessionID, tier, decision: schedulerForMode(decision, mode) });
|
|
958
|
+
return;
|
|
959
|
+
}
|
|
960
|
+
// Quota windows can expire while session.wait is pending. Snapshot the
|
|
961
|
+
// configured ladder length once so even those recoveries cannot loop.
|
|
962
|
+
const remaining = this.#quotaRecoveriesRemaining.get(sessionID) ??
|
|
963
|
+
(tier === undefined ? 0 : settings.tiers[tier].length);
|
|
964
|
+
if (decision.kind === "apply" && remaining === 0) {
|
|
965
|
+
if (probeTicket !== undefined)
|
|
966
|
+
this.#health.releaseProbe(probeTicket);
|
|
967
|
+
return;
|
|
968
|
+
}
|
|
969
|
+
await this.#recordReceipt({ sessionID, tier, decision });
|
|
970
|
+
if (decision.kind !== "apply") {
|
|
971
|
+
if (decision.kind === "park" && tier !== undefined) {
|
|
972
|
+
await this.#parked.park({
|
|
973
|
+
id: unit.id,
|
|
974
|
+
sessionID,
|
|
975
|
+
tier,
|
|
976
|
+
reason: "all-blocked",
|
|
977
|
+
detail: "quota exhausted",
|
|
978
|
+
parkedAt: this.#options.clock(),
|
|
979
|
+
});
|
|
980
|
+
}
|
|
981
|
+
return;
|
|
982
|
+
}
|
|
983
|
+
this.#quotaRecoveriesRemaining.set(sessionID, remaining - 1);
|
|
984
|
+
if (probeTicket !== undefined)
|
|
985
|
+
this.#sessionProbeTickets.set(sessionID, probeTicket);
|
|
986
|
+
// The host rejected this attempt without invoking its retry hook. Continue
|
|
987
|
+
// from durable session state rather than waiting for another user message.
|
|
988
|
+
void this.#ownRecovery(sessionID, decision.model, tier, () => this.#modeFor(sessionID, this.#settings.get()) === "active").catch((error) => {
|
|
989
|
+
if (probeTicket !== undefined && this.#sessionProbeTickets.get(sessionID) === probeTicket) {
|
|
990
|
+
this.#sessionProbeTickets.delete(sessionID);
|
|
991
|
+
this.#health.releaseProbe(probeTicket);
|
|
992
|
+
}
|
|
993
|
+
if (!(error instanceof RecoveryAbortedError))
|
|
994
|
+
logFailOpen("quota-recovery", error);
|
|
995
|
+
});
|
|
996
|
+
}
|
|
906
997
|
/**
|
|
907
998
|
* Escalation targets must be usable: the scheduler contract hands
|
|
908
999
|
* planEscalation "already gated" resources, so the wiring filters the next
|
|
@@ -1395,6 +1486,7 @@ export class Supervisor {
|
|
|
1395
1486
|
this.#promptAdmission.delete(sessionID);
|
|
1396
1487
|
this.#continuationPending.delete(sessionID);
|
|
1397
1488
|
this.#escalations.delete(sessionID);
|
|
1489
|
+
this.#quotaRecoveriesRemaining.delete(sessionID);
|
|
1398
1490
|
if (this.#sessionModes.get(sessionID) !== null) {
|
|
1399
1491
|
try {
|
|
1400
1492
|
await this.#sessionModes.set(sessionID, null);
|
|
@@ -1614,11 +1706,22 @@ function refKey(ref) {
|
|
|
1614
1706
|
...(variant !== undefined ? { variantID: variant } : {}),
|
|
1615
1707
|
};
|
|
1616
1708
|
}
|
|
1709
|
+
/** Quota survives variantless host resolution; only shared-limit evidence blocks siblings. */
|
|
1710
|
+
function failureKey(key, classification) {
|
|
1711
|
+
if (classification.quotaScope === "provider") {
|
|
1712
|
+
// Host model refs expose the configured provider, not a credential/account id.
|
|
1713
|
+
return { providerID: key.providerID };
|
|
1714
|
+
}
|
|
1715
|
+
if (classification.category === "quota_exhausted" && key.variantID !== undefined) {
|
|
1716
|
+
return { providerID: key.providerID, modelID: key.modelID };
|
|
1717
|
+
}
|
|
1718
|
+
return key;
|
|
1719
|
+
}
|
|
1617
1720
|
/** Health failure event from a classification (bounded structured evidence). */
|
|
1618
1721
|
function failureEvent(key, classification) {
|
|
1619
1722
|
return {
|
|
1620
1723
|
kind: "failure",
|
|
1621
|
-
key,
|
|
1724
|
+
key: failureKey(key, classification),
|
|
1622
1725
|
category: classification.category,
|
|
1623
1726
|
definitive: classification.definitive,
|
|
1624
1727
|
...(classification.retry.retryAfterMs !== undefined
|