@opengeni/runtime 4.0.3 → 4.1.0-canary.36239117573001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/codemode-client.json +1 -1
- package/dist/{chunk-GBVN7WSK.js → chunk-ORTF5RRC.js} +371 -28
- package/dist/chunk-ORTF5RRC.js.map +1 -0
- package/dist/{chunk-LL2YQO5H.js → chunk-YAA7WDNO.js} +638 -71
- package/dist/chunk-YAA7WDNO.js.map +1 -0
- package/dist/codex-tool-search.d.ts +12 -0
- package/dist/context-compaction.d.ts +4 -2
- package/dist/index.d.ts +27 -8
- package/dist/index.js +18 -2
- package/dist/knowledge-model-projection.d.ts +28 -0
- package/dist/metrics.d.ts +17 -0
- package/dist/model-provider-routing.d.ts +14 -0
- package/dist/operational-instructions.d.ts +1 -1
- package/dist/provider-quota.d.ts +76 -0
- package/dist/sandbox/channel-a.d.ts +43 -0
- package/dist/sandbox/index.d.ts +1 -1
- package/dist/sandbox/index.js +5 -1
- package/dist/tool-result-spill.d.ts +11 -3
- package/dist/workspace-tool-gateway.js +2 -2
- package/package.json +11 -11
- package/src/codex-tool-search.ts +54 -6
- package/src/context-compaction.ts +19 -3
- package/src/index.ts +277 -56
- package/src/knowledge-model-projection.ts +169 -0
- package/src/lazy-tool-transport.ts +4 -0
- package/src/metrics.ts +26 -0
- package/src/model-context-inspector.ts +1 -0
- package/src/model-provider-client.ts +37 -16
- package/src/model-provider-routing.ts +32 -6
- package/src/operational-instructions.ts +1 -1
- package/src/provider-quota.ts +412 -0
- package/src/sandbox/browser-control-client.ts +70 -25
- package/src/sandbox/channel-a.ts +370 -0
- package/src/sandbox/index.ts +3 -0
- package/src/sandbox/routing/routing-session.ts +25 -4
- package/src/tool-result-spill.ts +17 -5
- package/dist/chunk-GBVN7WSK.js.map +0 -1
- package/dist/chunk-LL2YQO5H.js.map +0 -1
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
import { APIError } from "openai";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Provider quota exhaustion versus ordinary rate limiting.
|
|
5
|
+
*
|
|
6
|
+
* Model providers use HTTP 429 for two different conditions. An ordinary rate
|
|
7
|
+
* limit (requests or tokens per minute) clears within seconds, so pacing the
|
|
8
|
+
* same request again is correct. An exhausted quota (a daily or monthly
|
|
9
|
+
* allowance, a free-tier day cap, or an account with no credits left) cannot
|
|
10
|
+
* clear inside a bounded retry budget, so every retry only delays the failure.
|
|
11
|
+
*
|
|
12
|
+
* This module is the single classifier for API-key provider responses. It
|
|
13
|
+
* reads only provider-owned evidence: the HTTP status, provider message/code
|
|
14
|
+
* strings, and the provider's own retry hint. The OpenAI SDK retry veto and the
|
|
15
|
+
* worker's turn classification both read that evidence from the same thrown
|
|
16
|
+
* SDK error shape (`classifyProviderQuotaError`), so one response can never be
|
|
17
|
+
* classified two ways. Subscription transports (Codex, SuperGrok) own their
|
|
18
|
+
* quota semantics through credential rotation and durable capacity waits and
|
|
19
|
+
* must not route through it.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
export const PROVIDER_QUOTA_EXHAUSTED_CODE = "provider_quota_exhausted";
|
|
23
|
+
|
|
24
|
+
/** What ran out. Chooses plain-language copy only; every scope is non-retryable. */
|
|
25
|
+
export type ProviderQuotaScope = "daily" | "monthly" | "credits" | "quota";
|
|
26
|
+
|
|
27
|
+
export type ProviderQuotaExhaustion = { scope: ProviderQuotaScope };
|
|
28
|
+
|
|
29
|
+
export type ProviderQuotaEvidence = {
|
|
30
|
+
/** HTTP status when one survived the error wrappers; null when statusless. */
|
|
31
|
+
status: number | null;
|
|
32
|
+
/** Provider-authored message, code and type strings, in any order. */
|
|
33
|
+
texts: readonly string[];
|
|
34
|
+
/** Provider retry hint from a header or structured field, in milliseconds. */
|
|
35
|
+
retryAfterMs: number | null;
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
/** A provider-stated wait at or below this is an ordinary short rate limit. */
|
|
39
|
+
export const PROVIDER_SHORT_RETRY_HINT_MS = 60_000;
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Without any quota wording, only a provider-stated wait beyond this proves a
|
|
43
|
+
* quota window rather than per-minute pacing.
|
|
44
|
+
*/
|
|
45
|
+
export const PROVIDER_QUOTA_RETRY_HINT_MS = 15 * 60_000;
|
|
46
|
+
|
|
47
|
+
// Bound the text inspected per field and in total. Provider errors are short;
|
|
48
|
+
// a pathological body must not turn failure classification into a hot loop.
|
|
49
|
+
const MAX_FIELD_CHARS = 4_096;
|
|
50
|
+
const MAX_FIELDS = 64;
|
|
51
|
+
|
|
52
|
+
// Account-level exhaustion that nothing clears in seconds. Structured codes are
|
|
53
|
+
// matched exactly; wording is matched as a phrase. OpenAI's
|
|
54
|
+
// `billing_hard_limit_reached` is the organization's own spending cap.
|
|
55
|
+
const CREDIT_CODES = new Set([
|
|
56
|
+
"insufficient_credits",
|
|
57
|
+
"insufficient_balance",
|
|
58
|
+
"billing_hard_limit_reached",
|
|
59
|
+
]);
|
|
60
|
+
const QUOTA_CODES = new Set(["insufficient_quota"]);
|
|
61
|
+
const CREDIT_TEXT =
|
|
62
|
+
/\binsufficient[ _](?:credits?|balance|funds)\b|\bcredit balance is too low\b|\brequires more credits\b|\bout of credits\b|\bused all (?:of )?(?:your |the )?(?:available )?credits\b|\bspending limit\b|\bbilling[ _]hard[ _]limit\b|\breached your specified api usage limits?\b/;
|
|
63
|
+
|
|
64
|
+
// Generic quota wording, including a reached "usage limit". Google reports
|
|
65
|
+
// per-minute limits with the same "exceeded your current quota" sentence, so
|
|
66
|
+
// this alone is not decisive: an explicit per-minute scope or a short provider
|
|
67
|
+
// retry hint keeps it retryable.
|
|
68
|
+
// Google's `RESOURCE_EXHAUSTED` status is deliberately absent: it accompanies
|
|
69
|
+
// every Google 429, including short dynamic-shared-quota capacity refusals, so
|
|
70
|
+
// it only marks the response as rate-shaped.
|
|
71
|
+
const QUOTA_TEXT =
|
|
72
|
+
/\bexceeded (?:your |the )?(?:current |allotted |daily |monthly )?quota\b|\bquota (?:has been |was |is )?(?:exceeded|exhausted|reached)\b|\bout of (?:call volume )?quota\b|\bquota[ _]?exceeded\b|\b(?:hit|reached|exceeded) (?:your |the |its )?(?:[a-z]+ )?usage limits?\b/;
|
|
73
|
+
|
|
74
|
+
// Explicit limit windows. "free-models-per-min" is OpenRouter's per-minute
|
|
75
|
+
// free tier; "free-models-per-day" its daily cap. GitHub/Azure AI inference
|
|
76
|
+
// name windows like "UserByModelByDay" or "per 86400s", and Google/Vertex
|
|
77
|
+
// metric ids join words with underscores
|
|
78
|
+
// ("generate_content_requests_per_minute_per_project"). Letter lookarounds
|
|
79
|
+
// rather than `\b` keep snake_case ids matching, since `_` is a word character.
|
|
80
|
+
const MINUTE_SCOPE =
|
|
81
|
+
/(?<![a-z])per[ _-]?min(?:ute)?s?(?![a-z])|perminute|byminute|(?<![a-z])(?:rpm|tpm)(?![a-z])|\/min(?![a-z])|\bper 60 ?s\b|(?<![a-z])per[ _-]?second(?![a-z])/;
|
|
82
|
+
const DAY_SCOPE =
|
|
83
|
+
/(?<![a-z])per[ _-]?day(?![a-z])|perday|byday|(?<![a-z])(?:rpd|tpd)(?![a-z])|(?<![a-z])daily(?![a-z])|\bper 86400 ?s\b/;
|
|
84
|
+
const MONTH_SCOPE = /(?<![a-z])per[ _-]?month(?![a-z])|permonth|bymonth|(?<![a-z])monthly(?![a-z])/;
|
|
85
|
+
|
|
86
|
+
const RATE_TEXT = /too many requests|rate.?limit|\b429\b|\bresource[ _]exhausted\b/;
|
|
87
|
+
|
|
88
|
+
const UNIT_MS: Record<string, number> = {
|
|
89
|
+
ms: 1,
|
|
90
|
+
millisecond: 1,
|
|
91
|
+
milliseconds: 1,
|
|
92
|
+
s: 1_000,
|
|
93
|
+
sec: 1_000,
|
|
94
|
+
secs: 1_000,
|
|
95
|
+
second: 1_000,
|
|
96
|
+
seconds: 1_000,
|
|
97
|
+
m: 60_000,
|
|
98
|
+
min: 60_000,
|
|
99
|
+
mins: 60_000,
|
|
100
|
+
minute: 60_000,
|
|
101
|
+
minutes: 60_000,
|
|
102
|
+
h: 3_600_000,
|
|
103
|
+
hr: 3_600_000,
|
|
104
|
+
hrs: 3_600_000,
|
|
105
|
+
hour: 3_600_000,
|
|
106
|
+
hours: 3_600_000,
|
|
107
|
+
d: 86_400_000,
|
|
108
|
+
day: 86_400_000,
|
|
109
|
+
days: 86_400_000,
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
const DURATION_TOKEN =
|
|
113
|
+
/^\s*(\d{1,9}(?:\.\d{1,12})?)\s*(milliseconds?|ms|seconds?|secs?|s|minutes?|mins?|m|hours?|hrs?|h|days?|d)(?![a-z])/i;
|
|
114
|
+
|
|
115
|
+
/** Parse a leading compound duration such as "16m32.64s", "6 seconds" or "7 days". */
|
|
116
|
+
function leadingDurationMs(text: string): number | null {
|
|
117
|
+
let rest = text;
|
|
118
|
+
let total = 0;
|
|
119
|
+
let matched = false;
|
|
120
|
+
for (let index = 0; index < 4; index += 1) {
|
|
121
|
+
const token = DURATION_TOKEN.exec(rest);
|
|
122
|
+
if (!token) break;
|
|
123
|
+
const unit = UNIT_MS[token[2]!.toLowerCase()];
|
|
124
|
+
if (unit === undefined) break;
|
|
125
|
+
total += Number(token[1]) * unit;
|
|
126
|
+
matched = true;
|
|
127
|
+
rest = rest.slice(token[0].length);
|
|
128
|
+
}
|
|
129
|
+
return matched && Number.isFinite(total) ? Math.ceil(total) : null;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Read a retry hint a provider wrote into its message ("Please retry after 6
|
|
134
|
+
* seconds", "Please try again in 16m32.64s", "Please wait 42567 seconds before
|
|
135
|
+
* retrying"). Used only to tell a short limit from an exhausted quota; it never
|
|
136
|
+
* paces a retry.
|
|
137
|
+
*/
|
|
138
|
+
export function providerMessageRetryHintMs(text: string): number | null {
|
|
139
|
+
let longest: number | null = null;
|
|
140
|
+
const prefixes = /\b(?:retry|try again)\s+(?:after|in)\s+|\bwait\s+/gi;
|
|
141
|
+
for (const match of text.slice(0, MAX_FIELD_CHARS * 4).matchAll(prefixes)) {
|
|
142
|
+
const start = (match.index ?? 0) + match[0].length;
|
|
143
|
+
const duration = leadingDurationMs(text.slice(start, start + 64));
|
|
144
|
+
if (duration !== null) longest = longest === null ? duration : Math.max(longest, duration);
|
|
145
|
+
}
|
|
146
|
+
return longest;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Decide whether a provider refusal is an exhausted quota. Returns null for an
|
|
151
|
+
* ordinary short rate limit and for anything that is not provider quota
|
|
152
|
+
* evidence at all (a sandbox "disk quota exceeded", an OpenGeni credit
|
|
153
|
+
* refusal, a 5xx), so those keep their existing classification.
|
|
154
|
+
*/
|
|
155
|
+
export function classifyProviderQuotaExhaustion(
|
|
156
|
+
evidence: ProviderQuotaEvidence,
|
|
157
|
+
): ProviderQuotaExhaustion | null {
|
|
158
|
+
const fields = evidence.texts
|
|
159
|
+
.slice(0, MAX_FIELDS)
|
|
160
|
+
.filter((value): value is string => typeof value === "string" && value.length > 0)
|
|
161
|
+
.map((value) => value.slice(0, MAX_FIELD_CHARS).toLowerCase());
|
|
162
|
+
const text = fields.join("\n");
|
|
163
|
+
if (text.includes("opengeni credits")) return null;
|
|
164
|
+
|
|
165
|
+
const explicitStatus =
|
|
166
|
+
evidence.status !== null && Number.isInteger(evidence.status) ? evidence.status : null;
|
|
167
|
+
// SDK wrappers sometimes keep only the "429 ..." message prefix.
|
|
168
|
+
const prefixedStatus = fields
|
|
169
|
+
.map((value) => /^\s*(4\d\d)\b/.exec(value)?.[1])
|
|
170
|
+
.find((value) => value !== undefined);
|
|
171
|
+
const status = explicitStatus ?? (prefixedStatus ? Number(prefixedStatus) : null);
|
|
172
|
+
// Only a rate-limit or payment refusal can be quota exhaustion. Every other
|
|
173
|
+
// status keeps its existing classification (a 5xx is transient, other 4xx
|
|
174
|
+
// are already terminal request faults).
|
|
175
|
+
if (status !== null && status !== 429 && status !== 402) return null;
|
|
176
|
+
|
|
177
|
+
const hasCode = (codes: ReadonlySet<string>) => fields.some((value) => codes.has(value.trim()));
|
|
178
|
+
const creditCode = hasCode(CREDIT_CODES);
|
|
179
|
+
const quotaCode = hasCode(QUOTA_CODES);
|
|
180
|
+
const rateShaped = status === 429 || RATE_TEXT.test(text);
|
|
181
|
+
// A statusless error must still look like a provider refusal; free text such
|
|
182
|
+
// as a sandbox "Disk quota exceeded" is not model-provider evidence.
|
|
183
|
+
if (status === null && !rateShaped && !creditCode && !quotaCode) return null;
|
|
184
|
+
|
|
185
|
+
// Payment Required and account credit/billing exhaustion never clear by
|
|
186
|
+
// waiting, whatever retry hint accompanies them.
|
|
187
|
+
if (status === 402 || creditCode || CREDIT_TEXT.test(text)) return { scope: "credits" };
|
|
188
|
+
if (quotaCode) return { scope: "quota" };
|
|
189
|
+
|
|
190
|
+
const hint = evidence.retryAfterMs ?? providerMessageRetryHintMs(text);
|
|
191
|
+
const minute = MINUTE_SCOPE.test(text);
|
|
192
|
+
const day = DAY_SCOPE.test(text);
|
|
193
|
+
const month = MONTH_SCOPE.test(text);
|
|
194
|
+
const quotaWording = QUOTA_TEXT.test(text);
|
|
195
|
+
|
|
196
|
+
if (!day && !month && !quotaWording) {
|
|
197
|
+
// No quota wording: only a provider wait far beyond per-minute pacing
|
|
198
|
+
// proves an exhausted window.
|
|
199
|
+
return hint !== null && hint > PROVIDER_QUOTA_RETRY_HINT_MS ? { scope: "quota" } : null;
|
|
200
|
+
}
|
|
201
|
+
// An explicit per-minute window is ordinary pacing even when the provider's
|
|
202
|
+
// message also advertises its daily allowance.
|
|
203
|
+
if (minute) return null;
|
|
204
|
+
// The provider itself says the limit clears within a minute.
|
|
205
|
+
if (hint !== null && hint <= PROVIDER_SHORT_RETRY_HINT_MS) return null;
|
|
206
|
+
if (day) return { scope: "daily" };
|
|
207
|
+
if (month) return { scope: "monthly" };
|
|
208
|
+
return { scope: "quota" };
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/** Plain-language turn failure copy for an exhausted provider quota. */
|
|
212
|
+
export function providerQuotaExhaustedMessage(scope: ProviderQuotaScope): string {
|
|
213
|
+
switch (scope) {
|
|
214
|
+
case "daily":
|
|
215
|
+
return "This model's daily limit at the model provider has been reached, so automatic retries stopped. Choose another model, or try again after the limit resets.";
|
|
216
|
+
case "monthly":
|
|
217
|
+
return "This model's monthly limit at the model provider has been reached, so automatic retries stopped. Choose another model, or try again after the limit resets.";
|
|
218
|
+
case "credits":
|
|
219
|
+
return "The model provider account for this model is out of credits, so automatic retries stopped. Choose another model, or add credits with the provider and try again.";
|
|
220
|
+
case "quota":
|
|
221
|
+
return "The model provider's usage quota for this model is used up, so automatic retries stopped. Choose another model, or try again after the quota resets.";
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
const MAX_RESPONSE_BODY_BYTES = 64 * 1024;
|
|
226
|
+
|
|
227
|
+
/** Read at most `maxBytes` of a cloned body; the original stays unread. */
|
|
228
|
+
async function boundedCloneText(response: Response, maxBytes: number): Promise<string | null> {
|
|
229
|
+
const body = response.clone().body;
|
|
230
|
+
if (!body) return "";
|
|
231
|
+
const reader = body.getReader();
|
|
232
|
+
const decoder = new TextDecoder();
|
|
233
|
+
let text = "";
|
|
234
|
+
let bytes = 0;
|
|
235
|
+
try {
|
|
236
|
+
for (;;) {
|
|
237
|
+
const chunk = await reader.read();
|
|
238
|
+
if (chunk.done) break;
|
|
239
|
+
bytes += chunk.value.byteLength;
|
|
240
|
+
if (bytes > maxBytes) return null;
|
|
241
|
+
text += decoder.decode(chunk.value, { stream: true });
|
|
242
|
+
}
|
|
243
|
+
return text + decoder.decode();
|
|
244
|
+
} finally {
|
|
245
|
+
await reader.cancel().catch(() => undefined);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
function statusOf(value: Record<string, unknown>): number | null {
|
|
250
|
+
const body =
|
|
251
|
+
value.error && typeof value.error === "object"
|
|
252
|
+
? (value.error as Record<string, unknown>)
|
|
253
|
+
: null;
|
|
254
|
+
const status = Number(value.status ?? value.statusCode ?? body?.status ?? body?.statusCode);
|
|
255
|
+
return Number.isInteger(status) && status >= 100 && status <= 599 ? status : null;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** The first HTTP status along an error and its cause chain. */
|
|
259
|
+
function errorHttpStatus(error: unknown): number | null {
|
|
260
|
+
let current: unknown = error;
|
|
261
|
+
for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
|
|
262
|
+
const status = statusOf(current as Record<string, unknown>);
|
|
263
|
+
if (status !== null) return status;
|
|
264
|
+
current = (current as Record<string, unknown>).cause;
|
|
265
|
+
}
|
|
266
|
+
return null;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* Provider-authored strings on an SDK error and its wrappers: message, code,
|
|
271
|
+
* type, name and param, recursing through `error`, `cause`, `response` and
|
|
272
|
+
* `data`. These are exactly the fields the worker's other failure classifiers
|
|
273
|
+
* read, so quota evidence never depends on a key only one reader sees.
|
|
274
|
+
*/
|
|
275
|
+
function errorTexts(value: unknown, out: string[], seen: WeakSet<object>): void {
|
|
276
|
+
if (out.length >= MAX_FIELDS) return;
|
|
277
|
+
if (typeof value === "string") {
|
|
278
|
+
out.push(value);
|
|
279
|
+
return;
|
|
280
|
+
}
|
|
281
|
+
if (!value || typeof value !== "object" || seen.has(value)) return;
|
|
282
|
+
seen.add(value);
|
|
283
|
+
const record = value as Record<string, unknown>;
|
|
284
|
+
for (const key of ["message", "code", "type", "name", "param"]) {
|
|
285
|
+
const field = record[key];
|
|
286
|
+
if (typeof field === "string" && field.length > 0) out.push(field);
|
|
287
|
+
}
|
|
288
|
+
for (const key of ["error", "cause", "response", "data"]) errorTexts(record[key], out, seen);
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
function headerValue(headers: unknown, name: string): string | null {
|
|
292
|
+
if (!headers || typeof headers !== "object") return null;
|
|
293
|
+
const getter = (headers as { get?: unknown }).get;
|
|
294
|
+
if (typeof getter === "function") {
|
|
295
|
+
const value = getter.call(headers, name);
|
|
296
|
+
return typeof value === "string" ? value : null;
|
|
297
|
+
}
|
|
298
|
+
const entry = Object.entries(headers as Record<string, unknown>).find(
|
|
299
|
+
([key, value]) => key.toLowerCase() === name && typeof value === "string",
|
|
300
|
+
);
|
|
301
|
+
return typeof entry?.[1] === "string" ? entry[1] : null;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/** A provider retry hint (`retry-after-ms`, `retry-after`, or a structured field) in milliseconds. */
|
|
305
|
+
function errorRetryAfterMs(error: unknown, nowMs: number): number | null {
|
|
306
|
+
let current: unknown = error;
|
|
307
|
+
for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
|
|
308
|
+
const value = current as Record<string, unknown>;
|
|
309
|
+
const body =
|
|
310
|
+
value.error && typeof value.error === "object"
|
|
311
|
+
? (value.error as Record<string, unknown>)
|
|
312
|
+
: null;
|
|
313
|
+
for (const headers of [value.headers, value.responseHeaders, body?.headers]) {
|
|
314
|
+
const millis = Number(headerValue(headers, "retry-after-ms") ?? Number.NaN);
|
|
315
|
+
if (Number.isFinite(millis) && millis > 0) return Math.ceil(millis);
|
|
316
|
+
const header = headerValue(headers, "retry-after");
|
|
317
|
+
if (header === null) continue;
|
|
318
|
+
const seconds = Number(header);
|
|
319
|
+
if (Number.isFinite(seconds) && seconds > 0) return Math.ceil(seconds * 1_000);
|
|
320
|
+
const date = Date.parse(header);
|
|
321
|
+
if (Number.isFinite(date) && date > nowMs) return Math.ceil(date - nowMs);
|
|
322
|
+
}
|
|
323
|
+
const seconds = Number(
|
|
324
|
+
value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds,
|
|
325
|
+
);
|
|
326
|
+
if (Number.isFinite(seconds) && seconds > 0) return Math.ceil(seconds * 1_000);
|
|
327
|
+
current = value.cause;
|
|
328
|
+
}
|
|
329
|
+
return null;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/**
|
|
333
|
+
* Classify a thrown provider error: an OpenAI SDK `APIError`, or any wrapper
|
|
334
|
+
* whose `error`/`cause` chain reaches one. This is the one evidence reader for
|
|
335
|
+
* both the SDK retry veto and the worker's turn failure.
|
|
336
|
+
*/
|
|
337
|
+
export function classifyProviderQuotaError(
|
|
338
|
+
error: unknown,
|
|
339
|
+
nowMs = Date.now(),
|
|
340
|
+
): ProviderQuotaExhaustion | null {
|
|
341
|
+
try {
|
|
342
|
+
const texts: string[] = [];
|
|
343
|
+
errorTexts(error, texts, new WeakSet());
|
|
344
|
+
return classifyProviderQuotaExhaustion({
|
|
345
|
+
status: errorHttpStatus(error),
|
|
346
|
+
texts,
|
|
347
|
+
retryAfterMs: errorRetryAfterMs(error, nowMs),
|
|
348
|
+
});
|
|
349
|
+
} catch {
|
|
350
|
+
// A hostile getter is not quota evidence; callers keep their existing path.
|
|
351
|
+
return null;
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
/**
|
|
356
|
+
* Classify a raw provider HTTP response without consuming its body. It builds
|
|
357
|
+
* the exact error the OpenAI SDK throws for this response (the same
|
|
358
|
+
* `APIError.generate` inputs as the SDK's own failure path) and classifies
|
|
359
|
+
* that, so the veto and the worker read identical evidence.
|
|
360
|
+
*/
|
|
361
|
+
export async function classifyProviderQuotaResponse(
|
|
362
|
+
response: Response,
|
|
363
|
+
nowMs = Date.now(),
|
|
364
|
+
): Promise<ProviderQuotaExhaustion | null> {
|
|
365
|
+
if (response.status !== 429) return null;
|
|
366
|
+
const text = await boundedCloneText(response, MAX_RESPONSE_BODY_BYTES);
|
|
367
|
+
if (text === null) return null;
|
|
368
|
+
let json: unknown;
|
|
369
|
+
try {
|
|
370
|
+
json = JSON.parse(text);
|
|
371
|
+
} catch {
|
|
372
|
+
json = undefined;
|
|
373
|
+
}
|
|
374
|
+
const sdkError = APIError.generate(
|
|
375
|
+
response.status,
|
|
376
|
+
json as object | undefined,
|
|
377
|
+
json ? undefined : text,
|
|
378
|
+
response.headers,
|
|
379
|
+
);
|
|
380
|
+
return classifyProviderQuotaError(sdkError, nowMs);
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
/**
|
|
384
|
+
* Tell the OpenAI SDK not to replay an exhausted-quota 429. The SDK otherwise
|
|
385
|
+
* retries every 429 up to `maxRetries` times, which only delays a refusal that
|
|
386
|
+
* cannot clear. The SDK honors the standard-for-it `x-should-retry` header, so
|
|
387
|
+
* the exact response (status, headers, body) is passed through with only that
|
|
388
|
+
* header added. Ordinary rate limits and every other response are untouched.
|
|
389
|
+
*/
|
|
390
|
+
export function withoutQuotaExhaustedRetries(inner: typeof fetch): typeof fetch {
|
|
391
|
+
return (async (input: Parameters<typeof fetch>[0], init?: Parameters<typeof fetch>[1]) => {
|
|
392
|
+
const response = await inner(input, init);
|
|
393
|
+
if (response.status !== 429 || response.headers.has("x-should-retry")) return response;
|
|
394
|
+
let exhausted: ProviderQuotaExhaustion | null = null;
|
|
395
|
+
try {
|
|
396
|
+
exhausted = await classifyProviderQuotaResponse(response);
|
|
397
|
+
} catch {
|
|
398
|
+
// Classification is advisory; the SDK keeps its ordinary retry decision.
|
|
399
|
+
}
|
|
400
|
+
if (!exhausted) return response;
|
|
401
|
+
const headers = new Headers(response.headers);
|
|
402
|
+
headers.set("x-should-retry", "false");
|
|
403
|
+
const vetoed = new Response(response.body, {
|
|
404
|
+
status: response.status,
|
|
405
|
+
statusText: response.statusText,
|
|
406
|
+
headers,
|
|
407
|
+
});
|
|
408
|
+
// Keep the request URL the SDK reports in its request logs.
|
|
409
|
+
if (response.url) Object.defineProperty(vetoed, "url", { value: response.url });
|
|
410
|
+
return vetoed;
|
|
411
|
+
}) as typeof fetch;
|
|
412
|
+
}
|
|
@@ -1042,6 +1042,7 @@ export class BrowserControlClient {
|
|
|
1042
1042
|
if (
|
|
1043
1043
|
retryNativeEndpoint &&
|
|
1044
1044
|
!requireHostFetch &&
|
|
1045
|
+
input.method === "GET" &&
|
|
1045
1046
|
error instanceof BrowserControlTransportError
|
|
1046
1047
|
) {
|
|
1047
1048
|
return await this.requestJson(input, false);
|
|
@@ -1153,7 +1154,12 @@ export class BrowserControlClient {
|
|
|
1153
1154
|
throw error;
|
|
1154
1155
|
}
|
|
1155
1156
|
if (error instanceof BrowserControlTransportError) {
|
|
1156
|
-
if (
|
|
1157
|
+
if (
|
|
1158
|
+
retryNativeEndpoint &&
|
|
1159
|
+
input.method === "GET" &&
|
|
1160
|
+
this.nativeAuthority &&
|
|
1161
|
+
this.session.ensureBrowserControl
|
|
1162
|
+
) {
|
|
1157
1163
|
nativeControllerPorts.delete(nativeControllerKey(this.nativeAuthority));
|
|
1158
1164
|
await this.controllerPort();
|
|
1159
1165
|
return await this.requestJson(input, false);
|
|
@@ -1941,21 +1947,25 @@ async function requestExposedController(
|
|
|
1941
1947
|
if (body !== undefined && Buffer.byteLength(body) > BROWSER_CONTROL_MAX_JSON_BYTES) {
|
|
1942
1948
|
throw new RangeError("browser controller request body is too large");
|
|
1943
1949
|
}
|
|
1944
|
-
const response = await
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1950
|
+
const response = await controllerTransport(
|
|
1951
|
+
async () =>
|
|
1952
|
+
await fetch(streamUrl, {
|
|
1953
|
+
method: input.method,
|
|
1954
|
+
headers: {
|
|
1955
|
+
...exposedPortHeaders(endpoint),
|
|
1956
|
+
authorization: `Bearer ${token}`,
|
|
1957
|
+
...(body === undefined ? {} : { "content-type": "application/json" }),
|
|
1958
|
+
},
|
|
1959
|
+
...(body === undefined ? {} : { body }),
|
|
1960
|
+
redirect: "manual",
|
|
1961
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
1962
|
+
}),
|
|
1963
|
+
);
|
|
1964
|
+
const responseText = await controllerTransport(async () => await response.text());
|
|
1956
1965
|
if (Buffer.byteLength(responseText) > BROWSER_CONTROL_MAX_JSON_BYTES) {
|
|
1957
1966
|
throw new BrowserControlProtocolError("browser controller response is too large");
|
|
1958
1967
|
}
|
|
1968
|
+
rejectProviderGatewayResponse(responseText, response.status);
|
|
1959
1969
|
if (
|
|
1960
1970
|
(response.status === 401 || response.status === 403) &&
|
|
1961
1971
|
!looksLikeBrowserControlEnvelope(responseText)
|
|
@@ -1985,21 +1995,25 @@ async function requestExposedControllerBytes(
|
|
|
1985
1995
|
const timeoutMs = boundedTimeout(input.timeoutMs ?? defaultTimeoutMs);
|
|
1986
1996
|
const streamUrl = exposedControllerUrl(endpoint, input.path);
|
|
1987
1997
|
streamUrl.protocol = streamUrl.protocol === "wss:" ? "https:" : "http:";
|
|
1988
|
-
const response = await
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
+
const response = await controllerTransport(
|
|
1999
|
+
async () =>
|
|
2000
|
+
await fetch(streamUrl, {
|
|
2001
|
+
method: input.method,
|
|
2002
|
+
headers: {
|
|
2003
|
+
...exposedPortHeaders(endpoint),
|
|
2004
|
+
authorization: `Bearer ${token}`,
|
|
2005
|
+
accept: "image/jpeg, image/png",
|
|
2006
|
+
},
|
|
2007
|
+
redirect: "manual",
|
|
2008
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
2009
|
+
}),
|
|
2010
|
+
);
|
|
1998
2011
|
if (!response.ok) {
|
|
1999
|
-
const responseText = await response.text();
|
|
2012
|
+
const responseText = await controllerTransport(async () => await response.text());
|
|
2000
2013
|
if (Buffer.byteLength(responseText) > BROWSER_CONTROL_MAX_JSON_BYTES) {
|
|
2001
2014
|
throw new BrowserControlProtocolError("browser controller response is too large");
|
|
2002
2015
|
}
|
|
2016
|
+
rejectProviderGatewayResponse(responseText, response.status);
|
|
2003
2017
|
parseEnvelope(responseText, response.status);
|
|
2004
2018
|
throw new BrowserControlProtocolError("browser controller image response is invalid");
|
|
2005
2019
|
}
|
|
@@ -2009,10 +2023,32 @@ async function requestExposedControllerBytes(
|
|
|
2009
2023
|
await response.body?.cancel("frame exceeds its byte bound").catch(() => undefined);
|
|
2010
2024
|
throw new RangeError("frame exceeds its byte bound");
|
|
2011
2025
|
}
|
|
2012
|
-
const data = await
|
|
2026
|
+
const data = await controllerTransport(
|
|
2027
|
+
async () => await readBoundedFrameResponse(response, maxBytes),
|
|
2028
|
+
);
|
|
2013
2029
|
return controllerImageFrame(data, response.headers, input);
|
|
2014
2030
|
}
|
|
2015
2031
|
|
|
2032
|
+
async function controllerTransport<T>(readOrDispatch: () => Promise<T>): Promise<T> {
|
|
2033
|
+
try {
|
|
2034
|
+
return await readOrDispatch();
|
|
2035
|
+
} catch (error) {
|
|
2036
|
+
if (
|
|
2037
|
+
error instanceof BrowserControlProtocolError ||
|
|
2038
|
+
error instanceof BrowserControlTransportError ||
|
|
2039
|
+
error instanceof RangeError
|
|
2040
|
+
) {
|
|
2041
|
+
throw error;
|
|
2042
|
+
}
|
|
2043
|
+
// Fetch or response-body failure cannot prove the controller did not
|
|
2044
|
+
// execute a mutation. Keep it typed so endpoint discovery's in-box fallback
|
|
2045
|
+
// cannot replay an already-dispatched request.
|
|
2046
|
+
throw new BrowserControlTransportError("browser controller host-fetch failed", {
|
|
2047
|
+
cause: error,
|
|
2048
|
+
});
|
|
2049
|
+
}
|
|
2050
|
+
}
|
|
2051
|
+
|
|
2016
2052
|
async function readBoundedFrameResponse(response: Response, maxBytes: number): Promise<Uint8Array> {
|
|
2017
2053
|
if (!response.body) throw new BrowserControlProtocolError("browser controller image is empty");
|
|
2018
2054
|
const reader = response.body.getReader();
|
|
@@ -2272,6 +2308,15 @@ function looksLikeBrowserControlEnvelope(body: string): boolean {
|
|
|
2272
2308
|
}
|
|
2273
2309
|
}
|
|
2274
2310
|
|
|
2311
|
+
function rejectProviderGatewayResponse(body: string, status: number): void {
|
|
2312
|
+
// A tunnel's transient upstream failure is not a browserd protocol response.
|
|
2313
|
+
// Preserve browserd envelopes, including their semantic retry policy. This
|
|
2314
|
+
// transport classification cannot prove that a mutation was not dispatched.
|
|
2315
|
+
if ([502, 503, 504].includes(status) && !looksLikeBrowserControlEnvelope(body)) {
|
|
2316
|
+
throw new BrowserControlTransportError(`browser controller returned HTTP ${status}`);
|
|
2317
|
+
}
|
|
2318
|
+
}
|
|
2319
|
+
|
|
2275
2320
|
function parseEnvelope(body: string, status: number): unknown {
|
|
2276
2321
|
let value: unknown;
|
|
2277
2322
|
try {
|