@centerforagenticai/pi-multi-account 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/packages/pi-anthropic-oauth/src/stream.ts +8 -0
- package/src/codex-adapter.ts +81 -7
- package/src/config.ts +151 -0
- package/src/host-final-stop-message.ts +16 -4
- package/src/index.ts +17 -0
- package/src/logical-provider.ts +127 -1
- package/src/model-fallback-policy.ts +384 -0
- package/src/recovery-engine.ts +354 -68
- package/src/recovery-plan.ts +7 -1
- package/src/recovery-send-evidence.ts +29 -0
- package/src/refusal-advice.ts +139 -0
- package/src/shared-usage.ts +21 -4
- package/src/usage-fetch.ts +42 -59
package/src/recovery-engine.ts
CHANGED
|
@@ -10,7 +10,8 @@ import {
|
|
|
10
10
|
createAcceptedOutputStream,
|
|
11
11
|
type AcceptedOutputErrorCode,
|
|
12
12
|
} from "./recovery-output.js";
|
|
13
|
-
import type { RecoveryCandidate } from "./recovery-plan.js";
|
|
13
|
+
import type { RecoveryActionKind, RecoveryCandidate } from "./recovery-plan.js";
|
|
14
|
+
import { classifyCodexRecoverySendEvidence } from "./recovery-send-evidence.js";
|
|
14
15
|
|
|
15
16
|
/** A request-owned timer. Cancelling it must prevent its callback from running. */
|
|
16
17
|
export interface RecoveryTimer {
|
|
@@ -33,8 +34,16 @@ export type RecoveryPreDispatchDecision =
|
|
|
33
34
|
|
|
34
35
|
export type RecoveryRetrySafety =
|
|
35
36
|
| {
|
|
36
|
-
readonly status: "
|
|
37
|
-
readonly
|
|
37
|
+
readonly status: "recoverable";
|
|
38
|
+
readonly action: "account";
|
|
39
|
+
readonly reason:
|
|
40
|
+
| "account-local-quota"
|
|
41
|
+
| "account-local-auth"
|
|
42
|
+
| "account-local-rate-limit";
|
|
43
|
+
}
|
|
44
|
+
| {
|
|
45
|
+
readonly status: "model-policy";
|
|
46
|
+
readonly reason: "structured-provider-error";
|
|
38
47
|
}
|
|
39
48
|
| {
|
|
40
49
|
readonly status: "unsafe";
|
|
@@ -42,38 +51,40 @@ export type RecoveryRetrySafety =
|
|
|
42
51
|
| "uncertain-external-effects"
|
|
43
52
|
| "completed-tool"
|
|
44
53
|
| "completed-call"
|
|
45
|
-
| "agent-run"
|
|
54
|
+
| "agent-run"
|
|
55
|
+
| "invalid-request"
|
|
56
|
+
| "refusal"
|
|
57
|
+
| "unknown";
|
|
46
58
|
};
|
|
47
59
|
|
|
60
|
+
export type RecoveryModelDecision =
|
|
61
|
+
| {
|
|
62
|
+
readonly status: "recoverable";
|
|
63
|
+
readonly action: "model";
|
|
64
|
+
readonly reason: "unsupported-model" | "unsupported-capability";
|
|
65
|
+
}
|
|
66
|
+
| { readonly status: "none" };
|
|
67
|
+
|
|
48
68
|
/** Request options after recovery has replaced the provider's inner retry allowance. */
|
|
49
69
|
export type RecoveryBoundStreamOptions = SimpleStreamOptions & {
|
|
50
70
|
readonly maxRetries: number;
|
|
51
71
|
};
|
|
52
72
|
|
|
53
73
|
/**
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
* may send three socket frames and then one SSE request. Recovery reserves all
|
|
60
|
-
* sends that could be charged even though fewer (often one) normally occur.
|
|
74
|
+
* Every supported bounded provider invocation reserves one possibly charged
|
|
75
|
+
* send. Pinned Codex non-SSE transport can reconnect or fall back internally,
|
|
76
|
+
* and the vendored Google Antigravity stream ignores `maxRetries` and loops over
|
|
77
|
+
* empty-response retries, runtime-model candidates and endpoint fallbacks, so
|
|
78
|
+
* both counts are explicitly unknown and can never authorize another send.
|
|
61
79
|
*/
|
|
62
80
|
export type RecoverySendReservation =
|
|
63
81
|
| {
|
|
64
82
|
readonly maximumPossiblyChargedSends: 1;
|
|
65
|
-
readonly
|
|
66
|
-
readonly basis: "bounded-http";
|
|
83
|
+
readonly basis: "bounded-provider-invocation";
|
|
67
84
|
}
|
|
68
85
|
| {
|
|
69
|
-
readonly maximumPossiblyChargedSends:
|
|
70
|
-
readonly
|
|
71
|
-
readonly basis: "adaptive-anthropic-provenance";
|
|
72
|
-
}
|
|
73
|
-
| {
|
|
74
|
-
readonly maximumPossiblyChargedSends: 4;
|
|
75
|
-
readonly overReservation: "deliberate";
|
|
76
|
-
readonly basis: "codex-websocket-uncertain";
|
|
86
|
+
readonly maximumPossiblyChargedSends: "unknown";
|
|
87
|
+
readonly basis: "codex-non-sse-unknown" | "antigravity-inner-unknown";
|
|
77
88
|
};
|
|
78
89
|
|
|
79
90
|
export interface RecoverySendReservationRequest {
|
|
@@ -90,18 +101,20 @@ export interface RecoverySendReservationRequest {
|
|
|
90
101
|
export type RecoveryChargedSendExposure =
|
|
91
102
|
| {
|
|
92
103
|
readonly status: "bounded";
|
|
93
|
-
readonly maximumPossiblyChargedSends: 1
|
|
104
|
+
readonly maximumPossiblyChargedSends: 1;
|
|
94
105
|
}
|
|
95
106
|
| {
|
|
96
107
|
readonly status: "unknown";
|
|
97
|
-
readonly reason: "codex-
|
|
108
|
+
readonly reason: "codex-non-sse-send-count" | "antigravity-inner-sends";
|
|
98
109
|
};
|
|
99
110
|
|
|
100
111
|
/**
|
|
101
112
|
* One supported provider invocation. `retrySafety` must classify the final
|
|
102
113
|
* outbound payload after caller payload replacement and every provider callback.
|
|
103
|
-
* Recovery independently overrides that classification for
|
|
104
|
-
*
|
|
114
|
+
* Recovery independently overrides that classification for non-SSE Codex
|
|
115
|
+
* attempts unless `classifyCodexRecoverySendEvidence` proves a single
|
|
116
|
+
* pre-execution rejection, because a socket send may have reconnected or fallen
|
|
117
|
+
* back to SSE with uncertain external effects.
|
|
105
118
|
*/
|
|
106
119
|
export interface RecoveryPhysicalAttempt {
|
|
107
120
|
readonly output: AsyncIterable<unknown> | Promise<AsyncIterable<unknown>>;
|
|
@@ -141,8 +154,37 @@ export interface RecoveryAttemptAccounting {
|
|
|
141
154
|
readonly cost: RecoveryCostCoverage;
|
|
142
155
|
}
|
|
143
156
|
|
|
157
|
+
/**
|
|
158
|
+
* Content-free structured projection of a failed terminal for model policy.
|
|
159
|
+
*
|
|
160
|
+
* The engine copies only these named fields; assistant content, thinking, tool
|
|
161
|
+
* calls, error text, raw stop reasons, response identifiers, usage and
|
|
162
|
+
* diagnostic messages/details never reach the hook. A policy must decide from
|
|
163
|
+
* these structured facts alone and must never scan provider or assistant text.
|
|
164
|
+
*/
|
|
165
|
+
export interface RecoveryModelFailure {
|
|
166
|
+
readonly stopReason: AssistantMessage["stopReason"];
|
|
167
|
+
readonly api: string;
|
|
168
|
+
readonly provider: string;
|
|
169
|
+
readonly model: string;
|
|
170
|
+
/** Only whether the terminal carried an error message, never its text. */
|
|
171
|
+
readonly hasErrorMessage: boolean;
|
|
172
|
+
/** Diagnostic `type` identifiers only, in terminal order. */
|
|
173
|
+
readonly diagnosticTypes: readonly string[];
|
|
174
|
+
/**
|
|
175
|
+
* Structured provider stop code, copied only from the two-value allowlist.
|
|
176
|
+
* Absent for any other value; never derived from error or assistant text.
|
|
177
|
+
*/
|
|
178
|
+
readonly code?: "refusal" | "unknown_stop";
|
|
179
|
+
}
|
|
180
|
+
|
|
144
181
|
export interface RecoveryEngineDependencies {
|
|
145
182
|
readonly clock: RecoveryClock;
|
|
183
|
+
/** #129 owns this structured policy hook. Omission means no model recovery. */
|
|
184
|
+
readonly classifyModelRecovery?: (
|
|
185
|
+
failure: RecoveryModelFailure,
|
|
186
|
+
signal: AbortSignal,
|
|
187
|
+
) => RecoveryModelDecision | Promise<RecoveryModelDecision>;
|
|
146
188
|
/** Re-read eligibility, live config authorization and credentials here. */
|
|
147
189
|
readonly recheck: (
|
|
148
190
|
candidate: RecoveryCandidate,
|
|
@@ -192,7 +234,10 @@ export type RecoveryTerminationReason =
|
|
|
192
234
|
| "uncertain-external-effects"
|
|
193
235
|
| "completed-tool"
|
|
194
236
|
| "completed-call"
|
|
195
|
-
| "agent-run"
|
|
237
|
+
| "agent-run"
|
|
238
|
+
| "invalid-request"
|
|
239
|
+
| "refusal"
|
|
240
|
+
| "unknown";
|
|
196
241
|
|
|
197
242
|
export type RecoveryResult =
|
|
198
243
|
| {
|
|
@@ -202,11 +247,16 @@ export type RecoveryResult =
|
|
|
202
247
|
readonly terminal: AssistantMessage;
|
|
203
248
|
readonly output: AssistantMessageEventStream;
|
|
204
249
|
}
|
|
205
|
-
| {
|
|
250
|
+
| {
|
|
251
|
+
readonly status: "exhausted";
|
|
252
|
+
readonly attempts: number;
|
|
253
|
+
readonly errorMessage: string;
|
|
254
|
+
}
|
|
206
255
|
| {
|
|
207
256
|
readonly status: "terminated";
|
|
208
257
|
readonly reason: RecoveryTerminationReason;
|
|
209
258
|
readonly attempts: number;
|
|
259
|
+
readonly errorMessage: string;
|
|
210
260
|
};
|
|
211
261
|
|
|
212
262
|
export interface RecoveryEngine {
|
|
@@ -235,15 +285,54 @@ interface ActiveRequest {
|
|
|
235
285
|
const GAP_USAGE: RecoveryUsageCoverage = Object.freeze({ coverage: "gap" });
|
|
236
286
|
const GAP_COST: RecoveryCostCoverage = Object.freeze({ coverage: "gap" });
|
|
237
287
|
|
|
288
|
+
/**
|
|
289
|
+
* One unified logical call owns at most two provider invocations: the initial
|
|
290
|
+
* invocation plus one recovery invocation. Raising this value allows a third
|
|
291
|
+
* invocation and must fail the named `RECOVERY-TWO-SEND-CAP` regression.
|
|
292
|
+
*
|
|
293
|
+
* For the bounded HTTP/SSE families (Anthropic, OpenAI, Codex SSE) inner
|
|
294
|
+
* provider retries are forced to zero, so each invocation is one possibly
|
|
295
|
+
* charged send and the call makes at most two sends. Both Anthropic
|
|
296
|
+
* paths honour that zero: the adaptive adapter
|
|
297
|
+
* (`src/anthropic-adaptive-stream.ts`) and the vendored pinned stream
|
|
298
|
+
* (`packages/pi-anthropic-oauth/src/stream.ts`, local patch 0004) forward a
|
|
299
|
+
* finite non-negative integer `maxRetries` into `client.messages.create`.
|
|
300
|
+
*
|
|
301
|
+
* A Codex non-SSE (auto/WebSocket) invocation has an unknown internal send
|
|
302
|
+
* count: pinned pi-ai 0.84.4 may reconnect or fall back to SSE inside one
|
|
303
|
+
* invocation. The engine therefore never uses such an invocation for the
|
|
304
|
+
* recovery send, but it cannot control the internal count of a non-SSE initial
|
|
305
|
+
* invocation. Consumers, including the production cutover, must not claim the
|
|
306
|
+
* at-most-two-sends guarantee for Codex auto/WebSocket calls; only the
|
|
307
|
+
* two-invocation bound holds there. Routed Codex calls are currently pinned to
|
|
308
|
+
* SSE by `forceCodexSseOptions` (`src/codex-adapter.ts`), so a caller that
|
|
309
|
+
* passes the forced options here receives the bounded-HTTP reservation.
|
|
310
|
+
*
|
|
311
|
+
* A Google Antigravity invocation also has an unknown internal send count: the
|
|
312
|
+
* vendored stream (`packages/pi-antigravity/src/stream/stream.ts`) ignores
|
|
313
|
+
* `maxRetries` and can send once per empty-response retry (up to three), runtime
|
|
314
|
+
* model candidate, and `ENDPOINT_FALLBACKS` entry
|
|
315
|
+
* (`packages/pi-antigravity/src/client/client.ts`, three endpoints). The engine
|
|
316
|
+
* reserves it as unknown, never uses it for the recovery send, and terminates
|
|
317
|
+
* with `uncertain-external-effects` after a failed initial Antigravity
|
|
318
|
+
* invocation. Consumers must not claim the at-most-two-sends guarantee for
|
|
319
|
+
* Antigravity calls; only the two-invocation bound holds there.
|
|
320
|
+
*/
|
|
321
|
+
export const RECOVERY_MAX_PROVIDER_SENDS_PER_CALL = 2 as const;
|
|
322
|
+
|
|
238
323
|
/**
|
|
239
324
|
* Supported request-local retry controls. Keep every managed family's entry
|
|
240
325
|
* separate: each provider path has its own compile-valid mutation control and
|
|
241
|
-
* regression test.
|
|
242
|
-
*
|
|
243
|
-
*
|
|
244
|
-
*
|
|
326
|
+
* regression test. Anthropic's zero bounds both Anthropic paths: the adaptive
|
|
327
|
+
* adapter (`ANTHROPIC-MAX-RETRIES-ZERO`) and the vendored pinned stream
|
|
328
|
+
* (`ANTHROPIC-PINNED-MAX-RETRIES`) each forward this request-local `maxRetries`
|
|
329
|
+
* into `client.messages.create`, so the SDK's two default retries never run.
|
|
330
|
+
* Google Antigravity receives the same fail-closed zero, but its vendored stream
|
|
331
|
+
* ignores it; its unknown inner send count is handled by the
|
|
332
|
+
* `antigravity-inner-unknown` reservation instead (see
|
|
333
|
+
* {@link RECOVERY_MAX_PROVIDER_SENDS_PER_CALL}).
|
|
245
334
|
*/
|
|
246
|
-
const RECOVERY_INNER_RETRY_LIMITS = Object.freeze({
|
|
335
|
+
export const RECOVERY_INNER_RETRY_LIMITS = Object.freeze({
|
|
247
336
|
anthropic: 0,
|
|
248
337
|
openai: 0,
|
|
249
338
|
"openai-codex": 0,
|
|
@@ -264,40 +353,63 @@ function sendReservationFor(
|
|
|
264
353
|
candidate: RecoveryCandidate,
|
|
265
354
|
options: RecoveryBoundStreamOptions,
|
|
266
355
|
): RecoverySendReservation {
|
|
267
|
-
if (candidate.family === "
|
|
268
|
-
//
|
|
269
|
-
//
|
|
356
|
+
if (candidate.family === "openai-codex" && options.transport !== "sse") {
|
|
357
|
+
// Pinned pi-ai 0.84.4 can resend `response.create` once for
|
|
358
|
+
// previous_response_not_found, once for a connection-limit error, then fall
|
|
359
|
+
// back to one SSE send (`dist/api/openai-codex-responses.js:214-245`), so one
|
|
360
|
+
// invocation may make up to four sends. The supported terminal carries no
|
|
361
|
+
// trustworthy complete count, so this invocation can never authorize a send.
|
|
362
|
+
// Recovery cannot bound the internal sends of an initial non-SSE invocation
|
|
363
|
+
// and never forces SSE; that measured limit remains open for the cutover.
|
|
270
364
|
return Object.freeze({
|
|
271
|
-
maximumPossiblyChargedSends:
|
|
272
|
-
|
|
273
|
-
basis: "adaptive-anthropic-provenance",
|
|
365
|
+
maximumPossiblyChargedSends: "unknown",
|
|
366
|
+
basis: "codex-non-sse-unknown",
|
|
274
367
|
});
|
|
275
368
|
}
|
|
276
|
-
if (candidate.family === "
|
|
277
|
-
//
|
|
278
|
-
//
|
|
369
|
+
if (candidate.family === "google-antigravity") {
|
|
370
|
+
// The vendored stream ignores `maxRetries` and loops over empty-response
|
|
371
|
+
// retries, runtime-model candidates and endpoint fallbacks inside one
|
|
372
|
+
// invocation, so it can make several sends with no trustworthy count.
|
|
279
373
|
return Object.freeze({
|
|
280
|
-
maximumPossiblyChargedSends:
|
|
281
|
-
|
|
282
|
-
basis: "codex-websocket-uncertain",
|
|
374
|
+
maximumPossiblyChargedSends: "unknown",
|
|
375
|
+
basis: "antigravity-inner-unknown",
|
|
283
376
|
});
|
|
284
377
|
}
|
|
285
378
|
return Object.freeze({
|
|
286
379
|
maximumPossiblyChargedSends: 1,
|
|
287
|
-
|
|
288
|
-
basis: "bounded-http",
|
|
380
|
+
basis: "bounded-provider-invocation",
|
|
289
381
|
});
|
|
290
382
|
}
|
|
291
383
|
|
|
292
384
|
function chargedSendExposureFor(
|
|
293
385
|
reservation: RecoverySendReservation,
|
|
294
386
|
): RecoveryChargedSendExposure {
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
387
|
+
switch (reservation.basis) {
|
|
388
|
+
case "codex-non-sse-unknown":
|
|
389
|
+
return Object.freeze({ status: "unknown", reason: "codex-non-sse-send-count" });
|
|
390
|
+
case "antigravity-inner-unknown":
|
|
391
|
+
return Object.freeze({ status: "unknown", reason: "antigravity-inner-sends" });
|
|
392
|
+
case "bounded-provider-invocation":
|
|
393
|
+
return Object.freeze({ status: "bounded", maximumPossiblyChargedSends: 1 });
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
/**
|
|
398
|
+
* Content-free terminal error text for an exhausted or terminated recovery.
|
|
399
|
+
*
|
|
400
|
+
* Host retries count toward the same two-send cap. Pinned pi-coding-agent 0.84.4
|
|
401
|
+
* `AgentSession._isRetryableError` (`dist/core/agent-session.js:2241-2246`)
|
|
402
|
+
* restarts the turn when `isRetryableAssistantError` from
|
|
403
|
+
* `@earendil-works/pi-ai/compat` matches the final `errorMessage`
|
|
404
|
+
* (`pi-ai/dist/utils/retry.js:166-173`, patterns at `:4-77`), and its overflow
|
|
405
|
+
* path compact-and-retries when `isContextOverflow` matches
|
|
406
|
+
* (`agent-session.js:1652-1690`, `pi-ai/dist/utils/overflow.js:130-156`). A
|
|
407
|
+
* production caller must surface this message, never a provider's own text, so
|
|
408
|
+
* neither host path can add a third send. The named `HOST-RETRY-BOUNDARY`
|
|
409
|
+
* regression checks both classifiers directly.
|
|
410
|
+
*/
|
|
411
|
+
export function buildBoundedRecoveryFinalErrorMessage(): string {
|
|
412
|
+
return "Unified recovery stopped after its bounded provider attempt.";
|
|
301
413
|
}
|
|
302
414
|
|
|
303
415
|
function finiteNonNegative(value: unknown): number | undefined {
|
|
@@ -352,6 +464,20 @@ function terminalFromEvent(event: unknown): unknown {
|
|
|
352
464
|
return undefined;
|
|
353
465
|
}
|
|
354
466
|
|
|
467
|
+
function assistantTerminal(value: unknown): AssistantMessage | undefined {
|
|
468
|
+
if (typeof value !== "object" || value === null) return undefined;
|
|
469
|
+
const terminal = value as Partial<AssistantMessage>;
|
|
470
|
+
return terminal.role === "assistant" &&
|
|
471
|
+
Array.isArray(terminal.content) &&
|
|
472
|
+
typeof terminal.api === "string" &&
|
|
473
|
+
typeof terminal.provider === "string" &&
|
|
474
|
+
typeof terminal.model === "string" &&
|
|
475
|
+
typeof terminal.stopReason === "string" &&
|
|
476
|
+
typeof terminal.timestamp === "number"
|
|
477
|
+
? (terminal as AssistantMessage)
|
|
478
|
+
: undefined;
|
|
479
|
+
}
|
|
480
|
+
|
|
355
481
|
const PROGRESS_EVENT_TYPES = new Set([
|
|
356
482
|
"start",
|
|
357
483
|
"text_start",
|
|
@@ -376,6 +502,7 @@ function isProgressEvent(event: unknown): boolean {
|
|
|
376
502
|
function observedOutput(
|
|
377
503
|
upstream: AsyncIterable<unknown> | Promise<AsyncIterable<unknown>>,
|
|
378
504
|
onProgress: () => void,
|
|
505
|
+
onTerminal: (terminal: AssistantMessage) => void,
|
|
379
506
|
onFacts: (facts: AttemptFacts) => void,
|
|
380
507
|
): AsyncIterable<unknown> {
|
|
381
508
|
return {
|
|
@@ -383,8 +510,12 @@ function observedOutput(
|
|
|
383
510
|
const source = await upstream;
|
|
384
511
|
for await (const event of source) {
|
|
385
512
|
if (isProgressEvent(event)) onProgress();
|
|
386
|
-
const
|
|
387
|
-
if (
|
|
513
|
+
const terminal = assistantTerminal(terminalFromEvent(event));
|
|
514
|
+
if (terminal !== undefined) {
|
|
515
|
+
onTerminal(terminal);
|
|
516
|
+
const facts = projectAttemptFacts(terminal);
|
|
517
|
+
if (facts !== undefined) onFacts(facts);
|
|
518
|
+
}
|
|
388
519
|
yield event;
|
|
389
520
|
}
|
|
390
521
|
},
|
|
@@ -451,10 +582,21 @@ function validCandidate(candidate: RecoveryCandidate): boolean {
|
|
|
451
582
|
// the way the prior hand-listed three-family check did for Google Antigravity.
|
|
452
583
|
isManagedFamily(candidate.family) &&
|
|
453
584
|
typeof candidate.modelId === "string" &&
|
|
454
|
-
candidate.modelId.length > 0
|
|
585
|
+
candidate.modelId.length > 0 &&
|
|
586
|
+
(candidate.recoveryAction === "account" || candidate.recoveryAction === "model")
|
|
455
587
|
);
|
|
456
588
|
}
|
|
457
589
|
|
|
590
|
+
function changesOnlyAuthorizedDimension(
|
|
591
|
+
initial: RecoveryCandidate,
|
|
592
|
+
candidate: RecoveryCandidate,
|
|
593
|
+
action: RecoveryActionKind,
|
|
594
|
+
): boolean {
|
|
595
|
+
return action === "account"
|
|
596
|
+
? candidate.providerId !== initial.providerId && candidate.modelId === initial.modelId
|
|
597
|
+
: candidate.modelId !== initial.modelId;
|
|
598
|
+
}
|
|
599
|
+
|
|
458
600
|
function observeLatePhysicalAttempt(
|
|
459
601
|
pending: RecoveryPhysicalAttempt | Promise<RecoveryPhysicalAttempt>,
|
|
460
602
|
signal: AbortSignal,
|
|
@@ -478,6 +620,45 @@ function observeLatePhysicalAttempt(
|
|
|
478
620
|
);
|
|
479
621
|
}
|
|
480
622
|
|
|
623
|
+
/**
|
|
624
|
+
* The first send must use the call's selected model on an account candidate.
|
|
625
|
+
* A model change happens only through `classifyModelRecovery` after a
|
|
626
|
+
* structured failure, never because recheck skipped every exact-model account.
|
|
627
|
+
*/
|
|
628
|
+
function eligibleInitialCandidate(candidate: RecoveryCandidate): boolean {
|
|
629
|
+
return (
|
|
630
|
+
candidate.recoveryAction === "account" &&
|
|
631
|
+
candidate.modelId === candidate.selectedModelId
|
|
632
|
+
);
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
function projectModelFailure(terminal: AssistantMessage): RecoveryModelFailure {
|
|
636
|
+
const diagnostics: readonly unknown[] = Array.isArray(terminal.diagnostics)
|
|
637
|
+
? terminal.diagnostics
|
|
638
|
+
: [];
|
|
639
|
+
const diagnosticTypes: string[] = [];
|
|
640
|
+
for (const diagnostic of diagnostics) {
|
|
641
|
+
const type =
|
|
642
|
+
typeof diagnostic === "object" && diagnostic !== null
|
|
643
|
+
? (diagnostic as { type?: unknown }).type
|
|
644
|
+
: undefined;
|
|
645
|
+
if (typeof type === "string") diagnosticTypes.push(type);
|
|
646
|
+
}
|
|
647
|
+
const rawCode = (terminal as { code?: unknown }).code;
|
|
648
|
+
const code =
|
|
649
|
+
rawCode === "refusal" || rawCode === "unknown_stop" ? rawCode : undefined;
|
|
650
|
+
return Object.freeze({
|
|
651
|
+
stopReason: terminal.stopReason,
|
|
652
|
+
api: terminal.api,
|
|
653
|
+
provider: terminal.provider,
|
|
654
|
+
model: terminal.model,
|
|
655
|
+
hasErrorMessage:
|
|
656
|
+
typeof terminal.errorMessage === "string" && terminal.errorMessage.length > 0,
|
|
657
|
+
diagnosticTypes: Object.freeze(diagnosticTypes),
|
|
658
|
+
...(code === undefined ? {} : { code }),
|
|
659
|
+
});
|
|
660
|
+
}
|
|
661
|
+
|
|
481
662
|
function validPreDispatchDecision(value: unknown): value is RecoveryPreDispatchDecision {
|
|
482
663
|
if (typeof value !== "object" || value === null) return false;
|
|
483
664
|
const decision = value as Record<string, unknown>;
|
|
@@ -492,18 +673,38 @@ function validPreDispatchDecision(value: unknown): value is RecoveryPreDispatchD
|
|
|
492
673
|
function validRetrySafety(value: unknown): value is RecoveryRetrySafety {
|
|
493
674
|
if (typeof value !== "object" || value === null) return false;
|
|
494
675
|
const safety = value as Record<string, unknown>;
|
|
495
|
-
if (safety.status === "
|
|
676
|
+
if (safety.status === "recoverable") {
|
|
496
677
|
return (
|
|
497
|
-
safety.
|
|
498
|
-
safety.reason === "
|
|
678
|
+
safety.action === "account" &&
|
|
679
|
+
(safety.reason === "account-local-quota" ||
|
|
680
|
+
safety.reason === "account-local-auth" ||
|
|
681
|
+
safety.reason === "account-local-rate-limit")
|
|
499
682
|
);
|
|
500
683
|
}
|
|
684
|
+
if (safety.status === "model-policy") {
|
|
685
|
+
return safety.reason === "structured-provider-error";
|
|
686
|
+
}
|
|
501
687
|
return (
|
|
502
688
|
safety.status === "unsafe" &&
|
|
503
689
|
(safety.reason === "uncertain-external-effects" ||
|
|
504
690
|
safety.reason === "completed-tool" ||
|
|
505
691
|
safety.reason === "completed-call" ||
|
|
506
|
-
safety.reason === "agent-run"
|
|
692
|
+
safety.reason === "agent-run" ||
|
|
693
|
+
safety.reason === "invalid-request" ||
|
|
694
|
+
safety.reason === "refusal" ||
|
|
695
|
+
safety.reason === "unknown")
|
|
696
|
+
);
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
function validModelDecision(value: unknown): value is RecoveryModelDecision {
|
|
700
|
+
if (typeof value !== "object" || value === null) return false;
|
|
701
|
+
const decision = value as Record<string, unknown>;
|
|
702
|
+
return (
|
|
703
|
+
decision.status === "none" ||
|
|
704
|
+
(decision.status === "recoverable" &&
|
|
705
|
+
decision.action === "model" &&
|
|
706
|
+
(decision.reason === "unsupported-model" ||
|
|
707
|
+
decision.reason === "unsupported-capability"))
|
|
507
708
|
);
|
|
508
709
|
}
|
|
509
710
|
|
|
@@ -558,16 +759,34 @@ async function runRecovery(
|
|
|
558
759
|
onProgress: () => void,
|
|
559
760
|
): Promise<RecoveryResult> {
|
|
560
761
|
let attempts = 0;
|
|
762
|
+
let initialCandidate: RecoveryCandidate | undefined;
|
|
763
|
+
let recoveryAction: RecoveryActionKind | undefined;
|
|
561
764
|
const terminated = (): RecoveryResult => ({
|
|
562
765
|
status: "terminated",
|
|
563
766
|
reason: request.reason ?? "callback-failure",
|
|
564
767
|
attempts,
|
|
768
|
+
errorMessage: buildBoundedRecoveryFinalErrorMessage(),
|
|
769
|
+
});
|
|
770
|
+
const exhausted = (): RecoveryResult => ({
|
|
771
|
+
status: "exhausted",
|
|
772
|
+
attempts,
|
|
773
|
+
errorMessage: buildBoundedRecoveryFinalErrorMessage(),
|
|
565
774
|
});
|
|
566
775
|
const consideredPairs = new Set<string>();
|
|
567
776
|
|
|
568
777
|
for (const candidate of [...input.candidates]) {
|
|
569
778
|
if (request.controller.signal.aborted) return terminated();
|
|
779
|
+
if (attempts >= RECOVERY_MAX_PROVIDER_SENDS_PER_CALL) return exhausted();
|
|
570
780
|
if (!validCandidate(candidate)) continue;
|
|
781
|
+
if (initialCandidate === undefined && !eligibleInitialCandidate(candidate)) continue;
|
|
782
|
+
if (
|
|
783
|
+
recoveryAction !== undefined &&
|
|
784
|
+
(initialCandidate === undefined ||
|
|
785
|
+
candidate.recoveryAction !== recoveryAction ||
|
|
786
|
+
!changesOnlyAuthorizedDimension(initialCandidate, candidate, recoveryAction))
|
|
787
|
+
) {
|
|
788
|
+
continue;
|
|
789
|
+
}
|
|
571
790
|
const pair = `${candidate.providerId}\u0000${candidate.modelId}`;
|
|
572
791
|
if (consideredPairs.has(pair)) continue;
|
|
573
792
|
consideredPairs.add(pair);
|
|
@@ -608,8 +827,16 @@ async function runRecovery(
|
|
|
608
827
|
abortRequest(request, "callback-failure");
|
|
609
828
|
return terminated();
|
|
610
829
|
}
|
|
830
|
+
if (attempts > 0 && reservation.maximumPossiblyChargedSends === "unknown") {
|
|
831
|
+
// The recovery send is the call's last send. A pinned Codex non-SSE or a
|
|
832
|
+
// vendored Antigravity invocation can make several internal sends, so it
|
|
833
|
+
// cannot fit a one-send remainder.
|
|
834
|
+
continue;
|
|
835
|
+
}
|
|
836
|
+
// A reservation keeps its ordinal even when the request aborts before dispatch.
|
|
837
|
+
const ordinal = attempts + 1;
|
|
611
838
|
const reserved = await reserveAttempt(deps, request, {
|
|
612
|
-
ordinal
|
|
839
|
+
ordinal,
|
|
613
840
|
candidate,
|
|
614
841
|
reservation,
|
|
615
842
|
});
|
|
@@ -625,6 +852,7 @@ async function runRecovery(
|
|
|
625
852
|
| { readonly terminal: AssistantMessage; readonly output: AssistantMessageEventStream }
|
|
626
853
|
| undefined;
|
|
627
854
|
let safety: RecoveryRetrySafety | undefined;
|
|
855
|
+
let terminal: AssistantMessage | undefined;
|
|
628
856
|
let callbackFailed = false;
|
|
629
857
|
const attemptController = new AbortController();
|
|
630
858
|
const abortAttempt = (): void => attemptController.abort();
|
|
@@ -635,6 +863,7 @@ async function runRecovery(
|
|
|
635
863
|
if (!request.controller.signal.aborted) {
|
|
636
864
|
try {
|
|
637
865
|
// A physical attempt begins exactly when this callback is invoked.
|
|
866
|
+
initialCandidate ??= candidate;
|
|
638
867
|
attempts += 1;
|
|
639
868
|
physicalValue = deps.dispatch({
|
|
640
869
|
candidate,
|
|
@@ -665,6 +894,9 @@ async function runRecovery(
|
|
|
665
894
|
observedOutput(
|
|
666
895
|
physical.output,
|
|
667
896
|
onProgress,
|
|
897
|
+
(observed) => {
|
|
898
|
+
terminal = observed;
|
|
899
|
+
},
|
|
668
900
|
(observed) => {
|
|
669
901
|
facts = observed;
|
|
670
902
|
},
|
|
@@ -686,10 +918,18 @@ async function runRecovery(
|
|
|
686
918
|
}
|
|
687
919
|
}
|
|
688
920
|
if (accepted === undefined) {
|
|
689
|
-
if (
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
921
|
+
if (
|
|
922
|
+
reservation.basis === "antigravity-inner-unknown" ||
|
|
923
|
+
(reservation.basis === "codex-non-sse-unknown" &&
|
|
924
|
+
(terminal === undefined ||
|
|
925
|
+
classifyCodexRecoverySendEvidence(terminal) !==
|
|
926
|
+
"pre-execution-rejected"))
|
|
927
|
+
) {
|
|
928
|
+
// A non-SSE Codex invocation may have reconnected or fallen back
|
|
929
|
+
// to SSE after socket.send; only structured proof of a single
|
|
930
|
+
// pre-execution rejection may consult the caller's classifier.
|
|
931
|
+
// An Antigravity invocation may already have sent to several
|
|
932
|
+
// endpoints or runtime models; no supported evidence bounds it.
|
|
693
933
|
safety = {
|
|
694
934
|
status: "unsafe",
|
|
695
935
|
reason: "uncertain-external-effects",
|
|
@@ -716,7 +956,7 @@ async function runRecovery(
|
|
|
716
956
|
}
|
|
717
957
|
|
|
718
958
|
const accounting = await accountAttempt(deps, request, {
|
|
719
|
-
ordinal
|
|
959
|
+
ordinal,
|
|
720
960
|
candidate,
|
|
721
961
|
disposition: accepted === undefined ? "failed" : "accepted",
|
|
722
962
|
...(accepted === undefined ? { failureCode } : {}),
|
|
@@ -745,12 +985,47 @@ async function runRecovery(
|
|
|
745
985
|
abortRequest(request, "callback-failure");
|
|
746
986
|
return terminated();
|
|
747
987
|
}
|
|
988
|
+
if (safety.status === "recoverable") {
|
|
989
|
+
recoveryAction = safety.action;
|
|
990
|
+
continue;
|
|
991
|
+
}
|
|
748
992
|
if (safety.status === "unsafe") {
|
|
749
993
|
abortRequest(request, safety.reason);
|
|
750
994
|
return terminated();
|
|
751
995
|
}
|
|
996
|
+
// The recovery send was the last send; do not consult model policy for a
|
|
997
|
+
// third send that the call can never make.
|
|
998
|
+
if (attempts >= RECOVERY_MAX_PROVIDER_SENDS_PER_CALL) return exhausted();
|
|
999
|
+
if (terminal === undefined || deps.classifyModelRecovery === undefined) {
|
|
1000
|
+
abortRequest(request, "unknown");
|
|
1001
|
+
return terminated();
|
|
1002
|
+
}
|
|
1003
|
+
let modelValue: ReturnType<NonNullable<RecoveryEngineDependencies["classifyModelRecovery"]>>;
|
|
1004
|
+
try {
|
|
1005
|
+
modelValue = deps.classifyModelRecovery(
|
|
1006
|
+
projectModelFailure(terminal),
|
|
1007
|
+
request.controller.signal,
|
|
1008
|
+
);
|
|
1009
|
+
} catch {
|
|
1010
|
+
abortRequest(request, "callback-failure");
|
|
1011
|
+
return terminated();
|
|
1012
|
+
}
|
|
1013
|
+
const modelDecision = await awaitRequest(modelValue, request);
|
|
1014
|
+
if (modelDecision.status === "aborted") return terminated();
|
|
1015
|
+
if (
|
|
1016
|
+
modelDecision.status === "rejected" ||
|
|
1017
|
+
!validModelDecision(modelDecision.value)
|
|
1018
|
+
) {
|
|
1019
|
+
abortRequest(request, "callback-failure");
|
|
1020
|
+
return terminated();
|
|
1021
|
+
}
|
|
1022
|
+
if (modelDecision.value.status === "none") {
|
|
1023
|
+
abortRequest(request, "unknown");
|
|
1024
|
+
return terminated();
|
|
1025
|
+
}
|
|
1026
|
+
recoveryAction = modelDecision.value.action;
|
|
752
1027
|
}
|
|
753
|
-
return
|
|
1028
|
+
return exhausted();
|
|
754
1029
|
}
|
|
755
1030
|
|
|
756
1031
|
export function createRecoveryEngine(
|
|
@@ -767,14 +1042,24 @@ export function createRecoveryEngine(
|
|
|
767
1042
|
return {
|
|
768
1043
|
async recover(input): Promise<RecoveryResult> {
|
|
769
1044
|
if (shutdown) {
|
|
770
|
-
return {
|
|
1045
|
+
return {
|
|
1046
|
+
status: "terminated",
|
|
1047
|
+
reason: "shutdown",
|
|
1048
|
+
attempts: 0,
|
|
1049
|
+
errorMessage: buildBoundedRecoveryFinalErrorMessage(),
|
|
1050
|
+
};
|
|
771
1051
|
}
|
|
772
1052
|
if (
|
|
773
1053
|
!validTiming(input.timing) ||
|
|
774
1054
|
(input.deadlineMs !== undefined &&
|
|
775
1055
|
(!Number.isFinite(input.deadlineMs) || input.deadlineMs < 0))
|
|
776
1056
|
) {
|
|
777
|
-
return {
|
|
1057
|
+
return {
|
|
1058
|
+
status: "terminated",
|
|
1059
|
+
reason: "malformed-config",
|
|
1060
|
+
attempts: 0,
|
|
1061
|
+
errorMessage: buildBoundedRecoveryFinalErrorMessage(),
|
|
1062
|
+
};
|
|
778
1063
|
}
|
|
779
1064
|
|
|
780
1065
|
const activeRequest: ActiveRequest = { controller: new AbortController() };
|
|
@@ -833,6 +1118,7 @@ export function createRecoveryEngine(
|
|
|
833
1118
|
status: "terminated",
|
|
834
1119
|
reason: activeRequest.reason ?? "callback-failure",
|
|
835
1120
|
attempts: 0,
|
|
1121
|
+
errorMessage: buildBoundedRecoveryFinalErrorMessage(),
|
|
836
1122
|
};
|
|
837
1123
|
}
|
|
838
1124
|
return await runRecovery(deps, input, activeRequest, resetIdle);
|