@centerforagenticai/pi-multi-account 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -3
- package/package.json +5 -5
- package/packages/pi-anthropic-oauth/package.json +2 -2
- package/packages/pi-anthropic-oauth/src/stream.ts +26 -8
- package/packages/pi-anthropic-oauth/src/transport-activity.ts +59 -0
- package/packages/pi-antigravity/package.json +2 -2
- package/packages/pi-antigravity/src/models/discovery.ts +2 -1
- package/packages/pi-antigravity/src/models/grouping.ts +12 -10
- package/packages/pi-antigravity/src/models/models.ts +11 -4
- package/src/account-group-failure.ts +149 -0
- package/src/account-group-members.ts +139 -0
- package/src/anthropic-adaptive-stream.ts +18 -9
- package/src/anthropic-alias-stream.ts +13 -68
- package/src/codex-adapter.ts +78 -50
- package/src/commands.ts +3 -3
- package/src/config.ts +197 -2
- package/src/diagnostic-store.ts +17 -15
- package/src/diagnostics.ts +97 -35
- package/src/host-final-stop-message.ts +20 -118
- package/src/index.ts +452 -87
- package/src/logical-dispatch.ts +12 -13
- package/src/logical-provider.ts +1282 -355
- package/src/model-fallback-policy.ts +384 -0
- package/src/models-declaration.ts +15 -8
- package/src/public-assistant-projection.ts +163 -0
- package/src/recovery-engine.ts +648 -110
- package/src/recovery-plan.ts +7 -1
- package/src/recovery-send-evidence.ts +29 -0
- package/src/refusal-advice.ts +139 -0
- package/src/routing.ts +4 -8
- package/src/runtime-state.ts +7 -0
- package/src/shared-usage.ts +21 -4
- package/src/upstream-anthropic.ts +12 -4
- package/src/upstream-antigravity.ts +2 -38
- package/src/usage-fetch.ts +42 -59
package/src/logical-provider.ts
CHANGED
|
@@ -14,16 +14,25 @@
|
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
16
|
import { logicalAccountEligible, recordFailureCooldown } from "./routing.js";
|
|
17
|
+
import { formatAccountGroupFailure, type AccountGroupFailurePolicy, type AccountGroupFailureCandidate } from "./account-group-failure.js";
|
|
17
18
|
import type { ManagedAccount } from "./routing.js";
|
|
18
19
|
import {
|
|
20
|
+
isContextOverflow,
|
|
19
21
|
isRetryableAssistantError,
|
|
20
22
|
type AssistantMessage,
|
|
21
23
|
type ProviderResponse,
|
|
22
24
|
type SimpleStreamOptions,
|
|
23
25
|
} from "@earendil-works/pi-ai";
|
|
24
|
-
import { RuntimeState,
|
|
26
|
+
import { RuntimeState, isCanonicalManagedProviderId } from "./runtime-state.js";
|
|
25
27
|
import { hostFinalStopMessage } from "./host-final-stop-message.js";
|
|
26
|
-
import {
|
|
28
|
+
import {
|
|
29
|
+
projectPublicAssistantMessage,
|
|
30
|
+
projectPublicAssistantEvent,
|
|
31
|
+
projectAssistantContent,
|
|
32
|
+
projectTerminalUsage,
|
|
33
|
+
finiteNonNegative,
|
|
34
|
+
} from "./public-assistant-projection.js";
|
|
35
|
+
import { DEFAULT_CONFIG, MAX_ACCOUNT_LIMIT, isAccountSlotIndex } from "./config.js";
|
|
27
36
|
import type {
|
|
28
37
|
AllowedFamily,
|
|
29
38
|
CrossFamilyChain,
|
|
@@ -37,6 +46,7 @@ import {
|
|
|
37
46
|
providerErrorCodeFromMessage,
|
|
38
47
|
} from "./error-classification.js";
|
|
39
48
|
import type {
|
|
49
|
+
FailureCategory,
|
|
40
50
|
ProviderErrorCode,
|
|
41
51
|
ProviderFailureSignal,
|
|
42
52
|
TransportFailureKind,
|
|
@@ -49,6 +59,37 @@ import type { ProviderType, Vendor } from "./vendor.js";
|
|
|
49
59
|
export { LOGICAL_PROVIDER_ID } from "./models-declaration.js";
|
|
50
60
|
import { LOGICAL_PROVIDER_ID } from "./models-declaration.js";
|
|
51
61
|
import { forceCodexSseOptions } from "./codex-adapter.js";
|
|
62
|
+
import {
|
|
63
|
+
buildBoundedRecoveryFinalErrorMessage,
|
|
64
|
+
createRecoveryEngine,
|
|
65
|
+
isRecoveryStartEvent,
|
|
66
|
+
type RecoveryClock,
|
|
67
|
+
type RecoveryDispatchRequest,
|
|
68
|
+
type RecoveryEngine,
|
|
69
|
+
type RecoveryPhysicalAttempt,
|
|
70
|
+
type RecoveryResult,
|
|
71
|
+
type RecoveryRetrySafety,
|
|
72
|
+
type RecoveryTimer,
|
|
73
|
+
type RecoveryTimingConfig,
|
|
74
|
+
} from "./recovery-engine.js";
|
|
75
|
+
import {
|
|
76
|
+
buildRecoveryCandidatePlan,
|
|
77
|
+
type RecoveryCandidate,
|
|
78
|
+
type RecoveryModelCapability,
|
|
79
|
+
type RecoveryPlanAccount,
|
|
80
|
+
} from "./recovery-plan.js";
|
|
81
|
+
|
|
82
|
+
/** Production recovery timers: monotonic time and cancellable host timers. */
|
|
83
|
+
const SYSTEM_RECOVERY_CLOCK: RecoveryClock = Object.freeze({
|
|
84
|
+
now: () => performance.now(),
|
|
85
|
+
setTimer: (delayMs: number, callback: () => void) => {
|
|
86
|
+
const handle = setTimeout(callback, delayMs);
|
|
87
|
+
if (typeof handle === "object" && handle !== null && "unref" in handle) {
|
|
88
|
+
(handle as { unref: () => void }).unref();
|
|
89
|
+
}
|
|
90
|
+
return { cancel: () => clearTimeout(handle) };
|
|
91
|
+
},
|
|
92
|
+
});
|
|
52
93
|
|
|
53
94
|
/** Fixed diagnostic label for a caller-supplied transport; never echoes the raw value. */
|
|
54
95
|
function codexTransportLabel(
|
|
@@ -97,6 +138,14 @@ export interface LogicalPhysicalAccount {
|
|
|
97
138
|
};
|
|
98
139
|
}
|
|
99
140
|
|
|
141
|
+
/** Safe physical cause of a failed logical call; never contains provider prose. */
|
|
142
|
+
export interface LogicalFailureEvidence {
|
|
143
|
+
readonly providerId?: string;
|
|
144
|
+
readonly category: FailureCategory | "context-overflow" | "host-stale-install";
|
|
145
|
+
/** Physical invocations, not the unknown inner send count of Antigravity. */
|
|
146
|
+
readonly attemptCount: number;
|
|
147
|
+
}
|
|
148
|
+
|
|
100
149
|
/** One physical attempt the logical provider makes. */
|
|
101
150
|
export interface LogicalDispatchCall {
|
|
102
151
|
providerId: string;
|
|
@@ -202,17 +251,21 @@ export interface LogicalAttributionLifecycle {
|
|
|
202
251
|
shutdown(): void;
|
|
203
252
|
}
|
|
204
253
|
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
readonly
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
254
|
+
/** One authorized selection generation; outside rows explain failure, never authorize sends. */
|
|
255
|
+
export interface LogicalSelectionSnapshot {
|
|
256
|
+
readonly accounts: LogicalPhysicalAccount[];
|
|
257
|
+
readonly group?: {
|
|
258
|
+
readonly policy: AccountGroupFailurePolicy;
|
|
259
|
+
readonly accountLimit: number;
|
|
260
|
+
readonly allAccounts: readonly LogicalPhysicalAccount[];
|
|
261
|
+
readonly otherCandidates: readonly AccountGroupFailureCandidate[];
|
|
262
|
+
};
|
|
212
263
|
}
|
|
213
264
|
|
|
214
265
|
export interface LogicalProviderDeps {
|
|
215
266
|
accounts: LogicalPhysicalAccount[];
|
|
267
|
+
/** Capture policy and account facts together, once per selection/preflight. */
|
|
268
|
+
captureSelectionSnapshot?: (modelId: string) => LogicalSelectionSnapshot;
|
|
216
269
|
dispatch: (
|
|
217
270
|
call: LogicalDispatchCall,
|
|
218
271
|
) => Promise<AsyncIterable<unknown>> | AsyncIterable<unknown>;
|
|
@@ -221,6 +274,12 @@ export interface LogicalProviderDeps {
|
|
|
221
274
|
/** Subscription-catalog owner of a declared logical model. */
|
|
222
275
|
modelVendor?: (modelId: string) => Vendor | undefined;
|
|
223
276
|
tierModelMap?: TierModelMap;
|
|
277
|
+
/**
|
|
278
|
+
* The live `sameFamilyFailover` setting. `false` makes every unified call
|
|
279
|
+
* one attempt: the in-call recovery never moves to another account.
|
|
280
|
+
* Omitted uses the default.
|
|
281
|
+
*/
|
|
282
|
+
sameFamilyFailover?: boolean;
|
|
224
283
|
crossFamilyChains?: readonly CrossFamilyChain[];
|
|
225
284
|
onObservation?: (observation: LogicalObservation) => void;
|
|
226
285
|
attribution?: LogicalAttributionLifecycle;
|
|
@@ -228,9 +287,24 @@ export interface LogicalProviderDeps {
|
|
|
228
287
|
onPublicTerminal?: (physical: AssistantMessage, publicMessage: AssistantMessage) => void;
|
|
229
288
|
onDiagnostic?: (message: string) => void;
|
|
230
289
|
onShutdownAbort?: () => void;
|
|
231
|
-
|
|
290
|
+
/**
|
|
291
|
+
* A failed physical terminal the call recovered past. It was never
|
|
292
|
+
* published, so no `message_end` will carry it; the session commits its
|
|
293
|
+
* account effects here instead.
|
|
294
|
+
*/
|
|
295
|
+
onSupersededTerminal?: (physical: AssistantMessage) => void;
|
|
232
296
|
state?: RuntimeState;
|
|
233
|
-
|
|
297
|
+
/**
|
|
298
|
+
* Per-invocation idle and absolute limits and the per-attempt stall limit;
|
|
299
|
+
* omitted fields use the defaults.
|
|
300
|
+
*/
|
|
301
|
+
recoveryTiming?: Partial<
|
|
302
|
+
RecoveryTimingConfig & Pick<MultiAccountConfig, "recoveryStallTimeoutMs">
|
|
303
|
+
>;
|
|
304
|
+
/** Injected only by tests; production uses the system clock. */
|
|
305
|
+
recoveryClock?: RecoveryClock;
|
|
306
|
+
/** Injected only by tests; production uses `TRANSPORT_SILENCE_TIMEOUT_MS`. */
|
|
307
|
+
transportSilenceTimeoutMs?: number;
|
|
234
308
|
}
|
|
235
309
|
|
|
236
310
|
export interface LogicalProvider {
|
|
@@ -353,68 +427,109 @@ const EXHAUSTION_LENGTH_ALLOWANCE_MULTIPLIER = 8;
|
|
|
353
427
|
const EXHAUSTION_LENGTH_MAX_CONTEXT_FRACTION = 0.8;
|
|
354
428
|
const EXHAUSTION_LENGTH_ERROR_MESSAGE = "provider returned error (usage-limit)";
|
|
355
429
|
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
430
|
+
/**
|
|
431
|
+
* Fixed public text for a setup-shaped context overflow. The pinned host's
|
|
432
|
+
* `isContextOverflow` matches it (so the host compacts and retries once) and
|
|
433
|
+
* `isRetryableAssistantError` does not (so the host does not fail over).
|
|
434
|
+
*/
|
|
435
|
+
export const SETUP_CONTEXT_OVERFLOW_MESSAGE = "context_length_exceeded (provider_error)";
|
|
436
|
+
|
|
437
|
+
/** Closed visible vocabulary: `quota-rate-limit` itself would trigger host retry. */
|
|
438
|
+
const VISIBLE_FAILURE_CAUSES: Readonly<Record<LogicalFailureEvidence["category"], string>> = Object.freeze({
|
|
439
|
+
"quota-rate-limit": "quota",
|
|
440
|
+
"terminal-auth": "terminal-auth",
|
|
441
|
+
"transient-auth": "transient-auth",
|
|
442
|
+
permission: "permission",
|
|
443
|
+
config: "config",
|
|
444
|
+
transport: "transport",
|
|
445
|
+
unknown: "unknown",
|
|
446
|
+
"context-overflow": "context-overflow",
|
|
447
|
+
"host-stale-install": "host_stale_install",
|
|
448
|
+
});
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* Public text for a local host fault: the running pi process tried to load a
|
|
452
|
+
* bundle chunk that an in-place upgrade replaced. No provider was reached, and
|
|
453
|
+
* no other account can load the missing file, so only a restart helps. Neither
|
|
454
|
+
* host retry predicate matches this text.
|
|
455
|
+
*/
|
|
456
|
+
export const HOST_STALE_INSTALL_MESSAGE =
|
|
457
|
+
"pi's installed files changed while this session was running; restart the pi session to load the current install.";
|
|
458
|
+
|
|
459
|
+
/**
|
|
460
|
+
* Node's own phrasing, anchored at the start: pi-ai `lazyStream` keeps
|
|
461
|
+
* `error.message` unprefixed. An unanchored match would misread a provider or
|
|
462
|
+
* gateway error body that quotes a module error as a local fault.
|
|
463
|
+
*/
|
|
464
|
+
const STALE_INSTALL_TEXT =
|
|
465
|
+
/^(?:Error(?: \[ERR_MODULE_NOT_FOUND\])?: )?(?:Cannot find (?:module|package) |Failed to (?:fetch|load) dynamically imported module|Error loading dynamically imported module)/;
|
|
466
|
+
|
|
467
|
+
/**
|
|
468
|
+
* Whether a setup failure is the host failing to load its own code, rather
|
|
469
|
+
* than a provider failing. Reads the Node error code first, then the message
|
|
470
|
+
* text, because pi-ai `lazyStream` keeps only `error.message`.
|
|
471
|
+
*/
|
|
472
|
+
export function isHostStaleInstallFailure(error: unknown): boolean {
|
|
473
|
+
try {
|
|
474
|
+
if (typeof error === "string") return STALE_INSTALL_TEXT.test(error);
|
|
475
|
+
if (typeof error !== "object" || error === null) return false;
|
|
476
|
+
const { code, message, errorMessage } = error as { code?: unknown; message?: unknown; errorMessage?: unknown };
|
|
477
|
+
if (code === "ERR_MODULE_NOT_FOUND") return true;
|
|
478
|
+
return [message, errorMessage].some((text) => typeof text === "string" && STALE_INSTALL_TEXT.test(text));
|
|
479
|
+
} catch {
|
|
480
|
+
return false;
|
|
481
|
+
}
|
|
360
482
|
}
|
|
361
483
|
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
484
|
+
/** A recognised provider code, HTTP status or transport kind outranks any text. */
|
|
485
|
+
function hasStructuredFailureEvidence(failure: ProviderFailureSignal | undefined): boolean {
|
|
486
|
+
return failure !== undefined &&
|
|
487
|
+
(failure.code !== undefined || failure.httpStatus !== undefined || failure.transportKind !== undefined);
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
/**
|
|
491
|
+
* Bounded diagnostic cause for a stale install: the Node error code and the
|
|
492
|
+
* missing file's base name. Directories are dropped so no local path is kept.
|
|
493
|
+
*/
|
|
494
|
+
function hostStaleInstallCause(error: unknown): string {
|
|
495
|
+
let text = "";
|
|
496
|
+
let code: unknown;
|
|
497
|
+
try {
|
|
498
|
+
if (typeof error === "string") text = error;
|
|
499
|
+
else if (typeof error === "object" && error !== null) {
|
|
500
|
+
const fields = error as { code?: unknown; message?: unknown; errorMessage?: unknown };
|
|
501
|
+
code = fields.code;
|
|
502
|
+
text = typeof fields.message === "string" ? fields.message : typeof fields.errorMessage === "string" ? fields.errorMessage : "";
|
|
503
|
+
}
|
|
504
|
+
} catch {
|
|
505
|
+
// Fall through with whatever was read.
|
|
382
506
|
}
|
|
383
|
-
const
|
|
384
|
-
const
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
507
|
+
const file = /['"]?([^'"\s]*\.(?:m?js|cjs|json|node))['"]?/.exec(text)?.[1]?.split(/[\\/]/).at(-1);
|
|
508
|
+
const kind = code === "ERR_MODULE_NOT_FOUND" || /ERR_MODULE_NOT_FOUND|Cannot find/.test(text) ? "ERR_MODULE_NOT_FOUND" : "dynamic-import-failed";
|
|
509
|
+
return file === undefined || file.length === 0 ? kind : `${kind} ${file.slice(0, 120)}`;
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
type SetupFailureDisposition = "context-overflow" | "retryable" | "host-final";
|
|
513
|
+
|
|
514
|
+
/**
|
|
515
|
+
* How the pinned host treats the raw setup text. The host checks the two
|
|
516
|
+
* predicates separately: `_handlePostAgentRun` compacts and retries once on
|
|
517
|
+
* `isContextOverflow`, while `_isRetryableError` excludes overflow and fails
|
|
518
|
+
* over on `isRetryableAssistantError`. An unreadable predicate result counts
|
|
519
|
+
* as host-final.
|
|
520
|
+
*/
|
|
521
|
+
function setupFailureDisposition(
|
|
522
|
+
message: AssistantMessage,
|
|
523
|
+
raw: unknown,
|
|
524
|
+
): SetupFailureDisposition {
|
|
525
|
+
if (typeof raw !== "string" || raw.length === 0) return "host-final";
|
|
526
|
+
try {
|
|
527
|
+
const probe = { ...message, errorMessage: raw };
|
|
528
|
+
if (isContextOverflow(probe, 0)) return "context-overflow";
|
|
529
|
+
return isRetryableAssistantError(probe) ? "retryable" : "host-final";
|
|
530
|
+
} catch {
|
|
531
|
+
return "host-final";
|
|
397
532
|
}
|
|
398
|
-
const cacheWrite1h = finiteNonNegative(usage.cacheWrite1h);
|
|
399
|
-
if (usage.cacheWrite1h !== undefined && cacheWrite1h === undefined) return undefined;
|
|
400
|
-
const reasoning = finiteNonNegative(usage.reasoning);
|
|
401
|
-
if (usage.reasoning !== undefined && reasoning === undefined) return undefined;
|
|
402
|
-
return {
|
|
403
|
-
input,
|
|
404
|
-
output,
|
|
405
|
-
cacheRead,
|
|
406
|
-
cacheWrite,
|
|
407
|
-
...(cacheWrite1h === undefined ? {} : { cacheWrite1h }),
|
|
408
|
-
...(reasoning === undefined ? {} : { reasoning }),
|
|
409
|
-
totalTokens,
|
|
410
|
-
cost: {
|
|
411
|
-
input: costInput,
|
|
412
|
-
output: costOutput,
|
|
413
|
-
cacheRead: costCacheRead,
|
|
414
|
-
cacheWrite: costCacheWrite,
|
|
415
|
-
total: costTotal,
|
|
416
|
-
},
|
|
417
|
-
};
|
|
418
533
|
}
|
|
419
534
|
|
|
420
535
|
interface ExhaustionLengthMatch {
|
|
@@ -542,6 +657,54 @@ function projectFailureSignal(
|
|
|
542
657
|
};
|
|
543
658
|
}
|
|
544
659
|
|
|
660
|
+
/**
|
|
661
|
+
* Whether a physical terminal is the host's setup-error shape: the first event
|
|
662
|
+
* of the stream is an `error` with no content, all-zero usage, no diagnostics,
|
|
663
|
+
* no structured stop code, and no structured failure evidence.
|
|
664
|
+
*
|
|
665
|
+
* That is what pi-ai `lazyStream` (`createSetupErrorMessage`) publishes when a
|
|
666
|
+
* provider stream throws or rejects before it starts, so its `errorMessage` is
|
|
667
|
+
* raw exception text, not provider-authored failure prose. The production cause
|
|
668
|
+
* of the observed setup `TypeError` is not known (see UPSTREAM.md). The same
|
|
669
|
+
* shape also carries transient pre-start failures ("fetch failed", a 503 before
|
|
670
|
+
* `start`), so the caller decides retryability from the text, never publishes
|
|
671
|
+
* it. A real provider failure that carries a recognized code or status keeps
|
|
672
|
+
* its own text and routing.
|
|
673
|
+
*/
|
|
674
|
+
function isUnclassifiedSetupFailure(
|
|
675
|
+
message: AssistantMessage,
|
|
676
|
+
failure: ProviderFailureSignal,
|
|
677
|
+
): boolean {
|
|
678
|
+
try {
|
|
679
|
+
if (message.stopReason !== "error") return false;
|
|
680
|
+
if (!Array.isArray(message.content) || message.content.length !== 0) return false;
|
|
681
|
+
const diagnostics = (message as { diagnostics?: unknown }).diagnostics;
|
|
682
|
+
if (diagnostics !== undefined && !(Array.isArray(diagnostics) && diagnostics.length === 0)) {
|
|
683
|
+
return false;
|
|
684
|
+
}
|
|
685
|
+
if ((message as { code?: unknown }).code !== undefined) return false;
|
|
686
|
+
const usage = projectTerminalUsage(message);
|
|
687
|
+
if (
|
|
688
|
+
usage === undefined ||
|
|
689
|
+
usage.input !== 0 ||
|
|
690
|
+
usage.output !== 0 ||
|
|
691
|
+
usage.cacheRead !== 0 ||
|
|
692
|
+
usage.cacheWrite !== 0 ||
|
|
693
|
+
usage.totalTokens !== 0 ||
|
|
694
|
+
usage.cost.total !== 0
|
|
695
|
+
) {
|
|
696
|
+
return false;
|
|
697
|
+
}
|
|
698
|
+
return (
|
|
699
|
+
failure.code === undefined &&
|
|
700
|
+
failure.httpStatus === undefined &&
|
|
701
|
+
failure.transportKind === undefined
|
|
702
|
+
);
|
|
703
|
+
} catch {
|
|
704
|
+
return false;
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
|
|
545
708
|
function safeProjectFailureSignal(
|
|
546
709
|
error: unknown,
|
|
547
710
|
modelId: string,
|
|
@@ -584,6 +747,128 @@ interface HostRetryCooldownReceipt {
|
|
|
584
747
|
readonly rollback: () => void;
|
|
585
748
|
}
|
|
586
749
|
|
|
750
|
+
/** Raised inside one physical attempt when its provider stops producing events. */
|
|
751
|
+
class PhysicalAttemptStall extends Error {
|
|
752
|
+
/** Read by `projectFailureSignal`: a stall cools and classifies as a timeout. */
|
|
753
|
+
readonly transportKind = "connection-timeout";
|
|
754
|
+
constructor() {
|
|
755
|
+
super("the provider stream stalled");
|
|
756
|
+
this.name = "PhysicalAttemptStall";
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
/**
|
|
761
|
+
* Longest silence on a live connection, keep-alive pings included, once the
|
|
762
|
+
* provider has reported transport activity. Anthropic pings roughly every 30
|
|
763
|
+
* seconds while a model thinks silently, so three missed pings mean the
|
|
764
|
+
* connection is gone rather than the model being slow.
|
|
765
|
+
*/
|
|
766
|
+
export const TRANSPORT_SILENCE_TIMEOUT_MS = 90_000;
|
|
767
|
+
|
|
768
|
+
interface StallGuardTiming {
|
|
769
|
+
/** Until the provider's `start` event, opening included. */
|
|
770
|
+
readonly timeoutMs: number;
|
|
771
|
+
/** After `start` while the provider reports no transport activity. */
|
|
772
|
+
readonly startedTimeoutMs: number;
|
|
773
|
+
/** Without a byte once the provider has reported transport activity. */
|
|
774
|
+
readonly silenceTimeoutMs: number;
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
/**
|
|
778
|
+
* One physical stream, ended by `PhysicalAttemptStall` when the provider leaves
|
|
779
|
+
* a single wait unanswered for too long: `timeoutMs` until the provider's
|
|
780
|
+
* `start` event (opening included), then `startedTimeoutMs`.
|
|
781
|
+
*
|
|
782
|
+
* After `start` the server has answered and keeps the connection alive with
|
|
783
|
+
* pings the event stream never surfaces. A model that thinks silently (omitted
|
|
784
|
+
* thinking display, interleaved thinking) can leave several minutes between
|
|
785
|
+
* two events, so the short limit would abort a healthy request.
|
|
786
|
+
*
|
|
787
|
+
* A provider that reports transport activity (`activity`, called for every
|
|
788
|
+
* response chunk including pings) proves liveness directly: each report
|
|
789
|
+
* restarts the current wait, and after the first report a silence longer than
|
|
790
|
+
* `silenceTimeoutMs` ends the attempt. A connection that died after `start`
|
|
791
|
+
* is then caught in seconds instead of at the whole-call limit.
|
|
792
|
+
*
|
|
793
|
+
* The timer runs only while this wrapper waits on the provider, so a slow
|
|
794
|
+
* consumer never counts as a stall. `onStall` aborts the physical request
|
|
795
|
+
* before the stall is raised; the abandoned source is closed without waiting.
|
|
796
|
+
*/
|
|
797
|
+
function stallGuarded(
|
|
798
|
+
open: () => Promise<AsyncIterable<unknown>>,
|
|
799
|
+
timing: StallGuardTiming,
|
|
800
|
+
clock: RecoveryClock,
|
|
801
|
+
onStall: () => void,
|
|
802
|
+
): { readonly output: AsyncIterable<unknown>; readonly activity: () => void } {
|
|
803
|
+
let started = false;
|
|
804
|
+
let reportsActivity = false;
|
|
805
|
+
let rearm: (() => void) | undefined;
|
|
806
|
+
const limit = (): number => {
|
|
807
|
+
const phaseLimit = started ? timing.startedTimeoutMs : timing.timeoutMs;
|
|
808
|
+
return reportsActivity ? Math.min(timing.silenceTimeoutMs, phaseLimit) : phaseLimit;
|
|
809
|
+
};
|
|
810
|
+
const activity = (): void => {
|
|
811
|
+
reportsActivity = true;
|
|
812
|
+
rearm?.();
|
|
813
|
+
};
|
|
814
|
+
const output: AsyncIterable<unknown> = {
|
|
815
|
+
async *[Symbol.asyncIterator]() {
|
|
816
|
+
let iterator: AsyncIterator<unknown> | undefined;
|
|
817
|
+
let finished = false;
|
|
818
|
+
const guarded = async <T>(pending: Promise<T>): Promise<T> => {
|
|
819
|
+
void pending.catch(() => {});
|
|
820
|
+
let timer: RecoveryTimer | undefined;
|
|
821
|
+
let fail!: (error: PhysicalAttemptStall) => void;
|
|
822
|
+
const stalled = new Promise<never>((_resolve, reject) => {
|
|
823
|
+
fail = reject;
|
|
824
|
+
});
|
|
825
|
+
void stalled.catch(() => {});
|
|
826
|
+
const arm = (): void => {
|
|
827
|
+
timer?.cancel();
|
|
828
|
+
timer = clock.setTimer(limit(), () => {
|
|
829
|
+
rearm = undefined;
|
|
830
|
+
try {
|
|
831
|
+
onStall();
|
|
832
|
+
} finally {
|
|
833
|
+
fail(new PhysicalAttemptStall());
|
|
834
|
+
}
|
|
835
|
+
});
|
|
836
|
+
};
|
|
837
|
+
arm();
|
|
838
|
+
rearm = arm;
|
|
839
|
+
try {
|
|
840
|
+
return await Promise.race([pending, stalled]);
|
|
841
|
+
} finally {
|
|
842
|
+
if (rearm === arm) rearm = undefined;
|
|
843
|
+
timer?.cancel();
|
|
844
|
+
}
|
|
845
|
+
};
|
|
846
|
+
try {
|
|
847
|
+
const source = await guarded(open());
|
|
848
|
+
iterator = source[Symbol.asyncIterator]();
|
|
849
|
+
for (;;) {
|
|
850
|
+
const step = await guarded(iterator.next());
|
|
851
|
+
if (step.done === true) {
|
|
852
|
+
finished = true;
|
|
853
|
+
return;
|
|
854
|
+
}
|
|
855
|
+
if (isRecoveryStartEvent(step.value)) started = true;
|
|
856
|
+
yield step.value;
|
|
857
|
+
}
|
|
858
|
+
} finally {
|
|
859
|
+
if (!finished && iterator !== undefined) {
|
|
860
|
+
try {
|
|
861
|
+
void Promise.resolve(iterator.return?.()).catch(() => {});
|
|
862
|
+
} catch {
|
|
863
|
+
// The abandoned source is already aborted.
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
},
|
|
868
|
+
};
|
|
869
|
+
return { output, activity };
|
|
870
|
+
}
|
|
871
|
+
|
|
587
872
|
/** Synchronous, cooldown-only bookkeeping shared with host retry selection. */
|
|
588
873
|
export interface HostRetryCoordinator {
|
|
589
874
|
readonly state: RuntimeState;
|
|
@@ -678,17 +963,34 @@ export function createLogicalProvider(
|
|
|
678
963
|
): LogicalProvider {
|
|
679
964
|
const coordinator = createHostRetryCoordinator(deps);
|
|
680
965
|
let codexTransportNoticeSent = false;
|
|
966
|
+
let shutDown = false;
|
|
967
|
+
/** Aborts every physical request, including attempts the engine handed over. */
|
|
968
|
+
const shutdownController = new AbortController();
|
|
681
969
|
|
|
682
970
|
const diagnose = (message: string): void => {
|
|
683
971
|
deps.onDiagnostic?.(message);
|
|
684
972
|
};
|
|
685
973
|
|
|
686
|
-
const selectAccount = (
|
|
687
|
-
modelId
|
|
688
|
-
|
|
689
|
-
|
|
974
|
+
const selectAccount = (modelId: string): LogicalServingAccount => {
|
|
975
|
+
const snapshot = deps.captureSelectionSnapshot?.(modelId);
|
|
976
|
+
const accounts = snapshot?.accounts ?? deps.accounts;
|
|
977
|
+
const tierModelMap = deps.tierModelMap ?? DEFAULT_CONFIG.tierModelMap;
|
|
978
|
+
const nowMs = Date.now();
|
|
979
|
+
const failureMessage = (fallback: string): string => {
|
|
980
|
+
const group = snapshot?.group;
|
|
981
|
+
if (group === undefined) return fallback;
|
|
982
|
+
const candidates = group.allAccounts.map((account): AccountGroupFailureCandidate => ({
|
|
983
|
+
providerId: account.providerId,
|
|
984
|
+
servesModel: modelVendor !== undefined && vendorForFamily(account.family) === modelVendor && resolveLogicalServingAccount(account, modelId, tierModelMap) !== undefined,
|
|
985
|
+
eligible: logicalAccountEligible(account, coordinator.state, nowMs),
|
|
986
|
+
...(account.authenticated === undefined ? {} : { authenticated: account.authenticated }),
|
|
987
|
+
...(account.exhausted === undefined ? {} : { exhausted: account.exhausted }),
|
|
988
|
+
coolingDown: coordinator.state.peekCooldown(account.providerId, nowMs) !== undefined,
|
|
989
|
+
}));
|
|
990
|
+
return formatAccountGroupFailure({ policy: group.policy, modelId, accountLimit: group.accountLimit, candidates: [...candidates, ...group.otherCandidates] })?.message ?? fallback;
|
|
991
|
+
};
|
|
690
992
|
const subscriptionVendors = new Set(
|
|
691
|
-
|
|
993
|
+
accounts
|
|
692
994
|
.filter(
|
|
693
995
|
(account) =>
|
|
694
996
|
logicalProviderType(account) === "subscription" &&
|
|
@@ -712,12 +1014,11 @@ export function createLogicalProvider(
|
|
|
712
1014
|
);
|
|
713
1015
|
}
|
|
714
1016
|
throw new Error(
|
|
715
|
-
`no managed account serves the exact model id ${modelId}
|
|
1017
|
+
failureMessage(`no managed account serves the exact model id ${modelId}`),
|
|
716
1018
|
);
|
|
717
1019
|
}
|
|
718
1020
|
|
|
719
|
-
const
|
|
720
|
-
const serving = deps.accounts.flatMap((account) => {
|
|
1021
|
+
const serving = accounts.flatMap((account) => {
|
|
721
1022
|
if (vendorForFamily(account.family) !== modelVendor) return [];
|
|
722
1023
|
const resolved = resolveLogicalServingAccount(account, modelId, tierModelMap);
|
|
723
1024
|
return resolved === undefined ? [] : [resolved];
|
|
@@ -727,7 +1028,7 @@ export function createLogicalProvider(
|
|
|
727
1028
|
// exact-identity contract: a virtual id that is neither exact nor explicitly
|
|
728
1029
|
// mapped to a catalog member must never reach a physical provider.
|
|
729
1030
|
throw new Error(
|
|
730
|
-
`no managed account serves the exact model id ${modelId}
|
|
1031
|
+
failureMessage(`no managed account serves the exact model id ${modelId}`),
|
|
731
1032
|
);
|
|
732
1033
|
}
|
|
733
1034
|
|
|
@@ -735,7 +1036,6 @@ export function createLogicalProvider(
|
|
|
735
1036
|
// (`openai`) accounts are one routing family: vendor `openai`. Ownership is
|
|
736
1037
|
// fixed from the declared subscription catalog before API exact/map matches are
|
|
737
1038
|
// admitted, so another vendor's API catalog cannot make a row ambiguous.
|
|
738
|
-
const nowMs = Date.now();
|
|
739
1039
|
const eligible = serving.filter(({ account }) =>
|
|
740
1040
|
logicalAccountEligible(
|
|
741
1041
|
{
|
|
@@ -750,31 +1050,6 @@ export function createLogicalProvider(
|
|
|
750
1050
|
nowMs,
|
|
751
1051
|
),
|
|
752
1052
|
);
|
|
753
|
-
if (consumeRoutePin) {
|
|
754
|
-
const pin = deps.routePin?.get();
|
|
755
|
-
if (pin !== undefined) {
|
|
756
|
-
const pinned = eligible.find(
|
|
757
|
-
({ account, resolvedModelId }) =>
|
|
758
|
-
account.providerId === pin.destinationProviderId &&
|
|
759
|
-
account.family === pin.destinationFamily &&
|
|
760
|
-
logicalProviderType(account) === "subscription" &&
|
|
761
|
-
resolvedModelId === modelId &&
|
|
762
|
-
pin.requestedModelId === modelId,
|
|
763
|
-
);
|
|
764
|
-
if (pinned === undefined) {
|
|
765
|
-
deps.routePin?.clear();
|
|
766
|
-
} else {
|
|
767
|
-
const consumed = deps.routePin?.consume(pin.generation, modelId);
|
|
768
|
-
if (
|
|
769
|
-
consumed !== undefined &&
|
|
770
|
-
consumed.destinationProviderId === pinned.account.providerId &&
|
|
771
|
-
consumed.destinationFamily === pinned.account.family
|
|
772
|
-
) {
|
|
773
|
-
return pinned;
|
|
774
|
-
}
|
|
775
|
-
}
|
|
776
|
-
}
|
|
777
|
-
}
|
|
778
1053
|
eligible.sort(
|
|
779
1054
|
(left, right) =>
|
|
780
1055
|
tierRank(logicalProviderType(left.account)) -
|
|
@@ -786,48 +1061,19 @@ export function createLogicalProvider(
|
|
|
786
1061
|
// ineligible. The turn is refused before any physical request is issued;
|
|
787
1062
|
// it is not parked, and no account is mutated by having been asked.
|
|
788
1063
|
throw new Error(
|
|
789
|
-
`every managed account serving ${modelId} is currently ineligible
|
|
1064
|
+
failureMessage(`every managed account serving ${modelId} is currently ineligible`),
|
|
790
1065
|
);
|
|
791
1066
|
}
|
|
792
1067
|
return chosen;
|
|
793
1068
|
};
|
|
794
1069
|
|
|
795
1070
|
/**
|
|
796
|
-
*
|
|
1071
|
+
* The terminal a physical event carries, if any.
|
|
797
1072
|
*
|
|
798
1073
|
* A provider can accept a request, return 200, and fail part-way through the
|
|
799
|
-
* stream
|
|
800
|
-
*
|
|
801
|
-
*
|
|
802
|
-
* healthy and the host would retry straight back onto it.
|
|
803
|
-
*
|
|
804
|
-
* It also guarantees the host sees a terminal event.
|
|
805
|
-
*
|
|
806
|
-
* The host settles a turn only on a `done` or `error` event: `EventStream`
|
|
807
|
-
* resolves its final result from `isComplete(event)` on push, and the
|
|
808
|
-
* `forwardStream` adapter that wraps every provider stream ends with
|
|
809
|
-
* `end(undefined)` when the source exposes no `result()`, which the
|
|
810
|
-
* `result !== undefined` guard makes a no-op. A dispatched stream that simply
|
|
811
|
-
* runs out therefore leaves the caller's `prompt()` pending forever, with no
|
|
812
|
-
* timeout anywhere to break it.
|
|
813
|
-
*
|
|
814
|
-
* `deps.dispatch` is caller-supplied, so that is a careless caller wedging the
|
|
815
|
-
* host permanently. Real Pi streams always terminate, so this is defense in
|
|
816
|
-
* depth at an injectable boundary rather than a repair of a reachable hang.
|
|
817
|
-
*
|
|
818
|
-
* The synthesized event is `error`, never `done`. A stream that stopped
|
|
819
|
-
* without terminating did not answer, and reporting `done` would invent an
|
|
820
|
-
* answer that never arrived.
|
|
821
|
-
*
|
|
822
|
-
* It records no cooldown. Not terminating is a protocol fault in the dispatch
|
|
823
|
-
* implementation, not evidence the account is rate limited, and
|
|
824
|
-
* `classifyFailure` would read an unrecognizable failure as
|
|
825
|
-
* `cooldown("unknown", …)` — cooling a healthy account for someone else's bug.
|
|
826
|
-
* A genuinely failing account still throws, and the catch below cools it.
|
|
827
|
-
*
|
|
828
|
-
* `result()` is deliberately not forwarded. Exposing it would make
|
|
829
|
-
* `forwardStream` take its `await source.result()` branch, which for exactly
|
|
830
|
-
* this non-terminating source never settles.
|
|
1074
|
+
* stream (Anthropic emits `event: error` that way). That failure is exactly as
|
|
1075
|
+
* real as a rejected dispatch, so it records exactly the same cooldown before
|
|
1076
|
+
* the engine decides whether the call may move to another account.
|
|
831
1077
|
*/
|
|
832
1078
|
const terminalAttribution = (
|
|
833
1079
|
event: unknown,
|
|
@@ -904,182 +1150,760 @@ export function createLogicalProvider(
|
|
|
904
1150
|
code: "quota_exhausted",
|
|
905
1151
|
});
|
|
906
1152
|
|
|
907
|
-
|
|
1153
|
+
/**
|
|
1154
|
+
* In-call retry safety for one failed physical terminal.
|
|
1155
|
+
*
|
|
1156
|
+
* Only an account-local quota, authentication or rate-limit failure may move
|
|
1157
|
+
* the call to another account serving the same model. A structured refusal or
|
|
1158
|
+
* unknown stop, an invalid request, a context overflow and every unclassified
|
|
1159
|
+
* failure end the call. The classification reads structured fields only.
|
|
1160
|
+
*/
|
|
1161
|
+
/**
|
|
1162
|
+
* Record a local host fault: no provider was reached, so the account is not
|
|
1163
|
+
* cooled. The bounded cause goes to the diagnostic sink, never to the reply.
|
|
1164
|
+
*/
|
|
1165
|
+
const markHostStaleInstall = (
|
|
1166
|
+
box: AttemptRecordBox,
|
|
1167
|
+
account: LogicalPhysicalAccount,
|
|
1168
|
+
cause: unknown,
|
|
1169
|
+
): void => {
|
|
1170
|
+
box.hostStaleInstall = true;
|
|
1171
|
+
box.overflow = false;
|
|
1172
|
+
box.preStartRetryable = false;
|
|
908
1173
|
try {
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
? error
|
|
914
|
-
: String(error);
|
|
915
|
-
return isRetryableAssistantError({
|
|
916
|
-
stopReason: "error",
|
|
917
|
-
errorMessage,
|
|
918
|
-
} as AssistantMessage);
|
|
1174
|
+
deps.onDiagnostic?.(
|
|
1175
|
+
`logical dispatch for ${account.providerId} failed loading host code (host_stale_install: ` +
|
|
1176
|
+
`${hostStaleInstallCause(cause)}); restart the pi session`,
|
|
1177
|
+
);
|
|
919
1178
|
} catch {
|
|
920
|
-
|
|
1179
|
+
// A diagnostic sink failure cannot replace a provider result.
|
|
921
1180
|
}
|
|
922
1181
|
};
|
|
1182
|
+
/** The account did nothing wrong; report it as handled so nothing cools it. */
|
|
1183
|
+
const hostFaultReceipt = (): HostRetryCooldownReceipt => ({ alreadyCooled: true, rollback: () => {} });
|
|
923
1184
|
|
|
924
|
-
const
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
1185
|
+
const retrySafetyFor = (box: AttemptRecordBox): RecoveryRetrySafety => {
|
|
1186
|
+
const { physicalTerminal: message, failure, overflow } = box;
|
|
1187
|
+
if (box.invalidated) return { status: "unsafe", reason: "unknown" };
|
|
1188
|
+
// Every same-family account loads the same missing host chunk: another send cannot help.
|
|
1189
|
+
if (box.hostStaleInstall === true) return { status: "unsafe", reason: "unknown" };
|
|
1190
|
+
const code = (message as { code?: unknown } | undefined)?.code;
|
|
1191
|
+
if (code === "refusal") return { status: "unsafe", reason: "refusal" };
|
|
1192
|
+
if (code === "unknown_stop") return { status: "unsafe", reason: "unknown" };
|
|
1193
|
+
if (overflow) return { status: "unsafe", reason: "invalid-request" };
|
|
1194
|
+
// Nothing after any output is ever sent again; the engine enforces the
|
|
1195
|
+
// same rule from the events it observed.
|
|
1196
|
+
if (box.sawOutput || (Array.isArray(message?.content) && message.content.length > 0)) {
|
|
1197
|
+
return { status: "unsafe", reason: "unknown" };
|
|
1198
|
+
}
|
|
1199
|
+
if (failure === undefined) return { status: "unsafe", reason: "unknown" };
|
|
1200
|
+
const category = classifyFailure(failure).category;
|
|
1201
|
+
if (category === "quota-rate-limit") {
|
|
1202
|
+
return {
|
|
1203
|
+
status: "recoverable",
|
|
1204
|
+
action: "account",
|
|
1205
|
+
reason:
|
|
1206
|
+
failure.code === "quota_exhausted"
|
|
1207
|
+
? "account-local-quota"
|
|
1208
|
+
: "account-local-rate-limit",
|
|
1209
|
+
};
|
|
1210
|
+
}
|
|
1211
|
+
if (category === "terminal-auth" || category === "transient-auth") {
|
|
1212
|
+
return { status: "recoverable", action: "account", reason: "account-local-auth" };
|
|
1213
|
+
}
|
|
1214
|
+
if (category === "config") return { status: "unsafe", reason: "invalid-request" };
|
|
1215
|
+
if (category === "transport" || box.preStartRetryable) {
|
|
1216
|
+
// A network or server failure before any content: the provider never
|
|
1217
|
+
// produced output, so one other account may serve the same request.
|
|
1218
|
+
return { status: "recoverable", action: "account", reason: "pre-start-transient" };
|
|
1219
|
+
}
|
|
1220
|
+
return { status: "unsafe", reason: "unknown" };
|
|
947
1221
|
};
|
|
948
1222
|
|
|
949
|
-
const projectMessage = (message: AssistantMessage, modelId: string): AssistantMessage =>
|
|
950
|
-
|
|
951
|
-
...hostFinalStopMessage(message),
|
|
952
|
-
});
|
|
1223
|
+
const projectMessage = (message: AssistantMessage, modelId: string): AssistantMessage =>
|
|
1224
|
+
projectPublicAssistantMessage(message, { api: LOGICAL_PROVIDER_ID, provider: LOGICAL_PROVIDER_ID, model: modelId });
|
|
953
1225
|
|
|
954
|
-
const projectEvent = (event: unknown, modelId: string): unknown =>
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
1226
|
+
const projectEvent = (event: unknown, modelId: string): unknown =>
|
|
1227
|
+
projectPublicAssistantEvent(event, { api: LOGICAL_PROVIDER_ID, provider: LOGICAL_PROVIDER_ID, model: modelId });
|
|
1228
|
+
|
|
1229
|
+
/** Outcome of one private physical attempt, recorded for the logical call. */
|
|
1230
|
+
interface AttemptRecordBox {
|
|
1231
|
+
readonly ordinal: number;
|
|
1232
|
+
physicalTerminal?: AssistantMessage;
|
|
1233
|
+
failure?: ProviderFailureSignal;
|
|
1234
|
+
overflow: boolean;
|
|
1235
|
+
/**
|
|
1236
|
+
* A non-terminal event other than `start` reached the engine; nothing
|
|
1237
|
+
* after it is resent. The provider's `start` only reports that response
|
|
1238
|
+
* headers arrived, so it is not output.
|
|
1239
|
+
*/
|
|
1240
|
+
sawOutput: boolean;
|
|
1241
|
+
/**
|
|
1242
|
+
* The attempt's first `start`, held by the engine until the first content.
|
|
1243
|
+
* A successful terminal with no content publishes this exact event.
|
|
1244
|
+
*/
|
|
1245
|
+
heldStart?: unknown;
|
|
1246
|
+
/** A setup-shaped first terminal whose raw text the host would retry. */
|
|
1247
|
+
preStartRetryable: boolean;
|
|
1248
|
+
/** Setup failed because this pi process could not load its own code. */
|
|
1249
|
+
hostStaleInstall?: boolean;
|
|
1250
|
+
/** Whether the caller or provider shutdown aborted this physical request. */
|
|
1251
|
+
cancelled?: () => boolean;
|
|
1252
|
+
/**
|
|
1253
|
+
* The session invalidated this attempt before its terminal (fresh input or
|
|
1254
|
+
* an operator stop, reset or enable): its cooldown was rolled back, so it
|
|
1255
|
+
* must end the call rather than buy another send.
|
|
1256
|
+
*/
|
|
1257
|
+
invalidated: boolean;
|
|
966
1258
|
|
|
967
|
-
|
|
1259
|
+
receipt?: HostRetryCooldownReceipt;
|
|
1260
|
+
attempt: LogicalAttributionAttempt;
|
|
1261
|
+
account: LogicalPhysicalAccount;
|
|
1262
|
+
}
|
|
1263
|
+
|
|
1264
|
+
/**
|
|
1265
|
+
* Observe one physical attempt privately.
|
|
1266
|
+
*
|
|
1267
|
+
* Every event is forwarded unchanged to the engine's accepted-output buffer,
|
|
1268
|
+
* which publishes nothing until a successful terminal. A failure records the
|
|
1269
|
+
* account cooldown before the engine decides whether to recover, so the second
|
|
1270
|
+
* attempt never reselects the failing account. A stream that runs out without
|
|
1271
|
+
* a terminal yields one synthetic error terminal instead of hanging.
|
|
1272
|
+
*/
|
|
1273
|
+
const observePhysicalAttempt = (
|
|
968
1274
|
stream: AsyncIterable<unknown>,
|
|
969
1275
|
model: unknown,
|
|
970
1276
|
options: SimpleStreamOptions | undefined,
|
|
971
|
-
|
|
1277
|
+
box: AttemptRecordBox,
|
|
972
1278
|
requestedModelId: string,
|
|
973
1279
|
dispatchedModelId: string,
|
|
974
|
-
attempt: LogicalAttributionAttempt,
|
|
975
1280
|
): AsyncIterable<unknown> => ({
|
|
976
1281
|
async *[Symbol.asyncIterator]() {
|
|
1282
|
+
const { account, attempt } = box;
|
|
977
1283
|
let sawTerminal = false;
|
|
978
|
-
|
|
1284
|
+
// Whether any event other than a leading `start` arrived. A `start`
|
|
1285
|
+
// only reports that response headers arrived, so a failure after it is
|
|
1286
|
+
// still the stream's first real event.
|
|
1287
|
+
let sawEvent = false;
|
|
979
1288
|
const recordFailureOnce = (error: unknown): HostRetryCooldownReceipt => {
|
|
980
|
-
|
|
1289
|
+
if (box.hostStaleInstall === true) box.receipt ??= hostFaultReceipt();
|
|
1290
|
+
box.receipt ??= coordinator.recordFailure({
|
|
981
1291
|
account,
|
|
982
1292
|
requestedModelId,
|
|
983
1293
|
dispatchedModelId,
|
|
984
1294
|
error,
|
|
985
1295
|
});
|
|
986
|
-
return
|
|
1296
|
+
return box.receipt;
|
|
987
1297
|
};
|
|
988
|
-
const attributeFailure = (
|
|
989
|
-
message
|
|
990
|
-
failure
|
|
991
|
-
|
|
992
|
-
): void => {
|
|
993
|
-
// Host retry needs the cooldown before the terminal reaches Pi, while
|
|
994
|
-
// exact identity cannot be accepted until association/message_end. The
|
|
995
|
-
// receipt makes the early write provisional and reverses it on uncertainty.
|
|
1298
|
+
const attributeFailure = (message: AssistantMessage, failure: ProviderFailureSignal): void => {
|
|
1299
|
+
const receipt = recordFailureOnce(message);
|
|
1300
|
+
box.failure = failure;
|
|
1301
|
+
box.physicalTerminal = message;
|
|
996
1302
|
try {
|
|
997
1303
|
const accepted = attempt.fail(message, {
|
|
998
1304
|
alreadyCooled: receipt.alreadyCooled,
|
|
999
1305
|
dispatchedModelId,
|
|
1000
1306
|
failure,
|
|
1001
|
-
...(receipt.alreadyCooled
|
|
1002
|
-
? { rollbackCooldown: receipt.rollback }
|
|
1003
|
-
: {}),
|
|
1307
|
+
...(receipt.alreadyCooled ? { rollbackCooldown: receipt.rollback } : {}),
|
|
1004
1308
|
});
|
|
1005
|
-
if (accepted === false && deps.attribution !== undefined)
|
|
1309
|
+
if (accepted === false && deps.attribution !== undefined) {
|
|
1310
|
+
receipt.rollback();
|
|
1311
|
+
box.invalidated = true;
|
|
1312
|
+
}
|
|
1006
1313
|
} catch {
|
|
1007
|
-
if (deps.attribution !== undefined)
|
|
1314
|
+
if (deps.attribution !== undefined) {
|
|
1315
|
+
receipt.rollback();
|
|
1316
|
+
box.invalidated = true;
|
|
1317
|
+
}
|
|
1008
1318
|
}
|
|
1009
1319
|
};
|
|
1010
1320
|
try {
|
|
1011
|
-
// Keep physical events intact. A corroborated subscription-exhaustion
|
|
1012
|
-
// length terminal still becomes a bounded retryable public error.
|
|
1013
1321
|
for await (const event of stream) {
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1322
|
+
if (!sawEvent && isRecoveryStartEvent(event)) {
|
|
1323
|
+
box.heldStart ??= event;
|
|
1324
|
+
yield event;
|
|
1325
|
+
continue;
|
|
1326
|
+
}
|
|
1327
|
+
const firstEvent = !sawEvent;
|
|
1328
|
+
sawEvent = true;
|
|
1019
1329
|
const terminal = terminalAttribution(event);
|
|
1020
|
-
if (terminal
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
const failure = safeProjectFailureSignal(message, dispatchedModelId);
|
|
1050
|
-
attributeFailure(message, failure, recordFailureOnce(message));
|
|
1330
|
+
if (terminal === undefined) {
|
|
1331
|
+
box.sawOutput = true;
|
|
1332
|
+
yield event;
|
|
1333
|
+
continue;
|
|
1334
|
+
}
|
|
1335
|
+
sawTerminal = true;
|
|
1336
|
+
const { message, outcome } = terminal;
|
|
1337
|
+
if (message === box.physicalTerminal) {
|
|
1338
|
+
// A rejected dispatch: its private terminal was already cooled,
|
|
1339
|
+
// attributed and classified from the thrown error itself.
|
|
1340
|
+
yield event;
|
|
1341
|
+
return;
|
|
1342
|
+
}
|
|
1343
|
+
if (outcome === "finish") {
|
|
1344
|
+
const match = exhaustionLengthMatch({
|
|
1345
|
+
message,
|
|
1346
|
+
model,
|
|
1347
|
+
options,
|
|
1348
|
+
account,
|
|
1349
|
+
currentAccounts: () => deps.accounts,
|
|
1350
|
+
});
|
|
1351
|
+
if (match !== undefined) {
|
|
1352
|
+
// A corroborated subscription-exhaustion length terminal is an
|
|
1353
|
+
// account-local quota failure, never an accepted answer.
|
|
1354
|
+
const replacement = exhaustionLengthError(requestedModelId, match);
|
|
1355
|
+
attributeFailure(replacement, safeProjectFailureSignal(replacement, dispatchedModelId));
|
|
1356
|
+
await attempt.waitForTerminal();
|
|
1357
|
+
yield { type: "error", reason: "error", error: replacement };
|
|
1358
|
+
return;
|
|
1051
1359
|
}
|
|
1360
|
+
box.physicalTerminal = message;
|
|
1361
|
+
safeAttributionCall(() => attempt.finish(message));
|
|
1362
|
+
await attempt.waitForTerminal();
|
|
1363
|
+
yield event;
|
|
1364
|
+
return;
|
|
1365
|
+
}
|
|
1366
|
+
if (outcome === "abort") {
|
|
1367
|
+
box.physicalTerminal = message;
|
|
1368
|
+
safeAttributionCall(() => attempt.abort(message));
|
|
1052
1369
|
await attempt.waitForTerminal();
|
|
1370
|
+
yield event;
|
|
1371
|
+
return;
|
|
1053
1372
|
}
|
|
1054
|
-
const
|
|
1055
|
-
if (
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1373
|
+
const failure = safeProjectFailureSignal(message, dispatchedModelId);
|
|
1374
|
+
if (firstEvent && isUnclassifiedSetupFailure(message, failure) && isHostStaleInstallFailure(message)) {
|
|
1375
|
+
markHostStaleInstall(box, account, message.errorMessage);
|
|
1376
|
+
} else if (firstEvent && isUnclassifiedSetupFailure(message, failure)) {
|
|
1377
|
+
// The raw setup text never leaves this attempt. Only its overflow
|
|
1378
|
+
// form changes the outcome: the host must still compact.
|
|
1379
|
+
const disposition = setupFailureDisposition(message, message.errorMessage);
|
|
1380
|
+
box.overflow = disposition === "context-overflow";
|
|
1381
|
+
// "fetch failed", a reset socket or a 503 before `start`: nothing
|
|
1382
|
+
// was produced, so one other account may serve the request.
|
|
1383
|
+
box.preStartRetryable = disposition === "retryable";
|
|
1384
|
+
try {
|
|
1385
|
+
deps.onDiagnostic?.(
|
|
1386
|
+
`logical dispatch for ${account.providerId} failed during stream setup; ` +
|
|
1387
|
+
"the raw setup error was not published",
|
|
1388
|
+
);
|
|
1389
|
+
} catch {
|
|
1390
|
+
// A diagnostic sink failure cannot replace a provider result.
|
|
1060
1391
|
}
|
|
1392
|
+
} else if (setupFailureDisposition(message, message.errorMessage) === "context-overflow") {
|
|
1393
|
+
box.overflow = (message as { code?: unknown }).code === undefined;
|
|
1061
1394
|
}
|
|
1062
|
-
|
|
1395
|
+
attributeFailure(message, failure);
|
|
1396
|
+
await attempt.waitForTerminal();
|
|
1397
|
+
yield event;
|
|
1398
|
+
return;
|
|
1063
1399
|
}
|
|
1064
1400
|
} catch (error) {
|
|
1401
|
+
// A thrown stream failure becomes this attempt's own private terminal,
|
|
1402
|
+
// so it is attributed and accounted exactly like an error event.
|
|
1403
|
+
if (
|
|
1404
|
+
box.physicalTerminal === undefined &&
|
|
1405
|
+
!sawEvent &&
|
|
1406
|
+
!hasStructuredFailureEvidence(safeProjectFailureSignal(error, dispatchedModelId)) &&
|
|
1407
|
+
isHostStaleInstallFailure(error)
|
|
1408
|
+
) {
|
|
1409
|
+
markHostStaleInstall(box, account, error);
|
|
1410
|
+
}
|
|
1065
1411
|
recordFailureOnce(error);
|
|
1412
|
+
if (box.hostStaleInstall === true) {
|
|
1413
|
+
// Already classified above; no provider disposition applies.
|
|
1414
|
+
} else if (box.physicalTerminal === undefined && !sawEvent) {
|
|
1415
|
+
// Thrown before any content (a reset socket after the response
|
|
1416
|
+
// headers, say): read like a failure while opening the stream.
|
|
1417
|
+
const raw =
|
|
1418
|
+
typeof error === "object" && error !== null
|
|
1419
|
+
? (error as { message?: unknown }).message
|
|
1420
|
+
: undefined;
|
|
1421
|
+
const disposition = setupFailureDisposition(
|
|
1422
|
+
syntheticErrorMessage(requestedModelId, buildBoundedRecoveryFinalErrorMessage()),
|
|
1423
|
+
raw,
|
|
1424
|
+
);
|
|
1425
|
+
box.overflow = disposition === "context-overflow";
|
|
1426
|
+
box.preStartRetryable = disposition === "retryable";
|
|
1427
|
+
}
|
|
1428
|
+
if (box.physicalTerminal === undefined) {
|
|
1429
|
+
attributeFailure(
|
|
1430
|
+
syntheticErrorMessage(requestedModelId, buildBoundedRecoveryFinalErrorMessage()),
|
|
1431
|
+
safeProjectFailureSignal(error, dispatchedModelId),
|
|
1432
|
+
);
|
|
1433
|
+
}
|
|
1066
1434
|
throw error;
|
|
1067
1435
|
}
|
|
1068
1436
|
if (sawTerminal) return;
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1437
|
+
try {
|
|
1438
|
+
deps.onDiagnostic?.(
|
|
1439
|
+
`logical dispatch stream for ${account.providerId} ended with no terminal event; ` +
|
|
1440
|
+
"reporting a synthetic failure so the turn cannot hang",
|
|
1441
|
+
);
|
|
1442
|
+
} catch {
|
|
1443
|
+
// A diagnostic sink failure cannot replace a provider result.
|
|
1444
|
+
}
|
|
1073
1445
|
const syntheticMessage = syntheticErrorMessage(
|
|
1074
1446
|
requestedModelId,
|
|
1075
1447
|
"the dispatched stream ended without a terminal event",
|
|
1076
1448
|
);
|
|
1449
|
+
box.physicalTerminal = syntheticMessage;
|
|
1077
1450
|
safeAttributionCall(() => attempt.fail(syntheticMessage));
|
|
1078
1451
|
await attempt.waitForTerminal();
|
|
1079
1452
|
yield { type: "error", reason: "error", error: syntheticMessage };
|
|
1080
1453
|
},
|
|
1081
1454
|
});
|
|
1082
1455
|
|
|
1456
|
+
/** Build the request-local engine candidates for one logical call. */
|
|
1457
|
+
const recoveryCandidates = (
|
|
1458
|
+
modelId: string,
|
|
1459
|
+
selected: LogicalServingAccount,
|
|
1460
|
+
): readonly RecoveryCandidate[] => {
|
|
1461
|
+
const nowMs = Date.now();
|
|
1462
|
+
const tierModelMap = deps.tierModelMap ?? DEFAULT_CONFIG.tierModelMap;
|
|
1463
|
+
const vendor = vendorForFamily(selected.account.family);
|
|
1464
|
+
const capability = (id: string): RecoveryModelCapability => ({
|
|
1465
|
+
modelId: id,
|
|
1466
|
+
input: [],
|
|
1467
|
+
supportsTools: true,
|
|
1468
|
+
contextWindow: 1,
|
|
1469
|
+
});
|
|
1470
|
+
const planAccount = (
|
|
1471
|
+
account: LogicalPhysicalAccount,
|
|
1472
|
+
modelIds: readonly string[],
|
|
1473
|
+
): RecoveryPlanAccount => ({
|
|
1474
|
+
providerId: account.providerId,
|
|
1475
|
+
family: account.family,
|
|
1476
|
+
providerType: logicalProviderType(account),
|
|
1477
|
+
eligible:
|
|
1478
|
+
account.providerId === selected.account.providerId ||
|
|
1479
|
+
logicalAccountEligible(
|
|
1480
|
+
{
|
|
1481
|
+
providerId: account.providerId,
|
|
1482
|
+
exhausted: account.exhausted,
|
|
1483
|
+
authenticated: account.authenticated,
|
|
1484
|
+
},
|
|
1485
|
+
coordinator.state,
|
|
1486
|
+
nowMs,
|
|
1487
|
+
),
|
|
1488
|
+
models: modelIds.map(capability),
|
|
1489
|
+
});
|
|
1490
|
+
const originFamily: AllowedFamily =
|
|
1491
|
+
selected.account.family === "openai" ? "openai-codex" : selected.account.family;
|
|
1492
|
+
if (selected.resolvedModelId !== modelId) {
|
|
1493
|
+
// An operator tier map routes this call to a different physical id. That
|
|
1494
|
+
// is one non-recoverable attempt: the plan admits only the selected pair.
|
|
1495
|
+
return buildRecoveryCandidatePlan({
|
|
1496
|
+
selectedModelId: selected.resolvedModelId,
|
|
1497
|
+
originFamily,
|
|
1498
|
+
accounts: [planAccount(selected.account, [selected.resolvedModelId])],
|
|
1499
|
+
config: {
|
|
1500
|
+
sameFamilyFailover: true,
|
|
1501
|
+
crossFamilyChainEnabled: false,
|
|
1502
|
+
crossFamilyChains: [],
|
|
1503
|
+
preferredModels: {},
|
|
1504
|
+
tierModelMap: {},
|
|
1505
|
+
},
|
|
1506
|
+
requiredInput: [],
|
|
1507
|
+
requiresTools: false,
|
|
1508
|
+
});
|
|
1509
|
+
}
|
|
1510
|
+
// Same vendor only, exact model only: the one recovery action in this slice
|
|
1511
|
+
// is another account serving the selected model.
|
|
1512
|
+
const accounts = deps.accounts
|
|
1513
|
+
.filter(
|
|
1514
|
+
(account) =>
|
|
1515
|
+
vendorForFamily(account.family) === vendor &&
|
|
1516
|
+
resolveLogicalServingAccount(account, modelId, tierModelMap)?.resolvedModelId === modelId,
|
|
1517
|
+
)
|
|
1518
|
+
.map((account) => planAccount(account, [modelId]));
|
|
1519
|
+
const ordered = [
|
|
1520
|
+
...accounts.filter(({ providerId }) => providerId === selected.account.providerId),
|
|
1521
|
+
...accounts.filter(({ providerId }) => providerId !== selected.account.providerId),
|
|
1522
|
+
];
|
|
1523
|
+
// With same-family failover off, only the selected account is planned:
|
|
1524
|
+
// the call is one attempt.
|
|
1525
|
+
const sameFamilyFailover =
|
|
1526
|
+
deps.sameFamilyFailover ?? DEFAULT_CONFIG.sameFamilyFailover;
|
|
1527
|
+
const plan = buildRecoveryCandidatePlan({
|
|
1528
|
+
selectedModelId: modelId,
|
|
1529
|
+
originFamily,
|
|
1530
|
+
accounts: sameFamilyFailover
|
|
1531
|
+
? ordered
|
|
1532
|
+
: ordered.filter(({ providerId }) => providerId === selected.account.providerId),
|
|
1533
|
+
config: {
|
|
1534
|
+
sameFamilyFailover: true,
|
|
1535
|
+
crossFamilyChainEnabled: false,
|
|
1536
|
+
crossFamilyChains: [],
|
|
1537
|
+
preferredModels: {},
|
|
1538
|
+
tierModelMap: {},
|
|
1539
|
+
},
|
|
1540
|
+
requiredInput: [],
|
|
1541
|
+
requiresTools: false,
|
|
1542
|
+
});
|
|
1543
|
+
// The selected account leads: it is the initial send.
|
|
1544
|
+
return [
|
|
1545
|
+
...plan.filter(({ providerId }) => providerId === selected.account.providerId),
|
|
1546
|
+
...plan.filter(({ providerId }) => providerId !== selected.account.providerId),
|
|
1547
|
+
];
|
|
1548
|
+
};
|
|
1549
|
+
|
|
1550
|
+
const activeEngines = new Set<RecoveryEngine>();
|
|
1551
|
+
|
|
1552
|
+
/**
|
|
1553
|
+
* Open one physical attempt for the engine.
|
|
1554
|
+
*
|
|
1555
|
+
* A rejected dispatch is converted into one private error terminal so its
|
|
1556
|
+
* failure is classified exactly like a stream failure. The retry-safety
|
|
1557
|
+
* promise settles once the private attempt ends, from structured facts only.
|
|
1558
|
+
*/
|
|
1559
|
+
const physicalAttempt = (input: {
|
|
1560
|
+
readonly request: RecoveryDispatchRequest;
|
|
1561
|
+
readonly account: LogicalPhysicalAccount;
|
|
1562
|
+
readonly model: unknown;
|
|
1563
|
+
readonly context: unknown;
|
|
1564
|
+
readonly callerOptions: SimpleStreamOptions | undefined;
|
|
1565
|
+
readonly requestedModelId: string;
|
|
1566
|
+
readonly box: AttemptRecordBox;
|
|
1567
|
+
readonly clock: RecoveryClock;
|
|
1568
|
+
readonly stallTimeoutMs: number;
|
|
1569
|
+
readonly startedStallTimeoutMs: number;
|
|
1570
|
+
readonly transportSilenceTimeoutMs: number;
|
|
1571
|
+
}): RecoveryPhysicalAttempt => {
|
|
1572
|
+
const { request, account, box, requestedModelId } = input;
|
|
1573
|
+
const dispatchedModelId = request.candidate.modelId;
|
|
1574
|
+
let settleSafety!: (safety: RecoveryRetrySafety) => void;
|
|
1575
|
+
const retrySafety = new Promise<RecoveryRetrySafety>((resolve) => {
|
|
1576
|
+
settleSafety = resolve;
|
|
1577
|
+
});
|
|
1578
|
+
const settle = (): void => settleSafety(retrySafetyFor(box));
|
|
1579
|
+
request.signal.addEventListener("abort", settle, { once: true });
|
|
1580
|
+
// The physical request outlives the engine once the attempt commits, so it
|
|
1581
|
+
// also listens to the caller and to provider shutdown directly.
|
|
1582
|
+
const physicalController = new AbortController();
|
|
1583
|
+
const abortPhysical = (): void => physicalController.abort();
|
|
1584
|
+
const abortSources = [request.signal, input.callerOptions?.signal, shutdownController.signal]
|
|
1585
|
+
.filter((signal): signal is AbortSignal => signal !== undefined);
|
|
1586
|
+
for (const signal of abortSources) {
|
|
1587
|
+
if (signal.aborted) abortPhysical();
|
|
1588
|
+
else signal.addEventListener("abort", abortPhysical, { once: true });
|
|
1589
|
+
}
|
|
1590
|
+
box.cancelled = () =>
|
|
1591
|
+
input.callerOptions?.signal?.aborted === true || shutdownController.signal.aborted;
|
|
1592
|
+
// Idempotent. Runs at the attempt's terminal as well as in the output's
|
|
1593
|
+
// finally: a consumer may abandon a committed stream after its terminal
|
|
1594
|
+
// without ever returning the iterator, and the session-lifetime shutdown
|
|
1595
|
+
// signal must not keep this attempt's listener.
|
|
1596
|
+
let released = false;
|
|
1597
|
+
const releaseAbortSources = (): void => {
|
|
1598
|
+
if (released) return;
|
|
1599
|
+
released = true;
|
|
1600
|
+
request.signal.removeEventListener("abort", settle);
|
|
1601
|
+
for (const signal of abortSources) signal.removeEventListener("abort", abortPhysical);
|
|
1602
|
+
};
|
|
1603
|
+
const { attempt } = box;
|
|
1604
|
+
const originalOnPayload = input.callerOptions?.onPayload;
|
|
1605
|
+
const originalOnResponse = input.callerOptions?.onResponse;
|
|
1606
|
+
const wrappedOnPayload: NonNullable<SimpleStreamOptions["onPayload"]> = async (
|
|
1607
|
+
payload,
|
|
1608
|
+
payloadModel,
|
|
1609
|
+
) => {
|
|
1610
|
+
safeAttributionCall(() => attempt.onPayload(payload));
|
|
1611
|
+
if (originalOnPayload === undefined) return undefined;
|
|
1612
|
+
return await originalOnPayload(payload, payloadModel);
|
|
1613
|
+
};
|
|
1614
|
+
const wrappedOnResponse: NonNullable<SimpleStreamOptions["onResponse"]> = async (
|
|
1615
|
+
response,
|
|
1616
|
+
responseModel,
|
|
1617
|
+
) => {
|
|
1618
|
+
safeAttributionCall(() => attempt.onResponse(response));
|
|
1619
|
+
if (originalOnResponse !== undefined) {
|
|
1620
|
+
await originalOnResponse(response, responseModel);
|
|
1621
|
+
}
|
|
1622
|
+
};
|
|
1623
|
+
// The engine-bound options carry `maxRetries: 0` and, for Codex only, the
|
|
1624
|
+
// forced SSE transport; every other caller field passes through unchanged.
|
|
1625
|
+
const physicalOptions: SimpleStreamOptions = {
|
|
1626
|
+
...request.options,
|
|
1627
|
+
signal: physicalController.signal,
|
|
1628
|
+
onPayload: wrappedOnPayload,
|
|
1629
|
+
onResponse: wrappedOnResponse,
|
|
1630
|
+
};
|
|
1631
|
+
// Bound after the stall guard exists. Anthropic streams call it for every
|
|
1632
|
+
// response chunk, pings included; other providers ignore the field.
|
|
1633
|
+
let reportTransportActivity: () => void = () => {};
|
|
1634
|
+
const originalOnTransportActivity = (
|
|
1635
|
+
input.callerOptions as { onTransportActivity?: unknown } | undefined
|
|
1636
|
+
)?.onTransportActivity;
|
|
1637
|
+
(physicalOptions as { onTransportActivity?: () => void }).onTransportActivity = () => {
|
|
1638
|
+
reportTransportActivity();
|
|
1639
|
+
if (typeof originalOnTransportActivity === "function") {
|
|
1640
|
+
try {
|
|
1641
|
+
(originalOnTransportActivity as () => void)();
|
|
1642
|
+
} catch {
|
|
1643
|
+
// A caller's liveness hook cannot break the physical attempt.
|
|
1644
|
+
}
|
|
1645
|
+
}
|
|
1646
|
+
};
|
|
1647
|
+
const open = async (): Promise<AsyncIterable<unknown>> => {
|
|
1648
|
+
try {
|
|
1649
|
+
return await deps.dispatch({
|
|
1650
|
+
providerId: account.providerId,
|
|
1651
|
+
modelId: dispatchedModelId,
|
|
1652
|
+
context: input.context,
|
|
1653
|
+
options: physicalOptions,
|
|
1654
|
+
});
|
|
1655
|
+
} catch (error) {
|
|
1656
|
+
// Cool synchronously, then surface the rejection as a private terminal
|
|
1657
|
+
// so classification and attribution match a stream failure.
|
|
1658
|
+
const syntheticMessage = syntheticErrorMessage(
|
|
1659
|
+
requestedModelId,
|
|
1660
|
+
buildBoundedRecoveryFinalErrorMessage(),
|
|
1661
|
+
);
|
|
1662
|
+
box.failure = safeProjectFailureSignal(error, dispatchedModelId);
|
|
1663
|
+
if (!hasStructuredFailureEvidence(box.failure) && isHostStaleInstallFailure(error)) {
|
|
1664
|
+
markHostStaleInstall(box, account, error);
|
|
1665
|
+
box.receipt = hostFaultReceipt();
|
|
1666
|
+
} else {
|
|
1667
|
+
const raw =
|
|
1668
|
+
typeof error === "object" && error !== null
|
|
1669
|
+
? (error as { message?: unknown }).message
|
|
1670
|
+
: undefined;
|
|
1671
|
+
const disposition = setupFailureDisposition(syntheticMessage, raw);
|
|
1672
|
+
box.overflow = disposition === "context-overflow";
|
|
1673
|
+
box.preStartRetryable = disposition === "retryable";
|
|
1674
|
+
box.receipt = coordinator.recordFailure({
|
|
1675
|
+
account,
|
|
1676
|
+
requestedModelId,
|
|
1677
|
+
dispatchedModelId,
|
|
1678
|
+
error,
|
|
1679
|
+
});
|
|
1680
|
+
}
|
|
1681
|
+
box.physicalTerminal = syntheticMessage;
|
|
1682
|
+
const receipt = box.receipt;
|
|
1683
|
+
try {
|
|
1684
|
+
const accepted = attempt.fail(syntheticMessage, {
|
|
1685
|
+
alreadyCooled: receipt.alreadyCooled,
|
|
1686
|
+
dispatchedModelId,
|
|
1687
|
+
failure: box.failure,
|
|
1688
|
+
...(receipt.alreadyCooled ? { rollbackCooldown: receipt.rollback } : {}),
|
|
1689
|
+
});
|
|
1690
|
+
if (accepted === false && deps.attribution !== undefined) {
|
|
1691
|
+
receipt.rollback();
|
|
1692
|
+
box.invalidated = true;
|
|
1693
|
+
}
|
|
1694
|
+
} catch {
|
|
1695
|
+
if (deps.attribution !== undefined) {
|
|
1696
|
+
receipt.rollback();
|
|
1697
|
+
box.invalidated = true;
|
|
1698
|
+
}
|
|
1699
|
+
}
|
|
1700
|
+
return (async function* () {
|
|
1701
|
+
await attempt.waitForTerminal();
|
|
1702
|
+
yield { type: "error", reason: "error", error: syntheticMessage };
|
|
1703
|
+
})();
|
|
1704
|
+
}
|
|
1705
|
+
};
|
|
1706
|
+
// The short stall limit covers opening and the wait for `start`; every
|
|
1707
|
+
// later wait uses the longer started limit, or the transport silence limit
|
|
1708
|
+
// once the provider reports byte-level activity. Before any content a
|
|
1709
|
+
// stall classifies as a pre-start transient; after content it ends the call.
|
|
1710
|
+
const stallGuard = stallGuarded(
|
|
1711
|
+
open,
|
|
1712
|
+
{
|
|
1713
|
+
timeoutMs: input.stallTimeoutMs,
|
|
1714
|
+
startedTimeoutMs: input.startedStallTimeoutMs,
|
|
1715
|
+
silenceTimeoutMs: input.transportSilenceTimeoutMs,
|
|
1716
|
+
},
|
|
1717
|
+
input.clock,
|
|
1718
|
+
abortPhysical,
|
|
1719
|
+
);
|
|
1720
|
+
const guarded = stallGuard.output;
|
|
1721
|
+
reportTransportActivity = () => {
|
|
1722
|
+
if (physicalController.signal.aborted) return;
|
|
1723
|
+
stallGuard.activity();
|
|
1724
|
+
request.onTransportActivity();
|
|
1725
|
+
};
|
|
1726
|
+
const output: AsyncIterable<unknown> = {
|
|
1727
|
+
async *[Symbol.asyncIterator]() {
|
|
1728
|
+
try {
|
|
1729
|
+
for await (const event of observePhysicalAttempt(
|
|
1730
|
+
guarded,
|
|
1731
|
+
input.model,
|
|
1732
|
+
input.callerOptions,
|
|
1733
|
+
box,
|
|
1734
|
+
requestedModelId,
|
|
1735
|
+
dispatchedModelId,
|
|
1736
|
+
)) {
|
|
1737
|
+
// Classify before the terminal reaches the buffer, so the engine
|
|
1738
|
+
// never waits on a source it has already released. Nothing after
|
|
1739
|
+
// the terminal can need an abort, so its listeners go now.
|
|
1740
|
+
if (terminalAttribution(event) !== undefined) {
|
|
1741
|
+
settle();
|
|
1742
|
+
releaseAbortSources();
|
|
1743
|
+
}
|
|
1744
|
+
yield event;
|
|
1745
|
+
}
|
|
1746
|
+
} finally {
|
|
1747
|
+
settle();
|
|
1748
|
+
releaseAbortSources();
|
|
1749
|
+
}
|
|
1750
|
+
},
|
|
1751
|
+
};
|
|
1752
|
+
return { output, retrySafety };
|
|
1753
|
+
};
|
|
1754
|
+
|
|
1755
|
+
/** Named terminal facts only: a provider may attach arbitrary private fields. */
|
|
1756
|
+
const safeFailureMessage = (
|
|
1757
|
+
modelId: string,
|
|
1758
|
+
box: AttemptRecordBox | undefined,
|
|
1759
|
+
errorMessage: string,
|
|
1760
|
+
keepContent = false,
|
|
1761
|
+
): AssistantMessage & { readonly logicalFailure: LogicalFailureEvidence } => {
|
|
1762
|
+
const physical = box?.physicalTerminal;
|
|
1763
|
+
const account = box?.account;
|
|
1764
|
+
const providerId = account !== undefined &&
|
|
1765
|
+
isCanonicalManagedProviderId(account.providerId, account.family) &&
|
|
1766
|
+
// Only registered slot identities: larger suffixes can look like HTTP
|
|
1767
|
+
// retry statuses (429/500) in the host's prose-only predicates.
|
|
1768
|
+
isAccountSlotIndex(
|
|
1769
|
+
account.providerId === account.family ? 1 : Number(account.providerId.slice(`${account.family}-account-`.length)),
|
|
1770
|
+
MAX_ACCOUNT_LIMIT,
|
|
1771
|
+
)
|
|
1772
|
+
? account.providerId : undefined;
|
|
1773
|
+
const classified = classifyFailure(box?.failure ?? {}).category;
|
|
1774
|
+
const stale = box?.hostStaleInstall === true;
|
|
1775
|
+
const category = stale ? "host-stale-install" : box?.overflow === true ? "context-overflow" :
|
|
1776
|
+
box?.preStartRetryable === true && classified === "unknown" ? "transport" : classified;
|
|
1777
|
+
const attemptCount = box?.ordinal ?? 0;
|
|
1778
|
+
const evidence = `[${providerId === undefined ? "" : `physical provider: ${providerId}; `}cause: ${VISIBLE_FAILURE_CAUSES[category]}; attempts: ${attemptCount}]`;
|
|
1779
|
+
return {
|
|
1780
|
+
...syntheticErrorMessage(
|
|
1781
|
+
modelId,
|
|
1782
|
+
`${stale ? HOST_STALE_INSTALL_MESSAGE : errorMessage} ${evidence}`,
|
|
1783
|
+
physical === undefined ? undefined : projectTerminalUsage(physical),
|
|
1784
|
+
physical === undefined ? undefined : finiteNonNegative(physical.timestamp),
|
|
1785
|
+
),
|
|
1786
|
+
...(keepContent && physical !== undefined ? { content: projectAssistantContent(physical.content) } : {}),
|
|
1787
|
+
...(box?.failure?.code === undefined ? {} : { code: box.failure.code }),
|
|
1788
|
+
logicalFailure: Object.freeze({
|
|
1789
|
+
...(providerId === undefined ? {} : { providerId }),
|
|
1790
|
+
category,
|
|
1791
|
+
attemptCount,
|
|
1792
|
+
}),
|
|
1793
|
+
};
|
|
1794
|
+
};
|
|
1795
|
+
|
|
1796
|
+
const withoutErrorMessage = (message: AssistantMessage): AssistantMessage => {
|
|
1797
|
+
const { errorMessage: _omitted, ...rest } = message;
|
|
1798
|
+
return rest as AssistantMessage;
|
|
1799
|
+
};
|
|
1800
|
+
|
|
1801
|
+
/**
|
|
1802
|
+
* The one public terminal for a call the engine neither accepted nor handed
|
|
1803
|
+
* over.
|
|
1804
|
+
*
|
|
1805
|
+
* A structured refusal or unknown stop is the provider's answer, not a
|
|
1806
|
+
* failure: it keeps response content and `code` with a fixed reason, which the host never
|
|
1807
|
+
* re-dispatches. A context overflow keeps the fixed overflow text so the host
|
|
1808
|
+
* still compacts. Every other failure publishes the bounded recovery text
|
|
1809
|
+
* with closed cause evidence; neither host retry predicate matches it.
|
|
1810
|
+
*/
|
|
1811
|
+
const publicFailure = (
|
|
1812
|
+
modelId: string,
|
|
1813
|
+
result: Extract<RecoveryResult, { status: "exhausted" | "terminated" }>,
|
|
1814
|
+
last: AttemptRecordBox | undefined,
|
|
1815
|
+
): { readonly event: unknown; readonly message: AssistantMessage } => {
|
|
1816
|
+
const physical = last?.physicalTerminal;
|
|
1817
|
+
// A provider's own aborted terminal stays aborted: the host never retries
|
|
1818
|
+
// it, and reporting it as an error would invite exactly that.
|
|
1819
|
+
const aborted =
|
|
1820
|
+
(result.status === "terminated" &&
|
|
1821
|
+
(result.reason === "caller-aborted" || result.reason === "shutdown" || result.reason === "reload")) ||
|
|
1822
|
+
physical?.stopReason === "aborted";
|
|
1823
|
+
if (!aborted && physical !== undefined && hostFinalStopMessage(physical) !== undefined) {
|
|
1824
|
+
const message = projectMessage(physical, modelId);
|
|
1825
|
+
return { event: { type: "error", reason: "error", error: message }, message };
|
|
1826
|
+
}
|
|
1827
|
+
const overflow =
|
|
1828
|
+
last?.overflow === true &&
|
|
1829
|
+
result.status === "terminated" &&
|
|
1830
|
+
result.reason === "invalid-request";
|
|
1831
|
+
const errorMessage = overflow ? SETUP_CONTEXT_OVERFLOW_MESSAGE : result.errorMessage;
|
|
1832
|
+
// Keep validated usage and named failure facts, never arbitrary physical
|
|
1833
|
+
// fields, content no consumer saw, or the provider's own error text. An
|
|
1834
|
+
// aborted terminal carries no error text at all.
|
|
1835
|
+
const projected = safeFailureMessage(modelId, last, errorMessage);
|
|
1836
|
+
const message: AssistantMessage = aborted
|
|
1837
|
+
? { ...withoutErrorMessage(projected), stopReason: "aborted" }
|
|
1838
|
+
: { ...projected, stopReason: "error" };
|
|
1839
|
+
return {
|
|
1840
|
+
event: { type: "error", reason: aborted ? "aborted" : "error", error: message },
|
|
1841
|
+
message,
|
|
1842
|
+
};
|
|
1843
|
+
};
|
|
1844
|
+
|
|
1845
|
+
/**
|
|
1846
|
+
* Publish a handed-over attempt live. Its earlier events are shown as they
|
|
1847
|
+
* arrive, so nothing after them is ever sent again: a later failure ends the
|
|
1848
|
+
* call. Refusal and unknown-stop outcomes keep their existing projection.
|
|
1849
|
+
* Other failures keep content already shown but project named facts only;
|
|
1850
|
+
* overflow gets fixed overflow text and other failures get host-final text.
|
|
1851
|
+
*/
|
|
1852
|
+
const publishCommitted = (
|
|
1853
|
+
modelId: string,
|
|
1854
|
+
output: AsyncIterable<unknown>,
|
|
1855
|
+
box: AttemptRecordBox,
|
|
1856
|
+
): AsyncIterable<unknown> => ({
|
|
1857
|
+
async *[Symbol.asyncIterator]() {
|
|
1858
|
+
const bind = (physical: AssistantMessage, publicMessage: AssistantMessage): void => {
|
|
1859
|
+
safeAttributionCall(() => box.attempt.bindPublicTerminal?.(physical, publicMessage));
|
|
1860
|
+
safeAttributionCall(() => deps.onPublicTerminal?.(physical, publicMessage));
|
|
1861
|
+
};
|
|
1862
|
+
const failed = (physical: AssistantMessage, aborted: boolean): AssistantMessage => {
|
|
1863
|
+
if (hostFinalStopMessage(physical) !== undefined) return projectMessage(physical, modelId);
|
|
1864
|
+
const projected = safeFailureMessage(
|
|
1865
|
+
modelId, box,
|
|
1866
|
+
box.overflow ? SETUP_CONTEXT_OVERFLOW_MESSAGE : buildBoundedRecoveryFinalErrorMessage(),
|
|
1867
|
+
true,
|
|
1868
|
+
);
|
|
1869
|
+
return aborted ? { ...withoutErrorMessage(projected), stopReason: "aborted" } : projected;
|
|
1870
|
+
};
|
|
1871
|
+
try {
|
|
1872
|
+
for await (const event of output) {
|
|
1873
|
+
const terminal = terminalAttribution(event);
|
|
1874
|
+
if (terminal === undefined) {
|
|
1875
|
+
yield projectEvent(event, modelId);
|
|
1876
|
+
continue;
|
|
1877
|
+
}
|
|
1878
|
+
if (terminal.outcome === "finish") {
|
|
1879
|
+
const publicMessage = projectMessage(terminal.message, modelId);
|
|
1880
|
+
bind(terminal.message, publicMessage);
|
|
1881
|
+
yield { type: "done", reason: publicMessage.stopReason, message: publicMessage };
|
|
1882
|
+
return;
|
|
1883
|
+
}
|
|
1884
|
+
const aborted = terminal.outcome === "abort";
|
|
1885
|
+
const publicMessage = failed(terminal.message, aborted);
|
|
1886
|
+
bind(terminal.message, publicMessage);
|
|
1887
|
+
yield { type: "error", reason: aborted ? "aborted" : "error", error: publicMessage };
|
|
1888
|
+
return;
|
|
1889
|
+
}
|
|
1890
|
+
} catch {
|
|
1891
|
+
// A stall or a thrown stream failure after content: the attempt
|
|
1892
|
+
// recorded its own synthetic terminal, and the call ends here.
|
|
1893
|
+
const physical =
|
|
1894
|
+
box.physicalTerminal ??
|
|
1895
|
+
syntheticErrorMessage(modelId, buildBoundedRecoveryFinalErrorMessage());
|
|
1896
|
+
const aborted = box.cancelled?.() === true;
|
|
1897
|
+
const publicMessage: AssistantMessage = {
|
|
1898
|
+
...failed(physical, aborted),
|
|
1899
|
+
...(aborted ? { stopReason: "aborted" as const } : {}),
|
|
1900
|
+
};
|
|
1901
|
+
bind(physical, publicMessage);
|
|
1902
|
+
yield { type: "error", reason: aborted ? "aborted" : "error", error: publicMessage };
|
|
1903
|
+
}
|
|
1904
|
+
},
|
|
1905
|
+
});
|
|
1906
|
+
|
|
1083
1907
|
return {
|
|
1084
1908
|
api: LOGICAL_PROVIDER_ID,
|
|
1085
1909
|
|
|
@@ -1097,54 +1921,18 @@ export function createLogicalProvider(
|
|
|
1097
1921
|
"the logical provider was asked for a request with no model id",
|
|
1098
1922
|
);
|
|
1099
1923
|
}
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
deps.onObservation?.({
|
|
1103
|
-
providerId: account.providerId,
|
|
1104
|
-
modelId: resolvedModelId,
|
|
1105
|
-
family: account.family,
|
|
1106
|
-
providerType,
|
|
1107
|
-
});
|
|
1108
|
-
const route: LogicalRouteFact = Object.freeze({
|
|
1109
|
-
providerId: account.providerId,
|
|
1110
|
-
family: account.family,
|
|
1111
|
-
providerType,
|
|
1112
|
-
accountFingerprint: account.providerId,
|
|
1113
|
-
});
|
|
1114
|
-
const attempt = safeBeginAttributionAttempt(deps.attribution, route);
|
|
1115
|
-
if (account.authenticated !== undefined) {
|
|
1116
|
-
safeAttributionCall(() => attempt.onAuthentication(account.authenticated!));
|
|
1117
|
-
}
|
|
1118
|
-
if (account.modelSupported !== undefined) {
|
|
1119
|
-
safeAttributionCall(() => attempt.onModelSupport(account.modelSupported!));
|
|
1924
|
+
if (shutDown) {
|
|
1925
|
+
throw new Error("the logical provider session has shut down");
|
|
1120
1926
|
}
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
payload,
|
|
1128
|
-
payloadModel,
|
|
1129
|
-
) => {
|
|
1130
|
-
safeAttributionCall(() => attempt.onPayload(payload));
|
|
1131
|
-
if (originalOnPayload === undefined) return undefined;
|
|
1132
|
-
return await originalOnPayload(payload, payloadModel);
|
|
1133
|
-
};
|
|
1134
|
-
const wrappedOnResponse: NonNullable<SimpleStreamOptions["onResponse"]> = async (
|
|
1135
|
-
response,
|
|
1136
|
-
responseModel,
|
|
1137
|
-
) => {
|
|
1138
|
-
safeAttributionCall(() => attempt.onResponse(response));
|
|
1139
|
-
if (originalOnResponse !== undefined) {
|
|
1140
|
-
await originalOnResponse(response, responseModel);
|
|
1141
|
-
}
|
|
1142
|
-
};
|
|
1143
|
-
let routedOptions = options;
|
|
1144
|
-
if (account.family === "openai-codex") {
|
|
1927
|
+
const selected = selectAccount(modelId);
|
|
1928
|
+
const candidates = recoveryCandidates(modelId, selected);
|
|
1929
|
+
// Codex routes are pinned to SSE before the engine sees the options, so
|
|
1930
|
+
// the engine reserves one bounded send for them instead of an unknown count.
|
|
1931
|
+
let engineOptions = options;
|
|
1932
|
+
if (selected.account.family === "openai-codex") {
|
|
1145
1933
|
// Temporary Pi 0.99 WebSocket containment; see CODEX_FORCED_TRANSPORT.
|
|
1146
1934
|
const forced = forceCodexSseOptions(options);
|
|
1147
|
-
|
|
1935
|
+
engineOptions = forced.options;
|
|
1148
1936
|
if (forced.overridden && options?.transport !== undefined && !codexTransportNoticeSent) {
|
|
1149
1937
|
codexTransportNoticeSent = true;
|
|
1150
1938
|
try {
|
|
@@ -1156,70 +1944,203 @@ export function createLogicalProvider(
|
|
|
1156
1944
|
}
|
|
1157
1945
|
}
|
|
1158
1946
|
}
|
|
1159
|
-
const
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
//
|
|
1165
|
-
//
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
account
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1947
|
+
const boxes: AttemptRecordBox[] = [];
|
|
1948
|
+
const clock = deps.recoveryClock ?? SYSTEM_RECOVERY_CLOCK;
|
|
1949
|
+
const stallTimeoutMs =
|
|
1950
|
+
deps.recoveryTiming?.recoveryStallTimeoutMs ??
|
|
1951
|
+
DEFAULT_CONFIG.recoveryStallTimeoutMs;
|
|
1952
|
+
// Once the provider has answered, only the whole-call limit bounds a
|
|
1953
|
+
// silent wait; the engine's absolute timer still ends the call.
|
|
1954
|
+
const startedStallTimeoutMs = Math.max(
|
|
1955
|
+
stallTimeoutMs,
|
|
1956
|
+
deps.recoveryTiming?.recoveryAbsoluteTimeoutMs ??
|
|
1957
|
+
DEFAULT_CONFIG.recoveryAbsoluteTimeoutMs,
|
|
1958
|
+
);
|
|
1959
|
+
const engine = createRecoveryEngine({
|
|
1960
|
+
clock,
|
|
1961
|
+
recheck: (candidate) => {
|
|
1962
|
+
const account = deps.accounts.find(
|
|
1963
|
+
({ providerId, family }) =>
|
|
1964
|
+
providerId === candidate.providerId && family === candidate.family,
|
|
1965
|
+
);
|
|
1966
|
+
if (account === undefined) return { status: "skip" };
|
|
1967
|
+
// Re-read live eligibility for every candidate: a second candidate is
|
|
1968
|
+
// checked after the first failure has already cooled its own account.
|
|
1969
|
+
return logicalAccountEligible(
|
|
1970
|
+
{
|
|
1971
|
+
providerId: account.providerId,
|
|
1972
|
+
exhausted: account.exhausted,
|
|
1973
|
+
authenticated: account.authenticated,
|
|
1974
|
+
},
|
|
1975
|
+
coordinator.state,
|
|
1976
|
+
Date.now(),
|
|
1977
|
+
)
|
|
1978
|
+
? { status: "eligible" }
|
|
1979
|
+
: { status: "skip" };
|
|
1980
|
+
},
|
|
1981
|
+
reserve: () => "reserved",
|
|
1982
|
+
dispatch: (request) => {
|
|
1983
|
+
const account = deps.accounts.find(
|
|
1984
|
+
({ providerId, family }) =>
|
|
1985
|
+
providerId === request.candidate.providerId &&
|
|
1986
|
+
family === request.candidate.family,
|
|
1987
|
+
);
|
|
1988
|
+
if (account === undefined) throw new Error("the candidate account disappeared");
|
|
1989
|
+
const providerType = logicalProviderType(account);
|
|
1990
|
+
deps.onObservation?.({
|
|
1991
|
+
providerId: account.providerId,
|
|
1992
|
+
modelId: request.candidate.modelId,
|
|
1993
|
+
family: account.family,
|
|
1994
|
+
providerType,
|
|
1995
|
+
});
|
|
1996
|
+
const route: LogicalRouteFact = Object.freeze({
|
|
1997
|
+
providerId: account.providerId,
|
|
1998
|
+
family: account.family,
|
|
1999
|
+
providerType,
|
|
2000
|
+
accountFingerprint: account.providerId,
|
|
2001
|
+
});
|
|
2002
|
+
const attempt = safeBeginAttributionAttempt(deps.attribution, route);
|
|
2003
|
+
if (account.authenticated !== undefined) {
|
|
2004
|
+
safeAttributionCall(() => attempt.onAuthentication(account.authenticated!));
|
|
2005
|
+
}
|
|
2006
|
+
if (account.modelSupported !== undefined) {
|
|
2007
|
+
safeAttributionCall(() => attempt.onModelSupport(account.modelSupported!));
|
|
2008
|
+
}
|
|
2009
|
+
if (account.health !== undefined) {
|
|
2010
|
+
safeAttributionCall(() => attempt.onHealth(account.health));
|
|
2011
|
+
}
|
|
2012
|
+
const box: AttemptRecordBox = {
|
|
2013
|
+
ordinal: boxes.length + 1,
|
|
2014
|
+
overflow: false,
|
|
2015
|
+
sawOutput: false,
|
|
2016
|
+
preStartRetryable: false,
|
|
2017
|
+
invalidated: false,
|
|
2018
|
+
attempt,
|
|
2019
|
+
account,
|
|
2020
|
+
};
|
|
2021
|
+
boxes.push(box);
|
|
2022
|
+
return physicalAttempt({
|
|
2023
|
+
request,
|
|
2024
|
+
account,
|
|
2025
|
+
model,
|
|
2026
|
+
context,
|
|
2027
|
+
callerOptions: options,
|
|
2028
|
+
requestedModelId: modelId,
|
|
2029
|
+
box,
|
|
2030
|
+
clock,
|
|
2031
|
+
stallTimeoutMs,
|
|
2032
|
+
startedStallTimeoutMs,
|
|
2033
|
+
transportSilenceTimeoutMs:
|
|
2034
|
+
deps.transportSilenceTimeoutMs ?? TRANSPORT_SILENCE_TIMEOUT_MS,
|
|
2035
|
+
});
|
|
2036
|
+
},
|
|
2037
|
+
// The attribution store retains every physical attempt's usage and cost
|
|
2038
|
+
// at its own terminal, under its own physical account. A failed attempt
|
|
2039
|
+
// whose retention failed cannot buy a second send that could not be
|
|
2040
|
+
// accounted for either; an accepted answer is never discarded for it,
|
|
2041
|
+
// and its missing record stays a reported coverage gap.
|
|
2042
|
+
account: async (record) => {
|
|
2043
|
+
const box = boxes[record.ordinal - 1];
|
|
2044
|
+
// An accepted or handed-over attempt has not reached its terminal yet;
|
|
2045
|
+
// its own terminal retains its facts.
|
|
2046
|
+
if (
|
|
2047
|
+
box === undefined ||
|
|
2048
|
+
record.disposition === "accepted" ||
|
|
2049
|
+
record.disposition === "committed"
|
|
2050
|
+
) {
|
|
2051
|
+
return "recorded";
|
|
2052
|
+
}
|
|
2053
|
+
const outcome = await box.attempt.waitForTerminal();
|
|
2054
|
+
return outcome.status === "failed" ? "rejected" : "recorded";
|
|
2055
|
+
},
|
|
2056
|
+
});
|
|
2057
|
+
activeEngines.add(engine);
|
|
2058
|
+
// The engine starts now and the stream is returned at once: a consumer
|
|
2059
|
+
// sees nothing until the call has a publishable outcome.
|
|
2060
|
+
const settled = (async (): Promise<AsyncIterable<unknown>> => {
|
|
2061
|
+
let result: RecoveryResult;
|
|
1189
2062
|
try {
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
2063
|
+
result = await engine.recover({
|
|
2064
|
+
candidates,
|
|
2065
|
+
context,
|
|
2066
|
+
...(engineOptions === undefined ? {} : { options: engineOptions }),
|
|
2067
|
+
// Only the provider's `start` is held: the first content event hands
|
|
2068
|
+
// the attempt over and streams live, so only a failure before any
|
|
2069
|
+
// content may move to another account.
|
|
2070
|
+
publication: "stream-after-first-content",
|
|
2071
|
+
timing: {
|
|
2072
|
+
recoveryIdleTimeoutMs:
|
|
2073
|
+
deps.recoveryTiming?.recoveryIdleTimeoutMs ??
|
|
2074
|
+
DEFAULT_CONFIG.recoveryIdleTimeoutMs,
|
|
2075
|
+
recoveryAbsoluteTimeoutMs:
|
|
2076
|
+
deps.recoveryTiming?.recoveryAbsoluteTimeoutMs ??
|
|
2077
|
+
DEFAULT_CONFIG.recoveryAbsoluteTimeoutMs,
|
|
2078
|
+
},
|
|
2079
|
+
...(options?.signal === undefined ? {} : { signal: options.signal }),
|
|
1197
2080
|
});
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
2081
|
+
} finally {
|
|
2082
|
+
activeEngines.delete(engine);
|
|
2083
|
+
}
|
|
2084
|
+
const last = boxes.at(-1);
|
|
2085
|
+
// Attempts the call moved past were never published. Their physical
|
|
2086
|
+
// failure is still real, so the session commits its account effects.
|
|
2087
|
+
for (const superseded of boxes.slice(0, -1)) {
|
|
2088
|
+
const physical = superseded.physicalTerminal;
|
|
2089
|
+
if (physical !== undefined) {
|
|
2090
|
+
safeAttributionCall(() => deps.onSupersededTerminal?.(physical));
|
|
2091
|
+
}
|
|
2092
|
+
}
|
|
2093
|
+
if (result.status === "accepted") {
|
|
2094
|
+
const physical = result.terminal;
|
|
2095
|
+
const publicMessage = projectMessage(physical, modelId);
|
|
2096
|
+
safeAttributionCall(() => last?.attempt.bindPublicTerminal?.(physical, publicMessage));
|
|
2097
|
+
safeAttributionCall(() => deps.onPublicTerminal?.(physical, publicMessage));
|
|
2098
|
+
const accepted = result.output;
|
|
2099
|
+
// Streaming publication accepts only an attempt that reached its
|
|
2100
|
+
// terminal before any content. One that sent a `start` is published
|
|
2101
|
+
// as its own held `start` and then its terminal, so the stream stays
|
|
2102
|
+
// well formed; one that sent nothing else is published as its bare
|
|
2103
|
+
// terminal.
|
|
2104
|
+
const heldStart = last?.heldStart;
|
|
2105
|
+
const replayProgress = last?.sawOutput === true || heldStart !== undefined;
|
|
2106
|
+
return (async function* () {
|
|
2107
|
+
let startPublished = false;
|
|
2108
|
+
for await (const event of accepted) {
|
|
2109
|
+
if (event.type === "done") yield { type: "done", reason: publicMessage.stopReason, message: publicMessage };
|
|
2110
|
+
else if (!replayProgress) continue;
|
|
2111
|
+
else if (event.type === "start" && heldStart !== undefined) {
|
|
2112
|
+
if (startPublished) continue;
|
|
2113
|
+
startPublished = true;
|
|
2114
|
+
yield projectEvent(heldStart, modelId);
|
|
2115
|
+
} else yield projectEvent(event, modelId);
|
|
2116
|
+
}
|
|
2117
|
+
})();
|
|
2118
|
+
}
|
|
2119
|
+
if (result.status === "committed") {
|
|
2120
|
+
// The live attempt is the last box: the engine dispatched nothing after it.
|
|
2121
|
+
return publishCommitted(modelId, result.output, last!);
|
|
2122
|
+
}
|
|
2123
|
+
const failure = publicFailure(modelId, result, last);
|
|
2124
|
+
const physical = last?.physicalTerminal;
|
|
2125
|
+
if (physical !== undefined) {
|
|
2126
|
+
safeAttributionCall(() => last?.attempt.bindPublicTerminal?.(physical, failure.message));
|
|
2127
|
+
safeAttributionCall(() => deps.onPublicTerminal?.(physical, failure.message));
|
|
1201
2128
|
}
|
|
1202
2129
|
return (async function* () {
|
|
1203
|
-
|
|
1204
|
-
yield { type: "error", reason: "error", error: syntheticMessage };
|
|
2130
|
+
yield failure.event;
|
|
1205
2131
|
})();
|
|
1206
|
-
}
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
account,
|
|
1212
|
-
modelId,
|
|
1213
|
-
resolvedModelId,
|
|
1214
|
-
attempt,
|
|
1215
|
-
);
|
|
2132
|
+
})();
|
|
2133
|
+
void settled.catch(() => {});
|
|
2134
|
+
return (async function* () {
|
|
2135
|
+
yield* await settled;
|
|
2136
|
+
})();
|
|
1216
2137
|
},
|
|
1217
2138
|
|
|
1218
2139
|
preflight(model) {
|
|
1219
2140
|
const modelId = requestedModelId(model);
|
|
1220
2141
|
if (modelId === undefined) return undefined;
|
|
1221
2142
|
try {
|
|
1222
|
-
const { account, resolvedModelId } = selectAccount(modelId
|
|
2143
|
+
const { account, resolvedModelId } = selectAccount(modelId);
|
|
1223
2144
|
const providerType = logicalProviderType(account);
|
|
1224
2145
|
// Everything below is read off the account that was actually
|
|
1225
2146
|
// chosen. The logical provider has no health, no credential and no
|
|
@@ -1260,6 +2181,12 @@ export function createLogicalProvider(
|
|
|
1260
2181
|
},
|
|
1261
2182
|
|
|
1262
2183
|
shutdown() {
|
|
2184
|
+
// No late dispatch survives the session: every in-flight logical call
|
|
2185
|
+
// aborts its active attempt and publishes no further send.
|
|
2186
|
+
shutDown = true;
|
|
2187
|
+
for (const engine of activeEngines) engine.shutdown();
|
|
2188
|
+
shutdownController.abort();
|
|
2189
|
+
activeEngines.clear();
|
|
1263
2190
|
safeAttributionCall(() => deps.attribution?.shutdown());
|
|
1264
2191
|
},
|
|
1265
2192
|
};
|