@centerforagenticai/pi-multi-account 0.1.5 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -14,6 +14,7 @@
14
14
  */
15
15
 
16
16
  import { logicalAccountEligible, recordFailureCooldown } from "./routing.js";
17
+ import { formatAccountGroupFailure, type AccountGroupFailurePolicy, type AccountGroupFailureCandidate } from "./account-group-failure.js";
17
18
  import type { ManagedAccount } from "./routing.js";
18
19
  import {
19
20
  isContextOverflow,
@@ -22,9 +23,16 @@ import {
22
23
  type ProviderResponse,
23
24
  type SimpleStreamOptions,
24
25
  } from "@earendil-works/pi-ai";
25
- import { RuntimeState, type LogicalRoutePin } from "./runtime-state.js";
26
+ import { RuntimeState, isCanonicalManagedProviderId } from "./runtime-state.js";
26
27
  import { hostFinalStopMessage } from "./host-final-stop-message.js";
27
- import { DEFAULT_CONFIG } from "./config.js";
28
+ import {
29
+ projectPublicAssistantMessage,
30
+ projectPublicAssistantEvent,
31
+ projectAssistantContent,
32
+ projectTerminalUsage,
33
+ finiteNonNegative,
34
+ } from "./public-assistant-projection.js";
35
+ import { DEFAULT_CONFIG, MAX_ACCOUNT_LIMIT, isAccountSlotIndex } from "./config.js";
28
36
  import type {
29
37
  AllowedFamily,
30
38
  CrossFamilyChain,
@@ -38,6 +46,7 @@ import {
38
46
  providerErrorCodeFromMessage,
39
47
  } from "./error-classification.js";
40
48
  import type {
49
+ FailureCategory,
41
50
  ProviderErrorCode,
42
51
  ProviderFailureSignal,
43
52
  TransportFailureKind,
@@ -50,6 +59,37 @@ import type { ProviderType, Vendor } from "./vendor.js";
50
59
  export { LOGICAL_PROVIDER_ID } from "./models-declaration.js";
51
60
  import { LOGICAL_PROVIDER_ID } from "./models-declaration.js";
52
61
  import { forceCodexSseOptions } from "./codex-adapter.js";
62
+ import {
63
+ buildBoundedRecoveryFinalErrorMessage,
64
+ createRecoveryEngine,
65
+ isRecoveryStartEvent,
66
+ type RecoveryClock,
67
+ type RecoveryDispatchRequest,
68
+ type RecoveryEngine,
69
+ type RecoveryPhysicalAttempt,
70
+ type RecoveryResult,
71
+ type RecoveryRetrySafety,
72
+ type RecoveryTimer,
73
+ type RecoveryTimingConfig,
74
+ } from "./recovery-engine.js";
75
+ import {
76
+ buildRecoveryCandidatePlan,
77
+ type RecoveryCandidate,
78
+ type RecoveryModelCapability,
79
+ type RecoveryPlanAccount,
80
+ } from "./recovery-plan.js";
81
+
82
+ /** Production recovery timers: monotonic time and cancellable host timers. */
83
+ const SYSTEM_RECOVERY_CLOCK: RecoveryClock = Object.freeze({
84
+ now: () => performance.now(),
85
+ setTimer: (delayMs: number, callback: () => void) => {
86
+ const handle = setTimeout(callback, delayMs);
87
+ if (typeof handle === "object" && handle !== null && "unref" in handle) {
88
+ (handle as { unref: () => void }).unref();
89
+ }
90
+ return { cancel: () => clearTimeout(handle) };
91
+ },
92
+ });
53
93
 
54
94
  /** Fixed diagnostic label for a caller-supplied transport; never echoes the raw value. */
55
95
  function codexTransportLabel(
@@ -98,6 +138,14 @@ export interface LogicalPhysicalAccount {
98
138
  };
99
139
  }
100
140
 
141
+ /** Safe physical cause of a failed logical call; never contains provider prose. */
142
+ export interface LogicalFailureEvidence {
143
+ readonly providerId?: string;
144
+ readonly category: FailureCategory | "context-overflow" | "host-stale-install";
145
+ /** Physical invocations, not the unknown inner send count of Antigravity. */
146
+ readonly attemptCount: number;
147
+ }
148
+
101
149
  /** One physical attempt the logical provider makes. */
102
150
  export interface LogicalDispatchCall {
103
151
  providerId: string;
@@ -203,17 +251,21 @@ export interface LogicalAttributionLifecycle {
203
251
  shutdown(): void;
204
252
  }
205
253
 
206
- export interface LogicalRoutePinAccess {
207
- readonly get: () => LogicalRoutePin | undefined;
208
- readonly consume: (
209
- expectedGeneration: number,
210
- requestedModelId: string,
211
- ) => LogicalRoutePin | undefined;
212
- readonly clear: () => void;
254
+ /** One authorized selection generation; outside rows explain failure, never authorize sends. */
255
+ export interface LogicalSelectionSnapshot {
256
+ readonly accounts: LogicalPhysicalAccount[];
257
+ readonly group?: {
258
+ readonly policy: AccountGroupFailurePolicy;
259
+ readonly accountLimit: number;
260
+ readonly allAccounts: readonly LogicalPhysicalAccount[];
261
+ readonly otherCandidates: readonly AccountGroupFailureCandidate[];
262
+ };
213
263
  }
214
264
 
215
265
  export interface LogicalProviderDeps {
216
266
  accounts: LogicalPhysicalAccount[];
267
+ /** Capture policy and account facts together, once per selection/preflight. */
268
+ captureSelectionSnapshot?: (modelId: string) => LogicalSelectionSnapshot;
217
269
  dispatch: (
218
270
  call: LogicalDispatchCall,
219
271
  ) => Promise<AsyncIterable<unknown>> | AsyncIterable<unknown>;
@@ -222,6 +274,12 @@ export interface LogicalProviderDeps {
222
274
  /** Subscription-catalog owner of a declared logical model. */
223
275
  modelVendor?: (modelId: string) => Vendor | undefined;
224
276
  tierModelMap?: TierModelMap;
277
+ /**
278
+ * The live `sameFamilyFailover` setting. `false` makes every unified call
279
+ * one attempt: the in-call recovery never moves to another account.
280
+ * Omitted uses the default.
281
+ */
282
+ sameFamilyFailover?: boolean;
225
283
  crossFamilyChains?: readonly CrossFamilyChain[];
226
284
  onObservation?: (observation: LogicalObservation) => void;
227
285
  attribution?: LogicalAttributionLifecycle;
@@ -229,9 +287,24 @@ export interface LogicalProviderDeps {
229
287
  onPublicTerminal?: (physical: AssistantMessage, publicMessage: AssistantMessage) => void;
230
288
  onDiagnostic?: (message: string) => void;
231
289
  onShutdownAbort?: () => void;
232
- scheduleContinuation?: (run: () => void) => void;
290
+ /**
291
+ * A failed physical terminal the call recovered past. It was never
292
+ * published, so no `message_end` will carry it; the session commits its
293
+ * account effects here instead.
294
+ */
295
+ onSupersededTerminal?: (physical: AssistantMessage) => void;
233
296
  state?: RuntimeState;
234
- routePin?: LogicalRoutePinAccess;
297
+ /**
298
+ * Per-invocation idle and absolute limits and the per-attempt stall limit;
299
+ * omitted fields use the defaults.
300
+ */
301
+ recoveryTiming?: Partial<
302
+ RecoveryTimingConfig & Pick<MultiAccountConfig, "recoveryStallTimeoutMs">
303
+ >;
304
+ /** Injected only by tests; production uses the system clock. */
305
+ recoveryClock?: RecoveryClock;
306
+ /** Injected only by tests; production uses `TRANSPORT_SILENCE_TIMEOUT_MS`. */
307
+ transportSilenceTimeoutMs?: number;
235
308
  }
236
309
 
237
310
  export interface LogicalProvider {
@@ -361,6 +434,105 @@ const EXHAUSTION_LENGTH_ERROR_MESSAGE = "provider returned error (usage-limit)";
361
434
  */
362
435
  export const SETUP_CONTEXT_OVERFLOW_MESSAGE = "context_length_exceeded (provider_error)";
363
436
 
437
+ /** Closed visible vocabulary: `quota-rate-limit` itself would trigger host retry. */
438
+ const VISIBLE_FAILURE_CAUSES: Readonly<Record<LogicalFailureEvidence["category"], string>> = Object.freeze({
439
+ "quota-rate-limit": "quota",
440
+ "terminal-auth": "terminal-auth",
441
+ "transient-auth": "transient-auth",
442
+ permission: "permission",
443
+ config: "config",
444
+ transport: "transport",
445
+ unknown: "unknown",
446
+ "context-overflow": "context-overflow",
447
+ "host-stale-install": "host_stale_install",
448
+ });
449
+
450
+ /**
451
+ * Public text for a local host fault: the running pi process tried to load a
452
+ * bundle chunk that an in-place upgrade replaced. No provider was reached, and
453
+ * no other account can load the missing file, so only a restart helps. Neither
454
+ * host retry predicate matches this text.
455
+ */
456
+ export const HOST_STALE_INSTALL_MESSAGE =
457
+ "pi's installed files changed while this session was running; restart the pi session to load the current install.";
458
+
459
+ /**
460
+ * Public text when the missing module is a bare package rather than one of
461
+ * pi's own bundle files: the dependency is absent from the install, so a
462
+ * restart loads the same broken tree and only a reinstall helps.
463
+ */
464
+ export const HOST_MISSING_DEPENDENCY_MESSAGE =
465
+ "pi could not load a package it depends on; reinstall pi (or the extension that needs the package), then restart the pi session.";
466
+
467
+ /**
468
+ * Node's own phrasing, anchored at the start: pi-ai `lazyStream` keeps
469
+ * `error.message` unprefixed. An unanchored match would misread a provider or
470
+ * gateway error body that quotes a module error as a local fault.
471
+ */
472
+ const STALE_INSTALL_TEXT =
473
+ /^(?:Error(?: \[ERR_MODULE_NOT_FOUND\])?: )?(?:Cannot find (?:module|package) |Failed to (?:fetch|load) dynamically imported module|Error loading dynamically imported module)/;
474
+
475
+ /**
476
+ * Whether a setup failure is the host failing to load its own code, rather
477
+ * than a provider failing. Reads the Node error code first, then the message
478
+ * text, because pi-ai `lazyStream` keeps only `error.message`.
479
+ */
480
+ export function isHostStaleInstallFailure(error: unknown): boolean {
481
+ try {
482
+ if (typeof error === "string") return STALE_INSTALL_TEXT.test(error);
483
+ if (typeof error !== "object" || error === null) return false;
484
+ const { code, message, errorMessage } = error as { code?: unknown; message?: unknown; errorMessage?: unknown };
485
+ if (code === "ERR_MODULE_NOT_FOUND") return true;
486
+ return [message, errorMessage].some((text) => typeof text === "string" && STALE_INSTALL_TEXT.test(text));
487
+ } catch {
488
+ return false;
489
+ }
490
+ }
491
+
492
+ /** A recognised provider code, HTTP status or transport kind outranks any text. */
493
+ function hasStructuredFailureEvidence(failure: ProviderFailureSignal | undefined): boolean {
494
+ return failure !== undefined &&
495
+ (failure.code !== undefined || failure.httpStatus !== undefined || failure.transportKind !== undefined);
496
+ }
497
+
498
+ /**
499
+ * Bounded diagnostic cause for a stale install: the Node error code and the
500
+ * missing file's base name. Directories are dropped so no local path is kept.
501
+ */
502
+ export function hostLoadFault(error: unknown): { cause: string; missingDependency: boolean } {
503
+ let text = "";
504
+ let code: unknown;
505
+ try {
506
+ if (typeof error === "string") text = error;
507
+ else if (typeof error === "object" && error !== null) {
508
+ const fields = error as { code?: unknown; message?: unknown; errorMessage?: unknown };
509
+ code = fields.code;
510
+ text = typeof fields.message === "string" ? fields.message : typeof fields.errorMessage === "string" ? fields.errorMessage : "";
511
+ }
512
+ } catch {
513
+ // Fall through with whatever was read.
514
+ }
515
+ const kind = code === "ERR_MODULE_NOT_FOUND" || /ERR_MODULE_NOT_FOUND|Cannot find/.test(text) ? "ERR_MODULE_NOT_FOUND" : "dynamic-import-failed";
516
+ // Node names the missing specifier first ("Cannot find package 'x' imported
517
+ // from /dir/importer.js"); the importer is not what is missing.
518
+ // Node does not escape quotes, so a path may contain one: read up to the
519
+ // quote that ends the specifier (before " imported from", a CJS
520
+ // ". Please verify…"/"Require stack" line, or the end of the text).
521
+ const specifier = /Cannot find (?:module|package) '(.+?)'(?=$| imported from |\.\s|\r?\n)/.exec(text)?.[1];
522
+ if (specifier !== undefined && specifier.length > 0) {
523
+ if (!/^(?:\.{1,2}[\\/]|[\\/]|file:|[A-Za-z]:[\\/])/.test(specifier)) {
524
+ // A bare specifier names a package (keep "@scope/name" or "name").
525
+ const parts = specifier.split("/");
526
+ const name = (specifier.startsWith("@") ? parts.slice(0, 2) : parts.slice(0, 1)).join("/");
527
+ return { cause: `${kind} package ${name.slice(0, 120)}`, missingDependency: true };
528
+ }
529
+ const base = specifier.split(/[\\/]/).at(-1) ?? "";
530
+ return { cause: base.length === 0 ? kind : `${kind} ${base.slice(0, 120)}`, missingDependency: false };
531
+ }
532
+ const file = /['"]?([^'"\s]*\.(?:m?js|cjs|json|node))['"]?/.exec(text)?.[1]?.split(/[\\/]/).at(-1);
533
+ return { cause: file === undefined || file.length === 0 ? kind : `${kind} ${file.slice(0, 120)}`, missingDependency: false };
534
+ }
535
+
364
536
  type SetupFailureDisposition = "context-overflow" | "retryable" | "host-final";
365
537
 
366
538
  /**
@@ -384,70 +556,6 @@ function setupFailureDisposition(
384
556
  }
385
557
  }
386
558
 
387
- function finiteNonNegative(value: unknown): number | undefined {
388
- return typeof value === "number" && Number.isFinite(value) && value >= 0
389
- ? value
390
- : undefined;
391
- }
392
-
393
- function projectTerminalUsage(message: AssistantMessage): AssistantMessage["usage"] | undefined {
394
- const rawUsage = (message as unknown as { usage?: unknown }).usage;
395
- if (typeof rawUsage !== "object" || rawUsage === null) return undefined;
396
- const usage = rawUsage as Record<string, unknown>;
397
- const input = finiteNonNegative(usage.input);
398
- const output = finiteNonNegative(usage.output);
399
- const cacheRead = finiteNonNegative(usage.cacheRead);
400
- const cacheWrite = finiteNonNegative(usage.cacheWrite);
401
- const totalTokens = finiteNonNegative(usage.totalTokens);
402
- const rawCost = usage.cost;
403
- if (
404
- input === undefined ||
405
- output === undefined ||
406
- cacheRead === undefined ||
407
- cacheWrite === undefined ||
408
- totalTokens === undefined ||
409
- typeof rawCost !== "object" ||
410
- rawCost === null
411
- ) {
412
- return undefined;
413
- }
414
- const cost = rawCost as Record<string, unknown>;
415
- const costInput = finiteNonNegative(cost.input);
416
- const costOutput = finiteNonNegative(cost.output);
417
- const costCacheRead = finiteNonNegative(cost.cacheRead);
418
- const costCacheWrite = finiteNonNegative(cost.cacheWrite);
419
- const costTotal = finiteNonNegative(cost.total);
420
- if (
421
- costInput === undefined ||
422
- costOutput === undefined ||
423
- costCacheRead === undefined ||
424
- costCacheWrite === undefined ||
425
- costTotal === undefined
426
- ) {
427
- return undefined;
428
- }
429
- const cacheWrite1h = finiteNonNegative(usage.cacheWrite1h);
430
- if (usage.cacheWrite1h !== undefined && cacheWrite1h === undefined) return undefined;
431
- const reasoning = finiteNonNegative(usage.reasoning);
432
- if (usage.reasoning !== undefined && reasoning === undefined) return undefined;
433
- return {
434
- input,
435
- output,
436
- cacheRead,
437
- cacheWrite,
438
- ...(cacheWrite1h === undefined ? {} : { cacheWrite1h }),
439
- ...(reasoning === undefined ? {} : { reasoning }),
440
- totalTokens,
441
- cost: {
442
- input: costInput,
443
- output: costOutput,
444
- cacheRead: costCacheRead,
445
- cacheWrite: costCacheWrite,
446
- total: costTotal,
447
- },
448
- };
449
- }
450
-
451
559
  interface ExhaustionLengthMatch {
452
560
  readonly usage: AssistantMessage["usage"];
453
561
  readonly timestamp: number;
@@ -663,6 +771,128 @@ interface HostRetryCooldownReceipt {
663
771
  readonly rollback: () => void;
664
772
  }
665
773
 
774
+ /** Raised inside one physical attempt when its provider stops producing events. */
775
+ class PhysicalAttemptStall extends Error {
776
+ /** Read by `projectFailureSignal`: a stall cools and classifies as a timeout. */
777
+ readonly transportKind = "connection-timeout";
778
+ constructor() {
779
+ super("the provider stream stalled");
780
+ this.name = "PhysicalAttemptStall";
781
+ }
782
+ }
783
+
784
+ /**
785
+ * Longest silence on a live connection, keep-alive pings included, once the
786
+ * provider has reported transport activity. Anthropic pings roughly every 30
787
+ * seconds while a model thinks silently, so three missed pings mean the
788
+ * connection is gone rather than the model being slow.
789
+ */
790
+ export const TRANSPORT_SILENCE_TIMEOUT_MS = 90_000;
791
+
792
+ interface StallGuardTiming {
793
+ /** Until the provider's `start` event, opening included. */
794
+ readonly timeoutMs: number;
795
+ /** After `start` while the provider reports no transport activity. */
796
+ readonly startedTimeoutMs: number;
797
+ /** Without a byte once the provider has reported transport activity. */
798
+ readonly silenceTimeoutMs: number;
799
+ }
800
+
801
+ /**
802
+ * One physical stream, ended by `PhysicalAttemptStall` when the provider leaves
803
+ * a single wait unanswered for too long: `timeoutMs` until the provider's
804
+ * `start` event (opening included), then `startedTimeoutMs`.
805
+ *
806
+ * After `start` the server has answered and keeps the connection alive with
807
+ * pings the event stream never surfaces. A model that thinks silently (omitted
808
+ * thinking display, interleaved thinking) can leave several minutes between
809
+ * two events, so the short limit would abort a healthy request.
810
+ *
811
+ * A provider that reports transport activity (`activity`, called for every
812
+ * response chunk including pings) proves liveness directly: each report
813
+ * restarts the current wait, and after the first report a silence longer than
814
+ * `silenceTimeoutMs` ends the attempt. A connection that died after `start`
815
+ * is then caught in seconds instead of at the whole-call limit.
816
+ *
817
+ * The timer runs only while this wrapper waits on the provider, so a slow
818
+ * consumer never counts as a stall. `onStall` aborts the physical request
819
+ * before the stall is raised; the abandoned source is closed without waiting.
820
+ */
821
+ function stallGuarded(
822
+ open: () => Promise<AsyncIterable<unknown>>,
823
+ timing: StallGuardTiming,
824
+ clock: RecoveryClock,
825
+ onStall: () => void,
826
+ ): { readonly output: AsyncIterable<unknown>; readonly activity: () => void } {
827
+ let started = false;
828
+ let reportsActivity = false;
829
+ let rearm: (() => void) | undefined;
830
+ const limit = (): number => {
831
+ const phaseLimit = started ? timing.startedTimeoutMs : timing.timeoutMs;
832
+ return reportsActivity ? Math.min(timing.silenceTimeoutMs, phaseLimit) : phaseLimit;
833
+ };
834
+ const activity = (): void => {
835
+ reportsActivity = true;
836
+ rearm?.();
837
+ };
838
+ const output: AsyncIterable<unknown> = {
839
+ async *[Symbol.asyncIterator]() {
840
+ let iterator: AsyncIterator<unknown> | undefined;
841
+ let finished = false;
842
+ const guarded = async <T>(pending: Promise<T>): Promise<T> => {
843
+ void pending.catch(() => {});
844
+ let timer: RecoveryTimer | undefined;
845
+ let fail!: (error: PhysicalAttemptStall) => void;
846
+ const stalled = new Promise<never>((_resolve, reject) => {
847
+ fail = reject;
848
+ });
849
+ void stalled.catch(() => {});
850
+ const arm = (): void => {
851
+ timer?.cancel();
852
+ timer = clock.setTimer(limit(), () => {
853
+ rearm = undefined;
854
+ try {
855
+ onStall();
856
+ } finally {
857
+ fail(new PhysicalAttemptStall());
858
+ }
859
+ });
860
+ };
861
+ arm();
862
+ rearm = arm;
863
+ try {
864
+ return await Promise.race([pending, stalled]);
865
+ } finally {
866
+ if (rearm === arm) rearm = undefined;
867
+ timer?.cancel();
868
+ }
869
+ };
870
+ try {
871
+ const source = await guarded(open());
872
+ iterator = source[Symbol.asyncIterator]();
873
+ for (;;) {
874
+ const step = await guarded(iterator.next());
875
+ if (step.done === true) {
876
+ finished = true;
877
+ return;
878
+ }
879
+ if (isRecoveryStartEvent(step.value)) started = true;
880
+ yield step.value;
881
+ }
882
+ } finally {
883
+ if (!finished && iterator !== undefined) {
884
+ try {
885
+ void Promise.resolve(iterator.return?.()).catch(() => {});
886
+ } catch {
887
+ // The abandoned source is already aborted.
888
+ }
889
+ }
890
+ }
891
+ },
892
+ };
893
+ return { output, activity };
894
+ }
895
+
666
896
  /** Synchronous, cooldown-only bookkeeping shared with host retry selection. */
667
897
  export interface HostRetryCoordinator {
668
898
  readonly state: RuntimeState;
@@ -757,17 +987,34 @@ export function createLogicalProvider(
757
987
  ): LogicalProvider {
758
988
  const coordinator = createHostRetryCoordinator(deps);
759
989
  let codexTransportNoticeSent = false;
990
+ let shutDown = false;
991
+ /** Aborts every physical request, including attempts the engine handed over. */
992
+ const shutdownController = new AbortController();
760
993
 
761
994
  const diagnose = (message: string): void => {
762
995
  deps.onDiagnostic?.(message);
763
996
  };
764
997
 
765
- const selectAccount = (
766
- modelId: string,
767
- consumeRoutePin: boolean,
768
- ): LogicalServingAccount => {
998
+ const selectAccount = (modelId: string): LogicalServingAccount => {
999
+ const snapshot = deps.captureSelectionSnapshot?.(modelId);
1000
+ const accounts = snapshot?.accounts ?? deps.accounts;
1001
+ const tierModelMap = deps.tierModelMap ?? DEFAULT_CONFIG.tierModelMap;
1002
+ const nowMs = Date.now();
1003
+ const failureMessage = (fallback: string): string => {
1004
+ const group = snapshot?.group;
1005
+ if (group === undefined) return fallback;
1006
+ const candidates = group.allAccounts.map((account): AccountGroupFailureCandidate => ({
1007
+ providerId: account.providerId,
1008
+ servesModel: modelVendor !== undefined && vendorForFamily(account.family) === modelVendor && resolveLogicalServingAccount(account, modelId, tierModelMap) !== undefined,
1009
+ eligible: logicalAccountEligible(account, coordinator.state, nowMs),
1010
+ ...(account.authenticated === undefined ? {} : { authenticated: account.authenticated }),
1011
+ ...(account.exhausted === undefined ? {} : { exhausted: account.exhausted }),
1012
+ coolingDown: coordinator.state.peekCooldown(account.providerId, nowMs) !== undefined,
1013
+ }));
1014
+ return formatAccountGroupFailure({ policy: group.policy, modelId, accountLimit: group.accountLimit, candidates: [...candidates, ...group.otherCandidates] })?.message ?? fallback;
1015
+ };
769
1016
  const subscriptionVendors = new Set(
770
- deps.accounts
1017
+ accounts
771
1018
  .filter(
772
1019
  (account) =>
773
1020
  logicalProviderType(account) === "subscription" &&
@@ -791,12 +1038,11 @@ export function createLogicalProvider(
791
1038
  );
792
1039
  }
793
1040
  throw new Error(
794
- `no managed account serves the exact model id ${modelId}`,
1041
+ failureMessage(`no managed account serves the exact model id ${modelId}`),
795
1042
  );
796
1043
  }
797
1044
 
798
- const tierModelMap = deps.tierModelMap ?? DEFAULT_CONFIG.tierModelMap;
799
- const serving = deps.accounts.flatMap((account) => {
1045
+ const serving = accounts.flatMap((account) => {
800
1046
  if (vendorForFamily(account.family) !== modelVendor) return [];
801
1047
  const resolved = resolveLogicalServingAccount(account, modelId, tierModelMap);
802
1048
  return resolved === undefined ? [] : [resolved];
@@ -806,7 +1052,7 @@ export function createLogicalProvider(
806
1052
  // exact-identity contract: a virtual id that is neither exact nor explicitly
807
1053
  // mapped to a catalog member must never reach a physical provider.
808
1054
  throw new Error(
809
- `no managed account serves the exact model id ${modelId}`,
1055
+ failureMessage(`no managed account serves the exact model id ${modelId}`),
810
1056
  );
811
1057
  }
812
1058
 
@@ -814,7 +1060,6 @@ export function createLogicalProvider(
814
1060
  // (`openai`) accounts are one routing family: vendor `openai`. Ownership is
815
1061
  // fixed from the declared subscription catalog before API exact/map matches are
816
1062
  // admitted, so another vendor's API catalog cannot make a row ambiguous.
817
- const nowMs = Date.now();
818
1063
  const eligible = serving.filter(({ account }) =>
819
1064
  logicalAccountEligible(
820
1065
  {
@@ -829,31 +1074,6 @@ export function createLogicalProvider(
829
1074
  nowMs,
830
1075
  ),
831
1076
  );
832
- if (consumeRoutePin) {
833
- const pin = deps.routePin?.get();
834
- if (pin !== undefined) {
835
- const pinned = eligible.find(
836
- ({ account, resolvedModelId }) =>
837
- account.providerId === pin.destinationProviderId &&
838
- account.family === pin.destinationFamily &&
839
- logicalProviderType(account) === "subscription" &&
840
- resolvedModelId === modelId &&
841
- pin.requestedModelId === modelId,
842
- );
843
- if (pinned === undefined) {
844
- deps.routePin?.clear();
845
- } else {
846
- const consumed = deps.routePin?.consume(pin.generation, modelId);
847
- if (
848
- consumed !== undefined &&
849
- consumed.destinationProviderId === pinned.account.providerId &&
850
- consumed.destinationFamily === pinned.account.family
851
- ) {
852
- return pinned;
853
- }
854
- }
855
- }
856
- }
857
1077
  eligible.sort(
858
1078
  (left, right) =>
859
1079
  tierRank(logicalProviderType(left.account)) -
@@ -865,48 +1085,19 @@ export function createLogicalProvider(
865
1085
  // ineligible. The turn is refused before any physical request is issued;
866
1086
  // it is not parked, and no account is mutated by having been asked.
867
1087
  throw new Error(
868
- `every managed account serving ${modelId} is currently ineligible`,
1088
+ failureMessage(`every managed account serving ${modelId} is currently ineligible`),
869
1089
  );
870
1090
  }
871
1091
  return chosen;
872
1092
  };
873
1093
 
874
1094
  /**
875
- * Watch a stream that opened successfully.
1095
+ * The terminal a physical event carries, if any.
876
1096
  *
877
1097
  * A provider can accept a request, return 200, and fail part-way through the
878
- * stream — Anthropic emits `event: error` that way. That failure is exactly
879
- * as real as a rejected dispatch, so it records exactly the same cooldown;
880
- * without this, a mid-stream rate limit would leave the account looking
881
- * healthy and the host would retry straight back onto it.
882
- *
883
- * It also guarantees the host sees a terminal event.
884
- *
885
- * The host settles a turn only on a `done` or `error` event: `EventStream`
886
- * resolves its final result from `isComplete(event)` on push, and the
887
- * `forwardStream` adapter that wraps every provider stream ends with
888
- * `end(undefined)` when the source exposes no `result()`, which the
889
- * `result !== undefined` guard makes a no-op. A dispatched stream that simply
890
- * runs out therefore leaves the caller's `prompt()` pending forever, with no
891
- * timeout anywhere to break it.
892
- *
893
- * `deps.dispatch` is caller-supplied, so that is a careless caller wedging the
894
- * host permanently. Real Pi streams always terminate, so this is defense in
895
- * depth at an injectable boundary rather than a repair of a reachable hang.
896
- *
897
- * The synthesized event is `error`, never `done`. A stream that stopped
898
- * without terminating did not answer, and reporting `done` would invent an
899
- * answer that never arrived.
900
- *
901
- * It records no cooldown. Not terminating is a protocol fault in the dispatch
902
- * implementation, not evidence the account is rate limited, and
903
- * `classifyFailure` would read an unrecognizable failure as
904
- * `cooldown("unknown", …)` — cooling a healthy account for someone else's bug.
905
- * A genuinely failing account still throws, and the catch below cools it.
906
- *
907
- * `result()` is deliberately not forwarded. Exposing it would make
908
- * `forwardStream` take its `await source.result()` branch, which for exactly
909
- * this non-terminating source never settles.
1098
+ * stream (Anthropic emits `event: error` that way). That failure is exactly as
1099
+ * real as a rejected dispatch, so it records exactly the same cooldown before
1100
+ * the engine decides whether the call may move to another account.
910
1101
  */
911
1102
  const terminalAttribution = (
912
1103
  event: unknown,
@@ -983,229 +1174,767 @@ export function createLogicalProvider(
983
1174
  code: "quota_exhausted",
984
1175
  });
985
1176
 
986
- const hostWouldRetry = (error: unknown): boolean => {
1177
+ /**
1178
+ * In-call retry safety for one failed physical terminal.
1179
+ *
1180
+ * Only an account-local quota, authentication or rate-limit failure may move
1181
+ * the call to another account serving the same model. A structured refusal or
1182
+ * unknown stop, an invalid request, a context overflow and every unclassified
1183
+ * failure end the call. The classification reads structured fields only.
1184
+ */
1185
+ /**
1186
+ * Record a local host fault: no provider was reached, so the account is not
1187
+ * cooled. The bounded cause goes to the diagnostic sink, never to the reply.
1188
+ */
1189
+ const markHostStaleInstall = (
1190
+ box: AttemptRecordBox,
1191
+ account: LogicalPhysicalAccount,
1192
+ cause: unknown,
1193
+ ): void => {
1194
+ const fault = hostLoadFault(cause);
1195
+ box.hostStaleInstall = true;
1196
+ box.hostMissingDependency = fault.missingDependency;
1197
+ box.overflow = false;
1198
+ box.preStartRetryable = false;
987
1199
  try {
988
- const errorMessage =
989
- error instanceof Error
990
- ? error.message
991
- : typeof error === "string"
992
- ? error
993
- : String(error);
994
- return isRetryableAssistantError({
995
- stopReason: "error",
996
- errorMessage,
997
- } as AssistantMessage);
1200
+ deps.onDiagnostic?.(
1201
+ `logical dispatch for ${account.providerId} failed loading host code (host_stale_install: ` +
1202
+ `${fault.cause}); ${fault.missingDependency ? "reinstall the missing package" : "restart the pi session"}`,
1203
+ );
998
1204
  } catch {
999
- return false;
1205
+ // A diagnostic sink failure cannot replace a provider result.
1000
1206
  }
1001
1207
  };
1002
-
1003
- const classifiedErrorMessage = (
1004
- failure: ProviderFailureSignal,
1005
- hostRetryable: boolean,
1006
- ): string => {
1007
- const label = (() => {
1008
- switch (classifyFailure(failure).category) {
1009
- case "quota-rate-limit":
1010
- return "usage-limit";
1011
- case "terminal-auth":
1012
- case "transient-auth":
1013
- return "authentication";
1014
- case "permission":
1015
- return "permission";
1016
- case "config":
1017
- return "configuration";
1018
- case "transport":
1019
- return "network";
1020
- case "unknown":
1021
- return "unknown";
1022
- }
1023
- })();
1024
- const prefix = hostRetryable ? "provider returned error" : "provider_error";
1025
- return `${prefix} (${label})`;
1026
- };
1027
-
1028
- const projectMessage = (message: AssistantMessage, modelId: string): AssistantMessage => ({
1029
- ...message, api: LOGICAL_PROVIDER_ID, provider: LOGICAL_PROVIDER_ID, model: modelId,
1030
- ...hostFinalStopMessage(message),
1031
- });
1032
-
1033
- const projectEvent = (event: unknown, modelId: string): unknown => {
1034
- if (typeof event !== "object" || event === null) return event;
1035
- const candidate = event as Record<string, unknown>;
1036
- const key = candidate.type === "done" ? "message" : candidate.type === "error" ? "error" :
1037
- ["start", "text_start", "text_delta", "text_end", "thinking_start",
1038
- "thinking_delta", "thinking_end", "toolcall_start", "toolcall_delta",
1039
- "toolcall_end"].includes(String(candidate.type)) ? "partial" : undefined;
1040
- if (key === undefined) return event;
1041
- const value = candidate[key];
1042
- if (typeof value !== "object" || value === null) return event;
1043
- return { ...candidate, [key]: projectMessage(value as AssistantMessage, modelId) };
1044
- };
1045
-
1046
- const withPublicErrorMessage = (event: unknown, errorMessage: string): unknown => {
1047
- if (typeof event !== "object" || event === null) return event;
1048
- const candidate = event as Record<string, unknown>;
1049
- if (candidate.type !== "error" || typeof candidate.error !== "object" || candidate.error === null) {
1050
- return event;
1208
+ /** The account did nothing wrong; report it as handled so nothing cools it. */
1209
+ const hostFaultReceipt = (): HostRetryCooldownReceipt => ({ alreadyCooled: true, rollback: () => {} });
1210
+
1211
+ const retrySafetyFor = (box: AttemptRecordBox): RecoveryRetrySafety => {
1212
+ const { physicalTerminal: message, failure, overflow } = box;
1213
+ if (box.invalidated) return { status: "unsafe", reason: "unknown" };
1214
+ // Every same-family account loads the same missing host chunk: another send cannot help.
1215
+ if (box.hostStaleInstall === true) return { status: "unsafe", reason: "unknown" };
1216
+ const code = (message as { code?: unknown } | undefined)?.code;
1217
+ if (code === "refusal") return { status: "unsafe", reason: "refusal" };
1218
+ if (code === "unknown_stop") return { status: "unsafe", reason: "unknown" };
1219
+ if (overflow) return { status: "unsafe", reason: "invalid-request" };
1220
+ // Nothing after any output is ever sent again; the engine enforces the
1221
+ // same rule from the events it observed.
1222
+ if (box.sawOutput || (Array.isArray(message?.content) && message.content.length > 0)) {
1223
+ return { status: "unsafe", reason: "unknown" };
1051
1224
  }
1052
- return {
1053
- ...candidate,
1054
- error: { ...(candidate.error as AssistantMessage), errorMessage },
1055
- };
1225
+ if (failure === undefined) return { status: "unsafe", reason: "unknown" };
1226
+ const category = classifyFailure(failure).category;
1227
+ if (category === "quota-rate-limit") {
1228
+ return {
1229
+ status: "recoverable",
1230
+ action: "account",
1231
+ reason:
1232
+ failure.code === "quota_exhausted"
1233
+ ? "account-local-quota"
1234
+ : "account-local-rate-limit",
1235
+ };
1236
+ }
1237
+ if (category === "terminal-auth" || category === "transient-auth") {
1238
+ return { status: "recoverable", action: "account", reason: "account-local-auth" };
1239
+ }
1240
+ if (category === "config") return { status: "unsafe", reason: "invalid-request" };
1241
+ if (category === "transport" || box.preStartRetryable) {
1242
+ // A network or server failure before any content: the provider never
1243
+ // produced output, so one other account may serve the same request.
1244
+ return { status: "recoverable", action: "account", reason: "pre-start-transient" };
1245
+ }
1246
+ return { status: "unsafe", reason: "unknown" };
1056
1247
  };
1057
1248
 
1058
- const watchStream = (
1249
+ const projectMessage = (message: AssistantMessage, modelId: string): AssistantMessage =>
1250
+ projectPublicAssistantMessage(message, { api: LOGICAL_PROVIDER_ID, provider: LOGICAL_PROVIDER_ID, model: modelId });
1251
+
1252
+ const projectEvent = (event: unknown, modelId: string): unknown =>
1253
+ projectPublicAssistantEvent(event, { api: LOGICAL_PROVIDER_ID, provider: LOGICAL_PROVIDER_ID, model: modelId });
1254
+
1255
+ /** Outcome of one private physical attempt, recorded for the logical call. */
1256
+ interface AttemptRecordBox {
1257
+ readonly ordinal: number;
1258
+ physicalTerminal?: AssistantMessage;
1259
+ failure?: ProviderFailureSignal;
1260
+ overflow: boolean;
1261
+ /**
1262
+ * A non-terminal event other than `start` reached the engine; nothing
1263
+ * after it is resent. The provider's `start` only reports that response
1264
+ * headers arrived, so it is not output.
1265
+ */
1266
+ sawOutput: boolean;
1267
+ /**
1268
+ * The attempt's first `start`, held by the engine until the first content.
1269
+ * A successful terminal with no content publishes this exact event.
1270
+ */
1271
+ heldStart?: unknown;
1272
+ /** A setup-shaped first terminal whose raw text the host would retry. */
1273
+ preStartRetryable: boolean;
1274
+ /** Setup failed because this pi process could not load its own code. */
1275
+ hostStaleInstall?: boolean;
1276
+ /** The missing module is a bare package, so a reinstall, not a restart, helps. */
1277
+ hostMissingDependency?: boolean;
1278
+ /** Whether the caller or provider shutdown aborted this physical request. */
1279
+ cancelled?: () => boolean;
1280
+ /**
1281
+ * The session invalidated this attempt before its terminal (fresh input or
1282
+ * an operator stop, reset or enable): its cooldown was rolled back, so it
1283
+ * must end the call rather than buy another send.
1284
+ */
1285
+ invalidated: boolean;
1286
+
1287
+ receipt?: HostRetryCooldownReceipt;
1288
+ attempt: LogicalAttributionAttempt;
1289
+ account: LogicalPhysicalAccount;
1290
+ }
1291
+
1292
+ /**
1293
+ * Observe one physical attempt privately.
1294
+ *
1295
+ * Every event is forwarded unchanged to the engine's accepted-output buffer,
1296
+ * which publishes nothing until a successful terminal. A failure records the
1297
+ * account cooldown before the engine decides whether to recover, so the second
1298
+ * attempt never reselects the failing account. A stream that runs out without
1299
+ * a terminal yields one synthetic error terminal instead of hanging.
1300
+ */
1301
+ const observePhysicalAttempt = (
1059
1302
  stream: AsyncIterable<unknown>,
1060
1303
  model: unknown,
1061
1304
  options: SimpleStreamOptions | undefined,
1062
- account: LogicalPhysicalAccount,
1305
+ box: AttemptRecordBox,
1063
1306
  requestedModelId: string,
1064
1307
  dispatchedModelId: string,
1065
- attempt: LogicalAttributionAttempt,
1066
1308
  ): AsyncIterable<unknown> => ({
1067
1309
  async *[Symbol.asyncIterator]() {
1310
+ const { account, attempt } = box;
1068
1311
  let sawTerminal = false;
1312
+ // Whether any event other than a leading `start` arrived. A `start`
1313
+ // only reports that response headers arrived, so a failure after it is
1314
+ // still the stream's first real event. Every other non-terminal event
1315
+ // is output (it also sets `box.sawOutput`), and a terminal ends the
1316
+ // loop, so on the thrown path `!sawEvent` means "no content yet": this
1317
+ // is the signal the setup-only classifications key on.
1069
1318
  let sawEvent = false;
1070
- let failureReceipt: HostRetryCooldownReceipt | undefined;
1071
1319
  const recordFailureOnce = (error: unknown): HostRetryCooldownReceipt => {
1072
- failureReceipt ??= coordinator.recordFailure({
1320
+ if (box.hostStaleInstall === true) box.receipt ??= hostFaultReceipt();
1321
+ box.receipt ??= coordinator.recordFailure({
1073
1322
  account,
1074
1323
  requestedModelId,
1075
1324
  dispatchedModelId,
1076
1325
  error,
1077
1326
  });
1078
- return failureReceipt;
1327
+ return box.receipt;
1079
1328
  };
1080
- const attributeFailure = (
1081
- message: AssistantMessage,
1082
- failure: ProviderFailureSignal,
1083
- receipt: HostRetryCooldownReceipt,
1084
- ): void => {
1085
- // Host retry needs the cooldown before the terminal reaches Pi, while
1086
- // exact identity cannot be accepted until association/message_end. The
1087
- // receipt makes the early write provisional and reverses it on uncertainty.
1329
+ const attributeFailure = (message: AssistantMessage, failure: ProviderFailureSignal): void => {
1330
+ const receipt = recordFailureOnce(message);
1331
+ box.failure = failure;
1332
+ box.physicalTerminal = message;
1088
1333
  try {
1089
1334
  const accepted = attempt.fail(message, {
1090
1335
  alreadyCooled: receipt.alreadyCooled,
1091
1336
  dispatchedModelId,
1092
1337
  failure,
1093
- ...(receipt.alreadyCooled
1094
- ? { rollbackCooldown: receipt.rollback }
1095
- : {}),
1338
+ ...(receipt.alreadyCooled ? { rollbackCooldown: receipt.rollback } : {}),
1096
1339
  });
1097
- if (accepted === false && deps.attribution !== undefined) receipt.rollback();
1340
+ if (accepted === false && deps.attribution !== undefined) {
1341
+ receipt.rollback();
1342
+ box.invalidated = true;
1343
+ }
1098
1344
  } catch {
1099
- if (deps.attribution !== undefined) receipt.rollback();
1345
+ if (deps.attribution !== undefined) {
1346
+ receipt.rollback();
1347
+ box.invalidated = true;
1348
+ }
1100
1349
  }
1101
1350
  };
1102
1351
  try {
1103
- // Keep physical events intact. A corroborated subscription-exhaustion
1104
- // length terminal still becomes a bounded retryable public error.
1105
1352
  for await (const event of stream) {
1106
- const eventType =
1107
- typeof event === "object" && event !== null
1108
- ? (event as { type?: unknown }).type
1109
- : undefined;
1110
- if (eventType === "done" || eventType === "error") sawTerminal = true;
1353
+ if (!sawEvent && isRecoveryStartEvent(event)) {
1354
+ box.heldStart ??= event;
1355
+ yield event;
1356
+ continue;
1357
+ }
1111
1358
  const firstEvent = !sawEvent;
1112
1359
  sawEvent = true;
1113
1360
  const terminal = terminalAttribution(event);
1114
- let setupFailureMessage: string | undefined;
1115
- if (terminal !== undefined) {
1116
- const { message, outcome } = terminal;
1117
- if (outcome === "finish") {
1118
- const match = exhaustionLengthMatch({
1119
- message,
1120
- model,
1121
- options,
1122
- account,
1123
- currentAccounts: () => deps.accounts,
1124
- });
1125
- if (match !== undefined) {
1126
- const replacement = exhaustionLengthError(requestedModelId, match);
1127
- const failure = safeProjectFailureSignal(
1128
- replacement,
1129
- dispatchedModelId,
1130
- );
1131
- attributeFailure(
1132
- replacement,
1133
- failure,
1134
- recordFailureOnce(replacement),
1135
- );
1136
- await attempt.waitForTerminal();
1137
- yield { type: "error", reason: "error", error: replacement };
1138
- continue;
1139
- }
1140
- safeAttributionCall(() => attempt.finish(message));
1141
- } else if (outcome === "abort") {
1142
- safeAttributionCall(() => attempt.abort(message));
1143
- } else {
1144
- const failure = safeProjectFailureSignal(message, dispatchedModelId);
1145
- attributeFailure(message, failure, recordFailureOnce(message));
1146
- if (firstEvent && isUnclassifiedSetupFailure(message, failure)) {
1147
- // The physical account is cooled above exactly like any other
1148
- // failure. Only the published text changes: the raw text is
1149
- // replaced by the bounded classified message the
1150
- // rejected-dispatch path uses, so it is never published. The
1151
- // setup shape is shared by transient pre-start failures ("fetch
1152
- // failed", a 503 before `start`), pre-start context overflows
1153
- // (a Codex 400 or Anthropic 413), and deterministic setup throws.
1154
- // The raw text decides the form: an overflow publishes a fixed
1155
- // overflow message so the host compacts instead of failing
1156
- // over; a host-retryable text keeps the retryable form and
1157
- // fails over; anything else is host-final (`provider_error`),
1158
- // so a deterministic fault is not repeated on the next account.
1159
- const disposition = setupFailureDisposition(message, message.errorMessage);
1160
- setupFailureMessage =
1161
- disposition === "context-overflow"
1162
- ? SETUP_CONTEXT_OVERFLOW_MESSAGE
1163
- : classifiedErrorMessage(failure, disposition === "retryable");
1164
- try {
1165
- deps.onDiagnostic?.(
1166
- `logical dispatch for ${account.providerId} failed during stream setup; ` +
1167
- "published a classified failure instead of the raw setup error",
1168
- );
1169
- } catch {
1170
- // A diagnostic sink failure cannot replace a provider result.
1171
- }
1172
- }
1361
+ if (terminal === undefined) {
1362
+ box.sawOutput = true;
1363
+ yield event;
1364
+ continue;
1365
+ }
1366
+ sawTerminal = true;
1367
+ const { message, outcome } = terminal;
1368
+ if (message === box.physicalTerminal) {
1369
+ // A rejected dispatch: its private terminal was already cooled,
1370
+ // attributed and classified from the thrown error itself.
1371
+ yield event;
1372
+ return;
1373
+ }
1374
+ if (outcome === "finish") {
1375
+ const match = exhaustionLengthMatch({
1376
+ message,
1377
+ model,
1378
+ options,
1379
+ account,
1380
+ currentAccounts: () => deps.accounts,
1381
+ });
1382
+ if (match !== undefined) {
1383
+ // A corroborated subscription-exhaustion length terminal is an
1384
+ // account-local quota failure, never an accepted answer.
1385
+ const replacement = exhaustionLengthError(requestedModelId, match);
1386
+ attributeFailure(replacement, safeProjectFailureSignal(replacement, dispatchedModelId));
1387
+ await attempt.waitForTerminal();
1388
+ yield { type: "error", reason: "error", error: replacement };
1389
+ return;
1173
1390
  }
1391
+ box.physicalTerminal = message;
1392
+ safeAttributionCall(() => attempt.finish(message));
1174
1393
  await attempt.waitForTerminal();
1394
+ yield event;
1395
+ return;
1175
1396
  }
1176
- const projectedEvent = projectEvent(event, requestedModelId);
1177
- const publicEvent =
1178
- setupFailureMessage === undefined
1179
- ? projectedEvent
1180
- : withPublicErrorMessage(projectedEvent, setupFailureMessage);
1181
- if (terminal !== undefined && publicEvent !== event) {
1182
- const publicTerminal = terminalAttribution(publicEvent);
1183
- if (publicTerminal !== undefined) {
1184
- safeAttributionCall(() => attempt.bindPublicTerminal?.(terminal.message, publicTerminal.message));
1185
- safeAttributionCall(() => deps.onPublicTerminal?.(terminal.message, publicTerminal.message));
1397
+ if (outcome === "abort") {
1398
+ box.physicalTerminal = message;
1399
+ safeAttributionCall(() => attempt.abort(message));
1400
+ await attempt.waitForTerminal();
1401
+ yield event;
1402
+ return;
1403
+ }
1404
+ const failure = safeProjectFailureSignal(message, dispatchedModelId);
1405
+ if (firstEvent && isUnclassifiedSetupFailure(message, failure) && isHostStaleInstallFailure(message)) {
1406
+ markHostStaleInstall(box, account, message.errorMessage);
1407
+ } else if (firstEvent && isUnclassifiedSetupFailure(message, failure)) {
1408
+ // The raw setup text never leaves this attempt. Only its overflow
1409
+ // form changes the outcome: the host must still compact.
1410
+ const disposition = setupFailureDisposition(message, message.errorMessage);
1411
+ box.overflow = disposition === "context-overflow";
1412
+ // "fetch failed", a reset socket or a 503 before `start`: nothing
1413
+ // was produced, so one other account may serve the request.
1414
+ box.preStartRetryable = disposition === "retryable";
1415
+ try {
1416
+ deps.onDiagnostic?.(
1417
+ `logical dispatch for ${account.providerId} failed during stream setup; ` +
1418
+ "the raw setup error was not published",
1419
+ );
1420
+ } catch {
1421
+ // A diagnostic sink failure cannot replace a provider result.
1186
1422
  }
1423
+ } else if (setupFailureDisposition(message, message.errorMessage) === "context-overflow") {
1424
+ box.overflow = (message as { code?: unknown }).code === undefined;
1187
1425
  }
1188
- yield publicEvent;
1426
+ attributeFailure(message, failure);
1427
+ await attempt.waitForTerminal();
1428
+ yield event;
1429
+ return;
1189
1430
  }
1190
1431
  } catch (error) {
1432
+ // A thrown stream failure becomes this attempt's own private terminal,
1433
+ // so it is attributed and accounted exactly like an error event.
1434
+ if (
1435
+ box.physicalTerminal === undefined &&
1436
+ !sawEvent &&
1437
+ !hasStructuredFailureEvidence(safeProjectFailureSignal(error, dispatchedModelId)) &&
1438
+ isHostStaleInstallFailure(error)
1439
+ ) {
1440
+ markHostStaleInstall(box, account, error);
1441
+ }
1191
1442
  recordFailureOnce(error);
1443
+ if (box.hostStaleInstall === true) {
1444
+ // Already classified above; no provider disposition applies.
1445
+ } else if (box.physicalTerminal === undefined && !sawEvent) {
1446
+ // Thrown before any content (a reset socket after the response
1447
+ // headers, say): read like a failure while opening the stream.
1448
+ const raw =
1449
+ typeof error === "object" && error !== null
1450
+ ? (error as { message?: unknown }).message
1451
+ : undefined;
1452
+ const disposition = setupFailureDisposition(
1453
+ syntheticErrorMessage(requestedModelId, buildBoundedRecoveryFinalErrorMessage()),
1454
+ raw,
1455
+ );
1456
+ box.overflow = disposition === "context-overflow";
1457
+ box.preStartRetryable = disposition === "retryable";
1458
+ }
1459
+ if (box.physicalTerminal === undefined) {
1460
+ attributeFailure(
1461
+ syntheticErrorMessage(requestedModelId, buildBoundedRecoveryFinalErrorMessage()),
1462
+ safeProjectFailureSignal(error, dispatchedModelId),
1463
+ );
1464
+ }
1192
1465
  throw error;
1193
1466
  }
1194
1467
  if (sawTerminal) return;
1195
- deps.onDiagnostic?.(
1196
- `logical dispatch stream for ${account.providerId} ended with no terminal event; ` +
1197
- "reporting a synthetic failure so the turn cannot hang",
1198
- );
1468
+ try {
1469
+ deps.onDiagnostic?.(
1470
+ `logical dispatch stream for ${account.providerId} ended with no terminal event; ` +
1471
+ "reporting a synthetic failure so the turn cannot hang",
1472
+ );
1473
+ } catch {
1474
+ // A diagnostic sink failure cannot replace a provider result.
1475
+ }
1199
1476
  const syntheticMessage = syntheticErrorMessage(
1200
1477
  requestedModelId,
1201
1478
  "the dispatched stream ended without a terminal event",
1202
1479
  );
1480
+ box.physicalTerminal = syntheticMessage;
1203
1481
  safeAttributionCall(() => attempt.fail(syntheticMessage));
1204
1482
  await attempt.waitForTerminal();
1205
1483
  yield { type: "error", reason: "error", error: syntheticMessage };
1206
1484
  },
1207
1485
  });
1208
1486
 
1487
+ /** Build the request-local engine candidates for one logical call. */
1488
+ const recoveryCandidates = (
1489
+ modelId: string,
1490
+ selected: LogicalServingAccount,
1491
+ ): readonly RecoveryCandidate[] => {
1492
+ const nowMs = Date.now();
1493
+ const tierModelMap = deps.tierModelMap ?? DEFAULT_CONFIG.tierModelMap;
1494
+ const vendor = vendorForFamily(selected.account.family);
1495
+ const capability = (id: string): RecoveryModelCapability => ({
1496
+ modelId: id,
1497
+ input: [],
1498
+ supportsTools: true,
1499
+ contextWindow: 1,
1500
+ });
1501
+ const planAccount = (
1502
+ account: LogicalPhysicalAccount,
1503
+ modelIds: readonly string[],
1504
+ ): RecoveryPlanAccount => ({
1505
+ providerId: account.providerId,
1506
+ family: account.family,
1507
+ providerType: logicalProviderType(account),
1508
+ eligible:
1509
+ account.providerId === selected.account.providerId ||
1510
+ logicalAccountEligible(
1511
+ {
1512
+ providerId: account.providerId,
1513
+ exhausted: account.exhausted,
1514
+ authenticated: account.authenticated,
1515
+ },
1516
+ coordinator.state,
1517
+ nowMs,
1518
+ ),
1519
+ models: modelIds.map(capability),
1520
+ });
1521
+ const originFamily: AllowedFamily =
1522
+ selected.account.family === "openai" ? "openai-codex" : selected.account.family;
1523
+ if (selected.resolvedModelId !== modelId) {
1524
+ // An operator tier map routes this call to a different physical id. That
1525
+ // is one non-recoverable attempt: the plan admits only the selected pair.
1526
+ return buildRecoveryCandidatePlan({
1527
+ selectedModelId: selected.resolvedModelId,
1528
+ originFamily,
1529
+ accounts: [planAccount(selected.account, [selected.resolvedModelId])],
1530
+ config: {
1531
+ sameFamilyFailover: true,
1532
+ crossFamilyChainEnabled: false,
1533
+ crossFamilyChains: [],
1534
+ preferredModels: {},
1535
+ tierModelMap: {},
1536
+ },
1537
+ requiredInput: [],
1538
+ requiresTools: false,
1539
+ });
1540
+ }
1541
+ // Same vendor only, exact model only: the one recovery action in this slice
1542
+ // is another account serving the selected model.
1543
+ const accounts = deps.accounts
1544
+ .filter(
1545
+ (account) =>
1546
+ vendorForFamily(account.family) === vendor &&
1547
+ resolveLogicalServingAccount(account, modelId, tierModelMap)?.resolvedModelId === modelId,
1548
+ )
1549
+ .map((account) => planAccount(account, [modelId]));
1550
+ const ordered = [
1551
+ ...accounts.filter(({ providerId }) => providerId === selected.account.providerId),
1552
+ ...accounts.filter(({ providerId }) => providerId !== selected.account.providerId),
1553
+ ];
1554
+ // With same-family failover off, only the selected account is planned:
1555
+ // the call is one attempt.
1556
+ const sameFamilyFailover =
1557
+ deps.sameFamilyFailover ?? DEFAULT_CONFIG.sameFamilyFailover;
1558
+ const plan = buildRecoveryCandidatePlan({
1559
+ selectedModelId: modelId,
1560
+ originFamily,
1561
+ accounts: sameFamilyFailover
1562
+ ? ordered
1563
+ : ordered.filter(({ providerId }) => providerId === selected.account.providerId),
1564
+ config: {
1565
+ sameFamilyFailover: true,
1566
+ crossFamilyChainEnabled: false,
1567
+ crossFamilyChains: [],
1568
+ preferredModels: {},
1569
+ tierModelMap: {},
1570
+ },
1571
+ requiredInput: [],
1572
+ requiresTools: false,
1573
+ });
1574
+ // The selected account leads: it is the initial send.
1575
+ return [
1576
+ ...plan.filter(({ providerId }) => providerId === selected.account.providerId),
1577
+ ...plan.filter(({ providerId }) => providerId !== selected.account.providerId),
1578
+ ];
1579
+ };
1580
+
1581
+ const activeEngines = new Set<RecoveryEngine>();
1582
+
1583
+ /**
1584
+ * Open one physical attempt for the engine.
1585
+ *
1586
+ * A rejected dispatch is converted into one private error terminal so its
1587
+ * failure is classified exactly like a stream failure. The retry-safety
1588
+ * promise settles once the private attempt ends, from structured facts only.
1589
+ */
1590
+ const physicalAttempt = (input: {
1591
+ readonly request: RecoveryDispatchRequest;
1592
+ readonly account: LogicalPhysicalAccount;
1593
+ readonly model: unknown;
1594
+ readonly context: unknown;
1595
+ readonly callerOptions: SimpleStreamOptions | undefined;
1596
+ readonly requestedModelId: string;
1597
+ readonly box: AttemptRecordBox;
1598
+ readonly clock: RecoveryClock;
1599
+ readonly stallTimeoutMs: number;
1600
+ readonly startedStallTimeoutMs: number;
1601
+ readonly transportSilenceTimeoutMs: number;
1602
+ }): RecoveryPhysicalAttempt => {
1603
+ const { request, account, box, requestedModelId } = input;
1604
+ const dispatchedModelId = request.candidate.modelId;
1605
+ let settleSafety!: (safety: RecoveryRetrySafety) => void;
1606
+ const retrySafety = new Promise<RecoveryRetrySafety>((resolve) => {
1607
+ settleSafety = resolve;
1608
+ });
1609
+ const settle = (): void => settleSafety(retrySafetyFor(box));
1610
+ request.signal.addEventListener("abort", settle, { once: true });
1611
+ // The physical request outlives the engine once the attempt commits, so it
1612
+ // also listens to the caller and to provider shutdown directly.
1613
+ const physicalController = new AbortController();
1614
+ const abortPhysical = (): void => physicalController.abort();
1615
+ const abortSources = [request.signal, input.callerOptions?.signal, shutdownController.signal]
1616
+ .filter((signal): signal is AbortSignal => signal !== undefined);
1617
+ for (const signal of abortSources) {
1618
+ if (signal.aborted) abortPhysical();
1619
+ else signal.addEventListener("abort", abortPhysical, { once: true });
1620
+ }
1621
+ box.cancelled = () =>
1622
+ input.callerOptions?.signal?.aborted === true || shutdownController.signal.aborted;
1623
+ // Idempotent. Runs at the attempt's terminal as well as in the output's
1624
+ // finally: a consumer may abandon a committed stream after its terminal
1625
+ // without ever returning the iterator, and the session-lifetime shutdown
1626
+ // signal must not keep this attempt's listener.
1627
+ let released = false;
1628
+ const releaseAbortSources = (): void => {
1629
+ if (released) return;
1630
+ released = true;
1631
+ request.signal.removeEventListener("abort", settle);
1632
+ for (const signal of abortSources) signal.removeEventListener("abort", abortPhysical);
1633
+ };
1634
+ const { attempt } = box;
1635
+ const originalOnPayload = input.callerOptions?.onPayload;
1636
+ const originalOnResponse = input.callerOptions?.onResponse;
1637
+ const wrappedOnPayload: NonNullable<SimpleStreamOptions["onPayload"]> = async (
1638
+ payload,
1639
+ payloadModel,
1640
+ ) => {
1641
+ safeAttributionCall(() => attempt.onPayload(payload));
1642
+ if (originalOnPayload === undefined) return undefined;
1643
+ return await originalOnPayload(payload, payloadModel);
1644
+ };
1645
+ const wrappedOnResponse: NonNullable<SimpleStreamOptions["onResponse"]> = async (
1646
+ response,
1647
+ responseModel,
1648
+ ) => {
1649
+ safeAttributionCall(() => attempt.onResponse(response));
1650
+ if (originalOnResponse !== undefined) {
1651
+ await originalOnResponse(response, responseModel);
1652
+ }
1653
+ };
1654
+ // The engine-bound options carry `maxRetries: 0` and, for Codex only, the
1655
+ // forced SSE transport; every other caller field passes through unchanged.
1656
+ const physicalOptions: SimpleStreamOptions = {
1657
+ ...request.options,
1658
+ signal: physicalController.signal,
1659
+ onPayload: wrappedOnPayload,
1660
+ onResponse: wrappedOnResponse,
1661
+ };
1662
+ // Bound after the stall guard exists. Anthropic streams call it for every
1663
+ // response chunk, pings included; other providers ignore the field.
1664
+ let reportTransportActivity: () => void = () => {};
1665
+ const originalOnTransportActivity = (
1666
+ input.callerOptions as { onTransportActivity?: unknown } | undefined
1667
+ )?.onTransportActivity;
1668
+ (physicalOptions as { onTransportActivity?: () => void }).onTransportActivity = () => {
1669
+ reportTransportActivity();
1670
+ if (typeof originalOnTransportActivity === "function") {
1671
+ try {
1672
+ (originalOnTransportActivity as () => void)();
1673
+ } catch {
1674
+ // A caller's liveness hook cannot break the physical attempt.
1675
+ }
1676
+ }
1677
+ };
1678
+ const open = async (): Promise<AsyncIterable<unknown>> => {
1679
+ try {
1680
+ return await deps.dispatch({
1681
+ providerId: account.providerId,
1682
+ modelId: dispatchedModelId,
1683
+ context: input.context,
1684
+ options: physicalOptions,
1685
+ });
1686
+ } catch (error) {
1687
+ // Cool synchronously, then surface the rejection as a private terminal
1688
+ // so classification and attribution match a stream failure.
1689
+ const syntheticMessage = syntheticErrorMessage(
1690
+ requestedModelId,
1691
+ buildBoundedRecoveryFinalErrorMessage(),
1692
+ );
1693
+ box.failure = safeProjectFailureSignal(error, dispatchedModelId);
1694
+ if (!hasStructuredFailureEvidence(box.failure) && isHostStaleInstallFailure(error)) {
1695
+ markHostStaleInstall(box, account, error);
1696
+ box.receipt = hostFaultReceipt();
1697
+ } else {
1698
+ const raw =
1699
+ typeof error === "object" && error !== null
1700
+ ? (error as { message?: unknown }).message
1701
+ : undefined;
1702
+ const disposition = setupFailureDisposition(syntheticMessage, raw);
1703
+ box.overflow = disposition === "context-overflow";
1704
+ box.preStartRetryable = disposition === "retryable";
1705
+ box.receipt = coordinator.recordFailure({
1706
+ account,
1707
+ requestedModelId,
1708
+ dispatchedModelId,
1709
+ error,
1710
+ });
1711
+ }
1712
+ box.physicalTerminal = syntheticMessage;
1713
+ const receipt = box.receipt;
1714
+ try {
1715
+ const accepted = attempt.fail(syntheticMessage, {
1716
+ alreadyCooled: receipt.alreadyCooled,
1717
+ dispatchedModelId,
1718
+ failure: box.failure,
1719
+ ...(receipt.alreadyCooled ? { rollbackCooldown: receipt.rollback } : {}),
1720
+ });
1721
+ if (accepted === false && deps.attribution !== undefined) {
1722
+ receipt.rollback();
1723
+ box.invalidated = true;
1724
+ }
1725
+ } catch {
1726
+ if (deps.attribution !== undefined) {
1727
+ receipt.rollback();
1728
+ box.invalidated = true;
1729
+ }
1730
+ }
1731
+ return (async function* () {
1732
+ await attempt.waitForTerminal();
1733
+ yield { type: "error", reason: "error", error: syntheticMessage };
1734
+ })();
1735
+ }
1736
+ };
1737
+ // The short stall limit covers opening and the wait for `start`; every
1738
+ // later wait uses the longer started limit, or the transport silence limit
1739
+ // once the provider reports byte-level activity. Before any content a
1740
+ // stall classifies as a pre-start transient; after content it ends the call.
1741
+ const stallGuard = stallGuarded(
1742
+ open,
1743
+ {
1744
+ timeoutMs: input.stallTimeoutMs,
1745
+ startedTimeoutMs: input.startedStallTimeoutMs,
1746
+ silenceTimeoutMs: input.transportSilenceTimeoutMs,
1747
+ },
1748
+ input.clock,
1749
+ abortPhysical,
1750
+ );
1751
+ const guarded = stallGuard.output;
1752
+ reportTransportActivity = () => {
1753
+ if (physicalController.signal.aborted) return;
1754
+ stallGuard.activity();
1755
+ request.onTransportActivity();
1756
+ };
1757
+ const output: AsyncIterable<unknown> = {
1758
+ async *[Symbol.asyncIterator]() {
1759
+ try {
1760
+ for await (const event of observePhysicalAttempt(
1761
+ guarded,
1762
+ input.model,
1763
+ input.callerOptions,
1764
+ box,
1765
+ requestedModelId,
1766
+ dispatchedModelId,
1767
+ )) {
1768
+ // Classify before the terminal reaches the buffer, so the engine
1769
+ // never waits on a source it has already released. Nothing after
1770
+ // the terminal can need an abort, so its listeners go now.
1771
+ if (terminalAttribution(event) !== undefined) {
1772
+ settle();
1773
+ releaseAbortSources();
1774
+ }
1775
+ yield event;
1776
+ }
1777
+ } finally {
1778
+ settle();
1779
+ releaseAbortSources();
1780
+ }
1781
+ },
1782
+ };
1783
+ return { output, retrySafety };
1784
+ };
1785
+
1786
+ /** Named terminal facts only: a provider may attach arbitrary private fields. */
1787
+ const safeFailureMessage = (
1788
+ modelId: string,
1789
+ box: AttemptRecordBox | undefined,
1790
+ errorMessage: string,
1791
+ keepContent = false,
1792
+ ): AssistantMessage & { readonly logicalFailure: LogicalFailureEvidence } => {
1793
+ const physical = box?.physicalTerminal;
1794
+ const account = box?.account;
1795
+ const providerId = account !== undefined &&
1796
+ isCanonicalManagedProviderId(account.providerId, account.family) &&
1797
+ // Only registered slot identities: larger suffixes can look like HTTP
1798
+ // retry statuses (429/500) in the host's prose-only predicates.
1799
+ isAccountSlotIndex(
1800
+ account.providerId === account.family ? 1 : Number(account.providerId.slice(`${account.family}-account-`.length)),
1801
+ MAX_ACCOUNT_LIMIT,
1802
+ )
1803
+ ? account.providerId : undefined;
1804
+ const classified = classifyFailure(box?.failure ?? {}).category;
1805
+ const stale = box?.hostStaleInstall === true;
1806
+ const category = stale ? "host-stale-install" : box?.overflow === true ? "context-overflow" :
1807
+ box?.preStartRetryable === true && classified === "unknown" ? "transport" : classified;
1808
+ const attemptCount = box?.ordinal ?? 0;
1809
+ const evidence = `[${providerId === undefined ? "" : `physical provider: ${providerId}; `}cause: ${VISIBLE_FAILURE_CAUSES[category]}; attempts: ${attemptCount}]`;
1810
+ return {
1811
+ ...syntheticErrorMessage(
1812
+ modelId,
1813
+ `${stale ? (box?.hostMissingDependency === true ? HOST_MISSING_DEPENDENCY_MESSAGE : HOST_STALE_INSTALL_MESSAGE) : errorMessage} ${evidence}`,
1814
+ physical === undefined ? undefined : projectTerminalUsage(physical),
1815
+ physical === undefined ? undefined : finiteNonNegative(physical.timestamp),
1816
+ ),
1817
+ ...(keepContent && physical !== undefined ? { content: projectAssistantContent(physical.content) } : {}),
1818
+ ...(box?.failure?.code === undefined ? {} : { code: box.failure.code }),
1819
+ logicalFailure: Object.freeze({
1820
+ ...(providerId === undefined ? {} : { providerId }),
1821
+ category,
1822
+ attemptCount,
1823
+ }),
1824
+ };
1825
+ };
1826
+
1827
+ const withoutErrorMessage = (message: AssistantMessage): AssistantMessage => {
1828
+ const { errorMessage: _omitted, ...rest } = message;
1829
+ return rest as AssistantMessage;
1830
+ };
1831
+
1832
+ /**
1833
+ * The one public terminal for a call the engine neither accepted nor handed
1834
+ * over.
1835
+ *
1836
+ * A structured refusal or unknown stop is the provider's answer, not a
1837
+ * failure: it keeps response content and `code` with a fixed reason, which the host never
1838
+ * re-dispatches. A context overflow keeps the fixed overflow text so the host
1839
+ * still compacts. Every other failure publishes the bounded recovery text
1840
+ * with closed cause evidence; neither host retry predicate matches it.
1841
+ */
1842
+ const publicFailure = (
1843
+ modelId: string,
1844
+ result: Extract<RecoveryResult, { status: "exhausted" | "terminated" }>,
1845
+ last: AttemptRecordBox | undefined,
1846
+ ): { readonly event: unknown; readonly message: AssistantMessage } => {
1847
+ const physical = last?.physicalTerminal;
1848
+ // A provider's own aborted terminal stays aborted: the host never retries
1849
+ // it, and reporting it as an error would invite exactly that.
1850
+ const aborted =
1851
+ (result.status === "terminated" &&
1852
+ (result.reason === "caller-aborted" || result.reason === "shutdown" || result.reason === "reload")) ||
1853
+ physical?.stopReason === "aborted";
1854
+ if (!aborted && physical !== undefined && hostFinalStopMessage(physical) !== undefined) {
1855
+ const message = projectMessage(physical, modelId);
1856
+ return { event: { type: "error", reason: "error", error: message }, message };
1857
+ }
1858
+ const overflow =
1859
+ last?.overflow === true &&
1860
+ result.status === "terminated" &&
1861
+ result.reason === "invalid-request";
1862
+ const errorMessage = overflow ? SETUP_CONTEXT_OVERFLOW_MESSAGE : result.errorMessage;
1863
+ // Keep validated usage and named failure facts, never arbitrary physical
1864
+ // fields, content no consumer saw, or the provider's own error text. An
1865
+ // aborted terminal carries no error text at all.
1866
+ const projected = safeFailureMessage(modelId, last, errorMessage);
1867
+ const message: AssistantMessage = aborted
1868
+ ? { ...withoutErrorMessage(projected), stopReason: "aborted" }
1869
+ : { ...projected, stopReason: "error" };
1870
+ return {
1871
+ event: { type: "error", reason: aborted ? "aborted" : "error", error: message },
1872
+ message,
1873
+ };
1874
+ };
1875
+
1876
+ /**
1877
+ * Publish a handed-over attempt live. Its earlier events are shown as they
1878
+ * arrive, so nothing after them is ever sent again: a later failure ends the
1879
+ * call. Refusal and unknown-stop outcomes keep their existing projection.
1880
+ * Other failures keep content already shown but project named facts only;
1881
+ * overflow gets fixed overflow text and other failures get host-final text.
1882
+ */
1883
+ const publishCommitted = (
1884
+ modelId: string,
1885
+ output: AsyncIterable<unknown>,
1886
+ box: AttemptRecordBox,
1887
+ ): AsyncIterable<unknown> => ({
1888
+ async *[Symbol.asyncIterator]() {
1889
+ const bind = (physical: AssistantMessage, publicMessage: AssistantMessage): void => {
1890
+ safeAttributionCall(() => box.attempt.bindPublicTerminal?.(physical, publicMessage));
1891
+ safeAttributionCall(() => deps.onPublicTerminal?.(physical, publicMessage));
1892
+ };
1893
+ const failed = (physical: AssistantMessage, aborted: boolean): AssistantMessage => {
1894
+ if (hostFinalStopMessage(physical) !== undefined) return projectMessage(physical, modelId);
1895
+ const projected = safeFailureMessage(
1896
+ modelId, box,
1897
+ box.overflow ? SETUP_CONTEXT_OVERFLOW_MESSAGE : buildBoundedRecoveryFinalErrorMessage(),
1898
+ true,
1899
+ );
1900
+ return aborted ? { ...withoutErrorMessage(projected), stopReason: "aborted" } : projected;
1901
+ };
1902
+ try {
1903
+ for await (const event of output) {
1904
+ const terminal = terminalAttribution(event);
1905
+ if (terminal === undefined) {
1906
+ yield projectEvent(event, modelId);
1907
+ continue;
1908
+ }
1909
+ if (terminal.outcome === "finish") {
1910
+ const publicMessage = projectMessage(terminal.message, modelId);
1911
+ bind(terminal.message, publicMessage);
1912
+ yield { type: "done", reason: publicMessage.stopReason, message: publicMessage };
1913
+ return;
1914
+ }
1915
+ const aborted = terminal.outcome === "abort";
1916
+ const publicMessage = failed(terminal.message, aborted);
1917
+ bind(terminal.message, publicMessage);
1918
+ yield { type: "error", reason: aborted ? "aborted" : "error", error: publicMessage };
1919
+ return;
1920
+ }
1921
+ } catch {
1922
+ // A stall or a thrown stream failure after content: the attempt
1923
+ // recorded its own synthetic terminal, and the call ends here.
1924
+ const physical =
1925
+ box.physicalTerminal ??
1926
+ syntheticErrorMessage(modelId, buildBoundedRecoveryFinalErrorMessage());
1927
+ const aborted = box.cancelled?.() === true;
1928
+ const publicMessage: AssistantMessage = {
1929
+ ...failed(physical, aborted),
1930
+ ...(aborted ? { stopReason: "aborted" as const } : {}),
1931
+ };
1932
+ bind(physical, publicMessage);
1933
+ yield { type: "error", reason: aborted ? "aborted" : "error", error: publicMessage };
1934
+ }
1935
+ },
1936
+ });
1937
+
1209
1938
  return {
1210
1939
  api: LOGICAL_PROVIDER_ID,
1211
1940
 
@@ -1223,54 +1952,18 @@ export function createLogicalProvider(
1223
1952
  "the logical provider was asked for a request with no model id",
1224
1953
  );
1225
1954
  }
1226
- const { account, resolvedModelId } = selectAccount(modelId, true);
1227
- const providerType = logicalProviderType(account);
1228
- deps.onObservation?.({
1229
- providerId: account.providerId,
1230
- modelId: resolvedModelId,
1231
- family: account.family,
1232
- providerType,
1233
- });
1234
- const route: LogicalRouteFact = Object.freeze({
1235
- providerId: account.providerId,
1236
- family: account.family,
1237
- providerType,
1238
- accountFingerprint: account.providerId,
1239
- });
1240
- const attempt = safeBeginAttributionAttempt(deps.attribution, route);
1241
- if (account.authenticated !== undefined) {
1242
- safeAttributionCall(() => attempt.onAuthentication(account.authenticated!));
1243
- }
1244
- if (account.modelSupported !== undefined) {
1245
- safeAttributionCall(() => attempt.onModelSupport(account.modelSupported!));
1246
- }
1247
- if (account.health !== undefined) {
1248
- safeAttributionCall(() => attempt.onHealth(account.health));
1955
+ if (shutDown) {
1956
+ throw new Error("the logical provider session has shut down");
1249
1957
  }
1250
- const originalOnPayload = options?.onPayload;
1251
- const originalOnResponse = options?.onResponse;
1252
- const wrappedOnPayload: NonNullable<SimpleStreamOptions["onPayload"]> = async (
1253
- payload,
1254
- payloadModel,
1255
- ) => {
1256
- safeAttributionCall(() => attempt.onPayload(payload));
1257
- if (originalOnPayload === undefined) return undefined;
1258
- return await originalOnPayload(payload, payloadModel);
1259
- };
1260
- const wrappedOnResponse: NonNullable<SimpleStreamOptions["onResponse"]> = async (
1261
- response,
1262
- responseModel,
1263
- ) => {
1264
- safeAttributionCall(() => attempt.onResponse(response));
1265
- if (originalOnResponse !== undefined) {
1266
- await originalOnResponse(response, responseModel);
1267
- }
1268
- };
1269
- let routedOptions = options;
1270
- if (account.family === "openai-codex") {
1958
+ const selected = selectAccount(modelId);
1959
+ const candidates = recoveryCandidates(modelId, selected);
1960
+ // Codex routes are pinned to SSE before the engine sees the options, so
1961
+ // the engine reserves one bounded send for them instead of an unknown count.
1962
+ let engineOptions = options;
1963
+ if (selected.account.family === "openai-codex") {
1271
1964
  // Temporary Pi 0.99 WebSocket containment; see CODEX_FORCED_TRANSPORT.
1272
1965
  const forced = forceCodexSseOptions(options);
1273
- routedOptions = forced.options;
1966
+ engineOptions = forced.options;
1274
1967
  if (forced.overridden && options?.transport !== undefined && !codexTransportNoticeSent) {
1275
1968
  codexTransportNoticeSent = true;
1276
1969
  try {
@@ -1282,70 +1975,203 @@ export function createLogicalProvider(
1282
1975
  }
1283
1976
  }
1284
1977
  }
1285
- const attributedOptions: SimpleStreamOptions = {
1286
- ...routedOptions,
1287
- onPayload: wrappedOnPayload,
1288
- onResponse: wrappedOnResponse,
1289
- };
1290
- // Dispatch only the exact requested id or the catalog-checked destination id
1291
- // from the operator-authored tier map. A catalog head is never a fallback.
1292
- let stream: AsyncIterable<unknown>;
1293
- try {
1294
- stream = await deps.dispatch({
1295
- providerId: account.providerId,
1296
- modelId: resolvedModelId,
1297
- context,
1298
- options: attributedOptions,
1299
- });
1300
- } catch (error) {
1301
- // Cool synchronously so host retry cannot reselect this account, then
1302
- // surface the rejection as a self-owned terminal that can be correlated
1303
- // by exact object identity at message_end.
1304
- const failure = safeProjectFailureSignal(error, resolvedModelId);
1305
- const receipt = coordinator.recordFailure({
1306
- account,
1307
- requestedModelId: modelId,
1308
- dispatchedModelId: resolvedModelId,
1309
- error,
1310
- });
1311
- const syntheticMessage = syntheticErrorMessage(
1312
- modelId,
1313
- classifiedErrorMessage(failure, hostWouldRetry(error)),
1314
- );
1978
+ const boxes: AttemptRecordBox[] = [];
1979
+ const clock = deps.recoveryClock ?? SYSTEM_RECOVERY_CLOCK;
1980
+ const stallTimeoutMs =
1981
+ deps.recoveryTiming?.recoveryStallTimeoutMs ??
1982
+ DEFAULT_CONFIG.recoveryStallTimeoutMs;
1983
+ // Once the provider has answered, only the whole-call limit bounds a
1984
+ // silent wait; the engine's absolute timer still ends the call.
1985
+ const startedStallTimeoutMs = Math.max(
1986
+ stallTimeoutMs,
1987
+ deps.recoveryTiming?.recoveryAbsoluteTimeoutMs ??
1988
+ DEFAULT_CONFIG.recoveryAbsoluteTimeoutMs,
1989
+ );
1990
+ const engine = createRecoveryEngine({
1991
+ clock,
1992
+ recheck: (candidate) => {
1993
+ const account = deps.accounts.find(
1994
+ ({ providerId, family }) =>
1995
+ providerId === candidate.providerId && family === candidate.family,
1996
+ );
1997
+ if (account === undefined) return { status: "skip" };
1998
+ // Re-read live eligibility for every candidate: a second candidate is
1999
+ // checked after the first failure has already cooled its own account.
2000
+ return logicalAccountEligible(
2001
+ {
2002
+ providerId: account.providerId,
2003
+ exhausted: account.exhausted,
2004
+ authenticated: account.authenticated,
2005
+ },
2006
+ coordinator.state,
2007
+ Date.now(),
2008
+ )
2009
+ ? { status: "eligible" }
2010
+ : { status: "skip" };
2011
+ },
2012
+ reserve: () => "reserved",
2013
+ dispatch: (request) => {
2014
+ const account = deps.accounts.find(
2015
+ ({ providerId, family }) =>
2016
+ providerId === request.candidate.providerId &&
2017
+ family === request.candidate.family,
2018
+ );
2019
+ if (account === undefined) throw new Error("the candidate account disappeared");
2020
+ const providerType = logicalProviderType(account);
2021
+ deps.onObservation?.({
2022
+ providerId: account.providerId,
2023
+ modelId: request.candidate.modelId,
2024
+ family: account.family,
2025
+ providerType,
2026
+ });
2027
+ const route: LogicalRouteFact = Object.freeze({
2028
+ providerId: account.providerId,
2029
+ family: account.family,
2030
+ providerType,
2031
+ accountFingerprint: account.providerId,
2032
+ });
2033
+ const attempt = safeBeginAttributionAttempt(deps.attribution, route);
2034
+ if (account.authenticated !== undefined) {
2035
+ safeAttributionCall(() => attempt.onAuthentication(account.authenticated!));
2036
+ }
2037
+ if (account.modelSupported !== undefined) {
2038
+ safeAttributionCall(() => attempt.onModelSupport(account.modelSupported!));
2039
+ }
2040
+ if (account.health !== undefined) {
2041
+ safeAttributionCall(() => attempt.onHealth(account.health));
2042
+ }
2043
+ const box: AttemptRecordBox = {
2044
+ ordinal: boxes.length + 1,
2045
+ overflow: false,
2046
+ sawOutput: false,
2047
+ preStartRetryable: false,
2048
+ invalidated: false,
2049
+ attempt,
2050
+ account,
2051
+ };
2052
+ boxes.push(box);
2053
+ return physicalAttempt({
2054
+ request,
2055
+ account,
2056
+ model,
2057
+ context,
2058
+ callerOptions: options,
2059
+ requestedModelId: modelId,
2060
+ box,
2061
+ clock,
2062
+ stallTimeoutMs,
2063
+ startedStallTimeoutMs,
2064
+ transportSilenceTimeoutMs:
2065
+ deps.transportSilenceTimeoutMs ?? TRANSPORT_SILENCE_TIMEOUT_MS,
2066
+ });
2067
+ },
2068
+ // The attribution store retains every physical attempt's usage and cost
2069
+ // at its own terminal, under its own physical account. A failed attempt
2070
+ // whose retention failed cannot buy a second send that could not be
2071
+ // accounted for either; an accepted answer is never discarded for it,
2072
+ // and its missing record stays a reported coverage gap.
2073
+ account: async (record) => {
2074
+ const box = boxes[record.ordinal - 1];
2075
+ // An accepted or handed-over attempt has not reached its terminal yet;
2076
+ // its own terminal retains its facts.
2077
+ if (
2078
+ box === undefined ||
2079
+ record.disposition === "accepted" ||
2080
+ record.disposition === "committed"
2081
+ ) {
2082
+ return "recorded";
2083
+ }
2084
+ const outcome = await box.attempt.waitForTerminal();
2085
+ return outcome.status === "failed" ? "rejected" : "recorded";
2086
+ },
2087
+ });
2088
+ activeEngines.add(engine);
2089
+ // The engine starts now and the stream is returned at once: a consumer
2090
+ // sees nothing until the call has a publishable outcome.
2091
+ const settled = (async (): Promise<AsyncIterable<unknown>> => {
2092
+ let result: RecoveryResult;
1315
2093
  try {
1316
- const accepted = attempt.fail(syntheticMessage, {
1317
- alreadyCooled: receipt.alreadyCooled,
1318
- dispatchedModelId: resolvedModelId,
1319
- failure,
1320
- ...(receipt.alreadyCooled
1321
- ? { rollbackCooldown: receipt.rollback }
1322
- : {}),
2094
+ result = await engine.recover({
2095
+ candidates,
2096
+ context,
2097
+ ...(engineOptions === undefined ? {} : { options: engineOptions }),
2098
+ // Only the provider's `start` is held: the first content event hands
2099
+ // the attempt over and streams live, so only a failure before any
2100
+ // content may move to another account.
2101
+ publication: "stream-after-first-content",
2102
+ timing: {
2103
+ recoveryIdleTimeoutMs:
2104
+ deps.recoveryTiming?.recoveryIdleTimeoutMs ??
2105
+ DEFAULT_CONFIG.recoveryIdleTimeoutMs,
2106
+ recoveryAbsoluteTimeoutMs:
2107
+ deps.recoveryTiming?.recoveryAbsoluteTimeoutMs ??
2108
+ DEFAULT_CONFIG.recoveryAbsoluteTimeoutMs,
2109
+ },
2110
+ ...(options?.signal === undefined ? {} : { signal: options.signal }),
1323
2111
  });
1324
- if (accepted === false && deps.attribution !== undefined) receipt.rollback();
1325
- } catch {
1326
- if (deps.attribution !== undefined) receipt.rollback();
2112
+ } finally {
2113
+ activeEngines.delete(engine);
2114
+ }
2115
+ const last = boxes.at(-1);
2116
+ // Attempts the call moved past were never published. Their physical
2117
+ // failure is still real, so the session commits its account effects.
2118
+ for (const superseded of boxes.slice(0, -1)) {
2119
+ const physical = superseded.physicalTerminal;
2120
+ if (physical !== undefined) {
2121
+ safeAttributionCall(() => deps.onSupersededTerminal?.(physical));
2122
+ }
2123
+ }
2124
+ if (result.status === "accepted") {
2125
+ const physical = result.terminal;
2126
+ const publicMessage = projectMessage(physical, modelId);
2127
+ safeAttributionCall(() => last?.attempt.bindPublicTerminal?.(physical, publicMessage));
2128
+ safeAttributionCall(() => deps.onPublicTerminal?.(physical, publicMessage));
2129
+ const accepted = result.output;
2130
+ // Streaming publication accepts only an attempt that reached its
2131
+ // terminal before any content. One that sent a `start` is published
2132
+ // as its own held `start` and then its terminal, so the stream stays
2133
+ // well formed; one that sent nothing else is published as its bare
2134
+ // terminal.
2135
+ const heldStart = last?.heldStart;
2136
+ const replayProgress = last?.sawOutput === true || heldStart !== undefined;
2137
+ return (async function* () {
2138
+ let startPublished = false;
2139
+ for await (const event of accepted) {
2140
+ if (event.type === "done") yield { type: "done", reason: publicMessage.stopReason, message: publicMessage };
2141
+ else if (!replayProgress) continue;
2142
+ else if (event.type === "start" && heldStart !== undefined) {
2143
+ if (startPublished) continue;
2144
+ startPublished = true;
2145
+ yield projectEvent(heldStart, modelId);
2146
+ } else yield projectEvent(event, modelId);
2147
+ }
2148
+ })();
2149
+ }
2150
+ if (result.status === "committed") {
2151
+ // The live attempt is the last box: the engine dispatched nothing after it.
2152
+ return publishCommitted(modelId, result.output, last!);
2153
+ }
2154
+ const failure = publicFailure(modelId, result, last);
2155
+ const physical = last?.physicalTerminal;
2156
+ if (physical !== undefined) {
2157
+ safeAttributionCall(() => last?.attempt.bindPublicTerminal?.(physical, failure.message));
2158
+ safeAttributionCall(() => deps.onPublicTerminal?.(physical, failure.message));
1327
2159
  }
1328
2160
  return (async function* () {
1329
- await attempt.waitForTerminal();
1330
- yield { type: "error", reason: "error", error: syntheticMessage };
2161
+ yield failure.event;
1331
2162
  })();
1332
- }
1333
- return watchStream(
1334
- stream,
1335
- model,
1336
- options,
1337
- account,
1338
- modelId,
1339
- resolvedModelId,
1340
- attempt,
1341
- );
2163
+ })();
2164
+ void settled.catch(() => {});
2165
+ return (async function* () {
2166
+ yield* await settled;
2167
+ })();
1342
2168
  },
1343
2169
 
1344
2170
  preflight(model) {
1345
2171
  const modelId = requestedModelId(model);
1346
2172
  if (modelId === undefined) return undefined;
1347
2173
  try {
1348
- const { account, resolvedModelId } = selectAccount(modelId, false);
2174
+ const { account, resolvedModelId } = selectAccount(modelId);
1349
2175
  const providerType = logicalProviderType(account);
1350
2176
  // Everything below is read off the account that was actually
1351
2177
  // chosen. The logical provider has no health, no credential and no
@@ -1386,6 +2212,12 @@ export function createLogicalProvider(
1386
2212
  },
1387
2213
 
1388
2214
  shutdown() {
2215
+ // No late dispatch survives the session: every in-flight logical call
2216
+ // aborts its active attempt and publishes no further send.
2217
+ shutDown = true;
2218
+ for (const engine of activeEngines) engine.shutdown();
2219
+ shutdownController.abort();
2220
+ activeEngines.clear();
1389
2221
  safeAttributionCall(() => deps.attribution?.shutdown());
1390
2222
  },
1391
2223
  };