@opengeni/worker-bundle 0.24.2-canary.0 → 0.26.0-canary.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/dist/activities/agent-turn/agent-build.d.ts +3 -10
  2. package/dist/activities/agent-turn/claim.d.ts +7 -0
  3. package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
  4. package/dist/activities/agent-turn/errors.d.ts +6 -5
  5. package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
  6. package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
  7. package/dist/activities/agent-turn/history-sink.d.ts +6 -2
  8. package/dist/activities/agent-turn/quiescence.d.ts +13 -8
  9. package/dist/activities/agent-turn/tool-environment.d.ts +2 -13
  10. package/dist/activities/agent-turn/tool-policy.d.ts +4 -0
  11. package/dist/activities/agent-turn/turn-context.d.ts +9 -1
  12. package/dist/activities/codemode-dispatcher.d.ts +10 -1
  13. package/dist/activities/codex-capacity.d.ts +32 -3
  14. package/dist/activities/codex-fleet-shadow.d.ts +1 -1
  15. package/dist/activities/codex-image-generation.d.ts +1 -1
  16. package/dist/activities/codex-rotation.d.ts +8 -67
  17. package/dist/activities/goals.d.ts +1 -1
  18. package/dist/activities/google-drive-publication.d.ts +2 -13
  19. package/dist/activities/image-generation-operation.d.ts +13 -4
  20. package/dist/activities/sandbox-lease.d.ts +7 -2
  21. package/dist/activities/session-state.d.ts +4 -2
  22. package/dist/activities/types.d.ts +11 -1
  23. package/dist/activities/xai-quota.d.ts +3 -0
  24. package/dist/{activities-control-ZWH7J4KZ.js → activities-control-KS3ZPQVY.js} +233 -88
  25. package/dist/activities-control-KS3ZPQVY.js.map +1 -0
  26. package/dist/activities-control.d.ts +2 -0
  27. package/dist/{activities-turn-I4OPLQLS.js → activities-turn-GHSFZB64.js} +1109 -1865
  28. package/dist/activities-turn-GHSFZB64.js.map +1 -0
  29. package/dist/activities.d.ts +2 -0
  30. package/dist/{chunk-65JEWP2Q.js → chunk-D4ZEFOYH.js} +1 -1
  31. package/dist/chunk-D4ZEFOYH.js.map +1 -0
  32. package/dist/{chunk-RQNH6CGR.js → chunk-O2LQWJIK.js} +3250 -2061
  33. package/dist/chunk-O2LQWJIK.js.map +1 -0
  34. package/dist/github-rest-mcp.d.ts +0 -2
  35. package/dist/index.d.ts +1 -0
  36. package/dist/index.js +2 -2
  37. package/dist/index.js.map +1 -1
  38. package/dist/observability-metrics.d.ts +1 -1
  39. package/dist/sandbox-routing.d.ts +6 -0
  40. package/dist/workflow-bundle.js +94 -9
  41. package/dist/workflows/activities.d.ts +1 -1
  42. package/dist/workflows/session.d.ts +3 -0
  43. package/package.json +17 -17
  44. package/src/activities/agent-turn/agent-build.ts +21 -26
  45. package/src/activities/agent-turn/claim.ts +26 -1
  46. package/src/activities/agent-turn/codex-capacity.ts +311 -234
  47. package/src/activities/agent-turn/credential-leases.ts +36 -0
  48. package/src/activities/agent-turn/errors.ts +73 -12
  49. package/src/activities/agent-turn/failure-settlement.ts +472 -331
  50. package/src/activities/agent-turn/history-prefix.ts +51 -0
  51. package/src/activities/agent-turn/history-sink.ts +30 -6
  52. package/src/activities/agent-turn/quiescence.ts +29 -17
  53. package/src/activities/agent-turn/run.ts +22 -5
  54. package/src/activities/agent-turn/stream-attempt.ts +35 -8
  55. package/src/activities/agent-turn/tool-environment.ts +42 -32
  56. package/src/activities/agent-turn/tool-policy.ts +14 -8
  57. package/src/activities/agent-turn/turn-context.ts +13 -0
  58. package/src/activities/agent-turn/xai-capacity.ts +11 -0
  59. package/src/activities/codemode-dispatcher.ts +235 -59
  60. package/src/activities/codex-capacity.ts +107 -10
  61. package/src/activities/codex-fleet-shadow.ts +8 -1
  62. package/src/activities/codex-image-generation.ts +19 -2
  63. package/src/activities/codex-rotation.ts +45 -142
  64. package/src/activities/goals.ts +9 -9
  65. package/src/activities/google-drive-publication.ts +4 -81
  66. package/src/activities/image-generation-operation.ts +28 -3
  67. package/src/activities/run-input.ts +6 -7
  68. package/src/activities/sandbox-lease.ts +246 -87
  69. package/src/activities/session-state.ts +20 -1
  70. package/src/activities/types.ts +16 -8
  71. package/src/activities/xai-quota.ts +35 -0
  72. package/src/activities.ts +1 -0
  73. package/src/github-rest-mcp.ts +0 -82
  74. package/src/observability-metrics.ts +4 -0
  75. package/src/sandbox-routing.ts +186 -16
  76. package/src/workflows/activities.ts +1 -0
  77. package/src/workflows/session.ts +114 -8
  78. package/dist/activities/api-integrations.d.ts +0 -21
  79. package/dist/activities-control-ZWH7J4KZ.js.map +0 -1
  80. package/dist/activities-turn-I4OPLQLS.js.map +0 -1
  81. package/dist/chunk-65JEWP2Q.js.map +0 -1
  82. package/dist/chunk-RQNH6CGR.js.map +0 -1
  83. package/src/activities/api-integrations.ts +0 -151
@@ -1,17 +1,10 @@
1
1
  import {
2
2
  requestSessionTurnRecovery,
3
3
  getSessionGoal,
4
- armCodexCapacityWait,
5
4
  armXaiCapacityWait,
6
5
  reconcileXaiCapacityWait,
7
- getCodexRotationSettings,
8
6
  listCodexAccountStatuses,
9
- getSessionCodexState,
10
- setSessionCodexPinInTransaction,
11
7
  quarantineCodexCredentialForLease,
12
- setCodexCredentialExhaustedWithWakeTargets,
13
- withSessionCodexCapacityMutation,
14
- countConsecutiveReactiveRotations,
15
8
  recordUsageEvent,
16
9
  getActiveSessionHistoryItemsPaged,
17
10
  settleCodexCredentialLeaseLoss,
@@ -19,21 +12,17 @@ import {
19
12
  readLease,
20
13
  SandboxLeaseSupersededError,
21
14
  isSessionEventPersistenceError,
15
+ type CodexLeaseAccountStatus,
22
16
  } from "@opengeni/db";
23
17
  import { publishDurableSessionEvents } from "@opengeni/events";
24
18
  import { maxTurnsExceededRunState } from "@opengeni/runtime";
25
19
  import { CancelledFailure } from "@temporalio/activity";
26
20
  import {
27
21
  authoritativeCodexCapacityResetAt,
28
- chooseRotationActive,
29
22
  classifyCodexPin,
30
- computeIdleDelayMs,
31
- computeReactiveRotationResume,
32
- shardCredentialForSession,
33
- earliestCodexReset,
23
+ selectCodexCredentialLeaseForTurn,
34
24
  type CodexRotationStrategy,
35
25
  } from "../codex-rotation";
36
- import { signalCodexCapacityWakeTargets } from "../codex-capacity";
37
26
  import type { Settings } from "@opengeni/config";
38
27
  import {
39
28
  classifyCodexEncryptedArtifactRejection,
@@ -48,7 +37,7 @@ import type {
48
37
  RunAgentTurnInput,
49
38
  RunAgentTurnResult,
50
39
  } from "../types";
51
- import { createTurnCredentialLeases } from "./credential-leases";
40
+ import { CodexCredentialLeaseLostError, createTurnCredentialLeases } from "./credential-leases";
52
41
  import { createTurnHistorySink } from "./history-sink";
53
42
 
54
43
  import { BudgetExhaustedError } from "./admission";
@@ -68,7 +57,6 @@ import {
68
57
  codexCredentialCooldownUntil,
69
58
  classifyCodexCredentialFailure,
70
59
  codexUsageLimitFailurePayload,
71
- CODEX_USAGE_LIMIT_MAX_RESUME_MS,
72
60
  } from "./errors";
73
61
  import { selectRejectedProviderArtifactHistoryIds } from "./history";
74
62
  import { waitForTurnFinalizerStep, turnFinalizerCancellationSignal } from "./quiescence";
@@ -84,6 +72,8 @@ import type {
84
72
  ProviderTurnState,
85
73
  TurnControlState,
86
74
  } from "./turn-context";
75
+ import type { CodexCredentialPolicySnapshotV1 } from "@opengeni/contracts";
76
+ import { armAndReconcileCodexCapacityWait } from "../codex-capacity";
87
77
 
88
78
  export type TurnFailureDeps = {
89
79
  error: unknown;
@@ -93,7 +83,6 @@ export type TurnFailureDeps = {
93
83
  bus: ActivityServices["bus"];
94
84
  observability: ActivityServices["observability"];
95
85
  wakeSessionWorkflow: ActivityServices["wakeSessionWorkflow"];
96
- signalCodexCapacityWorkflow: ActivityServices["signalCodexCapacityWorkflow"];
97
86
  cancellationSignal: AbortSignal | undefined;
98
87
  sandboxRotationController: AbortController;
99
88
  noteCancellationRequested: () => void;
@@ -116,6 +105,125 @@ export type TurnFailureDeps = {
116
105
  acknowledgeRecoveryQuiescence: () => void;
117
106
  };
118
107
 
108
+ export type CodexDefinitiveFailureDisposition = "failover" | "wait" | "terminal";
109
+
110
+ type CodexCapacityWaitFailurePayload = {
111
+ error: string;
112
+ code: string;
113
+ detail?: string;
114
+ retryable: false;
115
+ };
116
+
117
+ /**
118
+ * Pure policy for a definitive serving-credential refusal. A policy-constrained
119
+ * account and an all-unavailable pool wait for the same selected capacity to
120
+ * recover; only a truly empty/non-allocatable pool makes an auth/forbidden
121
+ * failure terminal. A different eligible account under rotation-on policy may
122
+ * recover the same durable turn immediately.
123
+ */
124
+ export function codexDefinitiveFailureDisposition(input: {
125
+ failureKind: "auth" | "forbidden" | "rate_limit" | "quota";
126
+ rotationEnabled: boolean;
127
+ pinDisposition: "manual" | "sharded" | "clearStale" | "unpinned";
128
+ decisionKind: "active" | "allCapped" | "none";
129
+ decisionCredentialId: string | null;
130
+ servingCredentialId: string;
131
+ }): CodexDefinitiveFailureDisposition {
132
+ const alternateAvailable =
133
+ input.rotationEnabled &&
134
+ input.pinDisposition !== "manual" &&
135
+ input.decisionKind === "active" &&
136
+ input.decisionCredentialId !== null &&
137
+ input.decisionCredentialId !== input.servingCredentialId;
138
+ if (alternateAvailable) return "failover";
139
+ if (
140
+ input.failureKind === "quota" ||
141
+ input.failureKind === "rate_limit" ||
142
+ input.decisionKind === "allCapped" ||
143
+ !input.rotationEnabled ||
144
+ input.pinDisposition === "manual"
145
+ ) {
146
+ return "wait";
147
+ }
148
+ return "terminal";
149
+ }
150
+
151
+ /**
152
+ * Bound one turn to the alternate credentials that policy actually permits.
153
+ * The effective account list is already workspace/organization scoped;
154
+ * allocator-disabled rows must not enlarge the retry budget. The database
155
+ * requires a positive bound even though one-account paths never fail over.
156
+ */
157
+ export function codexCredentialFailoverLimit(
158
+ accounts: ReadonlyArray<{ id: string; allocatorEnabled: boolean }>,
159
+ servingCredentialId: string,
160
+ ): number {
161
+ const allocatableAccounts = accounts.filter((account) => account.allocatorEnabled).length;
162
+ const servingIsAllocatable = accounts.some(
163
+ (account) => account.id === servingCredentialId && account.allocatorEnabled,
164
+ );
165
+ return Math.max(1, allocatableAccounts - (servingIsAllocatable ? 1 : 0));
166
+ }
167
+
168
+ /** Build the durable waiter payload without collapsing quota refusals into 403. */
169
+ export function codexCapacityWaitFailurePayload(input: {
170
+ failureKind: "auth" | "forbidden" | "rate_limit" | "quota";
171
+ usageLimit: { resetsInSeconds: number | null } | null;
172
+ cooldownSeconds: number | null;
173
+ detail: string;
174
+ allAccounts: boolean;
175
+ }): CodexCapacityWaitFailurePayload {
176
+ if (input.failureKind === "quota") {
177
+ return codexUsageLimitFailurePayload(
178
+ input.usageLimit ?? { resetsInSeconds: input.cooldownSeconds },
179
+ input.detail,
180
+ input.allAccounts ? { allAccounts: true } : undefined,
181
+ );
182
+ }
183
+ if (input.failureKind === "rate_limit") {
184
+ return {
185
+ error: "The serving Codex subscription is temporarily rate limited.",
186
+ code: "codex_account_rate_limited",
187
+ detail: input.detail,
188
+ retryable: false,
189
+ };
190
+ }
191
+ if (input.failureKind === "auth") {
192
+ return {
193
+ error: "The serving Codex account requires reconnection.",
194
+ code: "codex_relogin_required",
195
+ detail: "the same accepted turn is waiting for the selected account to recover",
196
+ retryable: false,
197
+ };
198
+ }
199
+ return {
200
+ error: "The serving Codex account is not authorized for this request.",
201
+ code: "codex_account_forbidden",
202
+ detail: "the same accepted turn is waiting for the selected account to recover",
203
+ retryable: false,
204
+ };
205
+ }
206
+
207
+ function codexLeaseAccountsForSelection(
208
+ accounts: Awaited<ReturnType<typeof listCodexAccountStatuses>>,
209
+ ): CodexLeaseAccountStatus[] {
210
+ return accounts.map((account) => ({
211
+ ...account,
212
+ activeLeaseCount: 0,
213
+ selectionCount: 0,
214
+ lastSelectedAt: null,
215
+ }));
216
+ }
217
+
218
+ function acceptedCodexPolicySnapshot(
219
+ providerTurn: ProviderTurnState,
220
+ ): CodexCredentialPolicySnapshotV1 {
221
+ if (!providerTurn.codexPolicySnapshot) {
222
+ throw new Error("Codex accepted policy snapshot is missing after durable lease acquisition");
223
+ }
224
+ return providerTurn.codexPolicySnapshot;
225
+ }
226
+
119
227
  export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgentTurnResult> {
120
228
  const {
121
229
  error,
@@ -125,7 +233,6 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
125
233
  bus,
126
234
  observability,
127
235
  wakeSessionWorkflow,
128
- signalCodexCapacityWorkflow,
129
236
  cancellationSignal,
130
237
  sandboxRotationController,
131
238
  noteCancellationRequested,
@@ -156,9 +263,10 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
156
263
  // finished yet.
157
264
  const recoveryTurnId = attempt.turnId;
158
265
  // A true epoch supersession and a provider lifecycle transition are both
159
- // recoverable control-plane states, never session failures. The latter is
160
- // paced briefly; the next acquire waits on the durable claim and resumes
161
- // immediately when it clears.
266
+ // recoverable control-plane states, never session failures. A rotation
267
+ // persists an exact group/epoch wait marker so the workflow parks before
268
+ // another turn-worker dispatch; shorter non-rotation transitions retain
269
+ // their existing paced retry.
162
270
  const lifecycleTransition = sandboxLifecycleTransitionDiagnostic(error);
163
271
  const leaseControlError =
164
272
  error instanceof SandboxLeaseSupersededError ? error : lifecycleTransition;
@@ -175,6 +283,14 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
175
283
  const transitionPending = lifecycleTransition !== null || rotationPending;
176
284
  const deadlineRotationPending =
177
285
  rotationPending && fencedLease?.rotationReason === "provider_deadline";
286
+ const sandboxLifecycleWait = rotationPending
287
+ ? {
288
+ version: 1 as const,
289
+ sandboxGroupId: leaseControlError.sandboxGroupId,
290
+ leaseEpoch: fencedLease?.leaseEpoch ?? leaseControlError.leaseEpoch,
291
+ reason: "rotation_in_progress" as const,
292
+ }
293
+ : undefined;
178
294
  const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
179
295
  sessionId: input.sessionId,
180
296
  turnId: recoveryTurnId,
@@ -199,6 +315,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
199
315
  },
200
316
  }
201
317
  : {}),
318
+ ...(sandboxLifecycleWait ? { sandboxLifecycleWait } : {}),
202
319
  });
203
320
  if (recovery.action === "stale") {
204
321
  acknowledgeLostAttemptOwnership();
@@ -212,7 +329,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
212
329
  control.turnMetricOutcome = "recovering";
213
330
  return claimedResult({
214
331
  status: "recovering",
215
- ...(transitionPending
332
+ ...(transitionPending && !sandboxLifecycleWait
216
333
  ? {
217
334
  continueDelayMs: sandboxDeadlineRotationRecoveryDelayMs(settings),
218
335
  }
@@ -286,6 +403,12 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
286
403
  sandboxGroupId: rotation.sandboxGroupId,
287
404
  leaseEpoch: rotation.leaseEpoch,
288
405
  },
406
+ sandboxLifecycleWait: {
407
+ version: 1,
408
+ sandboxGroupId: rotation.sandboxGroupId,
409
+ leaseEpoch: rotation.leaseEpoch,
410
+ reason: "rotation_in_progress",
411
+ },
289
412
  });
290
413
  if (recovery.action === "stale") {
291
414
  acknowledgeLostAttemptOwnership();
@@ -297,10 +420,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
297
420
  await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
298
421
  control.activityStatus = "recovering";
299
422
  control.turnMetricOutcome = "recovering";
300
- return claimedResult({
301
- status: "recovering",
302
- continueDelayMs: sandboxDeadlineRotationRecoveryDelayMs(settings),
303
- });
423
+ return claimedResult({ status: "recovering" });
304
424
  } catch (recoveryError) {
305
425
  console.error(
306
426
  "sandbox deadline rotation recovery failed",
@@ -508,8 +628,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
508
628
  // recoverable, but a successor attempt or worker recovery makes this activity
509
629
  // stale and unable to clobber the shared turn/session.
510
630
  if (
511
- leases.codex.lost &&
512
- settings.codexCredentialLeasingEnabled &&
631
+ (leases.codex.lost || error instanceof CodexCredentialLeaseLostError) &&
513
632
  billingState.isCodexTurn &&
514
633
  eventing.publish &&
515
634
  attempt.turnId &&
@@ -558,10 +677,9 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
558
677
  // malformed/partial 200 stream, invalid content, prompt 4xx, or provider
559
678
  // 5xx does not classify here and therefore cannot consume another
560
679
  // subscription or duplicate a side effect.
680
+ const usageLimit = isCodexTransportError(error) ? classifyCodexUsageLimitError(error) : null;
561
681
  const codexCredentialFailure =
562
- settings.codexCredentialLeasingEnabled &&
563
- billingState.isCodexTurn &&
564
- providerTurn.effectiveCodexCredentialId
682
+ billingState.isCodexTurn && providerTurn.effectiveCodexCredentialId
565
683
  ? classifyCodexCredentialFailure(error)
566
684
  : null;
567
685
  if (
@@ -625,15 +743,26 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
625
743
  }
626
744
  : null;
627
745
  const cooldownUntil = codexCredentialCooldownUntil(codexCredentialFailure, serving, now);
628
- const statePersisted =
629
- leases.codex.holderId && leases.codex.generation !== null
746
+ const quarantineResult =
747
+ leases.codex.holderId &&
748
+ leases.codex.generation !== null &&
749
+ providerTurn.effectiveCodexCredentialVersion !== null
630
750
  ? await quarantineCodexCredentialForLease(db, {
631
751
  accountId: input.accountId,
632
752
  workspaceId: input.workspaceId,
753
+ sessionId: input.sessionId,
633
754
  turnId: attempt.turnId,
755
+ attemptId: input.attemptId,
756
+ executionGeneration: attempt.executionGeneration,
757
+ workflowId: input.workflowId,
758
+ workflowRunId: input.workflowRunId,
759
+ dispatchId: attempt.dispatchId,
760
+ expectedRedispatches: attempt.redispatchesAtDispatch,
634
761
  credentialId: providerTurn.effectiveCodexCredentialId,
762
+ credentialVersion: providerTurn.effectiveCodexCredentialVersion,
635
763
  holderId: leases.codex.holderId,
636
764
  generation: leases.codex.generation,
765
+ maxFailovers: providerTurn.codexCredentialFailoverLimit,
637
766
  quarantine:
638
767
  codexCredentialFailure.kind === "auth"
639
768
  ? {
@@ -653,7 +782,46 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
653
782
  cooldownKind: codexCredentialFailure.kind,
654
783
  },
655
784
  })
656
- : false;
785
+ : null;
786
+ if (
787
+ quarantineResult?.action === "credential_changed" &&
788
+ leases.codex.holderId &&
789
+ leases.codex.generation !== null
790
+ ) {
791
+ const recovery = await settleCodexCredentialLeaseLoss(db, {
792
+ accountId: input.accountId,
793
+ workspaceId: input.workspaceId,
794
+ sessionId: input.sessionId,
795
+ turnId: attempt.turnId,
796
+ attemptId: input.attemptId,
797
+ holderId: leases.codex.holderId,
798
+ generation: leases.codex.generation,
799
+ expectedRedispatches: attempt.redispatchesAtDispatch,
800
+ checkpointDurable: true,
801
+ recoveryPayload: {
802
+ triggerEventId: attempt.triggerEventId!,
803
+ reason: "codex_credential_version_changed",
804
+ },
805
+ failedPayload: {},
806
+ });
807
+ if (recovery.action === "recovering") {
808
+ leases.codex.held = false;
809
+ await publishDurableSessionEvents(
810
+ bus,
811
+ input.workspaceId,
812
+ input.sessionId,
813
+ recovery.events,
814
+ );
815
+ control.activityStatus = "recovering";
816
+ control.turnMetricOutcome = "recovering";
817
+ return claimedResult({ status: "recovering" });
818
+ }
819
+ acknowledgeLostAttemptOwnership();
820
+ control.activityStatus = "cancelled";
821
+ control.turnMetricOutcome = "cancelled";
822
+ return claimedResult({ status: "cancelled" });
823
+ }
824
+ const statePersisted = quarantineResult?.action === "recorded";
657
825
  if (!statePersisted && leases.codex.holderId && leases.codex.generation !== null) {
658
826
  leases.codex.lost = true;
659
827
  return await settleLostCodexAttempt(
@@ -663,26 +831,129 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
663
831
  true,
664
832
  );
665
833
  }
666
- const [rotation, accounts] = await Promise.all([
667
- getCodexRotationSettings(db, input.workspaceId).catch(() => null),
668
- listCodexAccountStatuses(db, input.workspaceId).catch(() => []),
669
- ]);
670
- const decision = rotation
671
- ? chooseRotationActive({
672
- rotationStrategy: rotation.rotationStrategy as CodexRotationStrategy,
673
- activeCredentialId: rotation.activeCredentialId,
674
- priorCredentialId: providerTurn.effectiveCodexCredentialId,
675
- accounts,
676
- now: new Date(),
677
- })
678
- : ({ kind: "none" } as const);
679
- const candidateAvailable =
680
- statePersisted &&
681
- Boolean(rotation?.rotationEnabled && rotation?.leaseRotationEnabled) &&
682
- decision.kind === "active" &&
683
- decision.credentialId !== providerTurn.effectiveCodexCredentialId;
834
+ if (
835
+ quarantineResult?.action === "recorded" &&
836
+ quarantineResult.exhausted &&
837
+ leases.codex.holderId &&
838
+ leases.codex.generation !== null
839
+ ) {
840
+ const settlement = await settleCodexCredentialFailover(db, {
841
+ accountId: input.accountId,
842
+ workspaceId: input.workspaceId,
843
+ sessionId: input.sessionId,
844
+ turnId: attempt.turnId,
845
+ attemptId: input.attemptId,
846
+ holderId: leases.codex.holderId,
847
+ generation: leases.codex.generation,
848
+ expectedRedispatches: attempt.redispatchesAtDispatch,
849
+ maxFailovers: quarantineResult.maxFailovers,
850
+ recoveryPayload: {
851
+ triggerEventId: attempt.triggerEventId!,
852
+ reason: "codex_credential_failover",
853
+ credentialId: providerTurn.effectiveCodexCredentialId,
854
+ failureKind: codexCredentialFailure.kind,
855
+ },
856
+ failedPayload: {
857
+ error:
858
+ "Automatic Codex credential failover stopped after every bounded account attempt was consumed. Send a new message after checking account health or capacity.",
859
+ code: "codex_credential_failover_exhausted",
860
+ retryable: false,
861
+ recovery: "user_message",
862
+ failoverCount: quarantineResult.failoverCount,
863
+ maxFailovers: quarantineResult.maxFailovers,
864
+ },
865
+ });
866
+ if (settlement.action === "limit_exceeded") {
867
+ leases.codex.held = false;
868
+ await publishDurableSessionEvents(
869
+ bus,
870
+ input.workspaceId,
871
+ input.sessionId,
872
+ settlement.events,
873
+ );
874
+ control.activityError = error;
875
+ control.activityStatus = "idle";
876
+ control.turnMetricOutcome = "failed";
877
+ await deliverFailedChildTurnToParent(
878
+ { db, bus, settings, observability, wakeSessionWorkflow },
879
+ input.workspaceId,
880
+ input.sessionId,
881
+ attempt.turnId,
882
+ );
883
+ return claimedResult({ status: "idle" });
884
+ }
885
+ if (settlement.action === "stale") {
886
+ acknowledgeLostAttemptOwnership();
887
+ control.activityStatus = "cancelled";
888
+ control.turnMetricOutcome = "cancelled";
889
+ return claimedResult({ status: "cancelled" });
890
+ }
891
+ throw new Error("Exhausted Codex failover receipt unexpectedly recovered");
892
+ }
893
+ let accounts: Awaited<ReturnType<typeof listCodexAccountStatuses>>;
894
+ try {
895
+ accounts = await listCodexAccountStatuses(db, input.workspaceId);
896
+ } catch (metadataError) {
897
+ // Current account health/cooldown metadata is still required after
898
+ // quarantine. Operational database failures re-enter the existing
899
+ // exact-attempt recovery lane; no policy choice is made from partial
900
+ // account state.
901
+ const recoveryFailure = postClaimDatabaseRecoveryFailure({
902
+ error: metadataError,
903
+ turnId: attempt.turnId,
904
+ triggerEventId: attempt.triggerEventId!,
905
+ executionGeneration: attempt.executionGeneration,
906
+ });
907
+ if (recoveryFailure) {
908
+ control.activityStatus = "recovering";
909
+ control.turnMetricOutcome = "recovering";
910
+ control.activityError = metadataError;
911
+ throw recoveryFailure;
912
+ }
913
+ throw metadataError;
914
+ }
915
+ const acceptedPolicy = acceptedCodexPolicySnapshot(providerTurn);
916
+ const selected = selectCodexCredentialLeaseForTurn({
917
+ context: {
918
+ accounts: codexLeaseAccountsForSelection(accounts),
919
+ activeCredentialId: acceptedPolicy.activeCredentialId,
920
+ rotationEnabled: acceptedPolicy.rotationEnabled,
921
+ rotationStrategy: acceptedPolicy.rotationStrategy,
922
+ existingCredentialId: null,
923
+ failedCredentialIds: [providerTurn.effectiveCodexCredentialId],
924
+ policyScope: null,
925
+ unavailableDiagnostics: [],
926
+ },
927
+ sessionId: input.sessionId,
928
+ sessionPinnedCredentialId: acceptedPolicy.pinnedCredentialId,
929
+ sessionPinSource: acceptedPolicy.pinSource,
930
+ sessionLastCredentialId: acceptedPolicy.lastCredentialId,
931
+ now,
932
+ });
933
+ const decisionKind =
934
+ selected.decision.kind === "allocatorDisabled" ? "allCapped" : selected.decision.kind;
935
+ const pinDisposition = classifyCodexPin({
936
+ pinnedCredentialId: acceptedPolicy.pinnedCredentialId,
937
+ pinSource: acceptedPolicy.pinSource,
938
+ strategy: acceptedPolicy.rotationStrategy as CodexRotationStrategy,
939
+ rotationEnabled: acceptedPolicy.rotationEnabled,
940
+ });
941
+ const failureDisposition = codexDefinitiveFailureDisposition({
942
+ failureKind: codexCredentialFailure.kind,
943
+ rotationEnabled: acceptedPolicy.rotationEnabled,
944
+ pinDisposition,
945
+ decisionKind,
946
+ decisionCredentialId: selected.decision.kind === "active" ? selected.credentialId : null,
947
+ servingCredentialId: providerTurn.effectiveCodexCredentialId,
948
+ });
949
+ const maxFailovers = providerTurn.codexCredentialFailoverLimit;
684
950
 
685
- if (candidateAvailable && leases.codex.holderId && leases.codex.generation !== null) {
951
+ if (
952
+ statePersisted &&
953
+ failureDisposition === "failover" &&
954
+ leases.codex.holderId &&
955
+ leases.codex.generation !== null
956
+ ) {
686
957
  const settlement = await settleCodexCredentialFailover(db, {
687
958
  accountId: input.accountId,
688
959
  workspaceId: input.workspaceId,
@@ -692,7 +963,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
692
963
  holderId: leases.codex.holderId,
693
964
  generation: leases.codex.generation,
694
965
  expectedRedispatches: attempt.redispatchesAtDispatch,
695
- maxFailovers: Math.max(1, accounts.length),
966
+ maxFailovers,
696
967
  recoveryPayload: {
697
968
  triggerEventId: attempt.triggerEventId!,
698
969
  reason: "codex_credential_failover",
@@ -700,6 +971,15 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
700
971
  failureKind: codexCredentialFailure.kind,
701
972
  ...(cooldownUntil ? { cooldownUntil: cooldownUntil.toISOString() } : {}),
702
973
  },
974
+ failedPayload: {
975
+ error:
976
+ "Automatic Codex credential failover stopped after every bounded account attempt was consumed. Send a new message after checking account health or capacity.",
977
+ code: "codex_credential_failover_exhausted",
978
+ retryable: false,
979
+ recovery: "user_message",
980
+ failoverCount: quarantineResult.failoverCount,
981
+ maxFailovers: quarantineResult.maxFailovers,
982
+ },
703
983
  });
704
984
  observability.incrementCounter({
705
985
  name: "opengeni_codex_failover_settlements_total",
@@ -738,8 +1018,137 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
738
1018
  acknowledgeLostAttemptOwnership();
739
1019
  control.activityStatus = "cancelled";
740
1020
  control.turnMetricOutcome = "cancelled";
1021
+ return claimedResult({ status: "cancelled" });
1022
+ }
1023
+ if (settlement.action === "limit_exceeded") {
1024
+ leases.codex.held = false;
1025
+ await publishDurableSessionEvents(
1026
+ bus,
1027
+ input.workspaceId,
1028
+ input.sessionId,
1029
+ settlement.events,
1030
+ );
1031
+ control.activityError = error;
1032
+ control.activityStatus = "idle";
1033
+ control.turnMetricOutcome = "failed";
1034
+ await deliverFailedChildTurnToParent(
1035
+ { db, bus, settings, observability, wakeSessionWorkflow },
1036
+ input.workspaceId,
1037
+ input.sessionId,
1038
+ attempt.turnId,
1039
+ );
1040
+ return claimedResult({ status: "idle" });
1041
+ }
1042
+ }
1043
+
1044
+ if (
1045
+ statePersisted &&
1046
+ failureDisposition === "wait" &&
1047
+ leases.codex.holderId &&
1048
+ leases.codex.generation !== null
1049
+ ) {
1050
+ let goal: Awaited<ReturnType<typeof getSessionGoal>>;
1051
+ try {
1052
+ goal = await getSessionGoal(db, input.workspaceId, input.sessionId);
1053
+ } catch (metadataError) {
1054
+ const recoveryFailure = postClaimDatabaseRecoveryFailure({
1055
+ error: metadataError,
1056
+ turnId: attempt.turnId,
1057
+ triggerEventId: attempt.triggerEventId!,
1058
+ executionGeneration: attempt.executionGeneration,
1059
+ });
1060
+ if (recoveryFailure) {
1061
+ control.activityStatus = "recovering";
1062
+ control.turnMetricOutcome = "recovering";
1063
+ control.activityError = metadataError;
1064
+ throw recoveryFailure;
1065
+ }
1066
+ throw metadataError;
1067
+ }
1068
+ const activeGoal = goal?.status === "active" ? goal : null;
1069
+ const exactProviderReset =
1070
+ codexCredentialFailure.cooldownSeconds !== null &&
1071
+ Number.isFinite(codexCredentialFailure.cooldownSeconds) &&
1072
+ codexCredentialFailure.cooldownSeconds > 0;
1073
+ const policyCredentialId =
1074
+ pinDisposition === "manual" && acceptedPolicy.pinnedCredentialId
1075
+ ? acceptedPolicy.pinnedCredentialId
1076
+ : !acceptedPolicy.rotationEnabled
1077
+ ? acceptedPolicy.activeCredentialId
1078
+ : null;
1079
+ const capacityAccounts = policyCredentialId
1080
+ ? accounts.filter((account) => account.id === policyCredentialId)
1081
+ : accounts;
1082
+ const authoritativeResetAt = exactProviderReset
1083
+ ? (authoritativeCodexCapacityResetAt(capacityAccounts, now) ?? cooldownUntil)
1084
+ : null;
1085
+ const allAccounts =
1086
+ acceptedPolicy.rotationEnabled &&
1087
+ pinDisposition !== "manual" &&
1088
+ decisionKind === "allCapped";
1089
+ const failurePayload = codexCapacityWaitFailurePayload({
1090
+ failureKind: codexCredentialFailure.kind,
1091
+ usageLimit,
1092
+ cooldownSeconds: codexCredentialFailure.cooldownSeconds,
1093
+ detail:
1094
+ codexCredentialFailure.kind === "quota"
1095
+ ? error instanceof Error
1096
+ ? error.message
1097
+ : String(error)
1098
+ : "the same accepted turn is waiting for eligible credential capacity",
1099
+ allAccounts,
1100
+ });
1101
+ const evaluated = await armAndReconcileCodexCapacityWait(
1102
+ { db, bus },
1103
+ {
1104
+ accountId: input.accountId,
1105
+ workspaceId: input.workspaceId,
1106
+ sessionId: input.sessionId,
1107
+ turnId: attempt.turnId,
1108
+ attemptId: input.attemptId,
1109
+ workflowId: input.workflowId,
1110
+ goalId: activeGoal?.id ?? null,
1111
+ goalVersion: activeGoal?.version ?? null,
1112
+ earliestResetAt: authoritativeResetAt,
1113
+ resetKind: authoritativeResetAt
1114
+ ? "authoritative"
1115
+ : codexCredentialFailure.kind === "auth" ||
1116
+ codexCredentialFailure.kind === "forbidden"
1117
+ ? "mutation_only"
1118
+ : "bounded_refresh",
1119
+ failurePayload,
1120
+ leaseFence: {
1121
+ holderId: leases.codex.holderId,
1122
+ generation: leases.codex.generation,
1123
+ },
1124
+ expectedRedispatches: attempt.redispatchesAtDispatch,
1125
+ },
1126
+ { onArmed: () => (leases.codex.held = false) },
1127
+ );
1128
+ control.activityError = error;
1129
+ if (evaluated.action === "resumed") {
1130
+ control.activityStatus = "recovering";
1131
+ control.turnMetricOutcome = "recovering";
741
1132
  return claimedResult({ status: "recovering" });
742
1133
  }
1134
+ if (evaluated.action === "waiting") {
1135
+ control.activityError = error;
1136
+ control.activityStatus = "waiting_capacity";
1137
+ control.turnMetricOutcome = "recovering";
1138
+ return claimedResult({
1139
+ status: "waiting_capacity",
1140
+ capacityWait: {
1141
+ waiterId: evaluated.waiter.id,
1142
+ generation: evaluated.waiter.generation,
1143
+ nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
1144
+ wakeRevision: evaluated.waiter.wakeRevision,
1145
+ },
1146
+ });
1147
+ }
1148
+ acknowledgeLostAttemptOwnership();
1149
+ control.activityStatus = "cancelled";
1150
+ control.turnMetricOutcome = "cancelled";
1151
+ return claimedResult({ status: "cancelled" });
743
1152
  }
744
1153
  }
745
1154
  }
@@ -882,262 +1291,26 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
882
1291
  control.activityError = error;
883
1292
  return claimedResult({ status: "recovering" });
884
1293
  }
885
- // A ChatGPT/Codex usage cap (429 usage_limit_reached) is account state,
886
- // NOT an agent failure: surface the precise, actionable message (so the
887
- // user sees the reset window) but idle the session — never go terminal,
888
- // which would reject the user's next message after the cap lifts. The
889
- // payload is retryable:false so the generic provider-backpressure auto-retry
890
- // does not loop. For an active goal we hold the continuation for the reported
891
- // reset window (capped) so it resumes itself when access returns, instead of
892
- // hammering the capped backend.
893
- const usageLimit = isCodexTransportError(error) ? classifyCodexUsageLimitError(error) : null;
1294
+ // The leased credential path above normally quarantines quota state and
1295
+ // either recovers the same turn or arms a durable capacity wait. This narrow
1296
+ // fallback covers failures before a credential lease existed, or a failed
1297
+ // durable checkpoint where replay would be unsafe. Keep the session usable,
1298
+ // but never synthesize another turn or walk an unfenced legacy pointer.
894
1299
  if (usageLimit && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
895
- const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
896
- const goalActive = Boolean(goal && goal.status === "active");
897
1300
  await flushRuntimeBatcher();
898
1301
  await historySink.reconcileConversationTruth();
899
- // --- P3 reactive rotation (gated; re-fetch fresh state on this already-failed,
900
- // already-idling path). Mark THIS account cooling until its reset, then CONSULT the
901
- // engine over fresh accounts to decide continueDelayMs: a fast 0-delay re-dispatch
902
- // when another account is available, or idle-until-earliest when all are capped. The
903
- // catch deliberately does NOT move the active pointer — the re-dispatched turn's
904
- // proactive seam (turn-start) is the single authoritative pointer-move + strip site.
905
- let rotated = false;
906
- let rotationResumeMs: number | null = null; // 0 ⇒ a candidate is available; re-dispatch now
907
- let rotationResumeIdleUntilReset = false; // circuit-breaker fall (Finding 1b) ⇒ MANDATORY hold
908
- let allCappedResetAt: Date | null = null; // set ⇒ every account capped; idle until this
909
- let capacityAuthoritativeResetAt: Date | null = null;
910
- let capacityNeedsBoundedRefresh = false;
911
- if (providerTurn.effectiveCodexCredentialId) {
912
- const [rotation, sessionCodex] = await Promise.all([
913
- getCodexRotationSettings(db, input.workspaceId).catch(() => null),
914
- getSessionCodexState(db, input.workspaceId, input.sessionId).catch(() => null),
915
- ]);
916
- const reactiveStrategy = (rotation?.rotationStrategy ??
917
- "most_remaining") as CodexRotationStrategy;
918
- const reactiveDisposition = classifyCodexPin({
919
- pinnedCredentialId: sessionCodex?.pinnedCredentialId ?? null,
920
- pinSource: sessionCodex?.pinSource ?? null,
921
- strategy: reactiveStrategy,
922
- rotationEnabled: Boolean(rotation?.rotationEnabled),
923
- });
924
- const reactiveSharded = reactiveDisposition === "sharded";
925
- const rotating =
926
- Boolean(rotation?.rotationEnabled || rotation?.leaseRotationEnabled) &&
927
- reactiveDisposition !== "manual";
928
- if (rotating && rotation) {
929
- const accounts = await listCodexAccountStatuses(db, input.workspaceId).catch(() => []);
930
- const serving =
931
- accounts.find((a) => a.id === providerTurn.effectiveCodexCredentialId) ?? null;
932
- // Both provider allowance windows bind. Use the same canonical
933
- // quarantine calculation as the fenced failover path so a short
934
- // five-hour reset can never overwrite a later weekly reset.
935
- const until = codexCredentialCooldownUntil(
936
- { kind: "quota", cooldownSeconds: usageLimit.resetsInSeconds },
937
- serving,
938
- new Date(),
939
- )!;
940
- // Finding 1a: INSPECT the cooldown-write result. A swallowed best-effort
941
- // write whose failure went unnoticed is exactly what lets the next proactive
942
- // rank re-pick this just-capped account (stale-low cached usedPercent, not
943
- // cooling) — so capture whether it PERSISTED and feed it into the resume floor.
944
- const cooldownMutation = await setCodexCredentialExhaustedWithWakeTargets(
945
- db,
946
- input.workspaceId,
947
- providerTurn.effectiveCodexCredentialId,
948
- until,
949
- "quota",
950
- ).catch(() => null);
951
- const cooldownPersisted = cooldownMutation?.result ?? false;
952
- if (cooldownMutation) {
953
- await signalCodexCapacityWakeTargets(
954
- { signalCodexCapacityWorkflow, wakeSessionWorkflow },
955
- cooldownMutation.wakeTargets,
956
- );
957
- }
958
- // Re-rank over the fresh accounts; the in-memory list predates the cooldown
959
- // write, so stamp the just-cooled account so the engine excludes it now. The
960
- // serving account is thus walked AT MOST ONCE per turn (invariant 4: bounded).
961
- const fresh = accounts.map((a) =>
962
- a.id === providerTurn.effectiveCodexCredentialId
963
- ? { ...a, exhaustedUntil: until, exhaustedKind: "quota" as const }
964
- : a,
965
- );
966
- if (reactiveSharded) {
967
- // AM-5: RE-SHARD over the healthy survivors (the just-capped serving account is
968
- // marked cooling in `fresh` → excluded) so sessions sharing a capped account
969
- // spread across the pool rather than re-concentrating on one first-eligible
970
- // failover. AM-3: DURABLY REWRITE the session's POLICY pin to the new home —
971
- // selectCodexCredentialForTurn returns a cooling pinned account with NO
972
- // exhaustion check, so a pointer-only move would leave the re-dispatched turn on
973
- // the capped pin. Like the classic path we do NOT touch the workspace active
974
- // pointer; the session pin is the sharded home.
975
- const newHome = shardCredentialForSession({
976
- sessionId: input.sessionId,
977
- accounts: fresh,
978
- now: new Date(),
979
- });
980
- if (newHome) {
981
- rotated = true;
982
- const pinMutation = await withSessionCodexCapacityMutation(
983
- db,
984
- {
985
- workspaceId: input.workspaceId,
986
- reason: "codex_policy_pin_resharded",
987
- },
988
- async (tx) => {
989
- const changed = await setSessionCodexPinInTransaction(
990
- tx,
991
- input.workspaceId,
992
- input.sessionId,
993
- newHome,
994
- "policy",
995
- {
996
- expected: {
997
- pinnedCredentialId: sessionCodex?.pinnedCredentialId ?? null,
998
- pinSource: sessionCodex?.pinSource ?? null,
999
- },
1000
- },
1001
- );
1002
- return { result: changed, changed };
1003
- },
1004
- ).catch(() => null);
1005
- if (pinMutation) {
1006
- await signalCodexCapacityWakeTargets(
1007
- { signalCodexCapacityWorkflow, wakeSessionWorkflow },
1008
- pinMutation.wakeTargets,
1009
- );
1010
- }
1011
- const priorConsecutiveRotations = await countConsecutiveReactiveRotations(
1012
- db,
1013
- input.workspaceId,
1014
- input.sessionId,
1015
- ).catch(() => 0);
1016
- const resume = computeReactiveRotationResume({
1017
- cooldownPersisted,
1018
- priorConsecutiveRotations,
1019
- connectedAccountCount: accounts.length,
1020
- });
1021
- rotationResumeMs = resume.continueDelayMs;
1022
- rotationResumeIdleUntilReset = resume.idleUntilReset;
1023
- } else {
1024
- // Every account capped/cooling → idle until the earliest reset across all.
1025
- rotated = true;
1026
- allCappedResetAt = earliestCodexReset(fresh, new Date());
1027
- capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
1028
- capacityNeedsBoundedRefresh = true;
1029
- }
1030
- } else {
1031
- const decision = chooseRotationActive({
1032
- rotationStrategy: reactiveStrategy,
1033
- activeCredentialId: rotation.activeCredentialId,
1034
- priorCredentialId: providerTurn.effectiveCodexCredentialId,
1035
- accounts: fresh,
1036
- now: new Date(),
1037
- });
1038
- if (decision.kind === "active") {
1039
- rotated = true;
1040
- // Finding 1: a live candidate normally re-dispatches NOW (0). Two second-order
1041
- // faults would turn that 0 into a hot loop, so bound it. Count the consecutive
1042
- // reactive failovers since the last successful turn (this one is not yet
1043
- // published) and combine with the cooldown-persistence result.
1044
- const priorConsecutiveRotations = await countConsecutiveReactiveRotations(
1045
- db,
1046
- input.workspaceId,
1047
- input.sessionId,
1048
- ).catch(() => 0);
1049
- const resume = computeReactiveRotationResume({
1050
- cooldownPersisted,
1051
- priorConsecutiveRotations,
1052
- connectedAccountCount: accounts.length,
1053
- });
1054
- rotationResumeMs = resume.continueDelayMs; // 0 (happy path), a slow-retry floor, or the circuit-breaker idle
1055
- rotationResumeIdleUntilReset = resume.idleUntilReset; // true only on the circuit-breaker fall (MANDATORY hold)
1056
- } else if (decision.kind === "allCapped") {
1057
- rotated = true;
1058
- allCappedResetAt = decision.earliestResetAt;
1059
- capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
1060
- capacityNeedsBoundedRefresh = true;
1061
- }
1062
- // kind:"none" → fall through to today's single-account idle.
1063
- }
1064
- }
1065
- }
1066
-
1067
- const failurePayload = allCappedResetAt
1068
- ? codexUsageLimitFailurePayload(
1069
- {
1070
- resetsInSeconds: Math.ceil(Math.max(0, allCappedResetAt.getTime() - Date.now()) / 1000),
1071
- },
1072
- error instanceof Error ? error.message : String(error),
1073
- { allAccounts: true },
1074
- )
1075
- : codexUsageLimitFailurePayload(
1076
- usageLimit,
1077
- error instanceof Error ? error.message : String(error),
1078
- );
1079
- // A live alternate is still handled by the existing immediate,
1080
- // same-policy continuation path. When no alternate exists (all capped,
1081
- // or a single non-rotating account), persist the native capacity wait
1082
- // instead of an in-memory delay/user-message recovery.
1083
- if (rotationResumeMs === null) {
1084
- const providerResetAt =
1085
- capacityAuthoritativeResetAt ??
1086
- (usageLimit.resetsInSeconds !== null &&
1087
- Number.isFinite(usageLimit.resetsInSeconds) &&
1088
- usageLimit.resetsInSeconds > 0
1089
- ? new Date(Date.now() + Math.ceil(usageLimit.resetsInSeconds) * 1000)
1090
- : null);
1091
- const armed = await armCodexCapacityWait(db, {
1092
- accountId: input.accountId,
1093
- workspaceId: input.workspaceId,
1094
- sessionId: input.sessionId,
1095
- turnId: attempt.turnId,
1096
- attemptId: input.attemptId,
1097
- workflowId: input.workflowId,
1098
- goalId: goalActive && goal ? goal.id : null,
1099
- goalVersion: goalActive && goal ? goal.version : null,
1100
- earliestResetAt: providerResetAt,
1101
- resetKind:
1102
- providerResetAt && !capacityNeedsBoundedRefresh ? "authoritative" : "bounded_refresh",
1103
- failurePayload,
1104
- ...(leases.codex.holderId && leases.codex.generation !== null
1105
- ? {
1106
- leaseFence: {
1107
- holderId: leases.codex.holderId,
1108
- generation: leases.codex.generation,
1109
- },
1110
- expectedRedispatches: attempt.redispatchesAtDispatch,
1111
- }
1112
- : {}),
1113
- });
1114
- if (armed.action === "waiting") {
1115
- await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, armed.events);
1116
- control.turnMetricOutcome = "recovering";
1117
- control.activityStatus = "waiting_capacity";
1118
- control.activityError = error;
1119
- return claimedResult({
1120
- status: "waiting_capacity",
1121
- capacityWait: {
1122
- waiterId: armed.waiter.id,
1123
- generation: armed.waiter.generation,
1124
- nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
1125
- wakeRevision: armed.waiter.wakeRevision,
1126
- },
1127
- });
1128
- }
1129
- }
1302
+ const failurePayload = codexUsageLimitFailurePayload(
1303
+ usageLimit,
1304
+ error instanceof Error ? error.message : String(error),
1305
+ );
1130
1306
  if (
1131
1307
  !(await eventing.settle!({
1132
1308
  events: [
1133
- // `rotated:true` ONLY on the reactive rotation path tells evaluateGoalContinuation to
1134
- // freeze autoContinuations (a rotation walk must not burn the goal's continuation budget).
1135
1309
  {
1136
1310
  type: "turn.failed",
1137
1311
  payload: {
1138
1312
  ...failurePayload,
1139
- recovery: goalActive ? "goal_continuation" : "user_message",
1140
- ...(rotated ? { rotated: true } : {}),
1313
+ recovery: "user_message",
1141
1314
  },
1142
1315
  },
1143
1316
  { type: "session.status.changed", payload: { status: "idle" } },
@@ -1152,38 +1325,6 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
1152
1325
  control.turnMetricOutcome = "failed";
1153
1326
  control.activityStatus = "idle";
1154
1327
  control.activityError = error;
1155
- if (goalActive) {
1156
- // Rotation: a candidate is available → continue NOW (0). All-capped → idle until the
1157
- // earliest reset across all accounts (capped at 1h). Else the unchanged single-account idle.
1158
- if (rotationResumeMs !== null) {
1159
- // A candidate IS available. Normally the just-failed account is now cooling so
1160
- // the ranker cannot re-pick it → 0 (re-dispatch NOW, the legitimate skip-the-hold
1161
- // case). Finding 1 bounds the two exceptions: a persistence fault yields a positive
1162
- // slow-retry floor, and once consecutive failovers exceed the account count + margin
1163
- // the circuit breaker returns a fixed MANDATORY idle (idleUntilReset) — never a 0-delay
1164
- // hot loop against a capped backend + DB.
1165
- return claimedResult({
1166
- status: "idle",
1167
- continueDelayMs: rotationResumeMs,
1168
- ...(rotationResumeIdleUntilReset ? { idleUntilReset: true } : {}),
1169
- });
1170
- }
1171
- // All-capped: clamp to [MIN_IDLE_MS, max] — a POSITIVE, BOUNDED hold (never 0,
1172
- // so session.ts can never tight-loop). The post-idle continuation re-dispatch
1173
- // hits the proactive seam, which refreshes usage and self-heals.
1174
- const resumeMs = allCappedResetAt
1175
- ? computeIdleDelayMs(allCappedResetAt, new Date(), CODEX_USAGE_LIMIT_MAX_RESUME_MS)
1176
- : usageLimit.resetsInSeconds !== null &&
1177
- Number.isFinite(usageLimit.resetsInSeconds) &&
1178
- usageLimit.resetsInSeconds > 0
1179
- ? Math.min(Math.ceil(usageLimit.resetsInSeconds) * 1000, CODEX_USAGE_LIMIT_MAX_RESUME_MS)
1180
- : CODEX_USAGE_LIMIT_MAX_RESUME_MS;
1181
- return claimedResult({
1182
- status: "idle",
1183
- continueDelayMs: resumeMs,
1184
- ...(allCappedResetAt ? { idleUntilReset: true } : {}),
1185
- });
1186
- }
1187
1328
  return claimedResult({ status: "idle" });
1188
1329
  }
1189
1330
  // Budget/limit exhaustion between model calls is account state, not an