@opengeni/worker-bundle 0.23.4 → 0.26.0-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +17 -2
  2. package/dist/activities/agent-turn/agent-build.d.ts +3 -10
  3. package/dist/activities/agent-turn/claim.d.ts +8 -0
  4. package/dist/activities/agent-turn/codex.d.ts +2 -2
  5. package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
  6. package/dist/activities/agent-turn/errors.d.ts +1 -1
  7. package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
  8. package/dist/activities/agent-turn/list-models.d.ts +16 -0
  9. package/dist/activities/agent-turn/model-usage.d.ts +15 -1
  10. package/dist/activities/agent-turn/quiescence.d.ts +11 -6
  11. package/dist/activities/agent-turn/run.d.ts +9 -0
  12. package/dist/activities/agent-turn/sandbox-runtime.d.ts +15 -0
  13. package/dist/activities/agent-turn/session-title.d.ts +21 -0
  14. package/dist/activities/agent-turn/stream-attempt.d.ts +4 -0
  15. package/dist/activities/agent-turn/tool-environment.d.ts +4 -13
  16. package/dist/activities/agent-turn/tool-policy.d.ts +40 -5
  17. package/dist/activities/agent-turn/turn-context.d.ts +13 -2
  18. package/dist/activities/agent-turn.d.ts +1 -1
  19. package/dist/activities/automations.d.ts +1 -0
  20. package/dist/activities/capabilities.d.ts +3 -1
  21. package/dist/activities/codemode-dispatcher.d.ts +10 -1
  22. package/dist/activities/codex-capacity.d.ts +32 -3
  23. package/dist/activities/codex-fleet-shadow.d.ts +1 -1
  24. package/dist/activities/codex-image-generation.d.ts +1 -1
  25. package/dist/activities/codex-rotation.d.ts +8 -67
  26. package/dist/activities/context-compaction.d.ts +2 -0
  27. package/dist/activities/goals.d.ts +12 -1
  28. package/dist/activities/google-drive-publication.d.ts +2 -13
  29. package/dist/activities/image-generation-operation.d.ts +13 -4
  30. package/dist/activities/image-generation-references.d.ts +2 -1
  31. package/dist/activities/scheduled-tasks.d.ts +1 -0
  32. package/dist/activities/session-state.d.ts +4 -2
  33. package/dist/activities/types.d.ts +19 -1
  34. package/dist/{activities-control-52W5IDWM.js → activities-control-CTEMLTHG.js} +339 -195
  35. package/dist/activities-control-CTEMLTHG.js.map +1 -0
  36. package/dist/activities-control.d.ts +4 -0
  37. package/dist/{activities-turn-A2G2KLSY.js → activities-turn-S5IU347A.js} +1846 -1136
  38. package/dist/activities-turn-S5IU347A.js.map +1 -0
  39. package/dist/activities.d.ts +4 -0
  40. package/dist/{chunk-GFNXVZVQ.js → chunk-D4ZEFOYH.js} +107 -7
  41. package/dist/chunk-D4ZEFOYH.js.map +1 -0
  42. package/dist/{chunk-GRFQTTSQ.js → chunk-GDCLDKSH.js} +7 -3
  43. package/dist/{chunk-GRFQTTSQ.js.map → chunk-GDCLDKSH.js.map} +1 -1
  44. package/dist/{chunk-IPVMCAW5.js → chunk-TAUE3GYT.js} +225 -154
  45. package/dist/chunk-TAUE3GYT.js.map +1 -0
  46. package/dist/github-rest-mcp.d.ts +0 -2
  47. package/dist/index.d.ts +2 -0
  48. package/dist/index.js +43 -7
  49. package/dist/index.js.map +1 -1
  50. package/dist/observability-metrics.d.ts +35 -8
  51. package/dist/sandbox-reaper-timeout.d.ts +2 -2
  52. package/dist/sandbox-resume.d.ts +6 -1
  53. package/dist/workflow-bundle.js +96 -10
  54. package/dist/workflows/activities.d.ts +6 -1
  55. package/dist/workflows/automations.d.ts +5 -0
  56. package/dist/workflows/session.d.ts +3 -0
  57. package/package.json +17 -17
  58. package/src/activities/agent-run-admission.ts +7 -1
  59. package/src/activities/agent-turn/admission.ts +47 -6
  60. package/src/activities/agent-turn/agent-build.ts +20 -26
  61. package/src/activities/agent-turn/claim.ts +59 -6
  62. package/src/activities/agent-turn/codex-capacity.ts +311 -234
  63. package/src/activities/agent-turn/codex.ts +2 -2
  64. package/src/activities/agent-turn/compaction-prep.ts +8 -1
  65. package/src/activities/agent-turn/credential-leases.ts +36 -0
  66. package/src/activities/agent-turn/errors.ts +29 -1
  67. package/src/activities/agent-turn/failure-settlement.ts +451 -323
  68. package/src/activities/agent-turn/finalization.ts +31 -17
  69. package/src/activities/agent-turn/list-models.ts +101 -0
  70. package/src/activities/agent-turn/model-usage.ts +120 -45
  71. package/src/activities/agent-turn/quiescence.ts +32 -17
  72. package/src/activities/agent-turn/run.ts +98 -9
  73. package/src/activities/agent-turn/sandbox-runtime.ts +78 -81
  74. package/src/activities/agent-turn/session-title.ts +55 -2
  75. package/src/activities/agent-turn/stream-attempt.ts +281 -136
  76. package/src/activities/agent-turn/tool-environment.ts +132 -39
  77. package/src/activities/agent-turn/tool-policy.ts +71 -12
  78. package/src/activities/agent-turn/turn-context.ts +21 -2
  79. package/src/activities/agent-turn.ts +5 -1
  80. package/src/activities/automations.ts +122 -67
  81. package/src/activities/capabilities.ts +129 -2
  82. package/src/activities/codemode-dispatcher.ts +235 -59
  83. package/src/activities/codex-capacity.ts +89 -10
  84. package/src/activities/codex-fleet-shadow.ts +8 -1
  85. package/src/activities/codex-image-generation.ts +19 -2
  86. package/src/activities/codex-rotation.ts +45 -142
  87. package/src/activities/context-compaction.ts +3 -0
  88. package/src/activities/goals.ts +116 -39
  89. package/src/activities/google-drive-publication.ts +4 -81
  90. package/src/activities/image-generation-operation.ts +28 -3
  91. package/src/activities/image-generation-references.ts +9 -9
  92. package/src/activities/mcp-credentials.ts +53 -0
  93. package/src/activities/rig-verification.ts +5 -3
  94. package/src/activities/sandbox-lease.ts +29 -12
  95. package/src/activities/scheduled-tasks.ts +217 -110
  96. package/src/activities/session-state.ts +17 -0
  97. package/src/activities/site-auth-maintenance.ts +12 -12
  98. package/src/activities/types.ts +22 -8
  99. package/src/activities.ts +2 -0
  100. package/src/activity-services.ts +4 -1
  101. package/src/github-rest-mcp.ts +2 -83
  102. package/src/index.ts +34 -4
  103. package/src/observability-metrics.ts +158 -14
  104. package/src/sandbox-reaper-timeout.ts +10 -4
  105. package/src/sandbox-resume.ts +58 -8
  106. package/src/sandbox-routing.ts +71 -16
  107. package/src/workflows/activities.ts +16 -0
  108. package/src/workflows/automations.ts +21 -2
  109. package/src/workflows/session.ts +85 -7
  110. package/dist/activities/api-integrations.d.ts +0 -21
  111. package/dist/activities-control-52W5IDWM.js.map +0 -1
  112. package/dist/activities-turn-A2G2KLSY.js.map +0 -1
  113. package/dist/chunk-GFNXVZVQ.js.map +0 -1
  114. package/dist/chunk-IPVMCAW5.js.map +0 -1
  115. package/src/activities/api-integrations.ts +0 -141
@@ -1,20 +1,20 @@
1
1
  import {
2
2
  getSessionGoal,
3
3
  acquireCodexCredentialLease,
4
- armCodexCapacityWait,
4
+ CodexCredentialLeaseAttemptFencedError,
5
+ CodexCredentialFailoverExhaustedError,
5
6
  CODEX_CREDENTIAL_LEASE_TTL_MS,
6
- getCodexRotationSettings,
7
- listCodexAccountStatuses,
8
- getSessionCodexState,
9
7
  recordSessionActiveCodexCredential,
10
8
  setSessionCodexPinInTransaction,
11
- setActiveCodexCredential,
9
+ settleCodexCredentialFailover,
12
10
  withSessionCodexCapacityMutation,
13
11
  type CodexCredentialLeaseResult,
12
+ type CodexCredentialLeaseSessionState,
14
13
  type CodexCredentialLeaseSelectionContext,
15
14
  } from "@opengeni/db";
16
- import { publishDurableSessionEvents } from "@opengeni/events";
17
15
  import { type Settings } from "@opengeni/config";
16
+ import { CodexReloginRequired } from "@opengeni/codex";
17
+ import { publishDurableSessionEvents } from "@opengeni/events";
18
18
  import {
19
19
  authoritativeCodexCapacityResetAt,
20
20
  classifyCodexPin,
@@ -22,14 +22,17 @@ import {
22
22
  isCodexCredentialEligible,
23
23
  selectCodexCredentialLeaseForTurn,
24
24
  type CodexRotationStrategy,
25
- type RotationDecision,
25
+ type CodexTurnLeaseDecision,
26
26
  } from "../codex-rotation";
27
27
  import {
28
28
  codexFleetShadowDecisionMetricLabelsV1,
29
29
  codexFleetShadowErrorMetricLabelsV1,
30
30
  publishCodexFleetShadowDecisionV1,
31
31
  } from "../codex-fleet-shadow";
32
- import { signalCodexCapacityWakeTargets } from "../codex-capacity";
32
+ import {
33
+ armAndReconcileCodexCapacityWait,
34
+ signalCodexCapacityWakeTargets,
35
+ } from "../codex-capacity";
33
36
  import type {
34
37
  TurnActivityServices as ActivityServices,
35
38
  RunAgentTurnInput,
@@ -37,6 +40,7 @@ import type {
37
40
  } from "../types";
38
41
  import { recordTurnStartupPhase } from "../../observability-metrics";
39
42
  import { createTurnCredentialLeases } from "./credential-leases";
43
+ import { deliverFailedChildTurnToParent } from "../parent-wake";
40
44
  import { randomUUID } from "node:crypto";
41
45
 
42
46
  import { refreshCappedCodexUsageRows } from "./codex";
@@ -94,6 +98,7 @@ export async function selectCodexTurnCapacity(
94
98
  observability,
95
99
  wakeSessionWorkflow,
96
100
  signalCodexCapacityWorkflow,
101
+ dispatchId,
97
102
  control,
98
103
  attempt,
99
104
  billingState,
@@ -101,6 +106,7 @@ export async function selectCodexTurnCapacity(
101
106
  providerTurn,
102
107
  leases,
103
108
  claimedResult,
109
+ acknowledgeLostAttemptOwnership,
104
110
  turn,
105
111
  codexWorkspaceKey,
106
112
  } = deps;
@@ -117,129 +123,69 @@ export async function selectCodexTurnCapacity(
117
123
  const credentialSelectionStartedAt = performance.now();
118
124
  let credentialSelectionOutcome: "completed" | "failed" = "completed";
119
125
  try {
120
- const sessionCodex = await getSessionCodexState(db, input.workspaceId, input.sessionId);
121
- const sessionPin = sessionCodex?.pinnedCredentialId ?? null;
122
- const sessionPinSource = sessionCodex?.pinSource ?? null;
123
- const selectForTurn = (context: CodexCredentialLeaseSelectionContext) =>
126
+ const selectForTurn = (
127
+ context: CodexCredentialLeaseSelectionContext,
128
+ lockedSessionCodexState: CodexCredentialLeaseSessionState,
129
+ ) =>
124
130
  selectCodexCredentialLeaseForTurn({
125
131
  context,
126
- leasingEnabled: settings.codexCredentialLeasingEnabled,
127
132
  sessionId: input.sessionId,
128
- sessionPinnedCredentialId: sessionPin,
129
- sessionPinSource,
130
- sessionLastCredentialId: sessionCodex?.lastCredentialId ?? null,
133
+ sessionPinnedCredentialId: lockedSessionCodexState.pinnedCredentialId,
134
+ sessionPinSource: lockedSessionCodexState.pinSource,
135
+ sessionLastCredentialId: lockedSessionCodexState.lastCredentialId,
131
136
  now: new Date(),
132
137
  });
133
138
 
134
- // Rollout/rollback path is intentionally table-inert. With the flag off,
135
- // old and new workers both use legacy pin > active-pointer selection and
136
- // neither reads nor writes the additive lease/cursor schema.
137
- let leased: CodexCredentialLeaseResult<RotationDecision>;
138
- let leaseAcquisitionStartedAtMs: number | null = null;
139
- if (settings.codexCredentialLeasingEnabled) {
140
- leaseAcquisitionStartedAtMs = performance.now();
141
- leased = await acquireCodexCredentialLease(
139
+ let leaseAcquisitionStartedAtMs = performance.now();
140
+ let leased: CodexCredentialLeaseResult<CodexTurnLeaseDecision> =
141
+ await acquireCodexCredentialLease(
142
142
  db,
143
143
  {
144
144
  accountId: input.accountId,
145
145
  workspaceId: input.workspaceId,
146
+ sessionId: input.sessionId,
146
147
  turnId,
148
+ attemptId: input.attemptId,
149
+ executionGeneration: attempt.executionGeneration,
150
+ workflowId: input.workflowId,
151
+ workflowRunId: input.workflowRunId,
152
+ dispatchId,
153
+ expectedRedispatches: attempt.redispatchesAtDispatch,
147
154
  holderId,
148
- advanceActivePointer: sessionPin === null,
155
+ advanceActivePointer: true,
149
156
  },
150
157
  selectForTurn,
151
158
  );
152
- } else {
153
- const [rotation, accounts] = await Promise.all([
154
- getCodexRotationSettings(db, input.workspaceId),
155
- listCodexAccountStatuses(db, input.workspaceId),
156
- ]);
157
- const leaseAccounts = accounts.map((account) => ({
158
- ...account,
159
- activeLeaseCount: 0,
160
- selectionCount: 0,
161
- lastSelectedAt: null,
162
- }));
163
- const activeCredentialId = rotation?.activeCredentialId ?? null;
164
- const selected = selectForTurn({
165
- accounts: leaseAccounts,
166
- activeCredentialId,
167
- rotationEnabled: rotation?.rotationEnabled ?? false,
168
- leaseRotationEnabled: false,
169
- rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
170
- existingCredentialId: null,
171
- policyScope: null,
172
- unavailableDiagnostics: [],
173
- });
174
- leased = {
175
- ...selected,
176
- accounts: leaseAccounts,
177
- activeCredentialId,
178
- rotationEnabled: rotation?.rotationEnabled ?? false,
179
- rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
180
- reused: false,
181
- holderId: null,
182
- generation: null,
183
- leasedUntil: null,
184
- unavailableDiagnostics: [],
185
- advanceActivePointer: selected.advanceActivePointer !== false,
186
- };
187
- }
188
159
  if (leased.decision.kind === "allCapped") {
189
160
  // Bounded self-heal of stale usage cache, then ONE new atomic selection.
190
161
  await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
191
162
  signalCodexCapacityWorkflow,
192
163
  wakeSessionWorkflow,
193
164
  });
194
- if (settings.codexCredentialLeasingEnabled) {
195
- leaseAcquisitionStartedAtMs = performance.now();
196
- leased = await acquireCodexCredentialLease(
197
- db,
198
- {
199
- accountId: input.accountId,
200
- workspaceId: input.workspaceId,
201
- turnId,
202
- holderId,
203
- advanceActivePointer: sessionPin === null,
204
- },
205
- selectForTurn,
206
- );
207
- } else {
208
- const [rotation, accounts] = await Promise.all([
209
- getCodexRotationSettings(db, input.workspaceId),
210
- listCodexAccountStatuses(db, input.workspaceId),
211
- ]);
212
- const leaseAccounts = accounts.map((account) => ({
213
- ...account,
214
- activeLeaseCount: 0,
215
- selectionCount: 0,
216
- lastSelectedAt: null,
217
- }));
218
- const selected = selectForTurn({
219
- accounts: leaseAccounts,
220
- activeCredentialId: rotation?.activeCredentialId ?? null,
221
- rotationEnabled: rotation?.rotationEnabled ?? false,
222
- leaseRotationEnabled: false,
223
- rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
224
- existingCredentialId: null,
225
- policyScope: null,
226
- unavailableDiagnostics: [],
227
- });
228
- leased = {
229
- ...selected,
230
- accounts: leaseAccounts,
231
- activeCredentialId: rotation?.activeCredentialId ?? null,
232
- rotationEnabled: rotation?.rotationEnabled ?? false,
233
- rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
234
- reused: false,
235
- holderId: null,
236
- generation: null,
237
- leasedUntil: null,
238
- unavailableDiagnostics: [],
239
- advanceActivePointer: selected.advanceActivePointer !== false,
240
- };
241
- }
165
+ leaseAcquisitionStartedAtMs = performance.now();
166
+ leased = await acquireCodexCredentialLease(
167
+ db,
168
+ {
169
+ accountId: input.accountId,
170
+ workspaceId: input.workspaceId,
171
+ sessionId: input.sessionId,
172
+ turnId,
173
+ attemptId: input.attemptId,
174
+ executionGeneration: attempt.executionGeneration,
175
+ workflowId: input.workflowId,
176
+ workflowRunId: input.workflowRunId,
177
+ dispatchId,
178
+ expectedRedispatches: attempt.redispatchesAtDispatch,
179
+ holderId,
180
+ advanceActivePointer: true,
181
+ },
182
+ selectForTurn,
183
+ );
242
184
  }
185
+ const lockedSessionCodexState = leased.sessionCodexState;
186
+ providerTurn.codexPolicySnapshot = leased.codexPolicySnapshot;
187
+ const sessionPin = lockedSessionCodexState.pinnedCredentialId;
188
+ const sessionPinSource = lockedSessionCodexState.pinSource;
243
189
  const rotationDecision = leased.decision;
244
190
  const selectedPinDisposition = classifyCodexPin({
245
191
  pinnedCredentialId: sessionPin,
@@ -252,6 +198,7 @@ export async function selectCodexTurnCapacity(
252
198
  // movement for manual/policy homes; this write only records the NEXT turn's
253
199
  // policy home (or clears a policy pin whose strategy is no longer active).
254
200
  if (
201
+ !leased.codexPolicySnapshotReused &&
255
202
  selectedPinDisposition === "sharded" &&
256
203
  leased.credentialId !== null &&
257
204
  (sessionPinSource !== "policy" || sessionPin !== leased.credentialId)
@@ -312,16 +259,8 @@ export async function selectCodexTurnCapacity(
312
259
  pinMutation.wakeTargets,
313
260
  );
314
261
  }
315
- if (
316
- !settings.codexCredentialLeasingEnabled &&
317
- leased.advanceActivePointer &&
318
- sessionPin === null &&
319
- rotationDecision.kind === "active" &&
320
- rotationDecision.moved
321
- ) {
322
- await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
323
- }
324
262
  providerTurn.effectiveCodexCredentialId = leased.credentialId;
263
+ providerTurn.codexCredentialFailoverLimit = leased.failoverLimit;
325
264
  leases.codex.generation = leased.generation;
326
265
  leases.codex.confirmedUntilMs =
327
266
  leased.leasedUntil && leaseAcquisitionStartedAtMs !== null
@@ -336,7 +275,7 @@ export async function selectCodexTurnCapacity(
336
275
 
337
276
  const actualOutcome = providerTurn.effectiveCodexCredentialId
338
277
  ? "selected"
339
- : rotationDecision.kind === "allCapped"
278
+ : rotationDecision.kind === "allCapped" || rotationDecision.kind === "allocatorDisabled"
340
279
  ? "waiting"
341
280
  : "none";
342
281
  const actualReason = providerTurn.effectiveCodexCredentialId
@@ -349,7 +288,9 @@ export async function selectCodexTurnCapacity(
349
288
  : "active"
350
289
  : rotationDecision.kind === "allCapped"
351
290
  ? "all_capped"
352
- : "none";
291
+ : rotationDecision.kind === "allocatorDisabled"
292
+ ? "allocator_disabled"
293
+ : "none";
353
294
  const fencedInFlight = leased.reused;
354
295
  const shadowResult = await publishCodexFleetShadowDecisionV1({
355
296
  enabled: settings.codexFleetPolicyShadowEnabled,
@@ -360,7 +301,7 @@ export async function selectCodexTurnCapacity(
360
301
  actualReason,
361
302
  affinityCredentialId: fencedInFlight
362
303
  ? providerTurn.effectiveCodexCredentialId
363
- : (sessionPin ?? sessionCodex?.lastCredentialId ?? null),
304
+ : (sessionPin ?? lockedSessionCodexState.lastCredentialId ?? null),
364
305
  fencedInFlight,
365
306
  nearExhaustionPct: settings.codexRotationNearExhaustionPct,
366
307
  now: new Date(),
@@ -432,7 +373,8 @@ export async function selectCodexTurnCapacity(
432
373
  if (
433
374
  providerTurn.effectiveCodexCredentialId === null &&
434
375
  leased.accounts.length > 0 &&
435
- leased.accounts.every((account) => !account.allocatorEnabled) &&
376
+ (rotationDecision.kind === "allocatorDisabled" ||
377
+ leased.accounts.every((account) => !account.allocatorEnabled)) &&
436
378
  turnId
437
379
  ) {
438
380
  if (turn.source === "compaction") {
@@ -463,68 +405,175 @@ export async function selectCodexTurnCapacity(
463
405
  control.activityStatus = "idle";
464
406
  return { exit: claimedResult({ status: "idle", deferredUntilWake: true }) };
465
407
  }
466
- const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
408
+ const goal = await getSessionGoal(db, input.workspaceId, input.sessionId);
467
409
  const activeGoal = goal?.status === "active" ? goal : null;
468
- const armed = await armCodexCapacityWait(db, {
469
- accountId: input.accountId,
470
- workspaceId: input.workspaceId,
471
- sessionId: input.sessionId,
472
- turnId,
473
- attemptId: input.attemptId,
474
- workflowId: input.workflowId,
475
- goalId: activeGoal?.id ?? null,
476
- goalVersion: activeGoal?.version ?? null,
477
- earliestResetAt: null,
478
- resetKind: "bounded_refresh",
479
- failurePayload: {
480
- error: "All connected Codex subscriptions are disabled for new allocations.",
481
- code: "codex_allocator_disabled",
482
- detail: "waiting for a credential to be re-enabled, reconnected, or added",
410
+ const evaluated = await armAndReconcileCodexCapacityWait(
411
+ { db, bus },
412
+ {
413
+ accountId: input.accountId,
414
+ workspaceId: input.workspaceId,
415
+ sessionId: input.sessionId,
416
+ turnId,
417
+ attemptId: input.attemptId,
418
+ workflowId: input.workflowId,
419
+ goalId: activeGoal?.id ?? null,
420
+ goalVersion: activeGoal?.version ?? null,
421
+ earliestResetAt: null,
422
+ resetKind: "mutation_only",
423
+ failurePayload: {
424
+ error:
425
+ rotationDecision.kind === "allocatorDisabled"
426
+ ? "The policy-selected Codex subscription is disabled for new allocations."
427
+ : "All connected Codex subscriptions are disabled for new allocations.",
428
+ code: "codex_allocator_disabled",
429
+ detail:
430
+ rotationDecision.kind === "allocatorDisabled"
431
+ ? "waiting for the selected credential to be re-enabled under the accepted source policy"
432
+ : "waiting for a credential in the accepted source pool to be re-enabled, reconnected, or added",
433
+ },
483
434
  },
484
- });
485
- if (armed.action === "waiting") {
486
- await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, armed.events);
435
+ );
436
+ if (evaluated.action === "resumed") {
437
+ control.turnMetricOutcome = "recovering";
438
+ control.activityStatus = "recovering";
439
+ return { exit: claimedResult({ status: "recovering" }) };
440
+ }
441
+ if (evaluated.action === "waiting") {
487
442
  control.turnMetricOutcome = "recovering";
488
443
  control.activityStatus = "waiting_capacity";
489
444
  return {
490
445
  exit: claimedResult({
491
446
  status: "waiting_capacity",
492
447
  capacityWait: {
493
- waiterId: armed.waiter.id,
494
- generation: armed.waiter.generation,
495
- nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
496
- wakeRevision: armed.waiter.wakeRevision,
448
+ waiterId: evaluated.waiter.id,
449
+ generation: evaluated.waiter.generation,
450
+ nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
451
+ wakeRevision: evaluated.waiter.wakeRevision,
497
452
  },
498
453
  }),
499
454
  };
500
455
  }
456
+ acknowledgeLostAttemptOwnership();
457
+ control.turnMetricOutcome = "cancelled";
458
+ control.activityStatus = "cancelled";
459
+ return { exit: claimedResult({ status: "cancelled" }) };
460
+ }
461
+
462
+ if (rotationDecision.kind === "none") {
501
463
  if (
502
- !(await eventing.settle!({
503
- events: [
504
- {
505
- type: "turn.failed",
506
- payload: {
507
- error: "All connected Codex subscriptions are disabled for new allocations.",
508
- code: "codex_allocator_disabled",
509
- retryable: false,
510
- recovery: "user_message",
464
+ leased.poolAccountCount === 0 &&
465
+ !leased.codexPolicySnapshotReused &&
466
+ leased.codexPolicySnapshot?.source !== "disabled"
467
+ ) {
468
+ // Preserve the established no-account behavior. The empty-string
469
+ // token resolver used by the later model path raises this same typed
470
+ // reconnect outcome, but capacity must not report success and defer
471
+ // the failure to an unguarded provider setup.
472
+ throw new CodexReloginRequired("No Codex subscription is connected for this workspace.");
473
+ }
474
+ const noneReason =
475
+ sessionPinSource === "manual" && sessionPin !== null
476
+ ? {
477
+ code: "codex_manual_pin_unavailable",
478
+ error: "The pinned Codex subscription is unavailable for this turn.",
479
+ detail:
480
+ "waiting for the pinned account to reconnect or become allocatable under the accepted source policy",
481
+ }
482
+ : !leased.rotationEnabled && leased.activeCredentialId === null
483
+ ? {
484
+ code: "codex_active_pointer_unavailable",
485
+ error: "The Codex active subscription pointer is unavailable for this turn.",
486
+ detail:
487
+ "waiting for the accepted active account to be restored or become allocatable",
488
+ }
489
+ : leased.poolAccountCount > leased.accounts.length
490
+ ? {
491
+ code: "codex_policy_pool_unavailable",
492
+ error: "No Codex subscription matches the accepted account-selection policy.",
493
+ detail:
494
+ "waiting for a matching credential to reconnect or become allocatable under the accepted source policy",
495
+ }
496
+ : {
497
+ code: "codex_credential_unavailable",
498
+ error:
499
+ "No Codex subscription is currently available under the accepted policy.",
500
+ detail:
501
+ "waiting for a matching credential to reconnect or become allocatable under the accepted source policy",
502
+ };
503
+ if (turn.source === "compaction") {
504
+ if (
505
+ !(await eventing.settle!({
506
+ events: [
507
+ {
508
+ type: "turn.cancelled",
509
+ payload: {
510
+ maintenance: "context_compaction",
511
+ reason: noneReason.code,
512
+ requestPreserved: true,
513
+ },
511
514
  },
515
+ {
516
+ type: "session.status.changed",
517
+ payload: { status: "idle" },
518
+ },
519
+ ],
520
+ turnStatus: "cancelled",
521
+ sessionStatus: "idle",
522
+ activeTurnId: null,
523
+ }))
524
+ ) {
525
+ return { exit: claimedResult({ status: "cancelled" }) };
526
+ }
527
+ control.turnMetricOutcome = "cancelled";
528
+ control.activityStatus = "idle";
529
+ return { exit: claimedResult({ status: "idle", deferredUntilWake: true }) };
530
+ }
531
+ const goal = await getSessionGoal(db, input.workspaceId, input.sessionId);
532
+ const activeGoal = goal?.status === "active" ? goal : null;
533
+ const evaluated = await armAndReconcileCodexCapacityWait(
534
+ { db, bus },
535
+ {
536
+ accountId: input.accountId,
537
+ workspaceId: input.workspaceId,
538
+ sessionId: input.sessionId,
539
+ turnId,
540
+ attemptId: input.attemptId,
541
+ workflowId: input.workflowId,
542
+ goalId: activeGoal?.id ?? null,
543
+ goalVersion: activeGoal?.version ?? null,
544
+ earliestResetAt: null,
545
+ resetKind: "mutation_only",
546
+ failurePayload: {
547
+ error: noneReason.error,
548
+ code: noneReason.code,
549
+ detail: noneReason.detail,
550
+ },
551
+ },
552
+ );
553
+ if (evaluated.action === "resumed") {
554
+ control.turnMetricOutcome = "recovering";
555
+ control.activityStatus = "recovering";
556
+ return { exit: claimedResult({ status: "recovering" }) };
557
+ }
558
+ if (evaluated.action === "waiting") {
559
+ control.turnMetricOutcome = "recovering";
560
+ control.activityStatus = "waiting_capacity";
561
+ return {
562
+ exit: claimedResult({
563
+ status: "waiting_capacity",
564
+ capacityWait: {
565
+ waiterId: evaluated.waiter.id,
566
+ generation: evaluated.waiter.generation,
567
+ nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
568
+ wakeRevision: evaluated.waiter.wakeRevision,
512
569
  },
513
- {
514
- type: "session.status.changed",
515
- payload: { status: "idle" },
516
- },
517
- ],
518
- turnStatus: "failed",
519
- sessionStatus: "idle",
520
- activeTurnId: null,
521
- }))
522
- ) {
523
- return { exit: claimedResult({ status: "cancelled" }) };
570
+ }),
571
+ };
524
572
  }
525
- control.turnMetricOutcome = "failed";
526
- control.activityStatus = "idle";
527
- return { exit: claimedResult({ status: "idle" }) };
573
+ acknowledgeLostAttemptOwnership();
574
+ control.turnMetricOutcome = "cancelled";
575
+ control.activityStatus = "cancelled";
576
+ return { exit: claimedResult({ status: "cancelled" }) };
528
577
  }
529
578
 
530
579
  if (rotationDecision.kind === "allCapped" && turnId) {
@@ -561,7 +610,7 @@ export async function selectCodexTurnCapacity(
561
610
  // build) until the EARLIEST reset across all accounts — the multi-account
562
611
  // generalization of #143's single-account idle-until-reset. No saveRunState:
563
612
  // no model ran, nothing to freeze.
564
- const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
613
+ const goal = await getSessionGoal(db, input.workspaceId, input.sessionId);
565
614
  const goalActive = Boolean(goal && goal.status === "active");
566
615
  // BOUNDED + POSITIVE: clamp to [MIN_IDLE_MS, max] so a null/elapsed/unknown
567
616
  // reset can never yield a 0 (which session.ts would treat as "continue now",
@@ -571,87 +620,62 @@ export async function selectCodexTurnCapacity(
571
620
  new Date(),
572
621
  CODEX_USAGE_LIMIT_MAX_RESUME_MS,
573
622
  );
623
+ const allConnectedAccountsWereConsidered =
624
+ leased.rotationEnabled && selectedPinDisposition !== "manual";
625
+ const capacityDetail = allConnectedAccountsWereConsidered
626
+ ? "all connected Codex subscriptions are rate-limited"
627
+ : selectedPinDisposition === "manual"
628
+ ? "the pinned Codex subscription is rate-limited"
629
+ : "the active Codex subscription is rate-limited";
574
630
  const failurePayload = codexUsageLimitFailurePayload(
575
631
  { resetsInSeconds: Math.ceil(resumeMs / 1000) },
576
- "all connected Codex subscriptions are rate-limited",
577
- { allAccounts: true },
632
+ capacityDetail,
633
+ { allAccounts: allConnectedAccountsWereConsidered },
578
634
  );
579
635
  const authoritativeResetAt = authoritativeCodexCapacityResetAt(leased.accounts, new Date());
580
- const armed = await armCodexCapacityWait(db, {
581
- accountId: input.accountId,
582
- workspaceId: input.workspaceId,
583
- sessionId: input.sessionId,
584
- turnId,
585
- attemptId: input.attemptId,
586
- workflowId: input.workflowId,
587
- goalId: goalActive && goal ? goal.id : null,
588
- goalVersion: goalActive && goal ? goal.version : null,
589
- earliestResetAt: authoritativeResetAt,
590
- resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
591
- failurePayload,
592
- });
593
- if (armed.action === "waiting") {
594
- await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, armed.events);
636
+ const evaluated = await armAndReconcileCodexCapacityWait(
637
+ { db, bus },
638
+ {
639
+ accountId: input.accountId,
640
+ workspaceId: input.workspaceId,
641
+ sessionId: input.sessionId,
642
+ turnId,
643
+ attemptId: input.attemptId,
644
+ workflowId: input.workflowId,
645
+ goalId: goalActive && goal ? goal.id : null,
646
+ goalVersion: goalActive && goal ? goal.version : null,
647
+ earliestResetAt: authoritativeResetAt,
648
+ resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
649
+ failurePayload,
650
+ },
651
+ );
652
+ if (evaluated.action === "resumed") {
653
+ control.turnMetricOutcome = "recovering";
654
+ control.activityStatus = "recovering";
655
+ return { exit: claimedResult({ status: "recovering" }) };
656
+ }
657
+ if (evaluated.action === "waiting") {
595
658
  control.turnMetricOutcome = "recovering";
596
659
  control.activityStatus = "waiting_capacity";
597
660
  return {
598
661
  exit: claimedResult({
599
662
  status: "waiting_capacity",
600
663
  capacityWait: {
601
- waiterId: armed.waiter.id,
602
- generation: armed.waiter.generation,
603
- nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
604
- wakeRevision: armed.waiter.wakeRevision,
664
+ waiterId: evaluated.waiter.id,
665
+ generation: evaluated.waiter.generation,
666
+ nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
667
+ wakeRevision: evaluated.waiter.wakeRevision,
605
668
  },
606
669
  }),
607
670
  };
608
671
  }
609
- if (
610
- !(await eventing.settle!({
611
- events: [
612
- // `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
613
- // rotation-wait state as the reactive all-capped path, so it must freeze
614
- // autoContinuations identically (evaluateGoalContinuation reads this marker)
615
- // — a goal waiting out a long reset must not burn its continuation budget on
616
- // the proactive path while the reactive path spares it.
617
- {
618
- type: "turn.failed",
619
- payload: {
620
- ...failurePayload,
621
- recovery: goalActive ? "goal_continuation" : "user_message",
622
- rotated: true,
623
- },
624
- },
625
- {
626
- type: "session.status.changed",
627
- payload: { status: "idle" },
628
- },
629
- ],
630
- turnStatus: "failed",
631
- sessionStatus: "idle",
632
- activeTurnId: null,
633
- }))
634
- ) {
635
- return { exit: claimedResult({ status: "cancelled" }) };
636
- }
637
- control.turnMetricOutcome = "failed";
638
- control.activityStatus = "idle";
639
- // idleUntilReset marks this a MANDATORY hold: session.ts must wait the full
640
- // resumeMs even if a future change made it 0 — never a tight re-dispatch.
641
- return {
642
- exit: claimedResult(
643
- goalActive
644
- ? {
645
- status: "idle",
646
- continueDelayMs: resumeMs,
647
- idleUntilReset: true,
648
- }
649
- : { status: "idle" },
650
- ),
651
- };
672
+ acknowledgeLostAttemptOwnership();
673
+ control.turnMetricOutcome = "cancelled";
674
+ control.activityStatus = "cancelled";
675
+ return { exit: claimedResult({ status: "cancelled" }) };
652
676
  }
653
677
  if (providerTurn.effectiveCodexCredentialId) {
654
- const priorAccountId = sessionCodex?.lastCredentialId ?? null;
678
+ const priorAccountId = lockedSessionCodexState.lastCredentialId;
655
679
  if (priorAccountId !== providerTurn.effectiveCodexCredentialId) {
656
680
  await recordSessionActiveCodexCredential(
657
681
  db,
@@ -704,6 +728,59 @@ export async function selectCodexTurnCapacity(
704
728
  }
705
729
  } catch (error) {
706
730
  credentialSelectionOutcome = "failed";
731
+ if (error instanceof CodexCredentialLeaseAttemptFencedError) {
732
+ acknowledgeLostAttemptOwnership();
733
+ control.activityStatus = "cancelled";
734
+ control.turnMetricOutcome = "cancelled";
735
+ return { exit: claimedResult({ status: "cancelled" }) };
736
+ }
737
+ if (error instanceof CodexCredentialFailoverExhaustedError) {
738
+ const settlement = await settleCodexCredentialFailover(db, {
739
+ accountId: input.accountId,
740
+ workspaceId: input.workspaceId,
741
+ sessionId: input.sessionId,
742
+ turnId,
743
+ attemptId: input.attemptId,
744
+ holderId: null,
745
+ generation: null,
746
+ expectedRedispatches: attempt.redispatchesAtDispatch,
747
+ maxFailovers: error.maxFailovers,
748
+ recoveryPayload: {
749
+ triggerEventId: attempt.triggerEventId!,
750
+ reason: "codex_credential_failover_exhausted",
751
+ },
752
+ failedPayload: {
753
+ error:
754
+ "Automatic Codex credential failover stopped after every bounded account attempt was consumed. Send a new message after checking account health or capacity.",
755
+ code: "codex_credential_failover_exhausted",
756
+ retryable: false,
757
+ recovery: "user_message",
758
+ failoverCount: error.failoverCount,
759
+ maxFailovers: error.maxFailovers,
760
+ },
761
+ });
762
+ if (settlement.action === "limit_exceeded") {
763
+ await publishDurableSessionEvents(
764
+ bus,
765
+ input.workspaceId,
766
+ input.sessionId,
767
+ settlement.events,
768
+ );
769
+ control.activityStatus = "idle";
770
+ control.turnMetricOutcome = "failed";
771
+ await deliverFailedChildTurnToParent(
772
+ { db, bus, settings, observability, wakeSessionWorkflow },
773
+ input.workspaceId,
774
+ input.sessionId,
775
+ turnId,
776
+ );
777
+ return { exit: claimedResult({ status: "idle" }) };
778
+ }
779
+ acknowledgeLostAttemptOwnership();
780
+ control.activityStatus = "cancelled";
781
+ control.turnMetricOutcome = "cancelled";
782
+ return { exit: claimedResult({ status: "cancelled" }) };
783
+ }
707
784
  throw error;
708
785
  } finally {
709
786
  recordTurnStartupPhase(observability, {