@opengeni/worker-bundle 0.20.3 → 0.22.1-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +52 -0
  2. package/dist/activities/agent-turn/agent-build.d.ts +89 -0
  3. package/dist/activities/agent-turn/claim.d.ts +67 -0
  4. package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
  5. package/dist/activities/agent-turn/codex.d.ts +32 -0
  6. package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
  7. package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
  8. package/dist/activities/agent-turn/errors.d.ts +253 -0
  9. package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
  10. package/dist/activities/agent-turn/file-resources.d.ts +37 -0
  11. package/dist/activities/agent-turn/finalization.d.ts +42 -0
  12. package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
  13. package/dist/activities/agent-turn/governance-model.d.ts +66 -0
  14. package/dist/activities/agent-turn/history-sink.d.ts +52 -0
  15. package/dist/activities/agent-turn/history.d.ts +63 -0
  16. package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
  17. package/dist/activities/agent-turn/model-usage.d.ts +218 -0
  18. package/dist/activities/agent-turn/quiescence.d.ts +135 -0
  19. package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
  20. package/dist/activities/agent-turn/run.d.ts +3 -0
  21. package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
  22. package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
  23. package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
  24. package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
  25. package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
  26. package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
  27. package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
  28. package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
  29. package/dist/activities/agent-turn/turn-context.d.ts +144 -0
  30. package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
  31. package/dist/activities/agent-turn.d.ts +12 -954
  32. package/dist/activities/automations.d.ts +16 -0
  33. package/dist/activities/common.d.ts +1 -1
  34. package/dist/activities/connector-attachments.d.ts +13 -0
  35. package/dist/activities/documents.d.ts +1 -0
  36. package/dist/activities/environment.d.ts +26 -15
  37. package/dist/activities/generated-images.d.ts +1 -1
  38. package/dist/activities/git-credential-renewal.d.ts +3 -1
  39. package/dist/activities/goals.d.ts +4 -1
  40. package/dist/activities/google-drive-provider.d.ts +29 -0
  41. package/dist/activities/google-drive-publication.d.ts +17 -4
  42. package/dist/activities/incident-telemetry-authority.d.ts +52 -0
  43. package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
  44. package/dist/activities/knowledge-source-sync.d.ts +4 -0
  45. package/dist/activities/mcp-credentials.d.ts +9 -3
  46. package/dist/activities/open-suffix-resume.d.ts +77 -0
  47. package/dist/activities/packs.d.ts +3 -11
  48. package/dist/activities/parent-wake.d.ts +19 -0
  49. package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
  50. package/dist/activities/retained-screenshots.d.ts +1 -1
  51. package/dist/activities/rig-verification.d.ts +1 -1
  52. package/dist/activities/run-input.d.ts +9 -10
  53. package/dist/activities/sandbox-images.d.ts +1 -14
  54. package/dist/activities/sandbox-lease.d.ts +21 -3
  55. package/dist/activities/scheduled-tasks.d.ts +47 -0
  56. package/dist/activities/types.d.ts +23 -2
  57. package/dist/activities/video-media-validation.d.ts +1 -1
  58. package/dist/activities/video-output-retention.d.ts +1 -2
  59. package/dist/activities/xai-auth.d.ts +3 -0
  60. package/dist/{activities-control-BJSDRSIP.js → activities-control-DKG22ONJ.js} +2646 -551
  61. package/dist/activities-control-DKG22ONJ.js.map +1 -0
  62. package/dist/activities-control.d.ts +6 -2
  63. package/dist/activities-turn-4IMC3MAA.js +21326 -0
  64. package/dist/activities-turn-4IMC3MAA.js.map +1 -0
  65. package/dist/activities.d.ts +7 -3
  66. package/dist/artifact-materializer-entry.js +41 -22
  67. package/dist/artifact-materializer-entry.js.map +1 -1
  68. package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
  69. package/dist/chunk-CE7TPKBS.js.map +1 -0
  70. package/dist/chunk-CNYGVRKF.js +4698 -0
  71. package/dist/chunk-CNYGVRKF.js.map +1 -0
  72. package/dist/connection-provider-use.d.ts +18 -0
  73. package/dist/editable-artifact-materializer.d.ts +4 -3
  74. package/dist/github-rest-mcp.d.ts +26 -0
  75. package/dist/index.d.ts +10 -2
  76. package/dist/index.js +58 -14
  77. package/dist/index.js.map +1 -1
  78. package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
  79. package/dist/model-context-contributions.d.ts +8 -4
  80. package/dist/observability-metrics.d.ts +63 -4
  81. package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
  82. package/dist/personal-github-git-credentials.d.ts +9 -0
  83. package/dist/pr-review-credentials.d.ts +6 -0
  84. package/dist/sandbox-archive-storage.d.ts +45 -0
  85. package/dist/sandbox-resume.d.ts +65 -6
  86. package/dist/sandbox-routing.d.ts +38 -2
  87. package/dist/scheduled-alert-occurrence.d.ts +33 -0
  88. package/dist/workflow-bundle.js +87 -28
  89. package/dist/workflows/activities.d.ts +1 -0
  90. package/dist/workflows/automations.d.ts +6 -0
  91. package/dist/workflows/document-index.d.ts +1 -0
  92. package/dist/workflows.d.ts +1 -0
  93. package/package.json +17 -17
  94. package/src/activities/agent-turn/admission.ts +292 -0
  95. package/src/activities/agent-turn/agent-build.ts +802 -0
  96. package/src/activities/agent-turn/claim.ts +590 -0
  97. package/src/activities/agent-turn/codex-capacity.ts +720 -0
  98. package/src/activities/agent-turn/codex.ts +94 -0
  99. package/src/activities/agent-turn/compaction-prep.ts +722 -0
  100. package/src/activities/agent-turn/credential-leases.ts +240 -0
  101. package/src/activities/agent-turn/errors.ts +1068 -0
  102. package/src/activities/agent-turn/failure-settlement.ts +1436 -0
  103. package/src/activities/agent-turn/file-resources.ts +227 -0
  104. package/src/activities/agent-turn/finalization.ts +754 -0
  105. package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
  106. package/src/activities/agent-turn/governance-model.ts +465 -0
  107. package/src/activities/agent-turn/history-sink.ts +139 -0
  108. package/src/activities/agent-turn/history.ts +200 -0
  109. package/src/activities/agent-turn/media-artifacts.ts +359 -0
  110. package/src/activities/agent-turn/model-usage.ts +823 -0
  111. package/src/activities/agent-turn/quiescence.ts +353 -0
  112. package/src/activities/agent-turn/run-credentials.ts +761 -0
  113. package/src/activities/agent-turn/run.ts +1530 -0
  114. package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
  115. package/src/activities/agent-turn/sandbox-provision.ts +569 -0
  116. package/src/activities/agent-turn/sandbox-route.ts +358 -0
  117. package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
  118. package/src/activities/agent-turn/stream-attempt.ts +1670 -0
  119. package/src/activities/agent-turn/tool-environment.ts +729 -0
  120. package/src/activities/agent-turn/tool-policy.ts +216 -0
  121. package/src/activities/agent-turn/tool-result-spill.ts +265 -0
  122. package/src/activities/agent-turn/turn-context.ts +281 -0
  123. package/src/activities/agent-turn/xai-capacity.ts +229 -0
  124. package/src/activities/agent-turn.ts +12 -12701
  125. package/src/activities/api-integrations.ts +3 -0
  126. package/src/activities/automations.ts +214 -0
  127. package/src/activities/codemode-dispatcher.ts +12 -1
  128. package/src/activities/common.ts +1 -5
  129. package/src/activities/connector-attachments.ts +168 -0
  130. package/src/activities/context-compaction.ts +18 -0
  131. package/src/activities/environment.ts +151 -55
  132. package/src/activities/generated-images.ts +26 -11
  133. package/src/activities/git-credential-renewal.ts +13 -3
  134. package/src/activities/goals.ts +85 -19
  135. package/src/activities/google-drive-provider.ts +178 -0
  136. package/src/activities/google-drive-publication.ts +147 -49
  137. package/src/activities/incident-telemetry-authority.ts +327 -0
  138. package/src/activities/incident-telemetry-preflight.ts +317 -0
  139. package/src/activities/knowledge-source-sync.ts +178 -37
  140. package/src/activities/mcp-credentials.ts +191 -17
  141. package/src/activities/open-suffix-resume.ts +422 -0
  142. package/src/activities/packs.ts +12 -60
  143. package/src/activities/parent-wake.ts +64 -14
  144. package/src/activities/retained-screenshot-reaper.ts +6 -3
  145. package/src/activities/retained-screenshots.ts +15 -47
  146. package/src/activities/rig-verification.ts +1 -1
  147. package/src/activities/run-input.ts +102 -142
  148. package/src/activities/sandbox-images.ts +7 -166
  149. package/src/activities/sandbox-lease.ts +406 -15
  150. package/src/activities/scheduled-tasks.ts +2103 -411
  151. package/src/activities/session-state.ts +10 -2
  152. package/src/activities/types.ts +32 -2
  153. package/src/activities/video-media-validation.ts +3 -3
  154. package/src/activities/video-output-retention.ts +2 -57
  155. package/src/activities/xai-auth.ts +8 -0
  156. package/src/activities-control.ts +2 -0
  157. package/src/activities.ts +3 -0
  158. package/src/activity-services.ts +13 -4
  159. package/src/connection-provider-use.ts +36 -0
  160. package/src/editable-artifact-materializer-subprocess.ts +25 -14
  161. package/src/editable-artifact-materializer.ts +29 -18
  162. package/src/github-rest-mcp.ts +713 -0
  163. package/src/http.ts +16 -6
  164. package/src/index.ts +40 -5
  165. package/src/knowledge-source-sync-activity-policy.ts +2 -0
  166. package/src/model-context-contributions.ts +33 -10
  167. package/src/observability-metrics.ts +357 -0
  168. package/src/opensandbox-kubernetes-inventory.ts +292 -0
  169. package/src/personal-github-git-credentials.ts +331 -0
  170. package/src/pr-review-credentials.ts +163 -0
  171. package/src/sandbox-archive-storage.ts +159 -0
  172. package/src/sandbox-resume.ts +449 -63
  173. package/src/sandbox-routing.ts +314 -16
  174. package/src/scheduled-alert-occurrence.ts +176 -0
  175. package/src/workflows/activities.ts +18 -2
  176. package/src/workflows/automations.ts +11 -0
  177. package/src/workflows/session.ts +13 -3
  178. package/src/workflows.ts +1 -0
  179. package/dist/activities-control-BJSDRSIP.js.map +0 -1
  180. package/dist/activities-turn-J5YO7XXX.js +0 -17243
  181. package/dist/activities-turn-J5YO7XXX.js.map +0 -1
  182. package/dist/chunk-N5TJIFCE.js.map +0 -1
  183. package/dist/chunk-P6Z2SLAA.js +0 -3092
  184. package/dist/chunk-P6Z2SLAA.js.map +0 -1
@@ -0,0 +1,720 @@
1
+ import {
2
+ getSessionGoal,
3
+ acquireCodexCredentialLease,
4
+ armCodexCapacityWait,
5
+ CODEX_CREDENTIAL_LEASE_TTL_MS,
6
+ getCodexRotationSettings,
7
+ listCodexAccountStatuses,
8
+ getSessionCodexState,
9
+ recordSessionActiveCodexCredential,
10
+ setSessionCodexPinInTransaction,
11
+ setActiveCodexCredential,
12
+ withSessionCodexCapacityMutation,
13
+ type CodexCredentialLeaseResult,
14
+ type CodexCredentialLeaseSelectionContext,
15
+ } from "@opengeni/db";
16
+ import { publishDurableSessionEvents } from "@opengeni/events";
17
+ import { type Settings } from "@opengeni/config";
18
+ import {
19
+ authoritativeCodexCapacityResetAt,
20
+ classifyCodexPin,
21
+ computeIdleDelayMs,
22
+ isCodexCredentialEligible,
23
+ selectCodexCredentialLeaseForTurn,
24
+ type CodexRotationStrategy,
25
+ type RotationDecision,
26
+ } from "../codex-rotation";
27
+ import {
28
+ codexFleetShadowDecisionMetricLabelsV1,
29
+ codexFleetShadowErrorMetricLabelsV1,
30
+ publishCodexFleetShadowDecisionV1,
31
+ } from "../codex-fleet-shadow";
32
+ import { signalCodexCapacityWakeTargets } from "../codex-capacity";
33
+ import type {
34
+ TurnActivityServices as ActivityServices,
35
+ RunAgentTurnInput,
36
+ RunAgentTurnResult,
37
+ } from "../types";
38
+ import { recordTurnStartupPhase } from "../../observability-metrics";
39
+ import { createTurnCredentialLeases } from "./credential-leases";
40
+ import { randomUUID } from "node:crypto";
41
+
42
+ import { refreshCappedCodexUsageRows } from "./codex";
43
+ import { codexUsageLimitFailurePayload, CODEX_USAGE_LIMIT_MAX_RESUME_MS } from "./errors";
44
+ import type { ClaimTurnOk } from "./claim";
45
+ import type {
46
+ AttemptIdentityState,
47
+ BillingState,
48
+ ClaimedResult,
49
+ EventingState,
50
+ ProviderTurnState,
51
+ TurnControlState,
52
+ } from "./turn-context";
53
+
54
+ export type CapacityPhaseDeps = {
55
+ input: RunAgentTurnInput;
56
+ settings: Settings;
57
+ db: ActivityServices["db"];
58
+ bus: ActivityServices["bus"];
59
+ observability: ActivityServices["observability"];
60
+ wakeSessionWorkflow: ActivityServices["wakeSessionWorkflow"];
61
+ signalCodexCapacityWorkflow: ActivityServices["signalCodexCapacityWorkflow"];
62
+ cancellationSignal: AbortSignal | undefined;
63
+ dispatchId: string;
64
+ control: TurnControlState;
65
+ attempt: AttemptIdentityState;
66
+ billingState: BillingState;
67
+ eventing: EventingState & {
68
+ publish: NonNullable<EventingState["publish"]>;
69
+ settle: NonNullable<EventingState["settle"]>;
70
+ };
71
+ providerTurn: ProviderTurnState;
72
+ leases: ReturnType<typeof createTurnCredentialLeases>;
73
+ claimedResult: ClaimedResult;
74
+ acknowledgeLostAttemptOwnership: () => void;
75
+ acknowledgeRecoveryQuiescence: () => void;
76
+ setLastInputTokensFenced: (lastInputTokens: number | null) => Promise<void>;
77
+ turn: ClaimTurnOk["turn"];
78
+ session: ClaimTurnOk["session"];
79
+ turnExecutionPolicy: ClaimTurnOk["turnExecutionPolicy"];
80
+ trigger: ClaimTurnOk["trigger"];
81
+ codexWorkspaceKey: string;
82
+ };
83
+
84
+ export type CapacityPhaseOutcome = { exit: RunAgentTurnResult } | { ok: true };
85
+
86
+ export async function selectCodexTurnCapacity(
87
+ deps: CapacityPhaseDeps,
88
+ ): Promise<CapacityPhaseOutcome> {
89
+ const {
90
+ input,
91
+ settings,
92
+ db,
93
+ bus,
94
+ observability,
95
+ wakeSessionWorkflow,
96
+ signalCodexCapacityWorkflow,
97
+ control,
98
+ attempt,
99
+ billingState,
100
+ eventing,
101
+ providerTurn,
102
+ leases,
103
+ claimedResult,
104
+ turn,
105
+ codexWorkspaceKey,
106
+ } = deps;
107
+ const turnId = attempt.turnId;
108
+ const holderId = leases.codex.holderId;
109
+ if (!turnId) {
110
+ throw new Error("Turn id was not initialized");
111
+ }
112
+ if (!holderId) {
113
+ throw new Error("Codex lease holder was not initialized");
114
+ }
115
+
116
+ if (billingState.isCodexTurn) {
117
+ const credentialSelectionStartedAt = performance.now();
118
+ let credentialSelectionOutcome: "completed" | "failed" = "completed";
119
+ try {
120
+ const sessionCodex = await getSessionCodexState(db, input.workspaceId, input.sessionId);
121
+ const sessionPin = sessionCodex?.pinnedCredentialId ?? null;
122
+ const sessionPinSource = sessionCodex?.pinSource ?? null;
123
+ const selectForTurn = (context: CodexCredentialLeaseSelectionContext) =>
124
+ selectCodexCredentialLeaseForTurn({
125
+ context,
126
+ leasingEnabled: settings.codexCredentialLeasingEnabled,
127
+ sessionId: input.sessionId,
128
+ sessionPinnedCredentialId: sessionPin,
129
+ sessionPinSource,
130
+ sessionLastCredentialId: sessionCodex?.lastCredentialId ?? null,
131
+ now: new Date(),
132
+ });
133
+
134
+ // Rollout/rollback path is intentionally table-inert. With the flag off,
135
+ // old and new workers both use legacy pin > active-pointer selection and
136
+ // neither reads nor writes the additive lease/cursor schema.
137
+ let leased: CodexCredentialLeaseResult<RotationDecision>;
138
+ let leaseAcquisitionStartedAtMs: number | null = null;
139
+ if (settings.codexCredentialLeasingEnabled) {
140
+ leaseAcquisitionStartedAtMs = performance.now();
141
+ leased = await acquireCodexCredentialLease(
142
+ db,
143
+ {
144
+ accountId: input.accountId,
145
+ workspaceId: input.workspaceId,
146
+ turnId,
147
+ holderId,
148
+ advanceActivePointer: sessionPin === null,
149
+ },
150
+ selectForTurn,
151
+ );
152
+ } else {
153
+ const [rotation, accounts] = await Promise.all([
154
+ getCodexRotationSettings(db, input.workspaceId),
155
+ listCodexAccountStatuses(db, input.workspaceId),
156
+ ]);
157
+ const leaseAccounts = accounts.map((account) => ({
158
+ ...account,
159
+ activeLeaseCount: 0,
160
+ selectionCount: 0,
161
+ lastSelectedAt: null,
162
+ }));
163
+ const activeCredentialId = rotation?.activeCredentialId ?? null;
164
+ const selected = selectForTurn({
165
+ accounts: leaseAccounts,
166
+ activeCredentialId,
167
+ rotationEnabled: rotation?.rotationEnabled ?? false,
168
+ leaseRotationEnabled: false,
169
+ rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
170
+ existingCredentialId: null,
171
+ policyScope: null,
172
+ unavailableDiagnostics: [],
173
+ });
174
+ leased = {
175
+ ...selected,
176
+ accounts: leaseAccounts,
177
+ activeCredentialId,
178
+ rotationEnabled: rotation?.rotationEnabled ?? false,
179
+ rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
180
+ reused: false,
181
+ holderId: null,
182
+ generation: null,
183
+ leasedUntil: null,
184
+ unavailableDiagnostics: [],
185
+ advanceActivePointer: selected.advanceActivePointer !== false,
186
+ };
187
+ }
188
+ if (leased.decision.kind === "allCapped") {
189
+ // Bounded self-heal of stale usage cache, then ONE new atomic selection.
190
+ await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
191
+ signalCodexCapacityWorkflow,
192
+ wakeSessionWorkflow,
193
+ });
194
+ if (settings.codexCredentialLeasingEnabled) {
195
+ leaseAcquisitionStartedAtMs = performance.now();
196
+ leased = await acquireCodexCredentialLease(
197
+ db,
198
+ {
199
+ accountId: input.accountId,
200
+ workspaceId: input.workspaceId,
201
+ turnId,
202
+ holderId,
203
+ advanceActivePointer: sessionPin === null,
204
+ },
205
+ selectForTurn,
206
+ );
207
+ } else {
208
+ const [rotation, accounts] = await Promise.all([
209
+ getCodexRotationSettings(db, input.workspaceId),
210
+ listCodexAccountStatuses(db, input.workspaceId),
211
+ ]);
212
+ const leaseAccounts = accounts.map((account) => ({
213
+ ...account,
214
+ activeLeaseCount: 0,
215
+ selectionCount: 0,
216
+ lastSelectedAt: null,
217
+ }));
218
+ const selected = selectForTurn({
219
+ accounts: leaseAccounts,
220
+ activeCredentialId: rotation?.activeCredentialId ?? null,
221
+ rotationEnabled: rotation?.rotationEnabled ?? false,
222
+ leaseRotationEnabled: false,
223
+ rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
224
+ existingCredentialId: null,
225
+ policyScope: null,
226
+ unavailableDiagnostics: [],
227
+ });
228
+ leased = {
229
+ ...selected,
230
+ accounts: leaseAccounts,
231
+ activeCredentialId: rotation?.activeCredentialId ?? null,
232
+ rotationEnabled: rotation?.rotationEnabled ?? false,
233
+ rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
234
+ reused: false,
235
+ holderId: null,
236
+ generation: null,
237
+ leasedUntil: null,
238
+ unavailableDiagnostics: [],
239
+ advanceActivePointer: selected.advanceActivePointer !== false,
240
+ };
241
+ }
242
+ }
243
+ const rotationDecision = leased.decision;
244
+ const selectedPinDisposition = classifyCodexPin({
245
+ pinnedCredentialId: sessionPin,
246
+ pinSource: sessionPinSource,
247
+ strategy: leased.rotationStrategy as CodexRotationStrategy,
248
+ rotationEnabled: leased.rotationEnabled,
249
+ });
250
+ // pin policy pin persistence follows the atomic credential allocator selection. The selector
251
+ // already ran exact-turn reuse before policy filtering and vetoed pointer
252
+ // movement for manual/policy homes; this write only records the NEXT turn's
253
+ // policy home (or clears a policy pin whose strategy is no longer active).
254
+ if (
255
+ selectedPinDisposition === "sharded" &&
256
+ leased.credentialId !== null &&
257
+ (sessionPinSource !== "policy" || sessionPin !== leased.credentialId)
258
+ ) {
259
+ const pinMutation = await withSessionCodexCapacityMutation(
260
+ db,
261
+ {
262
+ workspaceId: input.workspaceId,
263
+ reason: "codex_policy_pin_changed",
264
+ },
265
+ async (tx) => {
266
+ const changed = await setSessionCodexPinInTransaction(
267
+ tx,
268
+ input.workspaceId,
269
+ input.sessionId,
270
+ leased.credentialId,
271
+ "policy",
272
+ {
273
+ expected: {
274
+ pinnedCredentialId: sessionPin,
275
+ pinSource: sessionPinSource,
276
+ },
277
+ },
278
+ );
279
+ return { result: changed, changed };
280
+ },
281
+ );
282
+ await signalCodexCapacityWakeTargets(
283
+ { signalCodexCapacityWorkflow, wakeSessionWorkflow },
284
+ pinMutation.wakeTargets,
285
+ );
286
+ } else if (selectedPinDisposition === "clearStale") {
287
+ const pinMutation = await withSessionCodexCapacityMutation(
288
+ db,
289
+ {
290
+ workspaceId: input.workspaceId,
291
+ reason: "codex_stale_policy_pin_cleared",
292
+ },
293
+ async (tx) => {
294
+ const changed = await setSessionCodexPinInTransaction(
295
+ tx,
296
+ input.workspaceId,
297
+ input.sessionId,
298
+ null,
299
+ "policy",
300
+ {
301
+ expected: {
302
+ pinnedCredentialId: sessionPin,
303
+ pinSource: sessionPinSource,
304
+ },
305
+ },
306
+ );
307
+ return { result: changed, changed };
308
+ },
309
+ );
310
+ await signalCodexCapacityWakeTargets(
311
+ { signalCodexCapacityWorkflow, wakeSessionWorkflow },
312
+ pinMutation.wakeTargets,
313
+ );
314
+ }
315
+ if (
316
+ !settings.codexCredentialLeasingEnabled &&
317
+ leased.advanceActivePointer &&
318
+ sessionPin === null &&
319
+ rotationDecision.kind === "active" &&
320
+ rotationDecision.moved
321
+ ) {
322
+ await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
323
+ }
324
+ providerTurn.effectiveCodexCredentialId = leased.credentialId;
325
+ leases.codex.generation = leased.generation;
326
+ leases.codex.confirmedUntilMs =
327
+ leased.leasedUntil && leaseAcquisitionStartedAtMs !== null
328
+ ? leaseAcquisitionStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS
329
+ : null;
330
+ leases.codex.held =
331
+ providerTurn.effectiveCodexCredentialId !== null &&
332
+ leased.holderId !== null &&
333
+ leased.generation !== null &&
334
+ leases.codex.confirmedUntilMs !== null;
335
+ if (leases.codex.held) leases.codex.startHeartbeat();
336
+
337
+ const actualOutcome = providerTurn.effectiveCodexCredentialId
338
+ ? "selected"
339
+ : rotationDecision.kind === "allCapped"
340
+ ? "waiting"
341
+ : "none";
342
+ const actualReason = providerTurn.effectiveCodexCredentialId
343
+ ? leased.reused
344
+ ? "lease_reused"
345
+ : sessionPin === providerTurn.effectiveCodexCredentialId
346
+ ? "pin"
347
+ : rotationDecision.kind === "active" && rotationDecision.moved
348
+ ? "rotation"
349
+ : "active"
350
+ : rotationDecision.kind === "allCapped"
351
+ ? "all_capped"
352
+ : "none";
353
+ const fencedInFlight = leased.reused;
354
+ const shadowResult = await publishCodexFleetShadowDecisionV1({
355
+ enabled: settings.codexFleetPolicyShadowEnabled,
356
+ decision: {
357
+ accounts: leased.accounts,
358
+ actualCredentialId: providerTurn.effectiveCodexCredentialId,
359
+ actualOutcome,
360
+ actualReason,
361
+ affinityCredentialId: fencedInFlight
362
+ ? providerTurn.effectiveCodexCredentialId
363
+ : (sessionPin ?? sessionCodex?.lastCredentialId ?? null),
364
+ fencedInFlight,
365
+ nearExhaustionPct: settings.codexRotationNearExhaustionPct,
366
+ now: new Date(),
367
+ aliasSeed: randomUUID(),
368
+ },
369
+ publish: eventing.publish,
370
+ });
371
+ if (shadowResult.outcome === "published") {
372
+ const shadowPayload = shadowResult.payload;
373
+ observability.incrementCounter({
374
+ name: "opengeni_codex_fleet_shadow_decisions_total",
375
+ help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
376
+ labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload),
377
+ });
378
+ observability.info("Codex adaptive fleet shadow decision", {
379
+ workspaceId: input.workspaceId,
380
+ policyVersion: shadowPayload.replay.policyVersion,
381
+ inputFingerprint: shadowPayload.replay.inputFingerprint,
382
+ decisionFingerprint: shadowPayload.replay.decisionFingerprint,
383
+ actualOutcome: shadowPayload.actual.outcome,
384
+ shadowOutcome: shadowPayload.replay.decision.outcome,
385
+ comparison: shadowPayload.comparison,
386
+ candidateCount: shadowPayload.replay.input.candidates.length,
387
+ truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
388
+ payloadBytes: shadowResult.payloadBytes,
389
+ });
390
+ } else if (shadowResult.outcome === "failed") {
391
+ // Shadow observability is explicitly non-authoritative. A malformed
392
+ // snapshot or event-write fault must never change the authoritative lease,
393
+ // capacity wait, failover, or the account serving this fenced turn.
394
+ observability.incrementCounter({
395
+ name: "opengeni_codex_fleet_shadow_errors_total",
396
+ help: "Shadow decision build/publication failures.",
397
+ labels: codexFleetShadowErrorMetricLabelsV1(shadowResult),
398
+ });
399
+ observability.warn("Codex adaptive fleet shadow decision failed open", {
400
+ stage: shadowResult.stage,
401
+ reason: shadowResult.reason,
402
+ errorClass: "CodexFleetShadowOperationError",
403
+ errorCode: "codex_fleet_shadow_failed",
404
+ origin: "worker",
405
+ payloadBytes: shadowResult.payloadBytes,
406
+ });
407
+ }
408
+
409
+ const eligibleCount = leased.accounts.filter((account) =>
410
+ isCodexCredentialEligible(account, new Date()),
411
+ ).length;
412
+ const poolDepth = eligibleCount === 0 ? "zero" : eligibleCount === 1 ? "one" : "many";
413
+ observability.incrementCounter({
414
+ name: "opengeni_codex_pool_observations_total",
415
+ help: "Observed eligible Codex pool depth buckets at turn selection.",
416
+ labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
417
+ });
418
+ if (eligibleCount <= 1) {
419
+ observability.incrementCounter({
420
+ name: "opengeni_codex_pool_low_total",
421
+ help: "Alert signal emitted when the eligible Codex pool is zero or one.",
422
+ labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
423
+ });
424
+ observability.warn("Codex eligible credential pool is low", {
425
+ workspaceId: input.workspaceId,
426
+ eligibleCount,
427
+ connectedCount: leased.accounts.length,
428
+ depth: poolDepth,
429
+ });
430
+ }
431
+
432
+ if (
433
+ providerTurn.effectiveCodexCredentialId === null &&
434
+ leased.accounts.length > 0 &&
435
+ leased.accounts.every((account) => !account.allocatorEnabled) &&
436
+ turnId
437
+ ) {
438
+ if (turn.source === "compaction") {
439
+ if (
440
+ !(await eventing.settle!({
441
+ events: [
442
+ {
443
+ type: "turn.cancelled",
444
+ payload: {
445
+ maintenance: "context_compaction",
446
+ reason: "codex_allocator_disabled",
447
+ requestPreserved: true,
448
+ },
449
+ },
450
+ {
451
+ type: "session.status.changed",
452
+ payload: { status: "idle" },
453
+ },
454
+ ],
455
+ turnStatus: "cancelled",
456
+ sessionStatus: "idle",
457
+ activeTurnId: null,
458
+ }))
459
+ ) {
460
+ return { exit: claimedResult({ status: "cancelled" }) };
461
+ }
462
+ control.turnMetricOutcome = "cancelled";
463
+ control.activityStatus = "idle";
464
+ return { exit: claimedResult({ status: "idle", deferredUntilWake: true }) };
465
+ }
466
+ const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
467
+ const activeGoal = goal?.status === "active" ? goal : null;
468
+ const armed = await armCodexCapacityWait(db, {
469
+ accountId: input.accountId,
470
+ workspaceId: input.workspaceId,
471
+ sessionId: input.sessionId,
472
+ turnId,
473
+ attemptId: input.attemptId,
474
+ workflowId: input.workflowId,
475
+ goalId: activeGoal?.id ?? null,
476
+ goalVersion: activeGoal?.version ?? null,
477
+ earliestResetAt: null,
478
+ resetKind: "bounded_refresh",
479
+ failurePayload: {
480
+ error: "All connected Codex subscriptions are disabled for new allocations.",
481
+ code: "codex_allocator_disabled",
482
+ detail: "waiting for a credential to be re-enabled, reconnected, or added",
483
+ },
484
+ });
485
+ if (armed.action === "waiting") {
486
+ await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, armed.events);
487
+ control.turnMetricOutcome = "recovering";
488
+ control.activityStatus = "waiting_capacity";
489
+ return {
490
+ exit: claimedResult({
491
+ status: "waiting_capacity",
492
+ capacityWait: {
493
+ waiterId: armed.waiter.id,
494
+ generation: armed.waiter.generation,
495
+ nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
496
+ wakeRevision: armed.waiter.wakeRevision,
497
+ },
498
+ }),
499
+ };
500
+ }
501
+ if (
502
+ !(await eventing.settle!({
503
+ events: [
504
+ {
505
+ type: "turn.failed",
506
+ payload: {
507
+ error: "All connected Codex subscriptions are disabled for new allocations.",
508
+ code: "codex_allocator_disabled",
509
+ retryable: false,
510
+ recovery: "user_message",
511
+ },
512
+ },
513
+ {
514
+ type: "session.status.changed",
515
+ payload: { status: "idle" },
516
+ },
517
+ ],
518
+ turnStatus: "failed",
519
+ sessionStatus: "idle",
520
+ activeTurnId: null,
521
+ }))
522
+ ) {
523
+ return { exit: claimedResult({ status: "cancelled" }) };
524
+ }
525
+ control.turnMetricOutcome = "failed";
526
+ control.activityStatus = "idle";
527
+ return { exit: claimedResult({ status: "idle" }) };
528
+ }
529
+
530
+ if (rotationDecision.kind === "allCapped" && turnId) {
531
+ if (turn.source === "compaction") {
532
+ if (
533
+ !(await eventing.settle!({
534
+ events: [
535
+ {
536
+ type: "turn.cancelled",
537
+ payload: {
538
+ maintenance: "context_compaction",
539
+ reason: "codex_capacity_unavailable",
540
+ requestPreserved: true,
541
+ },
542
+ },
543
+ {
544
+ type: "session.status.changed",
545
+ payload: { status: "idle" },
546
+ },
547
+ ],
548
+ turnStatus: "cancelled",
549
+ sessionStatus: "idle",
550
+ activeTurnId: null,
551
+ }))
552
+ ) {
553
+ return { exit: claimedResult({ status: "cancelled" }) };
554
+ }
555
+ control.turnMetricOutcome = "cancelled";
556
+ control.activityStatus = "idle";
557
+ return { exit: claimedResult({ status: "idle", deferredUntilWake: true }) };
558
+ }
559
+ // Every eligible account is capped/cooling (and a usage refresh did NOT
560
+ // surface a reset): idle the turn AT THE BOUNDARY (no wasted model/sandbox
561
+ // build) until the EARLIEST reset across all accounts — the multi-account
562
+ // generalization of #143's single-account idle-until-reset. No saveRunState:
563
+ // no model ran, nothing to freeze.
564
+ const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
565
+ const goalActive = Boolean(goal && goal.status === "active");
566
+ // BOUNDED + POSITIVE: clamp to [MIN_IDLE_MS, max] so a null/elapsed/unknown
567
+ // reset can never yield a 0 (which session.ts would treat as "continue now",
568
+ // re-entering this path in a tight CPU/DB-hammering loop).
569
+ const resumeMs = computeIdleDelayMs(
570
+ rotationDecision.earliestResetAt,
571
+ new Date(),
572
+ CODEX_USAGE_LIMIT_MAX_RESUME_MS,
573
+ );
574
+ const failurePayload = codexUsageLimitFailurePayload(
575
+ { resetsInSeconds: Math.ceil(resumeMs / 1000) },
576
+ "all connected Codex subscriptions are rate-limited",
577
+ { allAccounts: true },
578
+ );
579
+ const authoritativeResetAt = authoritativeCodexCapacityResetAt(leased.accounts, new Date());
580
+ const armed = await armCodexCapacityWait(db, {
581
+ accountId: input.accountId,
582
+ workspaceId: input.workspaceId,
583
+ sessionId: input.sessionId,
584
+ turnId,
585
+ attemptId: input.attemptId,
586
+ workflowId: input.workflowId,
587
+ goalId: goalActive && goal ? goal.id : null,
588
+ goalVersion: goalActive && goal ? goal.version : null,
589
+ earliestResetAt: authoritativeResetAt,
590
+ resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
591
+ failurePayload,
592
+ });
593
+ if (armed.action === "waiting") {
594
+ await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, armed.events);
595
+ control.turnMetricOutcome = "recovering";
596
+ control.activityStatus = "waiting_capacity";
597
+ return {
598
+ exit: claimedResult({
599
+ status: "waiting_capacity",
600
+ capacityWait: {
601
+ waiterId: armed.waiter.id,
602
+ generation: armed.waiter.generation,
603
+ nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
604
+ wakeRevision: armed.waiter.wakeRevision,
605
+ },
606
+ }),
607
+ };
608
+ }
609
+ if (
610
+ !(await eventing.settle!({
611
+ events: [
612
+ // `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
613
+ // rotation-wait state as the reactive all-capped path, so it must freeze
614
+ // autoContinuations identically (evaluateGoalContinuation reads this marker)
615
+ // — a goal waiting out a long reset must not burn its continuation budget on
616
+ // the proactive path while the reactive path spares it.
617
+ {
618
+ type: "turn.failed",
619
+ payload: {
620
+ ...failurePayload,
621
+ recovery: goalActive ? "goal_continuation" : "user_message",
622
+ rotated: true,
623
+ },
624
+ },
625
+ {
626
+ type: "session.status.changed",
627
+ payload: { status: "idle" },
628
+ },
629
+ ],
630
+ turnStatus: "failed",
631
+ sessionStatus: "idle",
632
+ activeTurnId: null,
633
+ }))
634
+ ) {
635
+ return { exit: claimedResult({ status: "cancelled" }) };
636
+ }
637
+ control.turnMetricOutcome = "failed";
638
+ control.activityStatus = "idle";
639
+ // idleUntilReset marks this a MANDATORY hold: session.ts must wait the full
640
+ // resumeMs even if a future change made it 0 — never a tight re-dispatch.
641
+ return {
642
+ exit: claimedResult(
643
+ goalActive
644
+ ? {
645
+ status: "idle",
646
+ continueDelayMs: resumeMs,
647
+ idleUntilReset: true,
648
+ }
649
+ : { status: "idle" },
650
+ ),
651
+ };
652
+ }
653
+ if (providerTurn.effectiveCodexCredentialId) {
654
+ const priorAccountId = sessionCodex?.lastCredentialId ?? null;
655
+ if (priorAccountId !== providerTurn.effectiveCodexCredentialId) {
656
+ await recordSessionActiveCodexCredential(
657
+ db,
658
+ input.workspaceId,
659
+ input.sessionId,
660
+ providerTurn.effectiveCodexCredentialId,
661
+ );
662
+ const rotated = rotationDecision.kind === "active" && rotationDecision.moved;
663
+ await eventing.publish([
664
+ {
665
+ type: "codex.account.switched",
666
+ payload: {
667
+ fromAccountId: priorAccountId,
668
+ toAccountId: providerTurn.effectiveCodexCredentialId,
669
+ reason: rotated ? "rotation" : "manual",
670
+ },
671
+ },
672
+ ]);
673
+ }
674
+
675
+ const selectionReason = leased.reused
676
+ ? "lease_reused"
677
+ : sessionPin === providerTurn.effectiveCodexCredentialId
678
+ ? "pin"
679
+ : rotationDecision.kind === "active" && rotationDecision.moved
680
+ ? "rotation"
681
+ : "active";
682
+ observability.incrementCounter({
683
+ name: "opengeni_codex_credential_selections_total",
684
+ help: "Codex credential selections by strategy and reason.",
685
+ labels: {
686
+ workspace_key: codexWorkspaceKey,
687
+ strategy: leased.rotationStrategy,
688
+ reason: selectionReason,
689
+ },
690
+ });
691
+ await eventing.publish([
692
+ {
693
+ type: "codex.credential.selected",
694
+ payload: {
695
+ credentialId: providerTurn.effectiveCodexCredentialId,
696
+ strategy: leased.rotationStrategy,
697
+ reason: selectionReason,
698
+ eligibleCount,
699
+ connectedCount: leased.accounts.length,
700
+ reused: leased.reused,
701
+ },
702
+ },
703
+ ]);
704
+ }
705
+ } catch (error) {
706
+ credentialSelectionOutcome = "failed";
707
+ throw error;
708
+ } finally {
709
+ recordTurnStartupPhase(observability, {
710
+ phase: "credential_selection",
711
+ provider: "codex-subscription",
712
+ backend: turn.sandboxBackend,
713
+ outcome: credentialSelectionOutcome,
714
+ durationSeconds: (performance.now() - credentialSelectionStartedAt) / 1_000,
715
+ });
716
+ }
717
+ }
718
+
719
+ return { ok: true };
720
+ }