@opengeni/worker-bundle 0.24.2-canary.0 → 0.26.0-canary.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/dist/activities/agent-turn/agent-build.d.ts +3 -10
  2. package/dist/activities/agent-turn/claim.d.ts +7 -0
  3. package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
  4. package/dist/activities/agent-turn/errors.d.ts +6 -5
  5. package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
  6. package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
  7. package/dist/activities/agent-turn/history-sink.d.ts +6 -2
  8. package/dist/activities/agent-turn/quiescence.d.ts +13 -8
  9. package/dist/activities/agent-turn/tool-environment.d.ts +2 -13
  10. package/dist/activities/agent-turn/tool-policy.d.ts +4 -0
  11. package/dist/activities/agent-turn/turn-context.d.ts +9 -1
  12. package/dist/activities/codemode-dispatcher.d.ts +10 -1
  13. package/dist/activities/codex-capacity.d.ts +32 -3
  14. package/dist/activities/codex-fleet-shadow.d.ts +1 -1
  15. package/dist/activities/codex-image-generation.d.ts +1 -1
  16. package/dist/activities/codex-rotation.d.ts +8 -67
  17. package/dist/activities/goals.d.ts +1 -1
  18. package/dist/activities/google-drive-publication.d.ts +2 -13
  19. package/dist/activities/image-generation-operation.d.ts +13 -4
  20. package/dist/activities/sandbox-lease.d.ts +7 -2
  21. package/dist/activities/session-state.d.ts +4 -2
  22. package/dist/activities/types.d.ts +11 -1
  23. package/dist/activities/xai-quota.d.ts +3 -0
  24. package/dist/{activities-control-ZWH7J4KZ.js → activities-control-KS3ZPQVY.js} +233 -88
  25. package/dist/activities-control-KS3ZPQVY.js.map +1 -0
  26. package/dist/activities-control.d.ts +2 -0
  27. package/dist/{activities-turn-I4OPLQLS.js → activities-turn-GHSFZB64.js} +1109 -1865
  28. package/dist/activities-turn-GHSFZB64.js.map +1 -0
  29. package/dist/activities.d.ts +2 -0
  30. package/dist/{chunk-65JEWP2Q.js → chunk-D4ZEFOYH.js} +1 -1
  31. package/dist/chunk-D4ZEFOYH.js.map +1 -0
  32. package/dist/{chunk-RQNH6CGR.js → chunk-O2LQWJIK.js} +3250 -2061
  33. package/dist/chunk-O2LQWJIK.js.map +1 -0
  34. package/dist/github-rest-mcp.d.ts +0 -2
  35. package/dist/index.d.ts +1 -0
  36. package/dist/index.js +2 -2
  37. package/dist/index.js.map +1 -1
  38. package/dist/observability-metrics.d.ts +1 -1
  39. package/dist/sandbox-routing.d.ts +6 -0
  40. package/dist/workflow-bundle.js +94 -9
  41. package/dist/workflows/activities.d.ts +1 -1
  42. package/dist/workflows/session.d.ts +3 -0
  43. package/package.json +17 -17
  44. package/src/activities/agent-turn/agent-build.ts +21 -26
  45. package/src/activities/agent-turn/claim.ts +26 -1
  46. package/src/activities/agent-turn/codex-capacity.ts +311 -234
  47. package/src/activities/agent-turn/credential-leases.ts +36 -0
  48. package/src/activities/agent-turn/errors.ts +73 -12
  49. package/src/activities/agent-turn/failure-settlement.ts +472 -331
  50. package/src/activities/agent-turn/history-prefix.ts +51 -0
  51. package/src/activities/agent-turn/history-sink.ts +30 -6
  52. package/src/activities/agent-turn/quiescence.ts +29 -17
  53. package/src/activities/agent-turn/run.ts +22 -5
  54. package/src/activities/agent-turn/stream-attempt.ts +35 -8
  55. package/src/activities/agent-turn/tool-environment.ts +42 -32
  56. package/src/activities/agent-turn/tool-policy.ts +14 -8
  57. package/src/activities/agent-turn/turn-context.ts +13 -0
  58. package/src/activities/agent-turn/xai-capacity.ts +11 -0
  59. package/src/activities/codemode-dispatcher.ts +235 -59
  60. package/src/activities/codex-capacity.ts +107 -10
  61. package/src/activities/codex-fleet-shadow.ts +8 -1
  62. package/src/activities/codex-image-generation.ts +19 -2
  63. package/src/activities/codex-rotation.ts +45 -142
  64. package/src/activities/goals.ts +9 -9
  65. package/src/activities/google-drive-publication.ts +4 -81
  66. package/src/activities/image-generation-operation.ts +28 -3
  67. package/src/activities/run-input.ts +6 -7
  68. package/src/activities/sandbox-lease.ts +246 -87
  69. package/src/activities/session-state.ts +20 -1
  70. package/src/activities/types.ts +16 -8
  71. package/src/activities/xai-quota.ts +35 -0
  72. package/src/activities.ts +1 -0
  73. package/src/github-rest-mcp.ts +0 -82
  74. package/src/observability-metrics.ts +4 -0
  75. package/src/sandbox-routing.ts +186 -16
  76. package/src/workflows/activities.ts +1 -0
  77. package/src/workflows/session.ts +114 -8
  78. package/dist/activities/api-integrations.d.ts +0 -21
  79. package/dist/activities-control-ZWH7J4KZ.js.map +0 -1
  80. package/dist/activities-turn-I4OPLQLS.js.map +0 -1
  81. package/dist/chunk-65JEWP2Q.js.map +0 -1
  82. package/dist/chunk-RQNH6CGR.js.map +0 -1
  83. package/src/activities/api-integrations.ts +0 -151
@@ -2,21 +2,27 @@ import { randomUUID } from "node:crypto";
2
2
  import {
3
3
  AttemptToolApprovalRequiredError,
4
4
  AttemptToolCatalogStaleError,
5
+ AttemptToolInputValidationError,
5
6
  AttemptToolNotFoundError,
6
7
  codemodeDispatchSubject,
7
8
  decodeCodemodeDispatchRequest,
8
9
  encodeCodemodeDispatchAck,
10
+ type AttemptToolCatalogEntry,
9
11
  type AttemptToolEnvironment,
10
12
  } from "@opengeni/codemode";
11
13
  import {
14
+ CODEMODE_CLAIM_LEASE_MS,
12
15
  CODEMODE_CLAIM_HEARTBEAT_MS,
13
16
  CODEMODE_MAX_CONCURRENT_CALLS_PER_ATTEMPT,
14
17
  type CodemodeOperation,
18
+ type SessionEvent,
15
19
  } from "@opengeni/contracts";
16
20
  import {
17
21
  cancelQueuedCodemodeOperationsForAttempt,
18
22
  claimCodemodeOperation,
19
23
  failCodemodeOperation,
24
+ getSessionEventByClientEventId,
25
+ getSessionToolCallCreatedEventByOperationId,
20
26
  markCodemodeOperationExecutionStarted,
21
27
  renewCodemodeOperationClaim,
22
28
  settleCodemodeOperationWithOutput,
@@ -37,6 +43,11 @@ export type CodemodeDispatcherScope = {
37
43
  executionGeneration: number;
38
44
  };
39
45
 
46
+ export type CodemodeDispatcherTimings = {
47
+ claimLeaseMs?: number;
48
+ claimHeartbeatMs?: number;
49
+ };
50
+
40
51
  /**
41
52
  * Owns the only Codemode execution edge for one exact attempt. NATS carries a
42
53
  * wake-up only; the durable row is claimed before any side effect and the call
@@ -48,6 +59,8 @@ export class CodemodeAttemptDispatcher {
48
59
  private pendingClaims = 0;
49
60
  private unsubscribe: (() => void) | null = null;
50
61
  private closing = false;
62
+ private readonly claimLeaseMs: number;
63
+ private readonly claimHeartbeatMs: number;
51
64
 
52
65
  constructor(
53
66
  private readonly db: Database,
@@ -56,6 +69,7 @@ export class CodemodeAttemptDispatcher {
56
69
  private readonly scope: CodemodeDispatcherScope,
57
70
  private readonly turnSignal?: AbortSignal,
58
71
  private readonly maxConcurrentCalls = CODEMODE_MAX_CONCURRENT_CALLS_PER_ATTEMPT,
72
+ timings: CodemodeDispatcherTimings = {},
59
73
  ) {
60
74
  if (
61
75
  environment.catalog.accountId !== scope.accountId ||
@@ -70,6 +84,22 @@ export class CodemodeAttemptDispatcher {
70
84
  if (!Number.isSafeInteger(maxConcurrentCalls) || maxConcurrentCalls < 1) {
71
85
  throw new Error("Codemode dispatcher concurrency must be a positive safe integer");
72
86
  }
87
+ this.claimLeaseMs = timings.claimLeaseMs ?? CODEMODE_CLAIM_LEASE_MS;
88
+ this.claimHeartbeatMs = timings.claimHeartbeatMs ?? CODEMODE_CLAIM_HEARTBEAT_MS;
89
+ if (
90
+ !Number.isSafeInteger(this.claimLeaseMs) ||
91
+ this.claimLeaseMs < 1_000 ||
92
+ this.claimLeaseMs > 10 * 60_000
93
+ ) {
94
+ throw new Error("Codemode dispatcher claim lease must be between 1 second and 10 minutes");
95
+ }
96
+ if (
97
+ !Number.isSafeInteger(this.claimHeartbeatMs) ||
98
+ this.claimHeartbeatMs < 1 ||
99
+ this.claimHeartbeatMs >= this.claimLeaseMs
100
+ ) {
101
+ throw new Error("Codemode dispatcher heartbeat must be positive and shorter than its lease");
102
+ }
73
103
  }
74
104
 
75
105
  start(): void {
@@ -103,11 +133,14 @@ export class CodemodeAttemptDispatcher {
103
133
  catalogDigest: request.catalogDigest,
104
134
  operationId: request.operationId,
105
135
  claimId: randomUUID(),
136
+ claimLeaseMs: this.claimLeaseMs,
106
137
  });
107
138
  if (claim.status === "claimed") {
108
- const execution = this.execute(claim.operation, claim.claimId).finally(() => {
109
- this.inFlight.delete(request.operationId);
110
- });
139
+ const execution = this.execute(claim.operation, claim.claimId, claim.reclaimed).finally(
140
+ () => {
141
+ this.inFlight.delete(request.operationId);
142
+ },
143
+ );
111
144
  this.inFlight.set(request.operationId, execution);
112
145
  return encodeCodemodeDispatchAck({
113
146
  version: 1,
@@ -115,6 +148,19 @@ export class CodemodeAttemptDispatcher {
115
148
  status: "accepted",
116
149
  });
117
150
  }
151
+ if (claim.status === "execution_owner_lost") {
152
+ await this.settleWithOutput(claim.operation, claim.claimId, {
153
+ state: "outcome_unknown",
154
+ errorCode: "worker_lost_during_execution",
155
+ errorMessage:
156
+ "The execution owner disappeared after the tool call began. Inspect actual state before retrying.",
157
+ });
158
+ return encodeCodemodeDispatchAck({
159
+ version: 1,
160
+ operationId: request.operationId,
161
+ status: "terminal",
162
+ });
163
+ }
118
164
  return encodeCodemodeDispatchAck({
119
165
  version: 1,
120
166
  operationId: request.operationId,
@@ -145,7 +191,39 @@ export class CodemodeAttemptDispatcher {
145
191
  await Promise.allSettled([...this.inFlight.values()]);
146
192
  }
147
193
 
148
- private async execute(operation: CodemodeOperation, claimId: string): Promise<void> {
194
+ private async execute(
195
+ operation: CodemodeOperation,
196
+ claimId: string,
197
+ reclaimed: boolean,
198
+ ): Promise<void> {
199
+ const claimAuthority = {
200
+ accountId: this.scope.accountId,
201
+ workspaceId: this.scope.workspaceId,
202
+ attemptId: this.scope.attemptId,
203
+ operationId: operation.operationId,
204
+ claimId,
205
+ claimLeaseMs: this.claimLeaseMs,
206
+ };
207
+ // Preparation may include provider preflight and policy resolution. Keep the
208
+ // durable claim alive from the moment execution is accepted, not only after
209
+ // the provider side-effect marker, so a slow preflight cannot be reclaimed.
210
+ const heartbeat = setInterval(() => {
211
+ void renewCodemodeOperationClaim(this.db, claimAuthority).catch(() => undefined);
212
+ }, this.claimHeartbeatMs);
213
+ heartbeat.unref?.();
214
+ try {
215
+ await this.executeClaimed(operation, claimId, reclaimed, claimAuthority);
216
+ } finally {
217
+ clearInterval(heartbeat);
218
+ }
219
+ }
220
+
221
+ private async executeClaimed(
222
+ operation: CodemodeOperation,
223
+ claimId: string,
224
+ reclaimed: boolean,
225
+ claimAuthority: Parameters<typeof markCodemodeOperationExecutionStarted>[1],
226
+ ): Promise<void> {
149
227
  const entry = this.environment.catalog.entries.find(
150
228
  (candidate) =>
151
229
  candidate.identity.serverId === operation.identity.serverId &&
@@ -170,38 +248,7 @@ export class CodemodeAttemptDispatcher {
170
248
  );
171
249
  return;
172
250
  }
173
- const created = await appendAndPublishTurnEventsFenced(
174
- this.db,
175
- this.bus,
176
- this.scope.workspaceId,
177
- this.scope.sessionId,
178
- this.scope.turnId,
179
- this.scope.executionGeneration,
180
- this.scope.attemptId,
181
- [
182
- {
183
- type: "agent.toolCall.created",
184
- turnId: this.scope.turnId,
185
- turnGeneration: this.scope.executionGeneration,
186
- turnAttemptId: this.scope.attemptId,
187
- producerId: operation.caller.subjectId,
188
- payload: {
189
- id: operation.operationId,
190
- name: entry.modelName,
191
- arguments: operation.arguments,
192
- origin: "codemode",
193
- subjectId: operation.caller.subjectId,
194
- raw: {
195
- type: "codemode_call",
196
- serverId: operation.identity.serverId,
197
- toolName: operation.identity.toolName,
198
- catalogDigest: operation.catalogDigest,
199
- },
200
- },
201
- },
202
- ],
203
- ).catch(() => ({ accepted: false }));
204
- if (!created.accepted) {
251
+ if (!(await this.ensureToolCallCreated(operation, entry, reclaimed))) {
205
252
  await this.failBeforeExecution(
206
253
  operation.operationId,
207
254
  claimId,
@@ -219,24 +266,9 @@ export class CodemodeAttemptDispatcher {
219
266
  return;
220
267
  }
221
268
 
222
- const claimAuthority = {
223
- accountId: this.scope.accountId,
224
- workspaceId: this.scope.workspaceId,
225
- attemptId: this.scope.attemptId,
226
- operationId: operation.operationId,
227
- claimId,
228
- };
229
- const crossedExecutionBoundary = await markCodemodeOperationExecutionStarted(
230
- this.db,
231
- claimAuthority,
232
- );
233
- if (!crossedExecutionBoundary) return;
234
- const heartbeat = setInterval(() => {
235
- void renewCodemodeOperationClaim(this.db, claimAuthority).catch(() => undefined);
236
- }, CODEMODE_CLAIM_HEARTBEAT_MS);
237
- heartbeat.unref?.();
269
+ let preparedCall: Awaited<ReturnType<AttemptToolEnvironment["prepareCall"]>>;
238
270
  try {
239
- const result = await this.environment.call(
271
+ preparedCall = await this.environment.prepareCall(
240
272
  {
241
273
  operationId: operation.operationId,
242
274
  catalogDigest: operation.catalogDigest,
@@ -246,7 +278,34 @@ export class CodemodeAttemptDispatcher {
246
278
  },
247
279
  { signal },
248
280
  );
249
- await this.settleWithOutput(operation, claimId, { state: "completed", result });
281
+ } catch (error) {
282
+ await this.settleWithOutput(operation, claimId, {
283
+ state: "failed",
284
+ errorCode: errorCode(error, signal, false),
285
+ errorMessage: safeErrorMessage(error),
286
+ });
287
+ return;
288
+ }
289
+ if (signal.aborted) {
290
+ await this.settleWithOutput(operation, claimId, {
291
+ state: "failed",
292
+ errorCode: "attempt_cancelled",
293
+ errorMessage: "Execution attempt ended before the Codemode call started",
294
+ });
295
+ return;
296
+ }
297
+
298
+ const crossedExecutionBoundary = await markCodemodeOperationExecutionStarted(
299
+ this.db,
300
+ claimAuthority,
301
+ );
302
+ if (!crossedExecutionBoundary) return;
303
+ try {
304
+ const result = await preparedCall.execute();
305
+ await this.settleWithOutput(operation, claimId, {
306
+ state: "completed",
307
+ result,
308
+ });
250
309
  } catch (error) {
251
310
  const knownPreExecution =
252
311
  error instanceof AttemptToolApprovalRequiredError ||
@@ -255,13 +314,90 @@ export class CodemodeAttemptDispatcher {
255
314
  connectorActionWasNotExecuted(error);
256
315
  await this.settleWithOutput(operation, claimId, {
257
316
  state: knownPreExecution ? "failed" : "outcome_unknown",
258
- errorCode: errorCode(error, signal),
317
+ errorCode: errorCode(error, signal, true),
259
318
  errorMessage: !knownPreExecution
260
319
  ? "Tool execution ended without a durable result. Its side-effect outcome is unknown; inspect actual state before retrying."
261
320
  : safeErrorMessage(error),
262
321
  });
263
- } finally {
264
- clearInterval(heartbeat);
322
+ }
323
+ }
324
+
325
+ private async ensureToolCallCreated(
326
+ operation: CodemodeOperation,
327
+ entry: AttemptToolCatalogEntry,
328
+ reclaimed: boolean,
329
+ ): Promise<boolean> {
330
+ const clientEventId = codemodeToolCallCreatedClientEventId(operation.operationId);
331
+ if (reclaimed) {
332
+ const existing = await getSessionEventByClientEventId(
333
+ this.db,
334
+ this.scope.workspaceId,
335
+ this.scope.sessionId,
336
+ clientEventId,
337
+ ).catch(() => null);
338
+ if (existing) {
339
+ assertMatchingCodemodeToolCallCreated(existing, operation, entry, this.scope);
340
+ return true;
341
+ }
342
+ const legacyCreated = await getSessionToolCallCreatedEventByOperationId(this.db, {
343
+ workspaceId: this.scope.workspaceId,
344
+ sessionId: this.scope.sessionId,
345
+ turnId: this.scope.turnId,
346
+ attemptId: this.scope.attemptId,
347
+ operationId: operation.operationId,
348
+ }).catch(() => null);
349
+ if (legacyCreated) {
350
+ assertMatchingCodemodeToolCallCreated(legacyCreated, operation, entry, this.scope);
351
+ return true;
352
+ }
353
+ }
354
+ try {
355
+ const created = await appendAndPublishTurnEventsFenced(
356
+ this.db,
357
+ this.bus,
358
+ this.scope.workspaceId,
359
+ this.scope.sessionId,
360
+ this.scope.turnId,
361
+ this.scope.executionGeneration,
362
+ this.scope.attemptId,
363
+ [
364
+ {
365
+ type: "agent.toolCall.created",
366
+ clientEventId,
367
+ turnId: this.scope.turnId,
368
+ turnGeneration: this.scope.executionGeneration,
369
+ turnAttemptId: this.scope.attemptId,
370
+ producerId: operation.caller.subjectId,
371
+ payload: {
372
+ id: operation.operationId,
373
+ name: entry.modelName,
374
+ arguments: operation.arguments,
375
+ origin: "codemode",
376
+ subjectId: operation.caller.subjectId,
377
+ raw: {
378
+ type: "codemode_call",
379
+ serverId: operation.identity.serverId,
380
+ toolName: operation.identity.toolName,
381
+ catalogDigest: operation.catalogDigest,
382
+ },
383
+ },
384
+ },
385
+ ],
386
+ );
387
+ return created.accepted;
388
+ } catch {
389
+ // A concurrent reclaimed claimant can win the unique client-event insert,
390
+ // or a committed append can lose its response. Reconcile durable truth
391
+ // before treating the attempt fence as closed.
392
+ const raced = await getSessionEventByClientEventId(
393
+ this.db,
394
+ this.scope.workspaceId,
395
+ this.scope.sessionId,
396
+ clientEventId,
397
+ ).catch(() => null);
398
+ if (!raced) return false;
399
+ assertMatchingCodemodeToolCallCreated(raced, operation, entry, this.scope);
400
+ return true;
265
401
  }
266
402
  }
267
403
 
@@ -311,6 +447,43 @@ export class CodemodeAttemptDispatcher {
311
447
  }
312
448
  }
313
449
 
450
+ export function codemodeToolCallCreatedClientEventId(operationId: string): string {
451
+ return `opengeni:codemode-tool-call-created:${operationId}`;
452
+ }
453
+
454
+ function assertMatchingCodemodeToolCallCreated(
455
+ event: SessionEvent,
456
+ operation: CodemodeOperation,
457
+ entry: AttemptToolCatalogEntry,
458
+ scope: CodemodeDispatcherScope,
459
+ ): void {
460
+ const payload = recordValue(event.payload);
461
+ const raw = recordValue(payload?.raw);
462
+ if (
463
+ event.type !== "agent.toolCall.created" ||
464
+ event.turnId !== scope.turnId ||
465
+ event.turnGeneration !== scope.executionGeneration ||
466
+ event.turnAttemptId !== scope.attemptId ||
467
+ event.turnAssociation !== "current" ||
468
+ payload?.id !== operation.operationId ||
469
+ payload?.name !== entry.modelName ||
470
+ payload?.origin !== "codemode" ||
471
+ payload?.subjectId !== operation.caller.subjectId ||
472
+ raw?.type !== "codemode_call" ||
473
+ raw?.serverId !== operation.identity.serverId ||
474
+ raw?.toolName !== operation.identity.toolName ||
475
+ raw?.catalogDigest !== operation.catalogDigest
476
+ ) {
477
+ throw new Error("Codemode tool-call creation id is bound to incompatible durable evidence");
478
+ }
479
+ }
480
+
481
+ function recordValue(value: unknown): Record<string, unknown> | null {
482
+ return value !== null && typeof value === "object" && !Array.isArray(value)
483
+ ? (value as Record<string, unknown>)
484
+ : null;
485
+ }
486
+
314
487
  function connectorActionWasNotExecuted(error: unknown): boolean {
315
488
  return Boolean(
316
489
  error &&
@@ -324,8 +497,10 @@ function combinedSignal(primary: AbortSignal, secondary?: AbortSignal): AbortSig
324
497
  return secondary ? AbortSignal.any([primary, secondary]) : primary;
325
498
  }
326
499
 
327
- function errorCode(error: unknown, signal: AbortSignal): string {
328
- if (signal.aborted) return "attempt_cancelled_during_execution";
500
+ function errorCode(error: unknown, signal: AbortSignal, executionStarted: boolean): string {
501
+ if (signal.aborted) {
502
+ return executionStarted ? "attempt_cancelled_during_execution" : "attempt_cancelled";
503
+ }
329
504
  if (connectorActionWasNotExecuted(error)) return "connector_action_not_executed";
330
505
  if (typeof error === "object" && error !== null && "code" in error) {
331
506
  const code = (error as { code?: unknown }).code;
@@ -338,6 +513,7 @@ function safeErrorMessage(error: unknown): string {
338
513
  if (
339
514
  error instanceof AttemptToolApprovalRequiredError ||
340
515
  error instanceof AttemptToolCatalogStaleError ||
516
+ error instanceof AttemptToolInputValidationError ||
341
517
  error instanceof AttemptToolNotFoundError
342
518
  ) {
343
519
  return error.message;
@@ -1,7 +1,9 @@
1
1
  import {
2
+ armCodexCapacityWait,
2
3
  fetchCodexUsageForAccount,
3
4
  getCodexCapacityWaitForSession,
4
5
  getXaiCapacityWaitForSession,
6
+ resolveXaiWaiterSubject,
5
7
  listCodexAccountStatuses,
6
8
  listPendingCodexCapacityWakeTargets,
7
9
  reconcileCodexCapacityWait as reconcileCodexCapacityWaitDb,
@@ -9,7 +11,8 @@ import {
9
11
  type CodexCapacityWakeTarget,
10
12
  type CodexCapacitySelectionContext,
11
13
  } from "@opengeni/db";
12
- import type { Settings } from "@opengeni/config";
14
+ import { publishDurableSessionEvents } from "@opengeni/events";
15
+ import { refreshExhaustedXaiQuota } from "./xai-quota";
13
16
  import {
14
17
  authoritativeCodexCapacityResetAt,
15
18
  codexAccountNeedsLiveCapacityRefresh,
@@ -85,14 +88,12 @@ export async function signalPendingCodexCapacityWakeTargets(
85
88
  await signalCodexCapacityWakeTargets(services, targets);
86
89
  }
87
90
 
88
- export function codexCapacityDecision(
89
- context: CodexCapacitySelectionContext,
90
- settings: Settings,
91
+ export function codexCapacityDecision<TPolicyScope = never, TUnavailableDiagnostic = never>(
92
+ context: CodexCapacitySelectionContext<TPolicyScope, TUnavailableDiagnostic>,
91
93
  ): ReturnType<Parameters<typeof reconcileCodexCapacityWaitDb>[2]> {
92
94
  const now = new Date();
93
95
  const selected = selectCodexCredentialLeaseForTurn({
94
96
  context,
95
- leasingEnabled: settings.codexCredentialLeasingEnabled,
96
97
  sessionId: context.sessionId,
97
98
  sessionPinnedCredentialId: context.sessionPinnedCredentialId,
98
99
  sessionPinSource: context.sessionPinSource,
@@ -118,8 +119,17 @@ export function codexCapacityDecision(
118
119
  },
119
120
  };
120
121
  }
121
- const authoritativeReset = authoritativeCodexCapacityResetAt(context.accounts, now);
122
- const hasReconcilableQuotaCooldown = context.accounts.some(
122
+ const policyCredentialId =
123
+ context.sessionPinSource === "manual" && context.sessionPinnedCredentialId
124
+ ? context.sessionPinnedCredentialId
125
+ : !context.rotationEnabled
126
+ ? context.activeCredentialId
127
+ : null;
128
+ const capacityAccounts = policyCredentialId
129
+ ? context.accounts.filter((account) => account.id === policyCredentialId)
130
+ : context.accounts;
131
+ const authoritativeReset = authoritativeCodexCapacityResetAt(capacityAccounts, now);
132
+ const hasReconcilableQuotaCooldown = capacityAccounts.some(
123
133
  (account) =>
124
134
  account.status === "active" &&
125
135
  account.allocatorEnabled &&
@@ -127,19 +137,85 @@ export function codexCapacityDecision(
127
137
  account.exhaustedUntil !== null &&
128
138
  account.exhaustedUntil > now,
129
139
  );
140
+ const policyAccount = capacityAccounts[0] ?? null;
141
+ const mutationOnlyStatusBlock =
142
+ (policyAccount != null &&
143
+ (!policyAccount.allocatorEnabled || policyAccount.status !== "active")) ||
144
+ (authoritativeReset === null &&
145
+ capacityAccounts.length > 0 &&
146
+ capacityAccounts.every(
147
+ (account) => !account.allocatorEnabled || account.status !== "active",
148
+ ));
149
+ const noneReason =
150
+ selected.decision.kind === "none"
151
+ ? context.sessionPinSource === "manual" && context.sessionPinnedCredentialId !== null
152
+ ? "manual_pin_missing"
153
+ : !context.rotationEnabled && context.activeCredentialId === null
154
+ ? "rotation_off_active_pointer_missing"
155
+ : context.policyScope !== null && context.accounts.length === 0
156
+ ? "policy_filtered_pool_empty"
157
+ : context.accounts.length === 0
158
+ ? "no_connected_credentials"
159
+ : "no_eligible_credential"
160
+ : null;
130
161
  return {
131
162
  kind: "unavailable",
132
163
  earliestResetAt: authoritativeReset,
133
164
  resetKind:
134
- authoritativeReset && !hasReconcilableQuotaCooldown ? "authoritative" : "bounded_refresh",
165
+ selected.decision.kind === "none" ||
166
+ selected.decision.kind === "allocatorDisabled" ||
167
+ mutationOnlyStatusBlock
168
+ ? "mutation_only"
169
+ : authoritativeReset && !hasReconcilableQuotaCooldown
170
+ ? "authoritative"
171
+ : "bounded_refresh",
135
172
  diagnostic: {
136
173
  connectedCount: context.accounts.length,
137
174
  allocatorEnabledCount: context.accounts.filter((account) => account.allocatorEnabled).length,
138
175
  policyHash: context.policyHash,
176
+ ...(noneReason ? { reason: noneReason } : {}),
139
177
  },
140
178
  };
141
179
  }
142
180
 
181
+ /**
182
+ * Arm a durable Codex waiter and immediately re-evaluate it under the
183
+ * allocator lock. A capacity mutation that commits just before the waiter is
184
+ * inserted cannot signal a row that does not exist yet, so every arm site must
185
+ * close that edge before returning an hours-away reset timer to the workflow.
186
+ * Mutations after the arm commit still advance the waiter's wake revision.
187
+ */
188
+ export async function armAndReconcileCodexCapacityWait(
189
+ services: Pick<ControlActivityServices, "db" | "bus">,
190
+ input: Parameters<typeof armCodexCapacityWait>[1],
191
+ options: { onArmed?: () => void } = {},
192
+ ) {
193
+ const armed = await armCodexCapacityWait(services.db, input);
194
+ if (armed.action !== "waiting") return armed;
195
+
196
+ options.onArmed?.();
197
+ await publishDurableSessionEvents(services.bus, input.workspaceId, input.sessionId, armed.events);
198
+ const evaluated = await reconcileCodexCapacityWaitDb(
199
+ services.db,
200
+ {
201
+ accountId: input.accountId,
202
+ workspaceId: input.workspaceId,
203
+ sessionId: input.sessionId,
204
+ waiterId: armed.waiter.id,
205
+ generation: armed.waiter.generation,
206
+ ...(input.now ? { now: input.now } : {}),
207
+ },
208
+ (context) => codexCapacityDecision(context),
209
+ );
210
+ await publishDurableSessionEvents(
211
+ services.bus,
212
+ input.workspaceId,
213
+ input.sessionId,
214
+ evaluated.events,
215
+ );
216
+ return evaluated;
217
+ }
218
+
143
219
  async function refreshCapacityMetadata(
144
220
  services: ControlActivityServices,
145
221
  workspaceId: string,
@@ -204,6 +280,22 @@ export function createCodexCapacityActivities(services: () => Promise<ControlAct
204
280
  if (!current || current.id !== input.waiterId || current.generation !== input.generation) {
205
281
  return { action: "stale" };
206
282
  }
283
+ const authority = await resolveXaiWaiterSubject(
284
+ resolved.db,
285
+ input.workspaceId,
286
+ input.sessionId,
287
+ );
288
+ if (authority)
289
+ await refreshExhaustedXaiQuota({
290
+ db: resolved.db,
291
+ settings: resolved.settings,
292
+ accountId: input.accountId,
293
+ workspaceId: input.workspaceId,
294
+ sessionId: input.sessionId,
295
+ turnId: authority.turnId,
296
+ subjectId: authority.subjectId,
297
+ authoritySnapshot: authority.snapshot,
298
+ });
207
299
  const result = await reconcileXaiCapacityWaitDb(resolved.db, {
208
300
  accountId: input.accountId,
209
301
  workspaceId: input.workspaceId,
@@ -239,7 +331,11 @@ export function createCodexCapacityActivities(services: () => Promise<ControlAct
239
331
  if (!current || current.id !== input.waiterId || current.generation !== input.generation) {
240
332
  return { action: "stale" };
241
333
  }
242
- if (input.cause === "timer" && current.nextCheckAt.getTime() <= Date.now()) {
334
+ const boundedRefreshAttempted =
335
+ current.resetKind === "bounded_refresh" &&
336
+ input.cause === "timer" &&
337
+ current.nextCheckAt.getTime() <= Date.now();
338
+ if (boundedRefreshAttempted) {
243
339
  // This is a bounded secret-safe control-plane quota refresh. It creates no
244
340
  // turn, model call, user message, schedule, or entitlement action.
245
341
  await refreshCapacityMetadata(resolved, input.workspaceId);
@@ -252,8 +348,9 @@ export function createCodexCapacityActivities(services: () => Promise<ControlAct
252
348
  sessionId: input.sessionId,
253
349
  waiterId: input.waiterId,
254
350
  generation: input.generation,
351
+ boundedRefreshAttempted,
255
352
  },
256
- (context) => codexCapacityDecision(context, resolved.settings),
353
+ (context) => codexCapacityDecision(context),
257
354
  );
258
355
  if (result.events.length > 0) {
259
356
  try {
@@ -16,7 +16,14 @@ export const CODEX_FLEET_SHADOW_MAX_PAYLOAD_BYTES = 34 * 1_024;
16
16
  export type CodexFleetActualDecisionV1 = {
17
17
  outcome: "selected" | "waiting" | "none";
18
18
  candidateKey: string | null;
19
- reason: "lease_reused" | "pin" | "rotation" | "active" | "all_capped" | "none";
19
+ reason:
20
+ | "lease_reused"
21
+ | "pin"
22
+ | "rotation"
23
+ | "active"
24
+ | "all_capped"
25
+ | "allocator_disabled"
26
+ | "none";
20
27
  };
21
28
 
22
29
  export type CodexFleetShadowComparisonV1 =
@@ -6,6 +6,7 @@ import {
6
6
  import type { Database } from "@opengeni/db";
7
7
  import type { ObjectStorage } from "@opengeni/storage";
8
8
  import type { GeneratedImageReceipt } from "./generated-images";
9
+ import { CodexCredentialLeaseLostError } from "./agent-turn/credential-leases";
9
10
  import {
10
11
  executeImageGenerationOperation,
11
12
  imageProviderBindingHash,
@@ -27,22 +28,38 @@ export async function executeCodexImageGeneration(input: {
27
28
  prompt: string;
28
29
  references?: readonly ResolvedImageGenerationReference[];
29
30
  credentialId: string;
30
- codexContext: Pick<CodexRequestContext, "clientVersion" | "getToken" | "refresh">;
31
+ codexContext: Pick<
32
+ CodexRequestContext,
33
+ "clientVersion" | "getToken" | "refresh" | "beforeProviderDispatch"
34
+ >;
31
35
  abortSignal?: AbortSignal;
32
36
  }): Promise<GeneratedImageReceipt> {
33
37
  const providerBindingHash = imageProviderBindingHash(CODEX_PROVIDER_ID, input.credentialId);
38
+ let providerDispatchAdmitted = false;
39
+ const codexContext: Pick<
40
+ CodexRequestContext,
41
+ "clientVersion" | "getToken" | "refresh" | "beforeProviderDispatch"
42
+ > = {
43
+ ...input.codexContext,
44
+ beforeProviderDispatch: async () => {
45
+ await input.codexContext.beforeProviderDispatch?.();
46
+ providerDispatchAdmitted = true;
47
+ },
48
+ };
34
49
  return await executeImageGenerationOperation({
35
50
  ...input,
36
51
  providerId: CODEX_PROVIDER_ID,
37
52
  providerBindingHash,
38
53
  modelId: CODEX_IMAGE_MODEL,
39
54
  ...(input.references ? { referenceDigests: input.references } : {}),
55
+ isProviderDispatchRejected: (error) =>
56
+ !providerDispatchAdmitted && error instanceof CodexCredentialLeaseLostError,
40
57
  generate: async () => {
41
58
  const generated = await generateCodexSubscriptionImage({
42
59
  prompt: input.prompt,
43
60
  ...(input.references ? { references: input.references } : {}),
44
61
  turnId: input.turnId,
45
- context: input.codexContext,
62
+ context: codexContext,
46
63
  ...(input.abortSignal ? { abortSignal: input.abortSignal } : {}),
47
64
  });
48
65
  return {