@opengeni/worker-bundle 0.23.4 → 0.26.0-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +17 -2
  2. package/dist/activities/agent-turn/agent-build.d.ts +3 -10
  3. package/dist/activities/agent-turn/claim.d.ts +8 -0
  4. package/dist/activities/agent-turn/codex.d.ts +2 -2
  5. package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
  6. package/dist/activities/agent-turn/errors.d.ts +1 -1
  7. package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
  8. package/dist/activities/agent-turn/list-models.d.ts +16 -0
  9. package/dist/activities/agent-turn/model-usage.d.ts +15 -1
  10. package/dist/activities/agent-turn/quiescence.d.ts +11 -6
  11. package/dist/activities/agent-turn/run.d.ts +9 -0
  12. package/dist/activities/agent-turn/sandbox-runtime.d.ts +15 -0
  13. package/dist/activities/agent-turn/session-title.d.ts +21 -0
  14. package/dist/activities/agent-turn/stream-attempt.d.ts +4 -0
  15. package/dist/activities/agent-turn/tool-environment.d.ts +4 -13
  16. package/dist/activities/agent-turn/tool-policy.d.ts +40 -5
  17. package/dist/activities/agent-turn/turn-context.d.ts +13 -2
  18. package/dist/activities/agent-turn.d.ts +1 -1
  19. package/dist/activities/automations.d.ts +1 -0
  20. package/dist/activities/capabilities.d.ts +3 -1
  21. package/dist/activities/codemode-dispatcher.d.ts +10 -1
  22. package/dist/activities/codex-capacity.d.ts +32 -3
  23. package/dist/activities/codex-fleet-shadow.d.ts +1 -1
  24. package/dist/activities/codex-image-generation.d.ts +1 -1
  25. package/dist/activities/codex-rotation.d.ts +8 -67
  26. package/dist/activities/context-compaction.d.ts +2 -0
  27. package/dist/activities/goals.d.ts +12 -1
  28. package/dist/activities/google-drive-publication.d.ts +2 -13
  29. package/dist/activities/image-generation-operation.d.ts +13 -4
  30. package/dist/activities/image-generation-references.d.ts +2 -1
  31. package/dist/activities/scheduled-tasks.d.ts +1 -0
  32. package/dist/activities/session-state.d.ts +4 -2
  33. package/dist/activities/types.d.ts +19 -1
  34. package/dist/{activities-control-52W5IDWM.js → activities-control-CTEMLTHG.js} +339 -195
  35. package/dist/activities-control-CTEMLTHG.js.map +1 -0
  36. package/dist/activities-control.d.ts +4 -0
  37. package/dist/{activities-turn-A2G2KLSY.js → activities-turn-S5IU347A.js} +1846 -1136
  38. package/dist/activities-turn-S5IU347A.js.map +1 -0
  39. package/dist/activities.d.ts +4 -0
  40. package/dist/{chunk-GFNXVZVQ.js → chunk-D4ZEFOYH.js} +107 -7
  41. package/dist/chunk-D4ZEFOYH.js.map +1 -0
  42. package/dist/{chunk-GRFQTTSQ.js → chunk-GDCLDKSH.js} +7 -3
  43. package/dist/{chunk-GRFQTTSQ.js.map → chunk-GDCLDKSH.js.map} +1 -1
  44. package/dist/{chunk-IPVMCAW5.js → chunk-TAUE3GYT.js} +225 -154
  45. package/dist/chunk-TAUE3GYT.js.map +1 -0
  46. package/dist/github-rest-mcp.d.ts +0 -2
  47. package/dist/index.d.ts +2 -0
  48. package/dist/index.js +43 -7
  49. package/dist/index.js.map +1 -1
  50. package/dist/observability-metrics.d.ts +35 -8
  51. package/dist/sandbox-reaper-timeout.d.ts +2 -2
  52. package/dist/sandbox-resume.d.ts +6 -1
  53. package/dist/workflow-bundle.js +96 -10
  54. package/dist/workflows/activities.d.ts +6 -1
  55. package/dist/workflows/automations.d.ts +5 -0
  56. package/dist/workflows/session.d.ts +3 -0
  57. package/package.json +17 -17
  58. package/src/activities/agent-run-admission.ts +7 -1
  59. package/src/activities/agent-turn/admission.ts +47 -6
  60. package/src/activities/agent-turn/agent-build.ts +20 -26
  61. package/src/activities/agent-turn/claim.ts +59 -6
  62. package/src/activities/agent-turn/codex-capacity.ts +311 -234
  63. package/src/activities/agent-turn/codex.ts +2 -2
  64. package/src/activities/agent-turn/compaction-prep.ts +8 -1
  65. package/src/activities/agent-turn/credential-leases.ts +36 -0
  66. package/src/activities/agent-turn/errors.ts +29 -1
  67. package/src/activities/agent-turn/failure-settlement.ts +451 -323
  68. package/src/activities/agent-turn/finalization.ts +31 -17
  69. package/src/activities/agent-turn/list-models.ts +101 -0
  70. package/src/activities/agent-turn/model-usage.ts +120 -45
  71. package/src/activities/agent-turn/quiescence.ts +32 -17
  72. package/src/activities/agent-turn/run.ts +98 -9
  73. package/src/activities/agent-turn/sandbox-runtime.ts +78 -81
  74. package/src/activities/agent-turn/session-title.ts +55 -2
  75. package/src/activities/agent-turn/stream-attempt.ts +281 -136
  76. package/src/activities/agent-turn/tool-environment.ts +132 -39
  77. package/src/activities/agent-turn/tool-policy.ts +71 -12
  78. package/src/activities/agent-turn/turn-context.ts +21 -2
  79. package/src/activities/agent-turn.ts +5 -1
  80. package/src/activities/automations.ts +122 -67
  81. package/src/activities/capabilities.ts +129 -2
  82. package/src/activities/codemode-dispatcher.ts +235 -59
  83. package/src/activities/codex-capacity.ts +89 -10
  84. package/src/activities/codex-fleet-shadow.ts +8 -1
  85. package/src/activities/codex-image-generation.ts +19 -2
  86. package/src/activities/codex-rotation.ts +45 -142
  87. package/src/activities/context-compaction.ts +3 -0
  88. package/src/activities/goals.ts +116 -39
  89. package/src/activities/google-drive-publication.ts +4 -81
  90. package/src/activities/image-generation-operation.ts +28 -3
  91. package/src/activities/image-generation-references.ts +9 -9
  92. package/src/activities/mcp-credentials.ts +53 -0
  93. package/src/activities/rig-verification.ts +5 -3
  94. package/src/activities/sandbox-lease.ts +29 -12
  95. package/src/activities/scheduled-tasks.ts +217 -110
  96. package/src/activities/session-state.ts +17 -0
  97. package/src/activities/site-auth-maintenance.ts +12 -12
  98. package/src/activities/types.ts +22 -8
  99. package/src/activities.ts +2 -0
  100. package/src/activity-services.ts +4 -1
  101. package/src/github-rest-mcp.ts +2 -83
  102. package/src/index.ts +34 -4
  103. package/src/observability-metrics.ts +158 -14
  104. package/src/sandbox-reaper-timeout.ts +10 -4
  105. package/src/sandbox-resume.ts +58 -8
  106. package/src/sandbox-routing.ts +71 -16
  107. package/src/workflows/activities.ts +16 -0
  108. package/src/workflows/automations.ts +21 -2
  109. package/src/workflows/session.ts +85 -7
  110. package/dist/activities/api-integrations.d.ts +0 -21
  111. package/dist/activities-control-52W5IDWM.js.map +0 -1
  112. package/dist/activities-turn-A2G2KLSY.js.map +0 -1
  113. package/dist/chunk-GFNXVZVQ.js.map +0 -1
  114. package/dist/chunk-IPVMCAW5.js.map +0 -1
  115. package/src/activities/api-integrations.ts +0 -141
@@ -2,21 +2,27 @@ import { randomUUID } from "node:crypto";
2
2
  import {
3
3
  AttemptToolApprovalRequiredError,
4
4
  AttemptToolCatalogStaleError,
5
+ AttemptToolInputValidationError,
5
6
  AttemptToolNotFoundError,
6
7
  codemodeDispatchSubject,
7
8
  decodeCodemodeDispatchRequest,
8
9
  encodeCodemodeDispatchAck,
10
+ type AttemptToolCatalogEntry,
9
11
  type AttemptToolEnvironment,
10
12
  } from "@opengeni/codemode";
11
13
  import {
14
+ CODEMODE_CLAIM_LEASE_MS,
12
15
  CODEMODE_CLAIM_HEARTBEAT_MS,
13
16
  CODEMODE_MAX_CONCURRENT_CALLS_PER_ATTEMPT,
14
17
  type CodemodeOperation,
18
+ type SessionEvent,
15
19
  } from "@opengeni/contracts";
16
20
  import {
17
21
  cancelQueuedCodemodeOperationsForAttempt,
18
22
  claimCodemodeOperation,
19
23
  failCodemodeOperation,
24
+ getSessionEventByClientEventId,
25
+ getSessionToolCallCreatedEventByOperationId,
20
26
  markCodemodeOperationExecutionStarted,
21
27
  renewCodemodeOperationClaim,
22
28
  settleCodemodeOperationWithOutput,
@@ -37,6 +43,11 @@ export type CodemodeDispatcherScope = {
37
43
  executionGeneration: number;
38
44
  };
39
45
 
46
+ export type CodemodeDispatcherTimings = {
47
+ claimLeaseMs?: number;
48
+ claimHeartbeatMs?: number;
49
+ };
50
+
40
51
  /**
41
52
  * Owns the only Codemode execution edge for one exact attempt. NATS carries a
42
53
  * wake-up only; the durable row is claimed before any side effect and the call
@@ -48,6 +59,8 @@ export class CodemodeAttemptDispatcher {
48
59
  private pendingClaims = 0;
49
60
  private unsubscribe: (() => void) | null = null;
50
61
  private closing = false;
62
+ private readonly claimLeaseMs: number;
63
+ private readonly claimHeartbeatMs: number;
51
64
 
52
65
  constructor(
53
66
  private readonly db: Database,
@@ -56,6 +69,7 @@ export class CodemodeAttemptDispatcher {
56
69
  private readonly scope: CodemodeDispatcherScope,
57
70
  private readonly turnSignal?: AbortSignal,
58
71
  private readonly maxConcurrentCalls = CODEMODE_MAX_CONCURRENT_CALLS_PER_ATTEMPT,
72
+ timings: CodemodeDispatcherTimings = {},
59
73
  ) {
60
74
  if (
61
75
  environment.catalog.accountId !== scope.accountId ||
@@ -70,6 +84,22 @@ export class CodemodeAttemptDispatcher {
70
84
  if (!Number.isSafeInteger(maxConcurrentCalls) || maxConcurrentCalls < 1) {
71
85
  throw new Error("Codemode dispatcher concurrency must be a positive safe integer");
72
86
  }
87
+ this.claimLeaseMs = timings.claimLeaseMs ?? CODEMODE_CLAIM_LEASE_MS;
88
+ this.claimHeartbeatMs = timings.claimHeartbeatMs ?? CODEMODE_CLAIM_HEARTBEAT_MS;
89
+ if (
90
+ !Number.isSafeInteger(this.claimLeaseMs) ||
91
+ this.claimLeaseMs < 1_000 ||
92
+ this.claimLeaseMs > 10 * 60_000
93
+ ) {
94
+ throw new Error("Codemode dispatcher claim lease must be between 1 second and 10 minutes");
95
+ }
96
+ if (
97
+ !Number.isSafeInteger(this.claimHeartbeatMs) ||
98
+ this.claimHeartbeatMs < 1 ||
99
+ this.claimHeartbeatMs >= this.claimLeaseMs
100
+ ) {
101
+ throw new Error("Codemode dispatcher heartbeat must be positive and shorter than its lease");
102
+ }
73
103
  }
74
104
 
75
105
  start(): void {
@@ -103,11 +133,14 @@ export class CodemodeAttemptDispatcher {
103
133
  catalogDigest: request.catalogDigest,
104
134
  operationId: request.operationId,
105
135
  claimId: randomUUID(),
136
+ claimLeaseMs: this.claimLeaseMs,
106
137
  });
107
138
  if (claim.status === "claimed") {
108
- const execution = this.execute(claim.operation, claim.claimId).finally(() => {
109
- this.inFlight.delete(request.operationId);
110
- });
139
+ const execution = this.execute(claim.operation, claim.claimId, claim.reclaimed).finally(
140
+ () => {
141
+ this.inFlight.delete(request.operationId);
142
+ },
143
+ );
111
144
  this.inFlight.set(request.operationId, execution);
112
145
  return encodeCodemodeDispatchAck({
113
146
  version: 1,
@@ -115,6 +148,19 @@ export class CodemodeAttemptDispatcher {
115
148
  status: "accepted",
116
149
  });
117
150
  }
151
+ if (claim.status === "execution_owner_lost") {
152
+ await this.settleWithOutput(claim.operation, claim.claimId, {
153
+ state: "outcome_unknown",
154
+ errorCode: "worker_lost_during_execution",
155
+ errorMessage:
156
+ "The execution owner disappeared after the tool call began. Inspect actual state before retrying.",
157
+ });
158
+ return encodeCodemodeDispatchAck({
159
+ version: 1,
160
+ operationId: request.operationId,
161
+ status: "terminal",
162
+ });
163
+ }
118
164
  return encodeCodemodeDispatchAck({
119
165
  version: 1,
120
166
  operationId: request.operationId,
@@ -145,7 +191,39 @@ export class CodemodeAttemptDispatcher {
145
191
  await Promise.allSettled([...this.inFlight.values()]);
146
192
  }
147
193
 
148
- private async execute(operation: CodemodeOperation, claimId: string): Promise<void> {
194
+ private async execute(
195
+ operation: CodemodeOperation,
196
+ claimId: string,
197
+ reclaimed: boolean,
198
+ ): Promise<void> {
199
+ const claimAuthority = {
200
+ accountId: this.scope.accountId,
201
+ workspaceId: this.scope.workspaceId,
202
+ attemptId: this.scope.attemptId,
203
+ operationId: operation.operationId,
204
+ claimId,
205
+ claimLeaseMs: this.claimLeaseMs,
206
+ };
207
+ // Preparation may include provider preflight and policy resolution. Keep the
208
+ // durable claim alive from the moment execution is accepted, not only after
209
+ // the provider side-effect marker, so a slow preflight cannot be reclaimed.
210
+ const heartbeat = setInterval(() => {
211
+ void renewCodemodeOperationClaim(this.db, claimAuthority).catch(() => undefined);
212
+ }, this.claimHeartbeatMs);
213
+ heartbeat.unref?.();
214
+ try {
215
+ await this.executeClaimed(operation, claimId, reclaimed, claimAuthority);
216
+ } finally {
217
+ clearInterval(heartbeat);
218
+ }
219
+ }
220
+
221
+ private async executeClaimed(
222
+ operation: CodemodeOperation,
223
+ claimId: string,
224
+ reclaimed: boolean,
225
+ claimAuthority: Parameters<typeof markCodemodeOperationExecutionStarted>[1],
226
+ ): Promise<void> {
149
227
  const entry = this.environment.catalog.entries.find(
150
228
  (candidate) =>
151
229
  candidate.identity.serverId === operation.identity.serverId &&
@@ -170,38 +248,7 @@ export class CodemodeAttemptDispatcher {
170
248
  );
171
249
  return;
172
250
  }
173
- const created = await appendAndPublishTurnEventsFenced(
174
- this.db,
175
- this.bus,
176
- this.scope.workspaceId,
177
- this.scope.sessionId,
178
- this.scope.turnId,
179
- this.scope.executionGeneration,
180
- this.scope.attemptId,
181
- [
182
- {
183
- type: "agent.toolCall.created",
184
- turnId: this.scope.turnId,
185
- turnGeneration: this.scope.executionGeneration,
186
- turnAttemptId: this.scope.attemptId,
187
- producerId: operation.caller.subjectId,
188
- payload: {
189
- id: operation.operationId,
190
- name: entry.modelName,
191
- arguments: operation.arguments,
192
- origin: "codemode",
193
- subjectId: operation.caller.subjectId,
194
- raw: {
195
- type: "codemode_call",
196
- serverId: operation.identity.serverId,
197
- toolName: operation.identity.toolName,
198
- catalogDigest: operation.catalogDigest,
199
- },
200
- },
201
- },
202
- ],
203
- ).catch(() => ({ accepted: false }));
204
- if (!created.accepted) {
251
+ if (!(await this.ensureToolCallCreated(operation, entry, reclaimed))) {
205
252
  await this.failBeforeExecution(
206
253
  operation.operationId,
207
254
  claimId,
@@ -219,24 +266,9 @@ export class CodemodeAttemptDispatcher {
219
266
  return;
220
267
  }
221
268
 
222
- const claimAuthority = {
223
- accountId: this.scope.accountId,
224
- workspaceId: this.scope.workspaceId,
225
- attemptId: this.scope.attemptId,
226
- operationId: operation.operationId,
227
- claimId,
228
- };
229
- const crossedExecutionBoundary = await markCodemodeOperationExecutionStarted(
230
- this.db,
231
- claimAuthority,
232
- );
233
- if (!crossedExecutionBoundary) return;
234
- const heartbeat = setInterval(() => {
235
- void renewCodemodeOperationClaim(this.db, claimAuthority).catch(() => undefined);
236
- }, CODEMODE_CLAIM_HEARTBEAT_MS);
237
- heartbeat.unref?.();
269
+ let preparedCall: Awaited<ReturnType<AttemptToolEnvironment["prepareCall"]>>;
238
270
  try {
239
- const result = await this.environment.call(
271
+ preparedCall = await this.environment.prepareCall(
240
272
  {
241
273
  operationId: operation.operationId,
242
274
  catalogDigest: operation.catalogDigest,
@@ -246,7 +278,34 @@ export class CodemodeAttemptDispatcher {
246
278
  },
247
279
  { signal },
248
280
  );
249
- await this.settleWithOutput(operation, claimId, { state: "completed", result });
281
+ } catch (error) {
282
+ await this.settleWithOutput(operation, claimId, {
283
+ state: "failed",
284
+ errorCode: errorCode(error, signal, false),
285
+ errorMessage: safeErrorMessage(error),
286
+ });
287
+ return;
288
+ }
289
+ if (signal.aborted) {
290
+ await this.settleWithOutput(operation, claimId, {
291
+ state: "failed",
292
+ errorCode: "attempt_cancelled",
293
+ errorMessage: "Execution attempt ended before the Codemode call started",
294
+ });
295
+ return;
296
+ }
297
+
298
+ const crossedExecutionBoundary = await markCodemodeOperationExecutionStarted(
299
+ this.db,
300
+ claimAuthority,
301
+ );
302
+ if (!crossedExecutionBoundary) return;
303
+ try {
304
+ const result = await preparedCall.execute();
305
+ await this.settleWithOutput(operation, claimId, {
306
+ state: "completed",
307
+ result,
308
+ });
250
309
  } catch (error) {
251
310
  const knownPreExecution =
252
311
  error instanceof AttemptToolApprovalRequiredError ||
@@ -255,13 +314,90 @@ export class CodemodeAttemptDispatcher {
255
314
  connectorActionWasNotExecuted(error);
256
315
  await this.settleWithOutput(operation, claimId, {
257
316
  state: knownPreExecution ? "failed" : "outcome_unknown",
258
- errorCode: errorCode(error, signal),
317
+ errorCode: errorCode(error, signal, true),
259
318
  errorMessage: !knownPreExecution
260
319
  ? "Tool execution ended without a durable result. Its side-effect outcome is unknown; inspect actual state before retrying."
261
320
  : safeErrorMessage(error),
262
321
  });
263
- } finally {
264
- clearInterval(heartbeat);
322
+ }
323
+ }
324
+
325
+ private async ensureToolCallCreated(
326
+ operation: CodemodeOperation,
327
+ entry: AttemptToolCatalogEntry,
328
+ reclaimed: boolean,
329
+ ): Promise<boolean> {
330
+ const clientEventId = codemodeToolCallCreatedClientEventId(operation.operationId);
331
+ if (reclaimed) {
332
+ const existing = await getSessionEventByClientEventId(
333
+ this.db,
334
+ this.scope.workspaceId,
335
+ this.scope.sessionId,
336
+ clientEventId,
337
+ ).catch(() => null);
338
+ if (existing) {
339
+ assertMatchingCodemodeToolCallCreated(existing, operation, entry, this.scope);
340
+ return true;
341
+ }
342
+ const legacyCreated = await getSessionToolCallCreatedEventByOperationId(this.db, {
343
+ workspaceId: this.scope.workspaceId,
344
+ sessionId: this.scope.sessionId,
345
+ turnId: this.scope.turnId,
346
+ attemptId: this.scope.attemptId,
347
+ operationId: operation.operationId,
348
+ }).catch(() => null);
349
+ if (legacyCreated) {
350
+ assertMatchingCodemodeToolCallCreated(legacyCreated, operation, entry, this.scope);
351
+ return true;
352
+ }
353
+ }
354
+ try {
355
+ const created = await appendAndPublishTurnEventsFenced(
356
+ this.db,
357
+ this.bus,
358
+ this.scope.workspaceId,
359
+ this.scope.sessionId,
360
+ this.scope.turnId,
361
+ this.scope.executionGeneration,
362
+ this.scope.attemptId,
363
+ [
364
+ {
365
+ type: "agent.toolCall.created",
366
+ clientEventId,
367
+ turnId: this.scope.turnId,
368
+ turnGeneration: this.scope.executionGeneration,
369
+ turnAttemptId: this.scope.attemptId,
370
+ producerId: operation.caller.subjectId,
371
+ payload: {
372
+ id: operation.operationId,
373
+ name: entry.modelName,
374
+ arguments: operation.arguments,
375
+ origin: "codemode",
376
+ subjectId: operation.caller.subjectId,
377
+ raw: {
378
+ type: "codemode_call",
379
+ serverId: operation.identity.serverId,
380
+ toolName: operation.identity.toolName,
381
+ catalogDigest: operation.catalogDigest,
382
+ },
383
+ },
384
+ },
385
+ ],
386
+ );
387
+ return created.accepted;
388
+ } catch {
389
+ // A concurrent reclaimed claimant can win the unique client-event insert,
390
+ // or a committed append can lose its response. Reconcile durable truth
391
+ // before treating the attempt fence as closed.
392
+ const raced = await getSessionEventByClientEventId(
393
+ this.db,
394
+ this.scope.workspaceId,
395
+ this.scope.sessionId,
396
+ clientEventId,
397
+ ).catch(() => null);
398
+ if (!raced) return false;
399
+ assertMatchingCodemodeToolCallCreated(raced, operation, entry, this.scope);
400
+ return true;
265
401
  }
266
402
  }
267
403
 
@@ -311,6 +447,43 @@ export class CodemodeAttemptDispatcher {
311
447
  }
312
448
  }
313
449
 
450
+ export function codemodeToolCallCreatedClientEventId(operationId: string): string {
451
+ return `opengeni:codemode-tool-call-created:${operationId}`;
452
+ }
453
+
454
+ function assertMatchingCodemodeToolCallCreated(
455
+ event: SessionEvent,
456
+ operation: CodemodeOperation,
457
+ entry: AttemptToolCatalogEntry,
458
+ scope: CodemodeDispatcherScope,
459
+ ): void {
460
+ const payload = recordValue(event.payload);
461
+ const raw = recordValue(payload?.raw);
462
+ if (
463
+ event.type !== "agent.toolCall.created" ||
464
+ event.turnId !== scope.turnId ||
465
+ event.turnGeneration !== scope.executionGeneration ||
466
+ event.turnAttemptId !== scope.attemptId ||
467
+ event.turnAssociation !== "current" ||
468
+ payload?.id !== operation.operationId ||
469
+ payload?.name !== entry.modelName ||
470
+ payload?.origin !== "codemode" ||
471
+ payload?.subjectId !== operation.caller.subjectId ||
472
+ raw?.type !== "codemode_call" ||
473
+ raw?.serverId !== operation.identity.serverId ||
474
+ raw?.toolName !== operation.identity.toolName ||
475
+ raw?.catalogDigest !== operation.catalogDigest
476
+ ) {
477
+ throw new Error("Codemode tool-call creation id is bound to incompatible durable evidence");
478
+ }
479
+ }
480
+
481
+ function recordValue(value: unknown): Record<string, unknown> | null {
482
+ return value !== null && typeof value === "object" && !Array.isArray(value)
483
+ ? (value as Record<string, unknown>)
484
+ : null;
485
+ }
486
+
314
487
  function connectorActionWasNotExecuted(error: unknown): boolean {
315
488
  return Boolean(
316
489
  error &&
@@ -324,8 +497,10 @@ function combinedSignal(primary: AbortSignal, secondary?: AbortSignal): AbortSig
324
497
  return secondary ? AbortSignal.any([primary, secondary]) : primary;
325
498
  }
326
499
 
327
- function errorCode(error: unknown, signal: AbortSignal): string {
328
- if (signal.aborted) return "attempt_cancelled_during_execution";
500
+ function errorCode(error: unknown, signal: AbortSignal, executionStarted: boolean): string {
501
+ if (signal.aborted) {
502
+ return executionStarted ? "attempt_cancelled_during_execution" : "attempt_cancelled";
503
+ }
329
504
  if (connectorActionWasNotExecuted(error)) return "connector_action_not_executed";
330
505
  if (typeof error === "object" && error !== null && "code" in error) {
331
506
  const code = (error as { code?: unknown }).code;
@@ -338,6 +513,7 @@ function safeErrorMessage(error: unknown): string {
338
513
  if (
339
514
  error instanceof AttemptToolApprovalRequiredError ||
340
515
  error instanceof AttemptToolCatalogStaleError ||
516
+ error instanceof AttemptToolInputValidationError ||
341
517
  error instanceof AttemptToolNotFoundError
342
518
  ) {
343
519
  return error.message;
@@ -1,4 +1,5 @@
1
1
  import {
2
+ armCodexCapacityWait,
2
3
  fetchCodexUsageForAccount,
3
4
  getCodexCapacityWaitForSession,
4
5
  getXaiCapacityWaitForSession,
@@ -9,7 +10,7 @@ import {
9
10
  type CodexCapacityWakeTarget,
10
11
  type CodexCapacitySelectionContext,
11
12
  } from "@opengeni/db";
12
- import type { Settings } from "@opengeni/config";
13
+ import { publishDurableSessionEvents } from "@opengeni/events";
13
14
  import {
14
15
  authoritativeCodexCapacityResetAt,
15
16
  codexAccountNeedsLiveCapacityRefresh,
@@ -85,14 +86,12 @@ export async function signalPendingCodexCapacityWakeTargets(
85
86
  await signalCodexCapacityWakeTargets(services, targets);
86
87
  }
87
88
 
88
- export function codexCapacityDecision(
89
- context: CodexCapacitySelectionContext,
90
- settings: Settings,
89
+ export function codexCapacityDecision<TPolicyScope = never, TUnavailableDiagnostic = never>(
90
+ context: CodexCapacitySelectionContext<TPolicyScope, TUnavailableDiagnostic>,
91
91
  ): ReturnType<Parameters<typeof reconcileCodexCapacityWaitDb>[2]> {
92
92
  const now = new Date();
93
93
  const selected = selectCodexCredentialLeaseForTurn({
94
94
  context,
95
- leasingEnabled: settings.codexCredentialLeasingEnabled,
96
95
  sessionId: context.sessionId,
97
96
  sessionPinnedCredentialId: context.sessionPinnedCredentialId,
98
97
  sessionPinSource: context.sessionPinSource,
@@ -118,8 +117,17 @@ export function codexCapacityDecision(
118
117
  },
119
118
  };
120
119
  }
121
- const authoritativeReset = authoritativeCodexCapacityResetAt(context.accounts, now);
122
- const hasReconcilableQuotaCooldown = context.accounts.some(
120
+ const policyCredentialId =
121
+ context.sessionPinSource === "manual" && context.sessionPinnedCredentialId
122
+ ? context.sessionPinnedCredentialId
123
+ : !context.rotationEnabled
124
+ ? context.activeCredentialId
125
+ : null;
126
+ const capacityAccounts = policyCredentialId
127
+ ? context.accounts.filter((account) => account.id === policyCredentialId)
128
+ : context.accounts;
129
+ const authoritativeReset = authoritativeCodexCapacityResetAt(capacityAccounts, now);
130
+ const hasReconcilableQuotaCooldown = capacityAccounts.some(
123
131
  (account) =>
124
132
  account.status === "active" &&
125
133
  account.allocatorEnabled &&
@@ -127,19 +135,85 @@ export function codexCapacityDecision(
127
135
  account.exhaustedUntil !== null &&
128
136
  account.exhaustedUntil > now,
129
137
  );
138
+ const policyAccount = capacityAccounts[0] ?? null;
139
+ const mutationOnlyStatusBlock =
140
+ (policyAccount != null &&
141
+ (!policyAccount.allocatorEnabled || policyAccount.status !== "active")) ||
142
+ (authoritativeReset === null &&
143
+ capacityAccounts.length > 0 &&
144
+ capacityAccounts.every(
145
+ (account) => !account.allocatorEnabled || account.status !== "active",
146
+ ));
147
+ const noneReason =
148
+ selected.decision.kind === "none"
149
+ ? context.sessionPinSource === "manual" && context.sessionPinnedCredentialId !== null
150
+ ? "manual_pin_missing"
151
+ : !context.rotationEnabled && context.activeCredentialId === null
152
+ ? "rotation_off_active_pointer_missing"
153
+ : context.policyScope !== null && context.accounts.length === 0
154
+ ? "policy_filtered_pool_empty"
155
+ : context.accounts.length === 0
156
+ ? "no_connected_credentials"
157
+ : "no_eligible_credential"
158
+ : null;
130
159
  return {
131
160
  kind: "unavailable",
132
161
  earliestResetAt: authoritativeReset,
133
162
  resetKind:
134
- authoritativeReset && !hasReconcilableQuotaCooldown ? "authoritative" : "bounded_refresh",
163
+ selected.decision.kind === "none" ||
164
+ selected.decision.kind === "allocatorDisabled" ||
165
+ mutationOnlyStatusBlock
166
+ ? "mutation_only"
167
+ : authoritativeReset && !hasReconcilableQuotaCooldown
168
+ ? "authoritative"
169
+ : "bounded_refresh",
135
170
  diagnostic: {
136
171
  connectedCount: context.accounts.length,
137
172
  allocatorEnabledCount: context.accounts.filter((account) => account.allocatorEnabled).length,
138
173
  policyHash: context.policyHash,
174
+ ...(noneReason ? { reason: noneReason } : {}),
139
175
  },
140
176
  };
141
177
  }
142
178
 
179
+ /**
180
+ * Arm a durable Codex waiter and immediately re-evaluate it under the
181
+ * allocator lock. A capacity mutation that commits just before the waiter is
182
+ * inserted cannot signal a row that does not exist yet, so every arm site must
183
+ * close that edge before returning an hours-away reset timer to the workflow.
184
+ * Mutations after the arm commit still advance the waiter's wake revision.
185
+ */
186
+ export async function armAndReconcileCodexCapacityWait(
187
+ services: Pick<ControlActivityServices, "db" | "bus">,
188
+ input: Parameters<typeof armCodexCapacityWait>[1],
189
+ options: { onArmed?: () => void } = {},
190
+ ) {
191
+ const armed = await armCodexCapacityWait(services.db, input);
192
+ if (armed.action !== "waiting") return armed;
193
+
194
+ options.onArmed?.();
195
+ await publishDurableSessionEvents(services.bus, input.workspaceId, input.sessionId, armed.events);
196
+ const evaluated = await reconcileCodexCapacityWaitDb(
197
+ services.db,
198
+ {
199
+ accountId: input.accountId,
200
+ workspaceId: input.workspaceId,
201
+ sessionId: input.sessionId,
202
+ waiterId: armed.waiter.id,
203
+ generation: armed.waiter.generation,
204
+ ...(input.now ? { now: input.now } : {}),
205
+ },
206
+ (context) => codexCapacityDecision(context),
207
+ );
208
+ await publishDurableSessionEvents(
209
+ services.bus,
210
+ input.workspaceId,
211
+ input.sessionId,
212
+ evaluated.events,
213
+ );
214
+ return evaluated;
215
+ }
216
+
143
217
  async function refreshCapacityMetadata(
144
218
  services: ControlActivityServices,
145
219
  workspaceId: string,
@@ -239,7 +313,11 @@ export function createCodexCapacityActivities(services: () => Promise<ControlAct
239
313
  if (!current || current.id !== input.waiterId || current.generation !== input.generation) {
240
314
  return { action: "stale" };
241
315
  }
242
- if (input.cause === "timer" && current.nextCheckAt.getTime() <= Date.now()) {
316
+ const boundedRefreshAttempted =
317
+ current.resetKind === "bounded_refresh" &&
318
+ input.cause === "timer" &&
319
+ current.nextCheckAt.getTime() <= Date.now();
320
+ if (boundedRefreshAttempted) {
243
321
  // This is a bounded secret-safe control-plane quota refresh. It creates no
244
322
  // turn, model call, user message, schedule, or entitlement action.
245
323
  await refreshCapacityMetadata(resolved, input.workspaceId);
@@ -252,8 +330,9 @@ export function createCodexCapacityActivities(services: () => Promise<ControlAct
252
330
  sessionId: input.sessionId,
253
331
  waiterId: input.waiterId,
254
332
  generation: input.generation,
333
+ boundedRefreshAttempted,
255
334
  },
256
- (context) => codexCapacityDecision(context, resolved.settings),
335
+ (context) => codexCapacityDecision(context),
257
336
  );
258
337
  if (result.events.length > 0) {
259
338
  try {
@@ -16,7 +16,14 @@ export const CODEX_FLEET_SHADOW_MAX_PAYLOAD_BYTES = 34 * 1_024;
16
16
  export type CodexFleetActualDecisionV1 = {
17
17
  outcome: "selected" | "waiting" | "none";
18
18
  candidateKey: string | null;
19
- reason: "lease_reused" | "pin" | "rotation" | "active" | "all_capped" | "none";
19
+ reason:
20
+ | "lease_reused"
21
+ | "pin"
22
+ | "rotation"
23
+ | "active"
24
+ | "all_capped"
25
+ | "allocator_disabled"
26
+ | "none";
20
27
  };
21
28
 
22
29
  export type CodexFleetShadowComparisonV1 =
@@ -6,6 +6,7 @@ import {
6
6
  import type { Database } from "@opengeni/db";
7
7
  import type { ObjectStorage } from "@opengeni/storage";
8
8
  import type { GeneratedImageReceipt } from "./generated-images";
9
+ import { CodexCredentialLeaseLostError } from "./agent-turn/credential-leases";
9
10
  import {
10
11
  executeImageGenerationOperation,
11
12
  imageProviderBindingHash,
@@ -27,22 +28,38 @@ export async function executeCodexImageGeneration(input: {
27
28
  prompt: string;
28
29
  references?: readonly ResolvedImageGenerationReference[];
29
30
  credentialId: string;
30
- codexContext: Pick<CodexRequestContext, "clientVersion" | "getToken" | "refresh">;
31
+ codexContext: Pick<
32
+ CodexRequestContext,
33
+ "clientVersion" | "getToken" | "refresh" | "beforeProviderDispatch"
34
+ >;
31
35
  abortSignal?: AbortSignal;
32
36
  }): Promise<GeneratedImageReceipt> {
33
37
  const providerBindingHash = imageProviderBindingHash(CODEX_PROVIDER_ID, input.credentialId);
38
+ let providerDispatchAdmitted = false;
39
+ const codexContext: Pick<
40
+ CodexRequestContext,
41
+ "clientVersion" | "getToken" | "refresh" | "beforeProviderDispatch"
42
+ > = {
43
+ ...input.codexContext,
44
+ beforeProviderDispatch: async () => {
45
+ await input.codexContext.beforeProviderDispatch?.();
46
+ providerDispatchAdmitted = true;
47
+ },
48
+ };
34
49
  return await executeImageGenerationOperation({
35
50
  ...input,
36
51
  providerId: CODEX_PROVIDER_ID,
37
52
  providerBindingHash,
38
53
  modelId: CODEX_IMAGE_MODEL,
39
54
  ...(input.references ? { referenceDigests: input.references } : {}),
55
+ isProviderDispatchRejected: (error) =>
56
+ !providerDispatchAdmitted && error instanceof CodexCredentialLeaseLostError,
40
57
  generate: async () => {
41
58
  const generated = await generateCodexSubscriptionImage({
42
59
  prompt: input.prompt,
43
60
  ...(input.references ? { references: input.references } : {}),
44
61
  turnId: input.turnId,
45
- context: input.codexContext,
62
+ context: codexContext,
46
63
  ...(input.abortSignal ? { abortSignal: input.abortSignal } : {}),
47
64
  });
48
65
  return {