@opengeni/worker-bundle 0.24.2-canary.0 → 0.26.0-canary.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/dist/activities/agent-turn/agent-build.d.ts +3 -10
  2. package/dist/activities/agent-turn/claim.d.ts +7 -0
  3. package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
  4. package/dist/activities/agent-turn/errors.d.ts +6 -5
  5. package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
  6. package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
  7. package/dist/activities/agent-turn/history-sink.d.ts +6 -2
  8. package/dist/activities/agent-turn/quiescence.d.ts +13 -8
  9. package/dist/activities/agent-turn/tool-environment.d.ts +2 -13
  10. package/dist/activities/agent-turn/tool-policy.d.ts +4 -0
  11. package/dist/activities/agent-turn/turn-context.d.ts +9 -1
  12. package/dist/activities/codemode-dispatcher.d.ts +10 -1
  13. package/dist/activities/codex-capacity.d.ts +32 -3
  14. package/dist/activities/codex-fleet-shadow.d.ts +1 -1
  15. package/dist/activities/codex-image-generation.d.ts +1 -1
  16. package/dist/activities/codex-rotation.d.ts +8 -67
  17. package/dist/activities/goals.d.ts +1 -1
  18. package/dist/activities/google-drive-publication.d.ts +2 -13
  19. package/dist/activities/image-generation-operation.d.ts +13 -4
  20. package/dist/activities/sandbox-lease.d.ts +7 -2
  21. package/dist/activities/session-state.d.ts +4 -2
  22. package/dist/activities/types.d.ts +11 -1
  23. package/dist/activities/xai-quota.d.ts +3 -0
  24. package/dist/{activities-control-ZWH7J4KZ.js → activities-control-KS3ZPQVY.js} +233 -88
  25. package/dist/activities-control-KS3ZPQVY.js.map +1 -0
  26. package/dist/activities-control.d.ts +2 -0
  27. package/dist/{activities-turn-I4OPLQLS.js → activities-turn-GHSFZB64.js} +1109 -1865
  28. package/dist/activities-turn-GHSFZB64.js.map +1 -0
  29. package/dist/activities.d.ts +2 -0
  30. package/dist/{chunk-65JEWP2Q.js → chunk-D4ZEFOYH.js} +1 -1
  31. package/dist/chunk-D4ZEFOYH.js.map +1 -0
  32. package/dist/{chunk-RQNH6CGR.js → chunk-O2LQWJIK.js} +3250 -2061
  33. package/dist/chunk-O2LQWJIK.js.map +1 -0
  34. package/dist/github-rest-mcp.d.ts +0 -2
  35. package/dist/index.d.ts +1 -0
  36. package/dist/index.js +2 -2
  37. package/dist/index.js.map +1 -1
  38. package/dist/observability-metrics.d.ts +1 -1
  39. package/dist/sandbox-routing.d.ts +6 -0
  40. package/dist/workflow-bundle.js +94 -9
  41. package/dist/workflows/activities.d.ts +1 -1
  42. package/dist/workflows/session.d.ts +3 -0
  43. package/package.json +17 -17
  44. package/src/activities/agent-turn/agent-build.ts +21 -26
  45. package/src/activities/agent-turn/claim.ts +26 -1
  46. package/src/activities/agent-turn/codex-capacity.ts +311 -234
  47. package/src/activities/agent-turn/credential-leases.ts +36 -0
  48. package/src/activities/agent-turn/errors.ts +73 -12
  49. package/src/activities/agent-turn/failure-settlement.ts +472 -331
  50. package/src/activities/agent-turn/history-prefix.ts +51 -0
  51. package/src/activities/agent-turn/history-sink.ts +30 -6
  52. package/src/activities/agent-turn/quiescence.ts +29 -17
  53. package/src/activities/agent-turn/run.ts +22 -5
  54. package/src/activities/agent-turn/stream-attempt.ts +35 -8
  55. package/src/activities/agent-turn/tool-environment.ts +42 -32
  56. package/src/activities/agent-turn/tool-policy.ts +14 -8
  57. package/src/activities/agent-turn/turn-context.ts +13 -0
  58. package/src/activities/agent-turn/xai-capacity.ts +11 -0
  59. package/src/activities/codemode-dispatcher.ts +235 -59
  60. package/src/activities/codex-capacity.ts +107 -10
  61. package/src/activities/codex-fleet-shadow.ts +8 -1
  62. package/src/activities/codex-image-generation.ts +19 -2
  63. package/src/activities/codex-rotation.ts +45 -142
  64. package/src/activities/goals.ts +9 -9
  65. package/src/activities/google-drive-publication.ts +4 -81
  66. package/src/activities/image-generation-operation.ts +28 -3
  67. package/src/activities/run-input.ts +6 -7
  68. package/src/activities/sandbox-lease.ts +246 -87
  69. package/src/activities/session-state.ts +20 -1
  70. package/src/activities/types.ts +16 -8
  71. package/src/activities/xai-quota.ts +35 -0
  72. package/src/activities.ts +1 -0
  73. package/src/github-rest-mcp.ts +0 -82
  74. package/src/observability-metrics.ts +4 -0
  75. package/src/sandbox-routing.ts +186 -16
  76. package/src/workflows/activities.ts +1 -0
  77. package/src/workflows/session.ts +114 -8
  78. package/dist/activities/api-integrations.d.ts +0 -21
  79. package/dist/activities-control-ZWH7J4KZ.js.map +0 -1
  80. package/dist/activities-turn-I4OPLQLS.js.map +0 -1
  81. package/dist/chunk-65JEWP2Q.js.map +0 -1
  82. package/dist/chunk-RQNH6CGR.js.map +0 -1
  83. package/src/activities/api-integrations.ts +0 -151
@@ -3,13 +3,14 @@ import {
3
3
  PostCompactionContinuationEmptyError,
4
4
  SandboxDeadlineRotationError,
5
5
  agentRunFailurePayload,
6
+ armAndReconcileCodexCapacityWait,
6
7
  armTurnQuiescenceWatchdog,
7
8
  assertAgentStreamNotCancelled,
8
9
  assertPhysicalToolQuiescenceForCancellation,
9
10
  assertSessionAttemptQuiescenceRecoveryDurable,
10
11
  assertSuccessfulAgentStreamCompletion,
11
12
  authoritativeCodexCapacityResetAt,
12
- chooseRotationActive,
13
+ buildXaiTurnRequestAuthorization,
13
14
  classifyCodexCredentialFailure,
14
15
  classifyCodexPin,
15
16
  classifyContextWindowOverflowError,
@@ -22,7 +23,6 @@ import {
22
23
  compactionFailureReason,
23
24
  compactionFailureReasonFromError,
24
25
  computeIdleDelayMs,
25
- computeReactiveRotationResume,
26
26
  createRuntimeBatcher,
27
27
  createSharedActivityServices,
28
28
  createTurnSandboxProvisioner,
@@ -30,13 +30,14 @@ import {
30
30
  deliverChildRequiresActionToParent,
31
31
  deliverFailedChildTurnToParent,
32
32
  drainAttemptOwnedSandboxWriters,
33
- earliestCodexReset,
34
33
  escapedMcpTimeoutRecoveryFailure,
34
+ establishSelfhostedTurnSession,
35
35
  finalizeDurableTurnOpStreams,
36
36
  isCodexCredentialEligible,
37
37
  isCompactionSummaryFailure,
38
38
  isLazySandboxProvisionRetryable,
39
39
  isWorkerShutdownCancellation,
40
+ lazyProvisionEnabled,
40
41
  maybePersistWarmWorkspaceSnapshot,
41
42
  mergeResourceRefs,
42
43
  nextStreamEvent,
@@ -49,12 +50,14 @@ import {
49
50
  providerRecoveryResult,
50
51
  providerRetryAfterMs,
51
52
  refreshCodexUsageAndRepairCapacityWaiters,
53
+ refreshExhaustedXaiQuota,
52
54
  releaseTurnSandboxAfterWriterDrain,
53
55
  requireAgentStreamFinalOutput,
54
56
  resolveWorkspaceInstalledSkillRuntime,
55
57
  resolveWorkspacePackRuntime,
56
58
  resumeBoxForTurn,
57
59
  rigProviderImageSourceImage,
60
+ routingEnabled,
58
61
  runMandatoryHistoryPersistenceStep,
59
62
  safeErrorDiagnostic,
60
63
  safeErrorForTelemetry,
@@ -64,7 +67,6 @@ import {
64
67
  sandboxRouteTransitionCode,
65
68
  selectCodexCredentialLeaseForTurn,
66
69
  settingsWithPackSandboxImage,
67
- shardCredentialForSession,
68
70
  shouldRecoverCompactionProviderFailure,
69
71
  shouldRunTurnEndWorkspacePersistence,
70
72
  shouldStartPeriodicWorkspaceSnapshot,
@@ -79,8 +81,10 @@ import {
79
81
  waitForTurnFinalizerStep,
80
82
  waitForTurnOperation,
81
83
  waitForWarmSnapshot,
82
- withFirstPartyTools
83
- } from "./chunk-RQNH6CGR.js";
84
+ withFirstPartyTools,
85
+ wrapLazyTurnBoxWithRouting,
86
+ wrapTurnBoxWithRouting
87
+ } from "./chunk-O2LQWJIK.js";
84
88
  import {
85
89
  StreamTimingMetrics,
86
90
  makeMachineOpObserver,
@@ -106,7 +110,7 @@ import {
106
110
  recordTurnWorkerPreparationTotal,
107
111
  runtimeMetricsHooksForObservability,
108
112
  turnLifecycleMetricsFor
109
- } from "./chunk-65JEWP2Q.js";
113
+ } from "./chunk-D4ZEFOYH.js";
110
114
 
111
115
  // src/activities-turn.ts
112
116
  import { createProductionAgentRuntime, summarizeForCompaction as summarizeForCompaction2 } from "@opengeni/runtime";
@@ -140,100 +144,6 @@ import {
140
144
  xaiSubscriptionRequestStorage
141
145
  } from "@opengeni/xai-subscription";
142
146
 
143
- // src/activities/xai-auth.ts
144
- import { environmentsEncryptionKeyBytes } from "@opengeni/config";
145
- import {
146
- materializeXaiCredentialForRun,
147
- refreshXaiSubscriptionCredentialSerialized
148
- } from "@opengeni/db";
149
- import {
150
- refreshXaiToken,
151
- XAI_CLIENT_VERSION,
152
- XAI_SUBSCRIPTION_MODEL_ID_PREFIX,
153
- XaiSubscriptionReloginRequired,
154
- xaiAccessTokenExpiry
155
- } from "@opengeni/xai-subscription";
156
- async function buildXaiTurnRequestAuthorization(input) {
157
- const encryptionKey = environmentsEncryptionKeyBytes(input.settings);
158
- if (!encryptionKey) {
159
- throw new Error("SuperGrok subscriptions require OPENGENI_ENVIRONMENTS_ENCRYPTION_KEY");
160
- }
161
- let credential = await materializeXaiCredentialForRun(input.db, {
162
- workspaceId: input.workspaceId,
163
- subjectId: input.subjectId,
164
- credentialId: input.credentialId,
165
- authoritySnapshot: input.authoritySnapshot,
166
- encryptionKey
167
- });
168
- const tokenSnapshot = () => {
169
- const accessToken = credential.secret.accessToken;
170
- const userId = credential.providerAccountId;
171
- if (!accessToken || !userId) {
172
- throw new XaiSubscriptionReloginRequired(
173
- "The SuperGrok connection is missing a verified token identity. Reconnect the account."
174
- );
175
- }
176
- return { accessToken, userId };
177
- };
178
- const refresh = async () => {
179
- const observedAccessToken = credential.secret.accessToken;
180
- const observedRefreshToken = credential.secret.refreshToken;
181
- if (!observedRefreshToken) {
182
- throw new XaiSubscriptionReloginRequired(
183
- "The SuperGrok connection cannot be refreshed. Reconnect the account."
184
- );
185
- }
186
- const result = await refreshXaiSubscriptionCredentialSerialized(input.db, {
187
- accountId: input.accountId,
188
- workspaceId: input.workspaceId,
189
- subjectId: input.subjectId,
190
- credentialId: input.credentialId,
191
- authoritySnapshot: input.authoritySnapshot,
192
- encryptionKey,
193
- observedAccessToken,
194
- observedRefreshToken,
195
- refresh: async (current) => {
196
- const refreshToken = current.secret.refreshToken;
197
- if (!refreshToken) {
198
- throw new XaiSubscriptionReloginRequired(
199
- "The SuperGrok connection cannot be refreshed. Reconnect the account."
200
- );
201
- }
202
- const tokens = await refreshXaiToken(refreshToken, {
203
- ...input.fetch ? { fetch: input.fetch } : {}
204
- });
205
- return {
206
- secret: {
207
- version: 1,
208
- accessToken: tokens.accessToken,
209
- refreshToken: tokens.refreshToken
210
- },
211
- expiresAt: xaiAccessTokenExpiry(tokens.accessToken) ?? new Date(Date.now() + tokens.expiresInSeconds * 1e3)
212
- };
213
- }
214
- });
215
- credential = result.credential;
216
- return tokenSnapshot();
217
- };
218
- return {
219
- credentialId: input.credentialId,
220
- context: {
221
- clientVersion: XAI_CLIENT_VERSION,
222
- sessionId: input.sessionId,
223
- turnId: input.turnId,
224
- getToken: async () => tokenSnapshot(),
225
- refresh,
226
- resolveModel: (model) => model.startsWith(XAI_SUBSCRIPTION_MODEL_ID_PREFIX) ? model.slice(XAI_SUBSCRIPTION_MODEL_ID_PREFIX.length) : model,
227
- ...input.hostedSearch ? { hostedSearch: input.hostedSearch } : {},
228
- ...input.onFinalContextUsage ? { onFinalContextUsage: input.onFinalContextUsage } : {},
229
- ...input.nextRequestId ? { nextRequestId: input.nextRequestId } : {},
230
- ...input.streamIdleTimeoutMs ? { streamIdleTimeoutMs: input.streamIdleTimeoutMs } : {},
231
- ...input.onModelRequestDiagnostic ? { onModelRequestDiagnostic: input.onModelRequestDiagnostic } : {},
232
- ...input.onModelRequestEvent ? { onModelRequestEvent: input.onModelRequestEvent } : {}
233
- }
234
- };
235
- }
236
-
237
147
  // src/activities/turn-attempt-fenced.ts
238
148
  var TurnAttemptFencedError = class extends Error {
239
149
  constructor(message) {
@@ -1259,12 +1169,21 @@ function codexCredentialLeaseDeadlineExpired(confirmedUntilMs, nowMs = performan
1259
1169
  }
1260
1170
 
1261
1171
  // src/activities/agent-turn/credential-leases.ts
1172
+ var CodexCredentialLeaseLostError = class extends Error {
1173
+ constructor(reason) {
1174
+ super("Codex credential lease is not usable for provider dispatch");
1175
+ this.reason = reason;
1176
+ this.name = "CodexCredentialLeaseLostError";
1177
+ }
1178
+ code = "codex_credential_lease_lost";
1179
+ };
1262
1180
  var CodexTurnLease = class {
1263
1181
  constructor(deps) {
1264
1182
  this.deps = deps;
1265
1183
  }
1266
1184
  held = false;
1267
1185
  lost = false;
1186
+ lossReason = null;
1268
1187
  holderId = null;
1269
1188
  generation = null;
1270
1189
  // Monotonic worker deadline, not a comparison between the Postgres and
@@ -1277,6 +1196,7 @@ var CodexTurnLease = class {
1277
1196
  markLost = (reason) => {
1278
1197
  if (this.lost) return;
1279
1198
  this.lost = true;
1199
+ this.lossReason = reason;
1280
1200
  this.deps.observability.incrementCounter({
1281
1201
  name: "opengeni_codex_lease_renewals_total",
1282
1202
  help: "Codex lease renewal checkpoints by outcome and reason.",
@@ -1288,6 +1208,16 @@ var CodexTurnLease = class {
1288
1208
  reason
1289
1209
  });
1290
1210
  };
1211
+ /** Fail closed at the last worker-confirmed lease deadline before dispatch. */
1212
+ assertUsable = () => {
1213
+ if (this.lost) {
1214
+ throw new CodexCredentialLeaseLostError(this.lossReason ?? "not_found");
1215
+ }
1216
+ if (!this.held || this.holderId === null || this.generation === null || codexCredentialLeaseDeadlineExpired(this.confirmedUntilMs)) {
1217
+ this.markLost("deadline");
1218
+ throw new CodexCredentialLeaseLostError("deadline");
1219
+ }
1220
+ };
1291
1221
  renew = async (reason) => {
1292
1222
  const turnId = this.deps.getTurnId();
1293
1223
  if (!turnId || !this.held || !this.holderId || this.generation === null || this.lost) {
@@ -1299,6 +1229,7 @@ var CodexTurnLease = class {
1299
1229
  }
1300
1230
  if (this.heartbeatInFlight) return;
1301
1231
  this.heartbeatInFlight = true;
1232
+ const priorConfirmedUntilMs = this.confirmedUntilMs;
1302
1233
  const renewalStartedAtMs = performance.now();
1303
1234
  try {
1304
1235
  const renewedUntil = await heartbeatCodexCredentialLeaseUntil(
@@ -1312,6 +1243,8 @@ var CodexTurnLease = class {
1312
1243
  );
1313
1244
  if (!renewedUntil) {
1314
1245
  this.markLost("not_found");
1246
+ } else if (codexCredentialLeaseDeadlineExpired(priorConfirmedUntilMs) || codexCredentialLeaseDeadlineExpired(renewalStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS)) {
1247
+ this.markLost("deadline");
1315
1248
  } else {
1316
1249
  this.confirmedUntilMs = renewalStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS;
1317
1250
  this.deps.observability.incrementCounter({
@@ -3272,11 +3205,63 @@ function completedToolCallFromSdkEvent(event) {
3272
3205
  };
3273
3206
  }
3274
3207
 
3208
+ // src/activities/agent-turn/history-prefix.ts
3209
+ import { createHash as createHash5 } from "crypto";
3210
+ import { normalizeProtocolJsonValue as normalizeProtocolJsonValue2, sanitizeHistoryItemsForModel as sanitizeHistoryItemsForModel2 } from "@opengeni/runtime";
3211
+ var HistoryPrefixGuard = class {
3212
+ persisted = [];
3213
+ seed(items, count, truncationTokens) {
3214
+ const keys = this.keys(items, truncationTokens);
3215
+ if (keys.length < count)
3216
+ throw new Error("Conversation history seed is shorter than its durable prefix");
3217
+ this.persisted = keys.slice(0, count);
3218
+ }
3219
+ verify(items, truncationTokens) {
3220
+ const keys = this.keys(items, truncationTokens);
3221
+ for (let i = 0; i < this.persisted.length; i++) {
3222
+ if (keys[i] !== this.persisted[i]) {
3223
+ throw new Error(
3224
+ `Conversation history durable prefix changed at item ${i}; refusing to skip unsaved work`
3225
+ );
3226
+ }
3227
+ }
3228
+ return keys;
3229
+ }
3230
+ acknowledge(keys, count) {
3231
+ this.persisted = keys.slice(0, count);
3232
+ }
3233
+ keys(items, truncationTokens) {
3234
+ return sanitizeHistoryItemsForModel2(items, truncationTokens).map(
3235
+ (item) => createHash5("sha256").update(JSON.stringify(sorted(normalizeProtocolJsonValue2(item)))).digest("hex")
3236
+ );
3237
+ }
3238
+ };
3239
+ function sorted(value) {
3240
+ if (Array.isArray(value)) return value.map(sorted);
3241
+ if (value !== null && typeof value === "object") {
3242
+ return Object.fromEntries(
3243
+ Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, entry]) => [key, sorted(entry)])
3244
+ );
3245
+ }
3246
+ return value;
3247
+ }
3248
+
3275
3249
  // src/activities/agent-turn/history-sink.ts
3276
3250
  var TurnHistorySink = class {
3277
3251
  constructor(deps) {
3278
3252
  this.deps = deps;
3279
3253
  }
3254
+ prefix = new HistoryPrefixGuard();
3255
+ seedHistory(input, count) {
3256
+ const items = typeof input === "string" && count === 0 ? [] : Array.isArray(input) ? input : input?.history;
3257
+ if (!Array.isArray(items)) throw new Error("Conversation history seed is unavailable");
3258
+ this.prefix.seed(
3259
+ items,
3260
+ count,
3261
+ this.deps.getModelRunSettings().modelToolOutputTruncationTokens
3262
+ );
3263
+ this.persistedHistoryCount = count;
3264
+ }
3280
3265
  persistedHistoryCount = 0;
3281
3266
  nextHistoryPosition = 0;
3282
3267
  providerArtifactCandidates = {
@@ -3295,6 +3280,10 @@ var TurnHistorySink = class {
3295
3280
  const rawHistory = stream.state.history;
3296
3281
  if (Array.isArray(rawHistory)) {
3297
3282
  const typedHistory = rawHistory;
3283
+ const verifiedKeys = this.prefix.verify(
3284
+ typedHistory,
3285
+ this.deps.getModelRunSettings().modelToolOutputTruncationTokens
3286
+ );
3298
3287
  await media.retainNativeGeneratedImagesFromHistory(typedHistory);
3299
3288
  const durableHistory = compactGeneratedImageHistory(
3300
3289
  compactRetainedScreenshotHistory(typedHistory, media.retainedScreenshotReceiptsByCallId),
@@ -3330,9 +3319,14 @@ var TurnHistorySink = class {
3330
3319
  });
3331
3320
  }
3332
3321
  if (shouldAppendRows || !options.skipInputOnlyRows) {
3322
+ this.prefix.acknowledge(verifiedKeys, nextWatermark);
3333
3323
  this.persistedHistoryCount = nextWatermark;
3334
3324
  this.nextHistoryPosition = nextPosition;
3335
3325
  }
3326
+ } else if (options.requireDurable) {
3327
+ throw new Error(
3328
+ "SDK conversation history is unavailable at a mandatory persistence boundary"
3329
+ );
3336
3330
  }
3337
3331
  const envelope = sandboxStateEntryFromRunState(stream.state);
3338
3332
  if (envelope) {
@@ -3347,10 +3341,7 @@ var TurnHistorySink = class {
3347
3341
  );
3348
3342
  }
3349
3343
  } catch (persistError) {
3350
- console.error(
3351
- "session history dual-write failed (run unaffected)",
3352
- safeErrorDiagnostic(persistError)
3353
- );
3344
+ console.error("session history persistence failed", safeErrorDiagnostic(persistError));
3354
3345
  if (options.requireDurable) throw persistError;
3355
3346
  }
3356
3347
  };
@@ -3458,6 +3449,7 @@ function createTurnContext(input) {
3458
3449
  },
3459
3450
  attempt: {
3460
3451
  turnId: void 0,
3452
+ dispatchId: "",
3461
3453
  triggerEventId: void 0,
3462
3454
  executionGeneration: 0,
3463
3455
  providerRecoveryCount: 0,
@@ -3530,6 +3522,9 @@ function createTurnContext(input) {
3530
3522
  },
3531
3523
  providerTurn: {
3532
3524
  effectiveCodexCredentialId: null,
3525
+ effectiveCodexCredentialVersion: null,
3526
+ codexCredentialFailoverLimit: 1,
3527
+ codexPolicySnapshot: null,
3533
3528
  effectiveXaiCredentialId: null,
3534
3529
  xaiRotationEnabled: false,
3535
3530
  xaiAuthoritySnapshot: null,
@@ -3556,7 +3551,7 @@ import { clearRunCredentialsForAttempt } from "@opengeni/runtime";
3556
3551
  import { fetchXaiSubscriptionQuota } from "@opengeni/xai-subscription";
3557
3552
 
3558
3553
  // src/activities/workspace-capture.ts
3559
- import { createHash as createHash5 } from "crypto";
3554
+ import { createHash as createHash6 } from "crypto";
3560
3555
  import {
3561
3556
  deleteWorkspaceCaptureRows,
3562
3557
  insertFailedWorkspaceCapture,
@@ -4310,7 +4305,7 @@ function joinRepoPath(repoRoot, repoRelPath) {
4310
4305
  return `${repoRoot.replace(/\/+$/, "")}/${repoRelPath}`;
4311
4306
  }
4312
4307
  function sha256(bytes) {
4313
- return createHash5("sha256").update(bytes).digest("hex");
4308
+ return createHash6("sha256").update(bytes).digest("hex");
4314
4309
  }
4315
4310
  function utf8(s) {
4316
4311
  return new TextEncoder().encode(s);
@@ -4913,17 +4908,10 @@ async function finalizeTurnAttempt(deps) {
4913
4908
  import {
4914
4909
  requestSessionTurnRecovery,
4915
4910
  getSessionGoal,
4916
- armCodexCapacityWait,
4917
4911
  armXaiCapacityWait,
4918
4912
  reconcileXaiCapacityWait,
4919
- getCodexRotationSettings,
4920
4913
  listCodexAccountStatuses as listCodexAccountStatuses2,
4921
- getSessionCodexState,
4922
- setSessionCodexPinInTransaction,
4923
4914
  quarantineCodexCredentialForLease,
4924
- setCodexCredentialExhaustedWithWakeTargets,
4925
- withSessionCodexCapacityMutation,
4926
- countConsecutiveReactiveRotations,
4927
4915
  recordUsageEvent as recordUsageEvent2,
4928
4916
  getActiveSessionHistoryItemsPaged,
4929
4917
  settleCodexCredentialLeaseLoss,
@@ -4948,7 +4936,7 @@ import {
4948
4936
  } from "@opengeni/config";
4949
4937
  import { selectCodexCredentialId } from "@opengeni/codex";
4950
4938
  import { directPersonalConnectionSubjectId } from "@opengeni/core";
4951
- import { createHash as createHash6 } from "crypto";
4939
+ import { createHash as createHash7 } from "crypto";
4952
4940
 
4953
4941
  // src/activities/agent-turn/model-usage.ts
4954
4942
  import {
@@ -5552,7 +5540,7 @@ function legacyTurnExecutionPolicyInput(turn) {
5552
5540
  };
5553
5541
  }
5554
5542
  function stableHumanInputRequestId(sessionId, turnId, toolCallId) {
5555
- const hex = createHash6("sha256").update("opengeni-human-input-v1\0").update(sessionId).update("\0").update(turnId).update("\0").update(toolCallId).digest("hex").slice(0, 32).split("");
5543
+ const hex = createHash7("sha256").update("opengeni-human-input-v1\0").update(sessionId).update("\0").update(turnId).update("\0").update(toolCallId).digest("hex").slice(0, 32).split("");
5556
5544
  hex[12] = "5";
5557
5545
  hex[16] = ["8", "9", "a", "b"][Number.parseInt(hex[16] ?? "0", 16) % 4] ?? "8";
5558
5546
  const value = hex.join("");
@@ -5575,7 +5563,7 @@ function stableInteractionInterventionOperationId(sessionId, turnId, toolCallId)
5575
5563
  );
5576
5564
  }
5577
5565
  function stableInteractionInterventionUuid(namespace, sessionId, turnId, toolCallId) {
5578
- const hex = createHash6("sha256").update(namespace).update("\0").update(sessionId).update("\0").update(turnId).update("\0").update(toolCallId).digest("hex").slice(0, 32).split("");
5566
+ const hex = createHash7("sha256").update(namespace).update("\0").update(sessionId).update("\0").update(turnId).update("\0").update(toolCallId).digest("hex").slice(0, 32).split("");
5579
5567
  hex[12] = "5";
5580
5568
  hex[16] = ["8", "9", "a", "b"][Number.parseInt(hex[16] ?? "0", 16) % 4] ?? "8";
5581
5569
  const value = hex.join("");
@@ -5633,6 +5621,59 @@ async function ensureRunAllowed(settings, db, accountId, workspaceId, isExternal
5633
5621
  }
5634
5622
 
5635
5623
  // src/activities/agent-turn/failure-settlement.ts
5624
+ function codexDefinitiveFailureDisposition(input) {
5625
+ const alternateAvailable = input.rotationEnabled && input.pinDisposition !== "manual" && input.decisionKind === "active" && input.decisionCredentialId !== null && input.decisionCredentialId !== input.servingCredentialId;
5626
+ if (alternateAvailable) return "failover";
5627
+ if (input.failureKind === "quota" || input.failureKind === "rate_limit" || input.decisionKind === "allCapped" || !input.rotationEnabled || input.pinDisposition === "manual") {
5628
+ return "wait";
5629
+ }
5630
+ return "terminal";
5631
+ }
5632
+ function codexCapacityWaitFailurePayload(input) {
5633
+ if (input.failureKind === "quota") {
5634
+ return codexUsageLimitFailurePayload(
5635
+ input.usageLimit ?? { resetsInSeconds: input.cooldownSeconds },
5636
+ input.detail,
5637
+ input.allAccounts ? { allAccounts: true } : void 0
5638
+ );
5639
+ }
5640
+ if (input.failureKind === "rate_limit") {
5641
+ return {
5642
+ error: "The serving Codex subscription is temporarily rate limited.",
5643
+ code: "codex_account_rate_limited",
5644
+ detail: input.detail,
5645
+ retryable: false
5646
+ };
5647
+ }
5648
+ if (input.failureKind === "auth") {
5649
+ return {
5650
+ error: "The serving Codex account requires reconnection.",
5651
+ code: "codex_relogin_required",
5652
+ detail: "the same accepted turn is waiting for the selected account to recover",
5653
+ retryable: false
5654
+ };
5655
+ }
5656
+ return {
5657
+ error: "The serving Codex account is not authorized for this request.",
5658
+ code: "codex_account_forbidden",
5659
+ detail: "the same accepted turn is waiting for the selected account to recover",
5660
+ retryable: false
5661
+ };
5662
+ }
5663
+ function codexLeaseAccountsForSelection(accounts) {
5664
+ return accounts.map((account) => ({
5665
+ ...account,
5666
+ activeLeaseCount: 0,
5667
+ selectionCount: 0,
5668
+ lastSelectedAt: null
5669
+ }));
5670
+ }
5671
+ function acceptedCodexPolicySnapshot(providerTurn) {
5672
+ if (!providerTurn.codexPolicySnapshot) {
5673
+ throw new Error("Codex accepted policy snapshot is missing after durable lease acquisition");
5674
+ }
5675
+ return providerTurn.codexPolicySnapshot;
5676
+ }
5636
5677
  async function settleTurnFailure(deps) {
5637
5678
  const {
5638
5679
  error,
@@ -5642,7 +5683,6 @@ async function settleTurnFailure(deps) {
5642
5683
  bus,
5643
5684
  observability,
5644
5685
  wakeSessionWorkflow,
5645
- signalCodexCapacityWorkflow,
5646
5686
  cancellationSignal,
5647
5687
  sandboxRotationController,
5648
5688
  noteCancellationRequested,
@@ -5672,6 +5712,12 @@ async function settleTurnFailure(deps) {
5672
5712
  const rotationPending = fencedLease?.rotationRequestedAt != null || lifecycleTransition?.reason === "rotation_in_progress";
5673
5713
  const transitionPending = lifecycleTransition !== null || rotationPending;
5674
5714
  const deadlineRotationPending = rotationPending && fencedLease?.rotationReason === "provider_deadline";
5715
+ const sandboxLifecycleWait = rotationPending ? {
5716
+ version: 1,
5717
+ sandboxGroupId: leaseControlError.sandboxGroupId,
5718
+ leaseEpoch: fencedLease?.leaseEpoch ?? leaseControlError.leaseEpoch,
5719
+ reason: "rotation_in_progress"
5720
+ } : void 0;
5675
5721
  const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
5676
5722
  sessionId: input.sessionId,
5677
5723
  turnId: recoveryTurnId,
@@ -5687,7 +5733,8 @@ async function settleTurnFailure(deps) {
5687
5733
  } : {},
5688
5734
  ...lifecycleTransition ? { transitionReason: lifecycleTransition.reason } : {}
5689
5735
  }
5690
- } : {}
5736
+ } : {},
5737
+ ...sandboxLifecycleWait ? { sandboxLifecycleWait } : {}
5691
5738
  });
5692
5739
  if (recovery.action === "stale") {
5693
5740
  acknowledgeLostAttemptOwnership();
@@ -5701,7 +5748,7 @@ async function settleTurnFailure(deps) {
5701
5748
  control.turnMetricOutcome = "recovering";
5702
5749
  return claimedResult({
5703
5750
  status: "recovering",
5704
- ...transitionPending ? {
5751
+ ...transitionPending && !sandboxLifecycleWait ? {
5705
5752
  continueDelayMs: sandboxDeadlineRotationRecoveryDelayMs(settings)
5706
5753
  } : {}
5707
5754
  });
@@ -5760,6 +5807,12 @@ async function settleTurnFailure(deps) {
5760
5807
  detail: {
5761
5808
  sandboxGroupId: rotation.sandboxGroupId,
5762
5809
  leaseEpoch: rotation.leaseEpoch
5810
+ },
5811
+ sandboxLifecycleWait: {
5812
+ version: 1,
5813
+ sandboxGroupId: rotation.sandboxGroupId,
5814
+ leaseEpoch: rotation.leaseEpoch,
5815
+ reason: "rotation_in_progress"
5763
5816
  }
5764
5817
  });
5765
5818
  if (recovery.action === "stale") {
@@ -5772,10 +5825,7 @@ async function settleTurnFailure(deps) {
5772
5825
  await publishDurableSessionEvents2(bus, input.workspaceId, input.sessionId, recovery.events);
5773
5826
  control.activityStatus = "recovering";
5774
5827
  control.turnMetricOutcome = "recovering";
5775
- return claimedResult({
5776
- status: "recovering",
5777
- continueDelayMs: sandboxDeadlineRotationRecoveryDelayMs(settings)
5778
- });
5828
+ return claimedResult({ status: "recovering" });
5779
5829
  } catch (recoveryError) {
5780
5830
  console.error(
5781
5831
  "sandbox deadline rotation recovery failed",
@@ -5936,7 +5986,7 @@ async function settleTurnFailure(deps) {
5936
5986
  control.turnMetricOutcome = "recovering";
5937
5987
  return claimedResult({ status: "recovering" });
5938
5988
  };
5939
- if (leases.codex.lost && settings.codexCredentialLeasingEnabled && billingState.isCodexTurn && eventing.publish && attempt.turnId && eventing.turnStartedPublished && leases.codex.holderId && leases.codex.generation !== null) {
5989
+ if ((leases.codex.lost || error instanceof CodexCredentialLeaseLostError) && billingState.isCodexTurn && eventing.publish && attempt.turnId && eventing.turnStartedPublished && leases.codex.holderId && leases.codex.generation !== null) {
5940
5990
  return await settleLostCodexAttempt(
5941
5991
  attempt.turnId,
5942
5992
  leases.codex.holderId,
@@ -5966,7 +6016,8 @@ async function settleTurnFailure(deps) {
5966
6016
  control.turnMetricOutcome = "recovering";
5967
6017
  return claimedResult({ status: "recovering" });
5968
6018
  }
5969
- const codexCredentialFailure = settings.codexCredentialLeasingEnabled && billingState.isCodexTurn && providerTurn.effectiveCodexCredentialId ? classifyCodexCredentialFailure(error) : null;
6019
+ const usageLimit = isCodexTransportError(error) ? classifyCodexUsageLimitError(error) : null;
6020
+ const codexCredentialFailure = billingState.isCodexTurn && providerTurn.effectiveCodexCredentialId ? classifyCodexCredentialFailure(error) : null;
5970
6021
  if (codexCredentialFailure && providerTurn.effectiveCodexCredentialId && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
5971
6022
  observability.incrementCounter({
5972
6023
  name: "opengeni_codex_credential_failures_total",
@@ -6017,13 +6068,22 @@ async function settleTurnFailure(deps) {
6017
6068
  } : {}
6018
6069
  } : null;
6019
6070
  const cooldownUntil = codexCredentialCooldownUntil(codexCredentialFailure, serving, now);
6020
- const statePersisted = leases.codex.holderId && leases.codex.generation !== null ? await quarantineCodexCredentialForLease(db, {
6071
+ const quarantineResult = leases.codex.holderId && leases.codex.generation !== null && providerTurn.effectiveCodexCredentialVersion !== null ? await quarantineCodexCredentialForLease(db, {
6021
6072
  accountId: input.accountId,
6022
6073
  workspaceId: input.workspaceId,
6074
+ sessionId: input.sessionId,
6023
6075
  turnId: attempt.turnId,
6076
+ attemptId: input.attemptId,
6077
+ executionGeneration: attempt.executionGeneration,
6078
+ workflowId: input.workflowId,
6079
+ workflowRunId: input.workflowRunId,
6080
+ dispatchId: attempt.dispatchId,
6081
+ expectedRedispatches: attempt.redispatchesAtDispatch,
6024
6082
  credentialId: providerTurn.effectiveCodexCredentialId,
6083
+ credentialVersion: providerTurn.effectiveCodexCredentialVersion,
6025
6084
  holderId: leases.codex.holderId,
6026
6085
  generation: leases.codex.generation,
6086
+ maxFailovers: providerTurn.codexCredentialFailoverLimit,
6027
6087
  quarantine: codexCredentialFailure.kind === "auth" ? {
6028
6088
  kind: "status",
6029
6089
  status: "needs_relogin",
@@ -6037,7 +6097,42 @@ async function settleTurnFailure(deps) {
6037
6097
  until: cooldownUntil,
6038
6098
  cooldownKind: codexCredentialFailure.kind
6039
6099
  }
6040
- }) : false;
6100
+ }) : null;
6101
+ if (quarantineResult?.action === "credential_changed" && leases.codex.holderId && leases.codex.generation !== null) {
6102
+ const recovery = await settleCodexCredentialLeaseLoss(db, {
6103
+ accountId: input.accountId,
6104
+ workspaceId: input.workspaceId,
6105
+ sessionId: input.sessionId,
6106
+ turnId: attempt.turnId,
6107
+ attemptId: input.attemptId,
6108
+ holderId: leases.codex.holderId,
6109
+ generation: leases.codex.generation,
6110
+ expectedRedispatches: attempt.redispatchesAtDispatch,
6111
+ checkpointDurable: true,
6112
+ recoveryPayload: {
6113
+ triggerEventId: attempt.triggerEventId,
6114
+ reason: "codex_credential_version_changed"
6115
+ },
6116
+ failedPayload: {}
6117
+ });
6118
+ if (recovery.action === "recovering") {
6119
+ leases.codex.held = false;
6120
+ await publishDurableSessionEvents2(
6121
+ bus,
6122
+ input.workspaceId,
6123
+ input.sessionId,
6124
+ recovery.events
6125
+ );
6126
+ control.activityStatus = "recovering";
6127
+ control.turnMetricOutcome = "recovering";
6128
+ return claimedResult({ status: "recovering" });
6129
+ }
6130
+ acknowledgeLostAttemptOwnership();
6131
+ control.activityStatus = "cancelled";
6132
+ control.turnMetricOutcome = "cancelled";
6133
+ return claimedResult({ status: "cancelled" });
6134
+ }
6135
+ const statePersisted = quarantineResult?.action === "recorded";
6041
6136
  if (!statePersisted && leases.codex.holderId && leases.codex.generation !== null) {
6042
6137
  leases.codex.lost = true;
6043
6138
  return await settleLostCodexAttempt(
@@ -6047,19 +6142,112 @@ async function settleTurnFailure(deps) {
6047
6142
  true
6048
6143
  );
6049
6144
  }
6050
- const [rotation, accounts] = await Promise.all([
6051
- getCodexRotationSettings(db, input.workspaceId).catch(() => null),
6052
- listCodexAccountStatuses2(db, input.workspaceId).catch(() => [])
6053
- ]);
6054
- const decision = rotation ? chooseRotationActive({
6055
- rotationStrategy: rotation.rotationStrategy,
6056
- activeCredentialId: rotation.activeCredentialId,
6057
- priorCredentialId: providerTurn.effectiveCodexCredentialId,
6058
- accounts,
6059
- now: /* @__PURE__ */ new Date()
6060
- }) : { kind: "none" };
6061
- const candidateAvailable = statePersisted && Boolean(rotation?.rotationEnabled && rotation?.leaseRotationEnabled) && decision.kind === "active" && decision.credentialId !== providerTurn.effectiveCodexCredentialId;
6062
- if (candidateAvailable && leases.codex.holderId && leases.codex.generation !== null) {
6145
+ if (quarantineResult?.action === "recorded" && quarantineResult.exhausted && leases.codex.holderId && leases.codex.generation !== null) {
6146
+ const settlement = await settleCodexCredentialFailover(db, {
6147
+ accountId: input.accountId,
6148
+ workspaceId: input.workspaceId,
6149
+ sessionId: input.sessionId,
6150
+ turnId: attempt.turnId,
6151
+ attemptId: input.attemptId,
6152
+ holderId: leases.codex.holderId,
6153
+ generation: leases.codex.generation,
6154
+ expectedRedispatches: attempt.redispatchesAtDispatch,
6155
+ maxFailovers: quarantineResult.maxFailovers,
6156
+ recoveryPayload: {
6157
+ triggerEventId: attempt.triggerEventId,
6158
+ reason: "codex_credential_failover",
6159
+ credentialId: providerTurn.effectiveCodexCredentialId,
6160
+ failureKind: codexCredentialFailure.kind
6161
+ },
6162
+ failedPayload: {
6163
+ error: "Automatic Codex credential failover stopped after every bounded account attempt was consumed. Send a new message after checking account health or capacity.",
6164
+ code: "codex_credential_failover_exhausted",
6165
+ retryable: false,
6166
+ recovery: "user_message",
6167
+ failoverCount: quarantineResult.failoverCount,
6168
+ maxFailovers: quarantineResult.maxFailovers
6169
+ }
6170
+ });
6171
+ if (settlement.action === "limit_exceeded") {
6172
+ leases.codex.held = false;
6173
+ await publishDurableSessionEvents2(
6174
+ bus,
6175
+ input.workspaceId,
6176
+ input.sessionId,
6177
+ settlement.events
6178
+ );
6179
+ control.activityError = error;
6180
+ control.activityStatus = "idle";
6181
+ control.turnMetricOutcome = "failed";
6182
+ await deliverFailedChildTurnToParent(
6183
+ { db, bus, settings, observability, wakeSessionWorkflow },
6184
+ input.workspaceId,
6185
+ input.sessionId,
6186
+ attempt.turnId
6187
+ );
6188
+ return claimedResult({ status: "idle" });
6189
+ }
6190
+ if (settlement.action === "stale") {
6191
+ acknowledgeLostAttemptOwnership();
6192
+ control.activityStatus = "cancelled";
6193
+ control.turnMetricOutcome = "cancelled";
6194
+ return claimedResult({ status: "cancelled" });
6195
+ }
6196
+ throw new Error("Exhausted Codex failover receipt unexpectedly recovered");
6197
+ }
6198
+ let accounts;
6199
+ try {
6200
+ accounts = await listCodexAccountStatuses2(db, input.workspaceId);
6201
+ } catch (metadataError) {
6202
+ const recoveryFailure = postClaimDatabaseRecoveryFailure({
6203
+ error: metadataError,
6204
+ turnId: attempt.turnId,
6205
+ triggerEventId: attempt.triggerEventId,
6206
+ executionGeneration: attempt.executionGeneration
6207
+ });
6208
+ if (recoveryFailure) {
6209
+ control.activityStatus = "recovering";
6210
+ control.turnMetricOutcome = "recovering";
6211
+ control.activityError = metadataError;
6212
+ throw recoveryFailure;
6213
+ }
6214
+ throw metadataError;
6215
+ }
6216
+ const acceptedPolicy = acceptedCodexPolicySnapshot(providerTurn);
6217
+ const selected = selectCodexCredentialLeaseForTurn({
6218
+ context: {
6219
+ accounts: codexLeaseAccountsForSelection(accounts),
6220
+ activeCredentialId: acceptedPolicy.activeCredentialId,
6221
+ rotationEnabled: acceptedPolicy.rotationEnabled,
6222
+ rotationStrategy: acceptedPolicy.rotationStrategy,
6223
+ existingCredentialId: null,
6224
+ failedCredentialIds: [providerTurn.effectiveCodexCredentialId],
6225
+ policyScope: null,
6226
+ unavailableDiagnostics: []
6227
+ },
6228
+ sessionId: input.sessionId,
6229
+ sessionPinnedCredentialId: acceptedPolicy.pinnedCredentialId,
6230
+ sessionPinSource: acceptedPolicy.pinSource,
6231
+ sessionLastCredentialId: acceptedPolicy.lastCredentialId,
6232
+ now
6233
+ });
6234
+ const decisionKind = selected.decision.kind === "allocatorDisabled" ? "allCapped" : selected.decision.kind;
6235
+ const pinDisposition = classifyCodexPin({
6236
+ pinnedCredentialId: acceptedPolicy.pinnedCredentialId,
6237
+ pinSource: acceptedPolicy.pinSource,
6238
+ strategy: acceptedPolicy.rotationStrategy,
6239
+ rotationEnabled: acceptedPolicy.rotationEnabled
6240
+ });
6241
+ const failureDisposition = codexDefinitiveFailureDisposition({
6242
+ failureKind: codexCredentialFailure.kind,
6243
+ rotationEnabled: acceptedPolicy.rotationEnabled,
6244
+ pinDisposition,
6245
+ decisionKind,
6246
+ decisionCredentialId: selected.decision.kind === "active" ? selected.credentialId : null,
6247
+ servingCredentialId: providerTurn.effectiveCodexCredentialId
6248
+ });
6249
+ const maxFailovers = providerTurn.codexCredentialFailoverLimit;
6250
+ if (statePersisted && failureDisposition === "failover" && leases.codex.holderId && leases.codex.generation !== null) {
6063
6251
  const settlement = await settleCodexCredentialFailover(db, {
6064
6252
  accountId: input.accountId,
6065
6253
  workspaceId: input.workspaceId,
@@ -6069,13 +6257,21 @@ async function settleTurnFailure(deps) {
6069
6257
  holderId: leases.codex.holderId,
6070
6258
  generation: leases.codex.generation,
6071
6259
  expectedRedispatches: attempt.redispatchesAtDispatch,
6072
- maxFailovers: Math.max(1, accounts.length),
6260
+ maxFailovers,
6073
6261
  recoveryPayload: {
6074
6262
  triggerEventId: attempt.triggerEventId,
6075
6263
  reason: "codex_credential_failover",
6076
6264
  credentialId: providerTurn.effectiveCodexCredentialId,
6077
6265
  failureKind: codexCredentialFailure.kind,
6078
6266
  ...cooldownUntil ? { cooldownUntil: cooldownUntil.toISOString() } : {}
6267
+ },
6268
+ failedPayload: {
6269
+ error: "Automatic Codex credential failover stopped after every bounded account attempt was consumed. Send a new message after checking account health or capacity.",
6270
+ code: "codex_credential_failover_exhausted",
6271
+ retryable: false,
6272
+ recovery: "user_message",
6273
+ failoverCount: quarantineResult.failoverCount,
6274
+ maxFailovers: quarantineResult.maxFailovers
6079
6275
  }
6080
6276
  });
6081
6277
  observability.incrementCounter({
@@ -6111,8 +6307,106 @@ async function settleTurnFailure(deps) {
6111
6307
  acknowledgeLostAttemptOwnership();
6112
6308
  control.activityStatus = "cancelled";
6113
6309
  control.turnMetricOutcome = "cancelled";
6310
+ return claimedResult({ status: "cancelled" });
6311
+ }
6312
+ if (settlement.action === "limit_exceeded") {
6313
+ leases.codex.held = false;
6314
+ await publishDurableSessionEvents2(
6315
+ bus,
6316
+ input.workspaceId,
6317
+ input.sessionId,
6318
+ settlement.events
6319
+ );
6320
+ control.activityError = error;
6321
+ control.activityStatus = "idle";
6322
+ control.turnMetricOutcome = "failed";
6323
+ await deliverFailedChildTurnToParent(
6324
+ { db, bus, settings, observability, wakeSessionWorkflow },
6325
+ input.workspaceId,
6326
+ input.sessionId,
6327
+ attempt.turnId
6328
+ );
6329
+ return claimedResult({ status: "idle" });
6330
+ }
6331
+ }
6332
+ if (statePersisted && failureDisposition === "wait" && leases.codex.holderId && leases.codex.generation !== null) {
6333
+ let goal;
6334
+ try {
6335
+ goal = await getSessionGoal(db, input.workspaceId, input.sessionId);
6336
+ } catch (metadataError) {
6337
+ const recoveryFailure = postClaimDatabaseRecoveryFailure({
6338
+ error: metadataError,
6339
+ turnId: attempt.turnId,
6340
+ triggerEventId: attempt.triggerEventId,
6341
+ executionGeneration: attempt.executionGeneration
6342
+ });
6343
+ if (recoveryFailure) {
6344
+ control.activityStatus = "recovering";
6345
+ control.turnMetricOutcome = "recovering";
6346
+ control.activityError = metadataError;
6347
+ throw recoveryFailure;
6348
+ }
6349
+ throw metadataError;
6350
+ }
6351
+ const activeGoal = goal?.status === "active" ? goal : null;
6352
+ const exactProviderReset = codexCredentialFailure.cooldownSeconds !== null && Number.isFinite(codexCredentialFailure.cooldownSeconds) && codexCredentialFailure.cooldownSeconds > 0;
6353
+ const policyCredentialId = pinDisposition === "manual" && acceptedPolicy.pinnedCredentialId ? acceptedPolicy.pinnedCredentialId : !acceptedPolicy.rotationEnabled ? acceptedPolicy.activeCredentialId : null;
6354
+ const capacityAccounts = policyCredentialId ? accounts.filter((account) => account.id === policyCredentialId) : accounts;
6355
+ const authoritativeResetAt = exactProviderReset ? authoritativeCodexCapacityResetAt(capacityAccounts, now) ?? cooldownUntil : null;
6356
+ const allAccounts = acceptedPolicy.rotationEnabled && pinDisposition !== "manual" && decisionKind === "allCapped";
6357
+ const failurePayload = codexCapacityWaitFailurePayload({
6358
+ failureKind: codexCredentialFailure.kind,
6359
+ usageLimit,
6360
+ cooldownSeconds: codexCredentialFailure.cooldownSeconds,
6361
+ detail: codexCredentialFailure.kind === "quota" ? error instanceof Error ? error.message : String(error) : "the same accepted turn is waiting for eligible credential capacity",
6362
+ allAccounts
6363
+ });
6364
+ const evaluated = await armAndReconcileCodexCapacityWait(
6365
+ { db, bus },
6366
+ {
6367
+ accountId: input.accountId,
6368
+ workspaceId: input.workspaceId,
6369
+ sessionId: input.sessionId,
6370
+ turnId: attempt.turnId,
6371
+ attemptId: input.attemptId,
6372
+ workflowId: input.workflowId,
6373
+ goalId: activeGoal?.id ?? null,
6374
+ goalVersion: activeGoal?.version ?? null,
6375
+ earliestResetAt: authoritativeResetAt,
6376
+ resetKind: authoritativeResetAt ? "authoritative" : codexCredentialFailure.kind === "auth" || codexCredentialFailure.kind === "forbidden" ? "mutation_only" : "bounded_refresh",
6377
+ failurePayload,
6378
+ leaseFence: {
6379
+ holderId: leases.codex.holderId,
6380
+ generation: leases.codex.generation
6381
+ },
6382
+ expectedRedispatches: attempt.redispatchesAtDispatch
6383
+ },
6384
+ { onArmed: () => leases.codex.held = false }
6385
+ );
6386
+ control.activityError = error;
6387
+ if (evaluated.action === "resumed") {
6388
+ control.activityStatus = "recovering";
6389
+ control.turnMetricOutcome = "recovering";
6114
6390
  return claimedResult({ status: "recovering" });
6115
6391
  }
6392
+ if (evaluated.action === "waiting") {
6393
+ control.activityError = error;
6394
+ control.activityStatus = "waiting_capacity";
6395
+ control.turnMetricOutcome = "recovering";
6396
+ return claimedResult({
6397
+ status: "waiting_capacity",
6398
+ capacityWait: {
6399
+ waiterId: evaluated.waiter.id,
6400
+ generation: evaluated.waiter.generation,
6401
+ nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
6402
+ wakeRevision: evaluated.waiter.wakeRevision
6403
+ }
6404
+ });
6405
+ }
6406
+ acknowledgeLostAttemptOwnership();
6407
+ control.activityStatus = "cancelled";
6408
+ control.turnMetricOutcome = "cancelled";
6409
+ return claimedResult({ status: "cancelled" });
6116
6410
  }
6117
6411
  }
6118
6412
  }
@@ -6223,201 +6517,20 @@ async function settleTurnFailure(deps) {
6223
6517
  control.activityError = error;
6224
6518
  return claimedResult({ status: "recovering" });
6225
6519
  }
6226
- const usageLimit = isCodexTransportError(error) ? classifyCodexUsageLimitError(error) : null;
6227
6520
  if (usageLimit && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
6228
- const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
6229
- const goalActive = Boolean(goal && goal.status === "active");
6230
6521
  await flushRuntimeBatcher();
6231
6522
  await historySink.reconcileConversationTruth();
6232
- let rotated = false;
6233
- let rotationResumeMs = null;
6234
- let rotationResumeIdleUntilReset = false;
6235
- let allCappedResetAt = null;
6236
- let capacityAuthoritativeResetAt = null;
6237
- let capacityNeedsBoundedRefresh = false;
6238
- if (providerTurn.effectiveCodexCredentialId) {
6239
- const [rotation, sessionCodex] = await Promise.all([
6240
- getCodexRotationSettings(db, input.workspaceId).catch(() => null),
6241
- getSessionCodexState(db, input.workspaceId, input.sessionId).catch(() => null)
6242
- ]);
6243
- const reactiveStrategy = rotation?.rotationStrategy ?? "most_remaining";
6244
- const reactiveDisposition = classifyCodexPin({
6245
- pinnedCredentialId: sessionCodex?.pinnedCredentialId ?? null,
6246
- pinSource: sessionCodex?.pinSource ?? null,
6247
- strategy: reactiveStrategy,
6248
- rotationEnabled: Boolean(rotation?.rotationEnabled)
6249
- });
6250
- const reactiveSharded = reactiveDisposition === "sharded";
6251
- const rotating = Boolean(rotation?.rotationEnabled || rotation?.leaseRotationEnabled) && reactiveDisposition !== "manual";
6252
- if (rotating && rotation) {
6253
- const accounts = await listCodexAccountStatuses2(db, input.workspaceId).catch(() => []);
6254
- const serving = accounts.find((a) => a.id === providerTurn.effectiveCodexCredentialId) ?? null;
6255
- const until = codexCredentialCooldownUntil(
6256
- { kind: "quota", cooldownSeconds: usageLimit.resetsInSeconds },
6257
- serving,
6258
- /* @__PURE__ */ new Date()
6259
- );
6260
- const cooldownMutation = await setCodexCredentialExhaustedWithWakeTargets(
6261
- db,
6262
- input.workspaceId,
6263
- providerTurn.effectiveCodexCredentialId,
6264
- until,
6265
- "quota"
6266
- ).catch(() => null);
6267
- const cooldownPersisted = cooldownMutation?.result ?? false;
6268
- if (cooldownMutation) {
6269
- await signalCodexCapacityWakeTargets(
6270
- { signalCodexCapacityWorkflow, wakeSessionWorkflow },
6271
- cooldownMutation.wakeTargets
6272
- );
6273
- }
6274
- const fresh = accounts.map(
6275
- (a) => a.id === providerTurn.effectiveCodexCredentialId ? { ...a, exhaustedUntil: until, exhaustedKind: "quota" } : a
6276
- );
6277
- if (reactiveSharded) {
6278
- const newHome = shardCredentialForSession({
6279
- sessionId: input.sessionId,
6280
- accounts: fresh,
6281
- now: /* @__PURE__ */ new Date()
6282
- });
6283
- if (newHome) {
6284
- rotated = true;
6285
- const pinMutation = await withSessionCodexCapacityMutation(
6286
- db,
6287
- {
6288
- workspaceId: input.workspaceId,
6289
- reason: "codex_policy_pin_resharded"
6290
- },
6291
- async (tx) => {
6292
- const changed = await setSessionCodexPinInTransaction(
6293
- tx,
6294
- input.workspaceId,
6295
- input.sessionId,
6296
- newHome,
6297
- "policy",
6298
- {
6299
- expected: {
6300
- pinnedCredentialId: sessionCodex?.pinnedCredentialId ?? null,
6301
- pinSource: sessionCodex?.pinSource ?? null
6302
- }
6303
- }
6304
- );
6305
- return { result: changed, changed };
6306
- }
6307
- ).catch(() => null);
6308
- if (pinMutation) {
6309
- await signalCodexCapacityWakeTargets(
6310
- { signalCodexCapacityWorkflow, wakeSessionWorkflow },
6311
- pinMutation.wakeTargets
6312
- );
6313
- }
6314
- const priorConsecutiveRotations = await countConsecutiveReactiveRotations(
6315
- db,
6316
- input.workspaceId,
6317
- input.sessionId
6318
- ).catch(() => 0);
6319
- const resume = computeReactiveRotationResume({
6320
- cooldownPersisted,
6321
- priorConsecutiveRotations,
6322
- connectedAccountCount: accounts.length
6323
- });
6324
- rotationResumeMs = resume.continueDelayMs;
6325
- rotationResumeIdleUntilReset = resume.idleUntilReset;
6326
- } else {
6327
- rotated = true;
6328
- allCappedResetAt = earliestCodexReset(fresh, /* @__PURE__ */ new Date());
6329
- capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, /* @__PURE__ */ new Date());
6330
- capacityNeedsBoundedRefresh = true;
6331
- }
6332
- } else {
6333
- const decision = chooseRotationActive({
6334
- rotationStrategy: reactiveStrategy,
6335
- activeCredentialId: rotation.activeCredentialId,
6336
- priorCredentialId: providerTurn.effectiveCodexCredentialId,
6337
- accounts: fresh,
6338
- now: /* @__PURE__ */ new Date()
6339
- });
6340
- if (decision.kind === "active") {
6341
- rotated = true;
6342
- const priorConsecutiveRotations = await countConsecutiveReactiveRotations(
6343
- db,
6344
- input.workspaceId,
6345
- input.sessionId
6346
- ).catch(() => 0);
6347
- const resume = computeReactiveRotationResume({
6348
- cooldownPersisted,
6349
- priorConsecutiveRotations,
6350
- connectedAccountCount: accounts.length
6351
- });
6352
- rotationResumeMs = resume.continueDelayMs;
6353
- rotationResumeIdleUntilReset = resume.idleUntilReset;
6354
- } else if (decision.kind === "allCapped") {
6355
- rotated = true;
6356
- allCappedResetAt = decision.earliestResetAt;
6357
- capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, /* @__PURE__ */ new Date());
6358
- capacityNeedsBoundedRefresh = true;
6359
- }
6360
- }
6361
- }
6362
- }
6363
- const failurePayload = allCappedResetAt ? codexUsageLimitFailurePayload(
6364
- {
6365
- resetsInSeconds: Math.ceil(Math.max(0, allCappedResetAt.getTime() - Date.now()) / 1e3)
6366
- },
6367
- error instanceof Error ? error.message : String(error),
6368
- { allAccounts: true }
6369
- ) : codexUsageLimitFailurePayload(
6523
+ const failurePayload = codexUsageLimitFailurePayload(
6370
6524
  usageLimit,
6371
6525
  error instanceof Error ? error.message : String(error)
6372
6526
  );
6373
- if (rotationResumeMs === null) {
6374
- const providerResetAt = capacityAuthoritativeResetAt ?? (usageLimit.resetsInSeconds !== null && Number.isFinite(usageLimit.resetsInSeconds) && usageLimit.resetsInSeconds > 0 ? new Date(Date.now() + Math.ceil(usageLimit.resetsInSeconds) * 1e3) : null);
6375
- const armed = await armCodexCapacityWait(db, {
6376
- accountId: input.accountId,
6377
- workspaceId: input.workspaceId,
6378
- sessionId: input.sessionId,
6379
- turnId: attempt.turnId,
6380
- attemptId: input.attemptId,
6381
- workflowId: input.workflowId,
6382
- goalId: goalActive && goal ? goal.id : null,
6383
- goalVersion: goalActive && goal ? goal.version : null,
6384
- earliestResetAt: providerResetAt,
6385
- resetKind: providerResetAt && !capacityNeedsBoundedRefresh ? "authoritative" : "bounded_refresh",
6386
- failurePayload,
6387
- ...leases.codex.holderId && leases.codex.generation !== null ? {
6388
- leaseFence: {
6389
- holderId: leases.codex.holderId,
6390
- generation: leases.codex.generation
6391
- },
6392
- expectedRedispatches: attempt.redispatchesAtDispatch
6393
- } : {}
6394
- });
6395
- if (armed.action === "waiting") {
6396
- await publishDurableSessionEvents2(bus, input.workspaceId, input.sessionId, armed.events);
6397
- control.turnMetricOutcome = "recovering";
6398
- control.activityStatus = "waiting_capacity";
6399
- control.activityError = error;
6400
- return claimedResult({
6401
- status: "waiting_capacity",
6402
- capacityWait: {
6403
- waiterId: armed.waiter.id,
6404
- generation: armed.waiter.generation,
6405
- nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
6406
- wakeRevision: armed.waiter.wakeRevision
6407
- }
6408
- });
6409
- }
6410
- }
6411
6527
  if (!await eventing.settle({
6412
6528
  events: [
6413
- // `rotated:true` ONLY on the reactive rotation path tells evaluateGoalContinuation to
6414
- // freeze autoContinuations (a rotation walk must not burn the goal's continuation budget).
6415
6529
  {
6416
6530
  type: "turn.failed",
6417
6531
  payload: {
6418
6532
  ...failurePayload,
6419
- recovery: goalActive ? "goal_continuation" : "user_message",
6420
- ...rotated ? { rotated: true } : {}
6533
+ recovery: "user_message"
6421
6534
  }
6422
6535
  },
6423
6536
  { type: "session.status.changed", payload: { status: "idle" } }
@@ -6431,21 +6544,6 @@ async function settleTurnFailure(deps) {
6431
6544
  control.turnMetricOutcome = "failed";
6432
6545
  control.activityStatus = "idle";
6433
6546
  control.activityError = error;
6434
- if (goalActive) {
6435
- if (rotationResumeMs !== null) {
6436
- return claimedResult({
6437
- status: "idle",
6438
- continueDelayMs: rotationResumeMs,
6439
- ...rotationResumeIdleUntilReset ? { idleUntilReset: true } : {}
6440
- });
6441
- }
6442
- const resumeMs = allCappedResetAt ? computeIdleDelayMs(allCappedResetAt, /* @__PURE__ */ new Date(), CODEX_USAGE_LIMIT_MAX_RESUME_MS) : usageLimit.resetsInSeconds !== null && Number.isFinite(usageLimit.resetsInSeconds) && usageLimit.resetsInSeconds > 0 ? Math.min(Math.ceil(usageLimit.resetsInSeconds) * 1e3, CODEX_USAGE_LIMIT_MAX_RESUME_MS) : CODEX_USAGE_LIMIT_MAX_RESUME_MS;
6443
- return claimedResult({
6444
- status: "idle",
6445
- continueDelayMs: resumeMs,
6446
- ...allCappedResetAt ? { idleUntilReset: true } : {}
6447
- });
6448
- }
6449
6547
  return claimedResult({ status: "idle" });
6450
6548
  }
6451
6549
  if (error instanceof BudgetExhaustedError && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
@@ -6690,6 +6788,7 @@ import {
6690
6788
  clearDurablePendingSessionToolCalls,
6691
6789
  isSessionCompactionRequested,
6692
6790
  nextSessionHistoryPosition as nextSessionHistoryPosition2,
6791
+ persistModelContextSnapshot,
6693
6792
  updateSessionTitleWithEvent
6694
6793
  } from "@opengeni/db";
6695
6794
  import { publishDurableSessionEvents as publishDurableSessionEvents3 } from "@opengeni/events";
@@ -6726,7 +6825,7 @@ import {
6726
6825
  latestCompactionReplacementFingerprint,
6727
6826
  prepareCompactionPromptInput,
6728
6827
  projectRemoteCompactionOverflowRetryInput,
6729
- sanitizeHistoryItemsForModel as sanitizeHistoryItemsForModel2,
6828
+ sanitizeHistoryItemsForModel as sanitizeHistoryItemsForModel3,
6730
6829
  summarizeForCompaction
6731
6830
  } from "@opengeni/runtime";
6732
6831
  import { contextInputBudgetTokens } from "@opengeni/config";
@@ -6737,7 +6836,7 @@ import {
6737
6836
  FileResourceRef,
6738
6837
  resourceMountPath as resourceMountPath2
6739
6838
  } from "@opengeni/contracts";
6740
- import { createHash as createHash7 } from "crypto";
6839
+ import { createHash as createHash8 } from "crypto";
6741
6840
  import {
6742
6841
  getActiveSessionHistoryItemsPaged as getActiveSessionHistoryItemsPaged2,
6743
6842
  getFilesForSubject as getFilesForSubject2,
@@ -6846,7 +6945,7 @@ async function modelAttachmentContentForFiles(files, readFileBytes) {
6846
6945
  });
6847
6946
  continue;
6848
6947
  }
6849
- if (createHash7("sha256").update(bytes).digest("hex") !== checksum) {
6948
+ if (createHash8("sha256").update(bytes).digest("hex") !== checksum) {
6850
6949
  console.error("model attachment checksum did not match finalized metadata", {
6851
6950
  fileId: file.id
6852
6951
  });
@@ -6975,13 +7074,8 @@ async function turnInput(db, runtime, agent, trigger, options) {
6975
7074
  options.turnId
6976
7075
  )
6977
7076
  );
6978
- if (updates.length > 0) {
6979
- const historyItemIds = new Set(
6980
- updates.map((update) => update.deliveredHistoryItemId).filter(Boolean)
6981
- );
6982
- if (historyItemIds.size !== 1 || updates.some((update) => !update.deliveredHistoryItemId)) {
6983
- throw new Error("Delivered internal updates have no single durable model-memory batch");
6984
- }
7077
+ if (updates.some((update) => !update.deliveredHistoryItemId)) {
7078
+ throw new Error("Delivered internal update has no durable model-memory batch");
6985
7079
  }
6986
7080
  const internalContext = joinInternalContext(
6987
7081
  options.recovering ? [
@@ -7245,7 +7339,7 @@ async function maybeCompactContext(db, settings, scope, lastInputTokens, summari
7245
7339
  const canonicalItems = projectRejectedProviderArtifacts(active);
7246
7340
  const projectForWire = async (input) => {
7247
7341
  const materialized = options.materializeHistory ? await options.materializeHistory(input) : input;
7248
- return sanitizeHistoryItemsForModel2(
7342
+ return sanitizeHistoryItemsForModel3(
7249
7343
  options.projectModelInput ? await options.projectModelInput(materialized) : materialized,
7250
7344
  settings.modelToolOutputTruncationTokens
7251
7345
  );
@@ -7531,7 +7625,7 @@ import {
7531
7625
  import {
7532
7626
  functionCallResultItem,
7533
7627
  invokePreparedAgentTool,
7534
- sanitizeHistoryItemsForModel as sanitizeHistoryItemsForModel3
7628
+ sanitizeHistoryItemsForModel as sanitizeHistoryItemsForModel4
7535
7629
  } from "@opengeni/runtime";
7536
7630
  function argumentsJsonFromCallItem(callItem) {
7537
7631
  const raw = callItem.arguments;
@@ -7561,7 +7655,7 @@ function remainingRunStatePendingApprovalsFromSuffix(rows) {
7561
7655
  return remainingSuffixApprovals(rows, /* @__PURE__ */ new Set(["approval", "interaction_intervention"]));
7562
7656
  }
7563
7657
  function openSuffixHistoryItems(row, resultItem) {
7564
- return sanitizeHistoryItemsForModel3([
7658
+ return sanitizeHistoryItemsForModel4([
7565
7659
  ...row.tiedReasoningItems,
7566
7660
  row.callItem,
7567
7661
  resultItem
@@ -8269,7 +8363,7 @@ async function runTurnStreamAttempt(deps) {
8269
8363
  });
8270
8364
  runInput = prepared.input;
8271
8365
  historySink.providerArtifactCandidates = prepared.providerArtifactCandidates;
8272
- historySink.persistedHistoryCount = prepared.persistedHistoryCount;
8366
+ historySink.seedHistory(prepared.input.input, prepared.persistedHistoryCount);
8273
8367
  preparedHistoryCount = prepared.persistedHistoryCount;
8274
8368
  const historyPositionStartedAt = performance.now();
8275
8369
  let historyPositionOutcome = "completed";
@@ -8520,6 +8614,17 @@ async function runTurnStreamAttempt(deps) {
8520
8614
  return await runtime.runStream(agent, runInput, eventing.modelRunSettings, {
8521
8615
  signal: runtimeCancellationSignal,
8522
8616
  sandboxEnvironment,
8617
+ onModelVisibleContext: async (snapshot) => {
8618
+ await persistModelContextSnapshot(db, {
8619
+ accountId: input.accountId,
8620
+ workspaceId: input.workspaceId,
8621
+ sessionId: input.sessionId,
8622
+ turnId: activeTurnId,
8623
+ attemptId: input.attemptId,
8624
+ executionGeneration: attempt.executionGeneration,
8625
+ snapshot
8626
+ });
8627
+ },
8523
8628
  onRuntimeEvent: async (event) => {
8524
8629
  await leases.renewServing("runtime_event");
8525
8630
  if (leases.servingLost()) {
@@ -8605,7 +8710,9 @@ async function runTurnStreamAttempt(deps) {
8605
8710
  });
8606
8711
  }
8607
8712
  };
8608
- if (leases.codex.lost) {
8713
+ if (billingState.isCodexTurn) {
8714
+ leases.codex.assertUsable();
8715
+ } else if (leases.codex.lost) {
8609
8716
  throw new Error("Codex credential lease expired before the model run");
8610
8717
  }
8611
8718
  if (leases.xai.lost) {
@@ -8657,6 +8764,7 @@ async function runTurnStreamAttempt(deps) {
8657
8764
  ]);
8658
8765
  };
8659
8766
  const iterator = eventing.stream.toStream()[Symbol.asyncIterator]();
8767
+ const closeStreamWaitAdmission = eventing.preparedTools?.inputWaitYield?.captureStreamClose();
8660
8768
  let streamDone = false;
8661
8769
  try {
8662
8770
  while (true) {
@@ -8957,6 +9065,7 @@ async function runTurnStreamAttempt(deps) {
8957
9065
  }
8958
9066
  }
8959
9067
  } catch (error) {
9068
+ closeStreamWaitAdmission?.();
8960
9069
  if (fallbackProviderRequestLifecycleStartedAt !== null) {
8961
9070
  const durationMs = Math.max(
8962
9071
  0,
@@ -9015,7 +9124,8 @@ async function runTurnStreamAttempt(deps) {
9015
9124
  temporalCancellationSignal: cancellationSignal,
9016
9125
  runtimeCancellationSignal
9017
9126
  });
9018
- if (options.requireTerminalModelResponse && eventing.stream.interruptions.length === 0 && modelResponseState.responseCount === responseCountBeforeStream) {
9127
+ await eventing.preparedTools?.inputWaitYield?.sealForSettlement(runtimeCancellationSignal);
9128
+ if (options.requireTerminalModelResponse && !eventing.preparedTools?.inputWaitYield?.yielded && eventing.stream.interruptions.length === 0 && modelResponseState.responseCount === responseCountBeforeStream) {
9019
9129
  throwIfWorkerShuttingDown();
9020
9130
  throwIfTurnCancelled();
9021
9131
  throw new PostCompactionContinuationEmptyError();
@@ -9239,16 +9349,16 @@ async function runTurnStreamAttempt(deps) {
9239
9349
  control.activityStatus = "requires_action";
9240
9350
  return claimedResult({ status: "requires_action" });
9241
9351
  }
9242
- const finalOutput = String(requireAgentStreamFinalOutput(eventing.stream.finalOutput));
9352
+ const inputWaitYielded = eventing.preparedTools?.inputWaitYield?.yielded === true;
9353
+ const finalOutput = String(
9354
+ requireAgentStreamFinalOutput(eventing.stream.finalOutput, inputWaitYielded)
9355
+ );
9243
9356
  await historySink.reconcileConversationTruth({ requireDurable: true });
9244
9357
  await finalizeTurnOpStreamOps();
9245
9358
  await finishParallelSessionTitle();
9246
9359
  if (!await eventing.settle({
9247
9360
  events: [
9248
- {
9249
- type: "agent.message.completed",
9250
- payload: { text: finalOutput }
9251
- },
9361
+ ...inputWaitYielded ? [] : [{ type: "agent.message.completed", payload: { text: finalOutput } }],
9252
9362
  { type: "turn.completed", payload: { output: finalOutput } },
9253
9363
  { type: "session.status.changed", payload: { status: "idle" } }
9254
9364
  ],
@@ -9481,7 +9591,7 @@ import {
9481
9591
 
9482
9592
  // src/activities/capabilities.ts
9483
9593
  import {
9484
- environmentsEncryptionKeyBytes as environmentsEncryptionKeyBytes2,
9594
+ environmentsEncryptionKeyBytes,
9485
9595
  WORKSPACE_GATEWAY_MODEL_ID_PREFIX,
9486
9596
  WORKSPACE_OPENROUTER_MODEL_ID_PREFIX,
9487
9597
  withCodexCatalogProvider,
@@ -9513,7 +9623,7 @@ import {
9513
9623
  loadOrganizationModelProviderApiKey
9514
9624
  } from "@opengeni/db";
9515
9625
  async function settingsWithSessionMcpServersForRun(db, workspaceId, sessionId, attemptId, settings, options) {
9516
- const encryptionKey = environmentsEncryptionKeyBytes2(settings);
9626
+ const encryptionKey = environmentsEncryptionKeyBytes(settings);
9517
9627
  if (!encryptionKey) {
9518
9628
  const metadata = await listSessionMcpServerMetadata(db, workspaceId, sessionId);
9519
9629
  if (metadata.length === 0) {
@@ -9664,6 +9774,13 @@ import { readTurnExecutionPolicyV1 } from "@opengeni/contracts";
9664
9774
  function turnAttemptProducerId(input, turnId) {
9665
9775
  return `${input.workflowId}:${turnId}:${input.attemptId}`;
9666
9776
  }
9777
+ function codexCredentialLeaseHolderId(input, turnId) {
9778
+ const parts = [input.workflowId, turnId, input.attemptId];
9779
+ if (parts.some((part) => part.trim().length === 0)) {
9780
+ throw new Error("Codex credential lease holder requires a complete turn attempt identity");
9781
+ }
9782
+ return `codex-turn:${parts.map((part) => encodeURIComponent(part)).join(":")}`;
9783
+ }
9667
9784
  async function claimTurnAttempt(deps) {
9668
9785
  const {
9669
9786
  input,
@@ -9712,9 +9829,11 @@ async function claimTurnAttempt(deps) {
9712
9829
  }
9713
9830
  const turn = claim.turn;
9714
9831
  attempt.turnId = turn.id;
9832
+ attempt.dispatchId = dispatchId;
9715
9833
  attempt.executionGeneration = turn.executionGeneration;
9716
9834
  attempt.providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
9717
9835
  attempt.triggerEventId = turn.triggerEventId;
9836
+ leases.codex.holderId = codexCredentialLeaseHolderId(input, turn.id);
9718
9837
  attempt.redispatchesAtDispatch = Number(
9719
9838
  turn.metadata?.workerDeathRedispatches ?? 0
9720
9839
  );
@@ -9863,7 +9982,6 @@ async function claimTurnAttempt(deps) {
9863
9982
  eventing.heartbeatTimer = startActivityHeartbeat(activityContext, heartbeatDetails);
9864
9983
  let producerSeq = 0;
9865
9984
  const producerId = turnAttemptProducerId(input, attempt.turnId);
9866
- leases.codex.holderId = dispatchId;
9867
9985
  const modelUsageDispatchId = input.attemptId;
9868
9986
  const claimedModelUsageSourceKeys = /* @__PURE__ */ new Set();
9869
9987
  const emittedModelUsageSourceKeys = /* @__PURE__ */ new Set();
@@ -9949,6 +10067,7 @@ async function claimTurnAttempt(deps) {
9949
10067
  turnStatus: inputSettlement.turnStatus,
9950
10068
  sessionStatus: inputSettlement.sessionStatus,
9951
10069
  activeTurnId: inputSettlement.activeTurnId,
10070
+ ...inputSettlement.suppressGoalContinuation !== void 0 ? { suppressGoalContinuation: inputSettlement.suppressGoalContinuation } : {},
9952
10071
  events: inputs,
9953
10072
  ...runState ? { runState } : {},
9954
10073
  ...compactionRequestFailure ? { compactionRequestFailure } : {}
@@ -10042,16 +10161,15 @@ async function claimTurnAttempt(deps) {
10042
10161
  import {
10043
10162
  getSessionGoal as getSessionGoal2,
10044
10163
  acquireCodexCredentialLease,
10045
- armCodexCapacityWait as armCodexCapacityWait2,
10164
+ CodexCredentialLeaseAttemptFencedError,
10165
+ CodexCredentialFailoverExhaustedError,
10046
10166
  CODEX_CREDENTIAL_LEASE_TTL_MS as CODEX_CREDENTIAL_LEASE_TTL_MS2,
10047
- getCodexRotationSettings as getCodexRotationSettings2,
10048
- listCodexAccountStatuses as listCodexAccountStatuses3,
10049
- getSessionCodexState as getSessionCodexState2,
10050
10167
  recordSessionActiveCodexCredential,
10051
- setSessionCodexPinInTransaction as setSessionCodexPinInTransaction2,
10052
- setActiveCodexCredential,
10053
- withSessionCodexCapacityMutation as withSessionCodexCapacityMutation2
10168
+ setSessionCodexPinInTransaction,
10169
+ settleCodexCredentialFailover as settleCodexCredentialFailover2,
10170
+ withSessionCodexCapacityMutation
10054
10171
  } from "@opengeni/db";
10172
+ import { CodexReloginRequired } from "@opengeni/codex";
10055
10173
  import { publishDurableSessionEvents as publishDurableSessionEvents5 } from "@opengeni/events";
10056
10174
 
10057
10175
  // src/activities/codex-fleet-shadow.ts
@@ -10255,6 +10373,7 @@ async function selectCodexTurnCapacity(deps) {
10255
10373
  observability,
10256
10374
  wakeSessionWorkflow,
10257
10375
  signalCodexCapacityWorkflow,
10376
+ dispatchId,
10258
10377
  control,
10259
10378
  attempt,
10260
10379
  billingState,
@@ -10262,6 +10381,7 @@ async function selectCodexTurnCapacity(deps) {
10262
10381
  providerTurn,
10263
10382
  leases,
10264
10383
  claimedResult,
10384
+ acknowledgeLostAttemptOwnership,
10265
10385
  turn,
10266
10386
  codexWorkspaceKey
10267
10387
  } = deps;
@@ -10277,123 +10397,62 @@ async function selectCodexTurnCapacity(deps) {
10277
10397
  const credentialSelectionStartedAt = performance.now();
10278
10398
  let credentialSelectionOutcome = "completed";
10279
10399
  try {
10280
- const sessionCodex = await getSessionCodexState2(db, input.workspaceId, input.sessionId);
10281
- const sessionPin = sessionCodex?.pinnedCredentialId ?? null;
10282
- const sessionPinSource = sessionCodex?.pinSource ?? null;
10283
- const selectForTurn = (context) => selectCodexCredentialLeaseForTurn({
10400
+ const selectForTurn = (context, lockedSessionCodexState2) => selectCodexCredentialLeaseForTurn({
10284
10401
  context,
10285
- leasingEnabled: settings.codexCredentialLeasingEnabled,
10286
10402
  sessionId: input.sessionId,
10287
- sessionPinnedCredentialId: sessionPin,
10288
- sessionPinSource,
10289
- sessionLastCredentialId: sessionCodex?.lastCredentialId ?? null,
10403
+ sessionPinnedCredentialId: lockedSessionCodexState2.pinnedCredentialId,
10404
+ sessionPinSource: lockedSessionCodexState2.pinSource,
10405
+ sessionLastCredentialId: lockedSessionCodexState2.lastCredentialId,
10290
10406
  now: /* @__PURE__ */ new Date()
10291
10407
  });
10292
- let leased;
10293
- let leaseAcquisitionStartedAtMs = null;
10294
- if (settings.codexCredentialLeasingEnabled) {
10408
+ let leaseAcquisitionStartedAtMs = performance.now();
10409
+ let leased = await acquireCodexCredentialLease(
10410
+ db,
10411
+ {
10412
+ accountId: input.accountId,
10413
+ workspaceId: input.workspaceId,
10414
+ sessionId: input.sessionId,
10415
+ turnId,
10416
+ attemptId: input.attemptId,
10417
+ executionGeneration: attempt.executionGeneration,
10418
+ workflowId: input.workflowId,
10419
+ workflowRunId: input.workflowRunId,
10420
+ dispatchId,
10421
+ expectedRedispatches: attempt.redispatchesAtDispatch,
10422
+ holderId,
10423
+ advanceActivePointer: true
10424
+ },
10425
+ selectForTurn
10426
+ );
10427
+ if (leased.decision.kind === "allCapped") {
10428
+ await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
10429
+ signalCodexCapacityWorkflow,
10430
+ wakeSessionWorkflow
10431
+ });
10295
10432
  leaseAcquisitionStartedAtMs = performance.now();
10296
10433
  leased = await acquireCodexCredentialLease(
10297
10434
  db,
10298
10435
  {
10299
10436
  accountId: input.accountId,
10300
10437
  workspaceId: input.workspaceId,
10438
+ sessionId: input.sessionId,
10301
10439
  turnId,
10440
+ attemptId: input.attemptId,
10441
+ executionGeneration: attempt.executionGeneration,
10442
+ workflowId: input.workflowId,
10443
+ workflowRunId: input.workflowRunId,
10444
+ dispatchId,
10445
+ expectedRedispatches: attempt.redispatchesAtDispatch,
10302
10446
  holderId,
10303
- advanceActivePointer: sessionPin === null
10447
+ advanceActivePointer: true
10304
10448
  },
10305
10449
  selectForTurn
10306
10450
  );
10307
- } else {
10308
- const [rotation, accounts] = await Promise.all([
10309
- getCodexRotationSettings2(db, input.workspaceId),
10310
- listCodexAccountStatuses3(db, input.workspaceId)
10311
- ]);
10312
- const leaseAccounts = accounts.map((account) => ({
10313
- ...account,
10314
- activeLeaseCount: 0,
10315
- selectionCount: 0,
10316
- lastSelectedAt: null
10317
- }));
10318
- const activeCredentialId = rotation?.activeCredentialId ?? null;
10319
- const selected = selectForTurn({
10320
- accounts: leaseAccounts,
10321
- activeCredentialId,
10322
- rotationEnabled: rotation?.rotationEnabled ?? false,
10323
- leaseRotationEnabled: false,
10324
- rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
10325
- existingCredentialId: null,
10326
- policyScope: null,
10327
- unavailableDiagnostics: []
10328
- });
10329
- leased = {
10330
- ...selected,
10331
- accounts: leaseAccounts,
10332
- activeCredentialId,
10333
- rotationEnabled: rotation?.rotationEnabled ?? false,
10334
- rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
10335
- reused: false,
10336
- holderId: null,
10337
- generation: null,
10338
- leasedUntil: null,
10339
- unavailableDiagnostics: [],
10340
- advanceActivePointer: selected.advanceActivePointer !== false
10341
- };
10342
- }
10343
- if (leased.decision.kind === "allCapped") {
10344
- await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
10345
- signalCodexCapacityWorkflow,
10346
- wakeSessionWorkflow
10347
- });
10348
- if (settings.codexCredentialLeasingEnabled) {
10349
- leaseAcquisitionStartedAtMs = performance.now();
10350
- leased = await acquireCodexCredentialLease(
10351
- db,
10352
- {
10353
- accountId: input.accountId,
10354
- workspaceId: input.workspaceId,
10355
- turnId,
10356
- holderId,
10357
- advanceActivePointer: sessionPin === null
10358
- },
10359
- selectForTurn
10360
- );
10361
- } else {
10362
- const [rotation, accounts] = await Promise.all([
10363
- getCodexRotationSettings2(db, input.workspaceId),
10364
- listCodexAccountStatuses3(db, input.workspaceId)
10365
- ]);
10366
- const leaseAccounts = accounts.map((account) => ({
10367
- ...account,
10368
- activeLeaseCount: 0,
10369
- selectionCount: 0,
10370
- lastSelectedAt: null
10371
- }));
10372
- const selected = selectForTurn({
10373
- accounts: leaseAccounts,
10374
- activeCredentialId: rotation?.activeCredentialId ?? null,
10375
- rotationEnabled: rotation?.rotationEnabled ?? false,
10376
- leaseRotationEnabled: false,
10377
- rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
10378
- existingCredentialId: null,
10379
- policyScope: null,
10380
- unavailableDiagnostics: []
10381
- });
10382
- leased = {
10383
- ...selected,
10384
- accounts: leaseAccounts,
10385
- activeCredentialId: rotation?.activeCredentialId ?? null,
10386
- rotationEnabled: rotation?.rotationEnabled ?? false,
10387
- rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
10388
- reused: false,
10389
- holderId: null,
10390
- generation: null,
10391
- leasedUntil: null,
10392
- unavailableDiagnostics: [],
10393
- advanceActivePointer: selected.advanceActivePointer !== false
10394
- };
10395
- }
10396
10451
  }
10452
+ const lockedSessionCodexState = leased.sessionCodexState;
10453
+ providerTurn.codexPolicySnapshot = leased.codexPolicySnapshot;
10454
+ const sessionPin = lockedSessionCodexState.pinnedCredentialId;
10455
+ const sessionPinSource = lockedSessionCodexState.pinSource;
10397
10456
  const rotationDecision = leased.decision;
10398
10457
  const selectedPinDisposition = classifyCodexPin({
10399
10458
  pinnedCredentialId: sessionPin,
@@ -10401,15 +10460,15 @@ async function selectCodexTurnCapacity(deps) {
10401
10460
  strategy: leased.rotationStrategy,
10402
10461
  rotationEnabled: leased.rotationEnabled
10403
10462
  });
10404
- if (selectedPinDisposition === "sharded" && leased.credentialId !== null && (sessionPinSource !== "policy" || sessionPin !== leased.credentialId)) {
10405
- const pinMutation = await withSessionCodexCapacityMutation2(
10463
+ if (!leased.codexPolicySnapshotReused && selectedPinDisposition === "sharded" && leased.credentialId !== null && (sessionPinSource !== "policy" || sessionPin !== leased.credentialId)) {
10464
+ const pinMutation = await withSessionCodexCapacityMutation(
10406
10465
  db,
10407
10466
  {
10408
10467
  workspaceId: input.workspaceId,
10409
10468
  reason: "codex_policy_pin_changed"
10410
10469
  },
10411
10470
  async (tx) => {
10412
- const changed = await setSessionCodexPinInTransaction2(
10471
+ const changed = await setSessionCodexPinInTransaction(
10413
10472
  tx,
10414
10473
  input.workspaceId,
10415
10474
  input.sessionId,
@@ -10430,14 +10489,14 @@ async function selectCodexTurnCapacity(deps) {
10430
10489
  pinMutation.wakeTargets
10431
10490
  );
10432
10491
  } else if (selectedPinDisposition === "clearStale") {
10433
- const pinMutation = await withSessionCodexCapacityMutation2(
10492
+ const pinMutation = await withSessionCodexCapacityMutation(
10434
10493
  db,
10435
10494
  {
10436
10495
  workspaceId: input.workspaceId,
10437
10496
  reason: "codex_stale_policy_pin_cleared"
10438
10497
  },
10439
10498
  async (tx) => {
10440
- const changed = await setSessionCodexPinInTransaction2(
10499
+ const changed = await setSessionCodexPinInTransaction(
10441
10500
  tx,
10442
10501
  input.workspaceId,
10443
10502
  input.sessionId,
@@ -10458,16 +10517,14 @@ async function selectCodexTurnCapacity(deps) {
10458
10517
  pinMutation.wakeTargets
10459
10518
  );
10460
10519
  }
10461
- if (!settings.codexCredentialLeasingEnabled && leased.advanceActivePointer && sessionPin === null && rotationDecision.kind === "active" && rotationDecision.moved) {
10462
- await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
10463
- }
10464
10520
  providerTurn.effectiveCodexCredentialId = leased.credentialId;
10521
+ providerTurn.codexCredentialFailoverLimit = leased.failoverLimit;
10465
10522
  leases.codex.generation = leased.generation;
10466
10523
  leases.codex.confirmedUntilMs = leased.leasedUntil && leaseAcquisitionStartedAtMs !== null ? leaseAcquisitionStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS2 : null;
10467
10524
  leases.codex.held = providerTurn.effectiveCodexCredentialId !== null && leased.holderId !== null && leased.generation !== null && leases.codex.confirmedUntilMs !== null;
10468
10525
  if (leases.codex.held) leases.codex.startHeartbeat();
10469
- const actualOutcome = providerTurn.effectiveCodexCredentialId ? "selected" : rotationDecision.kind === "allCapped" ? "waiting" : "none";
10470
- const actualReason = providerTurn.effectiveCodexCredentialId ? leased.reused ? "lease_reused" : sessionPin === providerTurn.effectiveCodexCredentialId ? "pin" : rotationDecision.kind === "active" && rotationDecision.moved ? "rotation" : "active" : rotationDecision.kind === "allCapped" ? "all_capped" : "none";
10526
+ const actualOutcome = providerTurn.effectiveCodexCredentialId ? "selected" : rotationDecision.kind === "allCapped" || rotationDecision.kind === "allocatorDisabled" ? "waiting" : "none";
10527
+ const actualReason = providerTurn.effectiveCodexCredentialId ? leased.reused ? "lease_reused" : sessionPin === providerTurn.effectiveCodexCredentialId ? "pin" : rotationDecision.kind === "active" && rotationDecision.moved ? "rotation" : "active" : rotationDecision.kind === "allCapped" ? "all_capped" : rotationDecision.kind === "allocatorDisabled" ? "allocator_disabled" : "none";
10471
10528
  const fencedInFlight = leased.reused;
10472
10529
  const shadowResult = await publishCodexFleetShadowDecisionV1({
10473
10530
  enabled: settings.codexFleetPolicyShadowEnabled,
@@ -10476,7 +10533,7 @@ async function selectCodexTurnCapacity(deps) {
10476
10533
  actualCredentialId: providerTurn.effectiveCodexCredentialId,
10477
10534
  actualOutcome,
10478
10535
  actualReason,
10479
- affinityCredentialId: fencedInFlight ? providerTurn.effectiveCodexCredentialId : sessionPin ?? sessionCodex?.lastCredentialId ?? null,
10536
+ affinityCredentialId: fencedInFlight ? providerTurn.effectiveCodexCredentialId : sessionPin ?? lockedSessionCodexState.lastCredentialId ?? null,
10480
10537
  fencedInFlight,
10481
10538
  nearExhaustionPct: settings.codexRotationNearExhaustionPct,
10482
10539
  now: /* @__PURE__ */ new Date(),
@@ -10540,7 +10597,7 @@ async function selectCodexTurnCapacity(deps) {
10540
10597
  depth: poolDepth
10541
10598
  });
10542
10599
  }
10543
- if (providerTurn.effectiveCodexCredentialId === null && leased.accounts.length > 0 && leased.accounts.every((account) => !account.allocatorEnabled) && turnId) {
10600
+ if (providerTurn.effectiveCodexCredentialId === null && leased.accounts.length > 0 && (rotationDecision.kind === "allocatorDisabled" || leased.accounts.every((account) => !account.allocatorEnabled)) && turnId) {
10544
10601
  if (turn.source === "compaction") {
10545
10602
  if (!await eventing.settle({
10546
10603
  events: [
@@ -10567,66 +10624,146 @@ async function selectCodexTurnCapacity(deps) {
10567
10624
  control.activityStatus = "idle";
10568
10625
  return { exit: claimedResult({ status: "idle", deferredUntilWake: true }) };
10569
10626
  }
10570
- const goal = await getSessionGoal2(db, input.workspaceId, input.sessionId).catch(() => null);
10627
+ const goal = await getSessionGoal2(db, input.workspaceId, input.sessionId);
10571
10628
  const activeGoal = goal?.status === "active" ? goal : null;
10572
- const armed = await armCodexCapacityWait2(db, {
10573
- accountId: input.accountId,
10574
- workspaceId: input.workspaceId,
10575
- sessionId: input.sessionId,
10576
- turnId,
10577
- attemptId: input.attemptId,
10578
- workflowId: input.workflowId,
10579
- goalId: activeGoal?.id ?? null,
10580
- goalVersion: activeGoal?.version ?? null,
10581
- earliestResetAt: null,
10582
- resetKind: "bounded_refresh",
10583
- failurePayload: {
10584
- error: "All connected Codex subscriptions are disabled for new allocations.",
10585
- code: "codex_allocator_disabled",
10586
- detail: "waiting for a credential to be re-enabled, reconnected, or added"
10629
+ const evaluated = await armAndReconcileCodexCapacityWait(
10630
+ { db, bus },
10631
+ {
10632
+ accountId: input.accountId,
10633
+ workspaceId: input.workspaceId,
10634
+ sessionId: input.sessionId,
10635
+ turnId,
10636
+ attemptId: input.attemptId,
10637
+ workflowId: input.workflowId,
10638
+ goalId: activeGoal?.id ?? null,
10639
+ goalVersion: activeGoal?.version ?? null,
10640
+ earliestResetAt: null,
10641
+ resetKind: "mutation_only",
10642
+ failurePayload: {
10643
+ error: rotationDecision.kind === "allocatorDisabled" ? "The policy-selected Codex subscription is disabled for new allocations." : "All connected Codex subscriptions are disabled for new allocations.",
10644
+ code: "codex_allocator_disabled",
10645
+ detail: rotationDecision.kind === "allocatorDisabled" ? "waiting for the selected credential to be re-enabled under the accepted source policy" : "waiting for a credential in the accepted source pool to be re-enabled, reconnected, or added"
10646
+ }
10587
10647
  }
10588
- });
10589
- if (armed.action === "waiting") {
10590
- await publishDurableSessionEvents5(bus, input.workspaceId, input.sessionId, armed.events);
10648
+ );
10649
+ if (evaluated.action === "resumed") {
10650
+ control.turnMetricOutcome = "recovering";
10651
+ control.activityStatus = "recovering";
10652
+ return { exit: claimedResult({ status: "recovering" }) };
10653
+ }
10654
+ if (evaluated.action === "waiting") {
10591
10655
  control.turnMetricOutcome = "recovering";
10592
10656
  control.activityStatus = "waiting_capacity";
10593
10657
  return {
10594
10658
  exit: claimedResult({
10595
10659
  status: "waiting_capacity",
10596
10660
  capacityWait: {
10597
- waiterId: armed.waiter.id,
10598
- generation: armed.waiter.generation,
10599
- nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
10600
- wakeRevision: armed.waiter.wakeRevision
10661
+ waiterId: evaluated.waiter.id,
10662
+ generation: evaluated.waiter.generation,
10663
+ nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
10664
+ wakeRevision: evaluated.waiter.wakeRevision
10601
10665
  }
10602
10666
  })
10603
10667
  };
10604
10668
  }
10605
- if (!await eventing.settle({
10606
- events: [
10607
- {
10608
- type: "turn.failed",
10609
- payload: {
10610
- error: "All connected Codex subscriptions are disabled for new allocations.",
10611
- code: "codex_allocator_disabled",
10612
- retryable: false,
10613
- recovery: "user_message"
10669
+ acknowledgeLostAttemptOwnership();
10670
+ control.turnMetricOutcome = "cancelled";
10671
+ control.activityStatus = "cancelled";
10672
+ return { exit: claimedResult({ status: "cancelled" }) };
10673
+ }
10674
+ if (rotationDecision.kind === "none") {
10675
+ if (leased.poolAccountCount === 0 && !leased.codexPolicySnapshotReused && leased.codexPolicySnapshot?.source !== "disabled") {
10676
+ throw new CodexReloginRequired("No Codex subscription is connected for this workspace.");
10677
+ }
10678
+ const noneReason = sessionPinSource === "manual" && sessionPin !== null ? {
10679
+ code: "codex_manual_pin_unavailable",
10680
+ error: "The pinned Codex subscription is unavailable for this turn.",
10681
+ detail: "waiting for the pinned account to reconnect or become allocatable under the accepted source policy"
10682
+ } : !leased.rotationEnabled && leased.activeCredentialId === null ? {
10683
+ code: "codex_active_pointer_unavailable",
10684
+ error: "The Codex active subscription pointer is unavailable for this turn.",
10685
+ detail: "waiting for the accepted active account to be restored or become allocatable"
10686
+ } : leased.poolAccountCount > leased.accounts.length ? {
10687
+ code: "codex_policy_pool_unavailable",
10688
+ error: "No Codex subscription matches the accepted account-selection policy.",
10689
+ detail: "waiting for a matching credential to reconnect or become allocatable under the accepted source policy"
10690
+ } : {
10691
+ code: "codex_credential_unavailable",
10692
+ error: "No Codex subscription is currently available under the accepted policy.",
10693
+ detail: "waiting for a matching credential to reconnect or become allocatable under the accepted source policy"
10694
+ };
10695
+ if (turn.source === "compaction") {
10696
+ if (!await eventing.settle({
10697
+ events: [
10698
+ {
10699
+ type: "turn.cancelled",
10700
+ payload: {
10701
+ maintenance: "context_compaction",
10702
+ reason: noneReason.code,
10703
+ requestPreserved: true
10704
+ }
10705
+ },
10706
+ {
10707
+ type: "session.status.changed",
10708
+ payload: { status: "idle" }
10614
10709
  }
10615
- },
10616
- {
10617
- type: "session.status.changed",
10618
- payload: { status: "idle" }
10710
+ ],
10711
+ turnStatus: "cancelled",
10712
+ sessionStatus: "idle",
10713
+ activeTurnId: null
10714
+ })) {
10715
+ return { exit: claimedResult({ status: "cancelled" }) };
10716
+ }
10717
+ control.turnMetricOutcome = "cancelled";
10718
+ control.activityStatus = "idle";
10719
+ return { exit: claimedResult({ status: "idle", deferredUntilWake: true }) };
10720
+ }
10721
+ const goal = await getSessionGoal2(db, input.workspaceId, input.sessionId);
10722
+ const activeGoal = goal?.status === "active" ? goal : null;
10723
+ const evaluated = await armAndReconcileCodexCapacityWait(
10724
+ { db, bus },
10725
+ {
10726
+ accountId: input.accountId,
10727
+ workspaceId: input.workspaceId,
10728
+ sessionId: input.sessionId,
10729
+ turnId,
10730
+ attemptId: input.attemptId,
10731
+ workflowId: input.workflowId,
10732
+ goalId: activeGoal?.id ?? null,
10733
+ goalVersion: activeGoal?.version ?? null,
10734
+ earliestResetAt: null,
10735
+ resetKind: "mutation_only",
10736
+ failurePayload: {
10737
+ error: noneReason.error,
10738
+ code: noneReason.code,
10739
+ detail: noneReason.detail
10619
10740
  }
10620
- ],
10621
- turnStatus: "failed",
10622
- sessionStatus: "idle",
10623
- activeTurnId: null
10624
- })) {
10625
- return { exit: claimedResult({ status: "cancelled" }) };
10741
+ }
10742
+ );
10743
+ if (evaluated.action === "resumed") {
10744
+ control.turnMetricOutcome = "recovering";
10745
+ control.activityStatus = "recovering";
10746
+ return { exit: claimedResult({ status: "recovering" }) };
10626
10747
  }
10627
- control.turnMetricOutcome = "failed";
10628
- control.activityStatus = "idle";
10629
- return { exit: claimedResult({ status: "idle" }) };
10748
+ if (evaluated.action === "waiting") {
10749
+ control.turnMetricOutcome = "recovering";
10750
+ control.activityStatus = "waiting_capacity";
10751
+ return {
10752
+ exit: claimedResult({
10753
+ status: "waiting_capacity",
10754
+ capacityWait: {
10755
+ waiterId: evaluated.waiter.id,
10756
+ generation: evaluated.waiter.generation,
10757
+ nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
10758
+ wakeRevision: evaluated.waiter.wakeRevision
10759
+ }
10760
+ })
10761
+ };
10762
+ }
10763
+ acknowledgeLostAttemptOwnership();
10764
+ control.turnMetricOutcome = "cancelled";
10765
+ control.activityStatus = "cancelled";
10766
+ return { exit: claimedResult({ status: "cancelled" }) };
10630
10767
  }
10631
10768
  if (rotationDecision.kind === "allCapped" && turnId) {
10632
10769
  if (turn.source === "compaction") {
@@ -10655,88 +10792,64 @@ async function selectCodexTurnCapacity(deps) {
10655
10792
  control.activityStatus = "idle";
10656
10793
  return { exit: claimedResult({ status: "idle", deferredUntilWake: true }) };
10657
10794
  }
10658
- const goal = await getSessionGoal2(db, input.workspaceId, input.sessionId).catch(() => null);
10795
+ const goal = await getSessionGoal2(db, input.workspaceId, input.sessionId);
10659
10796
  const goalActive = Boolean(goal && goal.status === "active");
10660
10797
  const resumeMs = computeIdleDelayMs(
10661
10798
  rotationDecision.earliestResetAt,
10662
10799
  /* @__PURE__ */ new Date(),
10663
10800
  CODEX_USAGE_LIMIT_MAX_RESUME_MS
10664
10801
  );
10802
+ const allConnectedAccountsWereConsidered = leased.rotationEnabled && selectedPinDisposition !== "manual";
10803
+ const capacityDetail = allConnectedAccountsWereConsidered ? "all connected Codex subscriptions are rate-limited" : selectedPinDisposition === "manual" ? "the pinned Codex subscription is rate-limited" : "the active Codex subscription is rate-limited";
10665
10804
  const failurePayload = codexUsageLimitFailurePayload(
10666
10805
  { resetsInSeconds: Math.ceil(resumeMs / 1e3) },
10667
- "all connected Codex subscriptions are rate-limited",
10668
- { allAccounts: true }
10806
+ capacityDetail,
10807
+ { allAccounts: allConnectedAccountsWereConsidered }
10669
10808
  );
10670
10809
  const authoritativeResetAt = authoritativeCodexCapacityResetAt(leased.accounts, /* @__PURE__ */ new Date());
10671
- const armed = await armCodexCapacityWait2(db, {
10672
- accountId: input.accountId,
10673
- workspaceId: input.workspaceId,
10674
- sessionId: input.sessionId,
10675
- turnId,
10676
- attemptId: input.attemptId,
10677
- workflowId: input.workflowId,
10678
- goalId: goalActive && goal ? goal.id : null,
10679
- goalVersion: goalActive && goal ? goal.version : null,
10680
- earliestResetAt: authoritativeResetAt,
10681
- resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
10682
- failurePayload
10683
- });
10684
- if (armed.action === "waiting") {
10685
- await publishDurableSessionEvents5(bus, input.workspaceId, input.sessionId, armed.events);
10810
+ const evaluated = await armAndReconcileCodexCapacityWait(
10811
+ { db, bus },
10812
+ {
10813
+ accountId: input.accountId,
10814
+ workspaceId: input.workspaceId,
10815
+ sessionId: input.sessionId,
10816
+ turnId,
10817
+ attemptId: input.attemptId,
10818
+ workflowId: input.workflowId,
10819
+ goalId: goalActive && goal ? goal.id : null,
10820
+ goalVersion: goalActive && goal ? goal.version : null,
10821
+ earliestResetAt: authoritativeResetAt,
10822
+ resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
10823
+ failurePayload
10824
+ }
10825
+ );
10826
+ if (evaluated.action === "resumed") {
10827
+ control.turnMetricOutcome = "recovering";
10828
+ control.activityStatus = "recovering";
10829
+ return { exit: claimedResult({ status: "recovering" }) };
10830
+ }
10831
+ if (evaluated.action === "waiting") {
10686
10832
  control.turnMetricOutcome = "recovering";
10687
10833
  control.activityStatus = "waiting_capacity";
10688
10834
  return {
10689
10835
  exit: claimedResult({
10690
10836
  status: "waiting_capacity",
10691
10837
  capacityWait: {
10692
- waiterId: armed.waiter.id,
10693
- generation: armed.waiter.generation,
10694
- nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
10695
- wakeRevision: armed.waiter.wakeRevision
10838
+ waiterId: evaluated.waiter.id,
10839
+ generation: evaluated.waiter.generation,
10840
+ nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
10841
+ wakeRevision: evaluated.waiter.wakeRevision
10696
10842
  }
10697
10843
  })
10698
10844
  };
10699
10845
  }
10700
- if (!await eventing.settle({
10701
- events: [
10702
- // `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
10703
- // rotation-wait state as the reactive all-capped path, so it must freeze
10704
- // autoContinuations identically (evaluateGoalContinuation reads this marker)
10705
- // — a goal waiting out a long reset must not burn its continuation budget on
10706
- // the proactive path while the reactive path spares it.
10707
- {
10708
- type: "turn.failed",
10709
- payload: {
10710
- ...failurePayload,
10711
- recovery: goalActive ? "goal_continuation" : "user_message",
10712
- rotated: true
10713
- }
10714
- },
10715
- {
10716
- type: "session.status.changed",
10717
- payload: { status: "idle" }
10718
- }
10719
- ],
10720
- turnStatus: "failed",
10721
- sessionStatus: "idle",
10722
- activeTurnId: null
10723
- })) {
10724
- return { exit: claimedResult({ status: "cancelled" }) };
10725
- }
10726
- control.turnMetricOutcome = "failed";
10727
- control.activityStatus = "idle";
10728
- return {
10729
- exit: claimedResult(
10730
- goalActive ? {
10731
- status: "idle",
10732
- continueDelayMs: resumeMs,
10733
- idleUntilReset: true
10734
- } : { status: "idle" }
10735
- )
10736
- };
10846
+ acknowledgeLostAttemptOwnership();
10847
+ control.turnMetricOutcome = "cancelled";
10848
+ control.activityStatus = "cancelled";
10849
+ return { exit: claimedResult({ status: "cancelled" }) };
10737
10850
  }
10738
10851
  if (providerTurn.effectiveCodexCredentialId) {
10739
- const priorAccountId = sessionCodex?.lastCredentialId ?? null;
10852
+ const priorAccountId = lockedSessionCodexState.lastCredentialId;
10740
10853
  if (priorAccountId !== providerTurn.effectiveCodexCredentialId) {
10741
10854
  await recordSessionActiveCodexCredential(
10742
10855
  db,
@@ -10782,8 +10895,60 @@ async function selectCodexTurnCapacity(deps) {
10782
10895
  }
10783
10896
  } catch (error) {
10784
10897
  credentialSelectionOutcome = "failed";
10785
- throw error;
10786
- } finally {
10898
+ if (error instanceof CodexCredentialLeaseAttemptFencedError) {
10899
+ acknowledgeLostAttemptOwnership();
10900
+ control.activityStatus = "cancelled";
10901
+ control.turnMetricOutcome = "cancelled";
10902
+ return { exit: claimedResult({ status: "cancelled" }) };
10903
+ }
10904
+ if (error instanceof CodexCredentialFailoverExhaustedError) {
10905
+ const settlement = await settleCodexCredentialFailover2(db, {
10906
+ accountId: input.accountId,
10907
+ workspaceId: input.workspaceId,
10908
+ sessionId: input.sessionId,
10909
+ turnId,
10910
+ attemptId: input.attemptId,
10911
+ holderId: null,
10912
+ generation: null,
10913
+ expectedRedispatches: attempt.redispatchesAtDispatch,
10914
+ maxFailovers: error.maxFailovers,
10915
+ recoveryPayload: {
10916
+ triggerEventId: attempt.triggerEventId,
10917
+ reason: "codex_credential_failover_exhausted"
10918
+ },
10919
+ failedPayload: {
10920
+ error: "Automatic Codex credential failover stopped after every bounded account attempt was consumed. Send a new message after checking account health or capacity.",
10921
+ code: "codex_credential_failover_exhausted",
10922
+ retryable: false,
10923
+ recovery: "user_message",
10924
+ failoverCount: error.failoverCount,
10925
+ maxFailovers: error.maxFailovers
10926
+ }
10927
+ });
10928
+ if (settlement.action === "limit_exceeded") {
10929
+ await publishDurableSessionEvents5(
10930
+ bus,
10931
+ input.workspaceId,
10932
+ input.sessionId,
10933
+ settlement.events
10934
+ );
10935
+ control.activityStatus = "idle";
10936
+ control.turnMetricOutcome = "failed";
10937
+ await deliverFailedChildTurnToParent(
10938
+ { db, bus, settings, observability, wakeSessionWorkflow },
10939
+ input.workspaceId,
10940
+ input.sessionId,
10941
+ turnId
10942
+ );
10943
+ return { exit: claimedResult({ status: "idle" }) };
10944
+ }
10945
+ acknowledgeLostAttemptOwnership();
10946
+ control.activityStatus = "cancelled";
10947
+ control.turnMetricOutcome = "cancelled";
10948
+ return { exit: claimedResult({ status: "cancelled" }) };
10949
+ }
10950
+ throw error;
10951
+ } finally {
10787
10952
  recordTurnStartupPhase(observability, {
10788
10953
  phase: "credential_selection",
10789
10954
  provider: "codex-subscription",
@@ -10841,9 +11006,10 @@ import {
10841
11006
  getGeneratedImageArtifact,
10842
11007
  markImageGenerationOperationOutcomeUnknown,
10843
11008
  markImageGenerationOperationRetentionFailed,
10844
- prepareImageGenerationOperation
11009
+ prepareImageGenerationOperation,
11010
+ resetImageGenerationOperationBeforeProviderDispatch
10845
11011
  } from "@opengeni/db";
10846
- import { createHash as createHash8 } from "crypto";
11012
+ import { createHash as createHash9 } from "crypto";
10847
11013
  var ImageGenerationOutcomeUnknownError = class extends Error {
10848
11014
  constructor(operationId) {
10849
11015
  super(
@@ -10866,6 +11032,7 @@ var ImageGenerationRetentionFailedError = class extends Error {
10866
11032
  var imageGenerationOperationPorts = {
10867
11033
  prepare: prepareImageGenerationOperation,
10868
11034
  begin: beginImageGenerationOperation,
11035
+ resetBeforeProviderDispatch: resetImageGenerationOperationBeforeProviderDispatch,
10869
11036
  retain: retainGeneratedImage,
10870
11037
  complete: completeImageGenerationOperation,
10871
11038
  markOutcomeUnknown: markImageGenerationOperationOutcomeUnknown,
@@ -10912,7 +11079,9 @@ async function executeImageGenerationOperation(input, ports = imageGenerationOpe
10912
11079
  accountId: input.accountId,
10913
11080
  workspaceId: input.workspaceId,
10914
11081
  operationId: identity.operationId,
10915
- operationKey: identity.operationKey
11082
+ operationKey: identity.operationKey,
11083
+ providerBindingHash: input.providerBindingHash,
11084
+ expectedArtifactId: identity.artifactId
10916
11085
  });
10917
11086
  if (!begun.started || begun.operation.status !== "provider_started") {
10918
11087
  const recovered = await ports.recover(input, {
@@ -10927,6 +11096,16 @@ async function executeImageGenerationOperation(input, ports = imageGenerationOpe
10927
11096
  try {
10928
11097
  output = await input.generate();
10929
11098
  } catch (error) {
11099
+ if (input.isProviderDispatchRejected?.(error) === true) {
11100
+ await ports.resetBeforeProviderDispatch(input.db, {
11101
+ accountId: input.accountId,
11102
+ workspaceId: input.workspaceId,
11103
+ operationId: identity.operationId,
11104
+ operationKey: identity.operationKey,
11105
+ error: error instanceof Error ? error.message : String(error)
11106
+ }).catch(() => void 0);
11107
+ throw error;
11108
+ }
10930
11109
  await ports.markOutcomeUnknown(input.db, {
10931
11110
  accountId: input.accountId,
10932
11111
  workspaceId: input.workspaceId,
@@ -11043,7 +11222,7 @@ async function recoverCompletedImage(input, identity) {
11043
11222
  return retainedGeneratedImageFromArtifact(recovered).receipt;
11044
11223
  }
11045
11224
  function digest(prefix, ...values) {
11046
- const hash = createHash8("sha256").update(prefix);
11225
+ const hash = createHash9("sha256").update(prefix);
11047
11226
  for (const value of values) hash.update(value).update("\0");
11048
11227
  return hash.digest("hex");
11049
11228
  }
@@ -11058,13 +11237,9 @@ function uuidFromDigest3(digestValue) {
11058
11237
  // src/activities/agent-turn/tool-policy.ts
11059
11238
  function structuredToolTransportForTurn(resolvedModel) {
11060
11239
  if (!resolvedModel) return true;
11061
- if (resolvedModel.provider.api === "chat") return false;
11062
- return ![
11063
- "codex-subscription",
11064
- "xai-subscription",
11065
- "vercel-gateway-managed",
11066
- "vercel-gateway-workspace"
11067
- ].includes(resolvedModel.provider.kind);
11240
+ const provider = resolvedModel.provider;
11241
+ if (provider.api === "chat" || provider.kind === "codex-subscription") return false;
11242
+ return provider.wireProfile === "azure-openai" || provider.builtin && provider.id === "openai" && provider.baseUrl === void 0;
11068
11243
  }
11069
11244
  function lazyToolTransportForTurn(resolvedModel) {
11070
11245
  if (!resolvedModel) return "openai_native";
@@ -12307,905 +12482,39 @@ function createSandboxTurnRuntime(deps) {
12307
12482
  };
12308
12483
  return {
12309
12484
  releaseLateSandbox,
12310
- requireResolvedSandboxForMutation,
12311
- finalizeTurnOpStreamOps,
12312
- onHomeSandboxRebound,
12313
- runWorkspaceMutationForSandbox,
12314
- stopLeaseHeartbeat,
12315
- flushRuntimeBatcher,
12316
- publishSandboxLifecycleEvents,
12317
- publishSandboxLost,
12318
- startLeaseHeartbeat
12319
- };
12320
- }
12321
-
12322
- // src/activities/agent-turn/sandbox-establish.ts
12323
- import {
12324
- getEnrollment,
12325
- getSandbox as getSandbox2,
12326
- readActiveSandbox as readActiveSandbox3,
12327
- getLiveEnrollmentConnection as getLiveEnrollmentConnection2,
12328
- assertPersonalMachineForAttempt
12329
- } from "@opengeni/db";
12330
- import {
12331
- withRunCredentialsSession as withRunCredentialsSession2,
12332
- runOwnedSandboxSetup as runOwnedSandboxSetup2,
12333
- markModelPreparationFirstSandboxOperation,
12334
- recordModelPreparationMeasurement,
12335
- sdkBackendIdForSandboxBackend,
12336
- resolveConnectedMachineWorkspaceRoot as resolveConnectedMachineWorkspaceRoot2
12337
- } from "@opengeni/runtime";
12338
- import { managedSessionGroupBackend, managedSessionGroupOs } from "@opengeni/core";
12339
-
12340
- // src/sandbox-routing.ts
12341
- import { sandboxLifecycleTransitionWaitMs as sandboxLifecycleTransitionWaitMs2 } from "@opengeni/config";
12342
- import {
12343
- adoptConnectedMachineSessionBackgroundCommand,
12344
- advanceWorkspaceGenerationForRetainedProcess,
12345
- advanceWorkspaceGeneration as advanceWorkspaceGeneration2,
12346
- getRetainedProcess,
12347
- retainWorkspaceMutationProcess,
12348
- retainedProcessSettlementIdentity,
12349
- settleRetainedProcess,
12350
- verifyRetainedProcessMutationSettlement,
12351
- verifyWorkspaceMutationSettlement as verifyWorkspaceMutationSettlement2,
12352
- getSandbox,
12353
- getLiveEnrollmentConnection,
12354
- markWarmLeaseInstanceLost,
12355
- readLease as readLease3,
12356
- readActiveSandbox,
12357
- resolvePersonalMachineConnectionForAttempt,
12358
- SandboxRetainedProcessPromotionFencedError
12359
- } from "@opengeni/db";
12360
- import {
12361
- settleConnectedMachineSessionBackgroundCommand,
12362
- settleSessionBackgroundCommandForRetainedProcess
12363
- } from "@opengeni/db/session-background-commands";
12364
- import {
12365
- buildSelfhostedBackendSession,
12366
- ActiveBackendUnresolvableError,
12367
- establishSandboxSessionFromEnvelope,
12368
- isProviderSandboxGoneDuringRoutedOperation,
12369
- isProviderSandboxNotFoundError,
12370
- makeActiveBackendResolver,
12371
- NatsControlRpc,
12372
- NatsOpStreamTransport,
12373
- RoutingBackendRecoveryRequiredError,
12374
- RoutingSandboxSession,
12375
- resolveModalCheckpointProviderBindingForSession,
12376
- resolveConnectedMachineWorkspaceRoot,
12377
- sandboxProviderInstanceIdFromEnvelope,
12378
- sandboxBackendForSdkBackendId,
12379
- selectBackend,
12380
- verifySandboxExecReadiness
12381
- } from "@opengeni/runtime";
12382
- function relayConfigFromSettings(settings) {
12383
- const raw = settings.selfhostedRelayUrl?.trim();
12384
- if (!raw) {
12385
- return { host: "relay.opengeni.local", port: 443, tls: true };
12386
- }
12387
- try {
12388
- const url = new URL(raw.includes("://") ? raw : `wss://${raw}`);
12389
- const tls = url.protocol === "wss:" || url.protocol === "https:";
12390
- const port = url.port ? Number(url.port) : tls ? 443 : 80;
12391
- return { host: url.hostname, port, tls };
12392
- } catch {
12393
- return { host: raw, port: 443, tls: true };
12394
- }
12395
- }
12396
- function selfhostedTimeoutsFromSettings(settings) {
12397
- return {
12398
- timeoutMs: settings.sandboxSelfhostedControlTimeoutMs,
12399
- execTimeoutMs: settings.sandboxSelfhostedExecTimeoutMs
12400
- };
12401
- }
12402
- function selfhostedResolverTimeouts(settings) {
12403
- const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
12404
- return {
12405
- selfhostedTimeoutMs: timeoutMs,
12406
- selfhostedExecTimeoutMs: execTimeoutMs
12407
- };
12408
- }
12409
- function homeRouteRecoveryDisposition(lease) {
12410
- if (!lease) return "unrecoverable";
12411
- if (lease.liveness === "warming") return "pending";
12412
- if (lease.liveness === "draining") return "superseded";
12413
- if (lease.recovery.restore.status === "pending" || lease.recovery.restore.status === "restoring" || lease.recovery.restore.status === "verifying") {
12414
- return "pending";
12415
- }
12416
- if (lease.recovery.restore.status === "degraded" || lease.recovery.workspace.status === "degraded" || lease.recovery.archive.status === "unverified" || lease.recovery.archive.status === "invalid") {
12417
- return "degraded";
12418
- }
12419
- return "unrecoverable";
12420
- }
12421
- function homeRouteRecoveryError(lease, fallbackEpoch) {
12422
- return new RoutingBackendRecoveryRequiredError(
12423
- "resolve_home_backend",
12424
- lease?.leaseEpoch ?? fallbackEpoch,
12425
- homeRouteRecoveryDisposition(lease)
12426
- );
12427
- }
12428
- function providerIdentityFromResumeState(resumeState) {
12429
- return sandboxProviderInstanceIdFromEnvelope(resumeState);
12430
- }
12431
- async function resolveCurrentHomeBackend(services, ids, established) {
12432
- const lease = await readLease3(services.db, ids.workspaceId, ids.sandboxGroupId);
12433
- const fallbackEpoch = lease?.leaseEpoch ?? 0;
12434
- if (!lease || lease.liveness !== "warm" || lease.recovery.provider.status !== "exists" || lease.instanceId === null || lease.recovery.provider.instanceId !== lease.instanceId || lease.recovery.workspace.status !== "ready" || lease.recovery.restore.status !== "not_required" && lease.recovery.restore.status !== "ready") {
12435
- throw homeRouteRecoveryError(lease, fallbackEpoch);
12436
- }
12437
- if (lease.instanceId === established.instanceId) {
12438
- services.onHomeSandboxRebound?.({
12439
- established,
12440
- leaseEpoch: lease.leaseEpoch
12441
- });
12442
- return {
12443
- session: established.session,
12444
- sandboxId: null,
12445
- kind: established.backendId,
12446
- leaseEpoch: lease.leaseEpoch,
12447
- providerInstanceId: lease.instanceId
12448
- };
12449
- }
12450
- const resumeState = lease.resumeState;
12451
- const durableResumeBackend = lease.resumeBackendId ?? lease.backend ?? ids.backend;
12452
- const resumeBackend = sandboxBackendForSdkBackendId(durableResumeBackend) ?? durableResumeBackend;
12453
- if (!resumeState || resumeBackend !== ids.backend || providerIdentityFromResumeState(resumeState) !== lease.instanceId) {
12454
- throw homeRouteRecoveryError(lease, lease.leaseEpoch);
12455
- }
12456
- let rebound;
12457
- try {
12458
- rebound = await establishSandboxSessionFromEnvelope(services.settings, resumeState, {
12459
- sessionId: ids.sessionId,
12460
- recovery: "resume-only",
12461
- backendOverride: resumeBackend
12462
- });
12463
- await verifySandboxExecReadiness(rebound);
12464
- } catch (error) {
12465
- if (!isProviderSandboxNotFoundError(resumeBackend, error)) {
12466
- throw new RoutingBackendRecoveryRequiredError(
12467
- "resolve_home_backend",
12468
- lease.leaseEpoch,
12469
- "pending"
12470
- );
12471
- }
12472
- const marked = await markWarmLeaseInstanceLost(services.db, {
12473
- accountId: ids.accountId,
12474
- workspaceId: ids.workspaceId,
12475
- sandboxGroupId: ids.sandboxGroupId,
12476
- expectedEpoch: lease.leaseEpoch,
12477
- expectedInstanceId: lease.instanceId,
12478
- diagnostic: "provider_not_found_during_home_route_rebind"
12479
- });
12480
- if (marked.status === "marked") {
12481
- await services.onHomeSandboxLost?.({
12482
- sandboxGroupId: ids.sandboxGroupId,
12483
- instanceId: lease.instanceId,
12484
- leaseEpoch: marked.lease.leaseEpoch
12485
- });
12486
- }
12487
- throw new RoutingBackendRecoveryRequiredError(
12488
- "resolve_home_backend",
12489
- marked.lease?.leaseEpoch ?? lease.leaseEpoch,
12490
- marked.status === "stale" ? "superseded" : homeRouteRecoveryDisposition(marked.lease)
12491
- );
12492
- }
12493
- const reboundBackend = sandboxBackendForSdkBackendId(rebound.backendId) ?? rebound.backendId;
12494
- if (rebound.instanceId !== lease.instanceId || reboundBackend !== resumeBackend) {
12495
- throw homeRouteRecoveryError(lease, lease.leaseEpoch);
12496
- }
12497
- services.onHomeSandboxRebound?.({
12498
- established: rebound,
12499
- leaseEpoch: lease.leaseEpoch
12500
- });
12501
- Object.assign(established, rebound);
12502
- return {
12503
- session: rebound.session,
12504
- sandboxId: null,
12505
- kind: rebound.backendId,
12506
- leaseEpoch: lease.leaseEpoch,
12507
- providerInstanceId: lease.instanceId
12508
- };
12509
- }
12510
- function beforePersistableHomeMutation(services, ids, home) {
12511
- const fence = ids.workspaceMutationFence;
12512
- if (!home || !fence) return void 0;
12513
- return async ({ op, backend, onCaptureWait }) => {
12514
- if (backend.sandboxId !== null || backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0) {
12515
- return null;
12516
- }
12517
- const providerBinding = home.backend === "modal" ? await resolveModalCheckpointProviderBindingForSession(services.settings, backend.session) : null;
12518
- const admission = await advanceWorkspaceGeneration2(services.db, {
12519
- accountId: fence.accountId,
12520
- workspaceId: ids.workspaceId,
12521
- sessionId: ids.sessionId,
12522
- turnId: fence.turnId,
12523
- executionGeneration: fence.executionGeneration,
12524
- attemptId: fence.attemptId,
12525
- holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
12526
- sandboxGroupId: home.sandboxGroupId,
12527
- expectedEpoch: backend.leaseEpoch,
12528
- expectedInstanceId: backend.providerInstanceId,
12529
- operation: op,
12530
- captureWaitMs: sandboxLifecycleTransitionWaitMs2(services.settings),
12531
- ...services.waitSignal ? { waitSignal: services.waitSignal } : {},
12532
- ...onCaptureWait ? { onCaptureWait } : {}
12533
- });
12534
- return { admission, providerBinding };
12535
- };
12536
- }
12537
- function afterPersistableHomeMutation(services, ids, home) {
12538
- const fence = ids.workspaceMutationFence;
12539
- if (!home || !fence) return void 0;
12540
- return async ({ op, backend, admission, outcome, retainedProcess }) => {
12541
- if (backend.sandboxId !== null || backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0 || !admission || typeof admission !== "object") {
12542
- return;
12543
- }
12544
- const boundAdmission = admission;
12545
- const exactAdmission = boundAdmission.admission;
12546
- if (!exactAdmission || typeof exactAdmission.id !== "string" || typeof exactAdmission.workspaceGeneration !== "number" || !("providerBinding" in boundAdmission)) {
12547
- throw new Error("Persistable home mutation settlement lacked its bound admission");
12548
- }
12549
- if (outcome === "resolved" && retainedProcess) {
12550
- try {
12551
- await retainWorkspaceMutationProcess(services.db, {
12552
- accountId: fence.accountId,
12553
- workspaceId: ids.workspaceId,
12554
- sessionId: ids.sessionId,
12555
- processId: retainedProcess.id,
12556
- providerSessionId: retainedProcess.providerSessionId,
12557
- admissionId: exactAdmission.id,
12558
- admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
12559
- operation: op,
12560
- providerBinding: boundAdmission.providerBinding ?? null,
12561
- backgroundCommand: {
12562
- commandId: retainedProcess.id,
12563
- command: op
12564
- },
12565
- owner: {
12566
- kind: "turn",
12567
- turnId: fence.turnId,
12568
- executionGeneration: fence.executionGeneration,
12569
- attemptId: fence.attemptId,
12570
- holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
12571
- sandboxGroupId: home.sandboxGroupId,
12572
- expectedEpoch: backend.leaseEpoch,
12573
- expectedInstanceId: backend.providerInstanceId,
12574
- routeKind: exactAdmission.routeKind,
12575
- routeTargetId: exactAdmission.routeTargetId,
12576
- routeEpoch: exactAdmission.routeEpoch
12577
- }
12578
- });
12579
- } catch (error) {
12580
- if (!(error instanceof SandboxRetainedProcessPromotionFencedError)) throw error;
12581
- const durable = error.process;
12582
- if (durable.id !== retainedProcess.id || durable.providerSessionId !== retainedProcess.providerSessionId || durable.sessionId !== ids.sessionId || durable.sandboxGroupId !== home.sandboxGroupId || durable.parentAdmissionId !== exactAdmission.id || durable.leaseEpoch !== backend.leaseEpoch || durable.providerInstanceId !== backend.providerInstanceId || durable.routeKind !== exactAdmission.routeKind || durable.routeTargetId !== exactAdmission.routeTargetId || durable.routeEpoch !== exactAdmission.routeEpoch || durable.state !== "active") {
12583
- throw new Error("Durable retained-process promotion returned a mismatched identity", {
12584
- cause: error
12585
- });
12586
- }
12587
- return {
12588
- status: "retained_process_durable_output_rejected",
12589
- retainedProcess: {
12590
- id: durable.id,
12591
- providerSessionId: durable.providerSessionId
12592
- }
12593
- };
12594
- }
12595
- return;
12596
- }
12597
- await verifyWorkspaceMutationSettlement2(services.db, {
12598
- accountId: fence.accountId,
12599
- workspaceId: ids.workspaceId,
12600
- sessionId: ids.sessionId,
12601
- turnId: fence.turnId,
12602
- executionGeneration: fence.executionGeneration,
12603
- attemptId: fence.attemptId,
12604
- holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
12605
- sandboxGroupId: home.sandboxGroupId,
12606
- expectedEpoch: backend.leaseEpoch,
12607
- expectedInstanceId: backend.providerInstanceId,
12608
- admission: exactAdmission,
12609
- operation: op,
12610
- outcome
12611
- });
12612
- };
12613
- }
12614
- function beforeRetainedProcessMutation(services, ids) {
12615
- const fence = ids.workspaceMutationFence;
12616
- if (!fence) return void 0;
12617
- return async ({ op, process: process2 }) => await advanceWorkspaceGenerationForRetainedProcess(services.db, {
12618
- accountId: fence.accountId,
12619
- workspaceId: ids.workspaceId,
12620
- sessionId: ids.sessionId,
12621
- processId: process2.id,
12622
- operation: op,
12623
- captureWaitMs: sandboxLifecycleTransitionWaitMs2(services.settings),
12624
- ...services.waitSignal ? { waitSignal: services.waitSignal } : {}
12625
- });
12626
- }
12627
- function afterRetainedProcessMutation(services, ids) {
12628
- const fence = ids.workspaceMutationFence;
12629
- if (!fence) return void 0;
12630
- return async ({ op, process: process2, admission, outcome }) => {
12631
- if (!admission || typeof admission !== "object" || typeof admission.id !== "string" || typeof admission.workspaceGeneration !== "number") {
12632
- throw new Error("Retained-process mutation settlement lacked its exact admission");
12633
- }
12634
- await verifyRetainedProcessMutationSettlement(services.db, {
12635
- accountId: fence.accountId,
12636
- workspaceId: ids.workspaceId,
12637
- sessionId: ids.sessionId,
12638
- processId: process2.id,
12639
- admission,
12640
- operation: op,
12641
- outcome
12642
- });
12643
- };
12644
- }
12645
- function settleRetainedProcessForTurn(services, ids) {
12646
- const fence = ids.workspaceMutationFence;
12647
- if (!fence) return void 0;
12648
- return async ({ backend, process: process2, proof }) => {
12649
- const durable = await getRetainedProcess(services.db, {
12650
- workspaceId: ids.workspaceId,
12651
- sessionId: ids.sessionId,
12652
- processId: process2.id
12653
- });
12654
- if (!durable || durable.providerSessionId !== process2.providerSessionId || durable.providerBackend !== backend.kind || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch) {
12655
- throw new Error("Retained-process settlement lost its exact durable backend identity");
12656
- }
12657
- const settlement = await settleRetainedProcess(services.db, {
12658
- accountId: fence.accountId,
12659
- workspaceId: ids.workspaceId,
12660
- sessionId: ids.sessionId,
12661
- processId: process2.id,
12662
- expected: retainedProcessSettlementIdentity(durable),
12663
- outcome: proof.outcome,
12664
- exitCode: proof.exitCode,
12665
- reason: proof.reason,
12666
- idleGraceMs: services.settings.sandboxIdleGraceMs
12667
- });
12668
- if (settlement.process.state === "active") {
12669
- throw new Error("Retained-process settlement returned an active durable process");
12670
- }
12671
- await settleSessionBackgroundCommandForRetainedProcess(services.db, {
12672
- accountId: fence.accountId,
12673
- workspaceId: ids.workspaceId,
12674
- sessionId: ids.sessionId,
12675
- retainedProcessId: process2.id,
12676
- outcome: settlement.process.state,
12677
- exitCode: settlement.process.exitCode,
12678
- reason: settlement.process.settlementReason ?? proof.reason
12679
- });
12680
- };
12681
- }
12682
- function controlRpcFactory(bus) {
12683
- return () => new NatsControlRpc(async () => {
12684
- if (!bus) {
12685
- return null;
12686
- }
12687
- return bus.getRequestConnection();
12688
- });
12689
- }
12690
- function wrapTurnBoxWithRouting(services, ids, established) {
12691
- const { db, settings, bus, onOp } = services;
12692
- const backgroundCommandAccountId = ids.workspaceMutationFence?.accountId;
12693
- const backgroundCommandAttempt = ids.workspaceMutationFence;
12694
- const beforeMutation = beforePersistableHomeMutation(services, ids, ids.homeLease);
12695
- const afterMutation = afterPersistableHomeMutation(services, ids, ids.homeLease);
12696
- const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
12697
- const afterProcessMutation = afterRetainedProcessMutation(services, ids);
12698
- const settleProcess = settleRetainedProcessForTurn(services, ids);
12699
- const resolver = makeActiveBackendResolver({
12700
- workspaceId: ids.workspaceId,
12701
- defaultBackend: established.session,
12702
- defaultKind: established.backendId,
12703
- getSandbox: async (sandboxId) => {
12704
- const sandbox = await getSandbox(
12705
- db,
12706
- ids.resourceAccountId && ids.resourceSubjectId ? {
12707
- accountId: ids.resourceAccountId,
12708
- workspaceId: ids.workspaceId,
12709
- subjectId: ids.resourceSubjectId
12710
- } : ids.workspaceId,
12711
- sandboxId
12712
- );
12713
- return sandbox ? {
12714
- id: sandbox.id,
12715
- workspaceId: sandbox.workspaceId,
12716
- kind: sandbox.kind,
12717
- name: sandbox.name,
12718
- ...sandbox.scope ? { scope: sandbox.scope } : {},
12719
- enrollmentId: sandbox.enrollmentId
12720
- } : null;
12721
- },
12722
- controlRpcFactory: controlRpcFactory(bus),
12723
- resolveSelfhostedConnection: async (sandbox) => {
12724
- if (!sandbox.enrollmentId) return null;
12725
- const enrollment = await resolveRoutedMachineConnection(db, ids, sandbox);
12726
- return connectionBindingFor(services, enrollment);
12727
- },
12728
- relay: relayConfigFromSettings(settings),
12729
- // A selfhosted swap target runs real commands too, so give it the same split
12730
- // deadlines the machine-primary establish path uses (short control, long exec).
12731
- ...selfhostedResolverTimeouts(settings),
12732
- ...onOp !== void 0 ? { selfhostedOnOp: onOp } : {},
12733
- ...backgroundCommandAccountId && backgroundCommandAttempt ? {
12734
- selfhostedAdoptBackgroundCommand: async (command) => {
12735
- const adopted = await adoptConnectedMachineSessionBackgroundCommand(db, {
12736
- accountId: backgroundCommandAccountId,
12737
- workspaceId: ids.workspaceId,
12738
- sessionId: ids.sessionId,
12739
- turnId: backgroundCommandAttempt.turnId,
12740
- executionGeneration: backgroundCommandAttempt.executionGeneration,
12741
- attemptId: backgroundCommandAttempt.attemptId,
12742
- commandId: crypto.randomUUID(),
12743
- controlWorkspaceId: command.controlWorkspaceId,
12744
- enrollmentId: command.enrollmentId,
12745
- connectionInstanceId: command.connectionInstanceId,
12746
- opId: command.opId,
12747
- command: command.command
12748
- });
12749
- return { commandId: adopted.id };
12750
- },
12751
- selfhostedSettleBackgroundCommand: async (command) => {
12752
- await settleConnectedMachineSessionBackgroundCommand(db, {
12753
- accountId: backgroundCommandAccountId,
12754
- workspaceId: ids.workspaceId,
12755
- sessionId: ids.sessionId,
12756
- commandId: command.commandId,
12757
- controlWorkspaceId: command.controlWorkspaceId,
12758
- enrollmentId: command.enrollmentId,
12759
- connectionInstanceId: command.connectionInstanceId,
12760
- opId: command.opId,
12761
- outcome: command.outcome,
12762
- exitCode: command.exitCode,
12763
- reason: command.reason
12764
- });
12765
- }
12766
- } : {},
12767
- // The turn's declared environment → a selfhosted swap target's manifest, so the
12768
- // SDK's per-turn manifest-env delta is empty (no "cannot change manifest
12769
- // environment variables" throw when the turn pins to a vm). Mirrors the group
12770
- // box, which is created WITH this same environment (resumeBoxForTurn).
12771
- ...ids.environment !== void 0 ? { environment: ids.environment } : {},
12772
- ...ids.transientExecEnvironment !== void 0 ? { transientExecEnvironment: ids.transientExecEnvironment } : {},
12773
- // Stage D machine-primary: pin THIS established SelfhostedSession for the machine
12774
- // pointer so the resolver returns the SAME instance (no two-instance manifest
12775
- // divergence). `established.session` is the SelfhostedSession the establish branch
12776
- // bound to (sandboxId, epoch).
12777
- ...ids.pinnedSelfhosted ? {
12778
- pinnedSelfhosted: {
12779
- sandboxId: ids.pinnedSelfhosted.sandboxId,
12780
- epoch: ids.pinnedSelfhosted.epoch,
12781
- session: established.session
12782
- }
12783
- } : {},
12784
- // A modal swap target in the turn path would need its own lease resume-by-id;
12785
- // that is a future cross-group-box concern. Until then a modal swap target is
12786
- // unresolvable (the swap tool validates liveness, so this only triggers if a
12787
- // session points at a sibling modal box the turn cannot resume here) and the
12788
- // op surfaces unresolvable — never a silent wrong-box landing.
12789
- //
12790
- // For a machine-primary turn of a Modal-HOME session (pinned to a machine, no
12791
- // group box established this turn), a mid-turn clear-to-null must NOT fall back to
12792
- // the pinned machine — passing defaultIsHome:false makes the null branch throw typed
12793
- // `home_unavailable_this_turn` instead. Forward the explicit boolean (including false).
12794
- ...ids.defaultIsHome !== void 0 ? { defaultIsHome: ids.defaultIsHome } : {},
12795
- ...ids.homeLease ? {
12796
- resolveDefaultBackend: () => resolveCurrentHomeBackend(
12797
- services,
12798
- {
12799
- workspaceId: ids.workspaceId,
12800
- sessionId: ids.sessionId,
12801
- accountId: ids.homeLease.accountId,
12802
- sandboxGroupId: ids.homeLease.sandboxGroupId,
12803
- backend: ids.homeLease.backend
12804
- },
12805
- established
12806
- )
12807
- } : {}
12808
- });
12809
- const proxy = new RoutingSandboxSession({
12810
- // Seed the DEFAULT backend (the established group box) at construction so
12811
- // `session.state` is the real backend's state object BEFORE the first op. The
12812
- // SDK reads `session.state.manifest` at turn START (and writes it back); an
12813
- // empty `{}` there crashes serializeManifestEnvironment /
12814
- // validateProvidedSessionManifestUpdate. This is byte-identical to what the
12815
- // resolver returns for the default pointer (`activeSandboxId === null`).
12816
- defaultResolved: {
12817
- session: established.session,
12818
- sandboxId: null,
12819
- kind: established.backendId,
12820
- ...ids.homeLease ? {
12821
- leaseEpoch: ids.homeLease.leaseEpoch,
12822
- providerInstanceId: ids.homeLease.instanceId
12823
- } : {}
12824
- },
12825
- readPointer: async () => {
12826
- if (!routingEnabled(settings)) {
12827
- return { activeSandboxId: null, activeEpoch: 0 };
12828
- }
12829
- const pointer = await readActiveSandbox(db, ids.workspaceId, ids.sessionId);
12830
- return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
12831
- },
12832
- resolveActiveBackend: resolver,
12833
- ...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
12834
- ...beforeMutation ? { beforeMutation } : {},
12835
- ...afterMutation ? { afterMutation } : {},
12836
- ...beforeProcessMutation ? { beforeProcessMutation } : {},
12837
- ...afterProcessMutation ? { afterProcessMutation } : {},
12838
- ...settleProcess ? { settleProcess } : {},
12839
- ...ids.homeLease ? {
12840
- onDefaultBackendError: async ({
12841
- error,
12842
- backend
12843
- }) => {
12844
- const home = ids.homeLease;
12845
- if (!isProviderSandboxGoneDuringRoutedOperation(home.backend, error)) return null;
12846
- const expectedEpoch = backend.leaseEpoch ?? home.leaseEpoch;
12847
- const expectedInstanceId = backend.providerInstanceId ?? home.instanceId;
12848
- const marked = await markWarmLeaseInstanceLost(db, {
12849
- accountId: home.accountId,
12850
- workspaceId: ids.workspaceId,
12851
- sandboxGroupId: home.sandboxGroupId,
12852
- expectedEpoch,
12853
- expectedInstanceId,
12854
- diagnostic: "provider_not_found_during_routed_operation"
12855
- });
12856
- if (marked.status === "marked") {
12857
- await services.onHomeSandboxLost?.({
12858
- sandboxGroupId: home.sandboxGroupId,
12859
- instanceId: expectedInstanceId,
12860
- leaseEpoch: marked.lease.leaseEpoch
12861
- });
12862
- }
12863
- const lease = marked.lease;
12864
- const restore = lease?.recovery.restore.status;
12865
- return {
12866
- leaseEpoch: lease?.leaseEpoch ?? expectedEpoch,
12867
- recovery: marked.status === "stale" ? "superseded" : restore === "pending" ? "pending" : restore === "degraded" ? "degraded" : "unrecoverable"
12868
- };
12869
- }
12870
- } : {}
12871
- });
12872
- return { ...established, session: proxy };
12873
- }
12874
- function wrapLazyTurnBoxWithRouting(services, ids, args) {
12875
- const { db, settings, bus, onOp } = services;
12876
- const backgroundCommandAccountId = ids.workspaceMutationFence?.accountId;
12877
- const backgroundCommandAttempt = ids.workspaceMutationFence;
12878
- const beforeMutation = beforePersistableHomeMutation(services, ids, args.homeLeaseIdentity);
12879
- const afterMutation = afterPersistableHomeMutation(services, ids, args.homeLeaseIdentity);
12880
- const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
12881
- const afterProcessMutation = afterRetainedProcessMutation(services, ids);
12882
- const settleProcess = settleRetainedProcessForTurn(services, ids);
12883
- const defaultBackend = sandboxBackendForSdkBackendId(args.backendId);
12884
- const defaultSupportsPty = defaultBackend ? selectBackend(defaultBackend).capabilities.Terminal.pty : false;
12885
- const syntheticSession = {
12886
- state: { manifest: args.agentDefaultManifest },
12887
- // The SDK binds shell tools synchronously before lazy provisioning. Preserve
12888
- // the authoritative descriptor's PTY shape so a Modal home exposes
12889
- // write_stdin on its first turn operation; the async cancellation transport
12890
- // below still resolves the exact live route before executing a command.
12891
- supportsPty: () => defaultSupportsPty
12892
- };
12893
- const routedResolver = makeActiveBackendResolver({
12894
- workspaceId: ids.workspaceId,
12895
- defaultBackend: syntheticSession,
12896
- defaultKind: "unprovisioned",
12897
- getSandbox: async (sandboxId) => {
12898
- const sandbox = await getSandbox(
12899
- db,
12900
- ids.resourceAccountId && ids.resourceSubjectId ? {
12901
- accountId: ids.resourceAccountId,
12902
- workspaceId: ids.workspaceId,
12903
- subjectId: ids.resourceSubjectId
12904
- } : ids.workspaceId,
12905
- sandboxId
12906
- );
12907
- return sandbox ? {
12908
- id: sandbox.id,
12909
- workspaceId: sandbox.workspaceId,
12910
- kind: sandbox.kind,
12911
- name: sandbox.name,
12912
- ...sandbox.scope ? { scope: sandbox.scope } : {},
12913
- enrollmentId: sandbox.enrollmentId
12914
- } : null;
12915
- },
12916
- controlRpcFactory: controlRpcFactory(bus),
12917
- resolveSelfhostedConnection: async (sandbox) => {
12918
- if (!sandbox.enrollmentId) return null;
12919
- const enrollment = await resolveRoutedMachineConnection(db, ids, sandbox);
12920
- return connectionBindingFor(services, enrollment);
12921
- },
12922
- relay: relayConfigFromSettings(settings),
12923
- ...selfhostedResolverTimeouts(settings),
12924
- ...onOp !== void 0 ? { selfhostedOnOp: onOp } : {},
12925
- ...backgroundCommandAccountId && backgroundCommandAttempt ? {
12926
- selfhostedAdoptBackgroundCommand: async (command) => {
12927
- const adopted = await adoptConnectedMachineSessionBackgroundCommand(db, {
12928
- accountId: backgroundCommandAccountId,
12929
- workspaceId: ids.workspaceId,
12930
- sessionId: ids.sessionId,
12931
- turnId: backgroundCommandAttempt.turnId,
12932
- executionGeneration: backgroundCommandAttempt.executionGeneration,
12933
- attemptId: backgroundCommandAttempt.attemptId,
12934
- commandId: crypto.randomUUID(),
12935
- controlWorkspaceId: command.controlWorkspaceId,
12936
- enrollmentId: command.enrollmentId,
12937
- connectionInstanceId: command.connectionInstanceId,
12938
- opId: command.opId,
12939
- command: command.command
12940
- });
12941
- return { commandId: adopted.id };
12942
- },
12943
- selfhostedSettleBackgroundCommand: async (command) => {
12944
- await settleConnectedMachineSessionBackgroundCommand(db, {
12945
- accountId: backgroundCommandAccountId,
12946
- workspaceId: ids.workspaceId,
12947
- sessionId: ids.sessionId,
12948
- commandId: command.commandId,
12949
- controlWorkspaceId: command.controlWorkspaceId,
12950
- enrollmentId: command.enrollmentId,
12951
- connectionInstanceId: command.connectionInstanceId,
12952
- opId: command.opId,
12953
- outcome: command.outcome,
12954
- exitCode: command.exitCode,
12955
- reason: command.reason
12956
- });
12957
- }
12958
- } : {},
12959
- ...ids.environment !== void 0 ? { environment: ids.environment } : {},
12960
- ...ids.transientExecEnvironment !== void 0 ? { transientExecEnvironment: ids.transientExecEnvironment } : {}
12961
- });
12962
- const proxy = new RoutingSandboxSession({
12963
- // Before the first op the SDK reads `state.manifest`; the synthetic backend
12964
- // points at agent.defaultManifest BY REFERENCE so the provided-session delta is
12965
- // empty. The first default-pointer op resolves the real box through the
12966
- // provisioner and `state` switches to that real backend by reference.
12967
- defaultResolved: {
12968
- session: syntheticSession,
12969
- sandboxId: null,
12970
- kind: "unprovisioned"
12971
- },
12972
- readPointer: async () => {
12973
- const pointer = await readActiveSandbox(db, ids.workspaceId, ids.sessionId);
12974
- return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
12975
- },
12976
- resolveActiveBackend: async (pointer) => {
12977
- if (pointer.activeSandboxId === null || !routingEnabled(settings)) {
12978
- const provisioned = await args.provisioner.get();
12979
- if (args.homeLeaseIdentity && provisioned.leaseEpoch !== void 0) {
12980
- return resolveCurrentHomeBackend(
12981
- services,
12982
- {
12983
- workspaceId: ids.workspaceId,
12984
- sessionId: ids.sessionId,
12985
- accountId: args.homeLeaseIdentity.accountId,
12986
- sandboxGroupId: args.homeLeaseIdentity.sandboxGroupId,
12987
- backend: args.homeLeaseIdentity.backend
12988
- },
12989
- provisioned.established
12990
- );
12991
- }
12992
- return {
12993
- session: provisioned.established.session,
12994
- sandboxId: null,
12995
- kind: provisioned.established.backendId
12996
- };
12997
- }
12998
- return routedResolver(pointer);
12999
- },
13000
- ...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
13001
- ...args.onFirstOperation ? { onFirstOperation: args.onFirstOperation } : {},
13002
- ...beforeMutation ? { beforeMutation } : {},
13003
- ...afterMutation ? { afterMutation } : {},
13004
- ...beforeProcessMutation ? { beforeProcessMutation } : {},
13005
- ...afterProcessMutation ? { afterProcessMutation } : {},
13006
- ...settleProcess ? { settleProcess } : {},
13007
- ...args.homeLeaseIdentity ? {
13008
- onDefaultBackendError: async ({
13009
- error,
13010
- backend
13011
- }) => {
13012
- const home = args.homeLeaseIdentity;
13013
- if (!isProviderSandboxGoneDuringRoutedOperation(home.backend, error)) return null;
13014
- if (backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0) {
13015
- return null;
13016
- }
13017
- const marked = await markWarmLeaseInstanceLost(db, {
13018
- accountId: home.accountId,
13019
- workspaceId: ids.workspaceId,
13020
- sandboxGroupId: home.sandboxGroupId,
13021
- expectedEpoch: backend.leaseEpoch,
13022
- expectedInstanceId: backend.providerInstanceId,
13023
- diagnostic: "provider_not_found_during_routed_operation"
13024
- });
13025
- if (marked.status === "marked") {
13026
- await services.onHomeSandboxLost?.({
13027
- sandboxGroupId: home.sandboxGroupId,
13028
- instanceId: backend.providerInstanceId,
13029
- leaseEpoch: marked.lease.leaseEpoch
13030
- });
13031
- }
13032
- const lease = marked.lease;
13033
- const restore = lease?.recovery.restore.status;
13034
- return {
13035
- leaseEpoch: lease?.leaseEpoch ?? backend.leaseEpoch,
13036
- recovery: marked.status === "stale" ? "superseded" : restore === "pending" ? "pending" : restore === "degraded" ? "degraded" : "unrecoverable"
13037
- };
13038
- }
13039
- } : {}
13040
- });
13041
- return {
13042
- client: args.client,
13043
- session: proxy,
13044
- sessionState: void 0,
13045
- instanceId: "unprovisioned",
13046
- backendId: args.backendId
13047
- };
13048
- }
13049
- function opStreamDepsFor(services, machineAdvertisesOpStream) {
13050
- const { settings, bus, opJournal } = services;
13051
- if (!machineAdvertisesOpStream || settings.agentOpStreamEnabled !== true || !bus) {
13052
- return void 0;
13053
- }
13054
- return {
13055
- transport: new NatsOpStreamTransport(async () => bus.getOpStreamConnection?.() ?? null),
13056
- ...opJournal !== void 0 ? { journal: opJournal } : {}
13057
- };
13058
- }
13059
- function connectionBindingFor(services, enrollment) {
13060
- if (!enrollment?.connectionInstanceId) return null;
13061
- const opStream = opStreamDepsFor(services, enrollment.opStream === true);
13062
- return {
13063
- workspaceId: enrollment.workspaceId,
13064
- connectionInstanceId: enrollment.connectionInstanceId,
13065
- workspaceRoot: enrollment.workspaceRoot,
13066
- ...opStream !== void 0 ? { opStream } : {},
13067
- operationResourcePolicy: enrollment.operationPolicy,
13068
- operationResourcePolicySupported: enrollment.agentCapabilities.operationResourcePolicy === true,
13069
- operationCpuQuotaSupported: enrollment.agentCapabilities.operationCpuQuota === true
13070
- };
13071
- }
13072
- async function resolveRoutedMachineConnection(db, ids, sandbox) {
13073
- if (!sandbox.enrollmentId) return null;
13074
- if (sandbox.scope !== "user") {
13075
- return await getLiveEnrollmentConnection(
13076
- db,
13077
- ids.resourceAccountId && ids.resourceSubjectId ? {
13078
- accountId: ids.resourceAccountId,
13079
- workspaceId: ids.workspaceId,
13080
- subjectId: ids.resourceSubjectId
13081
- } : sandbox.workspaceId,
13082
- sandbox.enrollmentId
13083
- );
13084
- }
13085
- const authority = ids.personalMachineAttempt;
13086
- if (!authority) return null;
13087
- return await resolvePersonalMachineConnectionForAttempt(db, {
13088
- accountId: authority.accountId,
13089
- workspaceId: ids.workspaceId,
13090
- subjectId: authority.subjectId,
13091
- sessionId: ids.sessionId,
13092
- turnId: authority.turnId,
13093
- attemptId: authority.attemptId,
13094
- executionGeneration: authority.executionGeneration,
13095
- enrollmentId: sandbox.enrollmentId,
13096
- requireActiveSandbox: true
13097
- });
13098
- }
13099
- async function establishSelfhostedTurnSession(services, args) {
13100
- const { db, settings, bus, onOp } = services;
13101
- const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
13102
- const opStream = opStreamDepsFor(services, args.opStream);
13103
- const { client, session } = await buildSelfhostedBackendSession({
13104
- workspaceId: args.workspaceId,
13105
- ...args.controlWorkspaceId !== void 0 ? { controlWorkspaceId: args.controlWorkspaceId } : {},
13106
- agentId: args.agentId,
13107
- connectionInstanceId: args.connectionInstanceId,
13108
- workspaceRoot: args.workspaceRoot,
13109
- relay: relayConfigFromSettings(settings),
13110
- controlRpcFactory: controlRpcFactory(bus),
13111
- epoch: args.epoch,
13112
- environment: args.environment,
13113
- ...args.transientExecEnvironment !== void 0 ? { transientExecEnvironment: args.transientExecEnvironment } : {},
13114
- // Give this turn's exec ops the long deadline (control ops stay short) so a real
13115
- // command is not killed at the control wall.
13116
- timeoutMs,
13117
- execTimeoutMs,
13118
- operationResourcePolicy: args.operationResourcePolicy,
13119
- operationResourcePolicySupported: args.operationResourcePolicySupported,
13120
- operationCpuQuotaSupported: args.operationCpuQuotaSupported,
13121
- resolveOperationAdmission: async () => {
13122
- const enrollment = args.personalMachineAttempt ? await resolvePersonalMachineConnectionForAttempt(db, {
13123
- accountId: args.personalMachineAttempt.accountId,
13124
- workspaceId: args.workspaceId,
13125
- subjectId: args.personalMachineAttempt.subjectId,
13126
- sessionId: args.personalMachineAttempt.sessionId,
13127
- turnId: args.personalMachineAttempt.turnId,
13128
- attemptId: args.personalMachineAttempt.attemptId,
13129
- executionGeneration: args.personalMachineAttempt.executionGeneration,
13130
- enrollmentId: args.agentId,
13131
- requireActiveSandbox: true
13132
- }) : await getLiveEnrollmentConnection(
13133
- db,
13134
- args.controlWorkspaceId ?? args.workspaceId,
13135
- args.agentId
13136
- );
13137
- const binding = connectionBindingFor(services, enrollment);
13138
- if (!binding) return null;
13139
- if (!binding.workspaceRoot) {
13140
- throw new ActiveBackendUnresolvableError(
13141
- "workspace_root_unavailable",
13142
- "Connected Machine no longer has a reported workspace root"
13143
- );
13144
- }
13145
- return {
13146
- ...binding,
13147
- workspaceRoot: resolveConnectedMachineWorkspaceRoot(binding.workspaceRoot, args.workingDir)
13148
- };
13149
- },
13150
- ...args.backgroundCommandAttempt ? {
13151
- adoptBackgroundCommand: async (command) => {
13152
- const adopted = await adoptConnectedMachineSessionBackgroundCommand(db, {
13153
- accountId: args.accountId,
13154
- workspaceId: args.workspaceId,
13155
- sessionId: args.sessionId,
13156
- turnId: args.backgroundCommandAttempt.turnId,
13157
- executionGeneration: args.backgroundCommandAttempt.executionGeneration,
13158
- attemptId: args.backgroundCommandAttempt.attemptId,
13159
- commandId: crypto.randomUUID(),
13160
- controlWorkspaceId: command.controlWorkspaceId,
13161
- enrollmentId: command.enrollmentId,
13162
- connectionInstanceId: command.connectionInstanceId,
13163
- opId: command.opId,
13164
- command: command.command
13165
- });
13166
- return { commandId: adopted.id };
13167
- }
13168
- } : {},
13169
- settleBackgroundCommand: async (command) => {
13170
- await settleConnectedMachineSessionBackgroundCommand(db, {
13171
- accountId: args.accountId,
13172
- workspaceId: args.workspaceId,
13173
- sessionId: args.sessionId,
13174
- commandId: command.commandId,
13175
- controlWorkspaceId: command.controlWorkspaceId,
13176
- enrollmentId: command.enrollmentId,
13177
- connectionInstanceId: command.connectionInstanceId,
13178
- opId: command.opId,
13179
- outcome: command.outcome,
13180
- exitCode: command.exitCode,
13181
- reason: command.reason
13182
- });
13183
- },
13184
- // Meter every control op (out-of-band telemetry) — no-op when unwired.
13185
- ...onOp !== void 0 ? { onOp } : {},
13186
- // The streaming exec transport — present iff the machine advertised the
13187
- // capability AND the server flag is on. Connected Machine exec fails closed
13188
- // when it is absent; there is no request/reply downgrade.
13189
- ...opStream !== void 0 ? { opStream } : {}
13190
- });
13191
- return {
13192
- client,
13193
- session,
13194
- sessionState: { agentId: args.agentId },
13195
- instanceId: args.agentId,
13196
- backendId: "selfhosted"
12485
+ requireResolvedSandboxForMutation,
12486
+ finalizeTurnOpStreamOps,
12487
+ onHomeSandboxRebound,
12488
+ runWorkspaceMutationForSandbox,
12489
+ stopLeaseHeartbeat,
12490
+ flushRuntimeBatcher,
12491
+ publishSandboxLifecycleEvents,
12492
+ publishSandboxLost,
12493
+ startLeaseHeartbeat
13197
12494
  };
13198
12495
  }
13199
- function routingEnabled(settings) {
13200
- return settings.sandboxSelfhostedEnabled === true;
13201
- }
13202
- function lazyProvisionEnabled(settings) {
13203
- return settings.sandboxLazyProvisionEnabled === true && settings.sandboxOwnershipEnabled === true;
13204
- }
13205
12496
 
13206
- // src/activities/agent-turn/sandbox-route.ts
12497
+ // src/activities/agent-turn/sandbox-establish.ts
13207
12498
  import {
12499
+ getEnrollment,
12500
+ getSandbox,
13208
12501
  readActiveSandbox as readActiveSandbox2,
12502
+ getLiveEnrollmentConnection,
12503
+ assertPersonalMachineForAttempt
12504
+ } from "@opengeni/db";
12505
+ import {
12506
+ withRunCredentialsSession as withRunCredentialsSession2,
12507
+ runOwnedSandboxSetup as runOwnedSandboxSetup2,
12508
+ markModelPreparationFirstSandboxOperation,
12509
+ recordModelPreparationMeasurement,
12510
+ sdkBackendIdForSandboxBackend,
12511
+ resolveConnectedMachineWorkspaceRoot
12512
+ } from "@opengeni/runtime";
12513
+ import { managedSessionGroupBackend, managedSessionGroupOs } from "@opengeni/core";
12514
+
12515
+ // src/activities/agent-turn/sandbox-route.ts
12516
+ import {
12517
+ readActiveSandbox,
13209
12518
  setActiveSandbox
13210
12519
  } from "@opengeni/db";
13211
12520
  import {
@@ -13318,7 +12627,7 @@ async function reconcileActiveSandboxPointer(db, ids, pointer, loadRecord, publi
13318
12627
  ).catch(() => void 0);
13319
12628
  return { pointer: reset.pointer, record: null };
13320
12629
  }
13321
- const reread = await readActiveSandbox2(db, ids.workspaceId, ids.sessionId).catch(() => null);
12630
+ const reread = await readActiveSandbox(db, ids.workspaceId, ids.sessionId).catch(() => null);
13322
12631
  if (!reread) {
13323
12632
  return { pointer, record: null };
13324
12633
  }
@@ -13387,7 +12696,7 @@ async function resolveSandboxRoute(deps) {
13387
12696
  logicalSandboxSettings
13388
12697
  } = deps;
13389
12698
  const routingOn = routingEnabled(settings);
13390
- let activeSandboxPointer = routingOn ? await readActiveSandbox3(db, input.workspaceId, input.sessionId).catch(() => null) : null;
12699
+ let activeSandboxPointer = routingOn ? await readActiveSandbox2(db, input.workspaceId, input.sessionId).catch(() => null) : null;
13391
12700
  let activeSandboxRecord = null;
13392
12701
  if (routingOn) {
13393
12702
  const reconciled = await reconcileActiveSandboxPointer(
@@ -13398,7 +12707,7 @@ async function resolveSandboxRoute(deps) {
13398
12707
  sessionId: input.sessionId
13399
12708
  },
13400
12709
  activeSandboxPointer,
13401
- (sandboxId) => getSandbox2(
12710
+ (sandboxId) => getSandbox(
13402
12711
  db,
13403
12712
  fileAuthoritySubjectId ? {
13404
12713
  accountId: input.accountId,
@@ -13541,7 +12850,7 @@ async function establishTurnSandbox(deps) {
13541
12850
  throw new Error("personal Connected Machine use was not admitted for this turn");
13542
12851
  }
13543
12852
  }
13544
- const liveMachineEnrollment = await getLiveEnrollmentConnection2(
12853
+ const liveMachineEnrollment = await getLiveEnrollmentConnection(
13545
12854
  db,
13546
12855
  activeSandboxRecord.workspaceId,
13547
12856
  activeSandboxRecord.enrollmentId
@@ -13556,7 +12865,7 @@ async function establishTurnSandbox(deps) {
13556
12865
  "Connected Machine has not reported an absolute workspace root; reconnect it with a current agent"
13557
12866
  );
13558
12867
  }
13559
- const machineWorkspaceRoot = resolveConnectedMachineWorkspaceRoot2(
12868
+ const machineWorkspaceRoot = resolveConnectedMachineWorkspaceRoot(
13560
12869
  machineEnrollment.workspaceRoot,
13561
12870
  activeSandboxPointer.workingDir
13562
12871
  );
@@ -14281,6 +13590,16 @@ async function selectXaiTurnCapacity(deps) {
14281
13590
  authoritySnapshot
14282
13591
  });
14283
13592
  const leaseStartedAtMs = performance.now();
13593
+ await refreshExhaustedXaiQuota({
13594
+ db,
13595
+ settings: deps.settings,
13596
+ accountId: input.accountId,
13597
+ workspaceId: input.workspaceId,
13598
+ subjectId,
13599
+ sessionId: input.sessionId,
13600
+ turnId: turn.id,
13601
+ authoritySnapshot
13602
+ });
14284
13603
  const leased = await acquireXaiCredentialLease(db, {
14285
13604
  accountId: input.accountId,
14286
13605
  workspaceId: input.workspaceId,
@@ -15354,6 +14673,8 @@ async function prepareRunCredentials(deps) {
15354
14673
 
15355
14674
  // src/activities/agent-turn/tool-environment.ts
15356
14675
  import {
14676
+ beginConnectorActionExecution,
14677
+ completeConnectorActionExecution,
15357
14678
  getScheduledVariableSetExpectedGenerationForAttempt,
15358
14679
  getWorkspaceModelPolicy as getWorkspaceModelPolicy2,
15359
14680
  listWorkspaceGatewayCustomModels as listWorkspaceGatewayCustomModels2,
@@ -15361,6 +14682,8 @@ import {
15361
14682
  listOrganizationModelProviderCustomModelsForWorkspace as listOrganizationModelProviderCustomModelsForWorkspace2,
15362
14683
  organizationModelProviderConnectionActiveForWorkspace,
15363
14684
  persistAttemptToolCatalog,
14685
+ prepareConnectorActionApproval,
14686
+ previewConnectorActionApproval,
15364
14687
  namedSubjectHasLiveWorkspaceAuthority as namedSubjectHasLiveWorkspaceAuthority2,
15365
14688
  updateSessionTitleWithEvent as updateSessionTitleWithEvent2,
15366
14689
  withCodexAppsRequestAuthorization,
@@ -15375,8 +14698,7 @@ import {
15375
14698
  } from "@opengeni/runtime";
15376
14699
 
15377
14700
  // src/activities/google-drive-publication.ts
15378
- import { createHash as createHash9 } from "crypto";
15379
- import { AttemptToolApprovalRequiredError } from "@opengeni/codemode";
14701
+ import { createHash as createHash10 } from "crypto";
15380
14702
  import {
15381
14703
  GOOGLE_DRIVE_FILE_SCOPE,
15382
14704
  GOOGLE_DRIVE_FULL_SCOPE,
@@ -15390,10 +14712,7 @@ import {
15390
14712
  googleDriveScopesAllowCapability
15391
14713
  } from "@opengeni/contracts/google-drive";
15392
14714
  import {
15393
- beginConnectorActionExecution,
15394
- completeConnectorActionExecution,
15395
14715
  getConnectionMetadata,
15396
- prepareConnectorActionApproval,
15397
14716
  requireFile as requireFile2,
15398
14717
  namedSubjectHasLiveWorkspaceAuthority
15399
14718
  } from "@opengeni/db";
@@ -15401,6 +14720,7 @@ import {
15401
14720
  readAuthorizedEditableArtifactMaterialization
15402
14721
  } from "@opengeni/db/editable-artifact-durable-export";
15403
14722
  import { readResponseJsonBounded, undiciFetch } from "@opengeni/network";
14723
+ import { ConnectorActionExecutionError } from "@opengeni/runtime";
15404
14724
  import { retryWhileMissing as retryWhileMissing4 } from "@opengeni/storage";
15405
14725
  var DRIVE_API = "https://www.googleapis.com/drive/v3";
15406
14726
  var DRIVE_UPLOAD_API = "https://www.googleapis.com/upload/drive/v3";
@@ -15416,9 +14736,6 @@ var defaultPorts = {
15416
14736
  getMembership: namedSubjectHasLiveWorkspaceAuthority,
15417
14737
  readMaterialization: readAuthorizedEditableArtifactMaterialization,
15418
14738
  requireFile: requireFile2,
15419
- prepare: prepareConnectorActionApproval,
15420
- begin: beginConnectorActionExecution,
15421
- complete: completeConnectorActionExecution,
15422
14739
  fetch: undiciFetch
15423
14740
  };
15424
14741
  async function resolveGoogleDrivePublicationTarget(db, workspace, delegations, ports = defaultPorts) {
@@ -15481,7 +14798,7 @@ function googleDrivePublicationConnectorCall(target, rawRequest, approvalId) {
15481
14798
  sourceSha256: request.file.sha256
15482
14799
  },
15483
14800
  canonicalReceipt: { ...request.file },
15484
- idempotencyKeyDigest: createHash9("sha256").update(request.idempotencyKey, "utf8").digest("hex")
14801
+ idempotencyKeyDigest: createHash10("sha256").update(request.idempotencyKey, "utf8").digest("hex")
15485
14802
  }
15486
14803
  };
15487
14804
  }
@@ -15589,24 +14906,6 @@ function createGoogleDrivePublicationAttemptTool(input) {
15589
14906
  approval: "policy",
15590
14907
  execute: async (raw, context) => {
15591
14908
  const request = GoogleDrivePublicationToolInput.parse(raw);
15592
- const connectorCall = googleDrivePublicationConnectorCall(
15593
- input.target,
15594
- request,
15595
- context.operationId
15596
- );
15597
- let requestId = null;
15598
- if (context.caller.kind === "codemode") {
15599
- const admission = await ports.begin(input.db, input.identity, connectorCall);
15600
- if (!admission.allowed) {
15601
- throw new Error(
15602
- admission.reason === "uncertain_retry" || admission.reason === "already_executed" ? "Google Drive publication outcome is unknown: a previous attempt may have already sent this publication to Google Drive. Verify in Drive before retrying with a new idempotency key." : admission.reason === "not_executed" ? "Google Drive publication was not executed: the previous attempt failed before any request reached Google Drive. Retrying with a new call is safe." : `Google Drive publication was not executed: ${admission.reason}. No request reached Google Drive.`
15603
- );
15604
- }
15605
- if (!admission.managed) {
15606
- throw new Error("Google Drive publication has no explicit connector action policy");
15607
- }
15608
- requestId = admission.requestId;
15609
- }
15610
14909
  let providerRequestStarted = false;
15611
14910
  try {
15612
14911
  const receipt = await executeGoogleDrivePublication(
@@ -15625,53 +14924,21 @@ function createGoogleDrivePublicationAttemptTool(input) {
15625
14924
  },
15626
14925
  ports
15627
14926
  );
15628
- if (requestId) {
15629
- await ports.complete(input.db, {
15630
- accountId: input.identity.accountId,
15631
- workspaceId: input.identity.workspaceId,
15632
- requestId,
15633
- attemptId: input.identity.attemptId,
15634
- outcome: "completed"
15635
- });
15636
- }
15637
14927
  return {
15638
14928
  content: [{ type: "text", text: JSON.stringify(receipt) }],
15639
14929
  structuredContent: receipt
15640
14930
  };
15641
14931
  } catch (error) {
15642
- if (requestId) {
15643
- await ports.complete(input.db, {
15644
- accountId: input.identity.accountId,
15645
- workspaceId: input.identity.workspaceId,
15646
- requestId,
15647
- attemptId: input.identity.attemptId,
15648
- outcome: providerRequestStarted ? "uncertain" : "not_executed"
15649
- });
15650
- }
15651
14932
  const detail = error instanceof Error ? error.message : String(error);
15652
- throw new Error(
14933
+ throw new ConnectorActionExecutionError(
15653
14934
  providerRequestStarted ? `Google Drive publication outcome is unknown: a provider request may have been sent before the failure. Verify in Drive before retrying. Cause: ${detail}` : `Google Drive publication was not executed: no request reached Google Drive. Cause: ${detail}`,
14935
+ providerRequestStarted ? "uncertain" : "not_executed",
15654
14936
  { cause: error }
15655
14937
  );
15656
14938
  }
15657
14939
  }
15658
14940
  };
15659
14941
  }
15660
- async function authorizeGoogleDrivePublicationAttempt(input) {
15661
- const request = GoogleDrivePublicationToolInput.parse(input.arguments);
15662
- const preparation = await (input.ports ?? defaultPorts).prepare(
15663
- input.db,
15664
- input.identity,
15665
- googleDrivePublicationConnectorCall(input.target, request, input.approvalId)
15666
- );
15667
- if (!preparation.managed) {
15668
- throw new Error("Google Drive publication has no explicit connector action policy");
15669
- }
15670
- if (preparation.decision === "ask") throw new AttemptToolApprovalRequiredError();
15671
- if (preparation.decision === "block") {
15672
- throw new Error("Google Drive publication is blocked by connector action policy");
15673
- }
15674
- }
15675
14942
  async function executeGoogleDrivePublication(input, ports = defaultPorts) {
15676
14943
  const request = GoogleDrivePublicationToolInput.parse(input.request);
15677
14944
  const connection = await ports.getConnection(
@@ -15719,7 +14986,7 @@ async function executeGoogleDrivePublication(input, ports = defaultPorts) {
15719
14986
  );
15720
14987
  if (!object) throw new Error("Editable artifact export object is missing");
15721
14988
  const bytes = object.bytes;
15722
- if (bytes.byteLength !== request.file.sizeBytes || createHash9("sha256").update(bytes).digest("hex") !== request.file.sha256) {
14989
+ if (bytes.byteLength !== request.file.sizeBytes || createHash10("sha256").update(bytes).digest("hex") !== request.file.sha256) {
15723
14990
  throw new Error("Editable artifact export bytes failed verification");
15724
14991
  }
15725
14992
  const authorizedFetch = async (url, init) => {
@@ -15801,7 +15068,7 @@ function assertPublicationConnection(input, connection) {
15801
15068
  }
15802
15069
  }
15803
15070
  function publicationActorReplicaId(identity) {
15804
- const value = createHash9("sha256").update(
15071
+ const value = createHash10("sha256").update(
15805
15072
  `google-drive-publication:${identity.sessionId}:${identity.turnId}:${identity.attemptId}:${identity.executionGeneration}`,
15806
15073
  "utf8"
15807
15074
  ).digest("hex").slice(0, 16);
@@ -15846,7 +15113,7 @@ function officeFormat(modality, contentType) {
15846
15113
  return expected.format;
15847
15114
  }
15848
15115
  function deterministicUuid(seed) {
15849
- const bytes = createHash9("sha256").update(seed, "utf8").digest().subarray(0, 16);
15116
+ const bytes = createHash10("sha256").update(seed, "utf8").digest().subarray(0, 16);
15850
15117
  bytes[6] = (bytes[6] ?? 0) & 15 | 80;
15851
15118
  bytes[8] = (bytes[8] ?? 0) & 63 | 128;
15852
15119
  const hex = bytes.toString("hex");
@@ -15984,7 +15251,7 @@ function driveReceipt(input, file, replayed) {
15984
15251
  });
15985
15252
  }
15986
15253
  function publicationOperationKey(input) {
15987
- return createHash9("sha256").update(
15254
+ return createHash10("sha256").update(
15988
15255
  JSON.stringify({
15989
15256
  workspaceId: input.workspaceId,
15990
15257
  connectionId: input.connectionId,
@@ -16202,100 +15469,6 @@ function isGitHubApiDestination(value) {
16202
15469
  }
16203
15470
  }
16204
15471
 
16205
- // src/activities/api-integrations.ts
16206
- import {
16207
- createGraphqlMcpServer,
16208
- createOpenApiMcpServer,
16209
- createPinnedIntegrationTransport
16210
- } from "@opengeni/capabilities";
16211
- function buildApiIntegrationServersForTurn(input) {
16212
- const transport = createPinnedIntegrationTransport({
16213
- network: input.settings,
16214
- ...input.fetchImpl ? { fetchImpl: input.fetchImpl } : {}
16215
- });
16216
- return input.integrations.map((integration) => {
16217
- const authority = {
16218
- ...input.authority,
16219
- ...integration.connectionRef?.connectionId ? { connectionRef: integration.connectionRef.connectionId } : {}
16220
- };
16221
- const credentialResolver = integration.connectionRef ? integrationCredentialResolver(input, integration) : void 0;
16222
- const server = integration.revision.protocol === "openapi" ? createOpenApiMcpServer({
16223
- revision: integration.revision,
16224
- transport,
16225
- authority,
16226
- ...credentialResolver ? { credentialResolver } : {}
16227
- }) : createGraphqlMcpServer({
16228
- revision: integration.revision,
16229
- endpoint: integration.baseUrl,
16230
- transport,
16231
- authority,
16232
- ...credentialResolver ? { credentialResolver } : {}
16233
- });
16234
- return {
16235
- id: integration.serverId,
16236
- server,
16237
- ...integration.connectionRef?.connectionId ? { resolvedConnectionId: integration.connectionRef.connectionId } : {}
16238
- };
16239
- });
16240
- }
16241
- function integrationCredentialResolver(input, integration) {
16242
- const connectionRef = integration.connectionRef;
16243
- if (!connectionRef) throw new Error("Integration credential resolver requires a connection");
16244
- return {
16245
- resolve: async (request) => {
16246
- const result = await input.resolveCredential({
16247
- workspaceId: input.authority.workspaceId,
16248
- serverId: integration.serverId,
16249
- toolName: request.operationKey,
16250
- connectionRef,
16251
- destinationUrl: request.destinationUrl,
16252
- credentialTarget: "http_api",
16253
- forceRefresh: request.forceRefresh === true
16254
- });
16255
- if (result.status === "auth_needed") {
16256
- await publishAuthNeeded(
16257
- input,
16258
- integration.serverId,
16259
- request.operationKey,
16260
- result,
16261
- connectionRef
16262
- );
16263
- return null;
16264
- }
16265
- const destination = new URL(request.destinationUrl);
16266
- return {
16267
- audience: { origin: destination.origin, pathPrefix: "/" },
16268
- placements: result.placements ?? Object.entries(result.headers).map(([name, value]) => ({
16269
- carrier: "header",
16270
- name,
16271
- value
16272
- })),
16273
- ...result.authorizeProviderRequest ? { authorizeProviderRequest: result.authorizeProviderRequest } : {},
16274
- ...result.expiresAt ? { expiresAt: result.expiresAt.toISOString() } : {},
16275
- ...connectionRef.scopes ? { scope: [...connectionRef.scopes] } : {}
16276
- };
16277
- }
16278
- };
16279
- }
16280
- async function publishAuthNeeded(input, serverId, toolName, result, connectionRef) {
16281
- try {
16282
- await input.onAuthNeeded?.({
16283
- serverId,
16284
- toolName,
16285
- providerDomain: result.providerDomain,
16286
- ...result.provider ? { provider: result.provider } : {},
16287
- reason: result.reason,
16288
- ...result.connectionId ? { connectionId: result.connectionId } : {},
16289
- ...result.authoritySource === "host" || connectionRef.authoritySource === "host" ? { authoritySource: "host" } : {},
16290
- ...result.scopes ? { scopes: result.scopes } : {},
16291
- ...result.resource ? { resource: result.resource } : {},
16292
- ...result.selectedResources ? { selectedResources: result.selectedResources } : {},
16293
- ...result.authorizationUrl ? { authorizationUrl: result.authorizationUrl } : {}
16294
- });
16295
- } catch {
16296
- }
16297
- }
16298
-
16299
15472
  // src/github-rest-mcp.ts
16300
15473
  import {
16301
15474
  mergeToolRefs
@@ -16306,18 +15479,12 @@ import {
16306
15479
  } from "@opengeni/contracts/personal-github";
16307
15480
  import { personalGitHubRepositoryResources as personalGitHubRepositoryResources2 } from "@opengeni/core";
16308
15481
  import {
16309
- beginConnectorActionExecution as beginConnectorActionExecution2,
16310
- completeConnectorActionExecution as completeConnectorActionExecution2,
16311
15482
  getConnectionMetadata as getConnectionMetadata2,
16312
15483
  getPersonalGitHubRepositorySelectionState,
16313
15484
  hasAuditableGitHubInstallationAuthority,
16314
15485
  listGitHubInstallationAccessForWorkspace,
16315
- prepareConnectorActionApproval as prepareConnectorActionApproval2,
16316
15486
  resolveAcceptedConnectionUse as resolveAcceptedConnectionUse2
16317
15487
  } from "@opengeni/db";
16318
- import {
16319
- AttemptToolApprovalRequiredError as AttemptToolApprovalRequiredError2
16320
- } from "@opengeni/codemode";
16321
15488
  import { createGitHubAppInstallationTokenWithExpiry as createGitHubAppInstallationTokenWithExpiry2, githubAppBotIdentity as githubAppBotIdentity2 } from "@opengeni/github";
16322
15489
  import {
16323
15490
  ConnectorActionBindingRejectedError,
@@ -16331,8 +15498,7 @@ import {
16331
15498
  GITHUB_REST_TOOL_NAMES,
16332
15499
  GitHubRestAuthorityError,
16333
15500
  githubRestConnectorActionOutcome,
16334
- GitHubRestMcpServer,
16335
- GitHubRestMutationNotExecutedError
15501
+ GitHubRestMcpServer
16336
15502
  } from "@opengeni/runtime/github-rest-mcp";
16337
15503
  var PERSONAL_GITHUB_DELEGATION_SERVER_ID = "github:personal";
16338
15504
  var INTERNAL_GITHUB_MCP_TIMEOUT_MS = 2e4;
@@ -16342,8 +15508,7 @@ async function buildGitHubRestMcpForTurn(input) {
16342
15508
  settings: input.settings,
16343
15509
  tools: input.tools,
16344
15510
  localMcpServers: [],
16345
- connectorBindings: [],
16346
- authorizeCodemodeCall: null
15511
+ connectorBindings: []
16347
15512
  };
16348
15513
  }
16349
15514
  const appRepositories = appRepositoryAuthority(input.resources);
@@ -16356,8 +15521,7 @@ async function buildGitHubRestMcpForTurn(input) {
16356
15521
  settings: input.settings,
16357
15522
  tools: input.tools,
16358
15523
  localMcpServers: [],
16359
- connectorBindings: [],
16360
- authorizeCodemodeCall: null
15524
+ connectorBindings: []
16361
15525
  };
16362
15526
  }
16363
15527
  const configuredIds = new Set(input.settings.mcpServers.map((server) => server.id));
@@ -16372,7 +15536,6 @@ async function buildGitHubRestMcpForTurn(input) {
16372
15536
  const localMcpServers = [];
16373
15537
  const toolRefs = [];
16374
15538
  const connectorBindings = [];
16375
- const codemodeActions = createCodemodeActionLifecycle(input, connectorBindings);
16376
15539
  if (appRepositories.length > 0) {
16377
15540
  const appResolver = workspaceAppAuthorityResolver(input, appRepositories);
16378
15541
  localMcpServers.push({
@@ -16381,8 +15544,7 @@ async function buildGitHubRestMcpForTurn(input) {
16381
15544
  serverId: GITHUB_REST_MCP_APP_SERVER_ID,
16382
15545
  authorityKind: "workspace_app",
16383
15546
  repositories: appRepositories,
16384
- resolveAuthority: appResolver,
16385
- completeCodemodeAction: codemodeActions.complete
15547
+ resolveAuthority: appResolver
16386
15548
  })
16387
15549
  });
16388
15550
  configs.push(internalMcpConfig(GITHUB_REST_MCP_APP_SERVER_ID, "GitHub \u2014 OpenGeni bot"));
@@ -16398,8 +15560,7 @@ async function buildGitHubRestMcpForTurn(input) {
16398
15560
  serverId: GITHUB_REST_MCP_PERSONAL_SERVER_ID,
16399
15561
  authorityKind: "personal_oauth",
16400
15562
  repositories: personalRepositories,
16401
- resolveAuthority: personalAuthorityResolver(input),
16402
- completeCodemodeAction: codemodeActions.complete
15563
+ resolveAuthority: personalAuthorityResolver(input)
16403
15564
  })
16404
15565
  });
16405
15566
  configs.push(internalMcpConfig(GITHUB_REST_MCP_PERSONAL_SERVER_ID, "GitHub \u2014 My account"));
@@ -16415,59 +15576,7 @@ async function buildGitHubRestMcpForTurn(input) {
16415
15576
  },
16416
15577
  tools: mergeToolRefs(input.tools, toolRefs),
16417
15578
  localMcpServers,
16418
- connectorBindings,
16419
- authorizeCodemodeCall: codemodeActions.authorize
16420
- };
16421
- }
16422
- function createCodemodeActionLifecycle(input, bindings) {
16423
- const identity = {
16424
- accountId: input.accountId,
16425
- workspaceId: input.workspaceId,
16426
- sessionId: input.sessionId,
16427
- turnId: input.turn.id,
16428
- attemptId: input.attemptId,
16429
- executionGeneration: input.turn.executionGeneration,
16430
- initiator: input.turn.initiator
16431
- };
16432
- const executing = /* @__PURE__ */ new Map();
16433
- return {
16434
- authorize: async ({ call, entry }) => {
16435
- if (call.caller.kind !== "codemode") return;
16436
- const binding = bindings.find((candidate) => candidate.modelName === entry.modelName);
16437
- if (!binding) return;
16438
- const invocation = binding.call(call.operationId, call.arguments);
16439
- const policy = await prepareConnectorActionApproval2(input.db, identity, invocation);
16440
- if (policy.managed && policy.decision === "ask") {
16441
- throw new AttemptToolApprovalRequiredError2();
16442
- }
16443
- if (policy.managed && policy.decision === "block") {
16444
- throw new GitHubRestMutationNotExecutedError(
16445
- "GitHub connector action is blocked by policy"
16446
- );
16447
- }
16448
- const admission = await beginConnectorActionExecution2(input.db, identity, invocation);
16449
- if (!admission.allowed) {
16450
- if (admission.reason === "approval_required") {
16451
- throw new AttemptToolApprovalRequiredError2();
16452
- }
16453
- throw new GitHubRestMutationNotExecutedError(
16454
- `GitHub connector action was not executed: ${admission.reason}`
16455
- );
16456
- }
16457
- if (admission.managed) executing.set(call.operationId, admission.requestId);
16458
- },
16459
- complete: async ({ operationId, outcome }) => {
16460
- const requestId = executing.get(operationId);
16461
- if (!requestId) return;
16462
- executing.delete(operationId);
16463
- await completeConnectorActionExecution2(input.db, {
16464
- accountId: input.accountId,
16465
- workspaceId: input.workspaceId,
16466
- requestId,
16467
- attemptId: input.attemptId,
16468
- outcome
16469
- });
16470
- }
15579
+ connectorBindings
16471
15580
  };
16472
15581
  }
16473
15582
  function internalMcpConfig(id, name) {
@@ -16811,7 +15920,7 @@ function canWrite(permissions) {
16811
15920
  }
16812
15921
 
16813
15922
  // src/activities/connector-attachments.ts
16814
- import { createHash as createHash10 } from "crypto";
15923
+ import { createHash as createHash11 } from "crypto";
16815
15924
  import {
16816
15925
  CONNECTOR_ATTACHMENT_RECEIPT_VERSION,
16817
15926
  ConnectorAttachmentReceiptEnvelope
@@ -16831,7 +15940,7 @@ var ConnectorAttachmentMaterializationError = class extends Error {
16831
15940
  }
16832
15941
  };
16833
15942
  function digestParts(...parts) {
16834
- const hash = createHash10("sha256");
15943
+ const hash = createHash11("sha256");
16835
15944
  for (const part of parts) {
16836
15945
  const bytes = Buffer.from(part, "utf8");
16837
15946
  hash.update(String(bytes.byteLength));
@@ -16943,14 +16052,16 @@ import { allowedFirstPartyMcpToolsForSession } from "@opengeni/config";
16943
16052
  // src/activities/codemode-dispatcher.ts
16944
16053
  import { randomUUID as randomUUID3 } from "crypto";
16945
16054
  import {
16946
- AttemptToolApprovalRequiredError as AttemptToolApprovalRequiredError3,
16055
+ AttemptToolApprovalRequiredError,
16947
16056
  AttemptToolCatalogStaleError,
16057
+ AttemptToolInputValidationError,
16948
16058
  AttemptToolNotFoundError,
16949
16059
  codemodeDispatchSubject,
16950
16060
  decodeCodemodeDispatchRequest,
16951
16061
  encodeCodemodeDispatchAck
16952
16062
  } from "@opengeni/codemode";
16953
16063
  import {
16064
+ CODEMODE_CLAIM_LEASE_MS,
16954
16065
  CODEMODE_CLAIM_HEARTBEAT_MS,
16955
16066
  CODEMODE_MAX_CONCURRENT_CALLS_PER_ATTEMPT
16956
16067
  } from "@opengeni/contracts";
@@ -16958,6 +16069,8 @@ import {
16958
16069
  cancelQueuedCodemodeOperationsForAttempt,
16959
16070
  claimCodemodeOperation,
16960
16071
  failCodemodeOperation,
16072
+ getSessionEventByClientEventId,
16073
+ getSessionToolCallCreatedEventByOperationId,
16961
16074
  markCodemodeOperationExecutionStarted,
16962
16075
  renewCodemodeOperationClaim,
16963
16076
  settleCodemodeOperationWithOutput
@@ -16967,7 +16080,7 @@ import {
16967
16080
  publishDurableSessionEvents as publishDurableSessionEvents8
16968
16081
  } from "@opengeni/events";
16969
16082
  var CodemodeAttemptDispatcher = class {
16970
- constructor(db, bus, environment, scope, turnSignal, maxConcurrentCalls = CODEMODE_MAX_CONCURRENT_CALLS_PER_ATTEMPT) {
16083
+ constructor(db, bus, environment, scope, turnSignal, maxConcurrentCalls = CODEMODE_MAX_CONCURRENT_CALLS_PER_ATTEMPT, timings = {}) {
16971
16084
  this.db = db;
16972
16085
  this.bus = bus;
16973
16086
  this.environment = environment;
@@ -16980,12 +16093,22 @@ var CodemodeAttemptDispatcher = class {
16980
16093
  if (!Number.isSafeInteger(maxConcurrentCalls) || maxConcurrentCalls < 1) {
16981
16094
  throw new Error("Codemode dispatcher concurrency must be a positive safe integer");
16982
16095
  }
16096
+ this.claimLeaseMs = timings.claimLeaseMs ?? CODEMODE_CLAIM_LEASE_MS;
16097
+ this.claimHeartbeatMs = timings.claimHeartbeatMs ?? CODEMODE_CLAIM_HEARTBEAT_MS;
16098
+ if (!Number.isSafeInteger(this.claimLeaseMs) || this.claimLeaseMs < 1e3 || this.claimLeaseMs > 10 * 6e4) {
16099
+ throw new Error("Codemode dispatcher claim lease must be between 1 second and 10 minutes");
16100
+ }
16101
+ if (!Number.isSafeInteger(this.claimHeartbeatMs) || this.claimHeartbeatMs < 1 || this.claimHeartbeatMs >= this.claimLeaseMs) {
16102
+ throw new Error("Codemode dispatcher heartbeat must be positive and shorter than its lease");
16103
+ }
16983
16104
  }
16984
16105
  stopController = new AbortController();
16985
16106
  inFlight = /* @__PURE__ */ new Map();
16986
16107
  pendingClaims = 0;
16987
16108
  unsubscribe = null;
16988
16109
  closing = false;
16110
+ claimLeaseMs;
16111
+ claimHeartbeatMs;
16989
16112
  start() {
16990
16113
  if (this.unsubscribe || this.closing)
16991
16114
  throw new Error("Codemode dispatcher cannot be restarted");
@@ -17013,12 +16136,15 @@ var CodemodeAttemptDispatcher = class {
17013
16136
  ...this.scope,
17014
16137
  catalogDigest: request.catalogDigest,
17015
16138
  operationId: request.operationId,
17016
- claimId: randomUUID3()
16139
+ claimId: randomUUID3(),
16140
+ claimLeaseMs: this.claimLeaseMs
17017
16141
  });
17018
16142
  if (claim.status === "claimed") {
17019
- const execution = this.execute(claim.operation, claim.claimId).finally(() => {
17020
- this.inFlight.delete(request.operationId);
17021
- });
16143
+ const execution = this.execute(claim.operation, claim.claimId, claim.reclaimed).finally(
16144
+ () => {
16145
+ this.inFlight.delete(request.operationId);
16146
+ }
16147
+ );
17022
16148
  this.inFlight.set(request.operationId, execution);
17023
16149
  return encodeCodemodeDispatchAck({
17024
16150
  version: 1,
@@ -17026,6 +16152,18 @@ var CodemodeAttemptDispatcher = class {
17026
16152
  status: "accepted"
17027
16153
  });
17028
16154
  }
16155
+ if (claim.status === "execution_owner_lost") {
16156
+ await this.settleWithOutput(claim.operation, claim.claimId, {
16157
+ state: "outcome_unknown",
16158
+ errorCode: "worker_lost_during_execution",
16159
+ errorMessage: "The execution owner disappeared after the tool call began. Inspect actual state before retrying."
16160
+ });
16161
+ return encodeCodemodeDispatchAck({
16162
+ version: 1,
16163
+ operationId: request.operationId,
16164
+ status: "terminal"
16165
+ });
16166
+ }
17029
16167
  return encodeCodemodeDispatchAck({
17030
16168
  version: 1,
17031
16169
  operationId: request.operationId,
@@ -17054,7 +16192,26 @@ var CodemodeAttemptDispatcher = class {
17054
16192
  });
17055
16193
  await Promise.allSettled([...this.inFlight.values()]);
17056
16194
  }
17057
- async execute(operation, claimId) {
16195
+ async execute(operation, claimId, reclaimed) {
16196
+ const claimAuthority = {
16197
+ accountId: this.scope.accountId,
16198
+ workspaceId: this.scope.workspaceId,
16199
+ attemptId: this.scope.attemptId,
16200
+ operationId: operation.operationId,
16201
+ claimId,
16202
+ claimLeaseMs: this.claimLeaseMs
16203
+ };
16204
+ const heartbeat = setInterval(() => {
16205
+ void renewCodemodeOperationClaim(this.db, claimAuthority).catch(() => void 0);
16206
+ }, this.claimHeartbeatMs);
16207
+ heartbeat.unref?.();
16208
+ try {
16209
+ await this.executeClaimed(operation, claimId, reclaimed, claimAuthority);
16210
+ } finally {
16211
+ clearInterval(heartbeat);
16212
+ }
16213
+ }
16214
+ async executeClaimed(operation, claimId, reclaimed, claimAuthority) {
17058
16215
  const entry = this.environment.catalog.entries.find(
17059
16216
  (candidate) => candidate.identity.serverId === operation.identity.serverId && candidate.identity.toolName === operation.identity.toolName
17060
16217
  );
@@ -17077,38 +16234,7 @@ var CodemodeAttemptDispatcher = class {
17077
16234
  );
17078
16235
  return;
17079
16236
  }
17080
- const created = await appendAndPublishTurnEventsFenced3(
17081
- this.db,
17082
- this.bus,
17083
- this.scope.workspaceId,
17084
- this.scope.sessionId,
17085
- this.scope.turnId,
17086
- this.scope.executionGeneration,
17087
- this.scope.attemptId,
17088
- [
17089
- {
17090
- type: "agent.toolCall.created",
17091
- turnId: this.scope.turnId,
17092
- turnGeneration: this.scope.executionGeneration,
17093
- turnAttemptId: this.scope.attemptId,
17094
- producerId: operation.caller.subjectId,
17095
- payload: {
17096
- id: operation.operationId,
17097
- name: entry.modelName,
17098
- arguments: operation.arguments,
17099
- origin: "codemode",
17100
- subjectId: operation.caller.subjectId,
17101
- raw: {
17102
- type: "codemode_call",
17103
- serverId: operation.identity.serverId,
17104
- toolName: operation.identity.toolName,
17105
- catalogDigest: operation.catalogDigest
17106
- }
17107
- }
17108
- }
17109
- ]
17110
- ).catch(() => ({ accepted: false }));
17111
- if (!created.accepted) {
16237
+ if (!await this.ensureToolCallCreated(operation, entry, reclaimed)) {
17112
16238
  await this.failBeforeExecution(
17113
16239
  operation.operationId,
17114
16240
  claimId,
@@ -17125,24 +16251,9 @@ var CodemodeAttemptDispatcher = class {
17125
16251
  });
17126
16252
  return;
17127
16253
  }
17128
- const claimAuthority = {
17129
- accountId: this.scope.accountId,
17130
- workspaceId: this.scope.workspaceId,
17131
- attemptId: this.scope.attemptId,
17132
- operationId: operation.operationId,
17133
- claimId
17134
- };
17135
- const crossedExecutionBoundary = await markCodemodeOperationExecutionStarted(
17136
- this.db,
17137
- claimAuthority
17138
- );
17139
- if (!crossedExecutionBoundary) return;
17140
- const heartbeat = setInterval(() => {
17141
- void renewCodemodeOperationClaim(this.db, claimAuthority).catch(() => void 0);
17142
- }, CODEMODE_CLAIM_HEARTBEAT_MS);
17143
- heartbeat.unref?.();
16254
+ let preparedCall;
17144
16255
  try {
17145
- const result = await this.environment.call(
16256
+ preparedCall = await this.environment.prepareCall(
17146
16257
  {
17147
16258
  operationId: operation.operationId,
17148
16259
  catalogDigest: operation.catalogDigest,
@@ -17152,16 +16263,111 @@ var CodemodeAttemptDispatcher = class {
17152
16263
  },
17153
16264
  { signal }
17154
16265
  );
17155
- await this.settleWithOutput(operation, claimId, { state: "completed", result });
17156
16266
  } catch (error) {
17157
- const knownPreExecution = error instanceof AttemptToolApprovalRequiredError3 || error instanceof AttemptToolCatalogStaleError || error instanceof AttemptToolNotFoundError || connectorActionWasNotExecuted(error);
16267
+ await this.settleWithOutput(operation, claimId, {
16268
+ state: "failed",
16269
+ errorCode: errorCode(error, signal, false),
16270
+ errorMessage: safeErrorMessage(error)
16271
+ });
16272
+ return;
16273
+ }
16274
+ if (signal.aborted) {
16275
+ await this.settleWithOutput(operation, claimId, {
16276
+ state: "failed",
16277
+ errorCode: "attempt_cancelled",
16278
+ errorMessage: "Execution attempt ended before the Codemode call started"
16279
+ });
16280
+ return;
16281
+ }
16282
+ const crossedExecutionBoundary = await markCodemodeOperationExecutionStarted(
16283
+ this.db,
16284
+ claimAuthority
16285
+ );
16286
+ if (!crossedExecutionBoundary) return;
16287
+ try {
16288
+ const result = await preparedCall.execute();
16289
+ await this.settleWithOutput(operation, claimId, {
16290
+ state: "completed",
16291
+ result
16292
+ });
16293
+ } catch (error) {
16294
+ const knownPreExecution = error instanceof AttemptToolApprovalRequiredError || error instanceof AttemptToolCatalogStaleError || error instanceof AttemptToolNotFoundError || connectorActionWasNotExecuted(error);
17158
16295
  await this.settleWithOutput(operation, claimId, {
17159
16296
  state: knownPreExecution ? "failed" : "outcome_unknown",
17160
- errorCode: errorCode(error, signal),
16297
+ errorCode: errorCode(error, signal, true),
17161
16298
  errorMessage: !knownPreExecution ? "Tool execution ended without a durable result. Its side-effect outcome is unknown; inspect actual state before retrying." : safeErrorMessage(error)
17162
16299
  });
17163
- } finally {
17164
- clearInterval(heartbeat);
16300
+ }
16301
+ }
16302
+ async ensureToolCallCreated(operation, entry, reclaimed) {
16303
+ const clientEventId = codemodeToolCallCreatedClientEventId(operation.operationId);
16304
+ if (reclaimed) {
16305
+ const existing = await getSessionEventByClientEventId(
16306
+ this.db,
16307
+ this.scope.workspaceId,
16308
+ this.scope.sessionId,
16309
+ clientEventId
16310
+ ).catch(() => null);
16311
+ if (existing) {
16312
+ assertMatchingCodemodeToolCallCreated(existing, operation, entry, this.scope);
16313
+ return true;
16314
+ }
16315
+ const legacyCreated = await getSessionToolCallCreatedEventByOperationId(this.db, {
16316
+ workspaceId: this.scope.workspaceId,
16317
+ sessionId: this.scope.sessionId,
16318
+ turnId: this.scope.turnId,
16319
+ attemptId: this.scope.attemptId,
16320
+ operationId: operation.operationId
16321
+ }).catch(() => null);
16322
+ if (legacyCreated) {
16323
+ assertMatchingCodemodeToolCallCreated(legacyCreated, operation, entry, this.scope);
16324
+ return true;
16325
+ }
16326
+ }
16327
+ try {
16328
+ const created = await appendAndPublishTurnEventsFenced3(
16329
+ this.db,
16330
+ this.bus,
16331
+ this.scope.workspaceId,
16332
+ this.scope.sessionId,
16333
+ this.scope.turnId,
16334
+ this.scope.executionGeneration,
16335
+ this.scope.attemptId,
16336
+ [
16337
+ {
16338
+ type: "agent.toolCall.created",
16339
+ clientEventId,
16340
+ turnId: this.scope.turnId,
16341
+ turnGeneration: this.scope.executionGeneration,
16342
+ turnAttemptId: this.scope.attemptId,
16343
+ producerId: operation.caller.subjectId,
16344
+ payload: {
16345
+ id: operation.operationId,
16346
+ name: entry.modelName,
16347
+ arguments: operation.arguments,
16348
+ origin: "codemode",
16349
+ subjectId: operation.caller.subjectId,
16350
+ raw: {
16351
+ type: "codemode_call",
16352
+ serverId: operation.identity.serverId,
16353
+ toolName: operation.identity.toolName,
16354
+ catalogDigest: operation.catalogDigest
16355
+ }
16356
+ }
16357
+ }
16358
+ ]
16359
+ );
16360
+ return created.accepted;
16361
+ } catch {
16362
+ const raced = await getSessionEventByClientEventId(
16363
+ this.db,
16364
+ this.scope.workspaceId,
16365
+ this.scope.sessionId,
16366
+ clientEventId
16367
+ ).catch(() => null);
16368
+ if (!raced) return false;
16369
+ assertMatchingCodemodeToolCallCreated(raced, operation, entry, this.scope);
16370
+ return true;
17165
16371
  }
17166
16372
  }
17167
16373
  async settleWithOutput(operation, claimId, settlement) {
@@ -17193,6 +16399,19 @@ var CodemodeAttemptDispatcher = class {
17193
16399
  });
17194
16400
  }
17195
16401
  };
16402
+ function codemodeToolCallCreatedClientEventId(operationId) {
16403
+ return `opengeni:codemode-tool-call-created:${operationId}`;
16404
+ }
16405
+ function assertMatchingCodemodeToolCallCreated(event, operation, entry, scope) {
16406
+ const payload = recordValue(event.payload);
16407
+ const raw = recordValue(payload?.raw);
16408
+ if (event.type !== "agent.toolCall.created" || event.turnId !== scope.turnId || event.turnGeneration !== scope.executionGeneration || event.turnAttemptId !== scope.attemptId || event.turnAssociation !== "current" || payload?.id !== operation.operationId || payload?.name !== entry.modelName || payload?.origin !== "codemode" || payload?.subjectId !== operation.caller.subjectId || raw?.type !== "codemode_call" || raw?.serverId !== operation.identity.serverId || raw?.toolName !== operation.identity.toolName || raw?.catalogDigest !== operation.catalogDigest) {
16409
+ throw new Error("Codemode tool-call creation id is bound to incompatible durable evidence");
16410
+ }
16411
+ }
16412
+ function recordValue(value) {
16413
+ return value !== null && typeof value === "object" && !Array.isArray(value) ? value : null;
16414
+ }
17196
16415
  function connectorActionWasNotExecuted(error) {
17197
16416
  return Boolean(
17198
16417
  error && typeof error === "object" && "connectorActionOutcome" in error && error.connectorActionOutcome === "not_executed"
@@ -17201,8 +16420,10 @@ function connectorActionWasNotExecuted(error) {
17201
16420
  function combinedSignal(primary, secondary) {
17202
16421
  return secondary ? AbortSignal.any([primary, secondary]) : primary;
17203
16422
  }
17204
- function errorCode(error, signal) {
17205
- if (signal.aborted) return "attempt_cancelled_during_execution";
16423
+ function errorCode(error, signal, executionStarted) {
16424
+ if (signal.aborted) {
16425
+ return executionStarted ? "attempt_cancelled_during_execution" : "attempt_cancelled";
16426
+ }
17206
16427
  if (connectorActionWasNotExecuted(error)) return "connector_action_not_executed";
17207
16428
  if (typeof error === "object" && error !== null && "code" in error) {
17208
16429
  const code = error.code;
@@ -17211,7 +16432,7 @@ function errorCode(error, signal) {
17211
16432
  return "tool_outcome_unknown";
17212
16433
  }
17213
16434
  function safeErrorMessage(error) {
17214
- if (error instanceof AttemptToolApprovalRequiredError3 || error instanceof AttemptToolCatalogStaleError || error instanceof AttemptToolNotFoundError) {
16435
+ if (error instanceof AttemptToolApprovalRequiredError || error instanceof AttemptToolCatalogStaleError || error instanceof AttemptToolInputValidationError || error instanceof AttemptToolNotFoundError) {
17215
16436
  return error.message;
17216
16437
  }
17217
16438
  return "Codemode call failed before tool execution";
@@ -17223,6 +16444,7 @@ import {
17223
16444
  defaultSessionMcpServerIds,
17224
16445
  loadRigDefaultVariableSetEnvironment,
17225
16446
  mergeRigDefaultVariableSetEnvironment,
16447
+ buildApiIntegrationMcpServers,
17226
16448
  resolveCatalogSettings as resolveCatalogSettings2,
17227
16449
  resolveWorkspaceModelSelection,
17228
16450
  withFrozenPersonalConnectionDelegations,
@@ -17555,7 +16777,7 @@ async function prepareTurnToolRuntime(deps) {
17555
16777
  );
17556
16778
  };
17557
16779
  const selectedApiIntegrationServerIds = new Set(turnTools.map((tool) => tool.id));
17558
- const apiIntegrationMcpServers = buildApiIntegrationServersForTurn({
16780
+ const apiIntegrationMcpServers = buildApiIntegrationMcpServers({
17559
16781
  settings: runSettings,
17560
16782
  integrations: installedApiIntegrations.filter(
17561
16783
  (integration) => selectedApiIntegrationServerIds.has(integration.serverId)
@@ -17633,6 +16855,22 @@ async function prepareTurnToolRuntime(deps) {
17633
16855
  )
17634
16856
  }
17635
16857
  ] : [];
16858
+ const attemptConnectorActionBindings = [
16859
+ ...googleDriveConnectorBindings,
16860
+ ...githubRestMcp.connectorBindings
16861
+ ];
16862
+ const connectorActionPolicy = {
16863
+ preview: async (call) => await previewConnectorActionApproval(db, connectorActionIdentity, call),
16864
+ prepare: async (call) => await prepareConnectorActionApproval(db, connectorActionIdentity, call),
16865
+ begin: async (call) => await beginConnectorActionExecution(db, connectorActionIdentity, call),
16866
+ complete: async ({ requestId, outcome }) => await completeConnectorActionExecution(db, {
16867
+ accountId: input.accountId,
16868
+ workspaceId: input.workspaceId,
16869
+ requestId,
16870
+ attemptId: input.attemptId,
16871
+ outcome
16872
+ })
16873
+ };
17636
16874
  const attemptToolDefinitions = [
17637
16875
  createListModelsAttemptToolDefinition({
17638
16876
  currentModelId: turnExecutionPolicy.productModelId,
@@ -17821,6 +17059,12 @@ async function prepareTurnToolRuntime(deps) {
17821
17059
  resolveCredential,
17822
17060
  onAuthNeeded: publishToolAuthNeeded,
17823
17061
  materializeConnectorAttachments,
17062
+ refreshOwnedCommand: async (commandId) => {
17063
+ throwIfWorkerShuttingDown();
17064
+ throwIfTurnCancelled();
17065
+ const owned = sandboxState.lazyOwnedSandbox?.session ?? sandboxState.resolvedSandbox?.established.session ?? media.sdkOwnedSandboxSession;
17066
+ return await owned?.refreshOwnedCommand?.(commandId) ?? false;
17067
+ },
17824
17068
  spillOversizedModelToolResult: async ({ operationId, result }) => await toolResultSpill.spill({ operationId, result }),
17825
17069
  localMcpServers,
17826
17070
  ...deferNonEagerToolPreparation ? { deferNonEagerUntilToolDemand: true } : {},
@@ -17844,21 +17088,8 @@ async function prepareTurnToolRuntime(deps) {
17844
17088
  nestedAgentDepth: session.nestedAgentDepth,
17845
17089
  effectiveMaxNestedAgentDepth: session.effectiveMaxNestedAgentDepth,
17846
17090
  attemptToolDefinitions,
17847
- ...googleDrivePublicationTarget && googleDrivePublicationAllowed || githubRestMcp.authorizeCodemodeCall ? {
17848
- attemptToolAuthorize: async (authorization) => {
17849
- const { call } = authorization;
17850
- if (googleDrivePublicationTarget && googleDrivePublicationAllowed && call.caller.kind === "codemode" && call.identity.serverId === "google-drive-publishing" && call.identity.toolName === "google_drive_publish_file") {
17851
- await authorizeGoogleDrivePublicationAttempt({
17852
- db,
17853
- identity: connectorActionIdentity,
17854
- target: googleDrivePublicationTarget,
17855
- approvalId: call.operationId,
17856
- arguments: call.arguments
17857
- });
17858
- }
17859
- await githubRestMcp.authorizeCodemodeCall?.(authorization);
17860
- }
17861
- } : {}
17091
+ connectorActionPolicy,
17092
+ attemptConnectorActionBindings
17862
17093
  }),
17863
17094
  cancellationSignal,
17864
17095
  async (latePreparedTools) => await latePreparedTools.close().catch(() => void 0)
@@ -17917,11 +17148,8 @@ async function prepareTurnToolRuntime(deps) {
17917
17148
  activatePreparedToolEnvironment(eventing.preparedTools);
17918
17149
  }
17919
17150
  return {
17920
- attemptConnectorActionBindings: [
17921
- ...googleDriveConnectorBindings,
17922
- ...githubRestMcp.connectorBindings
17923
- ],
17924
- connectorActionIdentity,
17151
+ attemptConnectorActionBindings,
17152
+ connectorActionPolicy,
17925
17153
  generateSessionTitleInParallel: titleToolPlan.generateTitleInParallel,
17926
17154
  postToolPreparationStartedAt,
17927
17155
  preparationIndependentToolNames: titleToolPlan.preparationIndependentToolNames
@@ -18114,19 +17342,16 @@ async function applyTurnGitHubRepositoryBindings(input) {
18114
17342
  // src/activities/agent-turn/agent-build.ts
18115
17343
  import {
18116
17344
  selectXaiCredentialForUse,
18117
- materializeXaiCredentialForRun as materializeXaiCredentialForRun2,
17345
+ materializeXaiCredentialForRun,
18118
17346
  resolveXaiProviderAccountAuthoritySnapshotForAcceptance,
18119
17347
  getXaiSessionAccountPin as getXaiSessionAccountPin2,
18120
17348
  setXaiSessionAccountPin as setXaiSessionAccountPin2,
18121
17349
  getWorkspaceVideoGenerationPolicy,
18122
- loadWorkspaceVercelAiGatewayCredentialLease,
18123
- beginConnectorActionExecution as beginConnectorActionExecution3,
18124
- completeConnectorActionExecution as completeConnectorActionExecution3,
18125
- prepareConnectorActionApproval as prepareConnectorActionApproval3
17350
+ loadWorkspaceVercelAiGatewayCredentialLease
18126
17351
  } from "@opengeni/db";
18127
17352
  import {
18128
17353
  serviceTierForLatencyMode as serviceTierForLatencyMode2,
18129
- environmentsEncryptionKeyBytes as environmentsEncryptionKeyBytes4,
17354
+ environmentsEncryptionKeyBytes as environmentsEncryptionKeyBytes3,
18130
17355
  WORKSPACE_GATEWAY_PROVIDER_ID as WORKSPACE_GATEWAY_PROVIDER_ID2,
18131
17356
  resolveModelProvider as resolveModelProvider2
18132
17357
  } from "@opengeni/config";
@@ -18205,9 +17430,9 @@ import {
18205
17430
  } from "@opengeni/db";
18206
17431
  import {
18207
17432
  calculateVideoGenerationCreditCostMicros,
18208
- environmentsEncryptionKeyBytes as environmentsEncryptionKeyBytes3
17433
+ environmentsEncryptionKeyBytes as environmentsEncryptionKeyBytes2
18209
17434
  } from "@opengeni/config";
18210
- import { createHash as createHash12, randomUUID as randomUUID5 } from "crypto";
17435
+ import { createHash as createHash13, randomUUID as randomUUID5 } from "crypto";
18211
17436
 
18212
17437
  // src/activities/video-reference-staging.ts
18213
17438
  import {
@@ -18216,7 +17441,7 @@ import {
18216
17441
 
18217
17442
  // src/activities/video-media-validation.ts
18218
17443
  import { retryWhileMissing as retryWhileMissing5 } from "@opengeni/storage";
18219
- import { createHash as createHash11, randomUUID as randomUUID4 } from "crypto";
17444
+ import { createHash as createHash12, randomUUID as randomUUID4 } from "crypto";
18220
17445
  import { open, mkdir, rm, stat } from "fs/promises";
18221
17446
  import { join } from "path";
18222
17447
  import { spawn } from "child_process";
@@ -18244,7 +17469,7 @@ async function copyVersionedObjectToVerifiedTemp(input) {
18244
17469
  const path = join(directory, "payload");
18245
17470
  await mkdir(directory, { recursive: false, mode: 448 });
18246
17471
  const handle = await open(path, "wx", 384);
18247
- const hash = createHash11("sha256");
17472
+ const hash = createHash12("sha256");
18248
17473
  let offset = 0;
18249
17474
  try {
18250
17475
  while (offset < input.expectedSizeBytes) {
@@ -18764,7 +17989,7 @@ function videoCapabilitiesForTurn(input) {
18764
17989
  }
18765
17990
  function managedVideoGenerationCredentialLease(settings) {
18766
17991
  if (!settings.vercelAiGatewayApiKey) return null;
18767
- const encryptionKey = environmentsEncryptionKeyBytes3(settings);
17992
+ const encryptionKey = environmentsEncryptionKeyBytes2(settings);
18768
17993
  if (!encryptionKey) {
18769
17994
  throw new Error("Managed video generation requires OPENGENI_ENVIRONMENTS_ENCRYPTION_KEY");
18770
17995
  }
@@ -18779,7 +18004,7 @@ function managedVideoGenerationCredentialLease(settings) {
18779
18004
  });
18780
18005
  }
18781
18006
  function xaiVideoGenerationCredentialLease(input) {
18782
- const encryptionKey = environmentsEncryptionKeyBytes3(input.settings);
18007
+ const encryptionKey = environmentsEncryptionKeyBytes2(input.settings);
18783
18008
  if (!encryptionKey) {
18784
18009
  throw new Error("SuperGrok video generation requires OPENGENI_ENVIRONMENTS_ENCRYPTION_KEY");
18785
18010
  }
@@ -18839,7 +18064,7 @@ async function admitVideoGenerationRequest(input) {
18839
18064
  operationId: candidateOperationId,
18840
18065
  requestDigest
18841
18066
  });
18842
- const encryptionKey = environmentsEncryptionKeyBytes3(input.settings);
18067
+ const encryptionKey = environmentsEncryptionKeyBytes2(input.settings);
18843
18068
  if (!encryptionKey) {
18844
18069
  throw new Error("Video generation requires OPENGENI_ENVIRONMENTS_ENCRYPTION_KEY");
18845
18070
  }
@@ -18863,7 +18088,7 @@ async function admitVideoGenerationRequest(input) {
18863
18088
  toolCallId: input.toolCallId,
18864
18089
  admissionKey,
18865
18090
  requestDigest,
18866
- promptDigest: createHash12("sha256").update(canonical.prompt).digest("hex"),
18091
+ promptDigest: createHash13("sha256").update(canonical.prompt).digest("hex"),
18867
18092
  requestEncrypted: encryptEnvironmentValue2(encryptionKey, JSON.stringify(canonical)),
18868
18093
  modelId: canonical.modelId,
18869
18094
  sourceMode: canonical.sourceMode,
@@ -19059,18 +18284,27 @@ import {
19059
18284
  var CODEX_IMAGE_MODEL = "gpt-image-2";
19060
18285
  async function executeCodexImageGeneration(input) {
19061
18286
  const providerBindingHash = imageProviderBindingHash(CODEX_PROVIDER_ID2, input.credentialId);
18287
+ let providerDispatchAdmitted = false;
18288
+ const codexContext = {
18289
+ ...input.codexContext,
18290
+ beforeProviderDispatch: async () => {
18291
+ await input.codexContext.beforeProviderDispatch?.();
18292
+ providerDispatchAdmitted = true;
18293
+ }
18294
+ };
19062
18295
  return await executeImageGenerationOperation({
19063
18296
  ...input,
19064
18297
  providerId: CODEX_PROVIDER_ID2,
19065
18298
  providerBindingHash,
19066
18299
  modelId: CODEX_IMAGE_MODEL,
19067
18300
  ...input.references ? { referenceDigests: input.references } : {},
18301
+ isProviderDispatchRejected: (error) => !providerDispatchAdmitted && error instanceof CodexCredentialLeaseLostError,
19068
18302
  generate: async () => {
19069
18303
  const generated = await generateCodexSubscriptionImage({
19070
18304
  prompt: input.prompt,
19071
18305
  ...input.references ? { references: input.references } : {},
19072
18306
  turnId: input.turnId,
19073
- context: input.codexContext,
18307
+ context: codexContext,
19074
18308
  ...input.abortSignal ? { abortSignal: input.abortSignal } : {}
19075
18309
  });
19076
18310
  return {
@@ -19326,7 +18560,8 @@ async function buildTurnAgent(deps) {
19326
18560
  sandboxCodemodeToken,
19327
18561
  fileResourceDownloads,
19328
18562
  attemptConnectorActionBindings,
19329
- connectorActionIdentity,
18563
+ connectorActionPolicy,
18564
+ trigger,
19330
18565
  preparationIndependentToolNames,
19331
18566
  videoGenerationAcceptancesByCallId,
19332
18567
  activeSandboxBackend,
@@ -19514,7 +18749,7 @@ async function buildTurnAgent(deps) {
19514
18749
  };
19515
18750
  }
19516
18751
  } else if (videoGenerationPolicy.fundingSource === "supergrok_subscription") {
19517
- const encryptionKey = environmentsEncryptionKeyBytes4(eventing.modelRunSettings);
18752
+ const encryptionKey = environmentsEncryptionKeyBytes3(eventing.modelRunSettings);
19518
18753
  const subjectId = leases.xai.subjectId ?? turn.initiatingHumanSubjectId;
19519
18754
  if (encryptionKey && subjectId) {
19520
18755
  const authoritySnapshot = providerTurn.xaiAuthoritySnapshot ?? await resolveXaiProviderAccountAuthoritySnapshotForAcceptance(db, {
@@ -19555,7 +18790,7 @@ async function buildTurnAgent(deps) {
19555
18790
  throw error;
19556
18791
  });
19557
18792
  }
19558
- const credential = await materializeXaiCredentialForRun2(db, {
18793
+ const credential = await materializeXaiCredentialForRun(db, {
19559
18794
  workspaceId: input.workspaceId,
19560
18795
  subjectId,
19561
18796
  credentialId: selected.credentialId,
@@ -19645,17 +18880,7 @@ async function buildTurnAgent(deps) {
19645
18880
  turnExecutionPolicy.providerId,
19646
18881
  turnExecutionPolicy.latencyMode
19647
18882
  );
19648
- const connectorActionPolicy = {
19649
- prepare: async (call) => await prepareConnectorActionApproval3(db, connectorActionIdentity, call),
19650
- begin: async (call) => await beginConnectorActionExecution3(db, connectorActionIdentity, call),
19651
- complete: async ({ requestId, outcome }) => await completeConnectorActionExecution3(db, {
19652
- accountId: input.accountId,
19653
- workspaceId: input.workspaceId,
19654
- requestId,
19655
- attemptId: input.attemptId,
19656
- outcome
19657
- })
19658
- };
18883
+ const approvedToolCallId = approvedConnectorActionCallId(trigger);
19659
18884
  const runtimeSkillActivations = [
19660
18885
  ...installedSkillRuntime.activations,
19661
18886
  ...packRuntime.skillActivations,
@@ -19695,6 +18920,7 @@ async function buildTurnAgent(deps) {
19695
18920
  let agentConstructionOutcome = "completed";
19696
18921
  try {
19697
18922
  return runtime.buildAgent(eventing.modelRunSettings, runtimeResources, {
18923
+ ...preparedTools.inputWaitYield ? { inputWaitYield: preparedTools.inputWaitYield } : {},
19698
18924
  reasoningEffort: turn.reasoningEffort,
19699
18925
  latencyMode: turnExecutionPolicy.latencyMode,
19700
18926
  ...serviceTier ? { serviceTier } : {},
@@ -19732,6 +18958,7 @@ async function buildTurnAgent(deps) {
19732
18958
  resolvedMcpConnectionIds: preparedTools.resolvedMcpConnectionIds,
19733
18959
  connectorActionPolicy,
19734
18960
  attemptConnectorActionBindings,
18961
+ ...approvedToolCallId ? { approvedToolCallId } : {},
19735
18962
  // LIVE by-reference connector namespaces (fills during this turn's
19736
18963
  // codex_apps tools/list): the codex tool_search description reads it per
19737
18964
  // model call so the model sees the account's real connected sources.
@@ -19838,6 +19065,13 @@ async function buildTurnAgent(deps) {
19838
19065
  postAgentPreparationStartedAt
19839
19066
  };
19840
19067
  }
19068
+ function approvedConnectorActionCallId(trigger) {
19069
+ if (trigger.type !== "user.approvalDecision") {
19070
+ return null;
19071
+ }
19072
+ const payload = trigger.payload;
19073
+ return payload.decision === "approve" && typeof payload.approvalId === "string" ? payload.approvalId : null;
19074
+ }
19841
19075
 
19842
19076
  // src/activities/agent-turn/run.ts
19843
19077
  function sessionTitleCodexRequestContext(context, nextRequestId) {
@@ -19848,6 +19082,7 @@ function sessionTitleCodexRequestContext(context, nextRequestId) {
19848
19082
  refresh: context.refresh,
19849
19083
  resolveModel: context.resolveModel,
19850
19084
  ...context.onUsageHeaders ? { onUsageHeaders: context.onUsageHeaders } : {},
19085
+ ...context.beforeProviderDispatch ? { beforeProviderDispatch: context.beforeProviderDispatch } : {},
19851
19086
  ...context.responseTimeoutPolicy ? { responseTimeoutPolicy: context.responseTimeoutPolicy } : {},
19852
19087
  nextRequestId,
19853
19088
  turnMetadata: { request_kind: "session_title" }
@@ -20177,6 +19412,11 @@ function createRunAgentTurnActivity(services) {
20177
19412
  input.workspaceId,
20178
19413
  providerTurn.effectiveCodexCredentialId ?? ""
20179
19414
  );
19415
+ const resolveTrackedToken = async (resolve) => {
19416
+ const token = await resolve();
19417
+ providerTurn.effectiveCodexCredentialVersion = token.credentialVersion;
19418
+ return token;
19419
+ };
20180
19420
  return {
20181
19421
  clientVersion: CODEX_CLIENT_VERSION2,
20182
19422
  // Backend sticky cache-routing key — the SAME id as the body's
@@ -20186,8 +19426,8 @@ function createRunAgentTurnActivity(services) {
20186
19426
  // (per-request shard lottery = prod's measured 48.6% on sol);
20187
19427
  // with it, resends pin to the warm shard (Codex CLI parity).
20188
19428
  sessionId: input.sessionId,
20189
- getToken: () => resolver.getToken(),
20190
- refresh: () => resolver.refresh(),
19429
+ getToken: () => resolveTrackedToken(resolver.getToken),
19430
+ refresh: () => resolveTrackedToken(resolver.refresh),
20191
19431
  resolveModel: buildModelResolver(
20192
19432
  CODEX_FALLBACK_MODEL_SLUGS,
20193
19433
  CODEX_FALLBACK_MODEL_SLUGS[0]
@@ -20196,6 +19436,9 @@ function createRunAgentTurnActivity(services) {
20196
19436
  providerTurn.latestCodexUsage = snapshot;
20197
19437
  },
20198
19438
  // latest wins; flushed once in finally
19439
+ beforeProviderDispatch: () => {
19440
+ leases.codex.assertUsable();
19441
+ },
20199
19442
  onRequestPreparationDiagnostic: (phase) => {
20200
19443
  if (eventing.firstModelRequestCheckpointAt === null || firstModelRequestCheckpoints.has(phase)) {
20201
19444
  return;
@@ -20748,7 +19991,7 @@ function createRunAgentTurnActivity(services) {
20748
19991
  });
20749
19992
  const {
20750
19993
  attemptConnectorActionBindings,
20751
- connectorActionIdentity,
19994
+ connectorActionPolicy,
20752
19995
  generateSessionTitleInParallel,
20753
19996
  postToolPreparationStartedAt,
20754
19997
  preparationIndependentToolNames
@@ -20802,7 +20045,8 @@ function createRunAgentTurnActivity(services) {
20802
20045
  sandboxCodemodeToken,
20803
20046
  fileResourceDownloads,
20804
20047
  attemptConnectorActionBindings,
20805
- connectorActionIdentity,
20048
+ connectorActionPolicy,
20049
+ trigger,
20806
20050
  preparationIndependentToolNames,
20807
20051
  videoGenerationAcceptancesByCallId,
20808
20052
  activeSandboxBackend,
@@ -21111,6 +20355,7 @@ function createRunAgentTurnActivity(services) {
21111
20355
  videoGenerationAcceptancesByCallId
21112
20356
  });
21113
20357
  } catch (error) {
20358
+ eventing.preparedTools?.inputWaitYield?.closeAdmission();
21114
20359
  return await settleTurnFailure({
21115
20360
  error,
21116
20361
  input,
@@ -21119,7 +20364,6 @@ function createRunAgentTurnActivity(services) {
21119
20364
  bus,
21120
20365
  observability,
21121
20366
  wakeSessionWorkflow,
21122
- signalCodexCapacityWorkflow,
21123
20367
  cancellationSignal,
21124
20368
  sandboxRotationController,
21125
20369
  noteCancellationRequested,
@@ -21173,7 +20417,7 @@ function createRunAgentTurnActivity(services) {
21173
20417
  }
21174
20418
 
21175
20419
  // src/activities/video-generation-reconciliation.ts
21176
- import { environmentsEncryptionKeyBytes as environmentsEncryptionKeyBytes5 } from "@opengeni/config";
20420
+ import { environmentsEncryptionKeyBytes as environmentsEncryptionKeyBytes4 } from "@opengeni/config";
21177
20421
  import { videoGenerationRequestDigest as videoGenerationRequestDigest2 } from "@opengeni/core";
21178
20422
  import {
21179
20423
  addSessionSystemUpdateWithSourceMutation,
@@ -21185,8 +20429,8 @@ import {
21185
20429
  markVideoGenerationRetaining,
21186
20430
  markVideoGenerationSubmissionIntent,
21187
20431
  markVideoGenerationSubmissionUncertain,
21188
- materializeXaiCredentialForRun as materializeXaiCredentialForRun3,
21189
- refreshXaiSubscriptionCredentialSerialized as refreshXaiSubscriptionCredentialSerialized2,
20432
+ materializeXaiCredentialForRun as materializeXaiCredentialForRun2,
20433
+ refreshXaiSubscriptionCredentialSerialized,
21190
20434
  rotateVideoGenerationCredential,
21191
20435
  markVideoGenerationTerminalUpdate,
21192
20436
  markVideoGenerationTerminalUpdateInTransaction,
@@ -21399,7 +20643,7 @@ function publicProviderReason(error, cancelled) {
21399
20643
  // src/activities/video-output-retention.ts
21400
20644
  import { GENERATED_VIDEO_MAX_BYTES as GENERATED_VIDEO_MAX_BYTES2 } from "@opengeni/contracts";
21401
20645
  import { pinnedFetch } from "@opengeni/network";
21402
- import { createHash as createHash13, randomUUID as randomUUID7 } from "crypto";
20646
+ import { createHash as createHash14, randomUUID as randomUUID7 } from "crypto";
21403
20647
  import { createReadStream } from "fs";
21404
20648
  import { mkdir as mkdir2, open as open2, rm as rm2 } from "fs/promises";
21405
20649
  import { join as join2 } from "path";
@@ -21437,7 +20681,7 @@ async function downloadGeneratedVideoToVerifiedTemp(input) {
21437
20681
  await mkdir2(directory, { recursive: false, mode: 448 });
21438
20682
  const handle = await open2(path, "wx", 384);
21439
20683
  const reader = response.body.getReader();
21440
- const digest2 = createHash13("sha256");
20684
+ const digest2 = createHash14("sha256");
21441
20685
  let sizeBytes = 0;
21442
20686
  try {
21443
20687
  for (; ; ) {
@@ -21606,9 +20850,9 @@ function throwIfAborted3(signal) {
21606
20850
 
21607
20851
  // src/activities/video-generation-reconciliation.ts
21608
20852
  import {
21609
- refreshXaiToken as refreshXaiToken2,
20853
+ refreshXaiToken,
21610
20854
  XaiSubscriptionError,
21611
- xaiAccessTokenExpiry as xaiAccessTokenExpiry2
20855
+ xaiAccessTokenExpiry
21612
20856
  } from "@opengeni/xai-subscription";
21613
20857
 
21614
20858
  // src/activities/xai-video-generation.ts
@@ -21677,7 +20921,7 @@ async function reconcileVideoGenerationOperation(service, input) {
21677
20921
  await deliverTerminalResult(service, operation);
21678
20922
  return { action: "terminal", status: operation.status };
21679
20923
  }
21680
- const key = environmentsEncryptionKeyBytes5(service.settings);
20924
+ const key = environmentsEncryptionKeyBytes4(service.settings);
21681
20925
  if (!key) throw new Error("Video generation recovery encryption key is unavailable");
21682
20926
  if (!operation.credentialEncrypted) {
21683
20927
  throw new Error("Video provider credential lease was erased too early");
@@ -22122,7 +21366,7 @@ async function durableXaiVideoAuth(service, operation, key, credential) {
22122
21366
  let currentEncrypted = frozenEncrypted;
22123
21367
  let connectedAccount = null;
22124
21368
  try {
22125
- connectedAccount = await materializeXaiCredentialForRun3(service.db, {
21369
+ connectedAccount = await materializeXaiCredentialForRun2(service.db, {
22126
21370
  workspaceId: operation.workspaceId,
22127
21371
  subjectId: current.subjectId,
22128
21372
  credentialId: current.credentialId,
@@ -22140,7 +21384,7 @@ async function durableXaiVideoAuth(service, operation, key, credential) {
22140
21384
  let next = null;
22141
21385
  if (connectedAccount) {
22142
21386
  try {
22143
- const result = await refreshXaiSubscriptionCredentialSerialized2(service.db, {
21387
+ const result = await refreshXaiSubscriptionCredentialSerialized(service.db, {
22144
21388
  accountId: operation.accountId,
22145
21389
  workspaceId: operation.workspaceId,
22146
21390
  subjectId: current.subjectId,
@@ -22154,14 +21398,14 @@ async function durableXaiVideoAuth(service, operation, key, credential) {
22154
21398
  if (!refreshToken) {
22155
21399
  throw new Error("The connected SuperGrok account cannot be refreshed");
22156
21400
  }
22157
- const tokens = await refreshXaiToken2(refreshToken);
21401
+ const tokens = await refreshXaiToken(refreshToken);
22158
21402
  return {
22159
21403
  secret: {
22160
21404
  version: 1,
22161
21405
  accessToken: tokens.accessToken,
22162
21406
  refreshToken: tokens.refreshToken
22163
21407
  },
22164
- expiresAt: xaiAccessTokenExpiry2(tokens.accessToken) ?? new Date(Date.now() + tokens.expiresInSeconds * 1e3)
21408
+ expiresAt: xaiAccessTokenExpiry(tokens.accessToken) ?? new Date(Date.now() + tokens.expiresInSeconds * 1e3)
22165
21409
  };
22166
21410
  }
22167
21411
  });
@@ -22179,7 +21423,7 @@ async function durableXaiVideoAuth(service, operation, key, credential) {
22179
21423
  }
22180
21424
  }
22181
21425
  if (!next) {
22182
- const tokens = await refreshXaiToken2(current.refreshToken);
21426
+ const tokens = await refreshXaiToken(current.refreshToken);
22183
21427
  next = Object.freeze({
22184
21428
  ...current,
22185
21429
  accessToken: tokens.accessToken,
@@ -22262,4 +21506,4 @@ export {
22262
21506
  createTurnActivities,
22263
21507
  createTurnActivitiesFromServices
22264
21508
  };
22265
- //# sourceMappingURL=activities-turn-I4OPLQLS.js.map
21509
+ //# sourceMappingURL=activities-turn-GHSFZB64.js.map