@opengeni/worker-bundle 0.12.7 → 0.13.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/dist/activities/agent-turn.d.ts +778 -0
  2. package/dist/activities/capabilities.d.ts +18 -0
  3. package/dist/activities/codex-auth.d.ts +1 -0
  4. package/dist/activities/codex-capacity.d.ts +31 -0
  5. package/dist/activities/codex-fleet-shadow.d.ts +74 -0
  6. package/dist/activities/codex-rotation.d.ts +261 -0
  7. package/dist/activities/common.d.ts +17 -0
  8. package/dist/activities/context-compaction.d.ts +81 -0
  9. package/dist/activities/documents.d.ts +39 -0
  10. package/dist/activities/environment.d.ts +97 -0
  11. package/dist/activities/expiring-material-renewal.d.ts +38 -0
  12. package/dist/activities/file-upload-reaper.d.ts +25 -0
  13. package/dist/activities/git-credential-renewal.d.ts +41 -0
  14. package/dist/activities/goals.d.ts +16 -0
  15. package/dist/activities/mcp-credentials.d.ts +14 -0
  16. package/dist/activities/packs.d.ts +77 -0
  17. package/dist/activities/parent-wake.d.ts +54 -0
  18. package/dist/activities/recording.d.ts +90 -0
  19. package/dist/activities/redaction.d.ts +1 -0
  20. package/dist/activities/rig-verification.d.ts +131 -0
  21. package/dist/activities/run-credential-renewal.d.ts +28 -0
  22. package/dist/activities/run-credentials.d.ts +36 -0
  23. package/dist/activities/run-input.d.ts +106 -0
  24. package/dist/activities/sandbox-lease.d.ts +128 -0
  25. package/dist/activities/scheduled-tasks.d.ts +4 -0
  26. package/dist/activities/session-state.d.ts +39 -0
  27. package/dist/activities/streaming.d.ts +24 -0
  28. package/dist/activities/toolspace-token-renewal.d.ts +23 -0
  29. package/dist/activities/turn-attempt-fenced.d.ts +11 -0
  30. package/dist/activities/types.d.ts +240 -0
  31. package/dist/activities/workflow-wake.d.ts +15 -0
  32. package/dist/activities/workspace-capture.d.ts +105 -0
  33. package/dist/activities.d.ts +312 -0
  34. package/dist/concurrency.d.ts +19 -0
  35. package/dist/host-export-pump.d.ts +66 -0
  36. package/dist/host-export.d.ts +6 -67
  37. package/dist/http.d.ts +32 -0
  38. package/dist/index.d.ts +27 -388
  39. package/dist/index.js +2730 -1211
  40. package/dist/index.js.map +1 -1
  41. package/dist/observability-metrics.d.ts +200 -0
  42. package/dist/op-journal.d.ts +10 -0
  43. package/dist/sandbox-resume.d.ts +152 -0
  44. package/dist/sandbox-routing.d.ts +159 -0
  45. package/dist/worker-service-lifecycle.d.ts +20 -0
  46. package/dist/workflow-bundle.js +88 -20
  47. package/dist/workflows/activities.d.ts +27 -0
  48. package/dist/workflows/document-index.d.ts +41 -0
  49. package/dist/workflows/file-upload-reaper.d.ts +2 -0
  50. package/dist/workflows/rig-verification.d.ts +3 -0
  51. package/dist/workflows/sandbox-reaper.d.ts +1 -0
  52. package/dist/workflows/scheduled-tasks.d.ts +17 -0
  53. package/dist/workflows/session.d.ts +58 -0
  54. package/dist/workflows/workflow-wake.d.ts +3 -0
  55. package/dist/workflows.d.ts +7 -0
  56. package/package.json +13 -13
  57. package/src/activities/agent-turn.ts +874 -134
  58. package/src/activities/context-compaction.ts +265 -76
  59. package/src/activities/environment.ts +1 -0
  60. package/src/activities/goals.ts +14 -0
  61. package/src/activities/rig-verification.ts +2 -0
  62. package/src/activities/run-input.ts +18 -4
  63. package/src/activities/sandbox-lease.ts +787 -133
  64. package/src/activities/session-state.ts +56 -0
  65. package/src/activities/types.ts +21 -0
  66. package/src/activities/workspace-capture.ts +95 -4
  67. package/src/activities.ts +3 -0
  68. package/src/observability-metrics.ts +94 -0
  69. package/src/sandbox-resume.ts +383 -90
  70. package/src/sandbox-routing.ts +32 -10
  71. package/src/workflows/activities.ts +1 -0
  72. package/src/workflows/session.ts +70 -0
@@ -42,6 +42,7 @@ import {
42
42
  countConsecutiveReactiveRotations,
43
43
  requireSession,
44
44
  recordUsageEvent,
45
+ recordModelCallFact,
45
46
  registerPendingSessionToolCall,
46
47
  recordPendingSessionToolCallResult,
47
48
  clearDurablePendingSessionToolCalls,
@@ -56,10 +57,12 @@ import {
56
57
  setSessionLastInputTokensForTurnAttempt,
57
58
  sumUsageQuantity,
58
59
  heartbeatLeaseHolder,
60
+ readLease,
59
61
  accrueWarmSeconds,
60
62
  getMaterializedSandboxFileResources,
61
63
  markSandboxFileResourcesMaterialized,
62
64
  areGitHubRepositoriesAllowedForWorkspace,
65
+ SandboxLeaseRecoveryBlockedError,
63
66
  SandboxLeaseSupersededError,
64
67
  SandboxImageConflictError,
65
68
  isSessionEventPersistenceError,
@@ -78,6 +81,7 @@ import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@
78
81
  import {
79
82
  sandboxStateEntryFromRunState,
80
83
  maxTurnsExceededRunState,
84
+ modelResponseServiceTierFromSdkEvent,
81
85
  modelResponseUsageFromSdkEvent,
82
86
  normalizeModelCallUsage,
83
87
  normalizeSdkEvent,
@@ -87,6 +91,9 @@ import {
87
91
  appendWorkspaceMemory,
88
92
  composeAgentInstructions,
89
93
  summarizeForCompaction,
94
+ requestRemoteCompactionV2,
95
+ REMOTE_COMPACTION_V2_BETA_FEATURE,
96
+ REMOTE_COMPACTION_V2_IMPLEMENTATION,
90
97
  CompactionProviderResponseError,
91
98
  EmptyCompactionSummaryError,
92
99
  ensureModalRegistryImage,
@@ -101,6 +108,7 @@ import {
101
108
  toolspaceTokenFileFromEnvironment,
102
109
  sandboxFileDownloadFailureNote,
103
110
  SUMMARY_BUFFER_TOKENS,
111
+ isMcpRequestTimeoutError,
104
112
  runOwnedSandboxSetup,
105
113
  RoutingMutationOutcomeUnknownError,
106
114
  WorkspaceArchiveIntegrityError,
@@ -130,7 +138,10 @@ import {
130
138
  calculateModelUsageCostMicros,
131
139
  configuredModelPricing,
132
140
  configuredStaticUsageLimits,
141
+ responseSatisfiesLatencyMode,
142
+ sandboxArchiveCaptureTimeoutMs,
133
143
  sandboxWarmRateMicrosPerSecond,
144
+ serviceTierForLatencyMode,
134
145
  settingsWithResolvedModelContext,
135
146
  resolveTurnExecutionPolicyV1,
136
147
  type ModelUsageInput,
@@ -138,7 +149,7 @@ import {
138
149
  type RegistryProviderKind,
139
150
  type Settings,
140
151
  } from "@opengeni/config";
141
- import { CancelledFailure } from "@temporalio/activity";
152
+ import { ApplicationFailure, CancelledFailure } from "@temporalio/activity";
142
153
  import {
143
154
  settingsWithCodexCredential,
144
155
  settingsWithEnabledCapabilityMcpServers,
@@ -174,21 +185,24 @@ import {
174
185
  buildModelResolver,
175
186
  CODEX_CLIENT_VERSION,
176
187
  CODEX_FALLBACK_MODEL_SLUGS,
188
+ CODEX_PROVIDER_ID,
177
189
  CodexReloginRequired,
190
+ classifyCodexEncryptedArtifactRejection,
178
191
  classifyCodexResponseTimeoutError,
179
192
  classifyCodexUsageLimitError,
180
193
  codexRequestStorage,
181
194
  isCodexTransportError,
195
+ withCodexRequestOverrides,
182
196
  type CodexRequestContext,
183
197
  type CodexUsageHeaderSnapshot,
184
198
  } from "@opengeni/codex";
185
199
  import { mergeResourceRefs } from "./common";
186
200
  import {
187
- enabledCapabilityMcpToolRefs,
201
+ assertSessionAllowsProductModel,
202
+ defaultSessionMcpServerIds,
188
203
  resolveSessionToolPolicy,
189
- sessionToolPolicyAllowsDefaultNativeTools,
190
204
  } from "@opengeni/core";
191
- import { maybeCompactContext } from "./context-compaction";
205
+ import { maybeCompactContext, settleFailedContextCompactionLandmark } from "./context-compaction";
192
206
  import { TurnAttemptFencedError } from "./turn-attempt-fenced";
193
207
  import {
194
208
  assertGitCredentialRenewalTransportUnchanged,
@@ -246,10 +260,15 @@ import {
246
260
  } from "./streaming";
247
261
  import type {
248
262
  ActivityServices,
263
+ EscapedMcpTimeoutRecoveryDetail,
249
264
  RunAgentTurnInput,
250
265
  RunAgentTurnResult,
251
266
  SessionAttemptQuiescenceProof,
252
267
  } from "./types";
268
+ import {
269
+ ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
270
+ ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
271
+ } from "./types";
253
272
  import {
254
273
  resumeBoxForTurn,
255
274
  sandboxLeaseHolderIdForAttempt,
@@ -286,7 +305,7 @@ import {
286
305
  prepareRecordingForSettlement,
287
306
  type ActiveRecording,
288
307
  } from "./recording";
289
- import { captureWorkspaceRevision } from "./workspace-capture";
308
+ import { captureWorkspaceRevision, openFreshWorkspaceCaptureSession } from "./workspace-capture";
290
309
  import type { ChannelASession } from "@opengeni/runtime/sandbox";
291
310
  import { createObjectStorage, type ObjectStorage } from "@opengeni/storage";
292
311
  import {
@@ -299,11 +318,13 @@ import {
299
318
  DEFAULT_FIRST_PARTY_MCP_TOOLS,
300
319
  evaluateWorkspaceModelPolicy,
301
320
  readTurnExecutionPolicyV1,
321
+ type LatencyMode,
302
322
  type ResourceRef,
303
323
  type SessionEvent,
304
324
  type SessionEventType,
305
325
  type SessionStatus,
306
326
  type SessionTurn,
327
+ type ToolAuthNeededPayload,
307
328
  type TurnExecutionPolicyV1,
308
329
  type TurnInitiator,
309
330
  } from "@opengeni/contracts";
@@ -322,6 +343,50 @@ export function credentialSubjectIdForTurnInitiator(
322
343
  return initiator.kind === "subject" ? initiator.subjectId : undefined;
323
344
  }
324
345
 
346
+ /**
347
+ * Direct authenticated-human work is the persisted human source with subject
348
+ * authority and no inherited agent or legacy provenance. Use this complete
349
+ * immutable turn authority instead of inferring causality from a
350
+ * `user.message` event shape or the root initiator alone.
351
+ */
352
+ export function isDirectHumanTurnInitiation(
353
+ turn: Pick<SessionTurn, "source" | "initiator" | "initiatorContext">,
354
+ ): boolean {
355
+ if (turn.source !== "user" || turn.initiator.kind !== "subject") {
356
+ return false;
357
+ }
358
+ return !["via", "viaTruncated", "provenanceError", "backfill"].some((key) =>
359
+ Object.prototype.hasOwnProperty.call(turn.initiatorContext, key),
360
+ );
361
+ }
362
+
363
+ /**
364
+ * A disconnected personal Slack server is prepared best-effort before the model
365
+ * runs. Its initialize/tools-list credential miss is setup state, not evidence
366
+ * that an unrelated turn wants Slack. Keep concrete tool-call failures
367
+ * actionable, but gate setup-time Slack prompts to a human message that names
368
+ * Slack explicitly. Other providers retain the existing generic behavior.
369
+ */
370
+ export function shouldPublishToolAuthNeededForTurn(
371
+ payload: Pick<ToolAuthNeededPayload, "providerDomain" | "toolName">,
372
+ trigger: Pick<SessionEvent, "type" | "payload">,
373
+ turn: Pick<SessionTurn, "source" | "initiator" | "initiatorContext">,
374
+ ): boolean {
375
+ if (typeof payload.toolName === "string" && payload.toolName.trim().length > 0) {
376
+ return true;
377
+ }
378
+ const providerDomain = payload.providerDomain.trim().toLowerCase();
379
+ const isSlack = providerDomain === "slack.com" || providerDomain.endsWith(".slack.com");
380
+ if (!isSlack) {
381
+ return true;
382
+ }
383
+ if (!isDirectHumanTurnInitiation(turn) || trigger.type !== "user.message") {
384
+ return false;
385
+ }
386
+ const text = (trigger.payload as { text?: unknown }).text;
387
+ return typeof text === "string" && /\bslack\b/i.test(text);
388
+ }
389
+
325
390
  export function turnExecutionPolicyBillingIdentity(policy: TurnExecutionPolicyV1): {
326
391
  externallyBilled: boolean;
327
392
  codexSubscription: boolean;
@@ -336,7 +401,7 @@ export function turnExecutionPolicyBillingIdentity(policy: TurnExecutionPolicyV1
336
401
  }
337
402
 
338
403
  export function legacyTurnExecutionPolicyInput(
339
- turn: Pick<SessionTurn, "source" | "model" | "reasoningEffort">,
404
+ turn: Pick<SessionTurn, "source" | "model" | "reasoningEffort" | "latencyMode">,
340
405
  ): Parameters<typeof resolveTurnExecutionPolicyV1>[1] {
341
406
  const explicit = turn.source === "user" || turn.source === "api";
342
407
  return {
@@ -345,6 +410,8 @@ export function legacyTurnExecutionPolicyInput(
345
410
  modelSource: explicit ? "explicit" : "continuation",
346
411
  reasoningEffort: turn.reasoningEffort,
347
412
  reasoningSource: explicit ? "explicit" : "continuation",
413
+ latencyMode: turn.latencyMode,
414
+ latencyModeSource: explicit ? "explicit" : "continuation",
348
415
  };
349
416
  }
350
417
 
@@ -361,6 +428,56 @@ export function providerRecoveryResult(): {
361
428
  };
362
429
  }
363
430
 
431
+ /**
432
+ * Tell the model when durable MCP policy references are unavailable for this
433
+ * exact turn. The policy projection already bounds and validates these ids;
434
+ * this notice prevents a graceful runtime drop from becoming a silent source-
435
+ * of-truth substitution or a false claim that the disconnected system was read.
436
+ */
437
+ export function unavailableMcpTurnInstructions(input: {
438
+ droppedIds: readonly string[];
439
+ droppedCount: number;
440
+ }): string | undefined {
441
+ if (input.droppedCount <= 0) {
442
+ return undefined;
443
+ }
444
+ const omittedCount = Math.max(0, input.droppedCount - input.droppedIds.length);
445
+ const listed = input.droppedIds.map((id) => `"${id}"`).join(", ");
446
+ const inventory = listed
447
+ ? `${listed}${omittedCount > 0 ? `, plus ${omittedCount} additional unavailable server(s)` : ""}`
448
+ : `${input.droppedCount} unavailable server(s)`;
449
+ return `MCP capability availability for this turn: the following session-selected server(s) are disconnected or no longer registered and were skipped: ${inventory}. Do not claim to have read or updated those systems. If the task depends on one as a source of truth, explain the limitation and ask the user to reconnect it or select another authoritative source; continue with unaffected work only when safe.`;
450
+ }
451
+
452
+ /**
453
+ * Preserve one precise recovery obligation across Temporal's activity boundary.
454
+ * This is intentionally narrower than the ordinary retryable-provider path:
455
+ * only a recovered turn (generation > 1) whose MCP timeout happened before a
456
+ * model request may ask the workflow's DB-only control activity to finish the
457
+ * same-turn checkpoint. The original transport/recovery errors are excluded
458
+ * from details so raw MCP response data can never enter workflow history.
459
+ */
460
+ export function escapedMcpTimeoutRecoveryFailure(input: {
461
+ failureCode: string | undefined;
462
+ modelRequestStarted: boolean;
463
+ detail: EscapedMcpTimeoutRecoveryDetail;
464
+ }): ApplicationFailure | null {
465
+ if (
466
+ input.failureCode !== "mcp_transport_timeout" ||
467
+ input.modelRequestStarted ||
468
+ !Number.isSafeInteger(input.detail.executionGeneration) ||
469
+ input.detail.executionGeneration <= 1
470
+ ) {
471
+ return null;
472
+ }
473
+ return ApplicationFailure.create({
474
+ message: ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
475
+ type: ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
476
+ nonRetryable: true,
477
+ details: [input.detail],
478
+ });
479
+ }
480
+
364
481
  /**
365
482
  * Resolve which Codex account a turn runs on (multi-account P1): session-pin >
366
483
  * workspace-active. No rotation in P1. The selected id must still be in the
@@ -762,14 +879,14 @@ export function classifyContextWindowOverflowError(
762
879
  * next-loop transport work can reject the stream iterator after a prior tool
763
880
  * output was already published. That is transient external backpressure, not a
764
881
  * terminal session error. Match MCP-qualified timeout text only: an unrelated
765
- * sandbox/model timeout and MCP's `-32001 Authentication required` signal must
882
+ * sandbox/model timeout and MCP's application-defined Authentication required signal must
766
883
  * retain their existing semantics.
767
884
  */
768
885
  export function classifyMcpTransportTimeoutError(
769
886
  error: unknown,
770
887
  ): { message: string; detail?: string } | null {
771
888
  const fields = collectErrorStrings(error);
772
- const matched = fields.find(
889
+ const matchedText = fields.find(
773
890
  (value) =>
774
891
  /\bmcp\b/i.test(value) &&
775
892
  /(?:request\s+timed\s+out|request\s+timeout|\btimed\s+out\b|\btimeout\b|ETIMEDOUT)/i.test(
@@ -777,13 +894,15 @@ export function classifyMcpTransportTimeoutError(
777
894
  ) &&
778
895
  !/authentication\s+required/i.test(value),
779
896
  );
780
- if (!matched) {
897
+ const sanitizedSdkTimeout = isMcpRequestTimeoutError(error);
898
+ if (!matchedText && !sanitizedSdkTimeout) {
781
899
  return null;
782
900
  }
783
901
  const message = error instanceof Error ? error.message : String(error);
902
+ const matched = matchedText ?? fields.find((value) => /\bmcp\b/i.test(value));
784
903
  return {
785
904
  message,
786
- ...(matched !== message ? { detail: matched } : {}),
905
+ ...(matched && matched !== message ? { detail: matched } : {}),
787
906
  };
788
907
  }
789
908
 
@@ -946,6 +1065,37 @@ export function modelResponseUsageContextSignal(
946
1065
  };
947
1066
  }
948
1067
 
1068
+ export function assertModelResponseLatencyMode(input: {
1069
+ event: Parameters<typeof modelResponseServiceTierFromSdkEvent>[0];
1070
+ requested: LatencyMode;
1071
+ model: string;
1072
+ /** When set to Codex ChatGPT auth, response `service_tier` is not an honor signal. */
1073
+ providerId?: string;
1074
+ }): void {
1075
+ if (input.requested === "standard") {
1076
+ return;
1077
+ }
1078
+ // ChatGPT-auth Codex (subscription): Fast maps to request `service_tier=priority`
1079
+ // (see openai/codex ServiceTier::Fast.request_value). The backend may still return
1080
+ // `response.service_tier=default`; OpenAI maintainers document that this does not
1081
+ // mean Fast was ignored. Native CLI also does not fail closed on that field.
1082
+ if (input.providerId === CODEX_PROVIDER_ID) {
1083
+ return;
1084
+ }
1085
+ const serviceTierEvent = modelResponseServiceTierFromSdkEvent(input.event);
1086
+ if (
1087
+ !serviceTierEvent ||
1088
+ (serviceTierEvent.source === "normalized" && serviceTierEvent.serviceTier === null)
1089
+ ) {
1090
+ return;
1091
+ }
1092
+ if (!responseSatisfiesLatencyMode(input.requested, serviceTierEvent.serviceTier)) {
1093
+ throw new Error(
1094
+ `Provider did not honor ${input.requested} latency mode for ${input.model}: response service_tier=${serviceTierEvent.serviceTier ?? "missing"}`,
1095
+ );
1096
+ }
1097
+ }
1098
+
949
1099
  /**
950
1100
  * Process one SDK stream event through the exact production usage path.
951
1101
  *
@@ -973,6 +1123,7 @@ export async function processModelResponseUsageEvent(input: {
973
1123
  provider: string;
974
1124
  providerApi: ModelProviderApi;
975
1125
  model: string;
1126
+ latencyMode?: LatencyMode;
976
1127
  metricProvider: string;
977
1128
  externallyBilled: boolean;
978
1129
  servingCredentialId: string | null;
@@ -1016,7 +1167,7 @@ export async function processModelResponseUsageEvent(input: {
1016
1167
  leaseLost: input.leaseLost,
1017
1168
  leaseLostMessage: input.leaseLostMessage,
1018
1169
  recordUsage: async () => {
1019
- await recordModelUsageAndDebitCredits(input.settings, input.db, {
1170
+ const billing = await recordModelUsageAndDebitCredits(input.settings, input.db, {
1020
1171
  accountId: input.accountId,
1021
1172
  workspaceId: input.workspaceId,
1022
1173
  sessionId: input.sessionId,
@@ -1027,6 +1178,7 @@ export async function processModelResponseUsageEvent(input: {
1027
1178
  usage: responseUsage.usage,
1028
1179
  normalizedUsage,
1029
1180
  sourceKey,
1181
+ ...(input.latencyMode ? { latencyMode: input.latencyMode } : {}),
1030
1182
  observability: input.observability,
1031
1183
  });
1032
1184
  authoritative = await emitModelCallUsage({
@@ -1046,6 +1198,22 @@ export async function processModelResponseUsageEvent(input: {
1046
1198
  accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
1047
1199
  emittedSourceKeys: input.emittedSourceKeys,
1048
1200
  });
1201
+ if (authoritative && billing) {
1202
+ await recordAuthoritativeModelCallFact({
1203
+ db: input.db,
1204
+ observability: input.observability,
1205
+ accountId: input.accountId,
1206
+ workspaceId: input.workspaceId,
1207
+ sessionId: input.sessionId,
1208
+ turnId: input.turnId,
1209
+ turnAttemptId: input.turnAttemptId,
1210
+ sourceKey,
1211
+ provider: input.provider,
1212
+ providerApi: input.providerApi,
1213
+ model: input.model,
1214
+ billing,
1215
+ });
1216
+ }
1049
1217
  const observedInput = normalizedUsage.telemetry.inputTokens;
1050
1218
  if (authoritative && observedInput !== null && observedInput > 0) {
1051
1219
  recordModelInputTokens(input.observability, input.metricProvider, observedInput);
@@ -1125,7 +1293,7 @@ export async function processCompactionModelUsageEvent(input: {
1125
1293
  leaseLost: input.leaseLost,
1126
1294
  leaseLostMessage: input.leaseLostMessage,
1127
1295
  recordUsage: async () => {
1128
- await recordModelUsageAndDebitCredits(input.settings, input.db, {
1296
+ const billing = await recordModelUsageAndDebitCredits(input.settings, input.db, {
1129
1297
  accountId: input.accountId,
1130
1298
  workspaceId: input.workspaceId,
1131
1299
  sessionId: input.sessionId,
@@ -1155,6 +1323,22 @@ export async function processCompactionModelUsageEvent(input: {
1155
1323
  accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
1156
1324
  emittedSourceKeys: input.emittedSourceKeys,
1157
1325
  });
1326
+ if (authoritative && billing) {
1327
+ await recordAuthoritativeModelCallFact({
1328
+ db: input.db,
1329
+ observability: input.observability,
1330
+ accountId: input.accountId,
1331
+ workspaceId: input.workspaceId,
1332
+ sessionId: input.sessionId,
1333
+ turnId: input.turnId,
1334
+ turnAttemptId: input.turnAttemptId,
1335
+ sourceKey,
1336
+ provider: input.provider,
1337
+ providerApi: input.providerApi,
1338
+ model: input.model,
1339
+ billing,
1340
+ });
1341
+ }
1158
1342
  },
1159
1343
  });
1160
1344
  return { status: "processed", sourceKey, authoritative };
@@ -1651,9 +1835,9 @@ export function shouldStartOnTurnRecording(params: {
1651
1835
  * and threaded to the runtime as an explicit flag (buildAgent → computerToolMode):
1652
1836
  * • codex-subscription → "function-image": the ChatGPT/Codex backend rejects
1653
1837
  * hosted tool types but SEES structured `input_image` tool results.
1654
- * • a "chat" (OpenAIChatCompletionsModel wire) provider → "function-text": it takes
1655
- * function tools but can't read structured image results, so screenshots render
1656
- * as a text data-URL.
1838
+ * • a "chat" (OpenAIChatCompletionsModel wire) provider → "disabled": it cannot
1839
+ * receive a screenshot through a proven visual transport, so computer use fails
1840
+ * closed instead of serializing the image as text.
1657
1841
  * • everything else — built-in Azure/OpenAI responses, registry "responses"
1658
1842
  * providers, AND the LEGACY global-client fallback (resolveTurnModel returned
1659
1843
  * null) — → "hosted": real Responses hosted-tool support.
@@ -1672,24 +1856,22 @@ export function computerToolModeForTurn(
1672
1856
  return "function-image";
1673
1857
  }
1674
1858
  if (resolvedModel.provider.api === "chat") {
1675
- return "function-text";
1859
+ return "disabled";
1676
1860
  }
1677
1861
  return "hosted";
1678
1862
  }
1679
1863
 
1680
1864
  /**
1681
- * Native web search requires two independent grants: the resolved provider
1682
- * must advertise runnable support, and the immutable session/turn policy must
1683
- * still track workspace defaults. The null model is the legacy built-in
1684
- * Responses path, whose deployment flag is its provider capability gate.
1685
- * There is deliberately no cross-provider or sandbox/curl fallback.
1865
+ * Native web search is a runtime capability, not part of the session's MCP
1866
+ * allow-list. Attach it whenever the resolved provider advertises runnable
1867
+ * support. The null model is the legacy built-in Responses path, whose
1868
+ * deployment flag is its provider capability gate. There is deliberately no
1869
+ * cross-provider or sandbox/curl fallback.
1686
1870
  */
1687
1871
  export function hostedWebSearchForTurn(
1688
1872
  resolvedModel: { configured: { hostedWebSearch: boolean } } | null,
1689
1873
  deploymentWebSearchEnabled: boolean,
1690
- defaultNativeToolsAllowed: boolean,
1691
1874
  ): boolean {
1692
- if (!defaultNativeToolsAllowed) return false;
1693
1875
  return resolvedModel?.configured.hostedWebSearch ?? deploymentWebSearchEnabled;
1694
1876
  }
1695
1877
 
@@ -1722,6 +1904,19 @@ export class TurnOperationCancelledError extends Error {
1722
1904
  }
1723
1905
  }
1724
1906
 
1907
+ export class SandboxDeadlineRotationError extends Error {
1908
+ readonly name = "SandboxDeadlineRotationError";
1909
+
1910
+ constructor(
1911
+ readonly sandboxGroupId: string,
1912
+ readonly leaseEpoch: number,
1913
+ ) {
1914
+ super(
1915
+ `Sandbox ${sandboxGroupId} reached its provider rotation boundary at lease epoch ${leaseEpoch}`,
1916
+ );
1917
+ }
1918
+ }
1919
+
1725
1920
  /**
1726
1921
  * Normalize a preparation/provisioning cancellation race back to the Temporal
1727
1922
  * cancellation that owns the activity. Several provider APIs expose no portable
@@ -1797,7 +1992,10 @@ function sleep(ms: number): Promise<void> {
1797
1992
  }
1798
1993
 
1799
1994
  export function isLazySandboxProvisionRetryable(error: unknown): boolean {
1800
- if (error instanceof SandboxImageConflictError) {
1995
+ if (
1996
+ error instanceof SandboxImageConflictError ||
1997
+ error instanceof SandboxLeaseRecoveryBlockedError
1998
+ ) {
1801
1999
  return false;
1802
2000
  }
1803
2001
  if (error instanceof WorkspaceArchiveIntegrityError) {
@@ -1812,6 +2010,20 @@ export function isLazySandboxProvisionRetryable(error: unknown): boolean {
1812
2010
  );
1813
2011
  }
1814
2012
 
2013
+ /**
2014
+ * Deadline recovery cannot re-admit the same turn until the global reaper has
2015
+ * observed the released holder, captured the final archive, terminated the old
2016
+ * provider box, and committed the lease cold. Give that full sequence one
2017
+ * snapshot budget plus two schedule periods. This is workflow-visible pacing,
2018
+ * not an in-memory sleep, so Pause/Steer/new user intent can still wake the
2019
+ * durable session immediately.
2020
+ */
2021
+ export function sandboxDeadlineRotationRecoveryDelayMs(
2022
+ settings: Pick<Settings, "sandboxLeaseReaperPeriodMs" | "sandboxSnapshotTimeoutMs">,
2023
+ ): number {
2024
+ return settings.sandboxSnapshotTimeoutMs + settings.sandboxLeaseReaperPeriodMs * 2;
2025
+ }
2026
+
1815
2027
  export function createTurnSandboxProvisioner<T>(
1816
2028
  establish: () => Promise<T>,
1817
2029
  options: {
@@ -1874,7 +2086,15 @@ export function createTurnSandboxProvisioner<T>(
1874
2086
  throw error;
1875
2087
  }
1876
2088
  })().catch((error) => {
1877
- memo = null;
2089
+ // A terminal lease/archive/configuration result cannot change within
2090
+ // this frozen turn. Keep its rejected promise memoized so later model
2091
+ // tool calls observe the same typed result without re-entering
2092
+ // provisioning, emitting another failure event, or creating another
2093
+ // provider attempt. Only a retryable failure releases the single-flight
2094
+ // for a later operation to re-read the lease/provider state.
2095
+ if (isLazySandboxProvisionRetryable(error)) {
2096
+ memo = null;
2097
+ }
1878
2098
  throw error;
1879
2099
  });
1880
2100
  }
@@ -2014,6 +2234,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2014
2234
  } = await services();
2015
2235
  const activityContext = currentActivityContext();
2016
2236
  const cancellationSignal = activityContext?.cancellationSignal;
2237
+ // Temporal cancellation is not the only way an activity loses ownership:
2238
+ // a database attempt fence or workflow abandonment can finalize the logical
2239
+ // turn while the SDK cancellation signal remains open. Bind sandbox
2240
+ // provisioning to both lifetimes so an uninterruptible, late provider
2241
+ // promise cannot resurrect a holder after this activity has finalized.
2242
+ const sandboxResumeController = new AbortController();
2243
+ const sandboxResumeSignal = cancellationSignal
2244
+ ? AbortSignal.any([cancellationSignal, sandboxResumeController.signal])
2245
+ : sandboxResumeController.signal;
2246
+ const sandboxRotationController = new AbortController();
2247
+ const runtimeCancellationSignal = cancellationSignal
2248
+ ? AbortSignal.any([cancellationSignal, sandboxRotationController.signal])
2249
+ : sandboxRotationController.signal;
2017
2250
  let cancellationRequestedAt: number | null = cancellationSignal?.aborted
2018
2251
  ? performance.now()
2019
2252
  : null;
@@ -2067,6 +2300,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2067
2300
  let isCodexTurn = false;
2068
2301
  let isExternallyBilledTurn = false;
2069
2302
  let executionGeneration = 0;
2303
+ let modelRequestStarted = false;
2070
2304
  // Still required by credential-loss/capacity settlements, whose own
2071
2305
  // recovery transactions fence against worker-death redispatches.
2072
2306
  let redispatchesAtDispatch = 0;
@@ -2274,6 +2508,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2274
2508
  expectedEpoch: sandbox.leaseEpoch,
2275
2509
  expectedInstanceId: sandbox.established.instanceId,
2276
2510
  operation,
2511
+ captureWaitMs: sandboxArchiveCaptureTimeoutMs(settings),
2277
2512
  };
2278
2513
  const admission = await advanceWorkspaceGeneration(db, identity);
2279
2514
  let result: T;
@@ -2314,6 +2549,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2314
2549
  // runs it refreshes expires_at epoch-fenced so a legit multi-day turn is
2315
2550
  // never TTL-reaped. Cleared in finally. Only set when the flag resolved a box.
2316
2551
  let leaseHeartbeatTimer: ReturnType<typeof setInterval> | undefined;
2552
+ const stopLeaseHeartbeat = (): void => {
2553
+ if (!leaseHeartbeatTimer) return;
2554
+ clearInterval(leaseHeartbeatTimer);
2555
+ leaseHeartbeatTimer = undefined;
2556
+ };
2557
+ let rotationInFlight: Promise<void> | null = null;
2317
2558
  // credential-renewal policy: the worker, not the model, owns renewal of run-scoped Git
2318
2559
  // credentials for a multi-day turn. The controller is attached only after
2319
2560
  // the initial seed reached a real cloud box and is drained before capture.
@@ -2337,6 +2578,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2337
2578
  // the atomic DB throttle discard the fresher one). Interval throttling
2338
2579
  // itself lives in maybePersistWarmWorkspaceSnapshot / persistWarmSnapshot.
2339
2580
  let snapshotInFlight: Promise<void> | null = null;
2581
+ // Turn-end capture needs the lease heartbeat to keep its holder alive, but
2582
+ // must prevent that same timer from starting another periodic snapshot
2583
+ // while it reads. This gate separates those two responsibilities.
2584
+ let turnEndCaptureInProgress = false;
2340
2585
  // Computer-use-only recording. Ordinary shell/filesystem turns leave this
2341
2586
  // null; the first actual computer action starts it after :0 is ready.
2342
2587
  let activeRecording: ActiveRecording | null = null;
@@ -2378,10 +2623,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2378
2623
  await current?.flush().catch(() => undefined);
2379
2624
  };
2380
2625
  let preparedTools: Awaited<ReturnType<OpenGeniRuntime["prepareTools"]>> | null = null;
2381
- // Resolved with the MCP policy at the immutable turn boundary. Provider
2382
- // capability is checked separately when buildAgent is called; this flag is
2383
- // only the durable omission/narrowing entitlement.
2384
- let defaultNativeToolsAllowed = false;
2385
2626
  const toolCancellationFenceRef: {
2386
2627
  current: TurnToolCancellationFence | null;
2387
2628
  } = {
@@ -2488,7 +2729,81 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2488
2729
  holderId: heartbeatHolderId,
2489
2730
  leaseTtlMs: settings.sandboxLeaseTtlMs,
2490
2731
  expectedEpoch: heartbeatEpoch,
2491
- }).catch(() => undefined);
2732
+ })
2733
+ .then(async (alive) => {
2734
+ if (alive || rotationInFlight || sandboxRotationController.signal.aborted) return;
2735
+ const rotatingLease = await readLease(db, input.workspaceId, heartbeatGroupId).catch(
2736
+ () => null,
2737
+ );
2738
+ if (
2739
+ !rotatingLease ||
2740
+ rotatingLease.leaseEpoch !== heartbeatEpoch ||
2741
+ rotatingLease.instanceId !== sandbox.established.instanceId ||
2742
+ rotatingLease.rotationRequestedAt === null
2743
+ ) {
2744
+ // The holder was reaped, the exact attempt closed, the epoch was
2745
+ // superseded, or the lease began draining. Do not leave a dead
2746
+ // interval issuing DB writes and snapshot probes forever.
2747
+ stopLeaseHeartbeat();
2748
+ return;
2749
+ }
2750
+ rotationInFlight = (async () => {
2751
+ // Rotation admission already fenced all new workspace mutations.
2752
+ // Wait for an earlier periodic capture, then produce the exact
2753
+ // generation-complete checkpoint that licenses aborting this run.
2754
+ if (snapshotInFlight) {
2755
+ await waitForWarmSnapshot(
2756
+ snapshotInFlight,
2757
+ settings.sandboxSnapshotTimeoutMs,
2758
+ cancellationSignal,
2759
+ );
2760
+ }
2761
+ const snapshotSession = setupBoxSession;
2762
+ const snapshotTurnId = turnId;
2763
+ if (snapshotSession && snapshotTurnId) {
2764
+ await maybePersistWarmWorkspaceSnapshot(
2765
+ { db, settings },
2766
+ {
2767
+ accountId: input.accountId,
2768
+ workspaceId: input.workspaceId,
2769
+ sessionId: input.sessionId,
2770
+ turnId: snapshotTurnId,
2771
+ attemptId: input.attemptId,
2772
+ sandboxGroupId: heartbeatGroupId,
2773
+ },
2774
+ snapshotSession,
2775
+ heartbeatEpoch,
2776
+ cancellationSignal,
2777
+ true,
2778
+ );
2779
+ }
2780
+ const checkpointed = await readLease(db, input.workspaceId, heartbeatGroupId);
2781
+ if (
2782
+ checkpointed?.leaseEpoch === heartbeatEpoch &&
2783
+ checkpointed.rotationRequestedAt !== null &&
2784
+ checkpointed.archiveComplete &&
2785
+ !cancellationSignal?.aborted
2786
+ ) {
2787
+ sandboxRotationController.abort(
2788
+ new SandboxDeadlineRotationError(heartbeatGroupId, heartbeatEpoch),
2789
+ );
2790
+ }
2791
+ })()
2792
+ .catch((error) => {
2793
+ observability.warn("sandbox deadline rotation checkpoint failed; retrying", {
2794
+ workspaceId: input.workspaceId,
2795
+ sessionId: input.sessionId,
2796
+ sandboxGroupId: heartbeatGroupId,
2797
+ leaseEpoch: heartbeatEpoch,
2798
+ error: error instanceof Error ? error.message : String(error),
2799
+ });
2800
+ })
2801
+ .finally(() => {
2802
+ rotationInFlight = null;
2803
+ });
2804
+ await rotationInFlight;
2805
+ })
2806
+ .catch(() => undefined);
2492
2807
  void accrueWarmSeconds(db, {
2493
2808
  accountId: input.accountId,
2494
2809
  workspaceId: input.workspaceId,
@@ -2510,7 +2825,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2510
2825
  // single-flight; throttling lives in the helper.
2511
2826
  const snapshotSession = setupBoxSession;
2512
2827
  const snapshotTurnId = turnId;
2513
- if (snapshotSession && snapshotTurnId && !snapshotInFlight) {
2828
+ if (snapshotSession && snapshotTurnId && !snapshotInFlight && !turnEndCaptureInProgress) {
2514
2829
  snapshotInFlight = maybePersistWarmWorkspaceSnapshot(
2515
2830
  { db, settings },
2516
2831
  {
@@ -2806,9 +3121,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2806
3121
  {
2807
3122
  modelId: turn.model,
2808
3123
  reasoningEffort: turn.reasoningEffort,
3124
+ latencyMode: turn.latencyMode,
2809
3125
  },
2810
3126
  );
2811
3127
  const turnExecutionPolicy = verifiedExecutionPolicy.policy;
3128
+ assertSessionAllowsProductModel(session, turnExecutionPolicy.productModelId);
2812
3129
  const billingIdentity = turnExecutionPolicyBillingIdentity(turnExecutionPolicy);
2813
3130
  isExternallyBilledTurn = billingIdentity.externallyBilled;
2814
3131
  isCodexTurn = billingIdentity.codexSubscription;
@@ -3780,12 +4097,22 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3780
4097
  },
3781
4098
  ]
3782
4099
  : !resolvedModel && fallbackProviderApiKey
3783
- ? [{ name: "MODEL_PROVIDER_API_KEY", value: fallbackProviderApiKey }]
4100
+ ? [
4101
+ {
4102
+ name: "MODEL_PROVIDER_API_KEY",
4103
+ value: fallbackProviderApiKey,
4104
+ },
4105
+ ]
3784
4106
  : []),
3785
4107
  ...Object.entries(selectedProvider?.defaultHeaders ?? {}).flatMap(([name, value]) =>
3786
4108
  publicProviderHeaders.has(name.toLowerCase())
3787
4109
  ? []
3788
- : [{ name: `MODEL_PROVIDER_HEADER_${name.toUpperCase()}`, value }],
4110
+ : [
4111
+ {
4112
+ name: `MODEL_PROVIDER_HEADER_${name.toUpperCase()}`,
4113
+ value,
4114
+ },
4115
+ ],
3789
4116
  ),
3790
4117
  ...Object.entries(selectedProvider?.defaultQuery ?? {}).flatMap(([name, value]) =>
3791
4118
  publicProviderQueries.has(name.toLowerCase())
@@ -3898,6 +4225,22 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3898
4225
  : null;
3899
4226
  const withCodex = <T>(fn: () => Promise<T>): Promise<T> =>
3900
4227
  codexContext ? codexRequestStorage.run(codexContext, fn) : fn();
4228
+ const withCodexRemoteCompaction = <T>(fn: () => Promise<T>): Promise<T> =>
4229
+ withCodex(() =>
4230
+ withCodexRequestOverrides(
4231
+ {
4232
+ betaFeatures: [REMOTE_COMPACTION_V2_BETA_FEATURE],
4233
+ turnMetadata: {
4234
+ request_kind: "compaction",
4235
+ compaction: {
4236
+ implementation: REMOTE_COMPACTION_V2_IMPLEMENTATION,
4237
+ strategy: "memento",
4238
+ },
4239
+ },
4240
+ },
4241
+ fn,
4242
+ ),
4243
+ );
3901
4244
  const promptCacheKey = acceptsPromptCacheKeyForTurn(resolvedModel)
3902
4245
  ? input.sessionId
3903
4246
  : undefined;
@@ -3952,6 +4295,41 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3952
4295
  ...(systemInstructions ? { systemInstructions } : {}),
3953
4296
  ...(promptCacheKey ? { promptCacheKey } : {}),
3954
4297
  });
4298
+ const remoteCompactionRequester =
4299
+ resolvedModel && isCodexTurn
4300
+ ? (s: Settings, m: Array<Record<string, unknown>>) =>
4301
+ withCodexRemoteCompaction(() =>
4302
+ requestRemoteCompactionV2(s, m, {
4303
+ client: resolvedModel.client,
4304
+ model: turnExecutionPolicy.upstreamModelId,
4305
+ onUsage: recordCompactionUsage,
4306
+ ...(promptCacheKey ? { promptCacheKey } : {}),
4307
+ }),
4308
+ )
4309
+ : undefined;
4310
+ const publishCompactionLiveEvents = async (events: SessionEvent[]) => {
4311
+ await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events);
4312
+ };
4313
+ const publishCompactionOutcomeEvents = async (events: SessionEvent[]) => {
4314
+ // `compaction.started` was already fanout via publishCompactionLiveEvents.
4315
+ await publishDurableSessionEvents(
4316
+ bus,
4317
+ input.workspaceId,
4318
+ input.sessionId,
4319
+ events.filter((event) => event.type !== "session.context.compaction.started"),
4320
+ );
4321
+ };
4322
+ const compactionModeOptions = {
4323
+ codexCompactionMode: session.codexCompactionMode,
4324
+ isCodexSubscriptionTurn: isCodexTurn,
4325
+ publishLiveEvents: publishCompactionLiveEvents,
4326
+ ...(remoteCompactionRequester
4327
+ ? { requestRemoteCompactionV2: remoteCompactionRequester }
4328
+ : {}),
4329
+ ...(effectiveCodexCredentialId
4330
+ ? { producerCodexCredentialId: effectiveCodexCredentialId }
4331
+ : {}),
4332
+ } as const;
3955
4333
 
3956
4334
  if (turn.source === "compaction") {
3957
4335
  const persistentSessionSettings = {
@@ -3997,6 +4375,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3997
4375
  force: true,
3998
4376
  clearRequestedCompaction: true,
3999
4377
  trigger: "operator",
4378
+ codexAccount: {
4379
+ currentCodexCredentialId: effectiveCodexCredentialId,
4380
+ },
4381
+ ...compactionModeOptions,
4000
4382
  },
4001
4383
  ),
4002
4384
  cancellationSignal,
@@ -4008,6 +4390,24 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4008
4390
  // request pending and let the ordinary same-turn provider/capacity
4009
4391
  // recovery path re-dispatch this exact maintenance execution.
4010
4392
  if (shouldRecoverCompactionProviderFailure(error)) throw error;
4393
+ if (error instanceof TurnAttemptFencedError) throw error;
4394
+ // `compaction.started` may already be live — settle skipped before
4395
+ // failing the turn so the timeline cannot stick on Compacting…
4396
+ const landmark = await settleFailedContextCompactionLandmark(
4397
+ db,
4398
+ {
4399
+ accountId: input.accountId,
4400
+ workspaceId: input.workspaceId,
4401
+ sessionId: input.sessionId,
4402
+ turnId: turn.id,
4403
+ executionGeneration,
4404
+ attemptId: input.attemptId,
4405
+ },
4406
+ {
4407
+ clearRequestedCompaction: true,
4408
+ publishLiveEvents: publishCompactionLiveEvents,
4409
+ },
4410
+ );
4011
4411
  if (!isCompactionSummaryFailure(error)) throw error;
4012
4412
  const errorMessage = String(redact(compactionFailureReasonFromError(error)));
4013
4413
  if (
@@ -4031,7 +4431,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4031
4431
  turnStatus: "failed",
4032
4432
  sessionStatus: "idle",
4033
4433
  activeTurnId: null,
4034
- consumeRequestedCompactionFailure: true,
4434
+ ...(landmark.requestConsumed ? {} : { consumeRequestedCompactionFailure: true }),
4035
4435
  }))
4036
4436
  ) {
4037
4437
  return claimedResult({ status: "cancelled" });
@@ -4045,12 +4445,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4045
4445
  if (outcome.compacted) {
4046
4446
  recordContextCompaction(observability, "operator");
4047
4447
  }
4048
- await publishDurableSessionEvents(
4049
- bus,
4050
- input.workspaceId,
4051
- input.sessionId,
4052
- outcome.events,
4053
- );
4448
+ await publishCompactionOutcomeEvents(outcome.events);
4054
4449
  }
4055
4450
  }
4056
4451
  if (
@@ -4084,23 +4479,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4084
4479
  // narrowed by the session's exact firstPartyMcpTools selection and
4085
4480
  // authorization. Idempotent: mergeToolRefs dedupes if already present.
4086
4481
  // Resolve the durable policy at the turn boundary. Workspace-default
4087
- // sessions may discover newly enabled capability MCPs and Codex Apps,
4482
+ // sessions follow the current configured MCP set,
4088
4483
  // while explicit, inherited-fixed, and legacy sessions remain narrowed
4089
4484
  // to their stored materialized allow-list.
4090
4485
  const resolvedToolPolicy = resolveSessionToolPolicy({
4091
- ...(session.toolPolicy ? { toolPolicy: session.toolPolicy } : {}),
4486
+ toolPolicy: session.toolPolicy,
4092
4487
  sessionTools: session.tools,
4093
- turnTools: turn.tools,
4094
- ...(turn.toolsProvided !== undefined ? { turnToolsProvided: turn.toolsProvided } : {}),
4095
4488
  availableMcpServerIds: runSettings.mcpServers.map((server) => server.id),
4096
- defaultMcpServerIds: enabledCapabilityMcpToolRefs(settings, mcpSettings).map(
4097
- (tool) => tool.id,
4098
- ),
4489
+ defaultMcpServerIds: defaultSessionMcpServerIds(capabilitySettings.mcpServers),
4490
+ });
4491
+ const mcpAvailabilityInstructions = unavailableMcpTurnInstructions({
4492
+ droppedIds: resolvedToolPolicy.effectivePolicy.droppedIds,
4493
+ droppedCount: resolvedToolPolicy.effectivePolicy.counts.dropped,
4099
4494
  });
4100
4495
  const effectivePolicyTools = resolvedToolPolicy.toolRefs;
4101
- defaultNativeToolsAllowed = sessionToolPolicyAllowsDefaultNativeTools(
4102
- resolvedToolPolicy.effectivePolicy,
4103
- );
4104
4496
  const turnTools = withFirstPartyTools(runSettings, effectivePolicyTools);
4105
4497
  // §7.6 connection-credential provider — load (and decrypt) the variable set via the host
4106
4498
  // `sandboxSecrets` provider when bound; unset → today's local decrypt.
@@ -4843,6 +5235,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4843
5235
  {
4844
5236
  db,
4845
5237
  settings,
5238
+ cancellationSignal: sandboxResumeSignal,
4846
5239
  sandboxMetrics: runtimeMetricsHooksForObservability(observability),
4847
5240
  onSandboxLost: publishSandboxLost,
4848
5241
  },
@@ -4860,9 +5253,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4860
5253
  environment: sandboxEnvironment,
4861
5254
  // IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
4862
5255
  // this run resolves. The lease stamps it + conflicts on a live shared box
4863
- // running a DIFFERENT image (solo → recreate on the new image; N-holders →
4864
- // SandboxImageConflictError surfaced as an actionable turn error). Prefer the
4865
- // explicit Modal image ref, else the docker image. The selfhosted branch
5256
+ // running a DIFFERENT image (solo → durable rotation; N-holders →
5257
+ // SandboxImageConflictError surfaced as an actionable turn error). Prefer
5258
+ // the explicit Modal image ref, else the docker image. The selfhosted branch
4866
5259
  // (establishSelfhostedTurnSession) NEVER passes
4867
5260
  // an image — B3 lives only on this Modal else-branch.
4868
5261
  ...((runSettings.modalImageRef ?? runSettings.dockerImage)
@@ -4872,14 +5265,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4872
5265
  : {}),
4873
5266
  // RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
4874
5267
  // conflicts on a live shared box set up under a different rig (solo
4875
- // recreate / N-holders SandboxRigConflictError). Omitted for a rig-less
4876
- // turn -> never stamped or enforced (shares exactly as today).
5268
+ // durable rotation / N-holders SandboxRigConflictError). Omitted for a
5269
+ // rig-less turn -> never stamped or enforced (shares exactly as today).
4877
5270
  ...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
4878
5271
  },
4879
5272
  "turn",
4880
5273
  managedOwnership!.holderId,
4881
5274
  ),
4882
- cancellationSignal,
5275
+ sandboxResumeSignal,
4883
5276
  async (lateSandbox) => await lateSandbox.release(),
4884
5277
  );
4885
5278
  setupBoxSession = resolvedSandbox.established.session;
@@ -4997,6 +5390,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4997
5390
  ...(credentialSubjectId ? { credentialSubjectId } : {}),
4998
5391
  resolveCredential,
4999
5392
  onAuthNeeded: async (payload) => {
5393
+ if (!shouldPublishToolAuthNeededForTurn(payload, trigger, turn)) {
5394
+ return;
5395
+ }
5000
5396
  await publish!([{ type: "tool.auth_needed", payload }], true);
5001
5397
  },
5002
5398
  // Manager-style sessions carry a creation-validated permission set
@@ -5047,13 +5443,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5047
5443
  },
5048
5444
  }
5049
5445
  : {};
5050
- const hostedWebSearch = hostedWebSearchForTurn(
5051
- resolvedModel,
5052
- runSettings.webSearchEnabled,
5053
- defaultNativeToolsAllowed,
5446
+ const hostedWebSearch = hostedWebSearchForTurn(resolvedModel, runSettings.webSearchEnabled);
5447
+ const serviceTier = serviceTierForLatencyMode(
5448
+ turnExecutionPolicy.providerId,
5449
+ turnExecutionPolicy.latencyMode,
5054
5450
  );
5055
5451
  const agent = runtime.buildAgent(modelRunSettings, turnResources, {
5056
5452
  reasoningEffort: turn.reasoningEffort,
5453
+ latencyMode: turnExecutionPolicy.latencyMode,
5454
+ ...(serviceTier ? { serviceTier } : {}),
5057
5455
  ...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
5058
5456
  genesisTitleHint: isGenesisTurn,
5059
5457
  persistentSessionSettings: {
@@ -5111,9 +5509,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5111
5509
  // store/compaction follow the provider's compaction mode (registry
5112
5510
  // providers resolve to "client"); encrypted reasoning is only
5113
5511
  // round-tripped on the Responses wire API; hosted web search is attached
5114
- // only when BOTH the model opts in and this immutable turn still tracks
5115
- // workspace defaults (an explicit session/turn remains narrowed); the
5116
- // effective context window drives the compaction threshold.
5512
+ // whenever the provider declares it runnable and is independent of the
5513
+ // session's MCP allow-list; the effective context window drives the
5514
+ // compaction threshold.
5117
5515
  hostedWebSearch,
5118
5516
  ...(resolvedModel
5119
5517
  ? {
@@ -5133,7 +5531,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5133
5531
  // default.
5134
5532
  structuredToolTransport: resolvedModel.provider.kind !== "codex-subscription",
5135
5533
  // EXPLICIT computer-use tool transport, derived from the resolved provider's
5136
- // authoritative wire identity (codex → function-image, chat → function-text,
5534
+ // authoritative wire identity (codex → function-image, chat → disabled,
5137
5535
  // responses → hosted) so the runtime never string-sniffs the model instance's
5138
5536
  // constructor name. See {@link computerToolModeForTurn}.
5139
5537
  computerToolMode: computerToolModeForTurn(resolvedModel),
@@ -5176,7 +5574,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5176
5574
  ...(session.instructions ? { sessionInstructions: session.instructions } : {}),
5177
5575
  // Exact host context captured when this turn was accepted. It is
5178
5576
  // system-level and disappears with the turn rather than entering chat.
5179
- ...(turn.turnInstructions ? { turnInstructions: turn.turnInstructions } : {}),
5577
+ ...(turn.turnInstructions || mcpAvailabilityInstructions
5578
+ ? {
5579
+ turnInstructions: [turn.turnInstructions, mcpAvailabilityInstructions]
5580
+ .filter((value): value is string => Boolean(value))
5581
+ .join(" "),
5582
+ }
5583
+ : {}),
5180
5584
  ...workspaceEnvironmentOption,
5181
5585
  // RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
5182
5586
  // the frozen version carries a non-empty script), and the rig credential
@@ -5237,6 +5641,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5237
5641
  {
5238
5642
  db,
5239
5643
  settings,
5644
+ cancellationSignal: sandboxResumeSignal,
5240
5645
  sandboxMetrics: runtimeMetricsHooksForObservability(observability),
5241
5646
  onSandboxLost: publishSandboxLost,
5242
5647
  },
@@ -5284,10 +5689,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5284
5689
  },
5285
5690
  ...(lazyGitTokens ? { gitTokenSeedsOverride: lazyGitTokens } : {}),
5286
5691
  ...(lazyGitCredentials?.bindings
5287
- ? { gitCredentialBindingsOverride: lazyGitCredentials.bindings }
5692
+ ? {
5693
+ gitCredentialBindingsOverride: lazyGitCredentials.bindings,
5694
+ }
5288
5695
  : {}),
5289
5696
  ...(lazyToolspaceToken
5290
- ? { toolspaceTokenSeedOverride: lazyToolspaceToken.token }
5697
+ ? {
5698
+ toolspaceTokenSeedOverride: lazyToolspaceToken.token,
5699
+ }
5291
5700
  : {}),
5292
5701
  ...(toolCancellationFenceRef.current
5293
5702
  ? {
@@ -5320,7 +5729,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5320
5729
  return provisioned;
5321
5730
  },
5322
5731
  {
5323
- ...(activityContext ? { signal: activityContext.cancellationSignal } : {}),
5732
+ signal: sandboxResumeSignal,
5324
5733
  onStarted: async () => {
5325
5734
  await publish?.(
5326
5735
  [
@@ -5347,7 +5756,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5347
5756
  ],
5348
5757
  true,
5349
5758
  );
5350
- throwIfTurnOperationCancelled(activityContext?.cancellationSignal);
5759
+ throwIfTurnOperationCancelled(sandboxResumeSignal);
5351
5760
  startLeaseHeartbeat(provisioned, activeSandboxBackend ?? groupBoxBackend);
5352
5761
  setupBoxSession = provisioned.established.session;
5353
5762
  resolvedSandbox = provisioned;
@@ -5446,24 +5855,47 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5446
5855
  force: true,
5447
5856
  clearRequestedCompaction: true,
5448
5857
  trigger: "operator",
5858
+ codexAccount: {
5859
+ currentCodexCredentialId: effectiveCodexCredentialId,
5860
+ },
5861
+ ...compactionModeOptions,
5449
5862
  }
5450
- : { trigger: "auto" },
5863
+ : {
5864
+ trigger: "auto",
5865
+ codexAccount: {
5866
+ currentCodexCredentialId: effectiveCodexCredentialId,
5867
+ },
5868
+ ...compactionModeOptions,
5869
+ },
5451
5870
  ),
5452
5871
  cancellationSignal,
5453
5872
  undefined,
5454
5873
  );
5455
- if (outcome.compacted) {
5456
- const compactionTrigger = forced ? "operator" : undefined;
5457
- recordContextCompaction(observability, compactionTrigger ?? "auto");
5458
- await publishDurableSessionEvents(
5459
- bus,
5460
- input.workspaceId,
5461
- input.sessionId,
5462
- outcome.events,
5463
- );
5874
+ if (outcome.events.length > 0) {
5875
+ if (outcome.compacted) {
5876
+ const compactionTrigger = forced ? "operator" : undefined;
5877
+ recordContextCompaction(observability, compactionTrigger ?? "auto");
5878
+ }
5879
+ await publishCompactionOutcomeEvents(outcome.events);
5464
5880
  }
5465
5881
  } catch (compactError) {
5466
5882
  if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
5883
+ if (compactError instanceof TurnAttemptFencedError) throw compactError;
5884
+ const landmark = await settleFailedContextCompactionLandmark(
5885
+ db,
5886
+ {
5887
+ accountId: input.accountId,
5888
+ workspaceId: input.workspaceId,
5889
+ sessionId: input.sessionId,
5890
+ turnId: turnId!,
5891
+ executionGeneration,
5892
+ attemptId: input.attemptId,
5893
+ },
5894
+ {
5895
+ clearRequestedCompaction: forced,
5896
+ publishLiveEvents: publishCompactionLiveEvents,
5897
+ },
5898
+ );
5467
5899
  if (!isCompactionSummaryFailure(compactError)) throw compactError;
5468
5900
  const errorMessage = String(redact(compactionFailureReasonFromError(compactError)));
5469
5901
  observability.error("context compaction failed", {
@@ -5489,7 +5921,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5489
5921
  turnStatus: "failed",
5490
5922
  sessionStatus: "idle",
5491
5923
  activeTurnId: null,
5492
- ...(forced ? { consumeRequestedCompactionFailure: true } : {}),
5924
+ ...(forced && !landmark.requestConsumed
5925
+ ? { consumeRequestedCompactionFailure: true }
5926
+ : {}),
5493
5927
  }))
5494
5928
  ) {
5495
5929
  return claimedResult({ status: "cancelled" });
@@ -5665,6 +6099,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5665
6099
  force: true,
5666
6100
  ...(triggerLabel === "operator" ? { clearRequestedCompaction: true } : {}),
5667
6101
  trigger: triggerLabel,
6102
+ codexAccount: {
6103
+ currentCodexCredentialId: effectiveCodexCredentialId,
6104
+ },
6105
+ ...compactionModeOptions,
5668
6106
  },
5669
6107
  ),
5670
6108
  cancellationSignal,
@@ -5674,12 +6112,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5674
6112
  if (outcome.compacted) {
5675
6113
  recordContextCompaction(observability, triggerLabel);
5676
6114
  }
5677
- await publishDurableSessionEvents(
5678
- bus,
5679
- input.workspaceId,
5680
- input.sessionId,
5681
- outcome.events,
5682
- );
6115
+ await publishCompactionOutcomeEvents(outcome.events);
5683
6116
  }
5684
6117
  return outcome;
5685
6118
  };
@@ -5752,8 +6185,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5752
6185
  initialGitCredentials,
5753
6186
  );
5754
6187
  }
6188
+ // Conservative request boundary: once control enters runStream, the
6189
+ // provider may have accepted work even if no response/event follows.
6190
+ // Escaped MCP setup timeouts are automatically recoverable only
6191
+ // before this line.
6192
+ modelRequestStarted = true;
5755
6193
  return await runtime.runStream(agent, runInput!, modelRunSettings, {
5756
- ...(activityContext ? { signal: activityContext.cancellationSignal } : {}),
6194
+ signal: runtimeCancellationSignal,
5757
6195
  sandboxEnvironment,
5758
6196
  onRuntimeEvent: async (event) => {
5759
6197
  await renewCodexLease("runtime_event");
@@ -5872,6 +6310,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5872
6310
  provider: resolvedModel?.provider.id ?? settings.openaiProvider,
5873
6311
  providerApi: resolvedModel?.provider.api ?? "responses",
5874
6312
  model: turn.model,
6313
+ latencyMode: turnExecutionPolicy.latencyMode,
5875
6314
  metricProvider: streamProvider,
5876
6315
  externallyBilled: isExternallyBilledTurn,
5877
6316
  servingCredentialId: effectiveCodexCredentialId,
@@ -5882,6 +6321,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5882
6321
  leaseLostMessage: "Codex credential lease expired during the active turn",
5883
6322
  setLastInputTokens: setLastInputTokensFenced,
5884
6323
  });
6324
+ assertModelResponseLatencyMode({
6325
+ event: next.value,
6326
+ requested: turnExecutionPolicy.latencyMode,
6327
+ model: turn.model,
6328
+ ...(resolvedModel?.provider.id ? { providerId: resolvedModel.provider.id } : {}),
6329
+ });
5885
6330
  if (responseUsageResult.status === "processed") {
5886
6331
  currentToolBatchCallIds = new Set<string>();
5887
6332
  currentToolBatchCompletedCallIds = new Set<string>();
@@ -6045,7 +6490,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6045
6490
  leaseLost: () => codexLeaseLost,
6046
6491
  leaseLostMessage: "Codex credential lease expired during the active turn",
6047
6492
  recordUsage: async () => {
6048
- await recordModelUsageAndDebitCredits(settings, db, {
6493
+ const billing = await recordModelUsageAndDebitCredits(settings, db, {
6049
6494
  accountId: input.accountId,
6050
6495
  workspaceId: input.workspaceId,
6051
6496
  sessionId: input.sessionId,
@@ -6056,8 +6501,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6056
6501
  usage: aggregateUsage,
6057
6502
  normalizedUsage: normalizedAggregateUsage,
6058
6503
  sourceKey: aggregateSourceKey,
6504
+ latencyMode: turnExecutionPolicy.latencyMode,
6059
6505
  observability,
6060
6506
  });
6507
+ const aggregateProvider = resolvedModel?.provider.id ?? settings.openaiProvider;
6508
+ const aggregateProviderApi = resolvedModel?.provider.api ?? "responses";
6061
6509
  aggregateAuthoritative = await emitModelCallUsage({
6062
6510
  observability,
6063
6511
  publish,
@@ -6065,8 +6513,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6065
6513
  workspaceId: input.workspaceId,
6066
6514
  sessionId: input.sessionId,
6067
6515
  turnId: activeTurnId,
6068
- provider: resolvedModel?.provider.id ?? settings.openaiProvider,
6069
- providerApi: resolvedModel?.provider.api ?? "responses",
6516
+ provider: aggregateProvider,
6517
+ providerApi: aggregateProviderApi,
6070
6518
  model: turn.model,
6071
6519
  sourceKey: aggregateSourceKey,
6072
6520
  usage: { usage: aggregateUsage },
@@ -6075,6 +6523,22 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6075
6523
  accountChangedFromPrevCall: aggregateAccountCtx.accountChangedFromPrevCall,
6076
6524
  emittedSourceKeys: emittedModelUsageSourceKeys,
6077
6525
  });
6526
+ if (aggregateAuthoritative && billing) {
6527
+ await recordAuthoritativeModelCallFact({
6528
+ db,
6529
+ observability,
6530
+ accountId: input.accountId,
6531
+ workspaceId: input.workspaceId,
6532
+ sessionId: input.sessionId,
6533
+ turnId: activeTurnId,
6534
+ turnAttemptId: input.attemptId,
6535
+ sourceKey: aggregateSourceKey,
6536
+ provider: aggregateProvider,
6537
+ providerApi: aggregateProviderApi,
6538
+ model: turn.model,
6539
+ billing,
6540
+ });
6541
+ }
6078
6542
  if (aggregateAuthoritative && aggregateInput !== null && aggregateInput > 0) {
6079
6543
  recordModelInputTokens(observability, streamProvider, aggregateInput);
6080
6544
  }
@@ -6147,7 +6611,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6147
6611
  events: [
6148
6612
  ...requestEvents,
6149
6613
  ...(approvals.length > 0
6150
- ? [{ type: "session.requiresAction" as const, payload: { approvals } }]
6614
+ ? [
6615
+ {
6616
+ type: "session.requiresAction" as const,
6617
+ payload: { approvals },
6618
+ },
6619
+ ]
6151
6620
  : []),
6152
6621
  {
6153
6622
  type: "session.status.changed",
@@ -6262,6 +6731,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6262
6731
  let compacted = false;
6263
6732
  let compactionHandled = false;
6264
6733
  let compactionFailureMessage: string | null = null;
6734
+ let compactionRequestCleared = false;
6265
6735
  try {
6266
6736
  const outcome = await forceContextCompaction(
6267
6737
  recoveryKind,
@@ -6281,6 +6751,23 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6281
6751
  // turn through the normal provider/capacity path. They are not an
6282
6752
  // empty summary and must not create a new goal continuation.
6283
6753
  if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
6754
+ if (compactError instanceof TurnAttemptFencedError) throw compactError;
6755
+ const landmark = await settleFailedContextCompactionLandmark(
6756
+ db,
6757
+ {
6758
+ accountId: input.accountId,
6759
+ workspaceId: input.workspaceId,
6760
+ sessionId: input.sessionId,
6761
+ turnId: activeTurnId!,
6762
+ executionGeneration,
6763
+ attemptId: input.attemptId,
6764
+ },
6765
+ {
6766
+ clearRequestedCompaction: recoveryKind === "operator",
6767
+ publishLiveEvents: publishCompactionLiveEvents,
6768
+ },
6769
+ );
6770
+ compactionRequestCleared = landmark.requestConsumed;
6284
6771
  if (!isCompactionSummaryFailure(compactError)) throw compactError;
6285
6772
  compactionFailureMessage = String(
6286
6773
  redact(compactionFailureReasonFromError(compactError)),
@@ -6316,7 +6803,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6316
6803
  turnStatus: "failed",
6317
6804
  sessionStatus: "idle",
6318
6805
  activeTurnId: null,
6319
- ...(recoveryKind === "operator" ? { consumeRequestedCompactionFailure: true } : {}),
6806
+ ...(recoveryKind === "operator" && !compactionRequestCleared
6807
+ ? { consumeRequestedCompactionFailure: true }
6808
+ : {}),
6320
6809
  }))
6321
6810
  ) {
6322
6811
  return claimedResult({ status: "cancelled" });
@@ -6367,12 +6856,29 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6367
6856
  // so its next attempt reattaches under the current epoch.
6368
6857
  if (error instanceof SandboxLeaseSupersededError && recoveryTurnId) {
6369
6858
  try {
6859
+ const fencedLease = await readLease(db, input.workspaceId, error.sandboxGroupId).catch(
6860
+ () => null,
6861
+ );
6862
+ const rotationPending = fencedLease?.rotationRequestedAt != null;
6863
+ const deadlineRotationPending =
6864
+ rotationPending && fencedLease?.rotationReason === "provider_deadline";
6370
6865
  const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
6371
6866
  sessionId: input.sessionId,
6372
6867
  turnId: recoveryTurnId,
6373
6868
  triggerEventId: triggerEventId!,
6374
6869
  attemptId: input.attemptId,
6375
- reason: "sandbox_lease_superseded",
6870
+ reason: deadlineRotationPending
6871
+ ? "sandbox_deadline_rotation"
6872
+ : "sandbox_lease_superseded",
6873
+ ...(rotationPending
6874
+ ? {
6875
+ detail: {
6876
+ sandboxGroupId: error.sandboxGroupId,
6877
+ leaseEpoch: error.leaseEpoch,
6878
+ rotationReason: fencedLease?.rotationReason ?? "operator",
6879
+ },
6880
+ }
6881
+ : {}),
6376
6882
  });
6377
6883
  if (recovery.action === "stale") {
6378
6884
  acknowledgeLostAttemptOwnership();
@@ -6388,7 +6894,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6388
6894
  );
6389
6895
  activityStatus = "recovering";
6390
6896
  turnMetricOutcome = "recovering";
6391
- return claimedResult({ status: "recovering" });
6897
+ return claimedResult({
6898
+ status: "recovering",
6899
+ ...(rotationPending
6900
+ ? {
6901
+ continueDelayMs: sandboxDeadlineRotationRecoveryDelayMs(settings),
6902
+ }
6903
+ : {}),
6904
+ });
6392
6905
  } catch (recoveryError) {
6393
6906
  console.error(
6394
6907
  "sandbox lease supersession recovery failed",
@@ -6397,6 +6910,53 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6397
6910
  throw recoveryError;
6398
6911
  }
6399
6912
  }
6913
+ if (
6914
+ sandboxRotationController.signal.aborted &&
6915
+ sandboxRotationController.signal.reason instanceof SandboxDeadlineRotationError &&
6916
+ !cancellationSignal?.aborted &&
6917
+ recoveryTurnId
6918
+ ) {
6919
+ try {
6920
+ await flushRuntimeBatcher();
6921
+ await reconcileConversationTruth({ requireDurable: true });
6922
+ const rotation = sandboxRotationController.signal.reason;
6923
+ const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
6924
+ sessionId: input.sessionId,
6925
+ turnId: recoveryTurnId,
6926
+ triggerEventId: triggerEventId!,
6927
+ attemptId: input.attemptId,
6928
+ reason: "sandbox_deadline_rotation",
6929
+ detail: {
6930
+ sandboxGroupId: rotation.sandboxGroupId,
6931
+ leaseEpoch: rotation.leaseEpoch,
6932
+ },
6933
+ });
6934
+ if (recovery.action === "stale") {
6935
+ acknowledgeLostAttemptOwnership();
6936
+ activityStatus = "cancelled";
6937
+ turnMetricOutcome = "cancelled";
6938
+ return claimedResult({ status: "cancelled" });
6939
+ }
6940
+ await publishDurableSessionEvents(
6941
+ bus,
6942
+ input.workspaceId,
6943
+ input.sessionId,
6944
+ recovery.events,
6945
+ );
6946
+ activityStatus = "recovering";
6947
+ turnMetricOutcome = "recovering";
6948
+ return claimedResult({
6949
+ status: "recovering",
6950
+ continueDelayMs: sandboxDeadlineRotationRecoveryDelayMs(settings),
6951
+ });
6952
+ } catch (recoveryError) {
6953
+ console.error(
6954
+ "sandbox deadline rotation recovery failed",
6955
+ safeErrorDiagnostic(recoveryError, (value) => String(redact(value))),
6956
+ );
6957
+ throw recoveryError;
6958
+ }
6959
+ }
6400
6960
  const cancellationFailure = turnOperationCancellationFailure(error);
6401
6961
  if (
6402
6962
  cancellationFailure &&
@@ -7157,6 +7717,62 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7157
7717
  activityStatus = "idle";
7158
7718
  return claimedResult({ status: "idle" });
7159
7719
  }
7720
+ // The Codex backend can reject an opaque reasoning artifact that it
7721
+ // minted on the immediately preceding successful request, even when the
7722
+ // credential-row UUID is unchanged. HTTP 400 + the exact provider
7723
+ // semantic proves this request never entered inference. Atomically mark
7724
+ // the active opaque artifacts rejected and recover the SAME logical turn
7725
+ // from durable history; messages, tool calls/results, summaries, and the
7726
+ // original audit rows remain intact. If no artifact can be invalidated,
7727
+ // fall through to the terminal path rather than resend an equivalent
7728
+ // request forever.
7729
+ const encryptedArtifactRejection =
7730
+ isCodexTurn && effectiveCodexCredentialId
7731
+ ? classifyCodexEncryptedArtifactRejection(error)
7732
+ : null;
7733
+ if (
7734
+ encryptedArtifactRejection &&
7735
+ effectiveCodexCredentialId &&
7736
+ publish &&
7737
+ turnId &&
7738
+ turnStartedPublished
7739
+ ) {
7740
+ await flushRuntimeBatcher();
7741
+ await reconcileConversationTruth({ requireDurable: true });
7742
+ const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
7743
+ sessionId: input.sessionId,
7744
+ turnId,
7745
+ triggerEventId: triggerEventId!,
7746
+ attemptId: input.attemptId,
7747
+ reason: encryptedArtifactRejection.kind,
7748
+ detail: {
7749
+ code: encryptedArtifactRejection.kind,
7750
+ retryable: true,
7751
+ },
7752
+ providerArtifactInvalidation: {
7753
+ codexCredentialId: effectiveCodexCredentialId,
7754
+ reason: encryptedArtifactRejection.kind,
7755
+ },
7756
+ });
7757
+ if (recovery.action === "stale") {
7758
+ acknowledgeLostAttemptOwnership();
7759
+ activityStatus = "cancelled";
7760
+ turnMetricOutcome = "cancelled";
7761
+ return claimedResult({ status: "cancelled" });
7762
+ }
7763
+ if (recovery.action === "recovering") {
7764
+ await publishDurableSessionEvents(
7765
+ bus,
7766
+ input.workspaceId,
7767
+ input.sessionId,
7768
+ recovery.events,
7769
+ );
7770
+ turnMetricOutcome = "recovering";
7771
+ activityStatus = "recovering";
7772
+ activityError = error;
7773
+ return claimedResult({ status: "recovering" });
7774
+ }
7775
+ }
7160
7776
  // A retryable provider/MCP failure is transient external backpressure,
7161
7777
  // not a session or goal failure. The in-client retry budget is already
7162
7778
  // exhausted by the time the error reaches here. Checkpoint conversation
@@ -7195,31 +7811,55 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7195
7811
  });
7196
7812
  }
7197
7813
  if (failure.retryable && publish && turnId && turnStartedPublished) {
7198
- const recoveryResult = providerRecoveryResult();
7199
- await flushRuntimeBatcher();
7200
- await reconcileConversationTruth({ requireDurable: true });
7201
- const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
7202
- sessionId: input.sessionId,
7203
- turnId,
7204
- triggerEventId: triggerEventId!,
7205
- attemptId: input.attemptId,
7206
- reason: failure.code ?? "provider_unavailable",
7207
- detail: {
7208
- ...failure,
7209
- continueDelayMs: recoveryResult.continueDelayMs,
7210
- },
7211
- });
7212
- if (recovery.action === "stale") {
7213
- acknowledgeLostAttemptOwnership();
7214
- activityStatus = "cancelled";
7215
- turnMetricOutcome = "cancelled";
7216
- return claimedResult({ status: "cancelled" });
7814
+ try {
7815
+ const recoveryResult = providerRecoveryResult();
7816
+ await flushRuntimeBatcher();
7817
+ await reconcileConversationTruth({ requireDurable: true });
7818
+ const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
7819
+ sessionId: input.sessionId,
7820
+ turnId,
7821
+ triggerEventId: triggerEventId!,
7822
+ attemptId: input.attemptId,
7823
+ reason: failure.code ?? "provider_unavailable",
7824
+ detail: {
7825
+ ...failure,
7826
+ continueDelayMs: recoveryResult.continueDelayMs,
7827
+ },
7828
+ });
7829
+ if (recovery.action === "stale") {
7830
+ acknowledgeLostAttemptOwnership();
7831
+ activityStatus = "cancelled";
7832
+ turnMetricOutcome = "cancelled";
7833
+ return claimedResult({ status: "cancelled" });
7834
+ }
7835
+ await publishDurableSessionEvents(
7836
+ bus,
7837
+ input.workspaceId,
7838
+ input.sessionId,
7839
+ recovery.events,
7840
+ );
7841
+ turnMetricOutcome = "recovering";
7842
+ activityStatus = "recovering";
7843
+ activityError = error;
7844
+ return claimedResult(recoveryResult);
7845
+ } catch (recoveryError) {
7846
+ const escaped = escapedMcpTimeoutRecoveryFailure({
7847
+ failureCode: failure.code,
7848
+ modelRequestStarted,
7849
+ detail: {
7850
+ turnId,
7851
+ triggerEventId: triggerEventId!,
7852
+ executionGeneration,
7853
+ },
7854
+ });
7855
+ if (escaped) {
7856
+ activityStatus = "recovering";
7857
+ turnMetricOutcome = "recovering";
7858
+ activityError = error;
7859
+ throw escaped;
7860
+ }
7861
+ throw recoveryError;
7217
7862
  }
7218
- await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
7219
- turnMetricOutcome = "recovering";
7220
- activityStatus = "recovering";
7221
- activityError = error;
7222
- return claimedResult(recoveryResult);
7223
7863
  }
7224
7864
  activityStatus = "failed";
7225
7865
  activityError = error;
@@ -7262,6 +7902,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7262
7902
  );
7263
7903
  return claimedResult({ status: "failed" });
7264
7904
  } finally {
7905
+ // This is the logical ownership boundary. Abort before any fallible
7906
+ // housekeeping so a still-pending provider establish releases its private
7907
+ // holder/timer even when Temporal itself never delivered cancellation.
7908
+ if (resolvedSandbox === null && !sandboxResumeController.signal.aborted) {
7909
+ sandboxResumeController.abort(
7910
+ cancellationSignal?.reason ?? new Error("TURN_ATTEMPT_FINALIZED"),
7911
+ );
7912
+ }
7265
7913
  const finalizationStarted = performance.now();
7266
7914
  let finalizationError: unknown;
7267
7915
  let physicalToolQuiescenceConfirmed = !acknowledgeQuiescence;
@@ -7488,7 +8136,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7488
8136
  // gives capture the full live-box margin instead of racing the teardown
7489
8137
  // tail, which was dropping 100% of captures on real Modal desktop boxes
7490
8138
  // ("request cancelled due to container exiting", 0 rows). External module:
7491
- // self-capped at 60s, best-effort (never throws past its boundary),
8139
+ // self-capped at 120s, best-effort (never throws past its boundary),
7492
8140
  // epoch-fenced, and it NEVER closes the box. The emitted
7493
8141
  // workspace.revision.captured event is ANNOUNCE-ONLY (metadata, never
7494
8142
  // content).
@@ -7508,14 +8156,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7508
8156
  setupBoxSession &&
7509
8157
  sandboxGroupId
7510
8158
  ) {
7511
- // Stop new heartbeat snapshot/meter ticks so a mid-turn snapshot cannot
7512
- // start concurrently with capture, then drain any in-flight snapshot
7513
- // (bounded) — capture and the warm snapshot both exec on the box, so
7514
- // sequence them, exactly as the turn-end snapshot placement did.
7515
- if (leaseHeartbeatTimer) {
7516
- clearInterval(leaseHeartbeatTimer);
7517
- leaseHeartbeatTimer = undefined;
7518
- }
8159
+ // Block new periodic snapshots, then drain any one already in flight.
8160
+ // Keep the lease heartbeat itself running: capture may legitimately
8161
+ // exceed the 90s holder TTL, and the reaper must remain unable to drain
8162
+ // the exact box while this holder still reads it.
8163
+ turnEndCaptureInProgress = true;
7519
8164
  if (snapshotInFlight) {
7520
8165
  await waitForWarmSnapshot(
7521
8166
  snapshotInFlight,
@@ -7523,6 +8168,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7523
8168
  finalizerSignal,
7524
8169
  );
7525
8170
  }
8171
+ const captureEstablished = resolvedSandbox.established;
7526
8172
  await captureWorkspaceRevision({
7527
8173
  db,
7528
8174
  objectStorage,
@@ -7531,6 +8177,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7531
8177
  await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events);
7532
8178
  },
7533
8179
  session: setupBoxSession as ChannelASession,
8180
+ openReadSession: async () =>
8181
+ await openFreshWorkspaceCaptureSession({
8182
+ backendId: captureEstablished.backendId,
8183
+ client: captureEstablished.client,
8184
+ session: setupBoxSession as ChannelASession,
8185
+ expectedInstanceId: captureEstablished.instanceId,
8186
+ }),
7534
8187
  leaseEpoch: resolvedSandbox.leaseEpoch,
7535
8188
  sandboxGroupId,
7536
8189
  accountId: input.accountId,
@@ -7564,7 +8217,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7564
8217
  // box rides the provider idle-timeout in the meantime. Best-effort: a
7565
8218
  // release failure must never mask the turn's real outcome.
7566
8219
  if (leaseHeartbeatTimer) {
7567
- clearInterval(leaseHeartbeatTimer);
8220
+ stopLeaseHeartbeat();
8221
+ }
8222
+ if (rotationInFlight) {
8223
+ await waitForTurnFinalizerStep(rotationInFlight, finalizerSignal).catch(() => undefined);
7568
8224
  }
7569
8225
  // A recording normally closes inside the attempt-fenced turn settlement.
7570
8226
  // Reaching finally with one still active means settlement threw, never ran,
@@ -7645,6 +8301,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7645
8301
  safeErrorDiagnostic(error, (value) => String(redact(value))),
7646
8302
  );
7647
8303
  } finally {
8304
+ // If fallible finalization exited before the ordinary exact-holder
8305
+ // release, the resume signal is the unconditional last line of defense.
8306
+ // Normal finalization already removed its listener in release().
8307
+ if (!sandboxResumeController.signal.aborted) {
8308
+ sandboxResumeController.abort(
8309
+ cancellationSignal?.reason ?? new Error("TURN_ATTEMPT_FINALIZED"),
8310
+ );
8311
+ }
7648
8312
  cancellationSignal?.removeEventListener("abort", noteCancellationRequested);
7649
8313
  const completedAt = performance.now();
7650
8314
  const durationSeconds = (completedAt - activityStarted) / 1000;
@@ -8268,6 +8932,13 @@ export async function ensureRunAllowed(
8268
8932
  }
8269
8933
  }
8270
8934
 
8935
+ export type ModelUsageBillingRecord = {
8936
+ billingPath: "opengeni_credits" | "external";
8937
+ /** Same quantity written to usage_events.model.cost when present; else 0. */
8938
+ pricedCostMicros: number;
8939
+ normalizedUsage: ModelCallUsageNormalization;
8940
+ };
8941
+
8271
8942
  // Exported for unit testing the external-billing bypass; not part of the activity surface.
8272
8943
  export async function recordModelUsageAndDebitCredits(
8273
8944
  settings: Settings,
@@ -8283,11 +8954,12 @@ export async function recordModelUsageAndDebitCredits(
8283
8954
  usage?: ModelUsageInput | ModelCallUsageInput | null;
8284
8955
  normalizedUsage?: ModelCallUsageNormalization;
8285
8956
  sourceKey: string;
8957
+ latencyMode?: LatencyMode;
8286
8958
  observability?: ActivityServices["observability"];
8287
8959
  },
8288
- ): Promise<void> {
8960
+ ): Promise<ModelUsageBillingRecord | null> {
8289
8961
  if (!input.usage) {
8290
- return;
8962
+ return null;
8291
8963
  }
8292
8964
  const normalizedUsage = input.normalizedUsage ?? normalizeModelCallUsage(input.usage);
8293
8965
  const sanitizedUsage = sanitizedModelUsageInput(normalizedUsage);
@@ -8317,7 +8989,11 @@ export async function recordModelUsageAndDebitCredits(
8317
8989
  turnAttemptId: input.turnAttemptId,
8318
8990
  idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`,
8319
8991
  });
8320
- return;
8992
+ return {
8993
+ billingPath: "external",
8994
+ pricedCostMicros: 0,
8995
+ normalizedUsage,
8996
+ };
8321
8997
  }
8322
8998
  if (totalTokens > 0) {
8323
8999
  await recordUsageEvent(db, {
@@ -8336,12 +9012,18 @@ export async function recordModelUsageAndDebitCredits(
8336
9012
  }
8337
9013
  const shouldDebit = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
8338
9014
  if (!shouldDebit || totalTokens === 0) {
8339
- return;
9015
+ return {
9016
+ billingPath: "opengeni_credits",
9017
+ pricedCostMicros: 0,
9018
+ normalizedUsage,
9019
+ };
8340
9020
  }
8341
9021
  if (!configuredModelPricing(settings)[input.model]) {
8342
9022
  throw new Error(`Missing model pricing for ${input.model}`);
8343
9023
  }
8344
- const costMicros = calculateModelUsageCostMicros(settings, input.model, sanitizedUsage);
9024
+ const costMicros = calculateModelUsageCostMicros(settings, input.model, sanitizedUsage, {
9025
+ latencyMode: input.latencyMode ?? "standard",
9026
+ });
8345
9027
  await recordUsageEvent(db, {
8346
9028
  accountId: input.accountId,
8347
9029
  workspaceId: input.workspaceId,
@@ -8369,6 +9051,7 @@ export async function recordModelUsageAndDebitCredits(
8369
9051
  sessionId: input.sessionId,
8370
9052
  turnId: input.turnId,
8371
9053
  sourceKey: input.sourceKey,
9054
+ latencyMode: input.latencyMode ?? "standard",
8372
9055
  inputTokens,
8373
9056
  outputTokens,
8374
9057
  totalTokens,
@@ -8380,6 +9063,59 @@ export async function recordModelUsageAndDebitCredits(
8380
9063
  });
8381
9064
  recordCreditMicros(input.observability, "usage", result.debitedMicros);
8382
9065
  }
9066
+ return {
9067
+ billingPath: "opengeni_credits",
9068
+ pricedCostMicros: costMicros,
9069
+ normalizedUsage,
9070
+ };
9071
+ }
9072
+
9073
+ /** Soft-fail Insights fact write — never throws into the billing/emit path. */
9074
+ export async function recordAuthoritativeModelCallFact(input: {
9075
+ db: ActivityServices["db"];
9076
+ observability: ActivityServices["observability"];
9077
+ accountId: string;
9078
+ workspaceId: string;
9079
+ sessionId: string;
9080
+ turnId: string;
9081
+ turnAttemptId: string;
9082
+ sourceKey: string;
9083
+ provider: string;
9084
+ providerApi: ModelProviderApi;
9085
+ model: string;
9086
+ billing: ModelUsageBillingRecord;
9087
+ }): Promise<void> {
9088
+ try {
9089
+ const telemetry = input.billing.normalizedUsage.telemetry;
9090
+ await recordModelCallFact(input.db, {
9091
+ accountId: input.accountId,
9092
+ workspaceId: input.workspaceId,
9093
+ sessionId: input.sessionId,
9094
+ turnId: input.turnId,
9095
+ turnAttemptId: input.turnAttemptId,
9096
+ sourceKey: input.sourceKey,
9097
+ provider: input.provider,
9098
+ providerApi: input.providerApi,
9099
+ model: input.model,
9100
+ billingPath: input.billing.billingPath,
9101
+ pricedCostMicros: input.billing.pricedCostMicros,
9102
+ inputTokens: telemetry.inputTokens,
9103
+ outputTokens: telemetry.outputTokens,
9104
+ cachedTokens: telemetry.cachedTokens,
9105
+ cacheWriteTokens: telemetry.cacheWriteTokens,
9106
+ reasoningTokens: telemetry.reasoningTokens,
9107
+ totalTokens: input.billing.normalizedUsage.totalTokens,
9108
+ });
9109
+ } catch (error) {
9110
+ input.observability.warn("model call fact persist failed", {
9111
+ accountId: input.accountId,
9112
+ workspaceId: input.workspaceId,
9113
+ sessionId: input.sessionId,
9114
+ turnId: input.turnId,
9115
+ sourceKey: input.sourceKey,
9116
+ error: error instanceof Error ? error.message : String(error),
9117
+ });
9118
+ }
8383
9119
  }
8384
9120
 
8385
9121
  function sanitizedModelUsageInput(normalized: ModelCallUsageNormalization): ModelUsageInput {
@@ -8392,7 +9128,11 @@ function sanitizedModelUsageInput(normalized: ModelCallUsageNormalization): Mode
8392
9128
  : {}),
8393
9129
  ...(normalized.totalTokens !== null ? { totalTokens: normalized.totalTokens } : {}),
8394
9130
  ...(normalized.telemetry.cachedTokens !== null
8395
- ? { inputTokensDetails: { cached_tokens: normalized.telemetry.cachedTokens } }
9131
+ ? {
9132
+ inputTokensDetails: {
9133
+ cached_tokens: normalized.telemetry.cachedTokens,
9134
+ },
9135
+ }
8396
9136
  : {}),
8397
9137
  };
8398
9138
  }