@opengeni/worker-bundle 0.16.29 → 0.17.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/dist/activities/agent-turn.d.ts +85 -19
  2. package/dist/activities/api-integrations.d.ts +21 -0
  3. package/dist/activities/codemode-dispatcher.d.ts +35 -0
  4. package/dist/activities/codemode-token-renewal.d.ts +23 -0
  5. package/dist/activities/codex-capacity.d.ts +5 -5
  6. package/dist/activities/codex-image-generation.d.ts +21 -0
  7. package/dist/activities/context-compaction.d.ts +1 -0
  8. package/dist/activities/documents.d.ts +3 -2
  9. package/dist/activities/environment.d.ts +11 -8
  10. package/dist/activities/expiring-material-renewal.d.ts +1 -1
  11. package/dist/activities/file-upload-reaper.d.ts +7 -2
  12. package/dist/activities/gateway-image-generation.d.ts +39 -0
  13. package/dist/activities/gateway-video-generation.d.ts +53 -0
  14. package/dist/activities/generated-images.d.ts +112 -0
  15. package/dist/activities/git-credential-renewal.d.ts +1 -1
  16. package/dist/activities/goals.d.ts +2 -2
  17. package/dist/activities/google-drive-changes.d.ts +99 -0
  18. package/dist/activities/image-generation-operation.d.ts +74 -0
  19. package/dist/activities/image-generation-references.d.ts +25 -0
  20. package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
  21. package/dist/activities/knowledge-source-sync.d.ts +75 -0
  22. package/dist/activities/packs.d.ts +13 -26
  23. package/dist/activities/parent-wake.d.ts +1 -1
  24. package/dist/activities/retained-screenshot-reaper.d.ts +2 -2
  25. package/dist/activities/retained-screenshots.d.ts +36 -0
  26. package/dist/activities/rig-verification.d.ts +82 -5
  27. package/dist/activities/run-credential-renewal.d.ts +1 -1
  28. package/dist/activities/run-input.d.ts +3 -1
  29. package/dist/activities/sandbox-images.d.ts +14 -0
  30. package/dist/activities/sandbox-lease.d.ts +47 -9
  31. package/dist/activities/scheduled-tasks.d.ts +2 -2
  32. package/dist/activities/session-state.d.ts +4 -4
  33. package/dist/activities/streaming.d.ts +1 -1
  34. package/dist/activities/types.d.ts +71 -5
  35. package/dist/activities/video-generation-admission.d.ts +38 -0
  36. package/dist/activities/video-generation-credential.d.ts +7 -0
  37. package/dist/activities/video-generation-reconciliation.d.ts +13 -0
  38. package/dist/activities/video-media-validation.d.ts +34 -0
  39. package/dist/activities/video-output-retention.d.ts +36 -0
  40. package/dist/activities/video-reference-staging.d.ts +49 -0
  41. package/dist/activities/workflow-wake.d.ts +2 -2
  42. package/dist/activities/workspace-capture.d.ts +10 -1
  43. package/dist/activities-control-44KBWLSM.js +6885 -0
  44. package/dist/activities-control-44KBWLSM.js.map +1 -0
  45. package/dist/activities-control.d.ts +289 -0
  46. package/dist/activities-turn-K4CEDEGJ.js +14675 -0
  47. package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
  48. package/dist/activities-turn.d.ts +17 -0
  49. package/dist/activities.d.ts +130 -150
  50. package/dist/activity-services.d.ts +10 -0
  51. package/dist/artifact-materializer-entry.d.ts +6 -0
  52. package/dist/artifact-materializer-entry.js +2662 -0
  53. package/dist/artifact-materializer-entry.js.map +1 -0
  54. package/dist/artifact-outbox-entry.d.ts +2 -0
  55. package/dist/artifact-outbox-entry.js +681 -0
  56. package/dist/artifact-outbox-entry.js.map +1 -0
  57. package/dist/chunk-E3RI3I33.js +828 -0
  58. package/dist/chunk-E3RI3I33.js.map +1 -0
  59. package/dist/chunk-GRFQTTSQ.js +448 -0
  60. package/dist/chunk-GRFQTTSQ.js.map +1 -0
  61. package/dist/chunk-KKCRGAGZ.js +3091 -0
  62. package/dist/chunk-KKCRGAGZ.js.map +1 -0
  63. package/dist/concurrency.d.ts +108 -5
  64. package/dist/editable-artifact-hint-broker.d.ts +49 -0
  65. package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
  66. package/dist/editable-artifact-materializer-service.d.ts +108 -0
  67. package/dist/editable-artifact-materializer-storage.d.ts +12 -0
  68. package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
  69. package/dist/editable-artifact-materializer.d.ts +262 -0
  70. package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
  71. package/dist/editable-artifact-outbox-service.d.ts +70 -0
  72. package/dist/host-export.d.ts +1 -1
  73. package/dist/index.d.ts +176 -10
  74. package/dist/index.js +662 -17374
  75. package/dist/index.js.map +1 -1
  76. package/dist/memory-pressure-guard.d.ts +57 -0
  77. package/dist/model-checkpoint-memory-collector.d.ts +22 -0
  78. package/dist/observability-metrics.d.ts +33 -0
  79. package/dist/sandbox-reaper-contract.d.ts +40 -0
  80. package/dist/sandbox-reaper-timeout.d.ts +14 -0
  81. package/dist/sandbox-resume.d.ts +34 -7
  82. package/dist/sandbox-routing.d.ts +2 -0
  83. package/dist/worker-service-lifecycle.d.ts +6 -2
  84. package/dist/workflow-bundle.js +1043 -363
  85. package/dist/workflows/activities.d.ts +7 -1
  86. package/dist/workflows/document-index.d.ts +1 -1
  87. package/dist/workflows/knowledge-source-sync.d.ts +13 -0
  88. package/dist/workflows/rig-verification.d.ts +1 -1
  89. package/dist/workflows/sandbox-reaper.d.ts +8 -0
  90. package/dist/workflows/scheduled-tasks.d.ts +1 -1
  91. package/dist/workflows/session.d.ts +2 -2
  92. package/dist/workflows/video-generation.d.ts +10 -0
  93. package/dist/workflows/workflow-wake.d.ts +1 -1
  94. package/dist/workflows.d.ts +9 -7
  95. package/package.json +34 -15
  96. package/src/activities/agent-turn.ts +1470 -246
  97. package/src/activities/api-integrations.ts +138 -0
  98. package/src/activities/codemode-dispatcher.ts +335 -0
  99. package/src/activities/codemode-token-renewal.ts +64 -0
  100. package/src/activities/codex-capacity.ts +6 -6
  101. package/src/activities/codex-image-generation.ts +56 -0
  102. package/src/activities/context-compaction.ts +38 -3
  103. package/src/activities/documents.ts +14 -4
  104. package/src/activities/environment.ts +45 -42
  105. package/src/activities/expiring-material-renewal.ts +1 -1
  106. package/src/activities/file-upload-reaper.ts +34 -3
  107. package/src/activities/gateway-image-generation.ts +146 -0
  108. package/src/activities/gateway-video-generation.ts +298 -0
  109. package/src/activities/generated-images.ts +860 -0
  110. package/src/activities/goals.ts +6 -2
  111. package/src/activities/google-drive-changes.ts +454 -0
  112. package/src/activities/image-generation-operation.ts +333 -0
  113. package/src/activities/image-generation-references.ts +118 -0
  114. package/src/activities/knowledge-source-sync-driver.ts +27 -0
  115. package/src/activities/knowledge-source-sync.ts +2840 -0
  116. package/src/activities/mcp-credentials.ts +1 -0
  117. package/src/activities/packs.ts +55 -55
  118. package/src/activities/retained-screenshot-reaper.ts +2 -2
  119. package/src/activities/retained-screenshots.ts +453 -28
  120. package/src/activities/rig-verification.ts +554 -43
  121. package/src/activities/run-input.ts +22 -6
  122. package/src/activities/sandbox-images.ts +167 -0
  123. package/src/activities/sandbox-lease.ts +1065 -483
  124. package/src/activities/scheduled-tasks.ts +77 -4
  125. package/src/activities/session-state.ts +4 -4
  126. package/src/activities/streaming.ts +42 -3
  127. package/src/activities/types.ts +87 -5
  128. package/src/activities/video-generation-admission.ts +270 -0
  129. package/src/activities/video-generation-credential.ts +34 -0
  130. package/src/activities/video-generation-reconciliation.ts +578 -0
  131. package/src/activities/video-media-validation.ts +326 -0
  132. package/src/activities/video-output-retention.ts +314 -0
  133. package/src/activities/video-reference-staging.ts +263 -0
  134. package/src/activities/workflow-wake.ts +2 -2
  135. package/src/activities/workspace-capture.ts +44 -10
  136. package/src/activities-control.ts +54 -0
  137. package/src/activities-turn.ts +40 -0
  138. package/src/activities.ts +48 -124
  139. package/src/activity-services.ts +62 -0
  140. package/src/artifact-materializer-entry.ts +35 -0
  141. package/src/artifact-outbox-entry.ts +29 -0
  142. package/src/concurrency.ts +484 -6
  143. package/src/editable-artifact-hint-broker.ts +235 -0
  144. package/src/editable-artifact-materializer-launcher.ts +515 -0
  145. package/src/editable-artifact-materializer-service.ts +787 -0
  146. package/src/editable-artifact-materializer-storage.ts +413 -0
  147. package/src/editable-artifact-materializer-subprocess.ts +956 -0
  148. package/src/editable-artifact-materializer.ts +1289 -0
  149. package/src/editable-artifact-outbox-dispatcher.ts +244 -0
  150. package/src/editable-artifact-outbox-service.ts +438 -0
  151. package/src/index.ts +277 -72
  152. package/src/memory-pressure-guard.ts +489 -0
  153. package/src/model-checkpoint-memory-collector.ts +116 -0
  154. package/src/observability-metrics.ts +199 -4
  155. package/src/sandbox-reaper-contract.ts +73 -0
  156. package/src/sandbox-reaper-timeout.ts +79 -0
  157. package/src/sandbox-resume.ts +294 -153
  158. package/src/sandbox-routing.ts +19 -24
  159. package/src/worker-service-lifecycle.ts +80 -14
  160. package/src/workflows/activities.ts +30 -0
  161. package/src/workflows/file-upload-reaper.ts +3 -1
  162. package/src/workflows/knowledge-source-sync.ts +83 -0
  163. package/src/workflows/sandbox-reaper.ts +183 -21
  164. package/src/workflows/scheduled-tasks.ts +42 -6
  165. package/src/workflows/video-generation.ts +36 -0
  166. package/src/workflows.ts +17 -1
  167. package/dist/activities/toolspace-token-renewal.d.ts +0 -23
  168. package/src/activities/toolspace-token-renewal.ts +0 -64
@@ -18,6 +18,7 @@ import {
18
18
  getHumanInputResumeForEvent,
19
19
  getSessionHumanInputRequest,
20
20
  installOrReadTurnExecutionPolicyForAttempt,
21
+ persistAttemptToolCatalog,
21
22
  workspaceCodexSubscriptionActive,
22
23
  acquireCodexCredentialLease,
23
24
  armCodexCapacityWait,
@@ -32,13 +33,13 @@ import {
32
33
  fetchCodexUsageForAccount,
33
34
  getSessionCodexState,
34
35
  recordSessionActiveCodexCredential,
35
- setSessionCodexPin,
36
+ setSessionCodexPinInTransaction,
36
37
  recordCodexAccountUsageWithWakeTargets,
37
38
  quarantineCodexCredentialForLease,
38
39
  setActiveCodexCredential,
39
40
  resolveWorkspaceMemoryBlock,
40
41
  setCodexCredentialExhaustedWithWakeTargets,
41
- withCodexCapacityMutation,
42
+ withSessionCodexCapacityMutation,
42
43
  countConsecutiveReactiveRotations,
43
44
  requireSession,
44
45
  recordUsageEvent,
@@ -48,8 +49,8 @@ import {
48
49
  clearDurablePendingSessionToolCalls,
49
50
  appendSessionHistoryItems,
50
51
  isSessionCompactionRequested,
52
+ getActiveSessionHistoryItemsPaged,
51
53
  countSessionHistoryItems,
52
- getActiveSessionHistoryItems,
53
54
  nextSessionHistoryPosition,
54
55
  settleCodexCredentialLeaseLoss,
55
56
  settleCodexCredentialFailover,
@@ -61,14 +62,22 @@ import {
61
62
  accrueWarmSeconds,
62
63
  getMaterializedSandboxFileResources,
63
64
  markSandboxFileResourcesMaterialized,
65
+ getGeneratedVideoArtifact,
66
+ listSessionSystemUpdatesForTurn,
67
+ getWorkspaceVideoGenerationPolicy,
68
+ loadWorkspaceVercelAiGatewayCredentialLease,
64
69
  areGitHubRepositoriesAllowedForWorkspace,
65
70
  SandboxLeaseRecoveryBlockedError,
66
71
  SandboxLeaseSupersededError,
72
+ SandboxLeaseTransitionError,
67
73
  SandboxImageConflictError,
74
+ ActiveSessionHistoryLimitExceededError,
75
+ ApprovalRunStateLimitExceededError,
68
76
  isSessionEventPersistenceError,
69
77
  getEnrollment,
70
78
  abandonRecordingForTurnAttempt,
71
- markSessionAttemptQuiesced,
79
+ commitSessionAttemptQuiescence,
80
+ getOrCreateCompanyProfileSnapshot,
72
81
  getOrCreatePreferenceRegistrySnapshot,
73
82
  getOrCreateWorkspaceInstructionPolicySnapshot,
74
83
  PreferenceRegistryInitiatorError,
@@ -82,10 +91,12 @@ import {
82
91
  type CodexCredentialLeaseResult,
83
92
  type CodexCredentialLeaseSelectionContext,
84
93
  type ApplySessionTurnSettlementInput,
94
+ type ApiIntegrationRuntime,
95
+ type SessionAttemptQuiescenceCommit,
85
96
  type SessionTurnRecordingSettlement,
86
97
  } from "@opengeni/db";
87
98
  import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@opengeni/events";
88
- import { sandboxOperationMetricObserver } from "@opengeni/observability";
99
+ import { sandboxLeaseTelemetryKey, sandboxOperationMetricObserver } from "@opengeni/observability";
89
100
  import {
90
101
  sandboxStateEntryFromRunState,
91
102
  maxTurnsExceededRunState,
@@ -95,6 +106,7 @@ import {
95
106
  normalizeSdkEvent,
96
107
  normalizeProtocolJsonValue,
97
108
  projectHistoryForProvider,
109
+ restoreGenericDispatchHistoryItems,
98
110
  sanitizeHistoryItemsForModel,
99
111
  projectModelInputForCapabilities,
100
112
  appendPersistentSessionSettings,
@@ -104,7 +116,6 @@ import {
104
116
  composeAgentInstructions,
105
117
  hasActiveWorkspaceInstructionPolicy,
106
118
  renderWorkspaceGovernanceContext,
107
- summarizeForCompaction,
108
119
  requestRemoteCompactionV2,
109
120
  serializedToolsForRemoteCompaction,
110
121
  REMOTE_COMPACTION_V2_BETA_FEATURE,
@@ -119,8 +130,8 @@ import {
119
130
  clearRunCredentialsForAttempt,
120
131
  withRunCredentialsSession,
121
132
  refreshGitCredentialBindingTokenFiles,
122
- refreshToolspaceTokenFile,
123
- toolspaceTokenFileFromEnvironment,
133
+ refreshCodemodeTokenFile,
134
+ codemodeTokenFileFromEnvironment,
124
135
  sandboxFileDownloadFailureNote,
125
136
  SUMMARY_BUFFER_TOKENS,
126
137
  isMcpRequestTimeoutError,
@@ -134,6 +145,7 @@ import {
134
145
  type SandboxFileDownloadFailure,
135
146
  type OpenGeniRuntime,
136
147
  type ComputerToolMode,
148
+ type RetainableSessionImageOutputHook,
137
149
  type ModelResponseUsage,
138
150
  type ModelCallUsageInput,
139
151
  type ModelCallUsageNormalization,
@@ -143,27 +155,33 @@ import {
143
155
  type BackendUnresolvableCode,
144
156
  type EstablishedSandboxSession,
145
157
  type GitCredentialTokenWriterSession,
158
+ type LazyToolTransport,
146
159
  type NormalizedRunCredentialMaterial,
147
160
  type RunCredentialCommandSession,
148
- type ToolspaceTokenWriterSession,
161
+ type CodemodeTokenWriterSession,
162
+ createFirstPartyInteractionAttemptToolDefinitions,
149
163
  deleteRecordingArtifacts,
150
164
  stopRecording as stopRecordingOnBox,
151
165
  } from "@opengeni/runtime";
152
166
  import { connectionTokenResolverForTurn } from "./mcp-credentials";
167
+ import { buildApiIntegrationServersForTurn } from "./api-integrations";
153
168
  import {
154
169
  assertTurnExecutionPolicyMatchesConfigV1,
155
- calculateGatewayReportedCostMicros,
156
- calculateModelUsageCostMicros,
157
- configuredModelPricing,
170
+ calculateGatewayReportedCostBreakdown,
171
+ calculateModelUsageCostBreakdown,
172
+ configuredModelPricingSchedules,
158
173
  configuredStaticUsageLimits,
159
174
  responseSatisfiesLatencyMode,
160
- sandboxArchiveCaptureTimeoutMs,
175
+ sandboxLifecycleTransitionWaitMs,
161
176
  sandboxWarmRateMicrosPerSecond,
162
177
  serviceTierForLatencyMode,
163
178
  settingsWithResolvedModelContext,
164
179
  resolveTurnExecutionPolicyV1,
165
180
  OPENGENI_GATEWAY_MODELS,
166
181
  OPENGENI_GATEWAY_PROVIDER_ID,
182
+ WORKSPACE_GATEWAY_PROVIDER_ID,
183
+ isDirectOpenAiApiBaseUrl,
184
+ resolveModelProvider,
167
185
  type ModelUsageInput,
168
186
  type ModelProviderApi,
169
187
  type RegistryProviderKind,
@@ -196,6 +214,7 @@ import {
196
214
  publishCodexFleetShadowDecisionV1,
197
215
  } from "./codex-fleet-shadow";
198
216
  import type { CodexAccountStatus } from "@opengeni/db";
217
+ import { CodemodeAttemptDispatcher } from "./codemode-dispatcher";
199
218
  import { buildCodexTokenResolver } from "./codex-auth";
200
219
  import {
201
220
  refreshCodexUsageAndRepairCapacityWaiters,
@@ -224,18 +243,25 @@ import {
224
243
  assertSessionAllowsProductModel,
225
244
  defaultSessionMcpServerIds,
226
245
  directPersonalConnectionSubjectId,
246
+ rigProviderImageContentHash,
227
247
  resolveCodexAppsCredentialIdForRun,
228
248
  withFrozenPersonalConnectionDelegations,
229
249
  resolveSessionToolPolicy,
250
+ videoGenerationCapabilitiesForPolicy,
230
251
  } from "@opengeni/core";
231
252
  import { maybeCompactContext, settleFailedContextCompactionLandmark } from "./context-compaction";
232
253
  import { TurnAttemptFencedError } from "./turn-attempt-fenced";
254
+ import {
255
+ admitVideoGenerationRequest,
256
+ managedVideoGenerationCredentialLease,
257
+ type VideoGenerationCredentialLease,
258
+ } from "./video-generation-admission";
233
259
  import {
234
260
  assertGitCredentialRenewalTransportUnchanged,
235
261
  gitCredentialAuthorityForTurn,
236
262
  gitHubTokenMintSelections,
237
263
  loadWorkspaceEnvironmentForRunWithCredentials,
238
- mintSandboxToolspaceToken,
264
+ mintSandboxCodemodeToken,
239
265
  mintRunGitCredentials,
240
266
  mintRunGitCredentialBinding,
241
267
  sandboxEnvironmentForRun,
@@ -252,10 +278,10 @@ import {
252
278
  type RunCredentialRenewalController,
253
279
  } from "./run-credential-renewal";
254
280
  import {
255
- TOOLSPACE_TOKEN_EXPIRY_LEAD_MS,
256
- startToolspaceTokenRenewalLoop,
257
- type ToolspaceTokenRenewalController,
258
- } from "./toolspace-token-renewal";
281
+ CODEMODE_TOKEN_EXPIRY_LEAD_MS,
282
+ startCodemodeTokenRenewalLoop,
283
+ type CodemodeTokenRenewalController,
284
+ } from "./codemode-token-renewal";
259
285
  import {
260
286
  bindRunCredentialResolver,
261
287
  runCredentialAuthNeededPayloads,
@@ -264,10 +290,12 @@ import {
264
290
  import { withFirstPartyTools } from "./goals";
265
291
  import {
266
292
  mergeRigDefaultVariableSetEnvironment,
293
+ rigProviderImageSourceImage,
267
294
  resolveWorkspacePackRuntime,
268
295
  resolveWorkspaceSkillLibraryRuntime,
269
296
  settingsWithPackSandboxImage,
270
297
  settingsWithRigImage,
298
+ settingsWithRigProviderImage,
271
299
  } from "./packs";
272
300
  import { deliverFailedChildTurnToParent } from "./parent-wake";
273
301
  import {
@@ -282,7 +310,7 @@ import {
282
310
  startActivityHeartbeat,
283
311
  } from "./streaming";
284
312
  import type {
285
- ActivityServices,
313
+ TurnActivityServices as ActivityServices,
286
314
  EscapedMcpTimeoutRecoveryDetail,
287
315
  RunAgentTurnInput,
288
316
  RunAgentTurnResult,
@@ -337,11 +365,36 @@ import {
337
365
  compactRetainedScreenshotRunState,
338
366
  materializeRetainedScreenshotHistory,
339
367
  materializeRetainedScreenshotRunState,
368
+ sdkEventContainsInlineImage,
340
369
  retainComputerScreenshot,
370
+ toolOutputContainsInlineImage,
341
371
  typedScreenshotFromSdkEvent,
372
+ typedScreenshotFromToolOutput,
373
+ unavailableRetainedSessionImage,
342
374
  } from "./retained-screenshots";
375
+ import {
376
+ assertGeneratedImageHistoryRetained,
377
+ collectGeneratedImageReceipts,
378
+ collectGeneratedImageArtifactReceipts,
379
+ collectGeneratedImageRunStateArtifactReceipts,
380
+ compactGeneratedImageHistory,
381
+ compactGeneratedImageRunState,
382
+ compactGeneratedImageSdkEvent,
383
+ generatedImageFromSdkEvent,
384
+ generatedImagesFromHistory,
385
+ isCompletedGeneratedImageSdkEvent,
386
+ projectGeneratedImageHistoryForModel,
387
+ projectGeneratedImageRunStateForModel,
388
+ retainGeneratedImage,
389
+ type GeneratedImageReceipt,
390
+ type GeneratedImageOutput,
391
+ } from "./generated-images";
392
+ import { executeGatewayImageGeneration } from "./gateway-image-generation";
393
+ import { executeCodexImageGeneration } from "./codex-image-generation";
394
+ import { imageProviderBindingHash } from "./image-generation-operation";
395
+ import { resolveImageGenerationReferences } from "./image-generation-references";
343
396
  import { captureWorkspaceRevision, openFreshWorkspaceCaptureSession } from "./workspace-capture";
344
- import type { ChannelASession } from "@opengeni/runtime/sandbox";
397
+ import { SandboxChannelAService, type ChannelASession } from "@opengeni/runtime/sandbox";
345
398
  import { createObjectStorage, type ObjectStorage } from "@opengeni/storage";
346
399
  import {
347
400
  desktopCapableBackend,
@@ -358,6 +411,7 @@ import {
358
411
  type LatencyMode,
359
412
  type ResourceRef,
360
413
  type RetainedArtifactMetadata,
414
+ type MediaGenerationResult,
361
415
  type SessionEvent,
362
416
  type SessionEventType,
363
417
  type SessionStatus,
@@ -366,6 +420,7 @@ import {
366
420
  type TurnExecutionPolicyV1,
367
421
  } from "@opengeni/contracts";
368
422
  import { createHash, randomUUID } from "node:crypto";
423
+ import { createModelCheckpointMemoryCollector } from "../model-checkpoint-memory-collector";
369
424
 
370
425
  // Retryable provider connectivity/5xx failures start quickly and back off to
371
426
  // this ceiling. Explicit rate limits retain the minute-granular fallback.
@@ -730,18 +785,22 @@ const QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS = 5_000;
730
785
  * state. The proof object never changes between attempts. */
731
786
  export async function persistOrSignalSessionAttemptQuiescence(input: {
732
787
  proof: SessionAttemptQuiescenceProof;
733
- persistReceipt: () => Promise<SessionEvent[]>;
788
+ persistReceipt: () => Promise<SessionAttemptQuiescenceCommit>;
789
+ deliverWorkflowWake?: (
790
+ wake: NonNullable<SessionAttemptQuiescenceCommit["workflowWake"]>,
791
+ ) => Promise<unknown>;
734
792
  publishEvents: (events: SessionEvent[]) => Promise<unknown>;
735
793
  signalProof: ActivityServices["signalSessionAttemptQuiesced"];
736
794
  sleep?: (ms: number) => Promise<void>;
737
795
  heartbeat?: (attempt: number, delayMs: number) => void;
738
796
  onReceiptFailure?: (error: unknown) => void;
797
+ onWakeFailure?: (error: unknown) => void;
739
798
  onPublishFailure?: (error: unknown) => void;
740
799
  onSignalFailure?: (error: unknown, attempt: number, delayMs: number) => void;
741
800
  }): Promise<"receipt" | "signal"> {
742
- let events: SessionEvent[];
801
+ let receipt: SessionAttemptQuiescenceCommit;
743
802
  try {
744
- events = await input.persistReceipt();
803
+ receipt = await input.persistReceipt();
745
804
  } catch (receiptError) {
746
805
  input.onReceiptFailure?.(receiptError);
747
806
  if (!input.signalProof) {
@@ -771,8 +830,17 @@ export async function persistOrSignalSessionAttemptQuiescence(input: {
771
830
  }
772
831
  }
773
832
 
833
+ if (receipt.workflowWake && input.deliverWorkflowWake) {
834
+ try {
835
+ await input.deliverWorkflowWake(receipt.workflowWake);
836
+ } catch (wakeError) {
837
+ // The exact revision is already durable. Immediate delivery is latency
838
+ // optimization only; the outbox dispatcher remains the repair path.
839
+ input.onWakeFailure?.(wakeError);
840
+ }
841
+ }
774
842
  try {
775
- await input.publishEvents(events);
843
+ await input.publishEvents(receipt.events);
776
844
  } catch (publishError) {
777
845
  // Postgres already committed quiesced_at, the queue event, and the wake.
778
846
  // NATS is live fanout only; never misclassify its failure as receipt loss.
@@ -792,7 +860,7 @@ export async function drainAttemptOwnedSandboxWriters(input: {
792
860
  toolCancellationFence: Pick<TurnToolCancellationFence, "cancel" | "waitForQuiescence"> | null;
793
861
  cancellationReason?: unknown;
794
862
  gitCredentialRenewals: readonly Pick<GitCredentialRenewalController, "stop">[];
795
- toolspaceTokenRenewal: Pick<ToolspaceTokenRenewalController, "stop"> | null;
863
+ codemodeTokenRenewal: Pick<CodemodeTokenRenewalController, "stop"> | null;
796
864
  runCredentialRenewal: Pick<RunCredentialRenewalController, "stop"> | null;
797
865
  }): Promise<void> {
798
866
  if (input.toolCancellationFence) {
@@ -802,10 +870,41 @@ export async function drainAttemptOwnedSandboxWriters(input: {
802
870
  await input.toolCancellationFence.waitForQuiescence();
803
871
  }
804
872
  await Promise.all(input.gitCredentialRenewals.map(async (renewal) => await renewal.stop()));
805
- await input.toolspaceTokenRenewal?.stop();
873
+ await input.codemodeTokenRenewal?.stop();
806
874
  await input.runCredentialRenewal?.stop();
807
875
  }
808
876
 
877
+ /** Persist the exact turn's second-stage release only after the caller has
878
+ * crossed the physical writer boundary. This wait is intentionally independent
879
+ * of Temporal cancellation: it is a short idempotent DB transaction that closes
880
+ * the durable archive-capture fence. Bounded retries cover a rollout connection
881
+ * reset without turning ordinary finalizer housekeeping into lifecycle state. */
882
+ export async function releaseTurnSandboxAfterWriterDrain(
883
+ sandbox: Pick<ResumedTurnSandbox, "release">,
884
+ options: {
885
+ maxAttempts?: number;
886
+ retryDelayMs?: number;
887
+ wait?: (delayMs: number) => Promise<void>;
888
+ } = {},
889
+ ): Promise<void> {
890
+ const maxAttempts = Math.max(1, options.maxAttempts ?? 3);
891
+ const retryDelayMs = Math.max(0, options.retryDelayMs ?? 50);
892
+ const wait = options.wait ?? sleep;
893
+ let lastError: unknown;
894
+ for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
895
+ try {
896
+ await sandbox.release({ workspaceWritersQuiesced: true });
897
+ return;
898
+ } catch (error) {
899
+ lastError = error;
900
+ if (attempt < maxAttempts) {
901
+ await wait(retryDelayMs * attempt);
902
+ }
903
+ }
904
+ }
905
+ throw lastError;
906
+ }
907
+
809
908
  /**
810
909
  * Await a finalizer operation only while this Temporal activity still owns its
811
910
  * execution window. Once Pause/Steer cancellation arrives, the operation keeps
@@ -1942,6 +2041,65 @@ export async function ensureTurnModalRegistryImage(
1942
2041
  await ensureRegistryImage(runSettings);
1943
2042
  }
1944
2043
 
2044
+ const SANDBOX_ARTIFACT_RUNTIME_MANIFEST = "/opt/opengeni/artifact-runtime/installation.json";
2045
+ const SANDBOX_ARTIFACT_TOOL_ENTRY = "/opt/opengeni/artifact-runtime/skill-facade-entry.mjs";
2046
+
2047
+ export type SandboxArtifactRuntimeAdmission = Readonly<{
2048
+ available: boolean;
2049
+ environment: Readonly<Record<string, string>>;
2050
+ }>;
2051
+
2052
+ /**
2053
+ * Admit the optional native standalone-file runtime only for the deployment's
2054
+ * exact base sandbox image contract. A pack/rig image override is an
2055
+ * independent filesystem and therefore fails closed even when the deployment
2056
+ * base image is capable. Collaborative artifact skills are admitted separately
2057
+ * from the frozen canonical tool catalog.
2058
+ *
2059
+ * This keeps lazy provisioning intact: CI/release proves the image closure,
2060
+ * while a before-agent-start doctor verifies the actual box before any model
2061
+ * call. No speculative sandbox is created merely to populate the skill index.
2062
+ */
2063
+ export function sandboxArtifactRuntimeAdmission(
2064
+ deploymentSettings: Settings,
2065
+ runSettings: Settings,
2066
+ backend: Settings["sandboxBackend"] | undefined,
2067
+ options: Readonly<{ production?: boolean }> = {},
2068
+ ): SandboxArtifactRuntimeAdmission {
2069
+ if (!deploymentSettings.sandboxArtifactRuntimeEnabled) {
2070
+ return { available: false, environment: {} };
2071
+ }
2072
+ const production = options.production ?? process.env.NODE_ENV === "production";
2073
+ if (backend === "docker") {
2074
+ const image = deploymentSettings.dockerImage;
2075
+ const localDevelopmentImage = /^opengeni-sandbox:local-[0-9a-f]{12}$/u.test(image);
2076
+ if (
2077
+ runSettings.dockerImage !== image ||
2078
+ (!/@sha256:[0-9a-f]{64}$/u.test(image) && (production || !localDevelopmentImage))
2079
+ ) {
2080
+ return { available: false, environment: {} };
2081
+ }
2082
+ } else if (backend === "modal") {
2083
+ if (
2084
+ !deploymentSettings.modalImageRef ||
2085
+ !/@sha256:[0-9a-f]{64}$/u.test(deploymentSettings.modalImageRef) ||
2086
+ runSettings.modalImageRef !== deploymentSettings.modalImageRef ||
2087
+ runSettings.modalImageId !== deploymentSettings.modalImageId
2088
+ ) {
2089
+ return { available: false, environment: {} };
2090
+ }
2091
+ } else {
2092
+ return { available: false, environment: {} };
2093
+ }
2094
+ return {
2095
+ available: true,
2096
+ environment: {
2097
+ OPENGENI_ARTIFACT_RUNTIME_MANIFEST: SANDBOX_ARTIFACT_RUNTIME_MANIFEST,
2098
+ OPENGENI_ARTIFACT_TOOL_ENTRY: SANDBOX_ARTIFACT_TOOL_ENTRY,
2099
+ },
2100
+ };
2101
+ }
2102
+
1945
2103
  /**
1946
2104
  * Decide whether the first actual computer action may start a proof recording.
1947
2105
  *
@@ -2041,6 +2199,36 @@ export function structuredToolTransportForTurn(
2041
2199
  );
2042
2200
  }
2043
2201
 
2202
+ /**
2203
+ * Progressive tool disclosure is universal for supported OpenGeni turns; only
2204
+ * its contained transport differs. Codex keeps its native path, built-in direct
2205
+ * OpenAI/Azure Responses use native client tool search, and every other ordinary
2206
+ * function-calling provider uses OpenGeni's stable search/invoke dispatcher.
2207
+ */
2208
+ export function lazyToolTransportForTurn(
2209
+ resolvedModel: {
2210
+ provider: {
2211
+ id: string;
2212
+ kind: RegistryProviderKind;
2213
+ api: ModelProviderApi;
2214
+ builtin: boolean;
2215
+ baseUrl?: string | undefined;
2216
+ };
2217
+ } | null,
2218
+ ): LazyToolTransport {
2219
+ if (!resolvedModel) return "openai_native";
2220
+ const provider = resolvedModel.provider;
2221
+ if (provider.kind === "codex-subscription") return "codex_native";
2222
+ const isDirectBuiltinResponses =
2223
+ provider.builtin &&
2224
+ provider.api === "responses" &&
2225
+ (provider.id === "azure" || (provider.id === "openai" && provider.baseUrl === undefined));
2226
+ if (isDirectBuiltinResponses) {
2227
+ return "openai_native";
2228
+ }
2229
+ return "generic_dispatch";
2230
+ }
2231
+
2044
2232
  /**
2045
2233
  * Native web search is a runtime capability, not part of the session's MCP
2046
2234
  * allow-list. Attach it whenever the resolved provider advertises runnable
@@ -2055,6 +2243,39 @@ export function hostedWebSearchForTurn(
2055
2243
  return resolvedModel?.configured.hostedWebSearch ?? deploymentWebSearchEnabled;
2056
2244
  }
2057
2245
 
2246
+ /** Direct OpenAI hosted image IDs are reusable only by the exact API credential. */
2247
+ export function openAiHostedImageProviderBindingForTurn(
2248
+ settings: Pick<Settings, "openaiProvider" | "openaiApiKey" | "openaiBaseUrl">,
2249
+ resolvedModel: {
2250
+ provider: {
2251
+ id: string;
2252
+ kind: RegistryProviderKind;
2253
+ builtin: boolean;
2254
+ baseUrl?: string | undefined;
2255
+ };
2256
+ configured: {
2257
+ capabilities: {
2258
+ hostedTools: { imageGeneration: { runnable: boolean } };
2259
+ };
2260
+ };
2261
+ } | null,
2262
+ ): { providerId: string; providerBindingHash: string } | null {
2263
+ if (!resolvedModel?.configured.capabilities.hostedTools.imageGeneration.runnable) return null;
2264
+ const providerId = resolvedModel.provider.id;
2265
+ const isDirectOpenAi =
2266
+ resolvedModel.provider.builtin &&
2267
+ resolvedModel.provider.id === "openai" &&
2268
+ isDirectOpenAiApiBaseUrl(resolvedModel.provider.baseUrl) &&
2269
+ settings.openaiProvider === "openai" &&
2270
+ isDirectOpenAiApiBaseUrl(settings.openaiBaseUrl);
2271
+ const bindingIdentity = isDirectOpenAi ? settings.openaiApiKey : null;
2272
+ if (!bindingIdentity) return null;
2273
+ return {
2274
+ providerId,
2275
+ providerBindingHash: imageProviderBindingHash(providerId, bindingIdentity),
2276
+ };
2277
+ }
2278
+
2058
2279
  /** Exact model-catalog modality gate; null preserves the legacy built-in path. */
2059
2280
  export function modelSupportsImageInputForTurn(
2060
2281
  resolvedModel: {
@@ -2080,7 +2301,10 @@ export function modelAttachmentInputPolicyForTurn(
2080
2301
  resolvedModel: {
2081
2302
  provider: { api: ModelProviderApi };
2082
2303
  configured: {
2083
- capabilities?: { inputModalities: string[]; inputFileMediaTypes?: string[] };
2304
+ capabilities?: {
2305
+ inputModalities: string[];
2306
+ inputFileMediaTypes?: string[];
2307
+ };
2084
2308
  };
2085
2309
  } | null,
2086
2310
  ): ModelAttachmentInputPolicy {
@@ -2207,27 +2431,30 @@ export function isLazySandboxProvisionRetryable(error: unknown): boolean {
2207
2431
  if (error instanceof WorkspaceArchiveIntegrityError) {
2208
2432
  return error.retryable;
2209
2433
  }
2210
- if (error instanceof SandboxLeaseSupersededError || error instanceof SandboxWarmingTimeoutError) {
2434
+ if (
2435
+ error instanceof SandboxLeaseSupersededError ||
2436
+ error instanceof SandboxLeaseTransitionError
2437
+ ) {
2211
2438
  return true;
2212
2439
  }
2213
- const message = error instanceof Error ? error.message : String(error);
2214
- return /(?:capacity|create|creation|provider|sandbox).*(?:timeout|timed out)|(?:timeout|timed out).*(?:capacity|create|creation|provider|sandbox)|ECONNRESET|ETIMEDOUT|EAI_AGAIN|temporar/i.test(
2215
- message,
2216
- );
2440
+ if (error instanceof SandboxWarmingTimeoutError) {
2441
+ return false;
2442
+ }
2443
+ // Provider/transport text is not durable evidence that creation never
2444
+ // happened. Retrying an ambiguous unknown here can create a second box; only
2445
+ // typed, ownership-fenced lifecycle outcomes above are safe to replay.
2446
+ return false;
2217
2447
  }
2218
2448
 
2219
- /**
2220
- * Deadline recovery cannot re-admit the same turn until the global reaper has
2221
- * observed the released holder, captured the final archive, terminated the old
2222
- * provider box, and committed the lease cold. Give that full sequence one
2223
- * snapshot budget plus two schedule periods. This is workflow-visible pacing,
2224
- * not an in-memory sleep, so Pause/Steer/new user intent can still wake the
2225
- * durable session immediately.
2226
- */
2449
+ /** Short workflow-visible anti-churn pacing after a lifecycle transition. The
2450
+ * next acquire waits on the exact durable claim itself, so this is deliberately
2451
+ * not a guessed snapshot-plus-schedules completion time. */
2227
2452
  export function sandboxDeadlineRotationRecoveryDelayMs(
2228
- settings: Pick<Settings, "sandboxLeaseReaperPeriodMs" | "sandboxSnapshotTimeoutMs">,
2453
+ settings: Pick<Settings, "sandboxLeaseReaperPeriodMs">,
2229
2454
  ): number {
2230
- return settings.sandboxSnapshotTimeoutMs + settings.sandboxLeaseReaperPeriodMs * 2;
2455
+ // The next attempt waits on the durable lifecycle claim itself. This delay is
2456
+ // only anti-churn pacing, not an estimate of snapshot + future schedules.
2457
+ return Math.min(5_000, settings.sandboxLeaseReaperPeriodMs);
2231
2458
  }
2232
2459
 
2233
2460
  export function createTurnSandboxProvisioner<T>(
@@ -2448,12 +2675,14 @@ export async function finalizeDurableTurnOpStreams(
2448
2675
  }
2449
2676
 
2450
2677
  export function createRunAgentTurnActivity(services: () => Promise<ActivityServices>) {
2678
+ const modelCheckpointMemoryCollector = createModelCheckpointMemoryCollector();
2451
2679
  return async function runAgentTurn(input: RunAgentTurnInput): Promise<RunAgentTurnResult> {
2452
2680
  const {
2453
2681
  settings,
2454
2682
  db,
2455
2683
  bus,
2456
2684
  runtime,
2685
+ summarizeContextForCompaction,
2457
2686
  objectStorage,
2458
2687
  observability,
2459
2688
  wakeSessionWorkflow,
@@ -2461,6 +2690,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2461
2690
  signalCodexCapacityWorkflow,
2462
2691
  entitlements,
2463
2692
  connectionCredentials,
2693
+ startVideoGenerationWorkflow,
2464
2694
  } = await services();
2465
2695
  const activityContext = currentActivityContext();
2466
2696
  const cancellationSignal = activityContext?.cancellationSignal;
@@ -2673,6 +2903,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2673
2903
  // run. null when the flag is off (byte-for-byte the legacy build-and-discard
2674
2904
  // path) OR when the backend is "none". Released + dropped in `finally`.
2675
2905
  let resolvedSandbox: ResumedTurnSandbox | null = null;
2906
+ let attemptWritersDrained = false;
2907
+ const lateSandboxesAwaitingWriterDrain = new Set<ResumedTurnSandbox>();
2908
+ const releaseLateSandbox = async (sandbox: ResumedTurnSandbox): Promise<void> => {
2909
+ lateSandboxesAwaitingWriterDrain.add(sandbox);
2910
+ // Drop the holder/timer immediately, but keep its null-outcome admissions
2911
+ // fenced until the shared attempt writer drain completes. The staged
2912
+ // release serializes a later proof-bearing call behind this one.
2913
+ await sandbox.release().catch(() => undefined);
2914
+ if (!attemptWritersDrained) return;
2915
+ try {
2916
+ await releaseTurnSandboxAfterWriterDrain(sandbox);
2917
+ lateSandboxesAwaitingWriterDrain.delete(sandbox);
2918
+ } catch (error) {
2919
+ console.error(
2920
+ "late sandbox quiesced release failed (turn outcome unaffected)",
2921
+ safeErrorDiagnostic(error),
2922
+ );
2923
+ }
2924
+ };
2676
2925
  const requireResolvedSandboxForMutation = (message: string): ResumedTurnSandbox => {
2677
2926
  if (!resolvedSandbox) throw new Error(message);
2678
2927
  return resolvedSandbox;
@@ -2754,7 +3003,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2754
3003
  expectedEpoch: sandbox.leaseEpoch,
2755
3004
  expectedInstanceId: sandbox.established.instanceId,
2756
3005
  operation,
2757
- captureWaitMs: sandboxArchiveCaptureTimeoutMs(settings),
3006
+ captureWaitMs: sandboxLifecycleTransitionWaitMs(settings),
3007
+ ...(cancellationSignal ? { waitSignal: cancellationSignal } : {}),
2758
3008
  };
2759
3009
  const admission = await advanceWorkspaceGeneration(db, identity);
2760
3010
  let result: T;
@@ -2811,11 +3061,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2811
3061
  let runCredentialRenewal: RunCredentialRenewalController | null = null;
2812
3062
  let runCredentialRenewalClosed = false;
2813
3063
  let runCredentialSession: RunCredentialCommandSession | null = null;
2814
- // The delegated Toolspace bearer has a one-hour TTL. Renewal is attempt-
3064
+ // The delegated Codemode bearer has a one-hour TTL. Renewal is attempt-
2815
3065
  // owned and attaches only after the initial token file reached a real
2816
3066
  // sandbox session; finalization drains an in-flight replacement.
2817
- let toolspaceTokenRenewal: ToolspaceTokenRenewalController | null = null;
2818
- let toolspaceTokenRenewalClosed = false;
3067
+ let codemodeTokenRenewal: CodemodeTokenRenewalController | null = null;
3068
+ let codemodeTokenRenewalClosed = false;
2819
3069
  // MID-SESSION snapshot single-flight guard: the heartbeat tick fires every
2820
3070
  // 10s but a Modal filesystem snapshot can take longer — never overlap two
2821
3071
  // captures on one box. The in-flight capture's promise is held so the
@@ -2869,6 +3119,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2869
3119
  await current?.flush().catch(() => undefined);
2870
3120
  };
2871
3121
  let preparedTools: Awaited<ReturnType<OpenGeniRuntime["prepareTools"]>> | null = null;
3122
+ let codemodeDispatcher: CodemodeAttemptDispatcher | null = null;
2872
3123
  const toolCancellationFenceRef: {
2873
3124
  current: TurnToolCancellationFence | null;
2874
3125
  } = {
@@ -3195,9 +3446,292 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3195
3446
  // remains the final call/result pairing guard for every other reconcile.
3196
3447
  let persistedHistoryCount = 0;
3197
3448
  const retainedScreenshotReceiptsByCallId = new Map<string, RetainedArtifactMetadata>();
3198
- const computerScreenshotCallIds = new Set<string>();
3449
+ // Set from the resolved provider wire before any model history is prepared.
3450
+ // Text-only/chat wires keep compact receipts and must never allocate image
3451
+ // bytes merely for the following request-local projection to remove them.
3452
+ let modelCanReceiveRetainedSessionImages = true;
3453
+ const generatedImageReceiptsByProviderItemId = new Map<string, GeneratedImageReceipt>();
3454
+ const generatedImageReceiptsByArtifactId = new Map<string, GeneratedImageReceipt>();
3455
+ const videoGenerationAcceptancesByCallId = new Map<
3456
+ string,
3457
+ { operationId: string; requestDigest: string }
3458
+ >();
3459
+ const requiredGeneratedVideoFiles: Array<{
3460
+ operationId: string;
3461
+ artifactId: string;
3462
+ fileId: string;
3463
+ objectKey: string;
3464
+ sizeBytes: number;
3465
+ sha256: string;
3466
+ filename: string;
3467
+ }> = [];
3468
+ let generatedImageMaterializationCache: {
3469
+ instanceId: string;
3470
+ fileIds: Set<string>;
3471
+ } | null = null;
3472
+ // Legacy ownership mode lets the Agents SDK create the sandbox inside
3473
+ // run(). Keep that exact session once the runtime exposes it so an image
3474
+ // generated later in the same model/tool loop can be copied immediately.
3475
+ let sdkOwnedSandboxSession: CodemodeTokenWriterSession | null = null;
3476
+ const prepareGeneratedImageDownload = async (receipt: GeneratedImageReceipt) => {
3477
+ if (!objectStorage) {
3478
+ throw new Error("Generated image sandbox materialization requires object storage");
3479
+ }
3480
+ const file = await requireFile(db, input.workspaceId, receipt.artifact.artifactId);
3481
+ const downloadStorage = objectStorageForSandboxDownloads(modelRunSettings, objectStorage);
3482
+ const signed = await downloadStorage.createGetUrl({
3483
+ key: file.objectKey,
3484
+ });
3485
+ return {
3486
+ file,
3487
+ download: {
3488
+ fileId: file.id,
3489
+ mountPath: "generated-images",
3490
+ filename: file.safeFilename,
3491
+ url: signed.url,
3492
+ expiresAt: signed.expiresAt,
3493
+ sizeBytes: file.sizeBytes,
3494
+ ...(file.sha256 ? { sha256: file.sha256 } : {}),
3495
+ } satisfies SandboxFileDownload,
3496
+ };
3497
+ };
3498
+ const writeGeneratedImageDownload = async (
3499
+ sessionForDownload: CodemodeTokenWriterSession,
3500
+ download: SandboxFileDownload,
3501
+ ): Promise<void> => {
3502
+ const runAs = sandboxRunAs(modelRunSettings);
3503
+ const materialized = await materializeSandboxFileDownloads(sessionForDownload, [download], {
3504
+ onRuntimeEvent: async (event) => {
3505
+ await publish?.([{ type: event.type, payload: event.payload }], true);
3506
+ },
3507
+ ...(runAs ? { runAs } : {}),
3508
+ ...(toolCancellationFenceRef.current
3509
+ ? {
3510
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
3511
+ toolCancellationFenceRef.current,
3512
+ ),
3513
+ }
3514
+ : {}),
3515
+ });
3516
+ if (materialized.failures.length > 0) {
3517
+ throw new Error(materialized.failures[0]!.reason);
3518
+ }
3519
+ };
3520
+ const warnGeneratedImageMaterializationDeferred = (error: unknown): void => {
3521
+ // Generation and permanent retention are already durable. Never turn a
3522
+ // transient sandbox copy failure into a replay of paid provider work;
3523
+ // an unmaterialized resource is retried on the next real sandbox.
3524
+ observability.warn("Generated image sandbox materialization deferred", {
3525
+ errorClass: error instanceof Error ? error.name : "UnknownError",
3526
+ errorCode: "generated_image_materialization_deferred",
3527
+ origin: "worker",
3528
+ });
3529
+ };
3530
+ const materializeGeneratedImageInSandbox = async (
3531
+ receipt: GeneratedImageReceipt,
3532
+ sandbox: ResumedTurnSandbox,
3533
+ sessionForDownload: unknown,
3534
+ ): Promise<boolean> => {
3535
+ try {
3536
+ const { file, download } = await prepareGeneratedImageDownload(receipt);
3537
+ const cacheable = sandbox.established.backendId !== "selfhosted";
3538
+ if (generatedImageMaterializationCache?.instanceId !== sandbox.established.instanceId) {
3539
+ const fileIds =
3540
+ cacheable && sandboxGroupId
3541
+ ? await getMaterializedSandboxFileResources(db, {
3542
+ accountId: input.accountId,
3543
+ workspaceId: input.workspaceId,
3544
+ sandboxGroupId,
3545
+ expectedEpoch: sandbox.leaseEpoch,
3546
+ instanceId: sandbox.established.instanceId,
3547
+ })
3548
+ : new Set<string>();
3549
+ generatedImageMaterializationCache = {
3550
+ instanceId: sandbox.established.instanceId,
3551
+ fileIds,
3552
+ };
3553
+ }
3554
+ if (generatedImageMaterializationCache.fileIds.has(file.id)) return true;
3555
+ await runWorkspaceMutationForSandbox(
3556
+ sandbox,
3557
+ "generatedImageMaterialization",
3558
+ async () =>
3559
+ await writeGeneratedImageDownload(
3560
+ sessionForDownload as CodemodeTokenWriterSession,
3561
+ download,
3562
+ ),
3563
+ );
3564
+ generatedImageMaterializationCache.fileIds.add(file.id);
3565
+ if (cacheable && sandboxGroupId) {
3566
+ await markSandboxFileResourcesMaterialized(db, {
3567
+ accountId: input.accountId,
3568
+ workspaceId: input.workspaceId,
3569
+ sandboxGroupId,
3570
+ expectedEpoch: sandbox.leaseEpoch,
3571
+ instanceId: sandbox.established.instanceId,
3572
+ fileIds: [file.id],
3573
+ });
3574
+ }
3575
+ return true;
3576
+ } catch (error) {
3577
+ warnGeneratedImageMaterializationDeferred(error);
3578
+ return false;
3579
+ }
3580
+ };
3581
+ const materializeGeneratedImageInOwnedSdkSession = async (
3582
+ receipt: GeneratedImageReceipt,
3583
+ sessionForDownload: unknown,
3584
+ ): Promise<boolean> => {
3585
+ try {
3586
+ const { download } = await prepareGeneratedImageDownload(receipt);
3587
+ await writeGeneratedImageDownload(
3588
+ sessionForDownload as CodemodeTokenWriterSession,
3589
+ download,
3590
+ );
3591
+ return true;
3592
+ } catch (error) {
3593
+ warnGeneratedImageMaterializationDeferred(error);
3594
+ return false;
3595
+ }
3596
+ };
3597
+ const materializeGeneratedImage = async (receipt: GeneratedImageReceipt): Promise<boolean> => {
3598
+ if (modelRunSettings.sandboxBackend === "none") return false;
3599
+ // Never allocate a cloud sandbox merely because durable history contains
3600
+ // an image. If this turn later needs one, its ready hook copies every
3601
+ // known receipt before releasing the first operation.
3602
+ if (resolvedSandbox && setupBoxSession) {
3603
+ return await materializeGeneratedImageInSandbox(receipt, resolvedSandbox, setupBoxSession);
3604
+ }
3605
+ if (sdkOwnedSandboxSession) {
3606
+ return await materializeGeneratedImageInOwnedSdkSession(receipt, sdkOwnedSandboxSession);
3607
+ }
3608
+ return false;
3609
+ };
3610
+ let nativeImageGenerationRetention: {
3611
+ providerId: string;
3612
+ providerBindingHash: string;
3613
+ sessionId: string;
3614
+ turnId: string;
3615
+ attemptId: string;
3616
+ } | null = null;
3617
+ const retainNativeGeneratedImage = async (
3618
+ output: GeneratedImageOutput,
3619
+ ): Promise<GeneratedImageReceipt> => {
3620
+ if (output.providerItemId) {
3621
+ const existing = generatedImageReceiptsByProviderItemId.get(output.providerItemId);
3622
+ if (existing) return existing;
3623
+ }
3624
+ const context = nativeImageGenerationRetention;
3625
+ if (!context || !output.providerItemId) {
3626
+ throw new Error("Native generated image arrived without provider retention identity");
3627
+ }
3628
+ const retained = await retainGeneratedImage({
3629
+ db,
3630
+ objectStorage,
3631
+ accountId: input.accountId,
3632
+ workspaceId: input.workspaceId,
3633
+ sessionId: context.sessionId,
3634
+ turnId: context.turnId,
3635
+ attemptId: context.attemptId,
3636
+ sourceStrategy: "native_hosted",
3637
+ providerId: context.providerId,
3638
+ providerBindingHash: context.providerBindingHash,
3639
+ output,
3640
+ });
3641
+ generatedImageReceiptsByProviderItemId.set(output.providerItemId, retained.receipt);
3642
+ generatedImageReceiptsByArtifactId.set(
3643
+ retained.receipt.artifact.artifactId,
3644
+ retained.receipt,
3645
+ );
3646
+ await materializeGeneratedImage(retained.receipt);
3647
+ return retained.receipt;
3648
+ };
3649
+ const retainNativeGeneratedImagesFromHistory = async (
3650
+ history: Array<Record<string, unknown>>,
3651
+ ): Promise<void> => {
3652
+ collectGeneratedImageReceipts(history, generatedImageReceiptsByProviderItemId);
3653
+ collectGeneratedImageArtifactReceipts(history, generatedImageReceiptsByArtifactId);
3654
+ for (const output of generatedImagesFromHistory(
3655
+ history,
3656
+ generatedImageReceiptsByProviderItemId,
3657
+ )) {
3658
+ if (
3659
+ !output.providerItemId ||
3660
+ generatedImageReceiptsByProviderItemId.has(output.providerItemId)
3661
+ ) {
3662
+ continue;
3663
+ }
3664
+ await retainNativeGeneratedImage(output);
3665
+ }
3666
+ assertGeneratedImageHistoryRetained(history, generatedImageReceiptsByProviderItemId);
3667
+ };
3668
+ const compactMediaRunState = (serialized: string): string =>
3669
+ compactGeneratedImageRunState(
3670
+ compactRetainedScreenshotRunState(serialized, retainedScreenshotReceiptsByCallId),
3671
+ generatedImageReceiptsByProviderItemId,
3672
+ );
3673
+ // Explicit image-producing tools cross the durable session-media boundary.
3674
+ // Incidental frames returned by click/scroll actions remain unretained; an
3675
+ // intentional computer_screenshot or view_image result must never be lost
3676
+ // when the event/history sanitizer removes its inline bytes.
3677
+ const retainedSessionImageCallIds = new Set<string>();
3678
+ const retainSessionImageAtToolBoundary: RetainableSessionImageOutputHook = async ({
3679
+ toolCallId,
3680
+ output,
3681
+ }) => {
3682
+ const activeTurnId = turnId;
3683
+ if (!activeTurnId) {
3684
+ throw new Error("Session image tool completed before turn initialization");
3685
+ }
3686
+ const typedScreenshot = typedScreenshotFromToolOutput({
3687
+ callId: toolCallId,
3688
+ output,
3689
+ });
3690
+ retainedSessionImageCallIds.add(toolCallId);
3691
+ if (!typedScreenshot) {
3692
+ if (toolOutputContainsInlineImage(output)) {
3693
+ retainedScreenshotReceiptsByCallId.set(
3694
+ toolCallId,
3695
+ unavailableRetainedSessionImage({
3696
+ sessionId: input.sessionId,
3697
+ turnId: activeTurnId,
3698
+ attemptId: input.attemptId,
3699
+ toolCallId,
3700
+ toolOutputId: toolCallId,
3701
+ reason: "unsupported",
3702
+ }),
3703
+ );
3704
+ }
3705
+ return;
3706
+ }
3707
+ retainedScreenshotReceiptsByCallId.set(
3708
+ toolCallId,
3709
+ unavailableRetainedSessionImage({
3710
+ sessionId: input.sessionId,
3711
+ turnId: activeTurnId,
3712
+ attemptId: input.attemptId,
3713
+ toolCallId,
3714
+ toolOutputId: toolCallId,
3715
+ reason: "pending",
3716
+ }),
3717
+ );
3718
+ retainedScreenshotReceiptsByCallId.set(
3719
+ toolCallId,
3720
+ await retainComputerScreenshot({
3721
+ db,
3722
+ objectStorage,
3723
+ accountId: input.accountId,
3724
+ workspaceId: input.workspaceId,
3725
+ sessionId: input.sessionId,
3726
+ turnId: activeTurnId,
3727
+ attemptId: input.attemptId,
3728
+ output: typedScreenshot,
3729
+ }),
3730
+ );
3731
+ };
3199
3732
  const materializeScreenshotHistory = async (history: Array<Record<string, unknown>>) => {
3200
3733
  collectRetainedScreenshotReceipts(history, retainedScreenshotReceiptsByCallId);
3734
+ if (!modelCanReceiveRetainedSessionImages) return history;
3201
3735
  return await materializeRetainedScreenshotHistory({
3202
3736
  db,
3203
3737
  objectStorage,
@@ -3208,13 +3742,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3208
3742
  };
3209
3743
  const materializeScreenshotRunState = async (serialized: string) => {
3210
3744
  collectRetainedScreenshotRunStateReceipts(serialized, retainedScreenshotReceiptsByCallId);
3211
- return await materializeRetainedScreenshotRunState({
3212
- db,
3213
- objectStorage,
3214
- workspaceId: input.workspaceId,
3215
- sessionId: input.sessionId,
3216
- serialized,
3217
- });
3745
+ collectGeneratedImageRunStateArtifactReceipts(serialized, generatedImageReceiptsByArtifactId);
3746
+ const withScreenshots = modelCanReceiveRetainedSessionImages
3747
+ ? await materializeRetainedScreenshotRunState({
3748
+ db,
3749
+ objectStorage,
3750
+ workspaceId: input.workspaceId,
3751
+ sessionId: input.sessionId,
3752
+ serialized,
3753
+ })
3754
+ : serialized;
3755
+ return projectGeneratedImageRunStateForModel(
3756
+ compactGeneratedImageRunState(withScreenshots, generatedImageReceiptsByProviderItemId),
3757
+ );
3218
3758
  };
3219
3759
  let providerArtifactCandidates: Awaited<
3220
3760
  ReturnType<typeof turnInput>
@@ -3237,9 +3777,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3237
3777
  try {
3238
3778
  const rawHistory = (stream.state as { history?: unknown[] }).history;
3239
3779
  if (Array.isArray(rawHistory)) {
3240
- const durableHistory = compactRetainedScreenshotHistory(
3241
- rawHistory as Array<Record<string, unknown>>,
3242
- retainedScreenshotReceiptsByCallId,
3780
+ const typedHistory = rawHistory as Array<Record<string, unknown>>;
3781
+ await retainNativeGeneratedImagesFromHistory(typedHistory);
3782
+ const durableHistory = compactGeneratedImageHistory(
3783
+ compactRetainedScreenshotHistory(typedHistory, retainedScreenshotReceiptsByCallId),
3784
+ generatedImageReceiptsByProviderItemId,
3243
3785
  );
3244
3786
  const { rows, nextWatermark, nextPosition } = historyRowsToAppend(
3245
3787
  durableHistory,
@@ -3324,10 +3866,39 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3324
3866
  // only exists in the RunState blob), never through a swapped trigger.
3325
3867
  let triggerType: string | null = null;
3326
3868
  try {
3869
+ const session = await requireSession(db, input.workspaceId, input.sessionId);
3870
+ const claim = await claimSessionWorkForAttempt(db, input.workspaceId, {
3871
+ sessionId: input.sessionId,
3872
+ workflowId: input.workflowId,
3873
+ workflowRunId: input.workflowRunId,
3874
+ attemptId: input.attemptId,
3875
+ dispatchId,
3876
+ trigger: input.trigger,
3877
+ });
3878
+ if (claim.action === "unclaimed") {
3879
+ activityStatus = "unclaimed";
3880
+ return { status: "unclaimed", reason: claim.reason };
3881
+ }
3882
+ const turn = claim.turn;
3883
+ turnId = turn.id;
3884
+ executionGeneration = turn.executionGeneration;
3885
+ providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
3886
+ let installedApiIntegrations: readonly ApiIntegrationRuntime[] = [];
3887
+ const credentialSubjectId = credentialSubjectIdForTurnInitiator(turn);
3327
3888
  const mcpSettings = await settingsWithEnabledCapabilityMcpServers(
3328
3889
  db,
3329
3890
  input.workspaceId,
3330
3891
  settings,
3892
+ {
3893
+ ...(credentialSubjectId
3894
+ ? { subjectId: credentialSubjectId }
3895
+ : {
3896
+ personalConnectionDelegations: turn.personalConnectionDelegations,
3897
+ }),
3898
+ onResolvedApiIntegrations: (integrations) => {
3899
+ installedApiIntegrations = integrations;
3900
+ },
3901
+ },
3331
3902
  );
3332
3903
  // Read the active-credential flag once for the runtime capability overlay.
3333
3904
  // Accepted billing/provider identity comes from the turn policy below,
@@ -3352,23 +3923,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3352
3923
  ? await resolveCodexAppsCredentialIdForRun(db, input.workspaceId)
3353
3924
  : null;
3354
3925
  runtime.configure(capabilitySettings);
3355
- const session = await requireSession(db, input.workspaceId, input.sessionId);
3356
- const claim = await claimSessionWorkForAttempt(db, input.workspaceId, {
3357
- sessionId: input.sessionId,
3358
- workflowId: input.workflowId,
3359
- workflowRunId: input.workflowRunId,
3360
- attemptId: input.attemptId,
3361
- dispatchId,
3362
- trigger: input.trigger,
3363
- });
3364
- if (claim.action === "unclaimed") {
3365
- activityStatus = "unclaimed";
3366
- return { status: "unclaimed", reason: claim.reason };
3367
- }
3368
- const turn = claim.turn;
3369
- turnId = turn.id;
3370
- executionGeneration = turn.executionGeneration;
3371
- providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
3372
3926
  const claimedPolicy = readTurnExecutionPolicyV1(turn.metadata);
3373
3927
  const policyForAbsent =
3374
3928
  claimedPolicy.kind === "valid"
@@ -3499,9 +4053,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3499
4053
  inputs,
3500
4054
  {
3501
4055
  onAppend: ({ durationSeconds }) =>
3502
- recordSessionEventAppendLatency(observability, { durationSeconds }),
4056
+ recordSessionEventAppendLatency(observability, {
4057
+ durationSeconds,
4058
+ }),
3503
4059
  onPublish: ({ durationSeconds }) =>
3504
- recordSessionEventPublishLatency(observability, { durationSeconds }),
4060
+ recordSessionEventPublishLatency(observability, {
4061
+ durationSeconds,
4062
+ }),
3505
4063
  },
3506
4064
  );
3507
4065
  if (inputs.length > 0 && !appended.accepted) {
@@ -3565,10 +4123,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3565
4123
  const runState = inputSettlement.runState
3566
4124
  ? {
3567
4125
  ...inputSettlement.runState,
3568
- serializedRunState: compactRetainedScreenshotRunState(
3569
- inputSettlement.runState.serializedRunState,
3570
- retainedScreenshotReceiptsByCallId,
3571
- ),
4126
+ serializedRunState: compactMediaRunState(inputSettlement.runState.serializedRunState),
3572
4127
  }
3573
4128
  : undefined;
3574
4129
  const result = await applySessionTurnSettlement(db, input.workspaceId, {
@@ -3797,14 +4352,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3797
4352
  leased.credentialId !== null &&
3798
4353
  (sessionPinSource !== "policy" || sessionPin !== leased.credentialId)
3799
4354
  ) {
3800
- const pinMutation = await withCodexCapacityMutation(
4355
+ const pinMutation = await withSessionCodexCapacityMutation(
3801
4356
  db,
3802
4357
  {
3803
4358
  workspaceId: input.workspaceId,
3804
4359
  reason: "codex_policy_pin_changed",
3805
4360
  },
3806
4361
  async (tx) => {
3807
- const changed = await setSessionCodexPin(
4362
+ const changed = await setSessionCodexPinInTransaction(
3808
4363
  tx,
3809
4364
  input.workspaceId,
3810
4365
  input.sessionId,
@@ -3825,14 +4380,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3825
4380
  pinMutation.wakeTargets,
3826
4381
  );
3827
4382
  } else if (selectedPinDisposition === "clearStale") {
3828
- const pinMutation = await withCodexCapacityMutation(
4383
+ const pinMutation = await withSessionCodexCapacityMutation(
3829
4384
  db,
3830
4385
  {
3831
4386
  workspaceId: input.workspaceId,
3832
4387
  reason: "codex_stale_policy_pin_cleared",
3833
4388
  },
3834
4389
  async (tx) => {
3835
- const changed = await setSessionCodexPin(
4390
+ const changed = await setSessionCodexPinInTransaction(
3836
4391
  tx,
3837
4392
  input.workspaceId,
3838
4393
  input.sessionId,
@@ -4292,38 +4847,49 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4292
4847
  attemptId: input.attemptId,
4293
4848
  executionGeneration: turn.executionGeneration,
4294
4849
  };
4295
- const [workspace, instructionPolicySnapshot, preferenceSnapshot] = await Promise.all([
4296
- getWorkspace(db, input.workspaceId),
4297
- getOrCreateWorkspaceInstructionPolicySnapshot(db, governanceClaims),
4298
- getOrCreatePreferenceRegistrySnapshot(db, governanceClaims).catch((error) => {
4299
- if (error instanceof PreferenceRegistryInitiatorError) return null;
4300
- throw error;
4301
- }),
4302
- ]);
4850
+ const [workspace, companyProfileSnapshot, instructionPolicySnapshot, preferenceSnapshot] =
4851
+ await Promise.all([
4852
+ getWorkspace(db, input.workspaceId),
4853
+ getOrCreateCompanyProfileSnapshot(db, governanceClaims),
4854
+ getOrCreateWorkspaceInstructionPolicySnapshot(db, governanceClaims),
4855
+ getOrCreatePreferenceRegistrySnapshot(db, governanceClaims).catch((error) => {
4856
+ if (error instanceof PreferenceRegistryInitiatorError) return null;
4857
+ throw error;
4858
+ }),
4859
+ ]);
4303
4860
  if (!workspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
4304
4861
  const workspaceAgentInstructions = workspace.agentInstructions;
4305
4862
  const agentHumanInputEnabled = resolveWorkspaceAgentHumanInputEnabled(workspace.settings);
4306
4863
  assertWorkspaceHumanInputAllowed(agentHumanInputEnabled, "resume", humanInputResume !== null);
4307
4864
  const workspaceGovernance = renderWorkspaceGovernanceContext({
4865
+ companyProfile: companyProfileSnapshot,
4308
4866
  instructionPolicy: instructionPolicySnapshot,
4309
4867
  preferences: preferenceSnapshot,
4310
4868
  });
4311
4869
  const structuredWorkspacePolicyActive =
4312
4870
  hasActiveWorkspaceInstructionPolicy(instructionPolicySnapshot);
4313
4871
  const workspaceMemory = await resolveWorkspaceMemoryBlock(db, input.workspaceId);
4314
- const baseRunSettings = {
4315
- // IMAGE PRECEDENCE (M3): rig > pack > deployment. settingsWithRigImage runs
4316
- // OUTERMOST so a rig-pinned image overrides both the pack image and the
4317
- // deployment default; a rig with no image (or a rig-less turn) is a
4318
- // pass-through, leaving the pack/deployment chain exactly as today.
4319
- ...settingsWithRigImage(
4320
- settingsWithPackSandboxImage(
4321
- capabilitySettings,
4322
- packRuntime.sandboxImage,
4323
- packRuntime.sandboxProviderImages,
4324
- ),
4325
- rigVersion?.image ?? null,
4872
+ const logicalSandboxSettings = settingsWithRigImage(
4873
+ settingsWithPackSandboxImage(
4874
+ capabilitySettings,
4875
+ packRuntime.sandboxImage,
4876
+ packRuntime.sandboxProviderImages,
4326
4877
  ),
4878
+ rigVersion?.image ?? null,
4879
+ );
4880
+ const providerImageSettings = await settingsWithRigProviderImage(
4881
+ logicalSandboxSettings,
4882
+ rigVersion,
4883
+ turn.sandboxBackend,
4884
+ );
4885
+ const baseRunSettings = {
4886
+ // IMAGE PRECEDENCE: rig > pre-V2 Pack compatibility > deployment.
4887
+ // resolveWorkspacePackRuntime returns no image for V2 Pack rows, so
4888
+ // settingsWithRigImage runs outermost over only the intentionally
4889
+ // retained legacy fallback. A matching verified provider-native ID is
4890
+ // then applied only to fresh creation without changing the logical
4891
+ // lease image.
4892
+ ...providerImageSettings,
4327
4893
  openaiModel: turn.model,
4328
4894
  openaiReasoningEffort: turn.reasoningEffort,
4329
4895
  sandboxBackend: turn.sandboxBackend,
@@ -4353,9 +4919,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4353
4919
  capabilitySettings,
4354
4920
  turnExecutionPolicy.productModelId,
4355
4921
  );
4356
- const supportsImageInput = modelSupportsImageInputForTurn(resolvedModel);
4357
4922
  const providerApi = resolvedModel?.provider.api ?? "responses";
4923
+ const nativeImageProviderBinding =
4924
+ providerApi === "responses"
4925
+ ? openAiHostedImageProviderBindingForTurn(capabilitySettings, resolvedModel)
4926
+ : null;
4927
+ const lazyToolTransport = lazyToolTransportForTurn(resolvedModel);
4358
4928
  const modelInputPolicy = modelAttachmentInputPolicyForTurn(resolvedModel);
4929
+ // Use the proven wire capability, not the catalogue modality alone. Chat
4930
+ // providers may advertise vision, but OpenGeni intentionally has no typed
4931
+ // image transport for that wire yet; exposing view_image there would turn
4932
+ // pixels into a multi-megabyte text/base64 function result.
4933
+ const supportsImageInput = modelInputPolicy.supportsImageInput;
4934
+ modelCanReceiveRetainedSessionImages = supportsImageInput;
4359
4935
  const attachmentProjector = createModelHistoryAttachmentProjector(
4360
4936
  db,
4361
4937
  input.workspaceId,
@@ -4364,8 +4940,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4364
4940
  );
4365
4941
  const modelHistoryProjector = async (items: Array<Record<string, unknown>>) =>
4366
4942
  projectModelInputForCapabilities(await attachmentProjector(items), modelInputPolicy);
4943
+ const generatedImageHistoryProjector = async (
4944
+ items: Array<Record<string, unknown>>,
4945
+ ): Promise<Array<Record<string, unknown>>> => {
4946
+ collectGeneratedImageReceipts(items, generatedImageReceiptsByProviderItemId);
4947
+ collectGeneratedImageArtifactReceipts(items, generatedImageReceiptsByArtifactId);
4948
+ return projectGeneratedImageHistoryForModel(items);
4949
+ };
4367
4950
  const compactionModelHistoryProjector = async (items: Array<Record<string, unknown>>) =>
4368
- await modelHistoryProjector(projectHistoryForProvider(items, providerApi));
4951
+ await modelHistoryProjector(
4952
+ projectHistoryForProvider(
4953
+ await generatedImageHistoryProjector(restoreGenericDispatchHistoryItems(items)),
4954
+ providerApi,
4955
+ ),
4956
+ );
4369
4957
  // Bind the provider/model catalog's context policy to every model-facing
4370
4958
  // path for this turn. In particular, Codex subscription turns must not
4371
4959
  // inherit the deployment's OpenAI/Azure mode or 1.05M context defaults:
@@ -4399,6 +4987,44 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4399
4987
  }
4400
4988
  }
4401
4989
  }
4990
+ // A recovered asynchronous video completion is model-visible only after
4991
+ // its durable File is present at the exact sandbox path carried by the
4992
+ // receipt. Resolve and verify the claimed update before sandbox policy is
4993
+ // chosen so this turn cannot remain lazy and expose an absent path.
4994
+ for (const update of await listSessionSystemUpdatesForTurn(
4995
+ db,
4996
+ input.workspaceId,
4997
+ input.sessionId,
4998
+ turn.id,
4999
+ )) {
5000
+ const payload = update.payload as MediaGenerationResult;
5001
+ if (payload.type !== "media_generation_result" || payload.status !== "ready") continue;
5002
+ const retained = await getGeneratedVideoArtifact(
5003
+ db,
5004
+ input.workspaceId,
5005
+ payload.receipt.artifact.artifactId,
5006
+ );
5007
+ if (
5008
+ !retained ||
5009
+ retained.artifact.deletedAt ||
5010
+ retained.file.status !== "ready" ||
5011
+ retained.file.contentType !== "video/mp4" ||
5012
+ retained.file.sizeBytes !== payload.receipt.artifact.originalBytes ||
5013
+ retained.file.sha256 !== payload.receipt.artifact.sha256 ||
5014
+ retained.artifact.sandboxFilename !== `generated-video-${retained.artifact.id}.mp4`
5015
+ ) {
5016
+ throw new Error("Generated video completion does not match its retained File");
5017
+ }
5018
+ requiredGeneratedVideoFiles.push({
5019
+ operationId: payload.operationId,
5020
+ artifactId: retained.artifact.id,
5021
+ fileId: retained.file.id,
5022
+ objectKey: retained.file.objectKey,
5023
+ sizeBytes: retained.file.sizeBytes,
5024
+ sha256: retained.file.sha256,
5025
+ filename: retained.artifact.sandboxFilename,
5026
+ });
5027
+ }
4402
5028
  // A codex-subscription turn resolves the bearer for THIS turn's effective
4403
5029
  // codex account (effectiveCodexCredentialId; pin > workspace-active) at
4404
5030
  // model-call time — multi-account P1 means a workspace can hold N accounts,
@@ -4517,8 +5143,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4517
5143
  resolvedModel
4518
5144
  ? (s: Settings, m: Array<Record<string, unknown>>) =>
4519
5145
  withCodex(() =>
4520
- summarizeForCompaction(s, m, {
5146
+ summarizeContextForCompaction(s, m, {
4521
5147
  client: resolvedModel.client,
5148
+ provider: resolvedModel.provider,
4522
5149
  api: resolvedModel.provider.api,
4523
5150
  model: turnExecutionPolicy.upstreamModelId,
4524
5151
  maxOutputTokens: SUMMARY_BUFFER_TOKENS,
@@ -4528,7 +5155,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4528
5155
  }),
4529
5156
  )
4530
5157
  : (s: Settings, m: Array<Record<string, unknown>>) =>
4531
- summarizeForCompaction(s, m, {
5158
+ summarizeContextForCompaction(s, m, {
4532
5159
  model: turnExecutionPolicy.upstreamModelId,
4533
5160
  maxOutputTokens: SUMMARY_BUFFER_TOKENS,
4534
5161
  onUsage: recordCompactionUsage,
@@ -4583,6 +5210,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4583
5210
  }
4584
5211
  return requestRemoteCompactionV2(s, m, {
4585
5212
  client: resolvedModel.client,
5213
+ provider: resolvedModel.provider,
4586
5214
  model: turnExecutionPolicy.upstreamModelId,
4587
5215
  systemInstructions: remoteCompactionInstructions,
4588
5216
  onUsage: recordCompactionUsage,
@@ -4965,7 +5593,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4965
5593
  // Resolved run credentials must be written to one exact leased sandbox
4966
5594
  // before agent execution. A warm active pointer can bypass the lazy
4967
5595
  // provisioner entirely, which previously skipped materialization.
4968
- !initialRunCredentialMaterial
5596
+ !initialRunCredentialMaterial &&
5597
+ requiredGeneratedVideoFiles.length === 0
4969
5598
  ? "on-demand"
4970
5599
  : "eager";
4971
5600
  // Computed exactly ONCE per turn and reused for BOTH the box manifest
@@ -5019,17 +5648,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5019
5648
  turn,
5020
5649
  })
5021
5650
  : undefined;
5022
- const toolspaceAuthority = {
5651
+ const codemodeAuthority = {
5023
5652
  sessionId: input.sessionId,
5653
+ turnId: turn.id,
5654
+ attemptId: input.attemptId,
5655
+ executionGeneration: turn.executionGeneration,
5024
5656
  };
5657
+ const sandboxArtifactRuntime = sandboxArtifactRuntimeAdmission(
5658
+ settings,
5659
+ runSettings,
5660
+ activeSandboxBackend ?? groupBoxBackend,
5661
+ );
5025
5662
  const {
5026
- environment: sandboxEnvironment,
5663
+ environment: baseSandboxEnvironment,
5027
5664
  gitToken: sandboxGitToken,
5028
5665
  gitTokens: sandboxGitTokens,
5029
5666
  gitTokenExpiresAt: sandboxGitTokenExpiresAt,
5030
5667
  gitCredentialBindings: sandboxGitCredentialBindings,
5031
- toolspaceToken: sandboxToolspaceToken,
5032
- toolspaceTokenExpiresAt: sandboxToolspaceTokenExpiresAt,
5668
+ codemodeToken: sandboxCodemodeToken,
5669
+ codemodeTokenExpiresAt: sandboxCodemodeTokenExpiresAt,
5033
5670
  } = await waitForTurnOperation(
5034
5671
  sandboxEnvironmentForRun(
5035
5672
  runSettings,
@@ -5039,22 +5676,28 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5039
5676
  sandboxWorkspaceEnvironmentValues,
5040
5677
  {
5041
5678
  skipGitHubToken: activeSandboxBackend === "selfhosted",
5042
- skipToolspace: activeSandboxBackend === "selfhosted",
5679
+ skipCodemode: activeSandboxBackend === "selfhosted",
5043
5680
  deferGitHubToken:
5044
5681
  activeSandboxBackend !== "selfhosted" && establishPolicy === "on-demand",
5045
5682
  scope: connectionScope,
5046
5683
  ...(gitCredentialAuthority ? { authority: gitCredentialAuthority } : {}),
5047
5684
  gitCredentials: connectionCredentials?.gitCredentials,
5048
5685
  authorizeGitHubTokenMint,
5049
- toolspaceAuthority,
5686
+ codemodeAuthority,
5050
5687
  },
5051
5688
  ),
5052
5689
  cancellationSignal,
5053
5690
  undefined,
5054
5691
  );
5055
-
5056
- const sandboxToolspaceTokenFile = sandboxToolspaceToken
5057
- ? toolspaceTokenFileFromEnvironment(sandboxEnvironment, input.sessionId)
5692
+ // Reserved, image-owned paths win over workspace/session variables. The
5693
+ // exact same merged object feeds both box manifest and agent declaration,
5694
+ // preserving the no-environment-delta invariant.
5695
+ const sandboxEnvironment = sandboxArtifactRuntime.available
5696
+ ? { ...baseSandboxEnvironment, ...sandboxArtifactRuntime.environment }
5697
+ : baseSandboxEnvironment;
5698
+
5699
+ const sandboxCodemodeTokenFile = sandboxCodemodeToken
5700
+ ? codemodeTokenFileFromEnvironment(sandboxEnvironment, input.sessionId)
5058
5701
  : undefined;
5059
5702
 
5060
5703
  const initialGitCredentials: MintedRunGitCredentials | undefined =
@@ -5167,22 +5810,22 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5167
5810
  gitCredentialRenewals = controllers;
5168
5811
  };
5169
5812
 
5170
- const attachToolspaceTokenRenewal = async (
5171
- tokenSession: ToolspaceTokenWriterSession,
5172
- initialExpiresAt = sandboxToolspaceTokenExpiresAt,
5813
+ const attachCodemodeTokenRenewal = async (
5814
+ tokenSession: CodemodeTokenWriterSession,
5815
+ initialExpiresAt = sandboxCodemodeTokenExpiresAt,
5173
5816
  initialSandbox?: ResumedTurnSandbox,
5174
5817
  ): Promise<void> => {
5175
- if (!sandboxToolspaceToken || !initialExpiresAt) return;
5176
- const previous = toolspaceTokenRenewal;
5177
- toolspaceTokenRenewal = null;
5818
+ if (!sandboxCodemodeToken || !initialExpiresAt) return;
5819
+ const previous = codemodeTokenRenewal;
5820
+ codemodeTokenRenewal = null;
5178
5821
  await previous?.stop();
5179
- if (toolspaceTokenRenewalClosed) return;
5822
+ if (codemodeTokenRenewalClosed) return;
5180
5823
 
5181
5824
  const mint = async () => {
5182
- const material = await mintSandboxToolspaceToken(
5825
+ const material = await mintSandboxCodemodeToken(
5183
5826
  runSettings,
5184
5827
  connectionScope,
5185
- toolspaceAuthority,
5828
+ codemodeAuthority,
5186
5829
  );
5187
5830
  if (material) {
5188
5831
  }
@@ -5192,18 +5835,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5192
5835
  const runAs = sandboxRunAs(runSettings);
5193
5836
  const targetSandbox = resolvedSandbox ?? initialSandbox;
5194
5837
  if (!targetSandbox) {
5195
- throw new Error("Toolspace token renewal has no exact sandbox lease target");
5838
+ throw new Error("Codemode token renewal has no exact sandbox lease target");
5196
5839
  }
5197
5840
  await runWorkspaceMutationForSandbox(
5198
5841
  targetSandbox,
5199
- "toolspaceTokenRenewal",
5842
+ "codemodeTokenRenewal",
5200
5843
  async () =>
5201
- await refreshToolspaceTokenFile(tokenSession, material.token, {
5844
+ await refreshCodemodeTokenFile(tokenSession, material.token, {
5202
5845
  ...(runAs ? { runAs } : {}),
5203
- ...(sandboxToolspaceTokenFile
5846
+ ...(sandboxCodemodeTokenFile
5204
5847
  ? {
5205
- tokenFile: sandboxToolspaceTokenFile,
5206
- legacyTokenFile: sandboxEnvironment.OPENGENI_TOOLSPACE_TOKEN_FILE!,
5848
+ tokenFile: sandboxCodemodeTokenFile,
5849
+ legacyTokenFile: sandboxEnvironment.OPENGENI_CODEMODE_TOKEN_FILE!,
5207
5850
  }
5208
5851
  : {}),
5209
5852
  ...(toolCancellationFenceRef.current
@@ -5217,32 +5860,32 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5217
5860
  );
5218
5861
  };
5219
5862
  let renewalExpiresAt = initialExpiresAt;
5220
- if (renewalExpiresAt.getTime() <= Date.now() + TOOLSPACE_TOKEN_EXPIRY_LEAD_MS) {
5863
+ if (renewalExpiresAt.getTime() <= Date.now() + CODEMODE_TOKEN_EXPIRY_LEAD_MS) {
5221
5864
  const fresh = await mint();
5222
5865
  if (!fresh) {
5223
- throw new Error("Toolspace token mint became unavailable during sandbox setup");
5866
+ throw new Error("Codemode token mint became unavailable during sandbox setup");
5224
5867
  }
5225
5868
  await write(fresh);
5226
5869
  renewalExpiresAt = fresh.expiresAt;
5227
5870
  }
5228
- const controller = startToolspaceTokenRenewalLoop({
5871
+ const controller = startCodemodeTokenRenewalLoop({
5229
5872
  initialExpiresAt: renewalExpiresAt,
5230
5873
  mint,
5231
5874
  write,
5232
5875
  onSuccess: () => {
5233
5876
  observability.incrementCounter({
5234
- name: "opengeni_toolspace_token_renewals_total",
5235
- help: "Sandbox Toolspace token renewal attempts by outcome.",
5877
+ name: "opengeni_codemode_token_renewals_total",
5878
+ help: "Sandbox Codemode token renewal attempts by outcome.",
5236
5879
  labels: { outcome: "completed" },
5237
5880
  });
5238
5881
  },
5239
5882
  onFailure: ({ retryDelayMs, errorClass }) => {
5240
5883
  observability.incrementCounter({
5241
- name: "opengeni_toolspace_token_renewals_total",
5242
- help: "Sandbox Toolspace token renewal attempts by outcome.",
5884
+ name: "opengeni_codemode_token_renewals_total",
5885
+ help: "Sandbox Codemode token renewal attempts by outcome.",
5243
5886
  labels: { outcome: "error" },
5244
5887
  });
5245
- observability.warn("Sandbox Toolspace token renewal failed; retry scheduled", {
5888
+ observability.warn("Sandbox Codemode token renewal failed; retry scheduled", {
5246
5889
  sessionId: input.sessionId,
5247
5890
  turnId,
5248
5891
  errorClass,
@@ -5250,11 +5893,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5250
5893
  });
5251
5894
  },
5252
5895
  });
5253
- if (toolspaceTokenRenewalClosed) {
5896
+ if (codemodeTokenRenewalClosed) {
5254
5897
  await controller.stop();
5255
5898
  return;
5256
5899
  }
5257
- toolspaceTokenRenewal = controller;
5900
+ codemodeTokenRenewal = controller;
5258
5901
  };
5259
5902
 
5260
5903
  const attachRunCredentialRenewal = async (
@@ -5473,6 +6116,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5473
6116
  onOp: machineOpObserver.observer,
5474
6117
  onSandboxOperation: sandboxOperationObserver,
5475
6118
  onHomeSandboxRebound,
6119
+ ...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
5476
6120
  },
5477
6121
  {
5478
6122
  workspaceId: input.workspaceId,
@@ -5517,6 +6161,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5517
6161
  {
5518
6162
  db,
5519
6163
  settings: runSettings,
6164
+ logicalFallbackSettings: logicalSandboxSettings,
5520
6165
  cancellationSignal: sandboxResumeSignal,
5521
6166
  sandboxMetrics: runtimeMetricsHooksForObservability(observability),
5522
6167
  onSandboxLost: publishSandboxLost,
@@ -5555,7 +6200,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5555
6200
  managedOwnership!.holderId,
5556
6201
  ),
5557
6202
  sandboxResumeSignal,
5558
- async (lateSandbox) => await lateSandbox.release(),
6203
+ releaseLateSandbox,
5559
6204
  );
5560
6205
  setupBoxSession = resolvedSandbox.established.session;
5561
6206
  // Durable box-lifecycle events (sandbox-file-persistence observability):
@@ -5583,6 +6228,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5583
6228
  onSandboxOperation: sandboxOperationObserver,
5584
6229
  onHomeSandboxLost: publishSandboxLost,
5585
6230
  onHomeSandboxRebound,
6231
+ ...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
5586
6232
  },
5587
6233
  // Thread the SAME declared environment the group box was created with
5588
6234
  // (resumeBoxForTurn, above) so a selfhosted swap target's manifest
@@ -5615,7 +6261,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5615
6261
  }
5616
6262
  }
5617
6263
 
5618
- const fileResourceDownloads = await waitForTurnOperation(
6264
+ const ordinaryFileResourceDownloads = await waitForTurnOperation(
5619
6265
  sandboxFileDownloadsForRun(
5620
6266
  runSettings,
5621
6267
  db,
@@ -5627,6 +6273,35 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5627
6273
  cancellationSignal,
5628
6274
  undefined,
5629
6275
  );
6276
+ if (requiredGeneratedVideoFiles.length > 0 && !objectStorage) {
6277
+ throw new Error("Generated video materialization requires object storage");
6278
+ }
6279
+ const generatedVideoDownloads: SandboxFileDownload[] = [];
6280
+ if (objectStorage && requiredGeneratedVideoFiles.length > 0) {
6281
+ const downloadStorage = objectStorageForSandboxDownloads(modelRunSettings, objectStorage);
6282
+ for (const file of requiredGeneratedVideoFiles) {
6283
+ const signed = await downloadStorage.createGetUrl({
6284
+ key: file.objectKey,
6285
+ });
6286
+ generatedVideoDownloads.push({
6287
+ fileId: file.fileId,
6288
+ mountPath: "generated-videos",
6289
+ filename: file.filename,
6290
+ url: signed.url,
6291
+ expiresAt: signed.expiresAt,
6292
+ sizeBytes: file.sizeBytes,
6293
+ sha256: file.sha256,
6294
+ });
6295
+ }
6296
+ }
6297
+ const fileResourceDownloads = [
6298
+ ...new Map(
6299
+ [...ordinaryFileResourceDownloads, ...generatedVideoDownloads].map((download) => [
6300
+ download.fileId,
6301
+ download,
6302
+ ]),
6303
+ ).values(),
6304
+ ];
5630
6305
  throwIfWorkerShuttingDown();
5631
6306
  throwIfTurnCancelled();
5632
6307
  const mcpCredentialRootSessionId =
@@ -5667,7 +6342,30 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5667
6342
  }
5668
6343
  return result;
5669
6344
  };
5670
- const credentialSubjectId = credentialSubjectIdForTurnInitiator(turn);
6345
+ const publishToolAuthNeeded = async (payload: ToolAuthNeededPayload): Promise<void> => {
6346
+ if (!shouldPublishToolAuthNeededForTurn(payload, trigger, turn)) {
6347
+ return;
6348
+ }
6349
+ await publish!([{ type: "tool.auth_needed", payload }], true);
6350
+ };
6351
+ const selectedApiIntegrationIds = new Set(turnTools.map((tool) => tool.id));
6352
+ const localMcpServers = buildApiIntegrationServersForTurn({
6353
+ settings: runSettings,
6354
+ integrations: installedApiIntegrations.filter((integration) =>
6355
+ selectedApiIntegrationIds.has(integration.serverId),
6356
+ ),
6357
+ authority: {
6358
+ accountId: input.accountId,
6359
+ workspaceId: input.workspaceId,
6360
+ sessionId: input.sessionId,
6361
+ rootSessionId: session.rootSessionId,
6362
+ turnId: turn.id,
6363
+ attemptId: input.attemptId,
6364
+ ...(credentialSubjectId ? { initiatingSubjectId: credentialSubjectId } : {}),
6365
+ },
6366
+ resolveCredential,
6367
+ onAuthNeeded: publishToolAuthNeeded,
6368
+ });
5671
6369
  const codexAppsAuth = codexAppsCredentialId
5672
6370
  ? (() => {
5673
6371
  const resolver = buildCodexTokenResolver(
@@ -5688,7 +6386,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5688
6386
 
5689
6387
  return await withCodexAppsRequestAuthorization(
5690
6388
  db,
5691
- { workspaceId: input.workspaceId, credentialId: codexAppsCredentialId },
6389
+ {
6390
+ workspaceId: input.workspaceId,
6391
+ credentialId: codexAppsCredentialId,
6392
+ },
5692
6393
  async () => await use(snapshot),
5693
6394
  );
5694
6395
  },
@@ -5711,11 +6412,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5711
6412
  ...(credentialSubjectId ? { credentialSubjectId } : {}),
5712
6413
  ...(codexAppsAuth ? { codexAppsAuth } : {}),
5713
6414
  resolveCredential,
5714
- onAuthNeeded: async (payload) => {
5715
- if (!shouldPublishToolAuthNeededForTurn(payload, trigger, turn)) {
5716
- return;
5717
- }
5718
- await publish!([{ type: "tool.auth_needed", payload }], true);
6415
+ onAuthNeeded: publishToolAuthNeeded,
6416
+ localMcpServers,
6417
+ onAttemptToolCatalog: async (catalog) => {
6418
+ await persistAttemptToolCatalog(db, catalog);
5719
6419
  },
5720
6420
  // Manager-style sessions carry a creation-validated permission set
5721
6421
  // for their first-party MCP token; null keeps the fixed default.
@@ -5723,10 +6423,44 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5723
6423
  ? { firstPartyPermissions: session.firstPartyMcpPermissions }
5724
6424
  : {}),
5725
6425
  firstPartyTools: session.firstPartyMcpTools ?? [...DEFAULT_FIRST_PARTY_MCP_TOOLS],
6426
+ attemptToolDefinitions: createFirstPartyInteractionAttemptToolDefinitions({
6427
+ settings: runSettings,
6428
+ scope: {
6429
+ accountId: input.accountId,
6430
+ workspaceId: input.workspaceId,
6431
+ sessionId: input.sessionId,
6432
+ turnId: turn.id,
6433
+ attemptId: input.attemptId,
6434
+ executionGeneration,
6435
+ },
6436
+ ...(session.firstPartyMcpPermissions?.length
6437
+ ? { permissions: session.firstPartyMcpPermissions }
6438
+ : {}),
6439
+ selectedTools: session.firstPartyMcpTools ?? [...DEFAULT_FIRST_PARTY_MCP_TOOLS],
6440
+ subjectId: "worker:first-party-mcp",
6441
+ subjectLabel: "OpenGeni worker",
6442
+ }),
5726
6443
  }),
5727
6444
  cancellationSignal,
5728
6445
  async (latePreparedTools) => await latePreparedTools.close().catch(() => undefined),
5729
6446
  );
6447
+ if (turnId && preparedTools.attemptToolEnvironment) {
6448
+ codemodeDispatcher = new CodemodeAttemptDispatcher(
6449
+ db,
6450
+ bus,
6451
+ preparedTools.attemptToolEnvironment,
6452
+ {
6453
+ accountId: input.accountId,
6454
+ workspaceId: input.workspaceId,
6455
+ sessionId: input.sessionId,
6456
+ turnId,
6457
+ attemptId: input.attemptId,
6458
+ executionGeneration,
6459
+ },
6460
+ cancellationSignal,
6461
+ );
6462
+ codemodeDispatcher.start();
6463
+ }
5730
6464
  // Genesis turn = the first user turn (no assistant history reconciled
5731
6465
  // yet). Durable Postgres state (countSessionHistoryItems includes
5732
6466
  // superseded rows after compaction), NOT a workflow counter (turnsThisRun
@@ -5765,6 +6499,204 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5765
6499
  }
5766
6500
  : {};
5767
6501
  const hostedWebSearch = hostedWebSearchForTurn(resolvedModel, runSettings.webSearchEnabled);
6502
+ const resolveImageReferences = async (
6503
+ references: Parameters<typeof resolveImageGenerationReferences>[0]["references"],
6504
+ ) =>
6505
+ await resolveImageGenerationReferences({
6506
+ db,
6507
+ objectStorage: objectStorage!,
6508
+ workspaceId: input.workspaceId,
6509
+ references,
6510
+ readSandboxFile: async (path, maxBytes) => {
6511
+ const imageReferenceSession = (setupBoxSession ??
6512
+ sdkOwnedSandboxSession) as ChannelASession | null;
6513
+ if (!imageReferenceSession) {
6514
+ throw new Error("Sandbox image reference is unavailable");
6515
+ }
6516
+ const relativePath = path.slice("/workspace/".length);
6517
+ const referenceRunAs = sandboxRunAs(modelRunSettings);
6518
+ const channel = new SandboxChannelAService({
6519
+ session: imageReferenceSession,
6520
+ workspaceRoot: "/workspace",
6521
+ leaseEpoch: resolvedSandbox?.leaseEpoch ?? 0,
6522
+ ...(referenceRunAs ? { runAs: referenceRunAs } : {}),
6523
+ });
6524
+ const read = await channel.fsRead({
6525
+ path: relativePath,
6526
+ encoding: "base64",
6527
+ maxBytes,
6528
+ });
6529
+ if (read.truncated) throw new Error("Sandbox image reference exceeds the byte limit");
6530
+ return Uint8Array.from(Buffer.from(read.content, "base64"));
6531
+ },
6532
+ });
6533
+ const imageGenerationOption: Pick<BuildAgentOptions, "imageGeneration"> = (() => {
6534
+ // Never expose a paid image operation unless its permanent artifact can
6535
+ // be committed. Failing after provider execution would leave an
6536
+ // unrecoverable outcome-unknown operation with no user-visible image.
6537
+ if (!objectStorage) return {};
6538
+ if (nativeImageProviderBinding) {
6539
+ nativeImageGenerationRetention = {
6540
+ ...nativeImageProviderBinding,
6541
+ sessionId: input.sessionId,
6542
+ turnId: turn.id,
6543
+ attemptId: input.attemptId,
6544
+ };
6545
+ return { imageGeneration: { kind: "native_hosted" } };
6546
+ }
6547
+
6548
+ if (resolvedModel?.provider.kind === "codex-subscription") {
6549
+ const codexImageContext = codexContext;
6550
+ const codexImageCredentialId = effectiveCodexCredentialId;
6551
+ if (!codexImageContext || !codexImageCredentialId) {
6552
+ throw new CodexReloginRequired(
6553
+ "Codex image generation requires a connected subscription account",
6554
+ );
6555
+ }
6556
+ return {
6557
+ imageGeneration: {
6558
+ kind: "provider_adapter",
6559
+ execute: async ({ prompt, references }, { toolCallId }) => {
6560
+ const resolvedReferences = await resolveImageReferences(references);
6561
+ const receipt = await executeCodexImageGeneration({
6562
+ db,
6563
+ objectStorage,
6564
+ accountId: input.accountId,
6565
+ workspaceId: input.workspaceId,
6566
+ sessionId: input.sessionId,
6567
+ turnId: turn.id,
6568
+ attemptId: input.attemptId,
6569
+ toolCallId,
6570
+ prompt,
6571
+ references: resolvedReferences,
6572
+ credentialId: codexImageCredentialId,
6573
+ codexContext: codexImageContext,
6574
+ ...(runtimeCancellationSignal ? { abortSignal: runtimeCancellationSignal } : {}),
6575
+ });
6576
+ generatedImageReceiptsByArtifactId.set(receipt.artifact.artifactId, receipt);
6577
+ await materializeGeneratedImage(receipt);
6578
+ return receipt;
6579
+ },
6580
+ },
6581
+ };
6582
+ }
6583
+
6584
+ const gatewayResolution = resolveModelProvider(
6585
+ capabilitySettings,
6586
+ WORKSPACE_GATEWAY_PROVIDER_ID,
6587
+ );
6588
+ const gateway = gatewayResolution?.provider;
6589
+ if (gateway?.kind !== "vercel-gateway-workspace" || !gateway.apiKey) return {};
6590
+ const gatewayApiKey = gateway.apiKey;
6591
+ return {
6592
+ imageGeneration: {
6593
+ kind: "provider_adapter",
6594
+ execute: async ({ prompt, references }, { toolCallId }) => {
6595
+ const resolvedReferences = await resolveImageReferences(references);
6596
+ const receipt = await executeGatewayImageGeneration({
6597
+ db,
6598
+ objectStorage,
6599
+ accountId: input.accountId,
6600
+ workspaceId: input.workspaceId,
6601
+ sessionId: input.sessionId,
6602
+ turnId: turn.id,
6603
+ attemptId: input.attemptId,
6604
+ apiKey: gatewayApiKey,
6605
+ modelId: capabilitySettings.imageGenerationModel,
6606
+ prompt,
6607
+ references: resolvedReferences,
6608
+ toolCallId,
6609
+ ...(runtimeCancellationSignal ? { abortSignal: runtimeCancellationSignal } : {}),
6610
+ });
6611
+ generatedImageReceiptsByArtifactId.set(receipt.artifact.artifactId, receipt);
6612
+ await materializeGeneratedImage(receipt);
6613
+ return receipt;
6614
+ },
6615
+ },
6616
+ };
6617
+ })();
6618
+ const videoGenerationPolicy = await getWorkspaceVideoGenerationPolicy(db, input.workspaceId);
6619
+ const videoGenerationEnabled =
6620
+ videoGenerationPolicy.defaultModelId !== null &&
6621
+ videoGenerationPolicy.enabledModelIds.length > 0;
6622
+ let videoGenerationCredential: VideoGenerationCredentialLease | null = null;
6623
+ if (objectStorage && videoGenerationEnabled) {
6624
+ if (videoGenerationPolicy.fundingSource === "opengeni_credits") {
6625
+ videoGenerationCredential = managedVideoGenerationCredentialLease(modelRunSettings);
6626
+ } else {
6627
+ const workspaceCredential = await loadWorkspaceVercelAiGatewayCredentialLease(
6628
+ db,
6629
+ modelRunSettings,
6630
+ input.workspaceId,
6631
+ );
6632
+ if (workspaceCredential) {
6633
+ videoGenerationCredential = {
6634
+ fundingSource: "workspace_gateway",
6635
+ ...workspaceCredential,
6636
+ };
6637
+ }
6638
+ }
6639
+ }
6640
+ const videoGenerationOption: Pick<BuildAgentOptions, "videoGeneration"> = (() => {
6641
+ if (
6642
+ !objectStorage ||
6643
+ modelRunSettings.sandboxBackend === "none" ||
6644
+ !videoGenerationCredential ||
6645
+ !videoGenerationEnabled
6646
+ ) {
6647
+ return {};
6648
+ }
6649
+ // Parse the frozen capability snapshot before advertising either tool.
6650
+ // Invalid or unsupported workspace policy therefore fails closed before
6651
+ // it can perturb the model's tool list.
6652
+ const capabilities = videoGenerationCapabilitiesForPolicy({
6653
+ policy: videoGenerationPolicy,
6654
+ credentialVersion: videoGenerationCredential.version,
6655
+ });
6656
+ return {
6657
+ videoGeneration: {
6658
+ capabilities: async () => capabilities,
6659
+ execute: async (toolInput, { toolCallId }) => {
6660
+ const sessionForReference =
6661
+ resolvedSandbox?.established.session ?? sdkOwnedSandboxSession;
6662
+ const fence = toolCancellationFenceRef.current;
6663
+ const runAs = sandboxRunAs(modelRunSettings);
6664
+ const accepted = await admitVideoGenerationRequest({
6665
+ db,
6666
+ storage: objectStorage,
6667
+ settings: modelRunSettings,
6668
+ accountId: input.accountId,
6669
+ workspaceId: input.workspaceId,
6670
+ sessionId: input.sessionId,
6671
+ turnId: turn.id,
6672
+ attemptId: input.attemptId,
6673
+ toolCallId,
6674
+ toolInput,
6675
+ policy: videoGenerationPolicy,
6676
+ credential: videoGenerationCredential,
6677
+ ...(sessionForReference && fence
6678
+ ? {
6679
+ runCommand: async (command) =>
6680
+ await fence.runSandboxCommandStructured(
6681
+ sessionForReference as import("@opengeni/runtime").TurnSandboxCommandSession,
6682
+ {
6683
+ ...command,
6684
+ ...(runAs ? { runAs } : {}),
6685
+ },
6686
+ ),
6687
+ }
6688
+ : {}),
6689
+ ...(runtimeCancellationSignal ? { signal: runtimeCancellationSignal } : {}),
6690
+ });
6691
+ videoGenerationAcceptancesByCallId.set(toolCallId, {
6692
+ operationId: accepted.operationId,
6693
+ requestDigest: accepted.requestDigest,
6694
+ });
6695
+ return accepted.receipt;
6696
+ },
6697
+ },
6698
+ };
6699
+ })();
5768
6700
  const serviceTier = serviceTierForLatencyMode(
5769
6701
  turnExecutionPolicy.providerId,
5770
6702
  turnExecutionPolicy.latencyMode,
@@ -5806,6 +6738,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5806
6738
  titleIsSet: Boolean(session.title?.trim()),
5807
6739
  },
5808
6740
  sandboxEnvironment,
6741
+ ...(preparedTools.attemptToolCatalog
6742
+ ? { attemptToolCatalog: preparedTools.attemptToolCatalog }
6743
+ : {}),
6744
+ ...(sandboxArtifactRuntime.available ? { artifactRuntimeAvailable: true } : {}),
5809
6745
  ...(cancellationSignal ? { turnCancellationSignal: cancellationSignal } : {}),
5810
6746
  onToolCancellationFence: (fence) => {
5811
6747
  toolCancellationFenceRef.current = fence;
@@ -5825,13 +6761,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5825
6761
  ...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
5826
6762
  ? { gitTokenSeed: sandboxGitToken }
5827
6763
  : {}),
5828
- // Toolspace delivery is managed-sandbox-only. Connected Machines own any
6764
+ // Codemode delivery is managed-sandbox-only. Connected Machines own any
5829
6765
  // manually configured API credentials and must never be contacted during
5830
6766
  // turn admission merely to seed OpenGeni tooling.
5831
- ...(sandboxToolspaceToken
6767
+ ...(sandboxCodemodeToken
5832
6768
  ? {
5833
- toolspaceTokenSeed: sandboxToolspaceToken,
5834
- toolspaceTokenSessionId: input.sessionId,
6769
+ codemodeTokenSeed: sandboxCodemodeToken,
6770
+ codemodeTokenSessionId: input.sessionId,
5835
6771
  }
5836
6772
  : {}),
5837
6773
  ...(activeSandboxBackend ? { activeSandboxBackend } : {}),
@@ -5858,6 +6794,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5858
6794
  // session's MCP allow-list; the effective context window drives the
5859
6795
  // compaction threshold.
5860
6796
  hostedWebSearch,
6797
+ ...imageGenerationOption,
6798
+ ...videoGenerationOption,
6799
+ lazyToolTransport,
5861
6800
  supportsImageInput,
5862
6801
  inputFileMediaTypes: modelInputPolicy.inputFileMediaTypes,
5863
6802
  ...(resolvedModel
@@ -5867,15 +6806,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5867
6806
  runSettings.openaiReasoningEncryptedContent,
5868
6807
  contextWindowTokens:
5869
6808
  resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
5870
- // The ChatGPT/Codex backend rejects the SDK's HOSTED sandbox tools —
5871
- // the `apply_patch` tool type ("Unsupported tool type: apply_patch")
5872
- // and structured tool output — which the OpenAIResponsesModel the SDK
5873
- // binds would otherwise select. Tell buildAgent to emit the function
5874
- // `apply_patch` + text `view_image` variants the backend accepts. Only
5875
- // the codex-subscription provider needs this; every other backend
5876
- // (built-in OpenAI/Azure = real hosted support; registry "chat"
5877
- // providers = the SDK's own ChatCompletions detection) keeps the SDK
5878
- // default.
6809
+ // The ChatGPT/Codex backend rejects the SDK's HOSTED apply_patch
6810
+ // tool. Gateway Responses routes likewise expose ordinary function
6811
+ // tools, not OpenAI-hosted sandbox tools. Tell buildAgent to use
6812
+ // function apply_patch and wrap successful view_image results as
6813
+ // typed input_image content. Chat wires have no proven typed image
6814
+ // result transport and therefore receive no view_image tool.
5879
6815
  structuredToolTransport: structuredToolTransportForTurn(resolvedModel),
5880
6816
  // EXPLICIT computer-use tool transport, derived from the resolved provider's
5881
6817
  // authoritative wire identity (codex → function-image, chat → disabled,
@@ -5905,6 +6841,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5905
6841
  didComputerUse = true;
5906
6842
  await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
5907
6843
  },
6844
+ onRetainableSessionImageOutput: retainSessionImageAtToolBoundary,
5908
6845
  ...(packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {}),
5909
6846
  ...(session.skills.length > 0 ? { sessionSkills: session.skills } : {}),
5910
6847
  ...(skillLibraryRuntime.skillLibrarySkills.length > 0
@@ -5946,6 +6883,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5946
6883
  rigName,
5947
6884
  script: rigVersion.setupScript,
5948
6885
  timeoutMs: runSettings.rigSetupTimeoutMs,
6886
+ contentHash: rigProviderImageContentHash({
6887
+ backend: turn.sandboxBackend,
6888
+ sourceImage: rigProviderImageSourceImage(
6889
+ logicalSandboxSettings,
6890
+ turn.sandboxBackend,
6891
+ ),
6892
+ definition: rigVersion,
6893
+ }),
5949
6894
  },
5950
6895
  }
5951
6896
  : {}),
@@ -5984,13 +6929,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5984
6929
  authorizeGitHubTokenMint,
5985
6930
  });
5986
6931
  const lazyGitTokens = lazyGitCredentials?.gitTokens;
5987
- const lazyToolspaceToken = sandboxToolspaceToken
5988
- ? await mintSandboxToolspaceToken(runSettings, connectionScope, toolspaceAuthority)
6932
+ const lazyCodemodeToken = sandboxCodemodeToken
6933
+ ? await mintSandboxCodemodeToken(runSettings, connectionScope, codemodeAuthority)
5989
6934
  : undefined;
5990
6935
  const provisioned = await resumeBoxForTurn(
5991
6936
  {
5992
6937
  db,
5993
6938
  settings: runSettings,
6939
+ logicalFallbackSettings: logicalSandboxSettings,
5994
6940
  cancellationSignal: sandboxResumeSignal,
5995
6941
  sandboxMetrics: runtimeMetricsHooksForObservability(observability),
5996
6942
  onSandboxLost: publishSandboxLost,
@@ -6043,9 +6989,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6043
6989
  gitCredentialBindingsOverride: lazyGitCredentials.bindings,
6044
6990
  }
6045
6991
  : {}),
6046
- ...(lazyToolspaceToken
6992
+ ...(lazyCodemodeToken
6047
6993
  ? {
6048
- toolspaceTokenSeedOverride: lazyToolspaceToken.token,
6994
+ codemodeTokenSeedOverride: lazyCodemodeToken.token,
6049
6995
  }
6050
6996
  : {}),
6051
6997
  ...(toolCancellationFenceRef.current
@@ -6058,9 +7004,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6058
7004
  },
6059
7005
  ),
6060
7006
  );
6061
- await attachToolspaceTokenRenewal(
6062
- provisioned.established.session as ToolspaceTokenWriterSession,
6063
- lazyToolspaceToken?.expiresAt,
7007
+ await attachCodemodeTokenRenewal(
7008
+ provisioned.established.session as CodemodeTokenWriterSession,
7009
+ lazyCodemodeToken?.expiresAt,
6064
7010
  provisioned,
6065
7011
  );
6066
7012
  await attachGitCredentialRenewal(
@@ -6110,6 +7056,17 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6110
7056
  startLeaseHeartbeat(provisioned, activeSandboxBackend ?? groupBoxBackend);
6111
7057
  setupBoxSession = provisioned.established.session;
6112
7058
  resolvedSandbox = provisioned;
7059
+ // `get()` does not release the first routed sandbox operation
7060
+ // until this hook returns. Materialize durable generated images
7061
+ // here so that operation can use their advertised paths without
7062
+ // racing a best-effort background copy.
7063
+ for (const receipt of generatedImageReceiptsByArtifactId.values()) {
7064
+ await materializeGeneratedImageInSandbox(
7065
+ receipt,
7066
+ provisioned,
7067
+ provisioned.established.session,
7068
+ );
7069
+ }
6113
7070
  },
6114
7071
  onFailed: async (error) => {
6115
7072
  await publish?.(
@@ -6126,7 +7083,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6126
7083
  );
6127
7084
  },
6128
7085
  disposeResult: async (provisioned) => {
6129
- await provisioned.release().catch(() => undefined);
7086
+ await releaseLateSandbox(provisioned);
6130
7087
  },
6131
7088
  },
6132
7089
  );
@@ -6140,6 +7097,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6140
7097
  onSandboxOperation: sandboxOperationObserver,
6141
7098
  onHomeSandboxLost: publishSandboxLost,
6142
7099
  onHomeSandboxRebound,
7100
+ ...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
6143
7101
  },
6144
7102
  {
6145
7103
  workspaceId: input.workspaceId,
@@ -6442,7 +7400,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6442
7400
  "fileMaterialization",
6443
7401
  async () =>
6444
7402
  await materializeSandboxFileDownloads(
6445
- setupBoxSession as any,
7403
+ setupBoxSession as CodemodeTokenWriterSession,
6446
7404
  downloadsToMaterialize,
6447
7405
  {
6448
7406
  onRuntimeEvent: async (event) => {
@@ -6477,6 +7435,37 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6477
7435
  }
6478
7436
  fileDownloadsMaterializedForRun = true;
6479
7437
  }
7438
+ const requiredVideoFileIds = new Set(requiredGeneratedVideoFiles.map((file) => file.fileId));
7439
+ const failedRequiredVideo = fileMaterializationFailures.find((failure) =>
7440
+ requiredVideoFileIds.has(failure.fileId),
7441
+ );
7442
+ if (failedRequiredVideo) {
7443
+ const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
7444
+ sessionId: input.sessionId,
7445
+ turnId: turn.id,
7446
+ triggerEventId: triggerEventId!,
7447
+ attemptId: input.attemptId,
7448
+ reason: "generated_video_materialization_failed",
7449
+ detail: {
7450
+ retryable: true,
7451
+ operationIds: requiredGeneratedVideoFiles.map((file) => file.operationId),
7452
+ },
7453
+ });
7454
+ if (recovery.action === "stale") {
7455
+ acknowledgeLostAttemptOwnership();
7456
+ activityStatus = "cancelled";
7457
+ turnMetricOutcome = "cancelled";
7458
+ return claimedResult({ status: "cancelled" });
7459
+ }
7460
+ if (recovery.action !== "recovering") {
7461
+ throw new Error("Generated video materialization could not enter recovery");
7462
+ }
7463
+ acknowledgeRecoveryQuiescence();
7464
+ await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
7465
+ activityStatus = "recovering";
7466
+ turnMetricOutcome = "recovering";
7467
+ return claimedResult({ status: "recovering", continueDelayMs: 1_000 });
7468
+ }
6480
7469
  const unavailableSandboxFilesNote = sandboxFileDownloadFailureNote(
6481
7470
  fileMaterializationFailures,
6482
7471
  );
@@ -6495,10 +7484,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6495
7484
  ...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
6496
7485
  ...(runCredentialsNote ? { runCredentialsNote } : {}),
6497
7486
  providerApi,
7487
+ projectCanonicalHistory: generatedImageHistoryProjector,
6498
7488
  materializeModelHistory: materializeScreenshotHistory,
6499
7489
  materializeSerializedRunState: materializeScreenshotRunState,
6500
7490
  projectModelHistory: modelHistoryProjector,
6501
7491
  });
7492
+ for (const receipt of generatedImageReceiptsByArtifactId.values()) {
7493
+ await materializeGeneratedImage(receipt);
7494
+ }
6502
7495
  runInput = prepared.input;
6503
7496
  providerArtifactCandidates = prepared.providerArtifactCandidates;
6504
7497
  // Slice index = the length of the model-facing (active) history this turn
@@ -6522,6 +7515,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6522
7515
  // and bricks the session (issue-61). The repaired seed is already
6523
7516
  // orphan-free, so it is a stable prefix of the re-sanitized history and the
6524
7517
  // slice begins exactly at the first genuinely-new item.
7518
+ // prepareInput already sanitized the exact durable prefix represented
7519
+ // by state.history. Carry its count forward instead of loading and
7520
+ // retaining the full active transcript a second time beside runInput.
6525
7521
  persistedHistoryCount = prepared.persistedHistoryCount;
6526
7522
  nextHistoryPosition = await nextSessionHistoryPosition(
6527
7523
  db,
@@ -6691,14 +7687,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6691
7687
  },
6692
7688
  }
6693
7689
  : {}),
6694
- ...(sandboxToolspaceToken && sandboxToolspaceTokenExpiresAt && !lazyOwnedSandbox
7690
+ ...(sandboxCodemodeToken && sandboxCodemodeTokenExpiresAt && !lazyOwnedSandbox
6695
7691
  ? {
6696
- onToolspaceTokenSessionReady: async (
6697
- tokenSession: ToolspaceTokenWriterSession,
6698
- ) => {
7692
+ onCodemodeTokenSessionReady: async (tokenSession: CodemodeTokenWriterSession) => {
6699
7693
  const renewalSession =
6700
- (setupBoxSession as ToolspaceTokenWriterSession | null) ?? tokenSession;
6701
- await attachToolspaceTokenRenewal(renewalSession);
7694
+ (setupBoxSession as CodemodeTokenWriterSession | null) ?? tokenSession;
7695
+ await attachCodemodeTokenRenewal(renewalSession);
6702
7696
  },
6703
7697
  }
6704
7698
  : {}),
@@ -6719,6 +7713,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6719
7713
  : {}),
6720
7714
  }
6721
7715
  : {}),
7716
+ ...(modelRunSettings.sandboxBackend !== "none"
7717
+ ? {
7718
+ onSandboxSessionReady: async (sandboxSession: CodemodeTokenWriterSession) => {
7719
+ sdkOwnedSandboxSession = sandboxSession;
7720
+ for (const receipt of generatedImageReceiptsByArtifactId.values()) {
7721
+ await materializeGeneratedImageInOwnedSdkSession(receipt, sandboxSession);
7722
+ }
7723
+ },
7724
+ }
7725
+ : {}),
6722
7726
  contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
6723
7727
  contextCompactionRequested: () =>
6724
7728
  isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
@@ -6760,6 +7764,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6760
7764
  let stableToolCallIdsToClear: string[] | null = null;
6761
7765
  let completedCurrentToolBatch = false;
6762
7766
  let retainedScreenshotMetadata: RetainedArtifactMetadata | null = null;
7767
+ const generatedImage = generatedImageFromSdkEvent(next.value);
7768
+ if (isCompletedGeneratedImageSdkEvent(next.value) && !generatedImage) {
7769
+ throw new Error(
7770
+ "Completed native image item could not cross the retained-artifact boundary",
7771
+ );
7772
+ }
7773
+ const generatedImageReceipt = generatedImage
7774
+ ? await retainNativeGeneratedImage(generatedImage)
7775
+ : null;
6763
7776
  const responseResult = await processModelResponseTerminalEvent({
6764
7777
  event: next.value,
6765
7778
  state: modelResponseState,
@@ -6794,10 +7807,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6794
7807
  ...(resolvedModel?.provider.id ? { providerId: resolvedModel.provider.id } : {}),
6795
7808
  });
6796
7809
  if (responseResult.status === "processed") {
7810
+ const rawStreamHistory = (stream.state as { history?: unknown[] }).history;
7811
+ if (Array.isArray(rawStreamHistory)) {
7812
+ // The completed image item is normally retained from its own
7813
+ // run-item event above. Scan once at the terminal response as
7814
+ // a compatibility backstop before reconciliation can persist
7815
+ // provider bytes. Never rescan history for every token delta.
7816
+ await retainNativeGeneratedImagesFromHistory(
7817
+ rawStreamHistory as Array<Record<string, unknown>>,
7818
+ );
7819
+ }
6797
7820
  streamSawPerResponseUsage ||= responseResult.usageReported;
6798
7821
  currentToolBatchCallIds = new Set<string>();
6799
7822
  currentToolBatchCompletedCallIds = new Set<string>();
6800
7823
  await reconcileConversationTruth();
7824
+ modelCheckpointMemoryCollector.schedule(observability);
6801
7825
  try {
6802
7826
  await ensureRunAllowed(
6803
7827
  settings,
@@ -6813,7 +7837,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6813
7837
  // conversation context preserved for the post-top-up resume.
6814
7838
  let serializedRunState: string | null = null;
6815
7839
  try {
6816
- serializedRunState = String(stream.state.toString());
7840
+ serializedRunState = compactMediaRunState(String(stream.state.toString()));
6817
7841
  } catch {
6818
7842
  serializedRunState = null;
6819
7843
  }
@@ -6823,7 +7847,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6823
7847
  );
6824
7848
  }
6825
7849
  }
6826
- const pendingToolCall = pendingToolCallFromSdkEvent(next.value);
7850
+ const durableSdkEvent = generatedImageReceipt
7851
+ ? compactGeneratedImageSdkEvent(next.value, generatedImageReceipt)
7852
+ : next.value;
7853
+ const pendingToolCall = pendingToolCallFromSdkEvent(durableSdkEvent);
6827
7854
  if (pendingToolCall) {
6828
7855
  const registered = await registerPendingSessionToolCall(db, {
6829
7856
  accountId: input.accountId,
@@ -6843,18 +7870,57 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6843
7870
  );
6844
7871
  }
6845
7872
  currentToolBatchCallIds.add(pendingToolCall.callId);
6846
- if (pendingToolCall.callName === "computer_screenshot") {
6847
- computerScreenshotCallIds.add(pendingToolCall.callId);
7873
+ if (toolCallProducesRetainableSessionImage(pendingToolCall.callName)) {
7874
+ retainedSessionImageCallIds.add(pendingToolCall.callId);
6848
7875
  }
6849
7876
  }
6850
- const completedToolCall = completedToolCallFromSdkEvent(next.value);
7877
+ const completedToolCall = completedToolCallFromSdkEvent(durableSdkEvent);
6851
7878
  if (completedToolCall) {
6852
- const typedScreenshot = typedScreenshotFromSdkEvent(next.value);
7879
+ retainedScreenshotMetadata =
7880
+ retainedScreenshotReceiptsByCallId.get(completedToolCall.callId) ?? null;
7881
+ const typedScreenshot = retainedScreenshotMetadata
7882
+ ? null
7883
+ : typedScreenshotFromSdkEvent(durableSdkEvent);
7884
+ if (
7885
+ !retainedScreenshotMetadata &&
7886
+ retainedSessionImageCallIds.has(completedToolCall.callId) &&
7887
+ sdkEventContainsInlineImage(durableSdkEvent) &&
7888
+ !typedScreenshot
7889
+ ) {
7890
+ retainedScreenshotMetadata = unavailableRetainedSessionImage({
7891
+ sessionId: input.sessionId,
7892
+ turnId: activeTurnId,
7893
+ attemptId: input.attemptId,
7894
+ toolCallId: completedToolCall.callId,
7895
+ toolOutputId: completedToolCall.callId,
7896
+ reason: "unsupported",
7897
+ });
7898
+ retainedScreenshotReceiptsByCallId.set(
7899
+ completedToolCall.callId,
7900
+ retainedScreenshotMetadata,
7901
+ );
7902
+ }
6853
7903
  if (
7904
+ !retainedScreenshotMetadata &&
6854
7905
  typedScreenshot &&
6855
7906
  typedScreenshot.callId === completedToolCall.callId &&
6856
- computerScreenshotCallIds.has(completedToolCall.callId)
7907
+ retainedSessionImageCallIds.has(completedToolCall.callId)
6857
7908
  ) {
7909
+ // Install a compact deterministic fallback before database or
7910
+ // object-storage I/O. Unexpected retention failure can then
7911
+ // reconcile/serialize without leaking inline bytes.
7912
+ retainedScreenshotMetadata = unavailableRetainedSessionImage({
7913
+ sessionId: input.sessionId,
7914
+ turnId: activeTurnId,
7915
+ attemptId: input.attemptId,
7916
+ toolCallId: typedScreenshot.callId,
7917
+ toolOutputId: typedScreenshot.toolOutputId,
7918
+ reason: "pending",
7919
+ });
7920
+ retainedScreenshotReceiptsByCallId.set(
7921
+ completedToolCall.callId,
7922
+ retainedScreenshotMetadata,
7923
+ );
6858
7924
  retainedScreenshotMetadata = await retainComputerScreenshot({
6859
7925
  db,
6860
7926
  objectStorage,
@@ -6891,12 +7957,38 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6891
7957
  callId: completedToolCall.callId,
6892
7958
  modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
6893
7959
  resultItem: durableResultItem as Record<string, unknown>,
7960
+ ...(videoGenerationAcceptancesByCallId.has(completedToolCall.callId)
7961
+ ? {
7962
+ videoGenerationAcceptance: videoGenerationAcceptancesByCallId.get(
7963
+ completedToolCall.callId,
7964
+ )!,
7965
+ }
7966
+ : {}),
6894
7967
  });
6895
7968
  if (!recorded.accepted) {
6896
7969
  throw new TurnAttemptFencedError(
6897
7970
  "turn attempt ended while recording a tool-call result",
6898
7971
  );
6899
7972
  }
7973
+ const videoAcceptance = videoGenerationAcceptancesByCallId.get(
7974
+ completedToolCall.callId,
7975
+ );
7976
+ if (videoAcceptance && startVideoGenerationWorkflow) {
7977
+ try {
7978
+ await startVideoGenerationWorkflow({
7979
+ accountId: input.accountId,
7980
+ workspaceId: input.workspaceId,
7981
+ operationId: videoAcceptance.operationId,
7982
+ });
7983
+ } catch (error) {
7984
+ // The accepted operation is durable. The recovery sweep starts
7985
+ // the same deterministic workflow ID if this nudge fails.
7986
+ observability.warn("Video generation workflow start deferred", {
7987
+ operationId: videoAcceptance.operationId,
7988
+ errorClass: error instanceof Error ? error.name : "UnknownError",
7989
+ });
7990
+ }
7991
+ }
6900
7992
  const belongsToCurrentBatch = currentToolBatchCallIds.has(completedToolCall.callId);
6901
7993
  if (belongsToCurrentBatch) {
6902
7994
  currentToolBatchCompletedCallIds.add(completedToolCall.callId);
@@ -6920,7 +8012,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6920
8012
  }
6921
8013
  }
6922
8014
  const normalized = normalizeSdkEvent(
6923
- next.value,
8015
+ durableSdkEvent as typeof next.value,
6924
8016
  retainedScreenshotMetadata
6925
8017
  ? {
6926
8018
  toolOutputOverride: retainedScreenshotMetadata,
@@ -6957,7 +8049,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6957
8049
  currentToolBatchCompletedCallIds = new Set<string>();
6958
8050
  }
6959
8051
  for (const callId of stableToolCallIdsToClear) {
6960
- computerScreenshotCallIds.delete(callId);
8052
+ retainedSessionImageCallIds.delete(callId);
6961
8053
  }
6962
8054
  }
6963
8055
  }
@@ -7144,7 +8236,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7144
8236
  sessionStatus: "requires_action",
7145
8237
  activeTurnId,
7146
8238
  runState: {
7147
- serializedRunState: stream.state.toString(),
8239
+ serializedRunState: compactMediaRunState(stream.state.toString()),
7148
8240
  pendingApprovals: approvals,
7149
8241
  humanInputRequests: humanInputRequests.map(
7150
8242
  ({ isNew: _isNew, ...request }) => request,
@@ -7360,15 +8452,24 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7360
8452
  // back to the workflow-claimed turn when the local lookup had not
7361
8453
  // finished yet.
7362
8454
  const recoveryTurnId = turnId;
7363
- // P1.2: a lease supersession during resume (a newer epoch re-established
7364
- // the box concurrently) is NOT a session failure. Recover the same turn
7365
- // so its next attempt reattaches under the current epoch.
7366
- if (error instanceof SandboxLeaseSupersededError && recoveryTurnId) {
8455
+ // A true epoch supersession and a provider lifecycle transition are both
8456
+ // recoverable control-plane states, never session failures. The latter is
8457
+ // paced briefly; the next acquire waits on the durable claim and resumes
8458
+ // immediately when it clears.
8459
+ if (
8460
+ (error instanceof SandboxLeaseSupersededError ||
8461
+ error instanceof SandboxLeaseTransitionError) &&
8462
+ recoveryTurnId
8463
+ ) {
7367
8464
  try {
7368
8465
  const fencedLease = await readLease(db, input.workspaceId, error.sandboxGroupId).catch(
7369
8466
  () => null,
7370
8467
  );
7371
- const rotationPending = fencedLease?.rotationRequestedAt != null;
8468
+ const lifecycleTransition = error instanceof SandboxLeaseTransitionError;
8469
+ const rotationPending =
8470
+ fencedLease?.rotationRequestedAt != null ||
8471
+ (lifecycleTransition && error.reason === "rotation_in_progress");
8472
+ const transitionPending = lifecycleTransition || rotationPending;
7372
8473
  const deadlineRotationPending =
7373
8474
  rotationPending && fencedLease?.rotationReason === "provider_deadline";
7374
8475
  const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
@@ -7378,13 +8479,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7378
8479
  attemptId: input.attemptId,
7379
8480
  reason: deadlineRotationPending
7380
8481
  ? "sandbox_deadline_rotation"
7381
- : "sandbox_lease_superseded",
7382
- ...(rotationPending
8482
+ : lifecycleTransition
8483
+ ? "sandbox_lifecycle_transition"
8484
+ : "sandbox_lease_superseded",
8485
+ ...(transitionPending
7383
8486
  ? {
7384
8487
  detail: {
7385
8488
  sandboxGroupId: error.sandboxGroupId,
7386
8489
  leaseEpoch: error.leaseEpoch,
7387
- rotationReason: fencedLease?.rotationReason ?? "operator",
8490
+ ...(rotationPending
8491
+ ? {
8492
+ rotationReason: fencedLease?.rotationReason ?? "operator",
8493
+ }
8494
+ : {}),
8495
+ ...(lifecycleTransition ? { transitionReason: error.reason } : {}),
7388
8496
  },
7389
8497
  }
7390
8498
  : {}),
@@ -7406,17 +8514,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7406
8514
  turnMetricOutcome = "recovering";
7407
8515
  return claimedResult({
7408
8516
  status: "recovering",
7409
- ...(rotationPending
8517
+ ...(transitionPending
7410
8518
  ? {
7411
8519
  continueDelayMs: sandboxDeadlineRotationRecoveryDelayMs(settings),
7412
8520
  }
7413
8521
  : {}),
7414
8522
  });
7415
8523
  } catch (recoveryError) {
7416
- console.error(
7417
- "sandbox lease supersession recovery failed",
7418
- safeErrorDiagnostic(recoveryError),
7419
- );
8524
+ console.error("sandbox lifecycle recovery failed", safeErrorDiagnostic(recoveryError));
7420
8525
  throw recoveryError;
7421
8526
  }
7422
8527
  }
@@ -7482,8 +8587,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7482
8587
  // the frozen RunState blob (the only representation of a turn
7483
8588
  // paused mid-flight), so swapping the trigger for a resume notice
7484
8589
  // could drop the user's decision. Re-applying an already-consumed
7485
- // approval re-executes at most the single approved step — the same
7486
- // bound every recovery already accepts.
8590
+ // approval re-enters at most the single approved step. Every
8591
+ // approval-gated MCP action crosses the durable execution admission
8592
+ // fence before its provider invocation, so a consumed step resumes
8593
+ // as already-executed or outcome-unknown rather than calling MCP again.
7487
8594
  const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
7488
8595
  sessionId: input.sessionId,
7489
8596
  turnId: recoveryTurnId,
@@ -7966,14 +9073,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7966
9073
  });
7967
9074
  if (newHome) {
7968
9075
  rotated = true;
7969
- const pinMutation = await withCodexCapacityMutation(
9076
+ const pinMutation = await withSessionCodexCapacityMutation(
7970
9077
  db,
7971
9078
  {
7972
9079
  workspaceId: input.workspaceId,
7973
9080
  reason: "codex_policy_pin_resharded",
7974
9081
  },
7975
9082
  async (tx) => {
7976
- const changed = await setSessionCodexPin(
9083
+ const changed = await setSessionCodexPinInTransaction(
7977
9084
  tx,
7978
9085
  input.workspaceId,
7979
9086
  input.sessionId,
@@ -8248,7 +9355,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8248
9355
  ) {
8249
9356
  await flushRuntimeBatcher();
8250
9357
  await reconcileConversationTruth({ requireDurable: true });
8251
- const activeHistory = await getActiveSessionHistoryItems(
9358
+ const activeHistory = await getActiveSessionHistoryItemsPaged(
8252
9359
  db,
8253
9360
  input.workspaceId,
8254
9361
  input.sessionId,
@@ -8304,9 +9411,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8304
9411
  // truth, recover this SAME accepted turn, then let the workflow re-claim
8305
9412
  // it after a pacing delay. This is independent of goal state and never
8306
9413
  // relies on a synthetic continuation prompt.
8307
- const failure = agentRunFailurePayload(error, { isCodexTurn }) as ReturnType<
8308
- typeof agentRunFailurePayload
8309
- >;
9414
+ const failure = agentRunFailurePayload(error, {
9415
+ isCodexTurn,
9416
+ }) as ReturnType<typeof agentRunFailurePayload>;
8310
9417
  if (isSessionEventPersistenceError(error)) {
8311
9418
  // Preserve the exact source message in the internal runtime diagnostic;
8312
9419
  // SQLSTATE/catalog facts remain separate classification attributes.
@@ -8454,10 +9561,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8454
9561
  gitCredentialRenewalClosed = true;
8455
9562
  const gitRenewalsToStop = gitCredentialRenewals;
8456
9563
  gitCredentialRenewals = [];
8457
- toolspaceTokenRenewalClosed = true;
8458
- const toolspaceRenewalToStop =
8459
- toolspaceTokenRenewal as ToolspaceTokenRenewalController | null;
8460
- toolspaceTokenRenewal = null;
9564
+ codemodeTokenRenewalClosed = true;
9565
+ const codemodeRenewalToStop = codemodeTokenRenewal as CodemodeTokenRenewalController | null;
9566
+ codemodeTokenRenewal = null;
8461
9567
  runCredentialRenewalClosed = true;
8462
9568
  const runRenewalToStop = runCredentialRenewal as RunCredentialRenewalController | null;
8463
9569
  runCredentialRenewal = null;
@@ -8509,9 +9615,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8509
9615
  toolCancellationFence,
8510
9616
  cancellationReason: cancellationSignal?.reason ?? new Error("TURN_ATTEMPT_FINALIZED"),
8511
9617
  gitCredentialRenewals: gitRenewalsToStop,
8512
- toolspaceTokenRenewal: toolspaceRenewalToStop,
9618
+ codemodeTokenRenewal: codemodeRenewalToStop,
8513
9619
  runCredentialRenewal: runRenewalToStop,
8514
9620
  });
9621
+ attemptWritersDrained = true;
8515
9622
  if (acknowledgeQuiescence) {
8516
9623
  // A cancellation before sandbox-backed capabilities exist still has
8517
9624
  // no tool controller to drain. Sandbox agent construction fails closed
@@ -8537,7 +9644,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8537
9644
  const recoveryMode = await persistOrSignalSessionAttemptQuiescence({
8538
9645
  proof,
8539
9646
  persistReceipt: async () =>
8540
- await markSessionAttemptQuiesced(db, {
9647
+ await commitSessionAttemptQuiescence(db, {
8541
9648
  accountId: input.accountId,
8542
9649
  workspaceId: input.workspaceId,
8543
9650
  sessionId: input.sessionId,
@@ -8547,6 +9654,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8547
9654
  temporalActivityId: dispatchId,
8548
9655
  allowUninterrupted: true,
8549
9656
  }),
9657
+ ...(wakeSessionWorkflow
9658
+ ? {
9659
+ deliverWorkflowWake: async (
9660
+ wake: NonNullable<SessionAttemptQuiescenceCommit["workflowWake"]>,
9661
+ ) =>
9662
+ await wakeSessionWorkflow({
9663
+ accountId: wake.accountId,
9664
+ workspaceId: wake.workspaceId,
9665
+ sessionId: wake.sessionId,
9666
+ workflowId: wake.temporalWorkflowId,
9667
+ wakeRevision: wake.wakeRevision,
9668
+ interruptionRequested: wake.interruptionRequested,
9669
+ }),
9670
+ }
9671
+ : {}),
8550
9672
  publishEvents: async (events) => {
8551
9673
  await waitForTurnFinalizerStep(
8552
9674
  publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events),
@@ -8570,6 +9692,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8570
9692
  safeErrorDiagnostic(error),
8571
9693
  );
8572
9694
  },
9695
+ onWakeFailure: (error) => {
9696
+ console.error(
9697
+ "agent turn quiescence immediate workflow wake failed; outbox repair retained",
9698
+ safeErrorDiagnostic(error),
9699
+ );
9700
+ },
8573
9701
  onPublishFailure: (error) => {
8574
9702
  console.error(
8575
9703
  "agent turn quiescence event fanout failed",
@@ -8733,6 +9861,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8733
9861
  ...(finalizerSignal ? { signal: finalizerSignal } : {}),
8734
9862
  });
8735
9863
  }
9864
+ if (codemodeDispatcher) {
9865
+ await waitForTurnFinalizerStep(
9866
+ codemodeDispatcher.close().catch(() => undefined),
9867
+ finalizerSignal,
9868
+ );
9869
+ codemodeDispatcher = null;
9870
+ }
8736
9871
  if (preparedTools) {
8737
9872
  await waitForTurnFinalizerStep(
8738
9873
  preparedTools.close().catch(() => undefined),
@@ -8820,17 +9955,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8820
9955
  // while the box is still solidly alive, instead of racing the turn-end
8821
9956
  // teardown that was killing 100% of captures on real Modal desktop boxes.
8822
9957
  }
8823
- const sandboxToRelease = resolvedSandbox;
8824
- resolvedSandbox = null; // drop ownership now; the exact-holder release may finish later
8825
- await waitForTurnFinalizerStep(
8826
- sandboxToRelease.release().catch((releaseError) => {
8827
- console.error(
8828
- "sandbox lease release failed (turn outcome unaffected)",
8829
- safeErrorDiagnostic(releaseError),
8830
- );
8831
- }),
8832
- finalizerSignal,
8833
- );
8834
9958
  }
8835
9959
  } catch (error) {
8836
9960
  finalizationError ??= error;
@@ -8839,9 +9963,42 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8839
9963
  safeErrorDiagnostic(error),
8840
9964
  );
8841
9965
  } finally {
8842
- // If fallible finalization exited before the ordinary exact-holder
8843
- // release, the resume signal is the unconditional last line of defense.
8844
- // Normal finalization already removed its listener in release().
9966
+ // The writer drain is the only authority that licenses settling an
9967
+ // abandoned turn admission. Keep this second-stage release outside all
9968
+ // later housekeeping so an event/cache/capture/recording failure cannot
9969
+ // strand a null-outcome admission forever. Do not race it against
9970
+ // Temporal cancellation: the eager listener may already have dropped
9971
+ // the holder, and this idempotent proof-bearing pass still must run.
9972
+ stopLeaseHeartbeat();
9973
+ const sandboxReleaseTargets = new Set(lateSandboxesAwaitingWriterDrain);
9974
+ lateSandboxesAwaitingWriterDrain.clear();
9975
+ if (resolvedSandbox) {
9976
+ sandboxReleaseTargets.add(resolvedSandbox);
9977
+ resolvedSandbox = null;
9978
+ }
9979
+ if (attemptWritersDrained) {
9980
+ for (const sandboxToRelease of sandboxReleaseTargets) {
9981
+ try {
9982
+ await releaseTurnSandboxAfterWriterDrain(sandboxToRelease);
9983
+ } catch (releaseError) {
9984
+ finalizationError ??= releaseError;
9985
+ console.error(
9986
+ "sandbox lease quiesced release failed (turn outcome unaffected)",
9987
+ safeErrorDiagnostic(releaseError),
9988
+ );
9989
+ }
9990
+ }
9991
+ } else {
9992
+ // No proof exists. Ensure every late result still receives the eager
9993
+ // leak-prevention release while preserving its admissions as a
9994
+ // fail-closed archive-capture fence.
9995
+ for (const sandboxToRelease of sandboxReleaseTargets) {
9996
+ await sandboxToRelease.release().catch(() => undefined);
9997
+ }
9998
+ }
9999
+ // Close provisioning after consuming the currently known targets. A
10000
+ // provider result that races in later is routed through releaseLateSandbox;
10001
+ // targets already released above synchronously removed their listener.
8845
10002
  if (!sandboxResumeController.signal.aborted) {
8846
10003
  sandboxResumeController.abort(
8847
10004
  cancellationSignal?.reason ?? new Error("TURN_ATTEMPT_FINALIZED"),
@@ -8868,12 +10025,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8868
10025
  value: physicalCancellationDurationSeconds,
8869
10026
  });
8870
10027
  observability.info("agent turn physical cancellation completed", {
8871
- "opengeni.session_id": input.sessionId,
8872
- "opengeni.turn_id": turnId ?? "",
8873
- "opengeni.attempt_id": input.attemptId,
8874
- "opengeni.physical_cancellation_duration_ms": Math.round(
8875
- physicalCancellationDurationSeconds * 1000,
8876
- ),
10028
+ durationMs: Math.round(physicalCancellationDurationSeconds * 1000),
10029
+ ...(sandboxGroupId
10030
+ ? {
10031
+ sandboxLeaseKey: sandboxLeaseTelemetryKey(input.workspaceId, sandboxGroupId),
10032
+ }
10033
+ : {}),
8877
10034
  });
8878
10035
  }
8879
10036
  observability.recordWorkerActivity({
@@ -9054,6 +10211,24 @@ export function agentRunFailurePayload(
9054
10211
  typeof error === "object" && error !== null && "code" in error
9055
10212
  ? String((error as { code?: unknown }).code)
9056
10213
  : undefined;
10214
+ if (error instanceof ActiveSessionHistoryLimitExceededError) {
10215
+ return {
10216
+ error:
10217
+ "The session's active conversation history exceeds the worker's safe materialization envelope. Clear the session context before retrying; an oversized history cannot be compacted safely in a serving worker.",
10218
+ code: error.code,
10219
+ retryable: false,
10220
+ detail: error.message,
10221
+ };
10222
+ }
10223
+ if (error instanceof ApprovalRunStateLimitExceededError) {
10224
+ return {
10225
+ error:
10226
+ "The saved approval state exceeds the worker's safe materialization envelope. Clear the pending approval context before retrying.",
10227
+ code: error.code,
10228
+ retryable: false,
10229
+ detail: error.message,
10230
+ };
10231
+ }
9057
10232
  // An accepted Codex stream with no terminal response is malformed/partial,
9058
10233
  // not provider backpressure. Replaying the same accepted turn could repeat
9059
10234
  // model or tool effects, so this marked transport failure must outrank the
@@ -9367,6 +10542,10 @@ export function pendingToolCallFromSdkEvent(event: unknown): {
9367
10542
  return null;
9368
10543
  }
9369
10544
  const raw = item.rawItem as Record<string, unknown>;
10545
+ // The hosted image call is a complete provider fact carried by one item; it
10546
+ // never receives a separate function result and therefore must not enter the
10547
+ // pending function-call ledger.
10548
+ if (raw.type === "hosted_tool_call" && raw.name === "image_generation_call") return null;
9370
10549
  const callId = raw.callId ?? raw.call_id ?? raw.id;
9371
10550
  const callType = raw.type;
9372
10551
  if (typeof callId !== "string" || callId.length === 0 || typeof callType !== "string") {
@@ -9381,6 +10560,11 @@ export function pendingToolCallFromSdkEvent(event: unknown): {
9381
10560
  };
9382
10561
  }
9383
10562
 
10563
+ /** Tools whose intentional visual result must survive inline-media sanitization. */
10564
+ export function toolCallProducesRetainableSessionImage(name: string | null): boolean {
10565
+ return name === "computer_screenshot" || name === "view_image";
10566
+ }
10567
+
9384
10568
  export function completedToolCallFromSdkEvent(event: unknown): {
9385
10569
  callId: string;
9386
10570
  resultItem: Record<string, unknown>;
@@ -9505,6 +10689,9 @@ export type ModelUsageBillingRecord = {
9505
10689
  billingPath: "opengeni_credits" | "external";
9506
10690
  /** Same quantity written to usage_events.model.cost when present; else 0. */
9507
10691
  pricedCostMicros: number;
10692
+ /** Hypothetical provider-rate USD micros; never an OpenGeni charge. */
10693
+ estimatedProviderCostMicros: number | null;
10694
+ pricingSource: "configured_list_price" | "gateway_reported" | null;
9508
10695
  normalizedUsage: ModelCallUsageNormalization;
9509
10696
  upstreamProvider?: string;
9510
10697
  };
@@ -9552,6 +10739,37 @@ export async function recordModelUsageAndDebitCredits(
9552
10739
  );
9553
10740
  }
9554
10741
  }
10742
+ const pricingSchedules = configuredModelPricingSchedules(settings);
10743
+ const configuredPricingModel = pricingSchedules[input.model]
10744
+ ? input.model
10745
+ : input.model.startsWith("codex/") && pricingSchedules[input.model.slice("codex/".length)]
10746
+ ? input.model.slice("codex/".length)
10747
+ : null;
10748
+ const pricingBreakdown = gatewayBilling
10749
+ ? calculateGatewayReportedCostBreakdown(
10750
+ settings,
10751
+ input.model,
10752
+ gatewayBilling.inferenceCostUsd,
10753
+ { inputTokens },
10754
+ )
10755
+ : configuredPricingModel
10756
+ ? calculateModelUsageCostBreakdown(settings, configuredPricingModel, sanitizedUsage, {
10757
+ latencyMode: input.latencyMode ?? "standard",
10758
+ })
10759
+ : null;
10760
+ const hasCompleteCoreTokenTelemetry =
10761
+ normalizedUsage.telemetry.inputTokens !== null &&
10762
+ normalizedUsage.telemetry.outputTokens !== null;
10763
+ const estimatedProviderCostMicros = gatewayBilling
10764
+ ? (pricingBreakdown?.providerCostMicros ?? null)
10765
+ : hasCompleteCoreTokenTelemetry
10766
+ ? (pricingBreakdown?.providerCostMicros ?? null)
10767
+ : null;
10768
+ const pricingSource = gatewayBilling
10769
+ ? ("gateway_reported" as const)
10770
+ : estimatedProviderCostMicros !== null
10771
+ ? ("configured_list_price" as const)
10772
+ : null;
9555
10773
  // An externally billed turn is paid outside OpenGeni, so it
9556
10774
  // consumes ZERO OpenGeni credits and must never feed an OpenGeni cap. A
9557
10775
  // codex/<slug> model has no entry in configuredModelPricing, so the normal path
@@ -9560,7 +10778,8 @@ export async function recordModelUsageAndDebitCredits(
9560
10778
  // the API tokens:consume cap sum `model.tokens` with NO cost dimension, so
9561
10779
  // any row would count against maxMonthlyTokensPerWorkspace);
9562
10780
  // - record a `model.cost = 0` audit marker (harmless to the monthly cost cap);
9563
- // - never look up pricing and never debit credits.
10781
+ // - optionally calculate a non-charging provider estimate for Insights when
10782
+ // core token telemetry is complete, but never debit credits.
9564
10783
  if (input.externallyBilled) {
9565
10784
  await recordUsageEvent(db, {
9566
10785
  accountId: input.accountId,
@@ -9578,6 +10797,8 @@ export async function recordModelUsageAndDebitCredits(
9578
10797
  return {
9579
10798
  billingPath: "external",
9580
10799
  pricedCostMicros: 0,
10800
+ estimatedProviderCostMicros,
10801
+ pricingSource,
9581
10802
  normalizedUsage,
9582
10803
  };
9583
10804
  }
@@ -9601,20 +10822,16 @@ export async function recordModelUsageAndDebitCredits(
9601
10822
  return {
9602
10823
  billingPath: "opengeni_credits",
9603
10824
  pricedCostMicros: 0,
10825
+ estimatedProviderCostMicros,
10826
+ pricingSource,
9604
10827
  normalizedUsage,
9605
10828
  ...(gatewayBilling ? { upstreamProvider: gatewayBilling.finalProvider } : {}),
9606
10829
  };
9607
10830
  }
9608
- if (!configuredModelPricing(settings)[input.model]) {
10831
+ if (!pricingBreakdown) {
9609
10832
  throw new Error(`Missing model pricing for ${input.model}`);
9610
10833
  }
9611
- const costMicros = gatewayBilling
9612
- ? calculateGatewayReportedCostMicros(settings, input.model, gatewayBilling.inferenceCostUsd, {
9613
- inputTokens,
9614
- })
9615
- : calculateModelUsageCostMicros(settings, input.model, sanitizedUsage, {
9616
- latencyMode: input.latencyMode ?? "standard",
9617
- });
10834
+ const costMicros = pricingBreakdown.creditCostMicros;
9618
10835
  await recordUsageEvent(db, {
9619
10836
  accountId: input.accountId,
9620
10837
  workspaceId: input.workspaceId,
@@ -9658,6 +10875,8 @@ export async function recordModelUsageAndDebitCredits(
9658
10875
  return {
9659
10876
  billingPath: "opengeni_credits",
9660
10877
  pricedCostMicros: costMicros,
10878
+ estimatedProviderCostMicros,
10879
+ pricingSource,
9661
10880
  normalizedUsage,
9662
10881
  ...(gatewayBilling ? { upstreamProvider: gatewayBilling.finalProvider } : {}),
9663
10882
  };
@@ -9692,6 +10911,8 @@ export async function recordAuthoritativeModelCallFact(input: {
9692
10911
  model: input.model,
9693
10912
  billingPath: input.billing.billingPath,
9694
10913
  pricedCostMicros: input.billing.pricedCostMicros,
10914
+ estimatedProviderCostMicros: input.billing.estimatedProviderCostMicros,
10915
+ pricingSource: input.billing.pricingSource,
9695
10916
  inputTokens: telemetry.inputTokens,
9696
10917
  outputTokens: telemetry.outputTokens,
9697
10918
  cachedTokens: telemetry.cachedTokens,
@@ -9722,6 +10943,9 @@ function sanitizedModelUsageInput(normalized: ModelCallUsageNormalization): Mode
9722
10943
  },
9723
10944
  }
9724
10945
  : {}),
10946
+ ...(normalized.requestUsageEntries
10947
+ ? { requestUsageEntries: normalized.requestUsageEntries }
10948
+ : {}),
9725
10949
  };
9726
10950
  }
9727
10951