@opengeni/worker-bundle 0.20.3 → 0.22.1-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +52 -0
  2. package/dist/activities/agent-turn/agent-build.d.ts +89 -0
  3. package/dist/activities/agent-turn/claim.d.ts +67 -0
  4. package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
  5. package/dist/activities/agent-turn/codex.d.ts +32 -0
  6. package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
  7. package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
  8. package/dist/activities/agent-turn/errors.d.ts +253 -0
  9. package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
  10. package/dist/activities/agent-turn/file-resources.d.ts +37 -0
  11. package/dist/activities/agent-turn/finalization.d.ts +42 -0
  12. package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
  13. package/dist/activities/agent-turn/governance-model.d.ts +66 -0
  14. package/dist/activities/agent-turn/history-sink.d.ts +52 -0
  15. package/dist/activities/agent-turn/history.d.ts +63 -0
  16. package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
  17. package/dist/activities/agent-turn/model-usage.d.ts +218 -0
  18. package/dist/activities/agent-turn/quiescence.d.ts +135 -0
  19. package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
  20. package/dist/activities/agent-turn/run.d.ts +3 -0
  21. package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
  22. package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
  23. package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
  24. package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
  25. package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
  26. package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
  27. package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
  28. package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
  29. package/dist/activities/agent-turn/turn-context.d.ts +144 -0
  30. package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
  31. package/dist/activities/agent-turn.d.ts +12 -954
  32. package/dist/activities/automations.d.ts +16 -0
  33. package/dist/activities/common.d.ts +1 -1
  34. package/dist/activities/connector-attachments.d.ts +13 -0
  35. package/dist/activities/documents.d.ts +1 -0
  36. package/dist/activities/environment.d.ts +26 -15
  37. package/dist/activities/generated-images.d.ts +1 -1
  38. package/dist/activities/git-credential-renewal.d.ts +3 -1
  39. package/dist/activities/goals.d.ts +4 -1
  40. package/dist/activities/google-drive-provider.d.ts +29 -0
  41. package/dist/activities/google-drive-publication.d.ts +17 -4
  42. package/dist/activities/incident-telemetry-authority.d.ts +52 -0
  43. package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
  44. package/dist/activities/knowledge-source-sync.d.ts +4 -0
  45. package/dist/activities/mcp-credentials.d.ts +9 -3
  46. package/dist/activities/open-suffix-resume.d.ts +77 -0
  47. package/dist/activities/packs.d.ts +3 -11
  48. package/dist/activities/parent-wake.d.ts +19 -0
  49. package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
  50. package/dist/activities/retained-screenshots.d.ts +1 -1
  51. package/dist/activities/rig-verification.d.ts +1 -1
  52. package/dist/activities/run-input.d.ts +9 -10
  53. package/dist/activities/sandbox-images.d.ts +1 -14
  54. package/dist/activities/sandbox-lease.d.ts +21 -3
  55. package/dist/activities/scheduled-tasks.d.ts +47 -0
  56. package/dist/activities/types.d.ts +23 -2
  57. package/dist/activities/video-media-validation.d.ts +1 -1
  58. package/dist/activities/video-output-retention.d.ts +1 -2
  59. package/dist/activities/xai-auth.d.ts +3 -0
  60. package/dist/{activities-control-BJSDRSIP.js → activities-control-DKG22ONJ.js} +2646 -551
  61. package/dist/activities-control-DKG22ONJ.js.map +1 -0
  62. package/dist/activities-control.d.ts +6 -2
  63. package/dist/activities-turn-4IMC3MAA.js +21326 -0
  64. package/dist/activities-turn-4IMC3MAA.js.map +1 -0
  65. package/dist/activities.d.ts +7 -3
  66. package/dist/artifact-materializer-entry.js +41 -22
  67. package/dist/artifact-materializer-entry.js.map +1 -1
  68. package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
  69. package/dist/chunk-CE7TPKBS.js.map +1 -0
  70. package/dist/chunk-CNYGVRKF.js +4698 -0
  71. package/dist/chunk-CNYGVRKF.js.map +1 -0
  72. package/dist/connection-provider-use.d.ts +18 -0
  73. package/dist/editable-artifact-materializer.d.ts +4 -3
  74. package/dist/github-rest-mcp.d.ts +26 -0
  75. package/dist/index.d.ts +10 -2
  76. package/dist/index.js +58 -14
  77. package/dist/index.js.map +1 -1
  78. package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
  79. package/dist/model-context-contributions.d.ts +8 -4
  80. package/dist/observability-metrics.d.ts +63 -4
  81. package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
  82. package/dist/personal-github-git-credentials.d.ts +9 -0
  83. package/dist/pr-review-credentials.d.ts +6 -0
  84. package/dist/sandbox-archive-storage.d.ts +45 -0
  85. package/dist/sandbox-resume.d.ts +65 -6
  86. package/dist/sandbox-routing.d.ts +38 -2
  87. package/dist/scheduled-alert-occurrence.d.ts +33 -0
  88. package/dist/workflow-bundle.js +87 -28
  89. package/dist/workflows/activities.d.ts +1 -0
  90. package/dist/workflows/automations.d.ts +6 -0
  91. package/dist/workflows/document-index.d.ts +1 -0
  92. package/dist/workflows.d.ts +1 -0
  93. package/package.json +17 -17
  94. package/src/activities/agent-turn/admission.ts +292 -0
  95. package/src/activities/agent-turn/agent-build.ts +802 -0
  96. package/src/activities/agent-turn/claim.ts +590 -0
  97. package/src/activities/agent-turn/codex-capacity.ts +720 -0
  98. package/src/activities/agent-turn/codex.ts +94 -0
  99. package/src/activities/agent-turn/compaction-prep.ts +722 -0
  100. package/src/activities/agent-turn/credential-leases.ts +240 -0
  101. package/src/activities/agent-turn/errors.ts +1068 -0
  102. package/src/activities/agent-turn/failure-settlement.ts +1436 -0
  103. package/src/activities/agent-turn/file-resources.ts +227 -0
  104. package/src/activities/agent-turn/finalization.ts +754 -0
  105. package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
  106. package/src/activities/agent-turn/governance-model.ts +465 -0
  107. package/src/activities/agent-turn/history-sink.ts +139 -0
  108. package/src/activities/agent-turn/history.ts +200 -0
  109. package/src/activities/agent-turn/media-artifacts.ts +359 -0
  110. package/src/activities/agent-turn/model-usage.ts +823 -0
  111. package/src/activities/agent-turn/quiescence.ts +353 -0
  112. package/src/activities/agent-turn/run-credentials.ts +761 -0
  113. package/src/activities/agent-turn/run.ts +1530 -0
  114. package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
  115. package/src/activities/agent-turn/sandbox-provision.ts +569 -0
  116. package/src/activities/agent-turn/sandbox-route.ts +358 -0
  117. package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
  118. package/src/activities/agent-turn/stream-attempt.ts +1670 -0
  119. package/src/activities/agent-turn/tool-environment.ts +729 -0
  120. package/src/activities/agent-turn/tool-policy.ts +216 -0
  121. package/src/activities/agent-turn/tool-result-spill.ts +265 -0
  122. package/src/activities/agent-turn/turn-context.ts +281 -0
  123. package/src/activities/agent-turn/xai-capacity.ts +229 -0
  124. package/src/activities/agent-turn.ts +12 -12701
  125. package/src/activities/api-integrations.ts +3 -0
  126. package/src/activities/automations.ts +214 -0
  127. package/src/activities/codemode-dispatcher.ts +12 -1
  128. package/src/activities/common.ts +1 -5
  129. package/src/activities/connector-attachments.ts +168 -0
  130. package/src/activities/context-compaction.ts +18 -0
  131. package/src/activities/environment.ts +151 -55
  132. package/src/activities/generated-images.ts +26 -11
  133. package/src/activities/git-credential-renewal.ts +13 -3
  134. package/src/activities/goals.ts +85 -19
  135. package/src/activities/google-drive-provider.ts +178 -0
  136. package/src/activities/google-drive-publication.ts +147 -49
  137. package/src/activities/incident-telemetry-authority.ts +327 -0
  138. package/src/activities/incident-telemetry-preflight.ts +317 -0
  139. package/src/activities/knowledge-source-sync.ts +178 -37
  140. package/src/activities/mcp-credentials.ts +191 -17
  141. package/src/activities/open-suffix-resume.ts +422 -0
  142. package/src/activities/packs.ts +12 -60
  143. package/src/activities/parent-wake.ts +64 -14
  144. package/src/activities/retained-screenshot-reaper.ts +6 -3
  145. package/src/activities/retained-screenshots.ts +15 -47
  146. package/src/activities/rig-verification.ts +1 -1
  147. package/src/activities/run-input.ts +102 -142
  148. package/src/activities/sandbox-images.ts +7 -166
  149. package/src/activities/sandbox-lease.ts +406 -15
  150. package/src/activities/scheduled-tasks.ts +2103 -411
  151. package/src/activities/session-state.ts +10 -2
  152. package/src/activities/types.ts +32 -2
  153. package/src/activities/video-media-validation.ts +3 -3
  154. package/src/activities/video-output-retention.ts +2 -57
  155. package/src/activities/xai-auth.ts +8 -0
  156. package/src/activities-control.ts +2 -0
  157. package/src/activities.ts +3 -0
  158. package/src/activity-services.ts +13 -4
  159. package/src/connection-provider-use.ts +36 -0
  160. package/src/editable-artifact-materializer-subprocess.ts +25 -14
  161. package/src/editable-artifact-materializer.ts +29 -18
  162. package/src/github-rest-mcp.ts +713 -0
  163. package/src/http.ts +16 -6
  164. package/src/index.ts +40 -5
  165. package/src/knowledge-source-sync-activity-policy.ts +2 -0
  166. package/src/model-context-contributions.ts +33 -10
  167. package/src/observability-metrics.ts +357 -0
  168. package/src/opensandbox-kubernetes-inventory.ts +292 -0
  169. package/src/personal-github-git-credentials.ts +331 -0
  170. package/src/pr-review-credentials.ts +163 -0
  171. package/src/sandbox-archive-storage.ts +159 -0
  172. package/src/sandbox-resume.ts +449 -63
  173. package/src/sandbox-routing.ts +314 -16
  174. package/src/scheduled-alert-occurrence.ts +176 -0
  175. package/src/workflows/activities.ts +18 -2
  176. package/src/workflows/automations.ts +11 -0
  177. package/src/workflows/session.ts +13 -3
  178. package/src/workflows.ts +1 -0
  179. package/dist/activities-control-BJSDRSIP.js.map +0 -1
  180. package/dist/activities-turn-J5YO7XXX.js +0 -17243
  181. package/dist/activities-turn-J5YO7XXX.js.map +0 -1
  182. package/dist/chunk-N5TJIFCE.js.map +0 -1
  183. package/dist/chunk-P6Z2SLAA.js +0 -3092
  184. package/dist/chunk-P6Z2SLAA.js.map +0 -1
@@ -0,0 +1,1207 @@
1
+ import {
2
+ getEnrollment,
3
+ getSandbox,
4
+ readActiveSandbox,
5
+ getLiveEnrollmentConnection,
6
+ assertPersonalMachineForAttempt,
7
+ type SandboxRecord,
8
+ } from "@opengeni/db";
9
+ import { sandboxOperationMetricObserver } from "@opengeni/observability";
10
+ import {
11
+ withRunCredentialsSession,
12
+ runOwnedSandboxSetup,
13
+ markModelPreparationFirstSandboxOperation,
14
+ recordModelPreparationMeasurement,
15
+ sdkBackendIdForSandboxBackend,
16
+ type EstablishedSandboxSession,
17
+ type GitCredentialTokenWriterSession,
18
+ type NormalizedRunCredentialMaterial,
19
+ type RunCredentialCommandSession,
20
+ type CodemodeTokenWriterSession,
21
+ type SelfhostedSession,
22
+ resolveConnectedMachineWorkspaceRoot,
23
+ } from "@opengeni/runtime";
24
+ import { type Settings } from "@opengeni/config";
25
+ import { mergeResourceRefs } from "../common";
26
+ import {
27
+ mintSandboxCodemodeToken,
28
+ type MintedRunGitCredentials,
29
+ type SandboxCodemodeAuthority,
30
+ } from "../environment";
31
+ import { rigProviderImageSourceImage } from "../packs";
32
+ import type { TurnActivityServices as ActivityServices, RunAgentTurnInput } from "../types";
33
+ import type { currentActivityContext } from "../streaming";
34
+ import { resumeBoxForTurn, type ResumedTurnSandbox } from "../../sandbox-resume";
35
+ import {
36
+ wrapTurnBoxWithRouting,
37
+ wrapLazyTurnBoxWithRouting,
38
+ establishSelfhostedTurnSession,
39
+ routingEnabled,
40
+ } from "../../sandbox-routing";
41
+ import {
42
+ makeMachineOpObserver,
43
+ recordSandboxLogicalProvision,
44
+ recordSandboxProvisionAttempt,
45
+ recordTurnStartupPhase,
46
+ runtimeMetricsHooksForObservability,
47
+ } from "../../observability-metrics";
48
+ import { ToolResultSpill } from "./tool-result-spill";
49
+ import type { createTurnMediaArtifacts } from "./media-artifacts";
50
+
51
+ import {
52
+ throwIfTurnOperationCancelled,
53
+ waitForTurnOperation,
54
+ classifySandboxLogicalProvisionFailure,
55
+ isLazySandboxProvisionRetryable,
56
+ createTurnSandboxProvisioner,
57
+ } from "./sandbox-provision";
58
+ import {
59
+ resolveActiveSandboxBackend,
60
+ managedSandboxOwnershipForTurn,
61
+ shouldEstablishSandboxForTurn,
62
+ sandboxEstablishPolicyDecision,
63
+ shouldPrefetchManagedSandbox,
64
+ reconcileActiveSandboxPointer,
65
+ } from "./sandbox-route";
66
+ import type { ClaimTurnOk } from "./claim";
67
+ import type { GovernanceModelOk } from "./governance-model";
68
+ import type { SandboxTurnRuntime } from "./sandbox-runtime";
69
+ import type { AttemptIdentityState, EventingState, SandboxRuntimeState } from "./turn-context";
70
+
71
+ export type SandboxRouteDeps = {
72
+ input: RunAgentTurnInput;
73
+ settings: Settings;
74
+ db: ActivityServices["db"];
75
+ eventing: EventingState;
76
+ sandboxState: SandboxRuntimeState;
77
+ media: ReturnType<typeof createTurnMediaArtifacts>;
78
+ fileAuthoritySubjectId: ClaimTurnOk["fileAuthoritySubjectId"];
79
+ runSettings: GovernanceModelOk["runSettings"];
80
+ };
81
+
82
+ export type SandboxRouteOk = {
83
+ routingOn: boolean;
84
+ activeSandboxPointer: Awaited<ReturnType<typeof readActiveSandbox>>;
85
+ activeSandboxRecord: SandboxRecord | null;
86
+ activeSandboxBackend: Settings["sandboxBackend"] | undefined;
87
+ machinePrimary: boolean;
88
+ groupBoxBackend: Settings["sandboxBackend"];
89
+ groupBoxImage: ReturnType<typeof rigProviderImageSourceImage>;
90
+ sandboxCreationBackend: Settings["sandboxBackend"];
91
+ effectiveRunCredentialBackend: Settings["sandboxBackend"];
92
+ };
93
+
94
+ export type EstablishTurnSandboxDeps = SandboxRouteOk & {
95
+ input: RunAgentTurnInput;
96
+ settings: Settings;
97
+ db: ActivityServices["db"];
98
+ bus: ActivityServices["bus"];
99
+ objectStorage: ActivityServices["objectStorage"];
100
+ observability: ActivityServices["observability"];
101
+ cancellationSignal: AbortSignal | undefined;
102
+ runtimeCancellationSignal: AbortSignal | undefined;
103
+ sandboxResumeSignal: AbortSignal;
104
+ activityContext: ReturnType<typeof currentActivityContext>;
105
+ opJournal: ClaimTurnOk["opJournal"];
106
+ sandboxState: SandboxRuntimeState;
107
+ eventing: EventingState;
108
+ attempt: AttemptIdentityState;
109
+ turn: ClaimTurnOk["turn"];
110
+ session: ClaimTurnOk["session"];
111
+ fileAuthoritySubjectId: ClaimTurnOk["fileAuthoritySubjectId"];
112
+ capabilitySettings: ClaimTurnOk["capabilitySettings"];
113
+ turnExecutionPolicy: ClaimTurnOk["turnExecutionPolicy"];
114
+ runSettings: GovernanceModelOk["runSettings"];
115
+ logicalSandboxSettings: GovernanceModelOk["logicalSandboxSettings"];
116
+ verifiedRigProviderImageId: GovernanceModelOk["verifiedRigProviderImageId"];
117
+ runtimePreparationStartedAt: number;
118
+ establishPolicy: ReturnType<typeof sandboxEstablishPolicyDecision>["policy"];
119
+ sandboxEnvironment: Record<string, string>;
120
+ startRunGitCredentialsMint: () => Promise<MintedRunGitCredentials | undefined>;
121
+ machineOpObserver: ReturnType<typeof makeMachineOpObserver>;
122
+ sandboxOperationObserver: ReturnType<typeof sandboxOperationMetricObserver>;
123
+ sandboxRuntime: SandboxTurnRuntime;
124
+ transientCodemodeEnvironment: (() => Readonly<Record<string, string>>) | undefined;
125
+ rigVersion: GovernanceModelOk["rigVersion"];
126
+ turnResources: ReturnType<typeof mergeResourceRefs>;
127
+ };
128
+
129
+ export type BindLazySandboxProvisionerDeps = EstablishTurnSandboxDeps & {
130
+ agent: ReturnType<ActivityServices["runtime"]["buildAgent"]>;
131
+ attachRunCredentialRenewal: (
132
+ credentialSession: RunCredentialCommandSession,
133
+ initialMaterial: NormalizedRunCredentialMaterial | null,
134
+ initialSandbox?: ResumedTurnSandbox,
135
+ ) => Promise<void>;
136
+ attachGitCredentialRenewal: (
137
+ tokenSession: GitCredentialTokenWriterSession,
138
+ initial: MintedRunGitCredentials | undefined,
139
+ initialSandbox?: ResumedTurnSandbox,
140
+ ) => Promise<void>;
141
+ attachCodemodeTokenRenewal: (
142
+ tokenSession?: CodemodeTokenWriterSession,
143
+ initialExpiresAt?: Date,
144
+ initialSandbox?: ResumedTurnSandbox,
145
+ ) => Promise<void>;
146
+ throwIfWorkerShuttingDown: () => void;
147
+ throwIfTurnCancelled: () => void;
148
+ initialRunCredentialMaterial: NormalizedRunCredentialMaterial | null;
149
+ sandboxCodemodeToken: string | undefined;
150
+ media: ReturnType<typeof createTurnMediaArtifacts>;
151
+ toolResultSpill: ToolResultSpill;
152
+ connectionScope: { accountId: string; workspaceId: string };
153
+ codemodeAuthority: SandboxCodemodeAuthority;
154
+ };
155
+
156
+ export async function resolveSandboxRoute(deps: SandboxRouteDeps): Promise<SandboxRouteOk> {
157
+ const {
158
+ input,
159
+ settings,
160
+ db,
161
+ eventing,
162
+ media,
163
+ sandboxState,
164
+ fileAuthoritySubjectId,
165
+ runSettings,
166
+ } = deps;
167
+
168
+ // EFFECTIVE compute backend, resolved ONCE at turn start (Case B + Stage D
169
+ // D1-lite) and reused for EVERY downstream decision: the env mint (skip
170
+ // inert platform git tokens for a machine turn), the establish path (no phantom Modal
171
+ // home box for a machine-primary turn), buildAgent (skip the repository clone
172
+ // hook so a private repo is never `git clone`d onto the user's real disk), and
173
+ // the warm-rate (a machine accrues ZERO cloud warm-seconds). The active pointer
174
+ // + its sandbox row are loaded ONCE here (best-effort, never throwing) and the
175
+ // SAME values feed resolveActiveSandboxBackend (the tested gate) AND the
176
+ // machine-primary establish branch (enrollmentId/epoch/workingDir) below — no
177
+ // double read, no read-skew between the gate decision and the establish. With
178
+ // routing OFF this is byte-for-byte the legacy path: no reads, undefined backend.
179
+ const routingOn = routingEnabled(settings);
180
+ let activeSandboxPointer = routingOn
181
+ ? await readActiveSandbox(db, input.workspaceId, input.sessionId).catch(() => null)
182
+ : null;
183
+ // TURN-START RECONCILE (issue #341 invariant B / Shapes 1+2): a persisted
184
+ // pointer whose target is STRUCTURALLY unestablishable at turn start would strand
185
+ // EVERY op of this turn — reset it to the session HOME under the epoch fence +
186
+ // emit a visible event, honoring a concurrent higher-epoch swap. The sandbox row
187
+ // is loaded HERE, inside reconcile, via a NON-swallowing lookup: a null decision
188
+ // then means the row is genuinely absent, never a suppressed transient DB error
189
+ // (which would wrongly clear a healthy user-chosen pointer). On a lookup throw the
190
+ // reconcile fails open — pointer untouched, record null (machinePrimary:false),
191
+ // no event — and the establish branch below reads the returned values.
192
+ let activeSandboxRecord: SandboxRecord | null = null;
193
+ if (routingOn) {
194
+ const reconciled = await reconcileActiveSandboxPointer(
195
+ db,
196
+ {
197
+ accountId: input.accountId,
198
+ workspaceId: input.workspaceId,
199
+ sessionId: input.sessionId,
200
+ },
201
+ activeSandboxPointer,
202
+ (sandboxId) =>
203
+ getSandbox(
204
+ db,
205
+ fileAuthoritySubjectId
206
+ ? {
207
+ accountId: input.accountId,
208
+ workspaceId: input.workspaceId,
209
+ subjectId: fileAuthoritySubjectId,
210
+ }
211
+ : input.workspaceId,
212
+ sandboxId,
213
+ ),
214
+ eventing.publish
215
+ ? async (events) => {
216
+ await eventing.publish!(events);
217
+ }
218
+ : undefined,
219
+ );
220
+ activeSandboxPointer = reconciled.pointer;
221
+ activeSandboxRecord = reconciled.record;
222
+ }
223
+ const activeSandboxBackend = await resolveActiveSandboxBackend(
224
+ routingOn,
225
+ async () => activeSandboxPointer,
226
+ async () => activeSandboxRecord?.kind ?? null,
227
+ );
228
+ // A machine-primary turn = the effective backend is selfhosted AND we have the
229
+ // machine's enrollment (agent id) + a non-null pointer to bind it. Anything
230
+ // missing (should not happen — the DB enforces selfhosted⇒enrollmentId) falls
231
+ // back to the cloud establish path (a correct, if phantom, box) rather than
232
+ // crashing the turn.
233
+ const machinePrimary =
234
+ activeSandboxBackend === "selfhosted" &&
235
+ Boolean(activeSandboxPointer?.activeSandboxId) &&
236
+ Boolean(activeSandboxRecord?.enrollmentId);
237
+ // `none` describes the durable home, not an explicit per-turn route. Give
238
+ // the runtime the effective backend so it builds a SandboxAgent for the
239
+ // attached Connected Machine without mutating the session or turn record.
240
+ if (machinePrimary && eventing.modelRunSettings.sandboxBackend === "none") {
241
+ eventing.modelRunSettings = {
242
+ ...eventing.modelRunSettings,
243
+ sandboxBackend: "selfhosted",
244
+ };
245
+ }
246
+ // The backend that can actually create a sandbox for this turn. In the
247
+ // common path this is runSettings.sandboxBackend. A selfhosted home turn
248
+ // that is NOT machine-primary falls back to the deployment cloud backend
249
+ // so swap-away / flag-off degrade to a real group box.
250
+ const groupBoxBackend: Settings["sandboxBackend"] =
251
+ runSettings.sandboxBackend === "selfhosted" && !machinePrimary
252
+ ? settings.sandboxBackend
253
+ : runSettings.sandboxBackend;
254
+ sandboxState.startupMilestoneBackend = activeSandboxBackend ?? groupBoxBackend;
255
+ media.sandboxFileDownloadBackend = sandboxState.startupMilestoneBackend;
256
+ const groupBoxImage = rigProviderImageSourceImage(runSettings, groupBoxBackend);
257
+ const sandboxCreationBackend: Settings["sandboxBackend"] =
258
+ settings.sandboxOwnershipEnabled && runSettings.sandboxBackend !== "none"
259
+ ? groupBoxBackend
260
+ : runSettings.sandboxBackend;
261
+ const effectiveRunCredentialBackend = activeSandboxBackend ?? groupBoxBackend;
262
+
263
+ return {
264
+ routingOn,
265
+ activeSandboxPointer,
266
+ activeSandboxRecord,
267
+ activeSandboxBackend,
268
+ machinePrimary,
269
+ groupBoxBackend,
270
+ groupBoxImage,
271
+ sandboxCreationBackend,
272
+ effectiveRunCredentialBackend,
273
+ };
274
+ }
275
+
276
+ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Promise<void> {
277
+ const {
278
+ input,
279
+ settings,
280
+ db,
281
+ bus,
282
+ objectStorage,
283
+ observability,
284
+ runtimeCancellationSignal,
285
+ sandboxResumeSignal,
286
+ opJournal,
287
+ sandboxState,
288
+ eventing,
289
+ attempt,
290
+ turn,
291
+ session,
292
+ fileAuthoritySubjectId,
293
+ turnExecutionPolicy,
294
+ runSettings,
295
+ logicalSandboxSettings,
296
+ runtimePreparationStartedAt,
297
+ establishPolicy,
298
+ sandboxEnvironment,
299
+ startRunGitCredentialsMint,
300
+ machineOpObserver,
301
+ sandboxOperationObserver,
302
+ sandboxRuntime,
303
+ transientCodemodeEnvironment,
304
+ activeSandboxPointer,
305
+ activeSandboxRecord,
306
+ activeSandboxBackend,
307
+ machinePrimary,
308
+ groupBoxBackend,
309
+ groupBoxImage,
310
+ rigVersion,
311
+ turnResources,
312
+ } = deps;
313
+ const {
314
+ releaseLateSandbox,
315
+ onHomeSandboxRebound,
316
+ publishSandboxLifecycleEvents,
317
+ publishSandboxLost,
318
+ startLeaseHeartbeat,
319
+ } = sandboxRuntime;
320
+
321
+ // P1.2 ownership inversion (gated, default OFF). With the flag off this
322
+ // block is skipped entirely: resolvedSandbox stays null and runStream
323
+ // takes the legacy per-run build-and-discard path — byte-for-byte today.
324
+ // With it on, acquire the group lease (holder = the durable attempt id),
325
+ // resume the one box by id, and inject it NON-OWNED into the run. The box
326
+ // backend is "none" -> never resolve (no box to touch).
327
+ //
328
+ // Established AFTER sandboxEnvironment is computed (not before) so the box's
329
+ // manifest is created with the SAME variableSet the agent declares — the SDK
330
+ // applies the agent's manifest to this provided session and throws on ANY
331
+ // variableSet delta (validateNoEnvironmentDelta). Passing sandboxEnvironment
332
+ // here makes current==target so the delta is empty.
333
+ recordTurnStartupPhase(observability, {
334
+ phase: "runtime_preparation",
335
+ provider: turnExecutionPolicy.providerId,
336
+ backend: activeSandboxBackend ?? groupBoxBackend,
337
+ outcome: "completed",
338
+ durationSeconds: (performance.now() - runtimePreparationStartedAt) / 1_000,
339
+ });
340
+ if (
341
+ shouldEstablishSandboxForTurn(
342
+ settings.sandboxOwnershipEnabled,
343
+ turn.sandboxBackend,
344
+ machinePrimary,
345
+ )
346
+ ) {
347
+ const sandboxEstablishStartedAt = performance.now();
348
+ let sandboxEstablishOutcome: "completed" | "failed" = "completed";
349
+ try {
350
+ const managedOwnership = managedSandboxOwnershipForTurn(
351
+ machinePrimary,
352
+ input.attemptId,
353
+ session.sandboxGroupId,
354
+ );
355
+ sandboxState.sandboxHolderId = managedOwnership?.holderId ?? null;
356
+ sandboxState.sandboxGroupId = managedOwnership?.sandboxGroupId ?? null;
357
+ // STAGE D honest-label guard: a machine-home session carries
358
+ // turn.sandboxBackend "selfhosted", but a turn is only machine-PRIMARY
359
+ // when a live machine pointer resolves (activeSandboxBackend==='selfhosted'
360
+ // + enrollmentId). When it is NOT primary — the agent swapped back to the
361
+ // group box (sandbox_swap 'session'/'default'/groupId clears the pointer) or
362
+ // selfhosted routing is flag-OFF (the pointer is ignored) — the else-branch
363
+ // must resume a REAL cloud group box, not a "selfhosted" one: the registry
364
+ // SelfhostedSandboxClient has no bound agentId and throws. Fall the group-box
365
+ // backend back to the deployment default cloud backend so swap-away / flag-off
366
+ // degrade to a genuine cloud box exactly like today (home=modal did).
367
+ if (machinePrimary) {
368
+ if (activeSandboxRecord!.scope === "user") {
369
+ if (!fileAuthoritySubjectId) {
370
+ throw new Error("personal machine use requires an initiating human subject");
371
+ }
372
+ const authorized = await assertPersonalMachineForAttempt(db, {
373
+ accountId: input.accountId,
374
+ workspaceId: input.workspaceId,
375
+ subjectId: fileAuthoritySubjectId,
376
+ sessionId: input.sessionId,
377
+ turnId: turn.id,
378
+ attemptId: input.attemptId,
379
+ executionGeneration: turn.executionGeneration,
380
+ enrollmentId: activeSandboxRecord!.enrollmentId!,
381
+ });
382
+ if (!authorized) {
383
+ throw new Error("personal Connected Machine use was not admitted for this turn");
384
+ }
385
+ }
386
+ // STAGE D D1-lite: the active sandbox is a connected machine, so DO NOT
387
+ // establish OR lease the managed home box. The active-sandbox pointer is
388
+ // the machine route's own epoch fence; the managed-home lease separately
389
+ // owns cloud recovery, snapshots, billing, and reaping. Mixing those two
390
+ // ownership domains lets an unrelated degraded cloud archive block a
391
+ // healthy explicitly selected machine.
392
+ // Whether the machine's latest Hello advertised the op-stream engine
393
+ // (refreshed on every connect). Read only when the server flag is on —
394
+ // one indexed lookup, and the flag off keeps this path byte-identical.
395
+ const liveMachineEnrollment = await getLiveEnrollmentConnection(
396
+ db,
397
+ activeSandboxRecord!.workspaceId,
398
+ activeSandboxRecord!.enrollmentId!,
399
+ );
400
+ const machineEnrollment =
401
+ liveMachineEnrollment ??
402
+ (await getEnrollment(
403
+ db,
404
+ activeSandboxRecord!.workspaceId,
405
+ activeSandboxRecord!.enrollmentId!,
406
+ ));
407
+ if (!machineEnrollment?.workspaceRoot) {
408
+ throw new Error(
409
+ "Connected Machine has not reported an absolute workspace root; reconnect it with a current agent",
410
+ );
411
+ }
412
+ const machineWorkspaceRoot = resolveConnectedMachineWorkspaceRoot(
413
+ machineEnrollment.workspaceRoot,
414
+ activeSandboxPointer!.workingDir,
415
+ );
416
+ const machineOpStream =
417
+ settings.agentOpStreamEnabled === true && liveMachineEnrollment?.opStream === true;
418
+ const established = await establishSelfhostedTurnSession(
419
+ {
420
+ db,
421
+ settings,
422
+ bus,
423
+ onOp: machineOpObserver.observer,
424
+ onSandboxOperation: sandboxOperationObserver,
425
+ opJournal,
426
+ },
427
+ {
428
+ accountId: input.accountId,
429
+ workspaceId: input.workspaceId,
430
+ sessionId: input.sessionId,
431
+ controlWorkspaceId: activeSandboxRecord!.workspaceId,
432
+ backgroundCommandAttempt: {
433
+ turnId: turn.id,
434
+ executionGeneration: attempt.executionGeneration,
435
+ attemptId: input.attemptId,
436
+ },
437
+ agentId: activeSandboxRecord!.enrollmentId!,
438
+ // An offline machine must not fail turn admission. Bind an
439
+ // intentionally unserved token so the model receives the normal
440
+ // typed agent_offline tool result; a later turn resolves a fresh
441
+ // claimed process instance.
442
+ connectionInstanceId: liveMachineEnrollment?.connectionInstanceId ?? "unavailable",
443
+ workspaceRoot: machineWorkspaceRoot,
444
+ opStream: machineOpStream,
445
+ operationResourcePolicy: liveMachineEnrollment?.operationPolicy ?? {
446
+ memoryMaxBytes: null,
447
+ memoryHighBytes: null,
448
+ cpuMaxMillicores: null,
449
+ revision: 0,
450
+ updatedAt: null,
451
+ },
452
+ operationResourcePolicySupported:
453
+ liveMachineEnrollment?.agentCapabilities.operationResourcePolicy === true,
454
+ operationCpuQuotaSupported:
455
+ liveMachineEnrollment?.agentCapabilities.operationCpuQuota === true,
456
+ ...(activeSandboxRecord!.scope === "user" && fileAuthoritySubjectId
457
+ ? {
458
+ personalMachineAttempt: {
459
+ accountId: input.accountId,
460
+ subjectId: fileAuthoritySubjectId,
461
+ sessionId: input.sessionId,
462
+ turnId: turn.id,
463
+ attemptId: input.attemptId,
464
+ executionGeneration: attempt.executionGeneration,
465
+ },
466
+ }
467
+ : {}),
468
+ epoch: activeSandboxPointer!.activeEpoch,
469
+ environment: sandboxEnvironment,
470
+ ...(transientCodemodeEnvironment
471
+ ? { transientExecEnvironment: transientCodemodeEnvironment }
472
+ : {}),
473
+ workingDir: activeSandboxPointer!.workingDir,
474
+ },
475
+ );
476
+ // The machine-primary establish narrows `session` to SelfhostedSession
477
+ // (buildSelfhostedBackendSession); EstablishedSandboxSession widens it.
478
+ sandboxState.machinePrimarySession = established.session as SelfhostedSession;
479
+ sandboxState.setupBoxSession = established.session;
480
+ sandboxState.resolvedSandbox = {
481
+ // Wrap in the SAME routing proxy so a mid-turn swap (to another machine
482
+ // or back to the group box) still re-routes per op. PIN this established
483
+ // SelfhostedSession for the machine pointer so the turn-start manifest
484
+ // write (via the proxy's `state` getter) and the per-op reads hit ONE
485
+ // instance — no two-instance manifest divergence.
486
+ established: wrapTurnBoxWithRouting(
487
+ {
488
+ db,
489
+ settings,
490
+ bus,
491
+ opJournal,
492
+ onOp: machineOpObserver.observer,
493
+ onSandboxOperation: sandboxOperationObserver,
494
+ onHomeSandboxRebound,
495
+ ...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
496
+ },
497
+ {
498
+ workspaceId: input.workspaceId,
499
+ sessionId: input.sessionId,
500
+ resourceAccountId: input.accountId,
501
+ ...(fileAuthoritySubjectId ? { resourceSubjectId: fileAuthoritySubjectId } : {}),
502
+ ...(fileAuthoritySubjectId
503
+ ? {
504
+ personalMachineAttempt: {
505
+ accountId: input.accountId,
506
+ subjectId: fileAuthoritySubjectId,
507
+ turnId: turn.id,
508
+ attemptId: input.attemptId,
509
+ executionGeneration: attempt.executionGeneration,
510
+ },
511
+ }
512
+ : {}),
513
+ environment: sandboxEnvironment,
514
+ ...(transientCodemodeEnvironment
515
+ ? { transientExecEnvironment: transientCodemodeEnvironment }
516
+ : {}),
517
+ workspaceMutationFence: {
518
+ accountId: input.accountId,
519
+ turnId: turn.id,
520
+ executionGeneration: attempt.executionGeneration,
521
+ attemptId: input.attemptId,
522
+ },
523
+ pinnedSelfhosted: {
524
+ sandboxId: activeSandboxPointer!.activeSandboxId!,
525
+ epoch: activeSandboxPointer!.activeEpoch,
526
+ },
527
+ // HOME semantics for a mid-turn clear-to-null: only a genuine
528
+ // machine-HOME session (its home IS this machine, session.sandboxBackend
529
+ // === "selfhosted") resolves null back to the pinned machine. A Modal-HOME
530
+ // session merely PINNED to a machine this turn never established its group
531
+ // box, so defaultIsHome:false makes a clear-to-null fail typed
532
+ // (`home_unavailable_this_turn`) rather than silently serving the machine.
533
+ // Failure settlement checkpoints the completed detach and continues the same
534
+ // logical turn in a fresh home-primary attempt; it never serves this machine
535
+ // after the pointer changed and never invents a cloud lease for machine-only work.
536
+ defaultIsHome: session.sandboxBackend === "selfhosted",
537
+ },
538
+ established,
539
+ ),
540
+ // For a machine route this is the active-pointer epoch, not a cloud
541
+ // lease epoch. Cloud-only heartbeat/snapshot/capture paths require
542
+ // sandboxGroupId and remain disabled for this branch.
543
+ leaseEpoch: activeSandboxPointer!.activeEpoch,
544
+ release: async () => undefined,
545
+ };
546
+ } else if (establishPolicy === "on-demand") {
547
+ // Lazy sandbox provisioning: holder/group ids are fixed at turn start,
548
+ // but the lease acquire + box establish + setup move behind the routing
549
+ // proxy's first default-pointer op. A chat-only turn never calls it, so
550
+ // no lease row, no provider box, no warm-meter interval.
551
+ //
552
+ // A repository still forces that first default-pointer op before HTTP
553
+ // (workspace-skill catalog in Agent.instructions). Start create now and
554
+ // join it at get(); do not await here or we serialize Modal before tools.
555
+ if (sandboxState.sandboxHolderId && sandboxState.sandboxGroupId) {
556
+ const lazyHolderId = sandboxState.sandboxHolderId;
557
+ const lazyGroupId = sandboxState.sandboxGroupId;
558
+ sandboxState.resumeManagedGroupBox = () =>
559
+ resumeBoxForTurn(
560
+ {
561
+ db,
562
+ settings: runSettings,
563
+ logicalFallbackSettings: logicalSandboxSettings,
564
+ cancellationSignal: sandboxResumeSignal,
565
+ sandboxMetrics: runtimeMetricsHooksForObservability(observability),
566
+ onSandboxLost: publishSandboxLost,
567
+ objectStorage,
568
+ },
569
+ {
570
+ accountId: input.accountId,
571
+ workspaceId: input.workspaceId,
572
+ sandboxGroupId: lazyGroupId,
573
+ sessionId: input.sessionId,
574
+ backend: groupBoxBackend,
575
+ os: session.sandboxOs,
576
+ environment: sandboxEnvironment,
577
+ ...(groupBoxImage
578
+ ? {
579
+ image: groupBoxImage,
580
+ }
581
+ : {}),
582
+ // The lazy acquire must enforce the same frozen rig authority
583
+ // as the eager acquire; otherwise a warm box for another rig
584
+ // can bypass the shared-state conflict/rotation fence.
585
+ ...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
586
+ },
587
+ "turn",
588
+ lazyHolderId,
589
+ );
590
+ if (
591
+ shouldPrefetchManagedSandbox({
592
+ establishPolicy,
593
+ machinePrimary,
594
+ groupBoxBackend,
595
+ hasRepositoryResources: turnResources.some(
596
+ (resource) => resource.kind === "repository",
597
+ ),
598
+ })
599
+ ) {
600
+ startRunGitCredentialsMint();
601
+ const started = waitForTurnOperation(
602
+ sandboxState.resumeManagedGroupBox(),
603
+ sandboxResumeSignal,
604
+ releaseLateSandbox,
605
+ );
606
+ const joined = started.catch((error) => {
607
+ if (isLazySandboxProvisionRetryable(error)) {
608
+ sandboxState.prefetchedManagedBox = null;
609
+ }
610
+ throw error;
611
+ });
612
+ sandboxState.prefetchedManagedBox = joined;
613
+ void joined.then(
614
+ (box) => {
615
+ if (sandboxResumeSignal.aborted) return;
616
+ startLeaseHeartbeat(box, activeSandboxBackend ?? groupBoxBackend);
617
+ },
618
+ () => undefined,
619
+ );
620
+ }
621
+ }
622
+ } else {
623
+ await eventing.publish!(
624
+ [
625
+ {
626
+ type: "sandbox.operation.started",
627
+ payload: { name: "sandbox.provision" },
628
+ },
629
+ ],
630
+ true,
631
+ );
632
+ try {
633
+ sandboxState.resolvedSandbox = await waitForTurnOperation(
634
+ resumeBoxForTurn(
635
+ {
636
+ db,
637
+ settings: runSettings,
638
+ logicalFallbackSettings: logicalSandboxSettings,
639
+ cancellationSignal: sandboxResumeSignal,
640
+ sandboxMetrics: runtimeMetricsHooksForObservability(observability),
641
+ onSandboxLost: publishSandboxLost,
642
+ objectStorage,
643
+ },
644
+ {
645
+ accountId: input.accountId,
646
+ workspaceId: input.workspaceId,
647
+ sandboxGroupId: session.sandboxGroupId,
648
+ sessionId: input.sessionId,
649
+ // groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
650
+ // is not machine-primary resumes a real cloud group box (the
651
+ // deployment default), never a "selfhosted" box (which would throw
652
+ // for lack of a bound agentId).
653
+ backend: groupBoxBackend,
654
+ os: session.sandboxOs,
655
+ environment: sandboxEnvironment,
656
+ // IMAGE IS SHARED STATE (B3): the container image
657
+ // this run resolves. The lease stamps it + conflicts on a live shared box
658
+ // running a DIFFERENT image (solo → durable rotation; N-holders →
659
+ // SandboxImageConflictError surfaced as an actionable turn error). Select
660
+ // the image for the actual group-box backend; a configured Modal image must
661
+ // never override a Docker run. The selfhosted branch
662
+ // (establishSelfhostedTurnSession) NEVER passes
663
+ // an image — B3 lives only on this managed-box branch.
664
+ ...(groupBoxImage
665
+ ? {
666
+ image: groupBoxImage,
667
+ }
668
+ : {}),
669
+ // RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
670
+ // conflicts on a live shared box set up under a different rig (solo
671
+ // durable rotation / N-holders SandboxRigConflictError). Omitted for a
672
+ // rig-less turn -> never stamped or enforced (shares exactly as today).
673
+ ...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
674
+ },
675
+ "turn",
676
+ managedOwnership!.holderId,
677
+ ),
678
+ sandboxResumeSignal,
679
+ releaseLateSandbox,
680
+ );
681
+ } catch (error) {
682
+ await eventing.publish!(
683
+ [
684
+ {
685
+ type: "sandbox.operation.failed",
686
+ payload: {
687
+ name: "sandbox.provision",
688
+ error: error instanceof Error ? error.message : String(error),
689
+ },
690
+ },
691
+ ],
692
+ true,
693
+ );
694
+ throw error;
695
+ }
696
+ sandboxState.setupBoxSession = sandboxState.resolvedSandbox.established.session;
697
+ // Durable box-lifecycle events (sandbox-file-persistence observability):
698
+ // record every box transition in session_events so the NEXT box loss is
699
+ // attributable from the DB alone — worker logs rotate within hours, which
700
+ // left both 2026-07-06 incidents without a durable trace. Best-effort.
701
+ await publishSandboxLifecycleEvents(sandboxState.resolvedSandbox);
702
+ await eventing.publish!(
703
+ [
704
+ {
705
+ type: "sandbox.operation.completed",
706
+ payload: {
707
+ name: "sandbox.provision",
708
+ ...(sandboxState.resolvedSandbox.established.origin
709
+ ? { origin: sandboxState.resolvedSandbox.established.origin }
710
+ : {}),
711
+ },
712
+ },
713
+ ],
714
+ true,
715
+ );
716
+ // M7 hot-swap: when the selfhosted feature is on, wrap the established
717
+ // group box in the STABLE routing proxy before it is injected NON-OWNED
718
+ // into the run. The SDK binds to this ONE object once and calls its
719
+ // methods per tool call; the proxy re-reads (active_sandbox_id,
720
+ // active_epoch) per op and dispatches to the currently-active backend, so
721
+ // a sandbox_swap mid-turn lands the NEXT tool call on the new box. With
722
+ // the flag off the established group box is injected unchanged (today's
723
+ // path). The lease still owns the group box lifecycle — the proxy is a
724
+ // routing veneer, not an owner.
725
+ sandboxState.resolvedSandbox = {
726
+ ...sandboxState.resolvedSandbox,
727
+ established: wrapTurnBoxWithRouting(
728
+ {
729
+ db,
730
+ settings,
731
+ bus,
732
+ opJournal,
733
+ onSandboxOperation: sandboxOperationObserver,
734
+ onHomeSandboxLost: publishSandboxLost,
735
+ onHomeSandboxRebound,
736
+ ...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
737
+ },
738
+ // Thread the SAME declared environment the group box was created with
739
+ // (resumeBoxForTurn, above) so a selfhosted swap target's manifest
740
+ // carries it too — the SDK's per-turn manifest-env delta stays empty
741
+ // (no "cannot change manifest environment variables" throw).
742
+ {
743
+ workspaceId: input.workspaceId,
744
+ sessionId: input.sessionId,
745
+ resourceAccountId: input.accountId,
746
+ ...(fileAuthoritySubjectId ? { resourceSubjectId: fileAuthoritySubjectId } : {}),
747
+ ...(fileAuthoritySubjectId
748
+ ? {
749
+ personalMachineAttempt: {
750
+ accountId: input.accountId,
751
+ subjectId: fileAuthoritySubjectId,
752
+ turnId: turn.id,
753
+ attemptId: input.attemptId,
754
+ executionGeneration: attempt.executionGeneration,
755
+ },
756
+ }
757
+ : {}),
758
+ environment: sandboxEnvironment,
759
+ ...(transientCodemodeEnvironment
760
+ ? { transientExecEnvironment: transientCodemodeEnvironment }
761
+ : {}),
762
+ workspaceMutationFence: {
763
+ accountId: input.accountId,
764
+ turnId: turn.id,
765
+ executionGeneration: attempt.executionGeneration,
766
+ attemptId: input.attemptId,
767
+ },
768
+ homeLease: {
769
+ accountId: input.accountId,
770
+ sandboxGroupId: session.sandboxGroupId,
771
+ leaseEpoch: sandboxState.resolvedSandbox.leaseEpoch,
772
+ instanceId: sandboxState.resolvedSandbox.established.instanceId,
773
+ backend: groupBoxBackend,
774
+ },
775
+ },
776
+ sandboxState.resolvedSandbox.established,
777
+ ),
778
+ };
779
+ }
780
+ if (sandboxState.resolvedSandbox) {
781
+ startLeaseHeartbeat(sandboxState.resolvedSandbox, activeSandboxBackend ?? groupBoxBackend);
782
+ }
783
+ } catch (error) {
784
+ sandboxEstablishOutcome = "failed";
785
+ throw error;
786
+ } finally {
787
+ recordTurnStartupPhase(observability, {
788
+ phase: "sandbox_establish",
789
+ provider: turnExecutionPolicy.providerId,
790
+ backend: machinePrimary ? "selfhosted" : groupBoxBackend,
791
+ outcome: sandboxEstablishOutcome,
792
+ durationSeconds: (performance.now() - sandboxEstablishStartedAt) / 1_000,
793
+ });
794
+ }
795
+ }
796
+ }
797
+
798
+ export async function bindLazySandboxProvisioner(
799
+ deps: BindLazySandboxProvisionerDeps,
800
+ ): Promise<void> {
801
+ const {
802
+ input,
803
+ settings,
804
+ db,
805
+ bus,
806
+ observability,
807
+ runtimeCancellationSignal,
808
+ sandboxResumeSignal,
809
+ opJournal,
810
+ sandboxState,
811
+ eventing,
812
+ attempt,
813
+ turn,
814
+ session,
815
+ fileAuthoritySubjectId,
816
+ runSettings,
817
+ sandboxEnvironment,
818
+ groupBoxBackend,
819
+ activeSandboxBackend,
820
+ establishPolicy,
821
+ startRunGitCredentialsMint,
822
+ machineOpObserver,
823
+ sandboxOperationObserver,
824
+ sandboxRuntime,
825
+ transientCodemodeEnvironment,
826
+ agent,
827
+ attachRunCredentialRenewal,
828
+ attachGitCredentialRenewal,
829
+ attachCodemodeTokenRenewal,
830
+ throwIfWorkerShuttingDown,
831
+ throwIfTurnCancelled,
832
+ initialRunCredentialMaterial,
833
+ sandboxCodemodeToken,
834
+ media,
835
+ toolResultSpill,
836
+ connectionScope,
837
+ codemodeAuthority,
838
+ } = deps;
839
+ const {
840
+ releaseLateSandbox,
841
+ onHomeSandboxRebound,
842
+ publishSandboxLifecycleEvents,
843
+ publishSandboxLost,
844
+ startLeaseHeartbeat,
845
+ runWorkspaceMutationForSandbox,
846
+ } = sandboxRuntime;
847
+
848
+ if (
849
+ establishPolicy === "on-demand" &&
850
+ sandboxState.sandboxHolderId &&
851
+ sandboxState.sandboxGroupId
852
+ ) {
853
+ const agentDefaultManifest = (agent as { defaultManifest?: unknown }).defaultManifest;
854
+ if (!agentDefaultManifest) {
855
+ throw new Error("Lazy sandbox provisioning requires a SandboxAgent defaultManifest");
856
+ }
857
+ const lazyClient = {
858
+ backendId: sdkBackendIdForSandboxBackend(groupBoxBackend),
859
+ } as EstablishedSandboxSession["client"];
860
+ let lazySandboxEstablishmentSettled = false;
861
+ sandboxState.turnSandboxProvisioner = createTurnSandboxProvisioner<ResumedTurnSandbox>(
862
+ async () => {
863
+ throwIfWorkerShuttingDown();
864
+ throwIfTurnCancelled();
865
+ if (!sandboxState.resumeManagedGroupBox) {
866
+ throw new Error("Lazy sandbox provisioning requires a managed group-box resume");
867
+ }
868
+ startRunGitCredentialsMint();
869
+ const provisioned = await (sandboxState.prefetchedManagedBox ??
870
+ sandboxState.resumeManagedGroupBox());
871
+ await publishSandboxLifecycleEvents(provisioned);
872
+ // Return the REAL established box (NOT a copy whose session is the routing
873
+ // proxy). resolveActiveBackend dispatches ops to `provisioned.established.session`;
874
+ // if that were the proxy itself, proxy.exec -> dispatch -> resolve ->
875
+ // provisioner.get() -> proxy.exec -> ... loops forever (an async infinite
876
+ // recursion that HANGS the turn — caught live on staging 2026-07-08). The SDK
877
+ // already holds the proxy directly (injected as lazyOwnedSandbox.session), so it
878
+ // gets per-op routing; the worker-side handle (resolvedSandbox: release,
879
+ // heartbeat, computer-use recording) wants the real box, unproxied.
880
+ return provisioned;
881
+ },
882
+ {
883
+ signal: sandboxResumeSignal,
884
+ onStarted: async ({ provisionId }) => {
885
+ await eventing.publish?.(
886
+ [
887
+ {
888
+ type: "sandbox.operation.started",
889
+ payload: { name: "sandbox.provision", provisionId },
890
+ },
891
+ ],
892
+ true,
893
+ );
894
+ },
895
+ onAttemptSettled: ({ result, error, outcome, durationMs }) => {
896
+ const successStage =
897
+ result?.established.origin === "created"
898
+ ? "create"
899
+ : result?.established.origin === "resumed"
900
+ ? "resume"
901
+ : result?.established.origin === "restored"
902
+ ? "archive_recovery"
903
+ : "unknown";
904
+ const failure = error
905
+ ? classifySandboxLogicalProvisionFailure(groupBoxBackend, error)
906
+ : null;
907
+ recordSandboxProvisionAttempt(observability, {
908
+ backend: groupBoxBackend,
909
+ stage: failure?.stage ?? successStage,
910
+ category: failure?.category ?? successStage,
911
+ outcome,
912
+ durationSeconds: durationMs / 1_000,
913
+ });
914
+ },
915
+ beforeCompleted: async (provisioned, settlement) => {
916
+ throwIfTurnOperationCancelled(sandboxResumeSignal);
917
+ startLeaseHeartbeat(provisioned, activeSandboxBackend ?? groupBoxBackend);
918
+ sandboxState.setupBoxSession = provisioned.established.session;
919
+ sandboxState.resolvedSandbox = provisioned;
920
+ // This durable completion and its logical metric close at actual box
921
+ // establishment. Credential, rig, repository, and file preparation below
922
+ // must never be attributed to "Starting sandbox" or provision latency.
923
+ await eventing.publish?.(
924
+ [
925
+ {
926
+ type: "sandbox.operation.completed",
927
+ payload: {
928
+ name: "sandbox.provision",
929
+ ...(provisioned.established.origin
930
+ ? { origin: provisioned.established.origin }
931
+ : {}),
932
+ provisionId: settlement.provisionId,
933
+ internalAttempts: settlement.internalAttempts,
934
+ },
935
+ },
936
+ ],
937
+ true,
938
+ );
939
+ const successStage =
940
+ provisioned.established.origin === "created"
941
+ ? "create"
942
+ : provisioned.established.origin === "resumed"
943
+ ? "resume"
944
+ : provisioned.established.origin === "restored"
945
+ ? "archive_recovery"
946
+ : "unknown";
947
+ recordSandboxLogicalProvision(observability, {
948
+ backend: groupBoxBackend,
949
+ stage: successStage,
950
+ category: "none",
951
+ outcome: "completed",
952
+ expected: false,
953
+ internalAttempts: settlement.internalAttempts,
954
+ durationSeconds: settlement.durationMs / 1_000,
955
+ });
956
+ lazySandboxEstablishmentSettled = true;
957
+
958
+ const lazyGitCredentials =
959
+ activeSandboxBackend === "selfhosted" ? undefined : await startRunGitCredentialsMint();
960
+ const lazyGitTokens = lazyGitCredentials?.gitTokens;
961
+ const lazyCodemodeToken = sandboxCodemodeToken
962
+ ? await mintSandboxCodemodeToken(runSettings, connectionScope, codemodeAuthority)
963
+ : undefined;
964
+ await attachRunCredentialRenewal(
965
+ provisioned.established.session as RunCredentialCommandSession,
966
+ initialRunCredentialMaterial,
967
+ provisioned,
968
+ );
969
+ const provisionedSetupSession = initialRunCredentialMaterial
970
+ ? withRunCredentialsSession(provisioned.established.session as object, input.sessionId)
971
+ : provisioned.established.session;
972
+ await runWorkspaceMutationForSandbox(
973
+ provisioned,
974
+ "lazyOwnedSandboxSetup",
975
+ async () =>
976
+ await runOwnedSandboxSetup(
977
+ agent,
978
+ provisioned.established.session as never,
979
+ provisionedSetupSession as never,
980
+ {
981
+ settings: runSettings,
982
+ environment: sandboxEnvironment,
983
+ onRuntimeEvent: async (event) => {
984
+ await eventing.publish?.([{ type: event.type, payload: event.payload }], true);
985
+ },
986
+ ...(lazyGitTokens ? { gitTokenSeedsOverride: lazyGitTokens } : {}),
987
+ ...(lazyGitCredentials?.bindings
988
+ ? {
989
+ gitCredentialBindingsOverride: lazyGitCredentials.bindings,
990
+ }
991
+ : {}),
992
+ ...(lazyCodemodeToken
993
+ ? {
994
+ codemodeTokenSeedOverride: lazyCodemodeToken.token,
995
+ }
996
+ : {}),
997
+ ...(eventing.toolCancellationFenceRef.current
998
+ ? {
999
+ commandRunner:
1000
+ eventing.toolCancellationFenceRef.current.runSandboxCommand.bind(
1001
+ eventing.toolCancellationFenceRef.current,
1002
+ ),
1003
+ }
1004
+ : {}),
1005
+ },
1006
+ ),
1007
+ (measurement) => {
1008
+ if (eventing.firstModelRequestPreparationRecorded) return;
1009
+ sandboxState.firstModelPreparationNestedSandboxMs +=
1010
+ measurement.durationSeconds * 1_000;
1011
+ sandboxState.firstModelPreparationNestedSandboxPhases.push(measurement);
1012
+ },
1013
+ );
1014
+ await attachCodemodeTokenRenewal(
1015
+ provisioned.established.session as CodemodeTokenWriterSession,
1016
+ lazyCodemodeToken?.expiresAt,
1017
+ provisioned,
1018
+ );
1019
+ await attachGitCredentialRenewal(
1020
+ provisioned.established.session as GitCredentialTokenWriterSession,
1021
+ lazyGitCredentials,
1022
+ provisioned,
1023
+ );
1024
+ // `get()` does not release the first routed sandbox operation
1025
+ // until this hook returns. Only images created during this exact
1026
+ // turn can require deferred delivery here; historical images stay
1027
+ // as durable receipts and are restored explicitly when requested.
1028
+ for (const receipt of media.generatedImageReceiptsCreatedThisTurn.values()) {
1029
+ await media.materializeGeneratedImageInSandbox(
1030
+ receipt,
1031
+ provisioned,
1032
+ provisioned.established.session,
1033
+ );
1034
+ }
1035
+ await toolResultSpill.materializeDeferred();
1036
+ throwIfTurnOperationCancelled(sandboxResumeSignal);
1037
+ },
1038
+ onFailed: async (error, settlement) => {
1039
+ if (lazySandboxEstablishmentSettled) return;
1040
+ const failure = classifySandboxLogicalProvisionFailure(groupBoxBackend, error);
1041
+ await eventing.publish?.(
1042
+ [
1043
+ {
1044
+ type: "sandbox.operation.failed",
1045
+ payload: {
1046
+ name: "sandbox.provision",
1047
+ error: error instanceof Error ? error.message : String(error),
1048
+ provisionId: settlement.provisionId,
1049
+ internalAttempts: settlement.internalAttempts,
1050
+ failureCategory: failure.category,
1051
+ failureStage: failure.stage,
1052
+ failureCode: failure.code,
1053
+ expectedTransition: failure.expected,
1054
+ retryable: failure.retryable,
1055
+ },
1056
+ },
1057
+ ],
1058
+ true,
1059
+ );
1060
+ recordSandboxLogicalProvision(observability, {
1061
+ backend: groupBoxBackend,
1062
+ stage: failure.stage,
1063
+ category: failure.category,
1064
+ outcome: failure.expected ? "expected_transition" : "failed",
1065
+ expected: failure.expected,
1066
+ internalAttempts: settlement.internalAttempts,
1067
+ durationSeconds: settlement.durationMs / 1_000,
1068
+ });
1069
+ },
1070
+ disposeResult: async (provisioned) => {
1071
+ await releaseLateSandbox(provisioned);
1072
+ },
1073
+ },
1074
+ );
1075
+ sandboxState.lazyOwnedSandbox = wrapLazyTurnBoxWithRouting(
1076
+ {
1077
+ db,
1078
+ settings,
1079
+ bus,
1080
+ opJournal,
1081
+ onOp: machineOpObserver.observer,
1082
+ onSandboxOperation: sandboxOperationObserver,
1083
+ onHomeSandboxLost: publishSandboxLost,
1084
+ onHomeSandboxRebound,
1085
+ ...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
1086
+ },
1087
+ {
1088
+ workspaceId: input.workspaceId,
1089
+ sessionId: input.sessionId,
1090
+ resourceAccountId: input.accountId,
1091
+ ...(fileAuthoritySubjectId ? { resourceSubjectId: fileAuthoritySubjectId } : {}),
1092
+ ...(fileAuthoritySubjectId
1093
+ ? {
1094
+ personalMachineAttempt: {
1095
+ accountId: input.accountId,
1096
+ subjectId: fileAuthoritySubjectId,
1097
+ turnId: turn.id,
1098
+ attemptId: input.attemptId,
1099
+ executionGeneration: attempt.executionGeneration,
1100
+ },
1101
+ }
1102
+ : {}),
1103
+ environment: sandboxEnvironment,
1104
+ ...(transientCodemodeEnvironment
1105
+ ? { transientExecEnvironment: transientCodemodeEnvironment }
1106
+ : {}),
1107
+ workspaceMutationFence: {
1108
+ accountId: input.accountId,
1109
+ turnId: turn.id,
1110
+ executionGeneration: attempt.executionGeneration,
1111
+ attemptId: input.attemptId,
1112
+ },
1113
+ },
1114
+ {
1115
+ client: lazyClient,
1116
+ backendId: sdkBackendIdForSandboxBackend(groupBoxBackend),
1117
+ agentDefaultManifest,
1118
+ provisioner: sandboxState.turnSandboxProvisioner,
1119
+ homeLeaseIdentity: {
1120
+ accountId: input.accountId,
1121
+ sandboxGroupId: session.sandboxGroupId,
1122
+ backend: groupBoxBackend,
1123
+ },
1124
+ onFirstOperation: (measurement) => {
1125
+ if (eventing.firstModelRequestPreparationRecorded) return;
1126
+ markModelPreparationFirstSandboxOperation(measurement.durationMs / 1_000);
1127
+
1128
+ for (const nested of sandboxState.firstModelPreparationNestedSandboxPhases) {
1129
+ if (nested.phase === "snapshot_wait") continue;
1130
+ recordModelPreparationMeasurement({
1131
+ phase:
1132
+ nested.phase === "admission"
1133
+ ? "sandbox_workspace_mutation_admission"
1134
+ : nested.phase === "provider"
1135
+ ? "sandbox_workspace_mutation_provider"
1136
+ : "sandbox_workspace_mutation_settlement",
1137
+ outcome: nested.outcome,
1138
+ durationSeconds: nested.durationSeconds,
1139
+ });
1140
+ }
1141
+
1142
+ const resolution = measurement.phases.resolution;
1143
+ if (resolution) {
1144
+ recordModelPreparationMeasurement({
1145
+ phase: "sandbox_first_routed_resolution_other",
1146
+ outcome: resolution.outcome,
1147
+ durationSeconds:
1148
+ Math.max(
1149
+ 0,
1150
+ resolution.durationMs - sandboxState.firstModelPreparationNestedSandboxMs,
1151
+ ) / 1_000,
1152
+ });
1153
+ }
1154
+
1155
+ const routedPhaseNames = [
1156
+ ["mutationAdmission", "sandbox_first_routed_mutation_admission"],
1157
+ ["providerOperation", "sandbox_first_routed_provider_operation"],
1158
+ ["mutationSettlement", "sandbox_first_routed_mutation_settlement"],
1159
+ ] as const;
1160
+ for (const [phaseName, metricPhase] of routedPhaseNames) {
1161
+ const phase = measurement.phases[phaseName];
1162
+ if (!phase) continue;
1163
+ recordModelPreparationMeasurement({
1164
+ phase: metricPhase,
1165
+ outcome: phase.outcome,
1166
+ durationSeconds: phase.durationMs / 1_000,
1167
+ });
1168
+ }
1169
+
1170
+ const nestedSnapshotPhases = sandboxState.firstModelPreparationNestedSandboxPhases.filter(
1171
+ (phase) => phase.phase === "snapshot_wait",
1172
+ );
1173
+ const routedSnapshot = measurement.phases.snapshotWait;
1174
+ const snapshotWaitMs =
1175
+ nestedSnapshotPhases.reduce(
1176
+ (total, phase) => total + phase.durationSeconds * 1_000,
1177
+ 0,
1178
+ ) + (routedSnapshot?.durationMs ?? 0);
1179
+ if (snapshotWaitMs > 0) {
1180
+ recordModelPreparationMeasurement({
1181
+ phase: "sandbox_snapshot_wait",
1182
+ outcome:
1183
+ routedSnapshot?.outcome === "failed" ||
1184
+ nestedSnapshotPhases.some((phase) => phase.outcome === "failed")
1185
+ ? "failed"
1186
+ : "completed",
1187
+ durationSeconds: snapshotWaitMs / 1_000,
1188
+ });
1189
+ }
1190
+
1191
+ const routedMeasuredMs = Object.values(measurement.phases).reduce(
1192
+ (total, phase) => total + (phase?.durationMs ?? 0),
1193
+ 0,
1194
+ );
1195
+ const routedOtherMs = Math.max(0, measurement.durationMs - routedMeasuredMs);
1196
+ if (routedOtherMs > 0) {
1197
+ recordModelPreparationMeasurement({
1198
+ phase: "sandbox_first_routed_other",
1199
+ outcome: measurement.outcome,
1200
+ durationSeconds: routedOtherMs / 1_000,
1201
+ });
1202
+ }
1203
+ },
1204
+ },
1205
+ );
1206
+ }
1207
+ }