@opengeni/worker-bundle 0.20.1 → 0.22.0-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +52 -0
  2. package/dist/activities/agent-turn/agent-build.d.ts +89 -0
  3. package/dist/activities/agent-turn/claim.d.ts +67 -0
  4. package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
  5. package/dist/activities/agent-turn/codex.d.ts +32 -0
  6. package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
  7. package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
  8. package/dist/activities/agent-turn/errors.d.ts +253 -0
  9. package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
  10. package/dist/activities/agent-turn/file-resources.d.ts +37 -0
  11. package/dist/activities/agent-turn/finalization.d.ts +42 -0
  12. package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
  13. package/dist/activities/agent-turn/governance-model.d.ts +66 -0
  14. package/dist/activities/agent-turn/history-sink.d.ts +52 -0
  15. package/dist/activities/agent-turn/history.d.ts +63 -0
  16. package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
  17. package/dist/activities/agent-turn/model-usage.d.ts +218 -0
  18. package/dist/activities/agent-turn/quiescence.d.ts +135 -0
  19. package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
  20. package/dist/activities/agent-turn/run.d.ts +3 -0
  21. package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
  22. package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
  23. package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
  24. package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
  25. package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
  26. package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
  27. package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
  28. package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
  29. package/dist/activities/agent-turn/turn-context.d.ts +144 -0
  30. package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
  31. package/dist/activities/agent-turn.d.ts +12 -954
  32. package/dist/activities/automations.d.ts +16 -0
  33. package/dist/activities/common.d.ts +1 -1
  34. package/dist/activities/connector-attachments.d.ts +13 -0
  35. package/dist/activities/documents.d.ts +1 -0
  36. package/dist/activities/environment.d.ts +26 -15
  37. package/dist/activities/generated-images.d.ts +1 -1
  38. package/dist/activities/git-credential-renewal.d.ts +3 -1
  39. package/dist/activities/goals.d.ts +4 -1
  40. package/dist/activities/google-drive-provider.d.ts +29 -0
  41. package/dist/activities/google-drive-publication.d.ts +66 -0
  42. package/dist/activities/image-generation-references.d.ts +2 -0
  43. package/dist/activities/incident-telemetry-authority.d.ts +52 -0
  44. package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
  45. package/dist/activities/knowledge-source-sync-driver.d.ts +21 -0
  46. package/dist/activities/knowledge-source-sync.d.ts +33 -1
  47. package/dist/activities/mcp-credentials.d.ts +9 -3
  48. package/dist/activities/open-suffix-resume.d.ts +77 -0
  49. package/dist/activities/packs.d.ts +3 -11
  50. package/dist/activities/parent-wake.d.ts +19 -0
  51. package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
  52. package/dist/activities/retained-screenshots.d.ts +1 -1
  53. package/dist/activities/rig-verification.d.ts +1 -1
  54. package/dist/activities/run-input.d.ts +16 -7
  55. package/dist/activities/sandbox-images.d.ts +1 -14
  56. package/dist/activities/sandbox-lease.d.ts +21 -3
  57. package/dist/activities/scheduled-tasks.d.ts +47 -0
  58. package/dist/activities/types.d.ts +23 -2
  59. package/dist/activities/video-media-validation.d.ts +1 -1
  60. package/dist/activities/video-output-retention.d.ts +1 -2
  61. package/dist/activities/xai-auth.d.ts +3 -0
  62. package/dist/{activities-control-SMKF2OBE.js → activities-control-DKG22ONJ.js} +2946 -549
  63. package/dist/activities-control-DKG22ONJ.js.map +1 -0
  64. package/dist/activities-control.d.ts +6 -2
  65. package/dist/activities-turn-4IMC3MAA.js +21326 -0
  66. package/dist/activities-turn-4IMC3MAA.js.map +1 -0
  67. package/dist/activities.d.ts +7 -3
  68. package/dist/artifact-materializer-entry.js +41 -22
  69. package/dist/artifact-materializer-entry.js.map +1 -1
  70. package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
  71. package/dist/chunk-CE7TPKBS.js.map +1 -0
  72. package/dist/chunk-CNYGVRKF.js +4698 -0
  73. package/dist/chunk-CNYGVRKF.js.map +1 -0
  74. package/dist/connection-provider-use.d.ts +18 -0
  75. package/dist/editable-artifact-materializer.d.ts +4 -3
  76. package/dist/github-rest-mcp.d.ts +26 -0
  77. package/dist/index.d.ts +10 -2
  78. package/dist/index.js +58 -14
  79. package/dist/index.js.map +1 -1
  80. package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
  81. package/dist/model-context-contributions.d.ts +8 -4
  82. package/dist/observability-metrics.d.ts +63 -4
  83. package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
  84. package/dist/personal-github-git-credentials.d.ts +9 -0
  85. package/dist/pr-review-credentials.d.ts +6 -0
  86. package/dist/sandbox-archive-storage.d.ts +45 -0
  87. package/dist/sandbox-resume.d.ts +65 -6
  88. package/dist/sandbox-routing.d.ts +38 -2
  89. package/dist/scheduled-alert-occurrence.d.ts +33 -0
  90. package/dist/workflow-bundle.js +87 -28
  91. package/dist/workflows/activities.d.ts +1 -0
  92. package/dist/workflows/automations.d.ts +6 -0
  93. package/dist/workflows/document-index.d.ts +1 -0
  94. package/dist/workflows.d.ts +1 -0
  95. package/package.json +17 -17
  96. package/src/activities/agent-turn/admission.ts +292 -0
  97. package/src/activities/agent-turn/agent-build.ts +802 -0
  98. package/src/activities/agent-turn/claim.ts +590 -0
  99. package/src/activities/agent-turn/codex-capacity.ts +720 -0
  100. package/src/activities/agent-turn/codex.ts +94 -0
  101. package/src/activities/agent-turn/compaction-prep.ts +722 -0
  102. package/src/activities/agent-turn/credential-leases.ts +240 -0
  103. package/src/activities/agent-turn/errors.ts +1068 -0
  104. package/src/activities/agent-turn/failure-settlement.ts +1436 -0
  105. package/src/activities/agent-turn/file-resources.ts +227 -0
  106. package/src/activities/agent-turn/finalization.ts +754 -0
  107. package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
  108. package/src/activities/agent-turn/governance-model.ts +465 -0
  109. package/src/activities/agent-turn/history-sink.ts +139 -0
  110. package/src/activities/agent-turn/history.ts +200 -0
  111. package/src/activities/agent-turn/media-artifacts.ts +359 -0
  112. package/src/activities/agent-turn/model-usage.ts +823 -0
  113. package/src/activities/agent-turn/quiescence.ts +353 -0
  114. package/src/activities/agent-turn/run-credentials.ts +761 -0
  115. package/src/activities/agent-turn/run.ts +1530 -0
  116. package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
  117. package/src/activities/agent-turn/sandbox-provision.ts +569 -0
  118. package/src/activities/agent-turn/sandbox-route.ts +358 -0
  119. package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
  120. package/src/activities/agent-turn/stream-attempt.ts +1670 -0
  121. package/src/activities/agent-turn/tool-environment.ts +729 -0
  122. package/src/activities/agent-turn/tool-policy.ts +216 -0
  123. package/src/activities/agent-turn/tool-result-spill.ts +265 -0
  124. package/src/activities/agent-turn/turn-context.ts +281 -0
  125. package/src/activities/agent-turn/xai-capacity.ts +229 -0
  126. package/src/activities/agent-turn.ts +12 -12575
  127. package/src/activities/api-integrations.ts +3 -0
  128. package/src/activities/automations.ts +214 -0
  129. package/src/activities/codemode-dispatcher.ts +12 -1
  130. package/src/activities/common.ts +1 -5
  131. package/src/activities/connector-attachments.ts +168 -0
  132. package/src/activities/context-compaction.ts +18 -0
  133. package/src/activities/documents.ts +6 -4
  134. package/src/activities/environment.ts +151 -55
  135. package/src/activities/generated-images.ts +26 -11
  136. package/src/activities/git-credential-renewal.ts +13 -3
  137. package/src/activities/goals.ts +85 -19
  138. package/src/activities/google-drive-provider.ts +178 -0
  139. package/src/activities/google-drive-publication.ts +938 -0
  140. package/src/activities/image-generation-references.ts +11 -2
  141. package/src/activities/incident-telemetry-authority.ts +327 -0
  142. package/src/activities/incident-telemetry-preflight.ts +317 -0
  143. package/src/activities/knowledge-source-sync-driver.ts +23 -0
  144. package/src/activities/knowledge-source-sync.ts +556 -40
  145. package/src/activities/mcp-credentials.ts +191 -17
  146. package/src/activities/open-suffix-resume.ts +422 -0
  147. package/src/activities/packs.ts +12 -60
  148. package/src/activities/parent-wake.ts +64 -14
  149. package/src/activities/retained-screenshot-reaper.ts +6 -3
  150. package/src/activities/retained-screenshots.ts +15 -47
  151. package/src/activities/rig-verification.ts +1 -1
  152. package/src/activities/run-input.ts +128 -136
  153. package/src/activities/sandbox-images.ts +7 -166
  154. package/src/activities/sandbox-lease.ts +406 -15
  155. package/src/activities/scheduled-tasks.ts +2103 -411
  156. package/src/activities/session-state.ts +10 -2
  157. package/src/activities/types.ts +32 -2
  158. package/src/activities/video-media-validation.ts +3 -3
  159. package/src/activities/video-output-retention.ts +2 -57
  160. package/src/activities/xai-auth.ts +8 -0
  161. package/src/activities-control.ts +2 -0
  162. package/src/activities.ts +3 -0
  163. package/src/activity-services.ts +13 -4
  164. package/src/connection-provider-use.ts +36 -0
  165. package/src/editable-artifact-materializer-subprocess.ts +25 -14
  166. package/src/editable-artifact-materializer.ts +29 -18
  167. package/src/github-rest-mcp.ts +713 -0
  168. package/src/http.ts +16 -6
  169. package/src/index.ts +40 -5
  170. package/src/knowledge-source-sync-activity-policy.ts +2 -0
  171. package/src/model-context-contributions.ts +33 -10
  172. package/src/observability-metrics.ts +357 -0
  173. package/src/opensandbox-kubernetes-inventory.ts +292 -0
  174. package/src/personal-github-git-credentials.ts +331 -0
  175. package/src/pr-review-credentials.ts +163 -0
  176. package/src/sandbox-archive-storage.ts +159 -0
  177. package/src/sandbox-resume.ts +449 -63
  178. package/src/sandbox-routing.ts +314 -16
  179. package/src/scheduled-alert-occurrence.ts +176 -0
  180. package/src/workflows/activities.ts +18 -2
  181. package/src/workflows/automations.ts +11 -0
  182. package/src/workflows/session.ts +13 -3
  183. package/src/workflows.ts +1 -0
  184. package/dist/activities-control-SMKF2OBE.js.map +0 -1
  185. package/dist/activities-turn-IDS3WEHA.js +0 -16532
  186. package/dist/activities-turn-IDS3WEHA.js.map +0 -1
  187. package/dist/chunk-N5TJIFCE.js.map +0 -1
  188. package/dist/chunk-P6Z2SLAA.js +0 -3092
  189. package/dist/chunk-P6Z2SLAA.js.map +0 -1
@@ -0,0 +1,731 @@
1
+ import {
2
+ advanceWorkspaceGeneration,
3
+ verifyWorkspaceMutationSettlement,
4
+ heartbeatLeaseHolder,
5
+ readLease,
6
+ accrueWarmSeconds,
7
+ abandonRecordingForTurnAttempt,
8
+ SandboxWorkspaceMutationFencedError,
9
+ } from "@opengeni/db";
10
+ import {
11
+ RoutingMutationOutcomeUnknownError,
12
+ type EstablishedSandboxSession,
13
+ stopRecording as stopRecordingOnBox,
14
+ } from "@opengeni/runtime";
15
+ import {
16
+ sandboxLifecycleTransitionWaitMs,
17
+ sandboxWarmRateMicrosPerSecond,
18
+ type Settings,
19
+ } from "@opengeni/config";
20
+ import { createRuntimeBatcher, currentActivityContext } from "../streaming";
21
+ import type { TurnActivityServices as ActivityServices, RunAgentTurnInput } from "../types";
22
+ import {
23
+ maybePersistWarmWorkspaceSnapshot,
24
+ persistSandboxDeadlineRotationCheckpoint,
25
+ waitForWarmSnapshot,
26
+ type ResumedTurnSandbox,
27
+ } from "../../sandbox-resume";
28
+ import { recordCreditMicros } from "../../observability-metrics";
29
+ import { beginRecording, discardUnpublishedRecording, type ActiveRecording } from "../recording";
30
+ import { ChannelAPartialMutationError } from "@opengeni/runtime/sandbox";
31
+ import { sandboxRunAs } from "@opengeni/runtime";
32
+ import { randomUUID } from "node:crypto";
33
+
34
+ import { safeErrorDiagnostic } from "./errors";
35
+ import {
36
+ shouldStartPeriodicWorkspaceSnapshot,
37
+ releaseTurnSandboxAfterWriterDrain,
38
+ finalizeDurableTurnOpStreams,
39
+ } from "./quiescence";
40
+ import { SandboxDeadlineRotationError } from "./sandbox-provision";
41
+ import { shouldStartOnTurnRecording } from "./tool-policy";
42
+ import type {
43
+ AttemptIdentityState,
44
+ EventingState,
45
+ RecordingState,
46
+ SandboxRuntimeState,
47
+ } from "./turn-context";
48
+
49
+ export type SandboxTurnRuntimeDeps = {
50
+ input: RunAgentTurnInput;
51
+ settings: Settings;
52
+ db: ActivityServices["db"];
53
+ objectStorage: ActivityServices["objectStorage"];
54
+ observability: ActivityServices["observability"];
55
+ cancellationSignal: AbortSignal | undefined;
56
+ activityContext: ReturnType<typeof currentActivityContext>;
57
+ sandboxRotationController: AbortController;
58
+ sandboxState: SandboxRuntimeState;
59
+ recordingState: RecordingState;
60
+ eventing: EventingState;
61
+ attempt: AttemptIdentityState;
62
+ };
63
+
64
+ export type SandboxTurnRuntime = ReturnType<typeof createSandboxTurnRuntime>;
65
+
66
+ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
67
+ const {
68
+ input,
69
+ settings,
70
+ db,
71
+ objectStorage,
72
+ observability,
73
+ cancellationSignal,
74
+ activityContext,
75
+ sandboxRotationController,
76
+ sandboxState,
77
+ recordingState,
78
+ eventing,
79
+ attempt,
80
+ } = deps;
81
+
82
+ // P1.2 ownership inversion: when sandboxOwnershipEnabled, the turn resolves
83
+ // the one box by id from the group lease and injects it NON-OWNED into the
84
+ // run. null when the flag is off (byte-for-byte the legacy build-and-discard
85
+ // path) OR when the backend is "none". Released + dropped in `finally`.
86
+ const releaseLateSandbox = async (sandbox: ResumedTurnSandbox): Promise<void> => {
87
+ sandboxState.lateSandboxesAwaitingWriterDrain.add(sandbox);
88
+ // Drop the holder/timer immediately, but keep its null-outcome admissions
89
+ // fenced until the shared attempt writer drain completes. The staged
90
+ // release serializes a later proof-bearing call behind this one.
91
+ await sandbox.release().catch(() => undefined);
92
+ if (!sandboxState.attemptWritersDrained) return;
93
+ try {
94
+ await releaseTurnSandboxAfterWriterDrain(sandbox);
95
+ sandboxState.lateSandboxesAwaitingWriterDrain.delete(sandbox);
96
+ } catch (error) {
97
+ console.error(
98
+ "late sandbox quiesced release failed (turn outcome unaffected)",
99
+ safeErrorDiagnostic(error),
100
+ );
101
+ }
102
+ };
103
+ const requireResolvedSandboxForMutation = (message: string): ResumedTurnSandbox => {
104
+ if (!sandboxState.resolvedSandbox) throw new Error(message);
105
+ return sandboxState.resolvedSandbox;
106
+ };
107
+ // The machine-primary SelfhostedSession (the UNWRAPPED backend, not the
108
+ // routing proxy). Kept as a fallback finalizer; the routing proxy normally
109
+ // aggregates it together with every machine reached after a mid-turn swap.
110
+ // The UN-PROXIED established box session, captured BEFORE wrapTurnBoxWithRouting.
111
+ // Platform setup (beforeAgentStart hooks + file materialization) execs against
112
+ // THIS handle so a mid-turn sandbox_swap can never re-route those execs onto a
113
+ // connected machine (the user's real computer).
114
+ const finalizeTurnOpStreamOps = async (): Promise<void> => {
115
+ await finalizeDurableTurnOpStreams(
116
+ [sandboxState.lazyOwnedSandbox?.session, sandboxState.resolvedSandbox?.established.session],
117
+ sandboxState.machinePrimarySession,
118
+ );
119
+ };
120
+ // A same-target API repair can replace the home provider while this turn is
121
+ // alive. Keep setup/snapshot persistence on the rebound raw session while
122
+ // preserving the SDK-owned routing proxy for eager turns. Lazy turns hold the
123
+ // proxy separately, so their worker-side handle may replace its raw session.
124
+ const onHomeSandboxRebound = (rebound: {
125
+ established: EstablishedSandboxSession;
126
+ leaseEpoch: number;
127
+ }): void => {
128
+ const current = sandboxState.resolvedSandbox;
129
+ const previousSession = current?.established.session;
130
+ const preserveRoutingProxy =
131
+ current !== null && previousSession !== sandboxState.setupBoxSession;
132
+ sandboxState.setupBoxSession = rebound.established.session;
133
+ if (!current) return;
134
+ current.leaseEpoch = rebound.leaseEpoch;
135
+ current.established = preserveRoutingProxy
136
+ ? {
137
+ ...current.established,
138
+ client: rebound.established.client,
139
+ // Keep the stable SDK-facing proxy; only its resolver changes the
140
+ // underlying backend. The worker's setupBoxSession above is raw.
141
+ session: previousSession,
142
+ sessionState: rebound.established.sessionState,
143
+ instanceId: rebound.established.instanceId,
144
+ backendId: rebound.established.backendId,
145
+ ...(rebound.established.origin ? { origin: rebound.established.origin } : {}),
146
+ ...(rebound.established.restoredArchive
147
+ ? { restoredArchive: rebound.established.restoredArchive }
148
+ : {}),
149
+ }
150
+ : rebound.established;
151
+ };
152
+ // The globally unique durable turn-attempt holder id + the group id,
153
+ // captured so the lease heartbeat can refresh the lease TTL epoch-fenced
154
+ // (a superseded owner self-evicts) and finally can release.
155
+ const runWorkspaceMutationForSandbox = async <T>(
156
+ sandbox: ResumedTurnSandbox,
157
+ operation: string,
158
+ mutation: () => Promise<T>,
159
+ observePhase?: (measurement: {
160
+ phase: "admission" | "provider" | "settlement" | "snapshot_wait";
161
+ outcome: "completed" | "failed";
162
+ durationSeconds: number;
163
+ }) => void,
164
+ ): Promise<T> => {
165
+ const observeMutationPhase = (
166
+ phase: "admission" | "provider" | "settlement" | "snapshot_wait",
167
+ outcome: "completed" | "failed",
168
+ durationMs: number,
169
+ ): void => {
170
+ try {
171
+ observePhase?.({
172
+ phase,
173
+ outcome,
174
+ durationSeconds: Math.max(0, durationMs) / 1_000,
175
+ });
176
+ } catch {
177
+ // Diagnostics must never alter workspace mutation authority.
178
+ }
179
+ };
180
+ // Connected machines are the user's own persistence and never dirty the
181
+ // cloud home archive. Every persistable raw-session write batch is fenced
182
+ // against the exact current lease/provider before the provider sees it.
183
+ if (sandbox.established.backendId === "selfhosted") {
184
+ const providerStartedAt = performance.now();
185
+ let providerOutcome: "completed" | "failed" = "failed";
186
+ try {
187
+ const result = await mutation();
188
+ providerOutcome = "completed";
189
+ return result;
190
+ } catch (error) {
191
+ if (error instanceof ChannelAPartialMutationError) {
192
+ throw new RoutingMutationOutcomeUnknownError(
193
+ operation,
194
+ `Connected Machine workspace mutation "${operation}" partially applied before a later batch item failed; the complete operation was not replayed`,
195
+ { cause: error },
196
+ );
197
+ }
198
+ throw error;
199
+ } finally {
200
+ observeMutationPhase("provider", providerOutcome, performance.now() - providerStartedAt);
201
+ }
202
+ }
203
+ if (
204
+ !sandboxState.sandboxGroupId ||
205
+ !sandboxState.sandboxHolderId ||
206
+ !attempt.turnId ||
207
+ attempt.executionGeneration <= 0
208
+ ) {
209
+ throw new Error("Workspace mutation attempted before exact turn sandbox admission");
210
+ }
211
+ let admissionCaptureWaitMs = 0;
212
+ const identity = {
213
+ accountId: input.accountId,
214
+ workspaceId: input.workspaceId,
215
+ sessionId: input.sessionId,
216
+ turnId: attempt.turnId,
217
+ executionGeneration: attempt.executionGeneration,
218
+ attemptId: input.attemptId,
219
+ holderId: sandboxState.sandboxHolderId,
220
+ sandboxGroupId: sandboxState.sandboxGroupId,
221
+ expectedEpoch: sandbox.leaseEpoch,
222
+ expectedInstanceId: sandbox.established.instanceId,
223
+ operation,
224
+ captureWaitMs: sandboxLifecycleTransitionWaitMs(settings),
225
+ ...(cancellationSignal ? { waitSignal: cancellationSignal } : {}),
226
+ ...(observePhase
227
+ ? {
228
+ onCaptureWait: (observation: {
229
+ durationMs: number;
230
+ outcome: "completed" | "failed";
231
+ }) => {
232
+ admissionCaptureWaitMs += Math.max(0, observation.durationMs);
233
+ observeMutationPhase("snapshot_wait", observation.outcome, observation.durationMs);
234
+ },
235
+ }
236
+ : {}),
237
+ };
238
+ const admissionStartedAt = performance.now();
239
+ let admissionOutcome: "completed" | "failed" = "failed";
240
+ let admission: Awaited<ReturnType<typeof advanceWorkspaceGeneration>>;
241
+ try {
242
+ admission = await advanceWorkspaceGeneration(db, identity);
243
+ admissionOutcome = "completed";
244
+ } catch (admissionError) {
245
+ if (
246
+ admissionError instanceof SandboxWorkspaceMutationFencedError &&
247
+ admissionError.code === "rotation_in_progress" &&
248
+ sandboxState.sandboxGroupId
249
+ ) {
250
+ // The lease still matches this turn's exact epoch/instance but a
251
+ // rotation was requested. Start the same orderly checkpoint the
252
+ // heartbeat would start on its next tick, so the model is not fed
253
+ // retry errors until then. Best-effort and single-flight; the
254
+ // admission failure itself is still surfaced to the caller.
255
+ void beginRotationCheckpoint(
256
+ sandbox,
257
+ sandbox.leaseEpoch,
258
+ sandboxState.sandboxGroupId,
259
+ ).catch(() => undefined);
260
+ }
261
+ throw admissionError;
262
+ } finally {
263
+ observeMutationPhase(
264
+ "admission",
265
+ admissionOutcome,
266
+ Math.max(0, performance.now() - admissionStartedAt - admissionCaptureWaitMs),
267
+ );
268
+ }
269
+ const settleMutation = async (outcome: "resolved" | "rejected"): Promise<void> => {
270
+ const settlementStartedAt = performance.now();
271
+ let settlementOutcome: "completed" | "failed" = "failed";
272
+ try {
273
+ await verifyWorkspaceMutationSettlement(db, {
274
+ ...identity,
275
+ admission,
276
+ outcome,
277
+ });
278
+ settlementOutcome = "completed";
279
+ } finally {
280
+ observeMutationPhase(
281
+ "settlement",
282
+ settlementOutcome,
283
+ performance.now() - settlementStartedAt,
284
+ );
285
+ }
286
+ };
287
+ let result: T;
288
+ const providerStartedAt = performance.now();
289
+ let providerOutcome: "completed" | "failed" = "failed";
290
+ try {
291
+ result = await mutation();
292
+ providerOutcome = "completed";
293
+ } catch (providerError) {
294
+ observeMutationPhase("provider", providerOutcome, performance.now() - providerStartedAt);
295
+ const partialMutation = providerError instanceof ChannelAPartialMutationError;
296
+ try {
297
+ await settleMutation(partialMutation ? "resolved" : "rejected");
298
+ } catch (settlementError) {
299
+ throw new RoutingMutationOutcomeUnknownError(
300
+ operation,
301
+ partialMutation
302
+ ? `Platform workspace mutation "${operation}" partially applied at the provider but lost its durable physical settlement; its outcome is unknown and it was not replayed`
303
+ : `Platform workspace mutation "${operation}" rejected at the provider but lost its durable physical settlement; its outcome is unknown and it was not replayed`,
304
+ { cause: settlementError },
305
+ );
306
+ }
307
+ if (partialMutation) {
308
+ throw new RoutingMutationOutcomeUnknownError(
309
+ operation,
310
+ `Platform workspace mutation "${operation}" partially applied before a later batch item failed; the complete operation was not replayed`,
311
+ { cause: providerError },
312
+ );
313
+ }
314
+ throw providerError;
315
+ }
316
+ observeMutationPhase("provider", providerOutcome, performance.now() - providerStartedAt);
317
+ try {
318
+ await settleMutation("resolved");
319
+ } catch (settlementError) {
320
+ throw new RoutingMutationOutcomeUnknownError(
321
+ operation,
322
+ `Platform workspace mutation "${operation}" returned from the provider but lost its durable settlement fence; its outcome is unknown and it was not replayed`,
323
+ { cause: settlementError },
324
+ );
325
+ }
326
+ return result;
327
+ };
328
+ // Lease-TTL refresh timer (parallels the activity heartbeat): while the turn
329
+ // runs it refreshes expires_at epoch-fenced so a legit multi-day turn is
330
+ // never TTL-reaped. Cleared in finally. Only set when the flag resolved a box.
331
+ const stopLeaseHeartbeat = (): void => {
332
+ if (!sandboxState.leaseHeartbeatTimer) return;
333
+ clearInterval(sandboxState.leaseHeartbeatTimer);
334
+ sandboxState.leaseHeartbeatTimer = undefined;
335
+ };
336
+ // credential-renewal policy: the worker, not the model, owns renewal of run-scoped Git
337
+ // credentials for a multi-day turn. The controller is attached only after
338
+ // the initial seed reached a real cloud box and is drained before capture.
339
+ // Generic host-owned run material has its own attempt-scoped renewal and
340
+ // write handle. It is always drained and wiped before workspace capture.
341
+ // The delegated Codemode bearer has a one-hour TTL. Renewal is attempt-
342
+ // owned and attaches only after the initial token file reached a real
343
+ // sandbox session; finalization drains an in-flight replacement.
344
+ // MID-SESSION snapshot single-flight guard: the heartbeat tick fires every
345
+ // 10s but a Modal filesystem snapshot can take longer — never overlap two
346
+ // captures on one box. The in-flight capture's promise is held so the
347
+ // turn-end persist can await it (its capture predates the turn's final
348
+ // writes; landing after the fresher turn-end capture started would make
349
+ // the atomic DB throttle discard the fresher one). Interval throttling
350
+ // itself lives in maybePersistWarmWorkspaceSnapshot / persistWarmSnapshot.
351
+ // The heartbeat snapshot is mid-session durability, not first-request
352
+ // preparation. Keep it off the startup critical path until a provider
353
+ // request has actually reached its transport boundary.
354
+ // Turn-end capture needs the lease heartbeat to keep its holder alive, but
355
+ // must prevent that same timer from starting another periodic snapshot
356
+ // while it reads. This gate separates those two responsibilities.
357
+ // Computer-use-only recording. Ordinary shell/filesystem turns leave this
358
+ // null; the first actual computer action starts it after :0 is ready.
359
+ // P4.3 recording gate: flips true in `onComputerUseReady`, the runtime's
360
+ // execution-time callback for the first real computer action. It must flip
361
+ // BEFORE awaiting recording startup: the SDK tool-call stream item can arrive
362
+ // before ffmpeg has finished starting. A plain text turn ("hey"/"continue")
363
+ // never invokes the callback, so settlement performs no storage PUT.
364
+ const abandonActiveRecording = async (
365
+ reason: string,
366
+ disposition: "failed" | "discard" = "failed",
367
+ ): Promise<void> => {
368
+ const recording = recordingState.activeRecording as ActiveRecording | null;
369
+ if (!recording) return;
370
+ recordingState.activeRecording = null;
371
+ if (sandboxState.resolvedSandbox) {
372
+ await stopRecordingOnBox(
373
+ sandboxState.resolvedSandbox.established.session,
374
+ recording.proc,
375
+ ).catch(() => undefined);
376
+ }
377
+ if (!attempt.turnId || attempt.executionGeneration <= 0) return;
378
+ await abandonRecordingForTurnAttempt(db, {
379
+ accountId: input.accountId,
380
+ workspaceId: input.workspaceId,
381
+ sessionId: input.sessionId,
382
+ turnId: attempt.turnId,
383
+ executionGeneration: attempt.executionGeneration,
384
+ attemptId: input.attemptId,
385
+ recordingId: recording.recordingId,
386
+ disposition,
387
+ reason,
388
+ }).catch(() => undefined);
389
+ };
390
+ const flushRuntimeBatcher = async () => {
391
+ const current = eventing.batcher as ReturnType<typeof createRuntimeBatcher> | null;
392
+ await current?.flush().catch(() => undefined);
393
+ };
394
+ // Reconciliation is declared before provider routing so every turn-end path
395
+ // can share one closure. It cannot run until `stream` exists, by which time
396
+ // this value has been rebound to the turn's resolved model policy.
397
+ const publishSandboxLifecycleEvents = async (sandbox: ResumedTurnSandbox): Promise<void> => {
398
+ const established = sandbox.established;
399
+ if (eventing.publish && established.origin && established.origin !== "resumed") {
400
+ const lifecycleEvents: Array<{
401
+ type: "sandbox.box.lost" | "sandbox.box.created";
402
+ payload: unknown;
403
+ }> = [];
404
+ if (established.lostInstanceId) {
405
+ lifecycleEvents.push({
406
+ type: "sandbox.box.lost",
407
+ payload: { sandboxId: established.lostInstanceId },
408
+ });
409
+ }
410
+ lifecycleEvents.push({
411
+ type: "sandbox.box.created",
412
+ payload: {
413
+ sandboxId: established.instanceId,
414
+ hydrated: established.origin === "restored" ? "archive" : "none",
415
+ },
416
+ });
417
+ await eventing.publish(lifecycleEvents).catch(() => undefined);
418
+ }
419
+ };
420
+ const publishSandboxLost = async (lostSandbox: { instanceId: string }): Promise<void> => {
421
+ if (!eventing.publish) return;
422
+ await eventing
423
+ .publish([
424
+ {
425
+ type: "sandbox.box.lost",
426
+ payload: { sandboxId: lostSandbox.instanceId },
427
+ },
428
+ ])
429
+ .catch((publishError) => {
430
+ // The lease transition is already authoritative. A fenced/failed audit
431
+ // append must not prevent the same logical turn from recovering.
432
+ console.error("sandbox box lost event publish failed", safeErrorDiagnostic(publishError));
433
+ });
434
+ };
435
+ /**
436
+ * Start (single-flight) the orderly provider-deadline rotation checkpoint for
437
+ * the established box. Entered from the lease heartbeat once the lease is
438
+ * rotation-fenced, and from a workspace-mutation admission that observed
439
+ * `rotation_in_progress` at the same epoch/instance, so the model is not fed
440
+ * retry errors for a whole heartbeat interval before the checkpoint runs.
441
+ * `not_rotating` means the exact epoch/instance no longer carries a requested
442
+ * rotation (holder reaped, attempt closed, epoch superseded, or draining).
443
+ */
444
+ const beginRotationCheckpoint = async (
445
+ sandbox: ResumedTurnSandbox,
446
+ rotationEpoch: number,
447
+ rotationGroupId: string,
448
+ ): Promise<"started" | "busy" | "not_rotating"> => {
449
+ // A cancelled attempt never starts (or reinstates a holder for) a
450
+ // checkpoint; its cancellation settlement owns the holder from here.
451
+ if (
452
+ sandboxState.rotationInFlight ||
453
+ sandboxRotationController.signal.aborted ||
454
+ cancellationSignal?.aborted
455
+ ) {
456
+ return "busy";
457
+ }
458
+ const rotatingLease = await readLease(db, input.workspaceId, rotationGroupId).catch(() => null);
459
+ if (
460
+ !rotatingLease ||
461
+ rotatingLease.leaseEpoch !== rotationEpoch ||
462
+ rotatingLease.instanceId !== sandbox.established.instanceId ||
463
+ rotatingLease.rotationRequestedAt === null
464
+ ) {
465
+ return "not_rotating";
466
+ }
467
+ if (sandboxState.rotationInFlight || sandboxRotationController.signal.aborted) return "busy";
468
+ sandboxState.rotationInFlight = (async () => {
469
+ // Rotation admission already fenced all new workspace mutations.
470
+ // Wait for an earlier periodic capture, then produce the exact
471
+ // generation-complete checkpoint that licenses aborting this run.
472
+ if (sandboxState.snapshotInFlight) {
473
+ await waitForWarmSnapshot(
474
+ sandboxState.snapshotInFlight,
475
+ settings.sandboxSnapshotTimeoutMs,
476
+ cancellationSignal,
477
+ );
478
+ }
479
+ const snapshotSession = sandboxState.setupBoxSession;
480
+ const snapshotTurnId = attempt.turnId;
481
+ if (!snapshotSession || !snapshotTurnId) return;
482
+ // The checkpoint helper first reinstates this turn's exact holder
483
+ // when it was lost at the same epoch/instance (defense in depth:
484
+ // the warm capture requires it), then forces the capture and
485
+ // reports whether the established epoch now carries a complete
486
+ // archive under its requested rotation.
487
+ const checkpoint = await persistSandboxDeadlineRotationCheckpoint(
488
+ { db, settings, objectStorage, observability },
489
+ {
490
+ accountId: input.accountId,
491
+ workspaceId: input.workspaceId,
492
+ sessionId: input.sessionId,
493
+ turnId: snapshotTurnId,
494
+ attemptId: input.attemptId,
495
+ sandboxGroupId: rotationGroupId,
496
+ },
497
+ snapshotSession,
498
+ { leaseEpoch: rotationEpoch, instanceId: sandbox.established.instanceId },
499
+ cancellationSignal,
500
+ );
501
+ if (checkpoint.checkpointed) {
502
+ sandboxRotationController.abort(
503
+ new SandboxDeadlineRotationError(rotationGroupId, rotationEpoch),
504
+ );
505
+ return;
506
+ }
507
+ if (checkpoint.holder === "attempt_fenced" || checkpoint.holder === "lease_fenced") {
508
+ // This attempt is no longer the active writer, or the exact
509
+ // epoch/instance is gone. Nothing left to checkpoint here.
510
+ stopLeaseHeartbeat();
511
+ }
512
+ })()
513
+ .catch((error) => {
514
+ observability.warn("sandbox deadline rotation checkpoint failed; retrying", {
515
+ workspaceId: input.workspaceId,
516
+ sessionId: input.sessionId,
517
+ sandboxGroupId: rotationGroupId,
518
+ leaseEpoch: rotationEpoch,
519
+ ...safeErrorDiagnostic(error),
520
+ });
521
+ })
522
+ .finally(() => {
523
+ sandboxState.rotationInFlight = null;
524
+ });
525
+ await sandboxState.rotationInFlight;
526
+ return "started";
527
+ };
528
+ const startLeaseHeartbeat = (
529
+ sandbox: ResumedTurnSandbox,
530
+ warmBackend: Settings["sandboxBackend"] | undefined,
531
+ ): void => {
532
+ if (sandboxState.leaseHeartbeatTimer) return;
533
+ if (!sandboxState.sandboxHolderId || !sandboxState.sandboxGroupId) {
534
+ return;
535
+ }
536
+ // Refresh the lease TTL on the activity-heartbeat cadence (10s, well
537
+ // inside the 90s lease TTL). EPOCH-FENCED: a superseded owner's refresh
538
+ // is rejected (returns false) and we stop refreshing — the box rides the
539
+ // provider idle-timeout and the next dispatch re-establishes it. Best-
540
+ // effort: a transient DB error must never fail the turn.
541
+ const heartbeatEpoch = sandbox.leaseEpoch;
542
+ const heartbeatHolderId = sandboxState.sandboxHolderId;
543
+ const heartbeatGroupId = sandboxState.sandboxGroupId;
544
+ // P2.1 warm-meter (tick A): while a turn runs, the heartbeat is also the
545
+ // warm-seconds tick. GROUP+epoch+tick keyed (one box = one stream, shared
546
+ // box metered once); epoch-fenced (a stale tick no-ops). Warm-cost is
547
+ // metered when a per-backend rate is configured. Best-effort: a metering
548
+ // failure must never fail the turn.
549
+ //
550
+ // Keyed off the EFFECTIVE backend (Stage D): a machine-primary turn has NO
551
+ // Modal box, so it must accrue ZERO cloud warm-seconds — `selfhosted` has no
552
+ // configured warm rate (0). Keying off turn.sandboxBackend (modal) would bill
553
+ // cloud seconds for a box that does not exist (a real money bug). Non-machine
554
+ // turns fall back to groupBoxBackend (the REAL box that ran): for a machine-
555
+ // home turn that degraded to the cloud group box (swap-away / flag-off), that
556
+ // is the deployment default (modal), so the fallback box is warm-metered at
557
+ // the cloud rate instead of selfhosted's rate-0 (which would under-bill).
558
+ const warmRate = sandboxWarmRateMicrosPerSecond(
559
+ settings,
560
+ warmBackend ?? (sandbox.established.backendId as Settings["sandboxBackend"]),
561
+ );
562
+ sandboxState.leaseHeartbeatTimer = setInterval(() => {
563
+ void heartbeatLeaseHolder(db, {
564
+ accountId: input.accountId,
565
+ workspaceId: input.workspaceId,
566
+ sandboxGroupId: heartbeatGroupId,
567
+ kind: "turn",
568
+ holderId: heartbeatHolderId,
569
+ leaseTtlMs: settings.sandboxLeaseTtlMs,
570
+ expectedEpoch: heartbeatEpoch,
571
+ })
572
+ .then(async (alive) => {
573
+ if (alive) return;
574
+ const rotation = await beginRotationCheckpoint(sandbox, heartbeatEpoch, heartbeatGroupId);
575
+ if (rotation === "not_rotating") {
576
+ // The holder was reaped, the exact attempt closed, the epoch was
577
+ // superseded, or the lease began draining. Do not leave a dead
578
+ // interval issuing DB writes and snapshot probes forever.
579
+ stopLeaseHeartbeat();
580
+ }
581
+ })
582
+ .catch(() => undefined);
583
+ void accrueWarmSeconds(db, {
584
+ accountId: input.accountId,
585
+ workspaceId: input.workspaceId,
586
+ sandboxGroupId: heartbeatGroupId,
587
+ expectedEpoch: heartbeatEpoch,
588
+ warmRateMicrosPerSecond: warmRate,
589
+ subjectId: input.sessionId,
590
+ })
591
+ .then((result) => recordCreditMicros(observability, "usage", result.costMicros))
592
+ .catch(() => undefined);
593
+ // MID-SESSION snapshot (sandbox-file-persistence): while the turn holds
594
+ // the box, fold a fresh /workspace snapshot onto the lease every
595
+ // sandboxSnapshotIntervalMs, so a box death the reaper never sees
596
+ // (Modal hard timeout mid-busy, OOM, infra) costs at most one interval
597
+ // of work — a legit multi-day turn is otherwise completely unprotected
598
+ // (the reaper only drain-persists IDLE leases). Uses the UN-proxied box
599
+ // session (setupBoxSession): the routing veneer could swap mid-op and a
600
+ // selfhosted target has no persistWorkspace anyway. Best-effort +
601
+ // single-flight; throttling lives in the helper.
602
+ const snapshotSession = sandboxState.setupBoxSession;
603
+ const snapshotTurnId = attempt.turnId;
604
+ if (
605
+ snapshotSession &&
606
+ snapshotTurnId &&
607
+ shouldStartPeriodicWorkspaceSnapshot({
608
+ firstProviderRequestStarted: sandboxState.firstProviderRequestStarted,
609
+ snapshotInFlight: Boolean(sandboxState.snapshotInFlight),
610
+ turnEndCaptureInProgress: sandboxState.turnEndCaptureInProgress,
611
+ })
612
+ ) {
613
+ sandboxState.snapshotInFlight = maybePersistWarmWorkspaceSnapshot(
614
+ { db, settings, objectStorage },
615
+ {
616
+ accountId: input.accountId,
617
+ workspaceId: input.workspaceId,
618
+ sessionId: input.sessionId,
619
+ turnId: snapshotTurnId,
620
+ attemptId: input.attemptId,
621
+ sandboxGroupId: heartbeatGroupId,
622
+ },
623
+ snapshotSession,
624
+ heartbeatEpoch,
625
+ activityContext?.cancellationSignal,
626
+ )
627
+ .then(async (persisted) => {
628
+ if (persisted && eventing.publish) {
629
+ await eventing.publish([
630
+ {
631
+ type: "sandbox.box.snapshot",
632
+ payload: { trigger: "heartbeat" },
633
+ },
634
+ ]);
635
+ }
636
+ })
637
+ .catch(() => undefined)
638
+ .finally(() => {
639
+ sandboxState.snapshotInFlight = null;
640
+ });
641
+ }
642
+ }, 10_000);
643
+ if (
644
+ "unref" in sandboxState.leaseHeartbeatTimer &&
645
+ typeof sandboxState.leaseHeartbeatTimer.unref === "function"
646
+ ) {
647
+ sandboxState.leaseHeartbeatTimer.unref();
648
+ }
649
+ };
650
+ const maybeStartOnTurnRecording = async (
651
+ sandbox: ResumedTurnSandbox,
652
+ effectiveBackend: Settings["sandboxBackend"] | undefined,
653
+ ): Promise<void> => {
654
+ if (recordingState.activeRecording) {
655
+ return;
656
+ }
657
+ if (recordingState.computerUseRecordingStart) {
658
+ await recordingState.computerUseRecordingStart;
659
+ return;
660
+ }
661
+ // Called only by the runtime's first-computer-action hook. Plain sandbox
662
+ // operations never start ffmpeg and never boot a display merely to record
663
+ // an unused desktop. Recording failure never fails the computer action.
664
+ if (
665
+ shouldStartOnTurnRecording({
666
+ recordingEnabled: settings.recordingEnabled,
667
+ desktopEnabled: settings.sandboxDesktopEnabled,
668
+ establishedBackendId: sandbox.established.backendId,
669
+ // EFFECTIVE (active) backend, not the session home: a machine-primary turn
670
+ // resolves to "selfhosted" and skips; a swap back to the cloud group box
671
+ // resolves to undefined and records as before.
672
+ effectiveBackend,
673
+ })
674
+ ) {
675
+ recordingState.computerUseRecordingStart = (async () => {
676
+ let begun: Awaited<ReturnType<typeof beginRecording>> | null = null;
677
+ try {
678
+ begun = await beginRecording({
679
+ settings,
680
+ db,
681
+ accountId: input.accountId,
682
+ workspaceId: input.workspaceId,
683
+ sessionId: input.sessionId,
684
+ turnId: attempt.turnId!,
685
+ recordingId: randomUUID(),
686
+ mode: "on-turn",
687
+ session: sandbox.established.session,
688
+ runAs: sandboxRunAs(settings),
689
+ reason: null,
690
+ });
691
+ if (!eventing.publish) {
692
+ throw new Error("recording started before the turn event publisher was ready");
693
+ }
694
+ await eventing.publish([{ type: "recording.started", payload: begun.started }]);
695
+ recordingState.activeRecording = begun.active;
696
+ } catch (recordingError) {
697
+ recordingState.activeRecording = null;
698
+ if (begun) {
699
+ await discardUnpublishedRecording({
700
+ db,
701
+ accountId: input.accountId,
702
+ workspaceId: input.workspaceId,
703
+ active: begun.active,
704
+ session: sandbox.established.session,
705
+ });
706
+ }
707
+ console.error(
708
+ "computer-use recording start failed (action outcome unaffected)",
709
+ safeErrorDiagnostic(recordingError),
710
+ );
711
+ }
712
+ })();
713
+ await recordingState.computerUseRecordingStart;
714
+ }
715
+ };
716
+
717
+ return {
718
+ releaseLateSandbox,
719
+ requireResolvedSandboxForMutation,
720
+ finalizeTurnOpStreamOps,
721
+ onHomeSandboxRebound,
722
+ runWorkspaceMutationForSandbox,
723
+ stopLeaseHeartbeat,
724
+ abandonActiveRecording,
725
+ flushRuntimeBatcher,
726
+ publishSandboxLifecycleEvents,
727
+ publishSandboxLost,
728
+ startLeaseHeartbeat,
729
+ maybeStartOnTurnRecording,
730
+ };
731
+ }