@opengeni/worker-bundle 0.20.1 → 0.22.0-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +52 -0
- package/dist/activities/agent-turn/agent-build.d.ts +89 -0
- package/dist/activities/agent-turn/claim.d.ts +67 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
- package/dist/activities/agent-turn/codex.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
- package/dist/activities/agent-turn/errors.d.ts +253 -0
- package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
- package/dist/activities/agent-turn/file-resources.d.ts +37 -0
- package/dist/activities/agent-turn/finalization.d.ts +42 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
- package/dist/activities/agent-turn/governance-model.d.ts +66 -0
- package/dist/activities/agent-turn/history-sink.d.ts +52 -0
- package/dist/activities/agent-turn/history.d.ts +63 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
- package/dist/activities/agent-turn/model-usage.d.ts +218 -0
- package/dist/activities/agent-turn/quiescence.d.ts +135 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
- package/dist/activities/agent-turn/run.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
- package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
- package/dist/activities/agent-turn/turn-context.d.ts +144 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
- package/dist/activities/agent-turn.d.ts +12 -954
- package/dist/activities/automations.d.ts +16 -0
- package/dist/activities/common.d.ts +1 -1
- package/dist/activities/connector-attachments.d.ts +13 -0
- package/dist/activities/documents.d.ts +1 -0
- package/dist/activities/environment.d.ts +26 -15
- package/dist/activities/generated-images.d.ts +1 -1
- package/dist/activities/git-credential-renewal.d.ts +3 -1
- package/dist/activities/goals.d.ts +4 -1
- package/dist/activities/google-drive-provider.d.ts +29 -0
- package/dist/activities/google-drive-publication.d.ts +66 -0
- package/dist/activities/image-generation-references.d.ts +2 -0
- package/dist/activities/incident-telemetry-authority.d.ts +52 -0
- package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +21 -0
- package/dist/activities/knowledge-source-sync.d.ts +33 -1
- package/dist/activities/mcp-credentials.d.ts +9 -3
- package/dist/activities/open-suffix-resume.d.ts +77 -0
- package/dist/activities/packs.d.ts +3 -11
- package/dist/activities/parent-wake.d.ts +19 -0
- package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
- package/dist/activities/retained-screenshots.d.ts +1 -1
- package/dist/activities/rig-verification.d.ts +1 -1
- package/dist/activities/run-input.d.ts +16 -7
- package/dist/activities/sandbox-images.d.ts +1 -14
- package/dist/activities/sandbox-lease.d.ts +21 -3
- package/dist/activities/scheduled-tasks.d.ts +47 -0
- package/dist/activities/types.d.ts +23 -2
- package/dist/activities/video-media-validation.d.ts +1 -1
- package/dist/activities/video-output-retention.d.ts +1 -2
- package/dist/activities/xai-auth.d.ts +3 -0
- package/dist/{activities-control-SMKF2OBE.js → activities-control-DKG22ONJ.js} +2946 -549
- package/dist/activities-control-DKG22ONJ.js.map +1 -0
- package/dist/activities-control.d.ts +6 -2
- package/dist/activities-turn-4IMC3MAA.js +21326 -0
- package/dist/activities-turn-4IMC3MAA.js.map +1 -0
- package/dist/activities.d.ts +7 -3
- package/dist/artifact-materializer-entry.js +41 -22
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
- package/dist/chunk-CE7TPKBS.js.map +1 -0
- package/dist/chunk-CNYGVRKF.js +4698 -0
- package/dist/chunk-CNYGVRKF.js.map +1 -0
- package/dist/connection-provider-use.d.ts +18 -0
- package/dist/editable-artifact-materializer.d.ts +4 -3
- package/dist/github-rest-mcp.d.ts +26 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +58 -14
- package/dist/index.js.map +1 -1
- package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
- package/dist/model-context-contributions.d.ts +8 -4
- package/dist/observability-metrics.d.ts +63 -4
- package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
- package/dist/personal-github-git-credentials.d.ts +9 -0
- package/dist/pr-review-credentials.d.ts +6 -0
- package/dist/sandbox-archive-storage.d.ts +45 -0
- package/dist/sandbox-resume.d.ts +65 -6
- package/dist/sandbox-routing.d.ts +38 -2
- package/dist/scheduled-alert-occurrence.d.ts +33 -0
- package/dist/workflow-bundle.js +87 -28
- package/dist/workflows/activities.d.ts +1 -0
- package/dist/workflows/automations.d.ts +6 -0
- package/dist/workflows/document-index.d.ts +1 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/admission.ts +292 -0
- package/src/activities/agent-turn/agent-build.ts +802 -0
- package/src/activities/agent-turn/claim.ts +590 -0
- package/src/activities/agent-turn/codex-capacity.ts +720 -0
- package/src/activities/agent-turn/codex.ts +94 -0
- package/src/activities/agent-turn/compaction-prep.ts +722 -0
- package/src/activities/agent-turn/credential-leases.ts +240 -0
- package/src/activities/agent-turn/errors.ts +1068 -0
- package/src/activities/agent-turn/failure-settlement.ts +1436 -0
- package/src/activities/agent-turn/file-resources.ts +227 -0
- package/src/activities/agent-turn/finalization.ts +754 -0
- package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
- package/src/activities/agent-turn/governance-model.ts +465 -0
- package/src/activities/agent-turn/history-sink.ts +139 -0
- package/src/activities/agent-turn/history.ts +200 -0
- package/src/activities/agent-turn/media-artifacts.ts +359 -0
- package/src/activities/agent-turn/model-usage.ts +823 -0
- package/src/activities/agent-turn/quiescence.ts +353 -0
- package/src/activities/agent-turn/run-credentials.ts +761 -0
- package/src/activities/agent-turn/run.ts +1530 -0
- package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
- package/src/activities/agent-turn/sandbox-provision.ts +569 -0
- package/src/activities/agent-turn/sandbox-route.ts +358 -0
- package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
- package/src/activities/agent-turn/stream-attempt.ts +1670 -0
- package/src/activities/agent-turn/tool-environment.ts +729 -0
- package/src/activities/agent-turn/tool-policy.ts +216 -0
- package/src/activities/agent-turn/tool-result-spill.ts +265 -0
- package/src/activities/agent-turn/turn-context.ts +281 -0
- package/src/activities/agent-turn/xai-capacity.ts +229 -0
- package/src/activities/agent-turn.ts +12 -12575
- package/src/activities/api-integrations.ts +3 -0
- package/src/activities/automations.ts +214 -0
- package/src/activities/codemode-dispatcher.ts +12 -1
- package/src/activities/common.ts +1 -5
- package/src/activities/connector-attachments.ts +168 -0
- package/src/activities/context-compaction.ts +18 -0
- package/src/activities/documents.ts +6 -4
- package/src/activities/environment.ts +151 -55
- package/src/activities/generated-images.ts +26 -11
- package/src/activities/git-credential-renewal.ts +13 -3
- package/src/activities/goals.ts +85 -19
- package/src/activities/google-drive-provider.ts +178 -0
- package/src/activities/google-drive-publication.ts +938 -0
- package/src/activities/image-generation-references.ts +11 -2
- package/src/activities/incident-telemetry-authority.ts +327 -0
- package/src/activities/incident-telemetry-preflight.ts +317 -0
- package/src/activities/knowledge-source-sync-driver.ts +23 -0
- package/src/activities/knowledge-source-sync.ts +556 -40
- package/src/activities/mcp-credentials.ts +191 -17
- package/src/activities/open-suffix-resume.ts +422 -0
- package/src/activities/packs.ts +12 -60
- package/src/activities/parent-wake.ts +64 -14
- package/src/activities/retained-screenshot-reaper.ts +6 -3
- package/src/activities/retained-screenshots.ts +15 -47
- package/src/activities/rig-verification.ts +1 -1
- package/src/activities/run-input.ts +128 -136
- package/src/activities/sandbox-images.ts +7 -166
- package/src/activities/sandbox-lease.ts +406 -15
- package/src/activities/scheduled-tasks.ts +2103 -411
- package/src/activities/session-state.ts +10 -2
- package/src/activities/types.ts +32 -2
- package/src/activities/video-media-validation.ts +3 -3
- package/src/activities/video-output-retention.ts +2 -57
- package/src/activities/xai-auth.ts +8 -0
- package/src/activities-control.ts +2 -0
- package/src/activities.ts +3 -0
- package/src/activity-services.ts +13 -4
- package/src/connection-provider-use.ts +36 -0
- package/src/editable-artifact-materializer-subprocess.ts +25 -14
- package/src/editable-artifact-materializer.ts +29 -18
- package/src/github-rest-mcp.ts +713 -0
- package/src/http.ts +16 -6
- package/src/index.ts +40 -5
- package/src/knowledge-source-sync-activity-policy.ts +2 -0
- package/src/model-context-contributions.ts +33 -10
- package/src/observability-metrics.ts +357 -0
- package/src/opensandbox-kubernetes-inventory.ts +292 -0
- package/src/personal-github-git-credentials.ts +331 -0
- package/src/pr-review-credentials.ts +163 -0
- package/src/sandbox-archive-storage.ts +159 -0
- package/src/sandbox-resume.ts +449 -63
- package/src/sandbox-routing.ts +314 -16
- package/src/scheduled-alert-occurrence.ts +176 -0
- package/src/workflows/activities.ts +18 -2
- package/src/workflows/automations.ts +11 -0
- package/src/workflows/session.ts +13 -3
- package/src/workflows.ts +1 -0
- package/dist/activities-control-SMKF2OBE.js.map +0 -1
- package/dist/activities-turn-IDS3WEHA.js +0 -16532
- package/dist/activities-turn-IDS3WEHA.js.map +0 -1
- package/dist/chunk-N5TJIFCE.js.map +0 -1
- package/dist/chunk-P6Z2SLAA.js +0 -3092
- package/dist/chunk-P6Z2SLAA.js.map +0 -1
|
@@ -0,0 +1,731 @@
|
|
|
1
|
+
import {
|
|
2
|
+
advanceWorkspaceGeneration,
|
|
3
|
+
verifyWorkspaceMutationSettlement,
|
|
4
|
+
heartbeatLeaseHolder,
|
|
5
|
+
readLease,
|
|
6
|
+
accrueWarmSeconds,
|
|
7
|
+
abandonRecordingForTurnAttempt,
|
|
8
|
+
SandboxWorkspaceMutationFencedError,
|
|
9
|
+
} from "@opengeni/db";
|
|
10
|
+
import {
|
|
11
|
+
RoutingMutationOutcomeUnknownError,
|
|
12
|
+
type EstablishedSandboxSession,
|
|
13
|
+
stopRecording as stopRecordingOnBox,
|
|
14
|
+
} from "@opengeni/runtime";
|
|
15
|
+
import {
|
|
16
|
+
sandboxLifecycleTransitionWaitMs,
|
|
17
|
+
sandboxWarmRateMicrosPerSecond,
|
|
18
|
+
type Settings,
|
|
19
|
+
} from "@opengeni/config";
|
|
20
|
+
import { createRuntimeBatcher, currentActivityContext } from "../streaming";
|
|
21
|
+
import type { TurnActivityServices as ActivityServices, RunAgentTurnInput } from "../types";
|
|
22
|
+
import {
|
|
23
|
+
maybePersistWarmWorkspaceSnapshot,
|
|
24
|
+
persistSandboxDeadlineRotationCheckpoint,
|
|
25
|
+
waitForWarmSnapshot,
|
|
26
|
+
type ResumedTurnSandbox,
|
|
27
|
+
} from "../../sandbox-resume";
|
|
28
|
+
import { recordCreditMicros } from "../../observability-metrics";
|
|
29
|
+
import { beginRecording, discardUnpublishedRecording, type ActiveRecording } from "../recording";
|
|
30
|
+
import { ChannelAPartialMutationError } from "@opengeni/runtime/sandbox";
|
|
31
|
+
import { sandboxRunAs } from "@opengeni/runtime";
|
|
32
|
+
import { randomUUID } from "node:crypto";
|
|
33
|
+
|
|
34
|
+
import { safeErrorDiagnostic } from "./errors";
|
|
35
|
+
import {
|
|
36
|
+
shouldStartPeriodicWorkspaceSnapshot,
|
|
37
|
+
releaseTurnSandboxAfterWriterDrain,
|
|
38
|
+
finalizeDurableTurnOpStreams,
|
|
39
|
+
} from "./quiescence";
|
|
40
|
+
import { SandboxDeadlineRotationError } from "./sandbox-provision";
|
|
41
|
+
import { shouldStartOnTurnRecording } from "./tool-policy";
|
|
42
|
+
import type {
|
|
43
|
+
AttemptIdentityState,
|
|
44
|
+
EventingState,
|
|
45
|
+
RecordingState,
|
|
46
|
+
SandboxRuntimeState,
|
|
47
|
+
} from "./turn-context";
|
|
48
|
+
|
|
49
|
+
export type SandboxTurnRuntimeDeps = {
|
|
50
|
+
input: RunAgentTurnInput;
|
|
51
|
+
settings: Settings;
|
|
52
|
+
db: ActivityServices["db"];
|
|
53
|
+
objectStorage: ActivityServices["objectStorage"];
|
|
54
|
+
observability: ActivityServices["observability"];
|
|
55
|
+
cancellationSignal: AbortSignal | undefined;
|
|
56
|
+
activityContext: ReturnType<typeof currentActivityContext>;
|
|
57
|
+
sandboxRotationController: AbortController;
|
|
58
|
+
sandboxState: SandboxRuntimeState;
|
|
59
|
+
recordingState: RecordingState;
|
|
60
|
+
eventing: EventingState;
|
|
61
|
+
attempt: AttemptIdentityState;
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
export type SandboxTurnRuntime = ReturnType<typeof createSandboxTurnRuntime>;
|
|
65
|
+
|
|
66
|
+
export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
|
|
67
|
+
const {
|
|
68
|
+
input,
|
|
69
|
+
settings,
|
|
70
|
+
db,
|
|
71
|
+
objectStorage,
|
|
72
|
+
observability,
|
|
73
|
+
cancellationSignal,
|
|
74
|
+
activityContext,
|
|
75
|
+
sandboxRotationController,
|
|
76
|
+
sandboxState,
|
|
77
|
+
recordingState,
|
|
78
|
+
eventing,
|
|
79
|
+
attempt,
|
|
80
|
+
} = deps;
|
|
81
|
+
|
|
82
|
+
// P1.2 ownership inversion: when sandboxOwnershipEnabled, the turn resolves
|
|
83
|
+
// the one box by id from the group lease and injects it NON-OWNED into the
|
|
84
|
+
// run. null when the flag is off (byte-for-byte the legacy build-and-discard
|
|
85
|
+
// path) OR when the backend is "none". Released + dropped in `finally`.
|
|
86
|
+
const releaseLateSandbox = async (sandbox: ResumedTurnSandbox): Promise<void> => {
|
|
87
|
+
sandboxState.lateSandboxesAwaitingWriterDrain.add(sandbox);
|
|
88
|
+
// Drop the holder/timer immediately, but keep its null-outcome admissions
|
|
89
|
+
// fenced until the shared attempt writer drain completes. The staged
|
|
90
|
+
// release serializes a later proof-bearing call behind this one.
|
|
91
|
+
await sandbox.release().catch(() => undefined);
|
|
92
|
+
if (!sandboxState.attemptWritersDrained) return;
|
|
93
|
+
try {
|
|
94
|
+
await releaseTurnSandboxAfterWriterDrain(sandbox);
|
|
95
|
+
sandboxState.lateSandboxesAwaitingWriterDrain.delete(sandbox);
|
|
96
|
+
} catch (error) {
|
|
97
|
+
console.error(
|
|
98
|
+
"late sandbox quiesced release failed (turn outcome unaffected)",
|
|
99
|
+
safeErrorDiagnostic(error),
|
|
100
|
+
);
|
|
101
|
+
}
|
|
102
|
+
};
|
|
103
|
+
const requireResolvedSandboxForMutation = (message: string): ResumedTurnSandbox => {
|
|
104
|
+
if (!sandboxState.resolvedSandbox) throw new Error(message);
|
|
105
|
+
return sandboxState.resolvedSandbox;
|
|
106
|
+
};
|
|
107
|
+
// The machine-primary SelfhostedSession (the UNWRAPPED backend, not the
|
|
108
|
+
// routing proxy). Kept as a fallback finalizer; the routing proxy normally
|
|
109
|
+
// aggregates it together with every machine reached after a mid-turn swap.
|
|
110
|
+
// The UN-PROXIED established box session, captured BEFORE wrapTurnBoxWithRouting.
|
|
111
|
+
// Platform setup (beforeAgentStart hooks + file materialization) execs against
|
|
112
|
+
// THIS handle so a mid-turn sandbox_swap can never re-route those execs onto a
|
|
113
|
+
// connected machine (the user's real computer).
|
|
114
|
+
const finalizeTurnOpStreamOps = async (): Promise<void> => {
|
|
115
|
+
await finalizeDurableTurnOpStreams(
|
|
116
|
+
[sandboxState.lazyOwnedSandbox?.session, sandboxState.resolvedSandbox?.established.session],
|
|
117
|
+
sandboxState.machinePrimarySession,
|
|
118
|
+
);
|
|
119
|
+
};
|
|
120
|
+
// A same-target API repair can replace the home provider while this turn is
|
|
121
|
+
// alive. Keep setup/snapshot persistence on the rebound raw session while
|
|
122
|
+
// preserving the SDK-owned routing proxy for eager turns. Lazy turns hold the
|
|
123
|
+
// proxy separately, so their worker-side handle may replace its raw session.
|
|
124
|
+
const onHomeSandboxRebound = (rebound: {
|
|
125
|
+
established: EstablishedSandboxSession;
|
|
126
|
+
leaseEpoch: number;
|
|
127
|
+
}): void => {
|
|
128
|
+
const current = sandboxState.resolvedSandbox;
|
|
129
|
+
const previousSession = current?.established.session;
|
|
130
|
+
const preserveRoutingProxy =
|
|
131
|
+
current !== null && previousSession !== sandboxState.setupBoxSession;
|
|
132
|
+
sandboxState.setupBoxSession = rebound.established.session;
|
|
133
|
+
if (!current) return;
|
|
134
|
+
current.leaseEpoch = rebound.leaseEpoch;
|
|
135
|
+
current.established = preserveRoutingProxy
|
|
136
|
+
? {
|
|
137
|
+
...current.established,
|
|
138
|
+
client: rebound.established.client,
|
|
139
|
+
// Keep the stable SDK-facing proxy; only its resolver changes the
|
|
140
|
+
// underlying backend. The worker's setupBoxSession above is raw.
|
|
141
|
+
session: previousSession,
|
|
142
|
+
sessionState: rebound.established.sessionState,
|
|
143
|
+
instanceId: rebound.established.instanceId,
|
|
144
|
+
backendId: rebound.established.backendId,
|
|
145
|
+
...(rebound.established.origin ? { origin: rebound.established.origin } : {}),
|
|
146
|
+
...(rebound.established.restoredArchive
|
|
147
|
+
? { restoredArchive: rebound.established.restoredArchive }
|
|
148
|
+
: {}),
|
|
149
|
+
}
|
|
150
|
+
: rebound.established;
|
|
151
|
+
};
|
|
152
|
+
// The globally unique durable turn-attempt holder id + the group id,
|
|
153
|
+
// captured so the lease heartbeat can refresh the lease TTL epoch-fenced
|
|
154
|
+
// (a superseded owner self-evicts) and finally can release.
|
|
155
|
+
const runWorkspaceMutationForSandbox = async <T>(
|
|
156
|
+
sandbox: ResumedTurnSandbox,
|
|
157
|
+
operation: string,
|
|
158
|
+
mutation: () => Promise<T>,
|
|
159
|
+
observePhase?: (measurement: {
|
|
160
|
+
phase: "admission" | "provider" | "settlement" | "snapshot_wait";
|
|
161
|
+
outcome: "completed" | "failed";
|
|
162
|
+
durationSeconds: number;
|
|
163
|
+
}) => void,
|
|
164
|
+
): Promise<T> => {
|
|
165
|
+
const observeMutationPhase = (
|
|
166
|
+
phase: "admission" | "provider" | "settlement" | "snapshot_wait",
|
|
167
|
+
outcome: "completed" | "failed",
|
|
168
|
+
durationMs: number,
|
|
169
|
+
): void => {
|
|
170
|
+
try {
|
|
171
|
+
observePhase?.({
|
|
172
|
+
phase,
|
|
173
|
+
outcome,
|
|
174
|
+
durationSeconds: Math.max(0, durationMs) / 1_000,
|
|
175
|
+
});
|
|
176
|
+
} catch {
|
|
177
|
+
// Diagnostics must never alter workspace mutation authority.
|
|
178
|
+
}
|
|
179
|
+
};
|
|
180
|
+
// Connected machines are the user's own persistence and never dirty the
|
|
181
|
+
// cloud home archive. Every persistable raw-session write batch is fenced
|
|
182
|
+
// against the exact current lease/provider before the provider sees it.
|
|
183
|
+
if (sandbox.established.backendId === "selfhosted") {
|
|
184
|
+
const providerStartedAt = performance.now();
|
|
185
|
+
let providerOutcome: "completed" | "failed" = "failed";
|
|
186
|
+
try {
|
|
187
|
+
const result = await mutation();
|
|
188
|
+
providerOutcome = "completed";
|
|
189
|
+
return result;
|
|
190
|
+
} catch (error) {
|
|
191
|
+
if (error instanceof ChannelAPartialMutationError) {
|
|
192
|
+
throw new RoutingMutationOutcomeUnknownError(
|
|
193
|
+
operation,
|
|
194
|
+
`Connected Machine workspace mutation "${operation}" partially applied before a later batch item failed; the complete operation was not replayed`,
|
|
195
|
+
{ cause: error },
|
|
196
|
+
);
|
|
197
|
+
}
|
|
198
|
+
throw error;
|
|
199
|
+
} finally {
|
|
200
|
+
observeMutationPhase("provider", providerOutcome, performance.now() - providerStartedAt);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
if (
|
|
204
|
+
!sandboxState.sandboxGroupId ||
|
|
205
|
+
!sandboxState.sandboxHolderId ||
|
|
206
|
+
!attempt.turnId ||
|
|
207
|
+
attempt.executionGeneration <= 0
|
|
208
|
+
) {
|
|
209
|
+
throw new Error("Workspace mutation attempted before exact turn sandbox admission");
|
|
210
|
+
}
|
|
211
|
+
let admissionCaptureWaitMs = 0;
|
|
212
|
+
const identity = {
|
|
213
|
+
accountId: input.accountId,
|
|
214
|
+
workspaceId: input.workspaceId,
|
|
215
|
+
sessionId: input.sessionId,
|
|
216
|
+
turnId: attempt.turnId,
|
|
217
|
+
executionGeneration: attempt.executionGeneration,
|
|
218
|
+
attemptId: input.attemptId,
|
|
219
|
+
holderId: sandboxState.sandboxHolderId,
|
|
220
|
+
sandboxGroupId: sandboxState.sandboxGroupId,
|
|
221
|
+
expectedEpoch: sandbox.leaseEpoch,
|
|
222
|
+
expectedInstanceId: sandbox.established.instanceId,
|
|
223
|
+
operation,
|
|
224
|
+
captureWaitMs: sandboxLifecycleTransitionWaitMs(settings),
|
|
225
|
+
...(cancellationSignal ? { waitSignal: cancellationSignal } : {}),
|
|
226
|
+
...(observePhase
|
|
227
|
+
? {
|
|
228
|
+
onCaptureWait: (observation: {
|
|
229
|
+
durationMs: number;
|
|
230
|
+
outcome: "completed" | "failed";
|
|
231
|
+
}) => {
|
|
232
|
+
admissionCaptureWaitMs += Math.max(0, observation.durationMs);
|
|
233
|
+
observeMutationPhase("snapshot_wait", observation.outcome, observation.durationMs);
|
|
234
|
+
},
|
|
235
|
+
}
|
|
236
|
+
: {}),
|
|
237
|
+
};
|
|
238
|
+
const admissionStartedAt = performance.now();
|
|
239
|
+
let admissionOutcome: "completed" | "failed" = "failed";
|
|
240
|
+
let admission: Awaited<ReturnType<typeof advanceWorkspaceGeneration>>;
|
|
241
|
+
try {
|
|
242
|
+
admission = await advanceWorkspaceGeneration(db, identity);
|
|
243
|
+
admissionOutcome = "completed";
|
|
244
|
+
} catch (admissionError) {
|
|
245
|
+
if (
|
|
246
|
+
admissionError instanceof SandboxWorkspaceMutationFencedError &&
|
|
247
|
+
admissionError.code === "rotation_in_progress" &&
|
|
248
|
+
sandboxState.sandboxGroupId
|
|
249
|
+
) {
|
|
250
|
+
// The lease still matches this turn's exact epoch/instance but a
|
|
251
|
+
// rotation was requested. Start the same orderly checkpoint the
|
|
252
|
+
// heartbeat would start on its next tick, so the model is not fed
|
|
253
|
+
// retry errors until then. Best-effort and single-flight; the
|
|
254
|
+
// admission failure itself is still surfaced to the caller.
|
|
255
|
+
void beginRotationCheckpoint(
|
|
256
|
+
sandbox,
|
|
257
|
+
sandbox.leaseEpoch,
|
|
258
|
+
sandboxState.sandboxGroupId,
|
|
259
|
+
).catch(() => undefined);
|
|
260
|
+
}
|
|
261
|
+
throw admissionError;
|
|
262
|
+
} finally {
|
|
263
|
+
observeMutationPhase(
|
|
264
|
+
"admission",
|
|
265
|
+
admissionOutcome,
|
|
266
|
+
Math.max(0, performance.now() - admissionStartedAt - admissionCaptureWaitMs),
|
|
267
|
+
);
|
|
268
|
+
}
|
|
269
|
+
const settleMutation = async (outcome: "resolved" | "rejected"): Promise<void> => {
|
|
270
|
+
const settlementStartedAt = performance.now();
|
|
271
|
+
let settlementOutcome: "completed" | "failed" = "failed";
|
|
272
|
+
try {
|
|
273
|
+
await verifyWorkspaceMutationSettlement(db, {
|
|
274
|
+
...identity,
|
|
275
|
+
admission,
|
|
276
|
+
outcome,
|
|
277
|
+
});
|
|
278
|
+
settlementOutcome = "completed";
|
|
279
|
+
} finally {
|
|
280
|
+
observeMutationPhase(
|
|
281
|
+
"settlement",
|
|
282
|
+
settlementOutcome,
|
|
283
|
+
performance.now() - settlementStartedAt,
|
|
284
|
+
);
|
|
285
|
+
}
|
|
286
|
+
};
|
|
287
|
+
let result: T;
|
|
288
|
+
const providerStartedAt = performance.now();
|
|
289
|
+
let providerOutcome: "completed" | "failed" = "failed";
|
|
290
|
+
try {
|
|
291
|
+
result = await mutation();
|
|
292
|
+
providerOutcome = "completed";
|
|
293
|
+
} catch (providerError) {
|
|
294
|
+
observeMutationPhase("provider", providerOutcome, performance.now() - providerStartedAt);
|
|
295
|
+
const partialMutation = providerError instanceof ChannelAPartialMutationError;
|
|
296
|
+
try {
|
|
297
|
+
await settleMutation(partialMutation ? "resolved" : "rejected");
|
|
298
|
+
} catch (settlementError) {
|
|
299
|
+
throw new RoutingMutationOutcomeUnknownError(
|
|
300
|
+
operation,
|
|
301
|
+
partialMutation
|
|
302
|
+
? `Platform workspace mutation "${operation}" partially applied at the provider but lost its durable physical settlement; its outcome is unknown and it was not replayed`
|
|
303
|
+
: `Platform workspace mutation "${operation}" rejected at the provider but lost its durable physical settlement; its outcome is unknown and it was not replayed`,
|
|
304
|
+
{ cause: settlementError },
|
|
305
|
+
);
|
|
306
|
+
}
|
|
307
|
+
if (partialMutation) {
|
|
308
|
+
throw new RoutingMutationOutcomeUnknownError(
|
|
309
|
+
operation,
|
|
310
|
+
`Platform workspace mutation "${operation}" partially applied before a later batch item failed; the complete operation was not replayed`,
|
|
311
|
+
{ cause: providerError },
|
|
312
|
+
);
|
|
313
|
+
}
|
|
314
|
+
throw providerError;
|
|
315
|
+
}
|
|
316
|
+
observeMutationPhase("provider", providerOutcome, performance.now() - providerStartedAt);
|
|
317
|
+
try {
|
|
318
|
+
await settleMutation("resolved");
|
|
319
|
+
} catch (settlementError) {
|
|
320
|
+
throw new RoutingMutationOutcomeUnknownError(
|
|
321
|
+
operation,
|
|
322
|
+
`Platform workspace mutation "${operation}" returned from the provider but lost its durable settlement fence; its outcome is unknown and it was not replayed`,
|
|
323
|
+
{ cause: settlementError },
|
|
324
|
+
);
|
|
325
|
+
}
|
|
326
|
+
return result;
|
|
327
|
+
};
|
|
328
|
+
// Lease-TTL refresh timer (parallels the activity heartbeat): while the turn
|
|
329
|
+
// runs it refreshes expires_at epoch-fenced so a legit multi-day turn is
|
|
330
|
+
// never TTL-reaped. Cleared in finally. Only set when the flag resolved a box.
|
|
331
|
+
const stopLeaseHeartbeat = (): void => {
|
|
332
|
+
if (!sandboxState.leaseHeartbeatTimer) return;
|
|
333
|
+
clearInterval(sandboxState.leaseHeartbeatTimer);
|
|
334
|
+
sandboxState.leaseHeartbeatTimer = undefined;
|
|
335
|
+
};
|
|
336
|
+
// credential-renewal policy: the worker, not the model, owns renewal of run-scoped Git
|
|
337
|
+
// credentials for a multi-day turn. The controller is attached only after
|
|
338
|
+
// the initial seed reached a real cloud box and is drained before capture.
|
|
339
|
+
// Generic host-owned run material has its own attempt-scoped renewal and
|
|
340
|
+
// write handle. It is always drained and wiped before workspace capture.
|
|
341
|
+
// The delegated Codemode bearer has a one-hour TTL. Renewal is attempt-
|
|
342
|
+
// owned and attaches only after the initial token file reached a real
|
|
343
|
+
// sandbox session; finalization drains an in-flight replacement.
|
|
344
|
+
// MID-SESSION snapshot single-flight guard: the heartbeat tick fires every
|
|
345
|
+
// 10s but a Modal filesystem snapshot can take longer — never overlap two
|
|
346
|
+
// captures on one box. The in-flight capture's promise is held so the
|
|
347
|
+
// turn-end persist can await it (its capture predates the turn's final
|
|
348
|
+
// writes; landing after the fresher turn-end capture started would make
|
|
349
|
+
// the atomic DB throttle discard the fresher one). Interval throttling
|
|
350
|
+
// itself lives in maybePersistWarmWorkspaceSnapshot / persistWarmSnapshot.
|
|
351
|
+
// The heartbeat snapshot is mid-session durability, not first-request
|
|
352
|
+
// preparation. Keep it off the startup critical path until a provider
|
|
353
|
+
// request has actually reached its transport boundary.
|
|
354
|
+
// Turn-end capture needs the lease heartbeat to keep its holder alive, but
|
|
355
|
+
// must prevent that same timer from starting another periodic snapshot
|
|
356
|
+
// while it reads. This gate separates those two responsibilities.
|
|
357
|
+
// Computer-use-only recording. Ordinary shell/filesystem turns leave this
|
|
358
|
+
// null; the first actual computer action starts it after :0 is ready.
|
|
359
|
+
// P4.3 recording gate: flips true in `onComputerUseReady`, the runtime's
|
|
360
|
+
// execution-time callback for the first real computer action. It must flip
|
|
361
|
+
// BEFORE awaiting recording startup: the SDK tool-call stream item can arrive
|
|
362
|
+
// before ffmpeg has finished starting. A plain text turn ("hey"/"continue")
|
|
363
|
+
// never invokes the callback, so settlement performs no storage PUT.
|
|
364
|
+
const abandonActiveRecording = async (
|
|
365
|
+
reason: string,
|
|
366
|
+
disposition: "failed" | "discard" = "failed",
|
|
367
|
+
): Promise<void> => {
|
|
368
|
+
const recording = recordingState.activeRecording as ActiveRecording | null;
|
|
369
|
+
if (!recording) return;
|
|
370
|
+
recordingState.activeRecording = null;
|
|
371
|
+
if (sandboxState.resolvedSandbox) {
|
|
372
|
+
await stopRecordingOnBox(
|
|
373
|
+
sandboxState.resolvedSandbox.established.session,
|
|
374
|
+
recording.proc,
|
|
375
|
+
).catch(() => undefined);
|
|
376
|
+
}
|
|
377
|
+
if (!attempt.turnId || attempt.executionGeneration <= 0) return;
|
|
378
|
+
await abandonRecordingForTurnAttempt(db, {
|
|
379
|
+
accountId: input.accountId,
|
|
380
|
+
workspaceId: input.workspaceId,
|
|
381
|
+
sessionId: input.sessionId,
|
|
382
|
+
turnId: attempt.turnId,
|
|
383
|
+
executionGeneration: attempt.executionGeneration,
|
|
384
|
+
attemptId: input.attemptId,
|
|
385
|
+
recordingId: recording.recordingId,
|
|
386
|
+
disposition,
|
|
387
|
+
reason,
|
|
388
|
+
}).catch(() => undefined);
|
|
389
|
+
};
|
|
390
|
+
const flushRuntimeBatcher = async () => {
|
|
391
|
+
const current = eventing.batcher as ReturnType<typeof createRuntimeBatcher> | null;
|
|
392
|
+
await current?.flush().catch(() => undefined);
|
|
393
|
+
};
|
|
394
|
+
// Reconciliation is declared before provider routing so every turn-end path
|
|
395
|
+
// can share one closure. It cannot run until `stream` exists, by which time
|
|
396
|
+
// this value has been rebound to the turn's resolved model policy.
|
|
397
|
+
const publishSandboxLifecycleEvents = async (sandbox: ResumedTurnSandbox): Promise<void> => {
|
|
398
|
+
const established = sandbox.established;
|
|
399
|
+
if (eventing.publish && established.origin && established.origin !== "resumed") {
|
|
400
|
+
const lifecycleEvents: Array<{
|
|
401
|
+
type: "sandbox.box.lost" | "sandbox.box.created";
|
|
402
|
+
payload: unknown;
|
|
403
|
+
}> = [];
|
|
404
|
+
if (established.lostInstanceId) {
|
|
405
|
+
lifecycleEvents.push({
|
|
406
|
+
type: "sandbox.box.lost",
|
|
407
|
+
payload: { sandboxId: established.lostInstanceId },
|
|
408
|
+
});
|
|
409
|
+
}
|
|
410
|
+
lifecycleEvents.push({
|
|
411
|
+
type: "sandbox.box.created",
|
|
412
|
+
payload: {
|
|
413
|
+
sandboxId: established.instanceId,
|
|
414
|
+
hydrated: established.origin === "restored" ? "archive" : "none",
|
|
415
|
+
},
|
|
416
|
+
});
|
|
417
|
+
await eventing.publish(lifecycleEvents).catch(() => undefined);
|
|
418
|
+
}
|
|
419
|
+
};
|
|
420
|
+
const publishSandboxLost = async (lostSandbox: { instanceId: string }): Promise<void> => {
|
|
421
|
+
if (!eventing.publish) return;
|
|
422
|
+
await eventing
|
|
423
|
+
.publish([
|
|
424
|
+
{
|
|
425
|
+
type: "sandbox.box.lost",
|
|
426
|
+
payload: { sandboxId: lostSandbox.instanceId },
|
|
427
|
+
},
|
|
428
|
+
])
|
|
429
|
+
.catch((publishError) => {
|
|
430
|
+
// The lease transition is already authoritative. A fenced/failed audit
|
|
431
|
+
// append must not prevent the same logical turn from recovering.
|
|
432
|
+
console.error("sandbox box lost event publish failed", safeErrorDiagnostic(publishError));
|
|
433
|
+
});
|
|
434
|
+
};
|
|
435
|
+
/**
|
|
436
|
+
* Start (single-flight) the orderly provider-deadline rotation checkpoint for
|
|
437
|
+
* the established box. Entered from the lease heartbeat once the lease is
|
|
438
|
+
* rotation-fenced, and from a workspace-mutation admission that observed
|
|
439
|
+
* `rotation_in_progress` at the same epoch/instance, so the model is not fed
|
|
440
|
+
* retry errors for a whole heartbeat interval before the checkpoint runs.
|
|
441
|
+
* `not_rotating` means the exact epoch/instance no longer carries a requested
|
|
442
|
+
* rotation (holder reaped, attempt closed, epoch superseded, or draining).
|
|
443
|
+
*/
|
|
444
|
+
const beginRotationCheckpoint = async (
|
|
445
|
+
sandbox: ResumedTurnSandbox,
|
|
446
|
+
rotationEpoch: number,
|
|
447
|
+
rotationGroupId: string,
|
|
448
|
+
): Promise<"started" | "busy" | "not_rotating"> => {
|
|
449
|
+
// A cancelled attempt never starts (or reinstates a holder for) a
|
|
450
|
+
// checkpoint; its cancellation settlement owns the holder from here.
|
|
451
|
+
if (
|
|
452
|
+
sandboxState.rotationInFlight ||
|
|
453
|
+
sandboxRotationController.signal.aborted ||
|
|
454
|
+
cancellationSignal?.aborted
|
|
455
|
+
) {
|
|
456
|
+
return "busy";
|
|
457
|
+
}
|
|
458
|
+
const rotatingLease = await readLease(db, input.workspaceId, rotationGroupId).catch(() => null);
|
|
459
|
+
if (
|
|
460
|
+
!rotatingLease ||
|
|
461
|
+
rotatingLease.leaseEpoch !== rotationEpoch ||
|
|
462
|
+
rotatingLease.instanceId !== sandbox.established.instanceId ||
|
|
463
|
+
rotatingLease.rotationRequestedAt === null
|
|
464
|
+
) {
|
|
465
|
+
return "not_rotating";
|
|
466
|
+
}
|
|
467
|
+
if (sandboxState.rotationInFlight || sandboxRotationController.signal.aborted) return "busy";
|
|
468
|
+
sandboxState.rotationInFlight = (async () => {
|
|
469
|
+
// Rotation admission already fenced all new workspace mutations.
|
|
470
|
+
// Wait for an earlier periodic capture, then produce the exact
|
|
471
|
+
// generation-complete checkpoint that licenses aborting this run.
|
|
472
|
+
if (sandboxState.snapshotInFlight) {
|
|
473
|
+
await waitForWarmSnapshot(
|
|
474
|
+
sandboxState.snapshotInFlight,
|
|
475
|
+
settings.sandboxSnapshotTimeoutMs,
|
|
476
|
+
cancellationSignal,
|
|
477
|
+
);
|
|
478
|
+
}
|
|
479
|
+
const snapshotSession = sandboxState.setupBoxSession;
|
|
480
|
+
const snapshotTurnId = attempt.turnId;
|
|
481
|
+
if (!snapshotSession || !snapshotTurnId) return;
|
|
482
|
+
// The checkpoint helper first reinstates this turn's exact holder
|
|
483
|
+
// when it was lost at the same epoch/instance (defense in depth:
|
|
484
|
+
// the warm capture requires it), then forces the capture and
|
|
485
|
+
// reports whether the established epoch now carries a complete
|
|
486
|
+
// archive under its requested rotation.
|
|
487
|
+
const checkpoint = await persistSandboxDeadlineRotationCheckpoint(
|
|
488
|
+
{ db, settings, objectStorage, observability },
|
|
489
|
+
{
|
|
490
|
+
accountId: input.accountId,
|
|
491
|
+
workspaceId: input.workspaceId,
|
|
492
|
+
sessionId: input.sessionId,
|
|
493
|
+
turnId: snapshotTurnId,
|
|
494
|
+
attemptId: input.attemptId,
|
|
495
|
+
sandboxGroupId: rotationGroupId,
|
|
496
|
+
},
|
|
497
|
+
snapshotSession,
|
|
498
|
+
{ leaseEpoch: rotationEpoch, instanceId: sandbox.established.instanceId },
|
|
499
|
+
cancellationSignal,
|
|
500
|
+
);
|
|
501
|
+
if (checkpoint.checkpointed) {
|
|
502
|
+
sandboxRotationController.abort(
|
|
503
|
+
new SandboxDeadlineRotationError(rotationGroupId, rotationEpoch),
|
|
504
|
+
);
|
|
505
|
+
return;
|
|
506
|
+
}
|
|
507
|
+
if (checkpoint.holder === "attempt_fenced" || checkpoint.holder === "lease_fenced") {
|
|
508
|
+
// This attempt is no longer the active writer, or the exact
|
|
509
|
+
// epoch/instance is gone. Nothing left to checkpoint here.
|
|
510
|
+
stopLeaseHeartbeat();
|
|
511
|
+
}
|
|
512
|
+
})()
|
|
513
|
+
.catch((error) => {
|
|
514
|
+
observability.warn("sandbox deadline rotation checkpoint failed; retrying", {
|
|
515
|
+
workspaceId: input.workspaceId,
|
|
516
|
+
sessionId: input.sessionId,
|
|
517
|
+
sandboxGroupId: rotationGroupId,
|
|
518
|
+
leaseEpoch: rotationEpoch,
|
|
519
|
+
...safeErrorDiagnostic(error),
|
|
520
|
+
});
|
|
521
|
+
})
|
|
522
|
+
.finally(() => {
|
|
523
|
+
sandboxState.rotationInFlight = null;
|
|
524
|
+
});
|
|
525
|
+
await sandboxState.rotationInFlight;
|
|
526
|
+
return "started";
|
|
527
|
+
};
|
|
528
|
+
const startLeaseHeartbeat = (
|
|
529
|
+
sandbox: ResumedTurnSandbox,
|
|
530
|
+
warmBackend: Settings["sandboxBackend"] | undefined,
|
|
531
|
+
): void => {
|
|
532
|
+
if (sandboxState.leaseHeartbeatTimer) return;
|
|
533
|
+
if (!sandboxState.sandboxHolderId || !sandboxState.sandboxGroupId) {
|
|
534
|
+
return;
|
|
535
|
+
}
|
|
536
|
+
// Refresh the lease TTL on the activity-heartbeat cadence (10s, well
|
|
537
|
+
// inside the 90s lease TTL). EPOCH-FENCED: a superseded owner's refresh
|
|
538
|
+
// is rejected (returns false) and we stop refreshing — the box rides the
|
|
539
|
+
// provider idle-timeout and the next dispatch re-establishes it. Best-
|
|
540
|
+
// effort: a transient DB error must never fail the turn.
|
|
541
|
+
const heartbeatEpoch = sandbox.leaseEpoch;
|
|
542
|
+
const heartbeatHolderId = sandboxState.sandboxHolderId;
|
|
543
|
+
const heartbeatGroupId = sandboxState.sandboxGroupId;
|
|
544
|
+
// P2.1 warm-meter (tick A): while a turn runs, the heartbeat is also the
|
|
545
|
+
// warm-seconds tick. GROUP+epoch+tick keyed (one box = one stream, shared
|
|
546
|
+
// box metered once); epoch-fenced (a stale tick no-ops). Warm-cost is
|
|
547
|
+
// metered when a per-backend rate is configured. Best-effort: a metering
|
|
548
|
+
// failure must never fail the turn.
|
|
549
|
+
//
|
|
550
|
+
// Keyed off the EFFECTIVE backend (Stage D): a machine-primary turn has NO
|
|
551
|
+
// Modal box, so it must accrue ZERO cloud warm-seconds — `selfhosted` has no
|
|
552
|
+
// configured warm rate (0). Keying off turn.sandboxBackend (modal) would bill
|
|
553
|
+
// cloud seconds for a box that does not exist (a real money bug). Non-machine
|
|
554
|
+
// turns fall back to groupBoxBackend (the REAL box that ran): for a machine-
|
|
555
|
+
// home turn that degraded to the cloud group box (swap-away / flag-off), that
|
|
556
|
+
// is the deployment default (modal), so the fallback box is warm-metered at
|
|
557
|
+
// the cloud rate instead of selfhosted's rate-0 (which would under-bill).
|
|
558
|
+
const warmRate = sandboxWarmRateMicrosPerSecond(
|
|
559
|
+
settings,
|
|
560
|
+
warmBackend ?? (sandbox.established.backendId as Settings["sandboxBackend"]),
|
|
561
|
+
);
|
|
562
|
+
sandboxState.leaseHeartbeatTimer = setInterval(() => {
|
|
563
|
+
void heartbeatLeaseHolder(db, {
|
|
564
|
+
accountId: input.accountId,
|
|
565
|
+
workspaceId: input.workspaceId,
|
|
566
|
+
sandboxGroupId: heartbeatGroupId,
|
|
567
|
+
kind: "turn",
|
|
568
|
+
holderId: heartbeatHolderId,
|
|
569
|
+
leaseTtlMs: settings.sandboxLeaseTtlMs,
|
|
570
|
+
expectedEpoch: heartbeatEpoch,
|
|
571
|
+
})
|
|
572
|
+
.then(async (alive) => {
|
|
573
|
+
if (alive) return;
|
|
574
|
+
const rotation = await beginRotationCheckpoint(sandbox, heartbeatEpoch, heartbeatGroupId);
|
|
575
|
+
if (rotation === "not_rotating") {
|
|
576
|
+
// The holder was reaped, the exact attempt closed, the epoch was
|
|
577
|
+
// superseded, or the lease began draining. Do not leave a dead
|
|
578
|
+
// interval issuing DB writes and snapshot probes forever.
|
|
579
|
+
stopLeaseHeartbeat();
|
|
580
|
+
}
|
|
581
|
+
})
|
|
582
|
+
.catch(() => undefined);
|
|
583
|
+
void accrueWarmSeconds(db, {
|
|
584
|
+
accountId: input.accountId,
|
|
585
|
+
workspaceId: input.workspaceId,
|
|
586
|
+
sandboxGroupId: heartbeatGroupId,
|
|
587
|
+
expectedEpoch: heartbeatEpoch,
|
|
588
|
+
warmRateMicrosPerSecond: warmRate,
|
|
589
|
+
subjectId: input.sessionId,
|
|
590
|
+
})
|
|
591
|
+
.then((result) => recordCreditMicros(observability, "usage", result.costMicros))
|
|
592
|
+
.catch(() => undefined);
|
|
593
|
+
// MID-SESSION snapshot (sandbox-file-persistence): while the turn holds
|
|
594
|
+
// the box, fold a fresh /workspace snapshot onto the lease every
|
|
595
|
+
// sandboxSnapshotIntervalMs, so a box death the reaper never sees
|
|
596
|
+
// (Modal hard timeout mid-busy, OOM, infra) costs at most one interval
|
|
597
|
+
// of work — a legit multi-day turn is otherwise completely unprotected
|
|
598
|
+
// (the reaper only drain-persists IDLE leases). Uses the UN-proxied box
|
|
599
|
+
// session (setupBoxSession): the routing veneer could swap mid-op and a
|
|
600
|
+
// selfhosted target has no persistWorkspace anyway. Best-effort +
|
|
601
|
+
// single-flight; throttling lives in the helper.
|
|
602
|
+
const snapshotSession = sandboxState.setupBoxSession;
|
|
603
|
+
const snapshotTurnId = attempt.turnId;
|
|
604
|
+
if (
|
|
605
|
+
snapshotSession &&
|
|
606
|
+
snapshotTurnId &&
|
|
607
|
+
shouldStartPeriodicWorkspaceSnapshot({
|
|
608
|
+
firstProviderRequestStarted: sandboxState.firstProviderRequestStarted,
|
|
609
|
+
snapshotInFlight: Boolean(sandboxState.snapshotInFlight),
|
|
610
|
+
turnEndCaptureInProgress: sandboxState.turnEndCaptureInProgress,
|
|
611
|
+
})
|
|
612
|
+
) {
|
|
613
|
+
sandboxState.snapshotInFlight = maybePersistWarmWorkspaceSnapshot(
|
|
614
|
+
{ db, settings, objectStorage },
|
|
615
|
+
{
|
|
616
|
+
accountId: input.accountId,
|
|
617
|
+
workspaceId: input.workspaceId,
|
|
618
|
+
sessionId: input.sessionId,
|
|
619
|
+
turnId: snapshotTurnId,
|
|
620
|
+
attemptId: input.attemptId,
|
|
621
|
+
sandboxGroupId: heartbeatGroupId,
|
|
622
|
+
},
|
|
623
|
+
snapshotSession,
|
|
624
|
+
heartbeatEpoch,
|
|
625
|
+
activityContext?.cancellationSignal,
|
|
626
|
+
)
|
|
627
|
+
.then(async (persisted) => {
|
|
628
|
+
if (persisted && eventing.publish) {
|
|
629
|
+
await eventing.publish([
|
|
630
|
+
{
|
|
631
|
+
type: "sandbox.box.snapshot",
|
|
632
|
+
payload: { trigger: "heartbeat" },
|
|
633
|
+
},
|
|
634
|
+
]);
|
|
635
|
+
}
|
|
636
|
+
})
|
|
637
|
+
.catch(() => undefined)
|
|
638
|
+
.finally(() => {
|
|
639
|
+
sandboxState.snapshotInFlight = null;
|
|
640
|
+
});
|
|
641
|
+
}
|
|
642
|
+
}, 10_000);
|
|
643
|
+
if (
|
|
644
|
+
"unref" in sandboxState.leaseHeartbeatTimer &&
|
|
645
|
+
typeof sandboxState.leaseHeartbeatTimer.unref === "function"
|
|
646
|
+
) {
|
|
647
|
+
sandboxState.leaseHeartbeatTimer.unref();
|
|
648
|
+
}
|
|
649
|
+
};
|
|
650
|
+
const maybeStartOnTurnRecording = async (
|
|
651
|
+
sandbox: ResumedTurnSandbox,
|
|
652
|
+
effectiveBackend: Settings["sandboxBackend"] | undefined,
|
|
653
|
+
): Promise<void> => {
|
|
654
|
+
if (recordingState.activeRecording) {
|
|
655
|
+
return;
|
|
656
|
+
}
|
|
657
|
+
if (recordingState.computerUseRecordingStart) {
|
|
658
|
+
await recordingState.computerUseRecordingStart;
|
|
659
|
+
return;
|
|
660
|
+
}
|
|
661
|
+
// Called only by the runtime's first-computer-action hook. Plain sandbox
|
|
662
|
+
// operations never start ffmpeg and never boot a display merely to record
|
|
663
|
+
// an unused desktop. Recording failure never fails the computer action.
|
|
664
|
+
if (
|
|
665
|
+
shouldStartOnTurnRecording({
|
|
666
|
+
recordingEnabled: settings.recordingEnabled,
|
|
667
|
+
desktopEnabled: settings.sandboxDesktopEnabled,
|
|
668
|
+
establishedBackendId: sandbox.established.backendId,
|
|
669
|
+
// EFFECTIVE (active) backend, not the session home: a machine-primary turn
|
|
670
|
+
// resolves to "selfhosted" and skips; a swap back to the cloud group box
|
|
671
|
+
// resolves to undefined and records as before.
|
|
672
|
+
effectiveBackend,
|
|
673
|
+
})
|
|
674
|
+
) {
|
|
675
|
+
recordingState.computerUseRecordingStart = (async () => {
|
|
676
|
+
let begun: Awaited<ReturnType<typeof beginRecording>> | null = null;
|
|
677
|
+
try {
|
|
678
|
+
begun = await beginRecording({
|
|
679
|
+
settings,
|
|
680
|
+
db,
|
|
681
|
+
accountId: input.accountId,
|
|
682
|
+
workspaceId: input.workspaceId,
|
|
683
|
+
sessionId: input.sessionId,
|
|
684
|
+
turnId: attempt.turnId!,
|
|
685
|
+
recordingId: randomUUID(),
|
|
686
|
+
mode: "on-turn",
|
|
687
|
+
session: sandbox.established.session,
|
|
688
|
+
runAs: sandboxRunAs(settings),
|
|
689
|
+
reason: null,
|
|
690
|
+
});
|
|
691
|
+
if (!eventing.publish) {
|
|
692
|
+
throw new Error("recording started before the turn event publisher was ready");
|
|
693
|
+
}
|
|
694
|
+
await eventing.publish([{ type: "recording.started", payload: begun.started }]);
|
|
695
|
+
recordingState.activeRecording = begun.active;
|
|
696
|
+
} catch (recordingError) {
|
|
697
|
+
recordingState.activeRecording = null;
|
|
698
|
+
if (begun) {
|
|
699
|
+
await discardUnpublishedRecording({
|
|
700
|
+
db,
|
|
701
|
+
accountId: input.accountId,
|
|
702
|
+
workspaceId: input.workspaceId,
|
|
703
|
+
active: begun.active,
|
|
704
|
+
session: sandbox.established.session,
|
|
705
|
+
});
|
|
706
|
+
}
|
|
707
|
+
console.error(
|
|
708
|
+
"computer-use recording start failed (action outcome unaffected)",
|
|
709
|
+
safeErrorDiagnostic(recordingError),
|
|
710
|
+
);
|
|
711
|
+
}
|
|
712
|
+
})();
|
|
713
|
+
await recordingState.computerUseRecordingStart;
|
|
714
|
+
}
|
|
715
|
+
};
|
|
716
|
+
|
|
717
|
+
return {
|
|
718
|
+
releaseLateSandbox,
|
|
719
|
+
requireResolvedSandboxForMutation,
|
|
720
|
+
finalizeTurnOpStreamOps,
|
|
721
|
+
onHomeSandboxRebound,
|
|
722
|
+
runWorkspaceMutationForSandbox,
|
|
723
|
+
stopLeaseHeartbeat,
|
|
724
|
+
abandonActiveRecording,
|
|
725
|
+
flushRuntimeBatcher,
|
|
726
|
+
publishSandboxLifecycleEvents,
|
|
727
|
+
publishSandboxLost,
|
|
728
|
+
startLeaseHeartbeat,
|
|
729
|
+
maybeStartOnTurnRecording,
|
|
730
|
+
};
|
|
731
|
+
}
|