@opengeni/worker-bundle 0.20.1 → 0.22.0-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +52 -0
- package/dist/activities/agent-turn/agent-build.d.ts +89 -0
- package/dist/activities/agent-turn/claim.d.ts +67 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
- package/dist/activities/agent-turn/codex.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
- package/dist/activities/agent-turn/errors.d.ts +253 -0
- package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
- package/dist/activities/agent-turn/file-resources.d.ts +37 -0
- package/dist/activities/agent-turn/finalization.d.ts +42 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
- package/dist/activities/agent-turn/governance-model.d.ts +66 -0
- package/dist/activities/agent-turn/history-sink.d.ts +52 -0
- package/dist/activities/agent-turn/history.d.ts +63 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
- package/dist/activities/agent-turn/model-usage.d.ts +218 -0
- package/dist/activities/agent-turn/quiescence.d.ts +135 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
- package/dist/activities/agent-turn/run.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
- package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
- package/dist/activities/agent-turn/turn-context.d.ts +144 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
- package/dist/activities/agent-turn.d.ts +12 -954
- package/dist/activities/automations.d.ts +16 -0
- package/dist/activities/common.d.ts +1 -1
- package/dist/activities/connector-attachments.d.ts +13 -0
- package/dist/activities/documents.d.ts +1 -0
- package/dist/activities/environment.d.ts +26 -15
- package/dist/activities/generated-images.d.ts +1 -1
- package/dist/activities/git-credential-renewal.d.ts +3 -1
- package/dist/activities/goals.d.ts +4 -1
- package/dist/activities/google-drive-provider.d.ts +29 -0
- package/dist/activities/google-drive-publication.d.ts +66 -0
- package/dist/activities/image-generation-references.d.ts +2 -0
- package/dist/activities/incident-telemetry-authority.d.ts +52 -0
- package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +21 -0
- package/dist/activities/knowledge-source-sync.d.ts +33 -1
- package/dist/activities/mcp-credentials.d.ts +9 -3
- package/dist/activities/open-suffix-resume.d.ts +77 -0
- package/dist/activities/packs.d.ts +3 -11
- package/dist/activities/parent-wake.d.ts +19 -0
- package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
- package/dist/activities/retained-screenshots.d.ts +1 -1
- package/dist/activities/rig-verification.d.ts +1 -1
- package/dist/activities/run-input.d.ts +16 -7
- package/dist/activities/sandbox-images.d.ts +1 -14
- package/dist/activities/sandbox-lease.d.ts +21 -3
- package/dist/activities/scheduled-tasks.d.ts +47 -0
- package/dist/activities/types.d.ts +23 -2
- package/dist/activities/video-media-validation.d.ts +1 -1
- package/dist/activities/video-output-retention.d.ts +1 -2
- package/dist/activities/xai-auth.d.ts +3 -0
- package/dist/{activities-control-SMKF2OBE.js → activities-control-DKG22ONJ.js} +2946 -549
- package/dist/activities-control-DKG22ONJ.js.map +1 -0
- package/dist/activities-control.d.ts +6 -2
- package/dist/activities-turn-4IMC3MAA.js +21326 -0
- package/dist/activities-turn-4IMC3MAA.js.map +1 -0
- package/dist/activities.d.ts +7 -3
- package/dist/artifact-materializer-entry.js +41 -22
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
- package/dist/chunk-CE7TPKBS.js.map +1 -0
- package/dist/chunk-CNYGVRKF.js +4698 -0
- package/dist/chunk-CNYGVRKF.js.map +1 -0
- package/dist/connection-provider-use.d.ts +18 -0
- package/dist/editable-artifact-materializer.d.ts +4 -3
- package/dist/github-rest-mcp.d.ts +26 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +58 -14
- package/dist/index.js.map +1 -1
- package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
- package/dist/model-context-contributions.d.ts +8 -4
- package/dist/observability-metrics.d.ts +63 -4
- package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
- package/dist/personal-github-git-credentials.d.ts +9 -0
- package/dist/pr-review-credentials.d.ts +6 -0
- package/dist/sandbox-archive-storage.d.ts +45 -0
- package/dist/sandbox-resume.d.ts +65 -6
- package/dist/sandbox-routing.d.ts +38 -2
- package/dist/scheduled-alert-occurrence.d.ts +33 -0
- package/dist/workflow-bundle.js +87 -28
- package/dist/workflows/activities.d.ts +1 -0
- package/dist/workflows/automations.d.ts +6 -0
- package/dist/workflows/document-index.d.ts +1 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/admission.ts +292 -0
- package/src/activities/agent-turn/agent-build.ts +802 -0
- package/src/activities/agent-turn/claim.ts +590 -0
- package/src/activities/agent-turn/codex-capacity.ts +720 -0
- package/src/activities/agent-turn/codex.ts +94 -0
- package/src/activities/agent-turn/compaction-prep.ts +722 -0
- package/src/activities/agent-turn/credential-leases.ts +240 -0
- package/src/activities/agent-turn/errors.ts +1068 -0
- package/src/activities/agent-turn/failure-settlement.ts +1436 -0
- package/src/activities/agent-turn/file-resources.ts +227 -0
- package/src/activities/agent-turn/finalization.ts +754 -0
- package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
- package/src/activities/agent-turn/governance-model.ts +465 -0
- package/src/activities/agent-turn/history-sink.ts +139 -0
- package/src/activities/agent-turn/history.ts +200 -0
- package/src/activities/agent-turn/media-artifacts.ts +359 -0
- package/src/activities/agent-turn/model-usage.ts +823 -0
- package/src/activities/agent-turn/quiescence.ts +353 -0
- package/src/activities/agent-turn/run-credentials.ts +761 -0
- package/src/activities/agent-turn/run.ts +1530 -0
- package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
- package/src/activities/agent-turn/sandbox-provision.ts +569 -0
- package/src/activities/agent-turn/sandbox-route.ts +358 -0
- package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
- package/src/activities/agent-turn/stream-attempt.ts +1670 -0
- package/src/activities/agent-turn/tool-environment.ts +729 -0
- package/src/activities/agent-turn/tool-policy.ts +216 -0
- package/src/activities/agent-turn/tool-result-spill.ts +265 -0
- package/src/activities/agent-turn/turn-context.ts +281 -0
- package/src/activities/agent-turn/xai-capacity.ts +229 -0
- package/src/activities/agent-turn.ts +12 -12575
- package/src/activities/api-integrations.ts +3 -0
- package/src/activities/automations.ts +214 -0
- package/src/activities/codemode-dispatcher.ts +12 -1
- package/src/activities/common.ts +1 -5
- package/src/activities/connector-attachments.ts +168 -0
- package/src/activities/context-compaction.ts +18 -0
- package/src/activities/documents.ts +6 -4
- package/src/activities/environment.ts +151 -55
- package/src/activities/generated-images.ts +26 -11
- package/src/activities/git-credential-renewal.ts +13 -3
- package/src/activities/goals.ts +85 -19
- package/src/activities/google-drive-provider.ts +178 -0
- package/src/activities/google-drive-publication.ts +938 -0
- package/src/activities/image-generation-references.ts +11 -2
- package/src/activities/incident-telemetry-authority.ts +327 -0
- package/src/activities/incident-telemetry-preflight.ts +317 -0
- package/src/activities/knowledge-source-sync-driver.ts +23 -0
- package/src/activities/knowledge-source-sync.ts +556 -40
- package/src/activities/mcp-credentials.ts +191 -17
- package/src/activities/open-suffix-resume.ts +422 -0
- package/src/activities/packs.ts +12 -60
- package/src/activities/parent-wake.ts +64 -14
- package/src/activities/retained-screenshot-reaper.ts +6 -3
- package/src/activities/retained-screenshots.ts +15 -47
- package/src/activities/rig-verification.ts +1 -1
- package/src/activities/run-input.ts +128 -136
- package/src/activities/sandbox-images.ts +7 -166
- package/src/activities/sandbox-lease.ts +406 -15
- package/src/activities/scheduled-tasks.ts +2103 -411
- package/src/activities/session-state.ts +10 -2
- package/src/activities/types.ts +32 -2
- package/src/activities/video-media-validation.ts +3 -3
- package/src/activities/video-output-retention.ts +2 -57
- package/src/activities/xai-auth.ts +8 -0
- package/src/activities-control.ts +2 -0
- package/src/activities.ts +3 -0
- package/src/activity-services.ts +13 -4
- package/src/connection-provider-use.ts +36 -0
- package/src/editable-artifact-materializer-subprocess.ts +25 -14
- package/src/editable-artifact-materializer.ts +29 -18
- package/src/github-rest-mcp.ts +713 -0
- package/src/http.ts +16 -6
- package/src/index.ts +40 -5
- package/src/knowledge-source-sync-activity-policy.ts +2 -0
- package/src/model-context-contributions.ts +33 -10
- package/src/observability-metrics.ts +357 -0
- package/src/opensandbox-kubernetes-inventory.ts +292 -0
- package/src/personal-github-git-credentials.ts +331 -0
- package/src/pr-review-credentials.ts +163 -0
- package/src/sandbox-archive-storage.ts +159 -0
- package/src/sandbox-resume.ts +449 -63
- package/src/sandbox-routing.ts +314 -16
- package/src/scheduled-alert-occurrence.ts +176 -0
- package/src/workflows/activities.ts +18 -2
- package/src/workflows/automations.ts +11 -0
- package/src/workflows/session.ts +13 -3
- package/src/workflows.ts +1 -0
- package/dist/activities-control-SMKF2OBE.js.map +0 -1
- package/dist/activities-turn-IDS3WEHA.js +0 -16532
- package/dist/activities-turn-IDS3WEHA.js.map +0 -1
- package/dist/chunk-N5TJIFCE.js.map +0 -1
- package/dist/chunk-P6Z2SLAA.js +0 -3092
- package/dist/chunk-P6Z2SLAA.js.map +0 -1
|
@@ -0,0 +1,1207 @@
|
|
|
1
|
+
import {
|
|
2
|
+
getEnrollment,
|
|
3
|
+
getSandbox,
|
|
4
|
+
readActiveSandbox,
|
|
5
|
+
getLiveEnrollmentConnection,
|
|
6
|
+
assertPersonalMachineForAttempt,
|
|
7
|
+
type SandboxRecord,
|
|
8
|
+
} from "@opengeni/db";
|
|
9
|
+
import { sandboxOperationMetricObserver } from "@opengeni/observability";
|
|
10
|
+
import {
|
|
11
|
+
withRunCredentialsSession,
|
|
12
|
+
runOwnedSandboxSetup,
|
|
13
|
+
markModelPreparationFirstSandboxOperation,
|
|
14
|
+
recordModelPreparationMeasurement,
|
|
15
|
+
sdkBackendIdForSandboxBackend,
|
|
16
|
+
type EstablishedSandboxSession,
|
|
17
|
+
type GitCredentialTokenWriterSession,
|
|
18
|
+
type NormalizedRunCredentialMaterial,
|
|
19
|
+
type RunCredentialCommandSession,
|
|
20
|
+
type CodemodeTokenWriterSession,
|
|
21
|
+
type SelfhostedSession,
|
|
22
|
+
resolveConnectedMachineWorkspaceRoot,
|
|
23
|
+
} from "@opengeni/runtime";
|
|
24
|
+
import { type Settings } from "@opengeni/config";
|
|
25
|
+
import { mergeResourceRefs } from "../common";
|
|
26
|
+
import {
|
|
27
|
+
mintSandboxCodemodeToken,
|
|
28
|
+
type MintedRunGitCredentials,
|
|
29
|
+
type SandboxCodemodeAuthority,
|
|
30
|
+
} from "../environment";
|
|
31
|
+
import { rigProviderImageSourceImage } from "../packs";
|
|
32
|
+
import type { TurnActivityServices as ActivityServices, RunAgentTurnInput } from "../types";
|
|
33
|
+
import type { currentActivityContext } from "../streaming";
|
|
34
|
+
import { resumeBoxForTurn, type ResumedTurnSandbox } from "../../sandbox-resume";
|
|
35
|
+
import {
|
|
36
|
+
wrapTurnBoxWithRouting,
|
|
37
|
+
wrapLazyTurnBoxWithRouting,
|
|
38
|
+
establishSelfhostedTurnSession,
|
|
39
|
+
routingEnabled,
|
|
40
|
+
} from "../../sandbox-routing";
|
|
41
|
+
import {
|
|
42
|
+
makeMachineOpObserver,
|
|
43
|
+
recordSandboxLogicalProvision,
|
|
44
|
+
recordSandboxProvisionAttempt,
|
|
45
|
+
recordTurnStartupPhase,
|
|
46
|
+
runtimeMetricsHooksForObservability,
|
|
47
|
+
} from "../../observability-metrics";
|
|
48
|
+
import { ToolResultSpill } from "./tool-result-spill";
|
|
49
|
+
import type { createTurnMediaArtifacts } from "./media-artifacts";
|
|
50
|
+
|
|
51
|
+
import {
|
|
52
|
+
throwIfTurnOperationCancelled,
|
|
53
|
+
waitForTurnOperation,
|
|
54
|
+
classifySandboxLogicalProvisionFailure,
|
|
55
|
+
isLazySandboxProvisionRetryable,
|
|
56
|
+
createTurnSandboxProvisioner,
|
|
57
|
+
} from "./sandbox-provision";
|
|
58
|
+
import {
|
|
59
|
+
resolveActiveSandboxBackend,
|
|
60
|
+
managedSandboxOwnershipForTurn,
|
|
61
|
+
shouldEstablishSandboxForTurn,
|
|
62
|
+
sandboxEstablishPolicyDecision,
|
|
63
|
+
shouldPrefetchManagedSandbox,
|
|
64
|
+
reconcileActiveSandboxPointer,
|
|
65
|
+
} from "./sandbox-route";
|
|
66
|
+
import type { ClaimTurnOk } from "./claim";
|
|
67
|
+
import type { GovernanceModelOk } from "./governance-model";
|
|
68
|
+
import type { SandboxTurnRuntime } from "./sandbox-runtime";
|
|
69
|
+
import type { AttemptIdentityState, EventingState, SandboxRuntimeState } from "./turn-context";
|
|
70
|
+
|
|
71
|
+
export type SandboxRouteDeps = {
|
|
72
|
+
input: RunAgentTurnInput;
|
|
73
|
+
settings: Settings;
|
|
74
|
+
db: ActivityServices["db"];
|
|
75
|
+
eventing: EventingState;
|
|
76
|
+
sandboxState: SandboxRuntimeState;
|
|
77
|
+
media: ReturnType<typeof createTurnMediaArtifacts>;
|
|
78
|
+
fileAuthoritySubjectId: ClaimTurnOk["fileAuthoritySubjectId"];
|
|
79
|
+
runSettings: GovernanceModelOk["runSettings"];
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
export type SandboxRouteOk = {
|
|
83
|
+
routingOn: boolean;
|
|
84
|
+
activeSandboxPointer: Awaited<ReturnType<typeof readActiveSandbox>>;
|
|
85
|
+
activeSandboxRecord: SandboxRecord | null;
|
|
86
|
+
activeSandboxBackend: Settings["sandboxBackend"] | undefined;
|
|
87
|
+
machinePrimary: boolean;
|
|
88
|
+
groupBoxBackend: Settings["sandboxBackend"];
|
|
89
|
+
groupBoxImage: ReturnType<typeof rigProviderImageSourceImage>;
|
|
90
|
+
sandboxCreationBackend: Settings["sandboxBackend"];
|
|
91
|
+
effectiveRunCredentialBackend: Settings["sandboxBackend"];
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
export type EstablishTurnSandboxDeps = SandboxRouteOk & {
|
|
95
|
+
input: RunAgentTurnInput;
|
|
96
|
+
settings: Settings;
|
|
97
|
+
db: ActivityServices["db"];
|
|
98
|
+
bus: ActivityServices["bus"];
|
|
99
|
+
objectStorage: ActivityServices["objectStorage"];
|
|
100
|
+
observability: ActivityServices["observability"];
|
|
101
|
+
cancellationSignal: AbortSignal | undefined;
|
|
102
|
+
runtimeCancellationSignal: AbortSignal | undefined;
|
|
103
|
+
sandboxResumeSignal: AbortSignal;
|
|
104
|
+
activityContext: ReturnType<typeof currentActivityContext>;
|
|
105
|
+
opJournal: ClaimTurnOk["opJournal"];
|
|
106
|
+
sandboxState: SandboxRuntimeState;
|
|
107
|
+
eventing: EventingState;
|
|
108
|
+
attempt: AttemptIdentityState;
|
|
109
|
+
turn: ClaimTurnOk["turn"];
|
|
110
|
+
session: ClaimTurnOk["session"];
|
|
111
|
+
fileAuthoritySubjectId: ClaimTurnOk["fileAuthoritySubjectId"];
|
|
112
|
+
capabilitySettings: ClaimTurnOk["capabilitySettings"];
|
|
113
|
+
turnExecutionPolicy: ClaimTurnOk["turnExecutionPolicy"];
|
|
114
|
+
runSettings: GovernanceModelOk["runSettings"];
|
|
115
|
+
logicalSandboxSettings: GovernanceModelOk["logicalSandboxSettings"];
|
|
116
|
+
verifiedRigProviderImageId: GovernanceModelOk["verifiedRigProviderImageId"];
|
|
117
|
+
runtimePreparationStartedAt: number;
|
|
118
|
+
establishPolicy: ReturnType<typeof sandboxEstablishPolicyDecision>["policy"];
|
|
119
|
+
sandboxEnvironment: Record<string, string>;
|
|
120
|
+
startRunGitCredentialsMint: () => Promise<MintedRunGitCredentials | undefined>;
|
|
121
|
+
machineOpObserver: ReturnType<typeof makeMachineOpObserver>;
|
|
122
|
+
sandboxOperationObserver: ReturnType<typeof sandboxOperationMetricObserver>;
|
|
123
|
+
sandboxRuntime: SandboxTurnRuntime;
|
|
124
|
+
transientCodemodeEnvironment: (() => Readonly<Record<string, string>>) | undefined;
|
|
125
|
+
rigVersion: GovernanceModelOk["rigVersion"];
|
|
126
|
+
turnResources: ReturnType<typeof mergeResourceRefs>;
|
|
127
|
+
};
|
|
128
|
+
|
|
129
|
+
export type BindLazySandboxProvisionerDeps = EstablishTurnSandboxDeps & {
|
|
130
|
+
agent: ReturnType<ActivityServices["runtime"]["buildAgent"]>;
|
|
131
|
+
attachRunCredentialRenewal: (
|
|
132
|
+
credentialSession: RunCredentialCommandSession,
|
|
133
|
+
initialMaterial: NormalizedRunCredentialMaterial | null,
|
|
134
|
+
initialSandbox?: ResumedTurnSandbox,
|
|
135
|
+
) => Promise<void>;
|
|
136
|
+
attachGitCredentialRenewal: (
|
|
137
|
+
tokenSession: GitCredentialTokenWriterSession,
|
|
138
|
+
initial: MintedRunGitCredentials | undefined,
|
|
139
|
+
initialSandbox?: ResumedTurnSandbox,
|
|
140
|
+
) => Promise<void>;
|
|
141
|
+
attachCodemodeTokenRenewal: (
|
|
142
|
+
tokenSession?: CodemodeTokenWriterSession,
|
|
143
|
+
initialExpiresAt?: Date,
|
|
144
|
+
initialSandbox?: ResumedTurnSandbox,
|
|
145
|
+
) => Promise<void>;
|
|
146
|
+
throwIfWorkerShuttingDown: () => void;
|
|
147
|
+
throwIfTurnCancelled: () => void;
|
|
148
|
+
initialRunCredentialMaterial: NormalizedRunCredentialMaterial | null;
|
|
149
|
+
sandboxCodemodeToken: string | undefined;
|
|
150
|
+
media: ReturnType<typeof createTurnMediaArtifacts>;
|
|
151
|
+
toolResultSpill: ToolResultSpill;
|
|
152
|
+
connectionScope: { accountId: string; workspaceId: string };
|
|
153
|
+
codemodeAuthority: SandboxCodemodeAuthority;
|
|
154
|
+
};
|
|
155
|
+
|
|
156
|
+
export async function resolveSandboxRoute(deps: SandboxRouteDeps): Promise<SandboxRouteOk> {
|
|
157
|
+
const {
|
|
158
|
+
input,
|
|
159
|
+
settings,
|
|
160
|
+
db,
|
|
161
|
+
eventing,
|
|
162
|
+
media,
|
|
163
|
+
sandboxState,
|
|
164
|
+
fileAuthoritySubjectId,
|
|
165
|
+
runSettings,
|
|
166
|
+
} = deps;
|
|
167
|
+
|
|
168
|
+
// EFFECTIVE compute backend, resolved ONCE at turn start (Case B + Stage D
|
|
169
|
+
// D1-lite) and reused for EVERY downstream decision: the env mint (skip
|
|
170
|
+
// inert platform git tokens for a machine turn), the establish path (no phantom Modal
|
|
171
|
+
// home box for a machine-primary turn), buildAgent (skip the repository clone
|
|
172
|
+
// hook so a private repo is never `git clone`d onto the user's real disk), and
|
|
173
|
+
// the warm-rate (a machine accrues ZERO cloud warm-seconds). The active pointer
|
|
174
|
+
// + its sandbox row are loaded ONCE here (best-effort, never throwing) and the
|
|
175
|
+
// SAME values feed resolveActiveSandboxBackend (the tested gate) AND the
|
|
176
|
+
// machine-primary establish branch (enrollmentId/epoch/workingDir) below — no
|
|
177
|
+
// double read, no read-skew between the gate decision and the establish. With
|
|
178
|
+
// routing OFF this is byte-for-byte the legacy path: no reads, undefined backend.
|
|
179
|
+
const routingOn = routingEnabled(settings);
|
|
180
|
+
let activeSandboxPointer = routingOn
|
|
181
|
+
? await readActiveSandbox(db, input.workspaceId, input.sessionId).catch(() => null)
|
|
182
|
+
: null;
|
|
183
|
+
// TURN-START RECONCILE (issue #341 invariant B / Shapes 1+2): a persisted
|
|
184
|
+
// pointer whose target is STRUCTURALLY unestablishable at turn start would strand
|
|
185
|
+
// EVERY op of this turn — reset it to the session HOME under the epoch fence +
|
|
186
|
+
// emit a visible event, honoring a concurrent higher-epoch swap. The sandbox row
|
|
187
|
+
// is loaded HERE, inside reconcile, via a NON-swallowing lookup: a null decision
|
|
188
|
+
// then means the row is genuinely absent, never a suppressed transient DB error
|
|
189
|
+
// (which would wrongly clear a healthy user-chosen pointer). On a lookup throw the
|
|
190
|
+
// reconcile fails open — pointer untouched, record null (machinePrimary:false),
|
|
191
|
+
// no event — and the establish branch below reads the returned values.
|
|
192
|
+
let activeSandboxRecord: SandboxRecord | null = null;
|
|
193
|
+
if (routingOn) {
|
|
194
|
+
const reconciled = await reconcileActiveSandboxPointer(
|
|
195
|
+
db,
|
|
196
|
+
{
|
|
197
|
+
accountId: input.accountId,
|
|
198
|
+
workspaceId: input.workspaceId,
|
|
199
|
+
sessionId: input.sessionId,
|
|
200
|
+
},
|
|
201
|
+
activeSandboxPointer,
|
|
202
|
+
(sandboxId) =>
|
|
203
|
+
getSandbox(
|
|
204
|
+
db,
|
|
205
|
+
fileAuthoritySubjectId
|
|
206
|
+
? {
|
|
207
|
+
accountId: input.accountId,
|
|
208
|
+
workspaceId: input.workspaceId,
|
|
209
|
+
subjectId: fileAuthoritySubjectId,
|
|
210
|
+
}
|
|
211
|
+
: input.workspaceId,
|
|
212
|
+
sandboxId,
|
|
213
|
+
),
|
|
214
|
+
eventing.publish
|
|
215
|
+
? async (events) => {
|
|
216
|
+
await eventing.publish!(events);
|
|
217
|
+
}
|
|
218
|
+
: undefined,
|
|
219
|
+
);
|
|
220
|
+
activeSandboxPointer = reconciled.pointer;
|
|
221
|
+
activeSandboxRecord = reconciled.record;
|
|
222
|
+
}
|
|
223
|
+
const activeSandboxBackend = await resolveActiveSandboxBackend(
|
|
224
|
+
routingOn,
|
|
225
|
+
async () => activeSandboxPointer,
|
|
226
|
+
async () => activeSandboxRecord?.kind ?? null,
|
|
227
|
+
);
|
|
228
|
+
// A machine-primary turn = the effective backend is selfhosted AND we have the
|
|
229
|
+
// machine's enrollment (agent id) + a non-null pointer to bind it. Anything
|
|
230
|
+
// missing (should not happen — the DB enforces selfhosted⇒enrollmentId) falls
|
|
231
|
+
// back to the cloud establish path (a correct, if phantom, box) rather than
|
|
232
|
+
// crashing the turn.
|
|
233
|
+
const machinePrimary =
|
|
234
|
+
activeSandboxBackend === "selfhosted" &&
|
|
235
|
+
Boolean(activeSandboxPointer?.activeSandboxId) &&
|
|
236
|
+
Boolean(activeSandboxRecord?.enrollmentId);
|
|
237
|
+
// `none` describes the durable home, not an explicit per-turn route. Give
|
|
238
|
+
// the runtime the effective backend so it builds a SandboxAgent for the
|
|
239
|
+
// attached Connected Machine without mutating the session or turn record.
|
|
240
|
+
if (machinePrimary && eventing.modelRunSettings.sandboxBackend === "none") {
|
|
241
|
+
eventing.modelRunSettings = {
|
|
242
|
+
...eventing.modelRunSettings,
|
|
243
|
+
sandboxBackend: "selfhosted",
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
// The backend that can actually create a sandbox for this turn. In the
|
|
247
|
+
// common path this is runSettings.sandboxBackend. A selfhosted home turn
|
|
248
|
+
// that is NOT machine-primary falls back to the deployment cloud backend
|
|
249
|
+
// so swap-away / flag-off degrade to a real group box.
|
|
250
|
+
const groupBoxBackend: Settings["sandboxBackend"] =
|
|
251
|
+
runSettings.sandboxBackend === "selfhosted" && !machinePrimary
|
|
252
|
+
? settings.sandboxBackend
|
|
253
|
+
: runSettings.sandboxBackend;
|
|
254
|
+
sandboxState.startupMilestoneBackend = activeSandboxBackend ?? groupBoxBackend;
|
|
255
|
+
media.sandboxFileDownloadBackend = sandboxState.startupMilestoneBackend;
|
|
256
|
+
const groupBoxImage = rigProviderImageSourceImage(runSettings, groupBoxBackend);
|
|
257
|
+
const sandboxCreationBackend: Settings["sandboxBackend"] =
|
|
258
|
+
settings.sandboxOwnershipEnabled && runSettings.sandboxBackend !== "none"
|
|
259
|
+
? groupBoxBackend
|
|
260
|
+
: runSettings.sandboxBackend;
|
|
261
|
+
const effectiveRunCredentialBackend = activeSandboxBackend ?? groupBoxBackend;
|
|
262
|
+
|
|
263
|
+
return {
|
|
264
|
+
routingOn,
|
|
265
|
+
activeSandboxPointer,
|
|
266
|
+
activeSandboxRecord,
|
|
267
|
+
activeSandboxBackend,
|
|
268
|
+
machinePrimary,
|
|
269
|
+
groupBoxBackend,
|
|
270
|
+
groupBoxImage,
|
|
271
|
+
sandboxCreationBackend,
|
|
272
|
+
effectiveRunCredentialBackend,
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Promise<void> {
|
|
277
|
+
const {
|
|
278
|
+
input,
|
|
279
|
+
settings,
|
|
280
|
+
db,
|
|
281
|
+
bus,
|
|
282
|
+
objectStorage,
|
|
283
|
+
observability,
|
|
284
|
+
runtimeCancellationSignal,
|
|
285
|
+
sandboxResumeSignal,
|
|
286
|
+
opJournal,
|
|
287
|
+
sandboxState,
|
|
288
|
+
eventing,
|
|
289
|
+
attempt,
|
|
290
|
+
turn,
|
|
291
|
+
session,
|
|
292
|
+
fileAuthoritySubjectId,
|
|
293
|
+
turnExecutionPolicy,
|
|
294
|
+
runSettings,
|
|
295
|
+
logicalSandboxSettings,
|
|
296
|
+
runtimePreparationStartedAt,
|
|
297
|
+
establishPolicy,
|
|
298
|
+
sandboxEnvironment,
|
|
299
|
+
startRunGitCredentialsMint,
|
|
300
|
+
machineOpObserver,
|
|
301
|
+
sandboxOperationObserver,
|
|
302
|
+
sandboxRuntime,
|
|
303
|
+
transientCodemodeEnvironment,
|
|
304
|
+
activeSandboxPointer,
|
|
305
|
+
activeSandboxRecord,
|
|
306
|
+
activeSandboxBackend,
|
|
307
|
+
machinePrimary,
|
|
308
|
+
groupBoxBackend,
|
|
309
|
+
groupBoxImage,
|
|
310
|
+
rigVersion,
|
|
311
|
+
turnResources,
|
|
312
|
+
} = deps;
|
|
313
|
+
const {
|
|
314
|
+
releaseLateSandbox,
|
|
315
|
+
onHomeSandboxRebound,
|
|
316
|
+
publishSandboxLifecycleEvents,
|
|
317
|
+
publishSandboxLost,
|
|
318
|
+
startLeaseHeartbeat,
|
|
319
|
+
} = sandboxRuntime;
|
|
320
|
+
|
|
321
|
+
// P1.2 ownership inversion (gated, default OFF). With the flag off this
|
|
322
|
+
// block is skipped entirely: resolvedSandbox stays null and runStream
|
|
323
|
+
// takes the legacy per-run build-and-discard path — byte-for-byte today.
|
|
324
|
+
// With it on, acquire the group lease (holder = the durable attempt id),
|
|
325
|
+
// resume the one box by id, and inject it NON-OWNED into the run. The box
|
|
326
|
+
// backend is "none" -> never resolve (no box to touch).
|
|
327
|
+
//
|
|
328
|
+
// Established AFTER sandboxEnvironment is computed (not before) so the box's
|
|
329
|
+
// manifest is created with the SAME variableSet the agent declares — the SDK
|
|
330
|
+
// applies the agent's manifest to this provided session and throws on ANY
|
|
331
|
+
// variableSet delta (validateNoEnvironmentDelta). Passing sandboxEnvironment
|
|
332
|
+
// here makes current==target so the delta is empty.
|
|
333
|
+
recordTurnStartupPhase(observability, {
|
|
334
|
+
phase: "runtime_preparation",
|
|
335
|
+
provider: turnExecutionPolicy.providerId,
|
|
336
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
337
|
+
outcome: "completed",
|
|
338
|
+
durationSeconds: (performance.now() - runtimePreparationStartedAt) / 1_000,
|
|
339
|
+
});
|
|
340
|
+
if (
|
|
341
|
+
shouldEstablishSandboxForTurn(
|
|
342
|
+
settings.sandboxOwnershipEnabled,
|
|
343
|
+
turn.sandboxBackend,
|
|
344
|
+
machinePrimary,
|
|
345
|
+
)
|
|
346
|
+
) {
|
|
347
|
+
const sandboxEstablishStartedAt = performance.now();
|
|
348
|
+
let sandboxEstablishOutcome: "completed" | "failed" = "completed";
|
|
349
|
+
try {
|
|
350
|
+
const managedOwnership = managedSandboxOwnershipForTurn(
|
|
351
|
+
machinePrimary,
|
|
352
|
+
input.attemptId,
|
|
353
|
+
session.sandboxGroupId,
|
|
354
|
+
);
|
|
355
|
+
sandboxState.sandboxHolderId = managedOwnership?.holderId ?? null;
|
|
356
|
+
sandboxState.sandboxGroupId = managedOwnership?.sandboxGroupId ?? null;
|
|
357
|
+
// STAGE D honest-label guard: a machine-home session carries
|
|
358
|
+
// turn.sandboxBackend "selfhosted", but a turn is only machine-PRIMARY
|
|
359
|
+
// when a live machine pointer resolves (activeSandboxBackend==='selfhosted'
|
|
360
|
+
// + enrollmentId). When it is NOT primary — the agent swapped back to the
|
|
361
|
+
// group box (sandbox_swap 'session'/'default'/groupId clears the pointer) or
|
|
362
|
+
// selfhosted routing is flag-OFF (the pointer is ignored) — the else-branch
|
|
363
|
+
// must resume a REAL cloud group box, not a "selfhosted" one: the registry
|
|
364
|
+
// SelfhostedSandboxClient has no bound agentId and throws. Fall the group-box
|
|
365
|
+
// backend back to the deployment default cloud backend so swap-away / flag-off
|
|
366
|
+
// degrade to a genuine cloud box exactly like today (home=modal did).
|
|
367
|
+
if (machinePrimary) {
|
|
368
|
+
if (activeSandboxRecord!.scope === "user") {
|
|
369
|
+
if (!fileAuthoritySubjectId) {
|
|
370
|
+
throw new Error("personal machine use requires an initiating human subject");
|
|
371
|
+
}
|
|
372
|
+
const authorized = await assertPersonalMachineForAttempt(db, {
|
|
373
|
+
accountId: input.accountId,
|
|
374
|
+
workspaceId: input.workspaceId,
|
|
375
|
+
subjectId: fileAuthoritySubjectId,
|
|
376
|
+
sessionId: input.sessionId,
|
|
377
|
+
turnId: turn.id,
|
|
378
|
+
attemptId: input.attemptId,
|
|
379
|
+
executionGeneration: turn.executionGeneration,
|
|
380
|
+
enrollmentId: activeSandboxRecord!.enrollmentId!,
|
|
381
|
+
});
|
|
382
|
+
if (!authorized) {
|
|
383
|
+
throw new Error("personal Connected Machine use was not admitted for this turn");
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
// STAGE D D1-lite: the active sandbox is a connected machine, so DO NOT
|
|
387
|
+
// establish OR lease the managed home box. The active-sandbox pointer is
|
|
388
|
+
// the machine route's own epoch fence; the managed-home lease separately
|
|
389
|
+
// owns cloud recovery, snapshots, billing, and reaping. Mixing those two
|
|
390
|
+
// ownership domains lets an unrelated degraded cloud archive block a
|
|
391
|
+
// healthy explicitly selected machine.
|
|
392
|
+
// Whether the machine's latest Hello advertised the op-stream engine
|
|
393
|
+
// (refreshed on every connect). Read only when the server flag is on —
|
|
394
|
+
// one indexed lookup, and the flag off keeps this path byte-identical.
|
|
395
|
+
const liveMachineEnrollment = await getLiveEnrollmentConnection(
|
|
396
|
+
db,
|
|
397
|
+
activeSandboxRecord!.workspaceId,
|
|
398
|
+
activeSandboxRecord!.enrollmentId!,
|
|
399
|
+
);
|
|
400
|
+
const machineEnrollment =
|
|
401
|
+
liveMachineEnrollment ??
|
|
402
|
+
(await getEnrollment(
|
|
403
|
+
db,
|
|
404
|
+
activeSandboxRecord!.workspaceId,
|
|
405
|
+
activeSandboxRecord!.enrollmentId!,
|
|
406
|
+
));
|
|
407
|
+
if (!machineEnrollment?.workspaceRoot) {
|
|
408
|
+
throw new Error(
|
|
409
|
+
"Connected Machine has not reported an absolute workspace root; reconnect it with a current agent",
|
|
410
|
+
);
|
|
411
|
+
}
|
|
412
|
+
const machineWorkspaceRoot = resolveConnectedMachineWorkspaceRoot(
|
|
413
|
+
machineEnrollment.workspaceRoot,
|
|
414
|
+
activeSandboxPointer!.workingDir,
|
|
415
|
+
);
|
|
416
|
+
const machineOpStream =
|
|
417
|
+
settings.agentOpStreamEnabled === true && liveMachineEnrollment?.opStream === true;
|
|
418
|
+
const established = await establishSelfhostedTurnSession(
|
|
419
|
+
{
|
|
420
|
+
db,
|
|
421
|
+
settings,
|
|
422
|
+
bus,
|
|
423
|
+
onOp: machineOpObserver.observer,
|
|
424
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
425
|
+
opJournal,
|
|
426
|
+
},
|
|
427
|
+
{
|
|
428
|
+
accountId: input.accountId,
|
|
429
|
+
workspaceId: input.workspaceId,
|
|
430
|
+
sessionId: input.sessionId,
|
|
431
|
+
controlWorkspaceId: activeSandboxRecord!.workspaceId,
|
|
432
|
+
backgroundCommandAttempt: {
|
|
433
|
+
turnId: turn.id,
|
|
434
|
+
executionGeneration: attempt.executionGeneration,
|
|
435
|
+
attemptId: input.attemptId,
|
|
436
|
+
},
|
|
437
|
+
agentId: activeSandboxRecord!.enrollmentId!,
|
|
438
|
+
// An offline machine must not fail turn admission. Bind an
|
|
439
|
+
// intentionally unserved token so the model receives the normal
|
|
440
|
+
// typed agent_offline tool result; a later turn resolves a fresh
|
|
441
|
+
// claimed process instance.
|
|
442
|
+
connectionInstanceId: liveMachineEnrollment?.connectionInstanceId ?? "unavailable",
|
|
443
|
+
workspaceRoot: machineWorkspaceRoot,
|
|
444
|
+
opStream: machineOpStream,
|
|
445
|
+
operationResourcePolicy: liveMachineEnrollment?.operationPolicy ?? {
|
|
446
|
+
memoryMaxBytes: null,
|
|
447
|
+
memoryHighBytes: null,
|
|
448
|
+
cpuMaxMillicores: null,
|
|
449
|
+
revision: 0,
|
|
450
|
+
updatedAt: null,
|
|
451
|
+
},
|
|
452
|
+
operationResourcePolicySupported:
|
|
453
|
+
liveMachineEnrollment?.agentCapabilities.operationResourcePolicy === true,
|
|
454
|
+
operationCpuQuotaSupported:
|
|
455
|
+
liveMachineEnrollment?.agentCapabilities.operationCpuQuota === true,
|
|
456
|
+
...(activeSandboxRecord!.scope === "user" && fileAuthoritySubjectId
|
|
457
|
+
? {
|
|
458
|
+
personalMachineAttempt: {
|
|
459
|
+
accountId: input.accountId,
|
|
460
|
+
subjectId: fileAuthoritySubjectId,
|
|
461
|
+
sessionId: input.sessionId,
|
|
462
|
+
turnId: turn.id,
|
|
463
|
+
attemptId: input.attemptId,
|
|
464
|
+
executionGeneration: attempt.executionGeneration,
|
|
465
|
+
},
|
|
466
|
+
}
|
|
467
|
+
: {}),
|
|
468
|
+
epoch: activeSandboxPointer!.activeEpoch,
|
|
469
|
+
environment: sandboxEnvironment,
|
|
470
|
+
...(transientCodemodeEnvironment
|
|
471
|
+
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
472
|
+
: {}),
|
|
473
|
+
workingDir: activeSandboxPointer!.workingDir,
|
|
474
|
+
},
|
|
475
|
+
);
|
|
476
|
+
// The machine-primary establish narrows `session` to SelfhostedSession
|
|
477
|
+
// (buildSelfhostedBackendSession); EstablishedSandboxSession widens it.
|
|
478
|
+
sandboxState.machinePrimarySession = established.session as SelfhostedSession;
|
|
479
|
+
sandboxState.setupBoxSession = established.session;
|
|
480
|
+
sandboxState.resolvedSandbox = {
|
|
481
|
+
// Wrap in the SAME routing proxy so a mid-turn swap (to another machine
|
|
482
|
+
// or back to the group box) still re-routes per op. PIN this established
|
|
483
|
+
// SelfhostedSession for the machine pointer so the turn-start manifest
|
|
484
|
+
// write (via the proxy's `state` getter) and the per-op reads hit ONE
|
|
485
|
+
// instance — no two-instance manifest divergence.
|
|
486
|
+
established: wrapTurnBoxWithRouting(
|
|
487
|
+
{
|
|
488
|
+
db,
|
|
489
|
+
settings,
|
|
490
|
+
bus,
|
|
491
|
+
opJournal,
|
|
492
|
+
onOp: machineOpObserver.observer,
|
|
493
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
494
|
+
onHomeSandboxRebound,
|
|
495
|
+
...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
|
|
496
|
+
},
|
|
497
|
+
{
|
|
498
|
+
workspaceId: input.workspaceId,
|
|
499
|
+
sessionId: input.sessionId,
|
|
500
|
+
resourceAccountId: input.accountId,
|
|
501
|
+
...(fileAuthoritySubjectId ? { resourceSubjectId: fileAuthoritySubjectId } : {}),
|
|
502
|
+
...(fileAuthoritySubjectId
|
|
503
|
+
? {
|
|
504
|
+
personalMachineAttempt: {
|
|
505
|
+
accountId: input.accountId,
|
|
506
|
+
subjectId: fileAuthoritySubjectId,
|
|
507
|
+
turnId: turn.id,
|
|
508
|
+
attemptId: input.attemptId,
|
|
509
|
+
executionGeneration: attempt.executionGeneration,
|
|
510
|
+
},
|
|
511
|
+
}
|
|
512
|
+
: {}),
|
|
513
|
+
environment: sandboxEnvironment,
|
|
514
|
+
...(transientCodemodeEnvironment
|
|
515
|
+
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
516
|
+
: {}),
|
|
517
|
+
workspaceMutationFence: {
|
|
518
|
+
accountId: input.accountId,
|
|
519
|
+
turnId: turn.id,
|
|
520
|
+
executionGeneration: attempt.executionGeneration,
|
|
521
|
+
attemptId: input.attemptId,
|
|
522
|
+
},
|
|
523
|
+
pinnedSelfhosted: {
|
|
524
|
+
sandboxId: activeSandboxPointer!.activeSandboxId!,
|
|
525
|
+
epoch: activeSandboxPointer!.activeEpoch,
|
|
526
|
+
},
|
|
527
|
+
// HOME semantics for a mid-turn clear-to-null: only a genuine
|
|
528
|
+
// machine-HOME session (its home IS this machine, session.sandboxBackend
|
|
529
|
+
// === "selfhosted") resolves null back to the pinned machine. A Modal-HOME
|
|
530
|
+
// session merely PINNED to a machine this turn never established its group
|
|
531
|
+
// box, so defaultIsHome:false makes a clear-to-null fail typed
|
|
532
|
+
// (`home_unavailable_this_turn`) rather than silently serving the machine.
|
|
533
|
+
// Failure settlement checkpoints the completed detach and continues the same
|
|
534
|
+
// logical turn in a fresh home-primary attempt; it never serves this machine
|
|
535
|
+
// after the pointer changed and never invents a cloud lease for machine-only work.
|
|
536
|
+
defaultIsHome: session.sandboxBackend === "selfhosted",
|
|
537
|
+
},
|
|
538
|
+
established,
|
|
539
|
+
),
|
|
540
|
+
// For a machine route this is the active-pointer epoch, not a cloud
|
|
541
|
+
// lease epoch. Cloud-only heartbeat/snapshot/capture paths require
|
|
542
|
+
// sandboxGroupId and remain disabled for this branch.
|
|
543
|
+
leaseEpoch: activeSandboxPointer!.activeEpoch,
|
|
544
|
+
release: async () => undefined,
|
|
545
|
+
};
|
|
546
|
+
} else if (establishPolicy === "on-demand") {
|
|
547
|
+
// Lazy sandbox provisioning: holder/group ids are fixed at turn start,
|
|
548
|
+
// but the lease acquire + box establish + setup move behind the routing
|
|
549
|
+
// proxy's first default-pointer op. A chat-only turn never calls it, so
|
|
550
|
+
// no lease row, no provider box, no warm-meter interval.
|
|
551
|
+
//
|
|
552
|
+
// A repository still forces that first default-pointer op before HTTP
|
|
553
|
+
// (workspace-skill catalog in Agent.instructions). Start create now and
|
|
554
|
+
// join it at get(); do not await here or we serialize Modal before tools.
|
|
555
|
+
if (sandboxState.sandboxHolderId && sandboxState.sandboxGroupId) {
|
|
556
|
+
const lazyHolderId = sandboxState.sandboxHolderId;
|
|
557
|
+
const lazyGroupId = sandboxState.sandboxGroupId;
|
|
558
|
+
sandboxState.resumeManagedGroupBox = () =>
|
|
559
|
+
resumeBoxForTurn(
|
|
560
|
+
{
|
|
561
|
+
db,
|
|
562
|
+
settings: runSettings,
|
|
563
|
+
logicalFallbackSettings: logicalSandboxSettings,
|
|
564
|
+
cancellationSignal: sandboxResumeSignal,
|
|
565
|
+
sandboxMetrics: runtimeMetricsHooksForObservability(observability),
|
|
566
|
+
onSandboxLost: publishSandboxLost,
|
|
567
|
+
objectStorage,
|
|
568
|
+
},
|
|
569
|
+
{
|
|
570
|
+
accountId: input.accountId,
|
|
571
|
+
workspaceId: input.workspaceId,
|
|
572
|
+
sandboxGroupId: lazyGroupId,
|
|
573
|
+
sessionId: input.sessionId,
|
|
574
|
+
backend: groupBoxBackend,
|
|
575
|
+
os: session.sandboxOs,
|
|
576
|
+
environment: sandboxEnvironment,
|
|
577
|
+
...(groupBoxImage
|
|
578
|
+
? {
|
|
579
|
+
image: groupBoxImage,
|
|
580
|
+
}
|
|
581
|
+
: {}),
|
|
582
|
+
// The lazy acquire must enforce the same frozen rig authority
|
|
583
|
+
// as the eager acquire; otherwise a warm box for another rig
|
|
584
|
+
// can bypass the shared-state conflict/rotation fence.
|
|
585
|
+
...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
|
|
586
|
+
},
|
|
587
|
+
"turn",
|
|
588
|
+
lazyHolderId,
|
|
589
|
+
);
|
|
590
|
+
if (
|
|
591
|
+
shouldPrefetchManagedSandbox({
|
|
592
|
+
establishPolicy,
|
|
593
|
+
machinePrimary,
|
|
594
|
+
groupBoxBackend,
|
|
595
|
+
hasRepositoryResources: turnResources.some(
|
|
596
|
+
(resource) => resource.kind === "repository",
|
|
597
|
+
),
|
|
598
|
+
})
|
|
599
|
+
) {
|
|
600
|
+
startRunGitCredentialsMint();
|
|
601
|
+
const started = waitForTurnOperation(
|
|
602
|
+
sandboxState.resumeManagedGroupBox(),
|
|
603
|
+
sandboxResumeSignal,
|
|
604
|
+
releaseLateSandbox,
|
|
605
|
+
);
|
|
606
|
+
const joined = started.catch((error) => {
|
|
607
|
+
if (isLazySandboxProvisionRetryable(error)) {
|
|
608
|
+
sandboxState.prefetchedManagedBox = null;
|
|
609
|
+
}
|
|
610
|
+
throw error;
|
|
611
|
+
});
|
|
612
|
+
sandboxState.prefetchedManagedBox = joined;
|
|
613
|
+
void joined.then(
|
|
614
|
+
(box) => {
|
|
615
|
+
if (sandboxResumeSignal.aborted) return;
|
|
616
|
+
startLeaseHeartbeat(box, activeSandboxBackend ?? groupBoxBackend);
|
|
617
|
+
},
|
|
618
|
+
() => undefined,
|
|
619
|
+
);
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
} else {
|
|
623
|
+
await eventing.publish!(
|
|
624
|
+
[
|
|
625
|
+
{
|
|
626
|
+
type: "sandbox.operation.started",
|
|
627
|
+
payload: { name: "sandbox.provision" },
|
|
628
|
+
},
|
|
629
|
+
],
|
|
630
|
+
true,
|
|
631
|
+
);
|
|
632
|
+
try {
|
|
633
|
+
sandboxState.resolvedSandbox = await waitForTurnOperation(
|
|
634
|
+
resumeBoxForTurn(
|
|
635
|
+
{
|
|
636
|
+
db,
|
|
637
|
+
settings: runSettings,
|
|
638
|
+
logicalFallbackSettings: logicalSandboxSettings,
|
|
639
|
+
cancellationSignal: sandboxResumeSignal,
|
|
640
|
+
sandboxMetrics: runtimeMetricsHooksForObservability(observability),
|
|
641
|
+
onSandboxLost: publishSandboxLost,
|
|
642
|
+
objectStorage,
|
|
643
|
+
},
|
|
644
|
+
{
|
|
645
|
+
accountId: input.accountId,
|
|
646
|
+
workspaceId: input.workspaceId,
|
|
647
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
648
|
+
sessionId: input.sessionId,
|
|
649
|
+
// groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
|
|
650
|
+
// is not machine-primary resumes a real cloud group box (the
|
|
651
|
+
// deployment default), never a "selfhosted" box (which would throw
|
|
652
|
+
// for lack of a bound agentId).
|
|
653
|
+
backend: groupBoxBackend,
|
|
654
|
+
os: session.sandboxOs,
|
|
655
|
+
environment: sandboxEnvironment,
|
|
656
|
+
// IMAGE IS SHARED STATE (B3): the container image
|
|
657
|
+
// this run resolves. The lease stamps it + conflicts on a live shared box
|
|
658
|
+
// running a DIFFERENT image (solo → durable rotation; N-holders →
|
|
659
|
+
// SandboxImageConflictError surfaced as an actionable turn error). Select
|
|
660
|
+
// the image for the actual group-box backend; a configured Modal image must
|
|
661
|
+
// never override a Docker run. The selfhosted branch
|
|
662
|
+
// (establishSelfhostedTurnSession) NEVER passes
|
|
663
|
+
// an image — B3 lives only on this managed-box branch.
|
|
664
|
+
...(groupBoxImage
|
|
665
|
+
? {
|
|
666
|
+
image: groupBoxImage,
|
|
667
|
+
}
|
|
668
|
+
: {}),
|
|
669
|
+
// RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
|
|
670
|
+
// conflicts on a live shared box set up under a different rig (solo
|
|
671
|
+
// durable rotation / N-holders SandboxRigConflictError). Omitted for a
|
|
672
|
+
// rig-less turn -> never stamped or enforced (shares exactly as today).
|
|
673
|
+
...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
|
|
674
|
+
},
|
|
675
|
+
"turn",
|
|
676
|
+
managedOwnership!.holderId,
|
|
677
|
+
),
|
|
678
|
+
sandboxResumeSignal,
|
|
679
|
+
releaseLateSandbox,
|
|
680
|
+
);
|
|
681
|
+
} catch (error) {
|
|
682
|
+
await eventing.publish!(
|
|
683
|
+
[
|
|
684
|
+
{
|
|
685
|
+
type: "sandbox.operation.failed",
|
|
686
|
+
payload: {
|
|
687
|
+
name: "sandbox.provision",
|
|
688
|
+
error: error instanceof Error ? error.message : String(error),
|
|
689
|
+
},
|
|
690
|
+
},
|
|
691
|
+
],
|
|
692
|
+
true,
|
|
693
|
+
);
|
|
694
|
+
throw error;
|
|
695
|
+
}
|
|
696
|
+
sandboxState.setupBoxSession = sandboxState.resolvedSandbox.established.session;
|
|
697
|
+
// Durable box-lifecycle events (sandbox-file-persistence observability):
|
|
698
|
+
// record every box transition in session_events so the NEXT box loss is
|
|
699
|
+
// attributable from the DB alone — worker logs rotate within hours, which
|
|
700
|
+
// left both 2026-07-06 incidents without a durable trace. Best-effort.
|
|
701
|
+
await publishSandboxLifecycleEvents(sandboxState.resolvedSandbox);
|
|
702
|
+
await eventing.publish!(
|
|
703
|
+
[
|
|
704
|
+
{
|
|
705
|
+
type: "sandbox.operation.completed",
|
|
706
|
+
payload: {
|
|
707
|
+
name: "sandbox.provision",
|
|
708
|
+
...(sandboxState.resolvedSandbox.established.origin
|
|
709
|
+
? { origin: sandboxState.resolvedSandbox.established.origin }
|
|
710
|
+
: {}),
|
|
711
|
+
},
|
|
712
|
+
},
|
|
713
|
+
],
|
|
714
|
+
true,
|
|
715
|
+
);
|
|
716
|
+
// M7 hot-swap: when the selfhosted feature is on, wrap the established
|
|
717
|
+
// group box in the STABLE routing proxy before it is injected NON-OWNED
|
|
718
|
+
// into the run. The SDK binds to this ONE object once and calls its
|
|
719
|
+
// methods per tool call; the proxy re-reads (active_sandbox_id,
|
|
720
|
+
// active_epoch) per op and dispatches to the currently-active backend, so
|
|
721
|
+
// a sandbox_swap mid-turn lands the NEXT tool call on the new box. With
|
|
722
|
+
// the flag off the established group box is injected unchanged (today's
|
|
723
|
+
// path). The lease still owns the group box lifecycle — the proxy is a
|
|
724
|
+
// routing veneer, not an owner.
|
|
725
|
+
sandboxState.resolvedSandbox = {
|
|
726
|
+
...sandboxState.resolvedSandbox,
|
|
727
|
+
established: wrapTurnBoxWithRouting(
|
|
728
|
+
{
|
|
729
|
+
db,
|
|
730
|
+
settings,
|
|
731
|
+
bus,
|
|
732
|
+
opJournal,
|
|
733
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
734
|
+
onHomeSandboxLost: publishSandboxLost,
|
|
735
|
+
onHomeSandboxRebound,
|
|
736
|
+
...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
|
|
737
|
+
},
|
|
738
|
+
// Thread the SAME declared environment the group box was created with
|
|
739
|
+
// (resumeBoxForTurn, above) so a selfhosted swap target's manifest
|
|
740
|
+
// carries it too — the SDK's per-turn manifest-env delta stays empty
|
|
741
|
+
// (no "cannot change manifest environment variables" throw).
|
|
742
|
+
{
|
|
743
|
+
workspaceId: input.workspaceId,
|
|
744
|
+
sessionId: input.sessionId,
|
|
745
|
+
resourceAccountId: input.accountId,
|
|
746
|
+
...(fileAuthoritySubjectId ? { resourceSubjectId: fileAuthoritySubjectId } : {}),
|
|
747
|
+
...(fileAuthoritySubjectId
|
|
748
|
+
? {
|
|
749
|
+
personalMachineAttempt: {
|
|
750
|
+
accountId: input.accountId,
|
|
751
|
+
subjectId: fileAuthoritySubjectId,
|
|
752
|
+
turnId: turn.id,
|
|
753
|
+
attemptId: input.attemptId,
|
|
754
|
+
executionGeneration: attempt.executionGeneration,
|
|
755
|
+
},
|
|
756
|
+
}
|
|
757
|
+
: {}),
|
|
758
|
+
environment: sandboxEnvironment,
|
|
759
|
+
...(transientCodemodeEnvironment
|
|
760
|
+
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
761
|
+
: {}),
|
|
762
|
+
workspaceMutationFence: {
|
|
763
|
+
accountId: input.accountId,
|
|
764
|
+
turnId: turn.id,
|
|
765
|
+
executionGeneration: attempt.executionGeneration,
|
|
766
|
+
attemptId: input.attemptId,
|
|
767
|
+
},
|
|
768
|
+
homeLease: {
|
|
769
|
+
accountId: input.accountId,
|
|
770
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
771
|
+
leaseEpoch: sandboxState.resolvedSandbox.leaseEpoch,
|
|
772
|
+
instanceId: sandboxState.resolvedSandbox.established.instanceId,
|
|
773
|
+
backend: groupBoxBackend,
|
|
774
|
+
},
|
|
775
|
+
},
|
|
776
|
+
sandboxState.resolvedSandbox.established,
|
|
777
|
+
),
|
|
778
|
+
};
|
|
779
|
+
}
|
|
780
|
+
if (sandboxState.resolvedSandbox) {
|
|
781
|
+
startLeaseHeartbeat(sandboxState.resolvedSandbox, activeSandboxBackend ?? groupBoxBackend);
|
|
782
|
+
}
|
|
783
|
+
} catch (error) {
|
|
784
|
+
sandboxEstablishOutcome = "failed";
|
|
785
|
+
throw error;
|
|
786
|
+
} finally {
|
|
787
|
+
recordTurnStartupPhase(observability, {
|
|
788
|
+
phase: "sandbox_establish",
|
|
789
|
+
provider: turnExecutionPolicy.providerId,
|
|
790
|
+
backend: machinePrimary ? "selfhosted" : groupBoxBackend,
|
|
791
|
+
outcome: sandboxEstablishOutcome,
|
|
792
|
+
durationSeconds: (performance.now() - sandboxEstablishStartedAt) / 1_000,
|
|
793
|
+
});
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
export async function bindLazySandboxProvisioner(
|
|
799
|
+
deps: BindLazySandboxProvisionerDeps,
|
|
800
|
+
): Promise<void> {
|
|
801
|
+
const {
|
|
802
|
+
input,
|
|
803
|
+
settings,
|
|
804
|
+
db,
|
|
805
|
+
bus,
|
|
806
|
+
observability,
|
|
807
|
+
runtimeCancellationSignal,
|
|
808
|
+
sandboxResumeSignal,
|
|
809
|
+
opJournal,
|
|
810
|
+
sandboxState,
|
|
811
|
+
eventing,
|
|
812
|
+
attempt,
|
|
813
|
+
turn,
|
|
814
|
+
session,
|
|
815
|
+
fileAuthoritySubjectId,
|
|
816
|
+
runSettings,
|
|
817
|
+
sandboxEnvironment,
|
|
818
|
+
groupBoxBackend,
|
|
819
|
+
activeSandboxBackend,
|
|
820
|
+
establishPolicy,
|
|
821
|
+
startRunGitCredentialsMint,
|
|
822
|
+
machineOpObserver,
|
|
823
|
+
sandboxOperationObserver,
|
|
824
|
+
sandboxRuntime,
|
|
825
|
+
transientCodemodeEnvironment,
|
|
826
|
+
agent,
|
|
827
|
+
attachRunCredentialRenewal,
|
|
828
|
+
attachGitCredentialRenewal,
|
|
829
|
+
attachCodemodeTokenRenewal,
|
|
830
|
+
throwIfWorkerShuttingDown,
|
|
831
|
+
throwIfTurnCancelled,
|
|
832
|
+
initialRunCredentialMaterial,
|
|
833
|
+
sandboxCodemodeToken,
|
|
834
|
+
media,
|
|
835
|
+
toolResultSpill,
|
|
836
|
+
connectionScope,
|
|
837
|
+
codemodeAuthority,
|
|
838
|
+
} = deps;
|
|
839
|
+
const {
|
|
840
|
+
releaseLateSandbox,
|
|
841
|
+
onHomeSandboxRebound,
|
|
842
|
+
publishSandboxLifecycleEvents,
|
|
843
|
+
publishSandboxLost,
|
|
844
|
+
startLeaseHeartbeat,
|
|
845
|
+
runWorkspaceMutationForSandbox,
|
|
846
|
+
} = sandboxRuntime;
|
|
847
|
+
|
|
848
|
+
if (
|
|
849
|
+
establishPolicy === "on-demand" &&
|
|
850
|
+
sandboxState.sandboxHolderId &&
|
|
851
|
+
sandboxState.sandboxGroupId
|
|
852
|
+
) {
|
|
853
|
+
const agentDefaultManifest = (agent as { defaultManifest?: unknown }).defaultManifest;
|
|
854
|
+
if (!agentDefaultManifest) {
|
|
855
|
+
throw new Error("Lazy sandbox provisioning requires a SandboxAgent defaultManifest");
|
|
856
|
+
}
|
|
857
|
+
const lazyClient = {
|
|
858
|
+
backendId: sdkBackendIdForSandboxBackend(groupBoxBackend),
|
|
859
|
+
} as EstablishedSandboxSession["client"];
|
|
860
|
+
let lazySandboxEstablishmentSettled = false;
|
|
861
|
+
sandboxState.turnSandboxProvisioner = createTurnSandboxProvisioner<ResumedTurnSandbox>(
|
|
862
|
+
async () => {
|
|
863
|
+
throwIfWorkerShuttingDown();
|
|
864
|
+
throwIfTurnCancelled();
|
|
865
|
+
if (!sandboxState.resumeManagedGroupBox) {
|
|
866
|
+
throw new Error("Lazy sandbox provisioning requires a managed group-box resume");
|
|
867
|
+
}
|
|
868
|
+
startRunGitCredentialsMint();
|
|
869
|
+
const provisioned = await (sandboxState.prefetchedManagedBox ??
|
|
870
|
+
sandboxState.resumeManagedGroupBox());
|
|
871
|
+
await publishSandboxLifecycleEvents(provisioned);
|
|
872
|
+
// Return the REAL established box (NOT a copy whose session is the routing
|
|
873
|
+
// proxy). resolveActiveBackend dispatches ops to `provisioned.established.session`;
|
|
874
|
+
// if that were the proxy itself, proxy.exec -> dispatch -> resolve ->
|
|
875
|
+
// provisioner.get() -> proxy.exec -> ... loops forever (an async infinite
|
|
876
|
+
// recursion that HANGS the turn — caught live on staging 2026-07-08). The SDK
|
|
877
|
+
// already holds the proxy directly (injected as lazyOwnedSandbox.session), so it
|
|
878
|
+
// gets per-op routing; the worker-side handle (resolvedSandbox: release,
|
|
879
|
+
// heartbeat, computer-use recording) wants the real box, unproxied.
|
|
880
|
+
return provisioned;
|
|
881
|
+
},
|
|
882
|
+
{
|
|
883
|
+
signal: sandboxResumeSignal,
|
|
884
|
+
onStarted: async ({ provisionId }) => {
|
|
885
|
+
await eventing.publish?.(
|
|
886
|
+
[
|
|
887
|
+
{
|
|
888
|
+
type: "sandbox.operation.started",
|
|
889
|
+
payload: { name: "sandbox.provision", provisionId },
|
|
890
|
+
},
|
|
891
|
+
],
|
|
892
|
+
true,
|
|
893
|
+
);
|
|
894
|
+
},
|
|
895
|
+
onAttemptSettled: ({ result, error, outcome, durationMs }) => {
|
|
896
|
+
const successStage =
|
|
897
|
+
result?.established.origin === "created"
|
|
898
|
+
? "create"
|
|
899
|
+
: result?.established.origin === "resumed"
|
|
900
|
+
? "resume"
|
|
901
|
+
: result?.established.origin === "restored"
|
|
902
|
+
? "archive_recovery"
|
|
903
|
+
: "unknown";
|
|
904
|
+
const failure = error
|
|
905
|
+
? classifySandboxLogicalProvisionFailure(groupBoxBackend, error)
|
|
906
|
+
: null;
|
|
907
|
+
recordSandboxProvisionAttempt(observability, {
|
|
908
|
+
backend: groupBoxBackend,
|
|
909
|
+
stage: failure?.stage ?? successStage,
|
|
910
|
+
category: failure?.category ?? successStage,
|
|
911
|
+
outcome,
|
|
912
|
+
durationSeconds: durationMs / 1_000,
|
|
913
|
+
});
|
|
914
|
+
},
|
|
915
|
+
beforeCompleted: async (provisioned, settlement) => {
|
|
916
|
+
throwIfTurnOperationCancelled(sandboxResumeSignal);
|
|
917
|
+
startLeaseHeartbeat(provisioned, activeSandboxBackend ?? groupBoxBackend);
|
|
918
|
+
sandboxState.setupBoxSession = provisioned.established.session;
|
|
919
|
+
sandboxState.resolvedSandbox = provisioned;
|
|
920
|
+
// This durable completion and its logical metric close at actual box
|
|
921
|
+
// establishment. Credential, rig, repository, and file preparation below
|
|
922
|
+
// must never be attributed to "Starting sandbox" or provision latency.
|
|
923
|
+
await eventing.publish?.(
|
|
924
|
+
[
|
|
925
|
+
{
|
|
926
|
+
type: "sandbox.operation.completed",
|
|
927
|
+
payload: {
|
|
928
|
+
name: "sandbox.provision",
|
|
929
|
+
...(provisioned.established.origin
|
|
930
|
+
? { origin: provisioned.established.origin }
|
|
931
|
+
: {}),
|
|
932
|
+
provisionId: settlement.provisionId,
|
|
933
|
+
internalAttempts: settlement.internalAttempts,
|
|
934
|
+
},
|
|
935
|
+
},
|
|
936
|
+
],
|
|
937
|
+
true,
|
|
938
|
+
);
|
|
939
|
+
const successStage =
|
|
940
|
+
provisioned.established.origin === "created"
|
|
941
|
+
? "create"
|
|
942
|
+
: provisioned.established.origin === "resumed"
|
|
943
|
+
? "resume"
|
|
944
|
+
: provisioned.established.origin === "restored"
|
|
945
|
+
? "archive_recovery"
|
|
946
|
+
: "unknown";
|
|
947
|
+
recordSandboxLogicalProvision(observability, {
|
|
948
|
+
backend: groupBoxBackend,
|
|
949
|
+
stage: successStage,
|
|
950
|
+
category: "none",
|
|
951
|
+
outcome: "completed",
|
|
952
|
+
expected: false,
|
|
953
|
+
internalAttempts: settlement.internalAttempts,
|
|
954
|
+
durationSeconds: settlement.durationMs / 1_000,
|
|
955
|
+
});
|
|
956
|
+
lazySandboxEstablishmentSettled = true;
|
|
957
|
+
|
|
958
|
+
const lazyGitCredentials =
|
|
959
|
+
activeSandboxBackend === "selfhosted" ? undefined : await startRunGitCredentialsMint();
|
|
960
|
+
const lazyGitTokens = lazyGitCredentials?.gitTokens;
|
|
961
|
+
const lazyCodemodeToken = sandboxCodemodeToken
|
|
962
|
+
? await mintSandboxCodemodeToken(runSettings, connectionScope, codemodeAuthority)
|
|
963
|
+
: undefined;
|
|
964
|
+
await attachRunCredentialRenewal(
|
|
965
|
+
provisioned.established.session as RunCredentialCommandSession,
|
|
966
|
+
initialRunCredentialMaterial,
|
|
967
|
+
provisioned,
|
|
968
|
+
);
|
|
969
|
+
const provisionedSetupSession = initialRunCredentialMaterial
|
|
970
|
+
? withRunCredentialsSession(provisioned.established.session as object, input.sessionId)
|
|
971
|
+
: provisioned.established.session;
|
|
972
|
+
await runWorkspaceMutationForSandbox(
|
|
973
|
+
provisioned,
|
|
974
|
+
"lazyOwnedSandboxSetup",
|
|
975
|
+
async () =>
|
|
976
|
+
await runOwnedSandboxSetup(
|
|
977
|
+
agent,
|
|
978
|
+
provisioned.established.session as never,
|
|
979
|
+
provisionedSetupSession as never,
|
|
980
|
+
{
|
|
981
|
+
settings: runSettings,
|
|
982
|
+
environment: sandboxEnvironment,
|
|
983
|
+
onRuntimeEvent: async (event) => {
|
|
984
|
+
await eventing.publish?.([{ type: event.type, payload: event.payload }], true);
|
|
985
|
+
},
|
|
986
|
+
...(lazyGitTokens ? { gitTokenSeedsOverride: lazyGitTokens } : {}),
|
|
987
|
+
...(lazyGitCredentials?.bindings
|
|
988
|
+
? {
|
|
989
|
+
gitCredentialBindingsOverride: lazyGitCredentials.bindings,
|
|
990
|
+
}
|
|
991
|
+
: {}),
|
|
992
|
+
...(lazyCodemodeToken
|
|
993
|
+
? {
|
|
994
|
+
codemodeTokenSeedOverride: lazyCodemodeToken.token,
|
|
995
|
+
}
|
|
996
|
+
: {}),
|
|
997
|
+
...(eventing.toolCancellationFenceRef.current
|
|
998
|
+
? {
|
|
999
|
+
commandRunner:
|
|
1000
|
+
eventing.toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
1001
|
+
eventing.toolCancellationFenceRef.current,
|
|
1002
|
+
),
|
|
1003
|
+
}
|
|
1004
|
+
: {}),
|
|
1005
|
+
},
|
|
1006
|
+
),
|
|
1007
|
+
(measurement) => {
|
|
1008
|
+
if (eventing.firstModelRequestPreparationRecorded) return;
|
|
1009
|
+
sandboxState.firstModelPreparationNestedSandboxMs +=
|
|
1010
|
+
measurement.durationSeconds * 1_000;
|
|
1011
|
+
sandboxState.firstModelPreparationNestedSandboxPhases.push(measurement);
|
|
1012
|
+
},
|
|
1013
|
+
);
|
|
1014
|
+
await attachCodemodeTokenRenewal(
|
|
1015
|
+
provisioned.established.session as CodemodeTokenWriterSession,
|
|
1016
|
+
lazyCodemodeToken?.expiresAt,
|
|
1017
|
+
provisioned,
|
|
1018
|
+
);
|
|
1019
|
+
await attachGitCredentialRenewal(
|
|
1020
|
+
provisioned.established.session as GitCredentialTokenWriterSession,
|
|
1021
|
+
lazyGitCredentials,
|
|
1022
|
+
provisioned,
|
|
1023
|
+
);
|
|
1024
|
+
// `get()` does not release the first routed sandbox operation
|
|
1025
|
+
// until this hook returns. Only images created during this exact
|
|
1026
|
+
// turn can require deferred delivery here; historical images stay
|
|
1027
|
+
// as durable receipts and are restored explicitly when requested.
|
|
1028
|
+
for (const receipt of media.generatedImageReceiptsCreatedThisTurn.values()) {
|
|
1029
|
+
await media.materializeGeneratedImageInSandbox(
|
|
1030
|
+
receipt,
|
|
1031
|
+
provisioned,
|
|
1032
|
+
provisioned.established.session,
|
|
1033
|
+
);
|
|
1034
|
+
}
|
|
1035
|
+
await toolResultSpill.materializeDeferred();
|
|
1036
|
+
throwIfTurnOperationCancelled(sandboxResumeSignal);
|
|
1037
|
+
},
|
|
1038
|
+
onFailed: async (error, settlement) => {
|
|
1039
|
+
if (lazySandboxEstablishmentSettled) return;
|
|
1040
|
+
const failure = classifySandboxLogicalProvisionFailure(groupBoxBackend, error);
|
|
1041
|
+
await eventing.publish?.(
|
|
1042
|
+
[
|
|
1043
|
+
{
|
|
1044
|
+
type: "sandbox.operation.failed",
|
|
1045
|
+
payload: {
|
|
1046
|
+
name: "sandbox.provision",
|
|
1047
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1048
|
+
provisionId: settlement.provisionId,
|
|
1049
|
+
internalAttempts: settlement.internalAttempts,
|
|
1050
|
+
failureCategory: failure.category,
|
|
1051
|
+
failureStage: failure.stage,
|
|
1052
|
+
failureCode: failure.code,
|
|
1053
|
+
expectedTransition: failure.expected,
|
|
1054
|
+
retryable: failure.retryable,
|
|
1055
|
+
},
|
|
1056
|
+
},
|
|
1057
|
+
],
|
|
1058
|
+
true,
|
|
1059
|
+
);
|
|
1060
|
+
recordSandboxLogicalProvision(observability, {
|
|
1061
|
+
backend: groupBoxBackend,
|
|
1062
|
+
stage: failure.stage,
|
|
1063
|
+
category: failure.category,
|
|
1064
|
+
outcome: failure.expected ? "expected_transition" : "failed",
|
|
1065
|
+
expected: failure.expected,
|
|
1066
|
+
internalAttempts: settlement.internalAttempts,
|
|
1067
|
+
durationSeconds: settlement.durationMs / 1_000,
|
|
1068
|
+
});
|
|
1069
|
+
},
|
|
1070
|
+
disposeResult: async (provisioned) => {
|
|
1071
|
+
await releaseLateSandbox(provisioned);
|
|
1072
|
+
},
|
|
1073
|
+
},
|
|
1074
|
+
);
|
|
1075
|
+
sandboxState.lazyOwnedSandbox = wrapLazyTurnBoxWithRouting(
|
|
1076
|
+
{
|
|
1077
|
+
db,
|
|
1078
|
+
settings,
|
|
1079
|
+
bus,
|
|
1080
|
+
opJournal,
|
|
1081
|
+
onOp: machineOpObserver.observer,
|
|
1082
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
1083
|
+
onHomeSandboxLost: publishSandboxLost,
|
|
1084
|
+
onHomeSandboxRebound,
|
|
1085
|
+
...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
|
|
1086
|
+
},
|
|
1087
|
+
{
|
|
1088
|
+
workspaceId: input.workspaceId,
|
|
1089
|
+
sessionId: input.sessionId,
|
|
1090
|
+
resourceAccountId: input.accountId,
|
|
1091
|
+
...(fileAuthoritySubjectId ? { resourceSubjectId: fileAuthoritySubjectId } : {}),
|
|
1092
|
+
...(fileAuthoritySubjectId
|
|
1093
|
+
? {
|
|
1094
|
+
personalMachineAttempt: {
|
|
1095
|
+
accountId: input.accountId,
|
|
1096
|
+
subjectId: fileAuthoritySubjectId,
|
|
1097
|
+
turnId: turn.id,
|
|
1098
|
+
attemptId: input.attemptId,
|
|
1099
|
+
executionGeneration: attempt.executionGeneration,
|
|
1100
|
+
},
|
|
1101
|
+
}
|
|
1102
|
+
: {}),
|
|
1103
|
+
environment: sandboxEnvironment,
|
|
1104
|
+
...(transientCodemodeEnvironment
|
|
1105
|
+
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
1106
|
+
: {}),
|
|
1107
|
+
workspaceMutationFence: {
|
|
1108
|
+
accountId: input.accountId,
|
|
1109
|
+
turnId: turn.id,
|
|
1110
|
+
executionGeneration: attempt.executionGeneration,
|
|
1111
|
+
attemptId: input.attemptId,
|
|
1112
|
+
},
|
|
1113
|
+
},
|
|
1114
|
+
{
|
|
1115
|
+
client: lazyClient,
|
|
1116
|
+
backendId: sdkBackendIdForSandboxBackend(groupBoxBackend),
|
|
1117
|
+
agentDefaultManifest,
|
|
1118
|
+
provisioner: sandboxState.turnSandboxProvisioner,
|
|
1119
|
+
homeLeaseIdentity: {
|
|
1120
|
+
accountId: input.accountId,
|
|
1121
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
1122
|
+
backend: groupBoxBackend,
|
|
1123
|
+
},
|
|
1124
|
+
onFirstOperation: (measurement) => {
|
|
1125
|
+
if (eventing.firstModelRequestPreparationRecorded) return;
|
|
1126
|
+
markModelPreparationFirstSandboxOperation(measurement.durationMs / 1_000);
|
|
1127
|
+
|
|
1128
|
+
for (const nested of sandboxState.firstModelPreparationNestedSandboxPhases) {
|
|
1129
|
+
if (nested.phase === "snapshot_wait") continue;
|
|
1130
|
+
recordModelPreparationMeasurement({
|
|
1131
|
+
phase:
|
|
1132
|
+
nested.phase === "admission"
|
|
1133
|
+
? "sandbox_workspace_mutation_admission"
|
|
1134
|
+
: nested.phase === "provider"
|
|
1135
|
+
? "sandbox_workspace_mutation_provider"
|
|
1136
|
+
: "sandbox_workspace_mutation_settlement",
|
|
1137
|
+
outcome: nested.outcome,
|
|
1138
|
+
durationSeconds: nested.durationSeconds,
|
|
1139
|
+
});
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1142
|
+
const resolution = measurement.phases.resolution;
|
|
1143
|
+
if (resolution) {
|
|
1144
|
+
recordModelPreparationMeasurement({
|
|
1145
|
+
phase: "sandbox_first_routed_resolution_other",
|
|
1146
|
+
outcome: resolution.outcome,
|
|
1147
|
+
durationSeconds:
|
|
1148
|
+
Math.max(
|
|
1149
|
+
0,
|
|
1150
|
+
resolution.durationMs - sandboxState.firstModelPreparationNestedSandboxMs,
|
|
1151
|
+
) / 1_000,
|
|
1152
|
+
});
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1155
|
+
const routedPhaseNames = [
|
|
1156
|
+
["mutationAdmission", "sandbox_first_routed_mutation_admission"],
|
|
1157
|
+
["providerOperation", "sandbox_first_routed_provider_operation"],
|
|
1158
|
+
["mutationSettlement", "sandbox_first_routed_mutation_settlement"],
|
|
1159
|
+
] as const;
|
|
1160
|
+
for (const [phaseName, metricPhase] of routedPhaseNames) {
|
|
1161
|
+
const phase = measurement.phases[phaseName];
|
|
1162
|
+
if (!phase) continue;
|
|
1163
|
+
recordModelPreparationMeasurement({
|
|
1164
|
+
phase: metricPhase,
|
|
1165
|
+
outcome: phase.outcome,
|
|
1166
|
+
durationSeconds: phase.durationMs / 1_000,
|
|
1167
|
+
});
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1170
|
+
const nestedSnapshotPhases = sandboxState.firstModelPreparationNestedSandboxPhases.filter(
|
|
1171
|
+
(phase) => phase.phase === "snapshot_wait",
|
|
1172
|
+
);
|
|
1173
|
+
const routedSnapshot = measurement.phases.snapshotWait;
|
|
1174
|
+
const snapshotWaitMs =
|
|
1175
|
+
nestedSnapshotPhases.reduce(
|
|
1176
|
+
(total, phase) => total + phase.durationSeconds * 1_000,
|
|
1177
|
+
0,
|
|
1178
|
+
) + (routedSnapshot?.durationMs ?? 0);
|
|
1179
|
+
if (snapshotWaitMs > 0) {
|
|
1180
|
+
recordModelPreparationMeasurement({
|
|
1181
|
+
phase: "sandbox_snapshot_wait",
|
|
1182
|
+
outcome:
|
|
1183
|
+
routedSnapshot?.outcome === "failed" ||
|
|
1184
|
+
nestedSnapshotPhases.some((phase) => phase.outcome === "failed")
|
|
1185
|
+
? "failed"
|
|
1186
|
+
: "completed",
|
|
1187
|
+
durationSeconds: snapshotWaitMs / 1_000,
|
|
1188
|
+
});
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1191
|
+
const routedMeasuredMs = Object.values(measurement.phases).reduce(
|
|
1192
|
+
(total, phase) => total + (phase?.durationMs ?? 0),
|
|
1193
|
+
0,
|
|
1194
|
+
);
|
|
1195
|
+
const routedOtherMs = Math.max(0, measurement.durationMs - routedMeasuredMs);
|
|
1196
|
+
if (routedOtherMs > 0) {
|
|
1197
|
+
recordModelPreparationMeasurement({
|
|
1198
|
+
phase: "sandbox_first_routed_other",
|
|
1199
|
+
outcome: measurement.outcome,
|
|
1200
|
+
durationSeconds: routedOtherMs / 1_000,
|
|
1201
|
+
});
|
|
1202
|
+
}
|
|
1203
|
+
},
|
|
1204
|
+
},
|
|
1205
|
+
);
|
|
1206
|
+
}
|
|
1207
|
+
}
|