@opengeni/worker-bundle 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +222 -0
- package/dist/index.js +3960 -0
- package/dist/index.js.map +1 -0
- package/package.json +60 -0
- package/src/activities/agent-turn.ts +2290 -0
- package/src/activities/capabilities.ts +131 -0
- package/src/activities/codex-auth.ts +9 -0
- package/src/activities/codex-rotation.ts +326 -0
- package/src/activities/common.ts +37 -0
- package/src/activities/context-compaction.ts +125 -0
- package/src/activities/documents.ts +67 -0
- package/src/activities/environment.ts +209 -0
- package/src/activities/goals.ts +281 -0
- package/src/activities/packs.ts +116 -0
- package/src/activities/parent-wake.ts +129 -0
- package/src/activities/recording.ts +248 -0
- package/src/activities/redaction.ts +54 -0
- package/src/activities/run-input.ts +331 -0
- package/src/activities/sandbox-lease.ts +612 -0
- package/src/activities/scheduled-tasks.ts +346 -0
- package/src/activities/session-state.ts +201 -0
- package/src/activities/streaming.ts +74 -0
- package/src/activities/types.ts +165 -0
- package/src/activities.ts +124 -0
- package/src/index.ts +184 -0
- package/src/sandbox-resume.ts +558 -0
- package/src/sandbox-routing.ts +232 -0
- package/src/workflows/activities.ts +23 -0
- package/src/workflows/document-index.ts +11 -0
- package/src/workflows/sandbox-reaper.ts +28 -0
- package/src/workflows/scheduled-tasks.ts +38 -0
- package/src/workflows/session.ts +376 -0
- package/src/workflows.ts +19 -0
|
@@ -0,0 +1,612 @@
|
|
|
1
|
+
// apps/worker/src/activities/sandbox-lease.ts — the SOLE liveness/GC/cost-stop
|
|
2
|
+
// driver (P1.3 / OD-3).
|
|
3
|
+
//
|
|
4
|
+
// There is exactly ONE reaper activity: `reapSandboxLeases`. It is fired by the
|
|
5
|
+
// ONE global reaper Temporal Schedule (registered in apps/worker/src/index.ts).
|
|
6
|
+
// There is NO ownerHeartbeat, NO per-session timer, NO per-RPC workflow, NO
|
|
7
|
+
// *ForViewer activity, NO resolveOwnerTaskQueue. Turn-holder lifecycle is bound
|
|
8
|
+
// to Temporal *activity* liveness (the turn activity acquires/releases the turn
|
|
9
|
+
// holder); a crashed founder's leaked turn holder becomes reapable via the lease
|
|
10
|
+
// TTL — TTL-exemption means a *live* turn is never idle-reaped, NOT that a dead
|
|
11
|
+
// turn's holder is immortal.
|
|
12
|
+
//
|
|
13
|
+
// One pass per fire:
|
|
14
|
+
// 1. reapStaleLeaseHoldersGlobal (the P1.1 SECURITY-DEFINER cross-workspace
|
|
15
|
+
// sweep): TTL-reaps stale viewer holders, resets warming-death rows to cold,
|
|
16
|
+
// recomputes refcounts + enters draining at refcount 0, and RETURNS the
|
|
17
|
+
// drainable rows (workspace, group, instance, epoch) whose drain grace has
|
|
18
|
+
// elapsed at refcount 0. DB-only — no provider call inside the sweep.
|
|
19
|
+
// 2. For each drainable row: resume/attach the provider box BY ID (off the
|
|
20
|
+
// lease's resume envelope, via createSandboxClientForBackend +
|
|
21
|
+
// establishSandboxSessionFromEnvelope), call the provider terminate, then
|
|
22
|
+
// confirmDrainCold (the CAS draining->cold under the epoch fence).
|
|
23
|
+
//
|
|
24
|
+
// IDEMPOTENT + safe to run concurrently with itself: the drain CAS is guarded on
|
|
25
|
+
// (draining AND refcount=0 AND lease_epoch=expected). If another sweep already
|
|
26
|
+
// drained the row, or a late re-arm flipped it warm, or a newer epoch snuck in,
|
|
27
|
+
// confirmDrainCold returns wentCold:false and we skip — provider stop() fires
|
|
28
|
+
// ONLY when the CAS proves the box is still the draining box we observed. We
|
|
29
|
+
// confirm the CAS would still pass (re-read the lease) BEFORE the provider call
|
|
30
|
+
// so a box that was re-armed mid-sweep is never terminated out from under a live
|
|
31
|
+
// holder.
|
|
32
|
+
|
|
33
|
+
import {
|
|
34
|
+
accrueWarmSeconds,
|
|
35
|
+
confirmDrainCold,
|
|
36
|
+
forceDrainOverLimitViewerOnlyBoxes,
|
|
37
|
+
getBillingBalance,
|
|
38
|
+
listMeterableWarmLeases,
|
|
39
|
+
persistDrainSnapshot,
|
|
40
|
+
readLease,
|
|
41
|
+
reapStaleLeaseHoldersGlobal,
|
|
42
|
+
rlsContextForWorkspace,
|
|
43
|
+
type MeterableWarmLease,
|
|
44
|
+
type ReapDrainable,
|
|
45
|
+
} from "@opengeni/db";
|
|
46
|
+
import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
|
|
47
|
+
import {
|
|
48
|
+
// The reaper attaches to the box by id to terminate it. It does NOT use
|
|
49
|
+
// establishSandboxSessionFromEnvelope (which cold-RESTORES via create() on a
|
|
50
|
+
// NotFound) — restoring a box just to immediately kill it is wrong. Instead it
|
|
51
|
+
// builds the client + resumes the envelope DIRECTLY: a live box resumes, gets
|
|
52
|
+
// its /workspace PERSISTED (snapshot/tar), then is terminated; a gone box
|
|
53
|
+
// (NotFound on resume) is already down, a clean no-op.
|
|
54
|
+
createSandboxClientForBackend,
|
|
55
|
+
deletePriorPersistedSnapshot,
|
|
56
|
+
deserializeSandboxSessionStateEnvelope,
|
|
57
|
+
isProviderSandboxNotFoundError,
|
|
58
|
+
} from "@opengeni/runtime";
|
|
59
|
+
import type { ActivityServices } from "./types";
|
|
60
|
+
|
|
61
|
+
export type ReapSandboxLeasesResult = {
|
|
62
|
+
/** Stale viewer holders + warming-death rows the sweep touched is folded into
|
|
63
|
+
* the DB pass; here we report the terminate outcomes. */
|
|
64
|
+
examined: number;
|
|
65
|
+
/** Boxes whose provider terminate fired AND whose lease went cold. */
|
|
66
|
+
terminated: number;
|
|
67
|
+
/** Drainable rows skipped because the CAS no longer held (re-armed / newer
|
|
68
|
+
* epoch / already drained by a concurrent sweep) — provider stop() did NOT
|
|
69
|
+
* fire. */
|
|
70
|
+
skipped: number;
|
|
71
|
+
/** Warm viewer-only leases that accrued warm-seconds this tick (P2.1). */
|
|
72
|
+
metered: number;
|
|
73
|
+
/** Viewer-only boxes force-drained because their workspace is over a limit
|
|
74
|
+
* (0 balance / over the warm cap) — turn-held boxes are never drained (P2.1). */
|
|
75
|
+
forceDrained: number;
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
// The structural slice of a provider SandboxClient we need to resume-by-id and
|
|
79
|
+
// terminate a box. `delete(state)` is the provider's stop()/terminate (the
|
|
80
|
+
// runtime SandboxClient maps it to the per-provider teardown). Narrowed so this
|
|
81
|
+
// stays agent-loop-free.
|
|
82
|
+
type TerminableClient = {
|
|
83
|
+
backendId: string;
|
|
84
|
+
resume?: (state: unknown) => Promise<unknown>;
|
|
85
|
+
deserializeSessionState?: (state: Record<string, unknown>) => Promise<unknown>;
|
|
86
|
+
delete?: (state: unknown) => Promise<unknown>;
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
// A live session handle may expose a kill/terminate/close itself (some providers
|
|
90
|
+
// tear the box down from the session, not the client). We try the client.delete
|
|
91
|
+
// first (the canonical teardown), then fall back to a session-level terminator.
|
|
92
|
+
type TerminableSession = {
|
|
93
|
+
kill?: () => Promise<unknown>;
|
|
94
|
+
terminate?: () => Promise<unknown>;
|
|
95
|
+
close?: () => Promise<unknown>;
|
|
96
|
+
closed?: boolean;
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
type CreateSandboxClientForBackendFn = typeof createSandboxClientForBackend;
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* The CAS-write the terminate seam calls to PERSIST the captured /workspace
|
|
103
|
+
* archive onto the lease under the epoch fence, BEFORE the box is terminated
|
|
104
|
+
* (sandbox-file-persistence). The caller (`terminateDrainableBox`) closes over
|
|
105
|
+
* db/accountId/row and delegates to persistDrainSnapshot. Returns:
|
|
106
|
+
* - wrote:false -> the CAS missed (re-armed / newer epoch / vanished). The box
|
|
107
|
+
* is wanted again; the seam MUST NOT terminate it.
|
|
108
|
+
* - priorArchive -> the superseded archive (if any) to GC the prior snapshot.
|
|
109
|
+
*
|
|
110
|
+
* Pass null for archiveBase64 to CAS-check WITHOUT writing (re-arm guard for
|
|
111
|
+
* backends with no persistWorkspace — ensures a re-arm during the snapshot window
|
|
112
|
+
* aborts the terminate before client.delete()). priorArchive is always null in
|
|
113
|
+
* this case.
|
|
114
|
+
*/
|
|
115
|
+
export type PersistArchiveFn = (
|
|
116
|
+
archiveBase64: string | null,
|
|
117
|
+
) => Promise<{ wrote: boolean; priorArchive: string | null }>;
|
|
118
|
+
|
|
119
|
+
/** The provider-terminate seam. Production wires the real resume-by-id ->
|
|
120
|
+
* persistWorkspace -> persist-onto-lease (epoch-fenced) -> snapshot-GC ->
|
|
121
|
+
* provider stop() (`terminateProviderBox`); a unit test injects a spy so the
|
|
122
|
+
* drain/CAS logic is exercised against a real DB without a live provider box.
|
|
123
|
+
* Returns true when the box was terminated (or already gone), false when persist
|
|
124
|
+
* found the lease re-armed and the box was deliberately LEFT RUNNING. */
|
|
125
|
+
export type TerminateBoxFn = (
|
|
126
|
+
settings: ActivityServices["settings"],
|
|
127
|
+
lease: NonNullable<Awaited<ReturnType<typeof readLease>>>,
|
|
128
|
+
observability: ActivityServices["observability"],
|
|
129
|
+
persistArchive: PersistArchiveFn,
|
|
130
|
+
) => Promise<boolean>;
|
|
131
|
+
|
|
132
|
+
export type SandboxLeaseActivityOptions = {
|
|
133
|
+
/** Override the provider terminate (tests spy this; defaults to the real
|
|
134
|
+
* resume-by-id + provider stop()). */
|
|
135
|
+
terminateBox?: TerminateBoxFn;
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
export function createSandboxLeaseActivities(
|
|
139
|
+
services: () => Promise<ActivityServices>,
|
|
140
|
+
options: SandboxLeaseActivityOptions = {},
|
|
141
|
+
) {
|
|
142
|
+
const terminateBox: TerminateBoxFn = options.terminateBox ?? terminateProviderBox;
|
|
143
|
+
/**
|
|
144
|
+
* The one global reaper sweep. Idempotent; concurrency-safe with itself.
|
|
145
|
+
* Gated by the caller (the Schedule is only registered when
|
|
146
|
+
* sandboxOwnershipEnabled); a defensive no-op here too so a manual trigger
|
|
147
|
+
* with the flag off can never terminate a box.
|
|
148
|
+
*/
|
|
149
|
+
async function reapSandboxLeases(): Promise<ReapSandboxLeasesResult> {
|
|
150
|
+
const { db, settings, observability } = await services();
|
|
151
|
+
if (!settings.sandboxOwnershipEnabled) {
|
|
152
|
+
return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0 };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// (0) Warm-meter tick (P2.1) — accrue warm-seconds for every WARM viewer-only
|
|
156
|
+
// box (turn-held boxes meter on the turn heartbeat, so the list fn excludes
|
|
157
|
+
// them). GROUP+epoch+tick idempotent → a shared box is one stream; an
|
|
158
|
+
// overlapping/re-fired sweep cannot double-charge. Best-effort per row.
|
|
159
|
+
const metered = await accrueWarmTick(db, settings, observability);
|
|
160
|
+
|
|
161
|
+
// (0b) Per-workspace warm-cap + force-drain (P2.1) — under the usage lock, a
|
|
162
|
+
// workspace at 0 balance / over its warm cap force-drains its VIEWER-ONLY
|
|
163
|
+
// boxes (guarded turn_holders=0 — a paying turn is NEVER killed). The newly
|
|
164
|
+
// draining rows are caught by the same sweep's terminate below.
|
|
165
|
+
const forceDrained = await forceDrainOverLimitWorkspaces(db, settings, metered.workspaceIds, observability);
|
|
166
|
+
|
|
167
|
+
// (1) The DB-only cross-workspace sweep. Returns the drainable rows.
|
|
168
|
+
const drainable: ReapDrainable[] = await reapStaleLeaseHoldersGlobal(db, {
|
|
169
|
+
viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
|
|
170
|
+
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
let terminated = 0;
|
|
174
|
+
let skipped = 0;
|
|
175
|
+
|
|
176
|
+
// (2) Terminate each drainable box, then CAS draining->cold. Per-row failures
|
|
177
|
+
// are isolated: one box's provider error must not abort the whole sweep (the
|
|
178
|
+
// next sweep retries it; the provider idle-timeout is the backstop).
|
|
179
|
+
for (const row of drainable) {
|
|
180
|
+
try {
|
|
181
|
+
const drainedCold = await terminateDrainableBox(db, settings, row, observability, terminateBox);
|
|
182
|
+
if (drainedCold) {
|
|
183
|
+
terminated += 1;
|
|
184
|
+
} else {
|
|
185
|
+
skipped += 1;
|
|
186
|
+
}
|
|
187
|
+
} catch (error) {
|
|
188
|
+
skipped += 1;
|
|
189
|
+
observability.warn("sandbox reaper: terminate failed for drainable lease", {
|
|
190
|
+
workspaceId: row.workspaceId,
|
|
191
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
192
|
+
leaseEpoch: row.leaseEpoch,
|
|
193
|
+
error: error instanceof Error ? error.message : String(error),
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0) {
|
|
199
|
+
observability.info("sandbox reaper swept", {
|
|
200
|
+
drainable: drainable.length,
|
|
201
|
+
terminated,
|
|
202
|
+
skipped,
|
|
203
|
+
metered: metered.accrued,
|
|
204
|
+
forceDrained,
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained };
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
return { reapSandboxLeases };
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* The reaper-tick warm-meter pass (P2.1). Accrues warm-seconds for every WARM
|
|
216
|
+
* viewer-only lease cross-workspace (the list fn excludes turn-held boxes — those
|
|
217
|
+
* meter on the turn heartbeat). Returns the count accrued + the distinct
|
|
218
|
+
* workspaces touched (so the force-drain pass only checks workspaces that have a
|
|
219
|
+
* live warm box, not the whole fleet). Per-row best-effort: one row's metering
|
|
220
|
+
* error must not abort the sweep.
|
|
221
|
+
*/
|
|
222
|
+
async function accrueWarmTick(
|
|
223
|
+
db: ActivityServices["db"],
|
|
224
|
+
settings: ActivityServices["settings"],
|
|
225
|
+
observability: ActivityServices["observability"],
|
|
226
|
+
): Promise<{ accrued: number; workspaceIds: Set<string> }> {
|
|
227
|
+
const workspaceIds = new Set<string>();
|
|
228
|
+
let accrued = 0;
|
|
229
|
+
let leases: MeterableWarmLease[] = [];
|
|
230
|
+
try {
|
|
231
|
+
leases = await listMeterableWarmLeases(db);
|
|
232
|
+
} catch (error) {
|
|
233
|
+
observability.warn("sandbox reaper: warm-lease read failed", {
|
|
234
|
+
error: error instanceof Error ? error.message : String(error),
|
|
235
|
+
});
|
|
236
|
+
return { accrued, workspaceIds };
|
|
237
|
+
}
|
|
238
|
+
for (const lease of leases) {
|
|
239
|
+
workspaceIds.add(lease.workspaceId);
|
|
240
|
+
try {
|
|
241
|
+
const rate = sandboxWarmRateMicrosPerSecond(settings, lease.backend);
|
|
242
|
+
const result = await accrueWarmSeconds(db, {
|
|
243
|
+
accountId: lease.accountId,
|
|
244
|
+
workspaceId: lease.workspaceId,
|
|
245
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
246
|
+
expectedEpoch: lease.leaseEpoch,
|
|
247
|
+
warmRateMicrosPerSecond: rate,
|
|
248
|
+
subjectId: lease.sandboxGroupId,
|
|
249
|
+
});
|
|
250
|
+
if (result.accrued) {
|
|
251
|
+
accrued += 1;
|
|
252
|
+
}
|
|
253
|
+
} catch (error) {
|
|
254
|
+
observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
|
|
255
|
+
workspaceId: lease.workspaceId,
|
|
256
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
257
|
+
error: error instanceof Error ? error.message : String(error),
|
|
258
|
+
});
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
return { accrued, workspaceIds };
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* The per-workspace warm-cap + force-drain pass (P2.1). For each workspace with a
|
|
266
|
+
* live warm box, under the usage lock: if it is at 0 balance (when a billing /
|
|
267
|
+
* managed mode is on) or over its warm cap, force-drain its VIEWER-ONLY boxes
|
|
268
|
+
* (guarded turn_holders=0 — a paying turn is never killed). Returns the count of
|
|
269
|
+
* viewer-only boxes force-drained. Per-workspace best-effort.
|
|
270
|
+
*/
|
|
271
|
+
async function forceDrainOverLimitWorkspaces(
|
|
272
|
+
db: ActivityServices["db"],
|
|
273
|
+
settings: ActivityServices["settings"],
|
|
274
|
+
workspaceIds: Set<string>,
|
|
275
|
+
observability: ActivityServices["observability"],
|
|
276
|
+
): Promise<number> {
|
|
277
|
+
const enforceBalance = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
|
|
278
|
+
const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
|
|
279
|
+
// Nothing to enforce → skip the whole pass (no lock churn).
|
|
280
|
+
if (!enforceBalance && cap <= 0) {
|
|
281
|
+
return 0;
|
|
282
|
+
}
|
|
283
|
+
let forceDrained = 0;
|
|
284
|
+
for (const workspaceId of workspaceIds) {
|
|
285
|
+
try {
|
|
286
|
+
const { accountId } = await rlsContextForWorkspace(db, workspaceId);
|
|
287
|
+
const balance = enforceBalance ? await getBillingBalance(db, accountId) : { balanceMicros: 1 } as { balanceMicros: number };
|
|
288
|
+
const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
|
|
289
|
+
workspaceId,
|
|
290
|
+
balanceMicros: balance.balanceMicros,
|
|
291
|
+
enforceBalance,
|
|
292
|
+
maxWarmSecondsPerWorkspace: cap,
|
|
293
|
+
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
294
|
+
});
|
|
295
|
+
if (result.overLimit && result.drained.length > 0) {
|
|
296
|
+
forceDrained += result.drained.length;
|
|
297
|
+
observability.info("sandbox reaper: force-drained viewer-only boxes (over limit)", {
|
|
298
|
+
workspaceId,
|
|
299
|
+
reason: result.reason,
|
|
300
|
+
drained: result.drained.length,
|
|
301
|
+
});
|
|
302
|
+
}
|
|
303
|
+
} catch (error) {
|
|
304
|
+
observability.warn("sandbox reaper: force-drain check failed for workspace", {
|
|
305
|
+
workspaceId,
|
|
306
|
+
error: error instanceof Error ? error.message : String(error),
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
return forceDrained;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/**
|
|
314
|
+
* Terminate one drainable box by id, then CAS its lease draining->cold under the
|
|
315
|
+
* epoch fence. Returns true when the lease went cold (the box is ours to stop
|
|
316
|
+
* and was stopped), false when a concurrent sweep / re-arm / newer epoch means
|
|
317
|
+
* we must NOT stop it (provider terminate is skipped).
|
|
318
|
+
*
|
|
319
|
+
* The ordering is deliberately CAS-gated on BOTH ends:
|
|
320
|
+
* - BEFORE provider terminate: re-read the lease and assert it is STILL
|
|
321
|
+
* draining at refcount 0 at the SAME epoch we observed. A re-arm (a viewer
|
|
322
|
+
* or turn arrived during the grace window) flips it back to warm and bumps
|
|
323
|
+
* no epoch but changes liveness/refcount — we skip and never touch the box.
|
|
324
|
+
* - AFTER provider terminate: confirmDrainCold's CAS (draining AND refcount=0
|
|
325
|
+
* AND lease_epoch=expected) is the authoritative commit; if it returns false
|
|
326
|
+
* a late re-arm raced us between our re-read and the stop — but the box is
|
|
327
|
+
* already torn down, so we let the next acquire cold-restore it (NEVER a
|
|
328
|
+
* double-spawn: the lease is the singleton, the box is just gone).
|
|
329
|
+
*/
|
|
330
|
+
async function terminateDrainableBox(
|
|
331
|
+
db: ActivityServices["db"],
|
|
332
|
+
settings: ActivityServices["settings"],
|
|
333
|
+
row: ReapDrainable,
|
|
334
|
+
observability: ActivityServices["observability"],
|
|
335
|
+
terminateBox: TerminateBoxFn,
|
|
336
|
+
): Promise<boolean> {
|
|
337
|
+
// Resolve the account for the RLS-scoped confirmDrainCold (the global sweep
|
|
338
|
+
// returns no account_id; the workspace->account map is the bootstrap read).
|
|
339
|
+
const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);
|
|
340
|
+
|
|
341
|
+
// Re-read the lease for the resume envelope AND the pre-terminate CAS guard.
|
|
342
|
+
const lease = await readLease(db, row.workspaceId, row.sandboxGroupId);
|
|
343
|
+
if (!lease) {
|
|
344
|
+
// Lease vanished (cold-reaped by a concurrent sweep). Nothing to stop.
|
|
345
|
+
return false;
|
|
346
|
+
}
|
|
347
|
+
if (
|
|
348
|
+
lease.liveness !== "draining"
|
|
349
|
+
|| lease.refcount !== 0
|
|
350
|
+
|| lease.leaseEpoch !== row.leaseEpoch
|
|
351
|
+
) {
|
|
352
|
+
// Re-armed (warm again) / a newer epoch / already drained by a concurrent
|
|
353
|
+
// sweep. Skip — provider stop() must fire ONLY past the drain grace at
|
|
354
|
+
// refcount=0 on the box we observed, never while a turn or viewer holds it.
|
|
355
|
+
return false;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
// The epoch-fenced PERSIST-onto-lease CAS the terminate seam calls AFTER it has
|
|
359
|
+
// snapshotted the live box and BEFORE it terminates (sandbox-file-persistence).
|
|
360
|
+
// Same guard as confirmDrainCold (draining AND refcount=0 AND lease_epoch=
|
|
361
|
+
// expected): a re-arm or newer epoch that snuck in writes ZERO rows → wrote:false
|
|
362
|
+
// → the seam leaves the box RUNNING and we skip the cold-commit below.
|
|
363
|
+
const persistArchive: PersistArchiveFn = async (archiveBase64: string | null) =>
|
|
364
|
+
await persistDrainSnapshot(db, {
|
|
365
|
+
accountId,
|
|
366
|
+
workspaceId: row.workspaceId,
|
|
367
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
368
|
+
expectedEpoch: row.leaseEpoch,
|
|
369
|
+
workspaceArchive: archiveBase64,
|
|
370
|
+
});
|
|
371
|
+
|
|
372
|
+
// Resume-by-id -> persistWorkspace -> persist-onto-lease -> GC prior snapshot ->
|
|
373
|
+
// provider terminate. A box that is already gone (NotFound) is success (the goal
|
|
374
|
+
// is "box is not running"). A genuine PERSIST failure (could not snapshot) or a
|
|
375
|
+
// resume failure on a box that IS alive re-throws -> the lease stays draining for
|
|
376
|
+
// the next sweep (NEVER terminate a box whose files we could not capture). A
|
|
377
|
+
// persist CAS-miss (re-armed) returns false -> the box was left running; skip the
|
|
378
|
+
// cold-commit so the re-armed box is not cold-leaked.
|
|
379
|
+
const terminated = await terminateBox(settings, lease, observability, persistArchive);
|
|
380
|
+
if (!terminated) {
|
|
381
|
+
return false;
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
// The authoritative commit: CAS draining->cold under the epoch fence. If a
|
|
385
|
+
// late re-arm or newer epoch raced in after our re-read, wentCold:false and we
|
|
386
|
+
// report a skip (the box is down; a fresh acquire cold-restores it).
|
|
387
|
+
const { wentCold } = await confirmDrainCold(db, {
|
|
388
|
+
accountId,
|
|
389
|
+
workspaceId: row.workspaceId,
|
|
390
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
391
|
+
expectedEpoch: row.leaseEpoch,
|
|
392
|
+
});
|
|
393
|
+
return wentCold;
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
// The persist-capable slice of a live provider session. persistWorkspace()
|
|
397
|
+
// snapshots /workspace (snapshot_filesystem → a Modal snapshot-ref; tar → a tar
|
|
398
|
+
// archive) and returns the archive bytes. modal carries the Modal SDK client used
|
|
399
|
+
// for snapshot-image GC (deletePriorPersistedSnapshot reads session.modal).
|
|
400
|
+
type PersistableSession = TerminableSession & {
|
|
401
|
+
persistWorkspace?: () => Promise<Uint8Array | undefined>;
|
|
402
|
+
};
|
|
403
|
+
|
|
404
|
+
/**
|
|
405
|
+
* Resume/attach the box by id, PERSIST its /workspace, fold the snapshot onto the
|
|
406
|
+
* lease under the epoch fence, GC the superseded snapshot, THEN terminate. Resumes
|
|
407
|
+
* DIRECTLY (no cold-restore fallback) so a gone box is a clean no-op, never a
|
|
408
|
+
* wasteful create-then-kill:
|
|
409
|
+
* - build the client for the lease's backend;
|
|
410
|
+
* - deserialize + resume the envelope (warm reattach by id, R4-safe);
|
|
411
|
+
* - session.persistWorkspace() -> capture the /workspace archive;
|
|
412
|
+
* - persistArchive(base64) -> CAS-fold it onto the lease (epoch-fenced);
|
|
413
|
+
* - GC the PRIOR snapshot (keep-latest-per-lease);
|
|
414
|
+
* - terminate the live handle (client.delete / session kill|terminate|close).
|
|
415
|
+
*
|
|
416
|
+
* Returns true when the box was terminated (or was already gone), false when the
|
|
417
|
+
* persist CAS found the lease re-armed and the box was deliberately LEFT RUNNING.
|
|
418
|
+
*
|
|
419
|
+
* Failure discipline (NEVER lose files, NEVER leak a box):
|
|
420
|
+
* - resume NotFound -> the box is already down; SUCCESS (return true), nothing
|
|
421
|
+
* to persist, the caller colds the lease.
|
|
422
|
+
* - a transient/auth/network resume failure on a box that may be ALIVE -> throw
|
|
423
|
+
* (the caller skips; the lease stays draining for the next sweep).
|
|
424
|
+
* - a persistWorkspace failure (snapshot timeout / provider error) -> throw
|
|
425
|
+
* BEFORE any terminate, so the box is NOT torn down with un-captured files;
|
|
426
|
+
* the next sweep retries, and the provider idle-timeout is the ultimate
|
|
427
|
+
* backstop. We never terminate a box whose snapshot we could not capture.
|
|
428
|
+
* For a draining lease with no envelope (a warming-death row that committed no
|
|
429
|
+
* box, or a 'none'-backed group) there is no live box — a no-op (return true).
|
|
430
|
+
*/
|
|
431
|
+
export async function terminateProviderBox(
|
|
432
|
+
settings: ActivityServices["settings"],
|
|
433
|
+
lease: NonNullable<Awaited<ReturnType<typeof readLease>>>,
|
|
434
|
+
observability: ActivityServices["observability"],
|
|
435
|
+
persistArchive: PersistArchiveFn,
|
|
436
|
+
createClientForBackend: CreateSandboxClientForBackendFn = createSandboxClientForBackend,
|
|
437
|
+
): Promise<boolean> {
|
|
438
|
+
const backend = (lease.resumeBackendId ?? lease.backend) as string;
|
|
439
|
+
// 'none' / no backend -> nothing to terminate.
|
|
440
|
+
if (!backend || backend === "none") {
|
|
441
|
+
return true;
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
// CRITICAL SAFETY (bring-your-own-compute, dossier §19/§21): NEVER provider-stop
|
|
445
|
+
// a selfhosted box. The "box" is a user's PHYSICAL machine — you cannot kill it,
|
|
446
|
+
// and a delete()/kill() reaching the agent would be catastrophic. The reaper
|
|
447
|
+
// DRAINS the lease to cold only (refcount→0 → draining → this returns true so
|
|
448
|
+
// the caller's confirmDrainCold flips draining→cold) but issues NO provider stop:
|
|
449
|
+
// no client is built, no resume, no persistWorkspace (the machine IS the
|
|
450
|
+
// persistence — nothing to snapshot), no delete/kill. The session simply detaches;
|
|
451
|
+
// the machine stays up under the agent's own process lifetime (§23.0). Checked on
|
|
452
|
+
// BOTH the lease backend and the resume envelope's backend so neither path leaks.
|
|
453
|
+
if (backend === "selfhosted" || lease.backend === "selfhosted" || lease.resumeBackendId === "selfhosted") {
|
|
454
|
+
observability.info("sandbox reaper: selfhosted lease drained to cold (NEVER provider-stopped — it is the user's machine)", {
|
|
455
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
456
|
+
backend,
|
|
457
|
+
});
|
|
458
|
+
return true;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
// resume_state is the folded group box-envelope (the provider sessionState the
|
|
462
|
+
// box was last persisted as). No envelope -> no live box to stop.
|
|
463
|
+
if (!lease.resumeState) {
|
|
464
|
+
return true;
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
const client = createClientForBackend(backend as never, settings) as TerminableClient | undefined;
|
|
468
|
+
if (!client) {
|
|
469
|
+
// 'none' backend resolved to no client.
|
|
470
|
+
return true;
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
// Resume by id (warm reattach) — NO cold-restore. A NotFound here = the box is
|
|
474
|
+
// already gone; success.
|
|
475
|
+
let session: PersistableSession | undefined;
|
|
476
|
+
let sessionState: unknown;
|
|
477
|
+
try {
|
|
478
|
+
if (!client.resume || !client.deserializeSessionState) {
|
|
479
|
+
// A backend that cannot resume cannot be attached-to for terminate; the
|
|
480
|
+
// provider idle-timeout is the backstop. No-op.
|
|
481
|
+
return true;
|
|
482
|
+
}
|
|
483
|
+
// resume_state is the lease ENVELOPE: `{ backendId, sessionState: {
|
|
484
|
+
// providerState: { sandboxId, ... }, manifest, ... } }` (the shape
|
|
485
|
+
// serializeEstablishedSandboxEnvelope folds onto the lease). The provider
|
|
486
|
+
// payload deserializeSandboxSessionStateEnvelope re-hydrates is the INNER
|
|
487
|
+
// `sessionState` — pass the WHOLE envelope and it reads `state.providerState`
|
|
488
|
+
// at the top level, finds nothing (providerState is nested under
|
|
489
|
+
// `sessionState`), drops `sandboxId`, and `client.resume(state)` throws
|
|
490
|
+
// "requires a persisted sandboxId". This is exactly what the working
|
|
491
|
+
// resume-by-id paths (establishSandboxSessionFromEnvelope) avoid: they unwrap
|
|
492
|
+
// `envelope.sessionState` first. Mirror that. (`?? lease.resumeState` keeps a
|
|
493
|
+
// legacy flat envelope — providerState at the top — resumable too.)
|
|
494
|
+
const envelopeSessionState =
|
|
495
|
+
(lease.resumeState as { sessionState?: unknown }).sessionState ?? lease.resumeState;
|
|
496
|
+
const resumedState = await deserializeSandboxSessionStateEnvelope(client as never, envelopeSessionState);
|
|
497
|
+
if (resumedState === undefined) {
|
|
498
|
+
return true;
|
|
499
|
+
}
|
|
500
|
+
session = (await client.resume(resumedState)) as PersistableSession;
|
|
501
|
+
sessionState = resumedState;
|
|
502
|
+
} catch (error) {
|
|
503
|
+
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
504
|
+
observability.info("sandbox reaper: drainable box already gone (NotFound on resume) — proceeding to cold", {
|
|
505
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
506
|
+
backend,
|
|
507
|
+
});
|
|
508
|
+
return true;
|
|
509
|
+
}
|
|
510
|
+
// Re-throw a non-NotFound resume failure so the caller SKIPS (the lease stays
|
|
511
|
+
// draining for the next sweep) — never cold a box we could not prove is gone.
|
|
512
|
+
throw error;
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
// PERSIST /workspace BEFORE terminating (sandbox-file-persistence). A failure to
|
|
516
|
+
// snapshot must re-throw BEFORE any terminate so files are never lost — the next
|
|
517
|
+
// sweep retries, the provider idle-timeout is the backstop. A NotFound here (the
|
|
518
|
+
// box raced gone between resume and persist) is success: nothing to persist.
|
|
519
|
+
let archiveBytes: Uint8Array | undefined;
|
|
520
|
+
try {
|
|
521
|
+
archiveBytes = session?.persistWorkspace ? await session.persistWorkspace() : undefined;
|
|
522
|
+
} catch (error) {
|
|
523
|
+
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
524
|
+
observability.info("sandbox reaper: box gone during persist (NotFound) — nothing to snapshot, proceeding to cold", {
|
|
525
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
526
|
+
backend,
|
|
527
|
+
});
|
|
528
|
+
return true;
|
|
529
|
+
}
|
|
530
|
+
observability.warn("sandbox reaper: persistWorkspace failed — leaving box draining (files NOT lost)", {
|
|
531
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
532
|
+
backend,
|
|
533
|
+
error: error instanceof Error ? error.message : String(error),
|
|
534
|
+
});
|
|
535
|
+
// NEVER terminate a box whose snapshot we could not capture.
|
|
536
|
+
throw error;
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
// Fold the captured archive onto the lease under the epoch fence, then GC the
|
|
540
|
+
// superseded snapshot. A CAS miss (wrote:false) means the lease was re-armed
|
|
541
|
+
// mid-drain: the box is wanted again — DO NOT terminate it (return false; the
|
|
542
|
+
// caller skips confirmDrainCold).
|
|
543
|
+
//
|
|
544
|
+
// Re-arm guard for no-archive path: even when persistWorkspace returned no bytes
|
|
545
|
+
// (a backend with no persistWorkspace, or an empty result), we MUST still CAS-
|
|
546
|
+
// check before delete(). The snapshot window (resume → persistWorkspace) can be
|
|
547
|
+
// long; a late acquireLease re-arm (draining→warm, same epoch) can land in it,
|
|
548
|
+
// so without this check we would delete a box the lease now treats as live. The
|
|
549
|
+
// null-archive path of persistArchive does exactly this: FOR UPDATE + liveness/
|
|
550
|
+
// refcount/epoch guard, no write. wrote:false → abort the terminate.
|
|
551
|
+
if (archiveBytes && archiveBytes.length > 0) {
|
|
552
|
+
const archiveBase64 = Buffer.from(archiveBytes).toString("base64");
|
|
553
|
+
const { wrote, priorArchive } = await persistArchive(archiveBase64);
|
|
554
|
+
if (!wrote) {
|
|
555
|
+
observability.info("sandbox reaper: lease re-armed during persist — leaving box RUNNING (no terminate)", {
|
|
556
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
557
|
+
backend,
|
|
558
|
+
});
|
|
559
|
+
return false;
|
|
560
|
+
}
|
|
561
|
+
// Keep-latest-per-lease GC: best-effort delete the prior snapshot image now,
|
|
562
|
+
// while the session (and its Modal client) is still live. Never throws.
|
|
563
|
+
const deleted = await deletePriorPersistedSnapshot(session, priorArchive);
|
|
564
|
+
if (deleted) {
|
|
565
|
+
observability.info("sandbox reaper: GC'd superseded workspace snapshot", {
|
|
566
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
567
|
+
backend,
|
|
568
|
+
snapshotId: deleted,
|
|
569
|
+
});
|
|
570
|
+
}
|
|
571
|
+
} else {
|
|
572
|
+
// No archive produced (backend has no persistWorkspace or returned empty).
|
|
573
|
+
// CAS-check via persistArchive(null): if the lease was re-armed during the
|
|
574
|
+
// snapshot window, wrote:false → leave the box RUNNING (abort terminate).
|
|
575
|
+
const { wrote } = await persistArchive(null);
|
|
576
|
+
if (!wrote) {
|
|
577
|
+
observability.info("sandbox reaper: lease re-armed during snapshot window (no-archive path) — leaving box RUNNING (no terminate)", {
|
|
578
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
579
|
+
backend,
|
|
580
|
+
});
|
|
581
|
+
return false;
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
// Provider terminate. Prefer the client.delete(state) teardown (the canonical
|
|
586
|
+
// provider stop()); fall back to a session-level kill/terminate/close. A
|
|
587
|
+
// terminate that fails because the box is already gone is success.
|
|
588
|
+
try {
|
|
589
|
+
if (client.delete && sessionState !== undefined) {
|
|
590
|
+
await client.delete(sessionState);
|
|
591
|
+
return true;
|
|
592
|
+
}
|
|
593
|
+
if (session?.kill) {
|
|
594
|
+
await session.kill();
|
|
595
|
+
return true;
|
|
596
|
+
}
|
|
597
|
+
if (session?.terminate) {
|
|
598
|
+
await session.terminate();
|
|
599
|
+
return true;
|
|
600
|
+
}
|
|
601
|
+
if (session?.close && !session.closed) {
|
|
602
|
+
await session.close();
|
|
603
|
+
}
|
|
604
|
+
} catch (error) {
|
|
605
|
+
observability.warn("sandbox reaper: provider terminate raised (box may already be gone)", {
|
|
606
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
607
|
+
backend,
|
|
608
|
+
error: error instanceof Error ? error.message : String(error),
|
|
609
|
+
});
|
|
610
|
+
}
|
|
611
|
+
return true;
|
|
612
|
+
}
|