@opengeni/worker-bundle 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -0
- package/dist/index.js +215 -40
- package/dist/index.js.map +1 -1
- package/package.json +8 -8
- package/src/activities/agent-turn.ts +4 -0
- package/src/activities/sandbox-lease.ts +97 -3
- package/src/sandbox-resume.ts +178 -59
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@opengeni/worker-bundle",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "OpenGeni worker entry: runOpenGeniWorker/createOpenGeniWorker(deps) + the un-bundled Temporal workflow source tree. An engine-distribution surface — its runtime closure includes engine-internal packages and the workflow bundle ships as on-disk TS source.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -38,14 +38,14 @@
|
|
|
38
38
|
},
|
|
39
39
|
"dependencies": {
|
|
40
40
|
"@opengeni/codex": "^0.2.1",
|
|
41
|
-
"@opengeni/config": "^0.2.
|
|
42
|
-
"@opengeni/contracts": "^0.
|
|
43
|
-
"@opengeni/db": "^0.
|
|
44
|
-
"@opengeni/documents": "^0.2.
|
|
45
|
-
"@opengeni/events": "^0.2.
|
|
41
|
+
"@opengeni/config": "^0.2.4",
|
|
42
|
+
"@opengeni/contracts": "^0.6.0",
|
|
43
|
+
"@opengeni/db": "^0.4.0",
|
|
44
|
+
"@opengeni/documents": "^0.2.4",
|
|
45
|
+
"@opengeni/events": "^0.2.4",
|
|
46
46
|
"@opengeni/observability": "^0.2.1",
|
|
47
|
-
"@opengeni/runtime": "^0.
|
|
48
|
-
"@opengeni/storage": "^0.2.
|
|
47
|
+
"@opengeni/runtime": "^0.3.0",
|
|
48
|
+
"@opengeni/storage": "^0.2.4",
|
|
49
49
|
"@temporalio/activity": "^1.17.0",
|
|
50
50
|
"@temporalio/client": "^1.17.0",
|
|
51
51
|
"@temporalio/worker": "^1.17.0",
|
|
@@ -1335,6 +1335,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1335
1335
|
: { computerToolMode: computerToolModeForTurn(null) }),
|
|
1336
1336
|
...(packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {}),
|
|
1337
1337
|
...(workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {}),
|
|
1338
|
+
// Per-session persona tier (session > workspace > deployment default).
|
|
1339
|
+
// Composed system-level AFTER the workspace persona so it refines it for
|
|
1340
|
+
// this one session; absent ⇒ byte-identical to today's composition.
|
|
1341
|
+
...(session.instructions ? { sessionInstructions: session.instructions } : {}),
|
|
1338
1342
|
...(workspaceEnvironment
|
|
1339
1343
|
? {
|
|
1340
1344
|
workspaceEnvironment: {
|
|
@@ -35,6 +35,7 @@ import {
|
|
|
35
35
|
confirmDrainCold,
|
|
36
36
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
37
37
|
getBillingBalance,
|
|
38
|
+
listLiveModalSandboxLeaseAttributions,
|
|
38
39
|
listMeterableWarmLeases,
|
|
39
40
|
persistDrainSnapshot,
|
|
40
41
|
readLease,
|
|
@@ -55,6 +56,8 @@ import {
|
|
|
55
56
|
deletePriorPersistedSnapshot,
|
|
56
57
|
deserializeSandboxSessionStateEnvelope,
|
|
57
58
|
isProviderSandboxNotFoundError,
|
|
59
|
+
sweepModalOrphanSandboxes,
|
|
60
|
+
terminateModalSandboxById,
|
|
58
61
|
} from "@opengeni/runtime";
|
|
59
62
|
import type { ActivityServices } from "./types";
|
|
60
63
|
|
|
@@ -73,6 +76,8 @@ export type ReapSandboxLeasesResult = {
|
|
|
73
76
|
/** Viewer-only boxes force-drained because their workspace is over a limit
|
|
74
77
|
* (0 balance / over the warm cap) — turn-held boxes are never drained (P2.1). */
|
|
75
78
|
forceDrained: number;
|
|
79
|
+
/** Provider-side Modal orphan sandboxes terminated by the defensive sweep. */
|
|
80
|
+
modalOrphansTerminated: number;
|
|
76
81
|
};
|
|
77
82
|
|
|
78
83
|
// The structural slice of a provider SandboxClient we need to resume-by-id and
|
|
@@ -129,10 +134,19 @@ export type TerminateBoxFn = (
|
|
|
129
134
|
persistArchive: PersistArchiveFn,
|
|
130
135
|
) => Promise<boolean>;
|
|
131
136
|
|
|
137
|
+
export type SweepModalOrphansFn = (
|
|
138
|
+
settings: ActivityServices["settings"],
|
|
139
|
+
db: ActivityServices["db"],
|
|
140
|
+
observability: ActivityServices["observability"],
|
|
141
|
+
) => Promise<number>;
|
|
142
|
+
|
|
132
143
|
export type SandboxLeaseActivityOptions = {
|
|
133
144
|
/** Override the provider terminate (tests spy this; defaults to the real
|
|
134
145
|
* resume-by-id + provider stop()). */
|
|
135
146
|
terminateBox?: TerminateBoxFn;
|
|
147
|
+
/** Override the provider-side Modal orphan sweep (tests spy this; defaults to
|
|
148
|
+
* Modal list+tag comparison when Modal is configured). */
|
|
149
|
+
sweepModalOrphans?: SweepModalOrphansFn;
|
|
136
150
|
};
|
|
137
151
|
|
|
138
152
|
export function createSandboxLeaseActivities(
|
|
@@ -140,6 +154,7 @@ export function createSandboxLeaseActivities(
|
|
|
140
154
|
options: SandboxLeaseActivityOptions = {},
|
|
141
155
|
) {
|
|
142
156
|
const terminateBox: TerminateBoxFn = options.terminateBox ?? terminateProviderBox;
|
|
157
|
+
const sweepModalOrphans: SweepModalOrphansFn = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
143
158
|
/**
|
|
144
159
|
* The one global reaper sweep. Idempotent; concurrency-safe with itself.
|
|
145
160
|
* Gated by the caller (the Schedule is only registered when
|
|
@@ -149,7 +164,7 @@ export function createSandboxLeaseActivities(
|
|
|
149
164
|
async function reapSandboxLeases(): Promise<ReapSandboxLeasesResult> {
|
|
150
165
|
const { db, settings, observability } = await services();
|
|
151
166
|
if (!settings.sandboxOwnershipEnabled) {
|
|
152
|
-
return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0 };
|
|
167
|
+
return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
|
|
153
168
|
}
|
|
154
169
|
|
|
155
170
|
// (0) Warm-meter tick (P2.1) — accrue warm-seconds for every WARM viewer-only
|
|
@@ -172,6 +187,7 @@ export function createSandboxLeaseActivities(
|
|
|
172
187
|
|
|
173
188
|
let terminated = 0;
|
|
174
189
|
let skipped = 0;
|
|
190
|
+
let modalOrphansTerminated = 0;
|
|
175
191
|
|
|
176
192
|
// (2) Terminate each drainable box, then CAS draining->cold. Per-row failures
|
|
177
193
|
// are isolated: one box's provider error must not abort the whole sweep (the
|
|
@@ -195,17 +211,26 @@ export function createSandboxLeaseActivities(
|
|
|
195
211
|
}
|
|
196
212
|
}
|
|
197
213
|
|
|
198
|
-
|
|
214
|
+
try {
|
|
215
|
+
modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
|
|
216
|
+
} catch (error) {
|
|
217
|
+
observability.warn("sandbox reaper: Modal orphan sweep failed", {
|
|
218
|
+
error: error instanceof Error ? error.message : String(error),
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
|
|
199
223
|
observability.info("sandbox reaper swept", {
|
|
200
224
|
drainable: drainable.length,
|
|
201
225
|
terminated,
|
|
202
226
|
skipped,
|
|
203
227
|
metered: metered.accrued,
|
|
204
228
|
forceDrained,
|
|
229
|
+
modalOrphansTerminated,
|
|
205
230
|
});
|
|
206
231
|
}
|
|
207
232
|
|
|
208
|
-
return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained };
|
|
233
|
+
return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained, modalOrphansTerminated };
|
|
209
234
|
}
|
|
210
235
|
|
|
211
236
|
return { reapSandboxLeases };
|
|
@@ -310,6 +335,46 @@ async function forceDrainOverLimitWorkspaces(
|
|
|
310
335
|
return forceDrained;
|
|
311
336
|
}
|
|
312
337
|
|
|
338
|
+
async function sweepModalOrphansForConfiguredBackend(
|
|
339
|
+
settings: ActivityServices["settings"],
|
|
340
|
+
db: ActivityServices["db"],
|
|
341
|
+
observability: ActivityServices["observability"],
|
|
342
|
+
): Promise<number> {
|
|
343
|
+
// Keep the provider-side sweep tightly scoped to deployments that have a Modal
|
|
344
|
+
// app path configured. Local/docker-only workers should not attempt Modal API
|
|
345
|
+
// calls just because modalAppName has a default.
|
|
346
|
+
if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret) {
|
|
347
|
+
return 0;
|
|
348
|
+
}
|
|
349
|
+
let liveLeases: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>;
|
|
350
|
+
try {
|
|
351
|
+
liveLeases = await listLiveModalSandboxLeaseAttributions(db);
|
|
352
|
+
} catch (error) {
|
|
353
|
+
observability.warn("sandbox reaper: live Modal lease attribution read failed", {
|
|
354
|
+
error: error instanceof Error ? error.message : String(error),
|
|
355
|
+
});
|
|
356
|
+
return 0;
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
const result = await sweepModalOrphanSandboxes(settings, liveLeases);
|
|
360
|
+
for (const terminated of result.terminated) {
|
|
361
|
+
observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
|
|
362
|
+
sandboxId: terminated.sandboxId,
|
|
363
|
+
reason: terminated.reason,
|
|
364
|
+
tags: JSON.stringify(terminated.tags),
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
if (result.examined > 0) {
|
|
368
|
+
observability.info("sandbox reaper: Modal orphan sweep completed", {
|
|
369
|
+
examined: result.examined,
|
|
370
|
+
terminated: result.terminated.length,
|
|
371
|
+
skipped: result.skipped,
|
|
372
|
+
appName: settings.modalAppName,
|
|
373
|
+
});
|
|
374
|
+
}
|
|
375
|
+
return result.terminated.length;
|
|
376
|
+
}
|
|
377
|
+
|
|
313
378
|
/**
|
|
314
379
|
* Terminate one drainable box by id, then CAS its lease draining->cold under the
|
|
315
380
|
* epoch fence. Returns true when the lease went cold (the box is ours to stop
|
|
@@ -458,6 +523,35 @@ export async function terminateProviderBox(
|
|
|
458
523
|
return true;
|
|
459
524
|
}
|
|
460
525
|
|
|
526
|
+
// A warming-death row can have a provider instance id recorded immediately
|
|
527
|
+
// after create() but no resumable envelope yet. For Modal, instance_id is
|
|
528
|
+
// enough to terminate directly; CAS-check first so a re-arm during the sweep
|
|
529
|
+
// leaves the box running.
|
|
530
|
+
if (backend === "modal" && !lease.resumeState && lease.instanceId) {
|
|
531
|
+
const { wrote } = await persistArchive(null);
|
|
532
|
+
if (!wrote) {
|
|
533
|
+
observability.info("sandbox reaper: Modal lease re-armed before direct terminate — leaving sandbox RUNNING", {
|
|
534
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
535
|
+
backend,
|
|
536
|
+
instanceId: lease.instanceId,
|
|
537
|
+
});
|
|
538
|
+
return false;
|
|
539
|
+
}
|
|
540
|
+
try {
|
|
541
|
+
await terminateModalSandboxById(settings, lease.instanceId);
|
|
542
|
+
} catch (error) {
|
|
543
|
+
if (!isProviderSandboxNotFoundError("modal", error)) {
|
|
544
|
+
throw error;
|
|
545
|
+
}
|
|
546
|
+
observability.info("sandbox reaper: Modal sandbox already gone during direct terminate", {
|
|
547
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
548
|
+
backend,
|
|
549
|
+
instanceId: lease.instanceId,
|
|
550
|
+
});
|
|
551
|
+
}
|
|
552
|
+
return true;
|
|
553
|
+
}
|
|
554
|
+
|
|
461
555
|
// resume_state is the folded group box-envelope (the provider sessionState the
|
|
462
556
|
// box was last persisted as). No envelope -> no live box to stop.
|
|
463
557
|
if (!lease.resumeState) {
|
package/src/sandbox-resume.ts
CHANGED
|
@@ -28,6 +28,7 @@ import {
|
|
|
28
28
|
failWarmingToCold,
|
|
29
29
|
getSandboxSessionEnvelope,
|
|
30
30
|
readLease,
|
|
31
|
+
recordWarmingSandboxCreated,
|
|
31
32
|
releaseLeaseHolder,
|
|
32
33
|
SandboxLeaseSupersededError,
|
|
33
34
|
type Database,
|
|
@@ -42,6 +43,8 @@ import {
|
|
|
42
43
|
DisplayStackUnsupportedError,
|
|
43
44
|
buildStreamUrl,
|
|
44
45
|
StreamPortUnavailableError,
|
|
46
|
+
modalSandboxAttributionEnvironment,
|
|
47
|
+
tagModalSandbox,
|
|
45
48
|
type EstablishedSandboxSession,
|
|
46
49
|
type ExposedPortEndpoint,
|
|
47
50
|
} from "@opengeni/runtime";
|
|
@@ -105,14 +108,102 @@ export type ResumedTurnSandbox = {
|
|
|
105
108
|
release: () => Promise<void>;
|
|
106
109
|
};
|
|
107
110
|
|
|
108
|
-
|
|
109
|
-
|
|
111
|
+
export class SandboxWarmingTimeoutError extends Error {
|
|
112
|
+
readonly code = "sandbox_warming_timeout";
|
|
113
|
+
|
|
114
|
+
constructor(public readonly backend: string, public readonly timeoutMs: number) {
|
|
115
|
+
super(
|
|
116
|
+
`Sandbox backend "${backend}" capacity or creation timed out after ${Math.ceil(timeoutMs / 1000)}s while warming the sandbox lease. Please try again; if this persists, sandbox capacity may be exhausted.`,
|
|
117
|
+
);
|
|
118
|
+
this.name = "SandboxWarmingTimeoutError";
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// Bounded poll while a sibling spawner is mid cold-restore. The wait budget is
|
|
123
|
+
// user-facing and separate from the lease TTL heartbeat/reaper horizon.
|
|
110
124
|
const WARMING_POLL_INTERVAL_MS = 250;
|
|
111
125
|
|
|
112
126
|
async function sleep(ms: number): Promise<void> {
|
|
113
127
|
await new Promise<void>((resolve) => setTimeout(resolve, ms));
|
|
114
128
|
}
|
|
115
129
|
|
|
130
|
+
async function terminateEstablishedSandbox(established: EstablishedSandboxSession | null): Promise<boolean> {
|
|
131
|
+
if (!established) {
|
|
132
|
+
return true;
|
|
133
|
+
}
|
|
134
|
+
const client = established.client as { delete?: (state: unknown) => Promise<unknown> };
|
|
135
|
+
if (typeof client.delete === "function" && established.sessionState !== undefined) {
|
|
136
|
+
try {
|
|
137
|
+
await client.delete(established.sessionState);
|
|
138
|
+
return true;
|
|
139
|
+
} catch {
|
|
140
|
+
return false;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
const session = established.session as {
|
|
144
|
+
close?: () => Promise<unknown>;
|
|
145
|
+
terminate?: () => Promise<unknown>;
|
|
146
|
+
kill?: () => Promise<unknown>;
|
|
147
|
+
closed?: boolean;
|
|
148
|
+
};
|
|
149
|
+
try {
|
|
150
|
+
if (session.terminate) {
|
|
151
|
+
await session.terminate();
|
|
152
|
+
return true;
|
|
153
|
+
} else if (session.kill) {
|
|
154
|
+
await session.kill();
|
|
155
|
+
return true;
|
|
156
|
+
} else if (session.close) {
|
|
157
|
+
if (!session.closed) {
|
|
158
|
+
await session.close();
|
|
159
|
+
}
|
|
160
|
+
return true;
|
|
161
|
+
}
|
|
162
|
+
return false;
|
|
163
|
+
} catch {
|
|
164
|
+
// Best-effort cleanup. A provider-side orphan sweep is the backstop.
|
|
165
|
+
return false;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function asSandboxWarmingError(error: unknown, backend: string, timeoutMs: number): unknown {
|
|
170
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
171
|
+
return /sandbox creation timed out|warming timed out|capacity.*timed out/i.test(message)
|
|
172
|
+
? new SandboxWarmingTimeoutError(backend, timeoutMs)
|
|
173
|
+
: error;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
function workspaceArchiveFromEnvelope(envelope: Record<string, unknown> | null | undefined): string | null {
|
|
177
|
+
const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null
|
|
178
|
+
? envelope.sessionState as Record<string, unknown>
|
|
179
|
+
: null;
|
|
180
|
+
const archive = sessionState?.workspaceArchive;
|
|
181
|
+
return typeof archive === "string" && archive.length > 0 ? archive : null;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function preserveWorkspaceArchiveOnInterimResumeState(
|
|
185
|
+
resumeState: Record<string, unknown> | null,
|
|
186
|
+
archiveSource: Record<string, unknown> | null,
|
|
187
|
+
): Record<string, unknown> | null {
|
|
188
|
+
const archive = workspaceArchiveFromEnvelope(archiveSource);
|
|
189
|
+
if (!archive) {
|
|
190
|
+
return resumeState;
|
|
191
|
+
}
|
|
192
|
+
const existingSessionState = resumeState && typeof resumeState.sessionState === "object" && resumeState.sessionState !== null
|
|
193
|
+
? resumeState.sessionState as Record<string, unknown>
|
|
194
|
+
: {};
|
|
195
|
+
return {
|
|
196
|
+
...(resumeState ?? {}),
|
|
197
|
+
...(resumeState?.backendId === undefined && archiveSource?.backendId !== undefined
|
|
198
|
+
? { backendId: archiveSource.backendId }
|
|
199
|
+
: {}),
|
|
200
|
+
sessionState: {
|
|
201
|
+
...existingSessionState,
|
|
202
|
+
workspaceArchive: archive,
|
|
203
|
+
},
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
|
|
116
207
|
/**
|
|
117
208
|
* Resume the one box for a single turn (or any worker-side resume op). Returns
|
|
118
209
|
* the live non-owned session + the fence epoch + a release fn. The CALLER owns
|
|
@@ -178,6 +269,14 @@ export async function resumeBoxForTurn(
|
|
|
178
269
|
// expose the stream port, then commit warm (lease_epoch++).
|
|
179
270
|
if (acquired.role === "spawner") {
|
|
180
271
|
const expectedEpoch = acquired.lease.leaseEpoch;
|
|
272
|
+
let createdEstablished: EstablishedSandboxSession | null = null;
|
|
273
|
+
if (ids.environment && ids.backend === "modal") {
|
|
274
|
+
Object.assign(ids.environment, modalSandboxAttributionEnvironment({
|
|
275
|
+
leaseId: acquired.lease.id,
|
|
276
|
+
workspaceId: ids.workspaceId,
|
|
277
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
278
|
+
}));
|
|
279
|
+
}
|
|
181
280
|
try {
|
|
182
281
|
const envelope = await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
|
|
183
282
|
// Prefer the COLD lease's preserved resume_state when it carries a persisted
|
|
@@ -194,7 +293,35 @@ export async function resumeBoxForTurn(
|
|
|
194
293
|
sessionId: ids.sessionId,
|
|
195
294
|
backendOverride: ids.backend as never,
|
|
196
295
|
...(ids.environment ? { environment: ids.environment } : {}),
|
|
296
|
+
onSandboxCreated: async (created) => {
|
|
297
|
+
createdEstablished = created;
|
|
298
|
+
const resumeEnvelope = preserveWorkspaceArchiveOnInterimResumeState(
|
|
299
|
+
(await serializeEstablishedSandboxEnvelope(created)) ?? null,
|
|
300
|
+
spawnEnvelope,
|
|
301
|
+
);
|
|
302
|
+
const recorded = await recordWarmingSandboxCreated(db, {
|
|
303
|
+
accountId: ids.accountId,
|
|
304
|
+
workspaceId: ids.workspaceId,
|
|
305
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
306
|
+
expectedEpoch,
|
|
307
|
+
instanceId: created.instanceId,
|
|
308
|
+
resumeBackendId: created.backendId,
|
|
309
|
+
resumeState: resumeEnvelope,
|
|
310
|
+
leaseTtlMs,
|
|
311
|
+
});
|
|
312
|
+
if (!recorded.recorded) {
|
|
313
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
314
|
+
}
|
|
315
|
+
if (created.backendId === "modal") {
|
|
316
|
+
await tagModalSandbox(settings, created.instanceId, {
|
|
317
|
+
leaseId: acquired.lease.id,
|
|
318
|
+
workspaceId: ids.workspaceId,
|
|
319
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
320
|
+
}).catch(() => undefined);
|
|
321
|
+
}
|
|
322
|
+
},
|
|
197
323
|
});
|
|
324
|
+
createdEstablished = established;
|
|
198
325
|
await ensureDisplayStack(settings, established);
|
|
199
326
|
const endpoint = await exposeStreamPort(settings, established);
|
|
200
327
|
// Fold the LIVE box into a re-resumable envelope and persist it as the
|
|
@@ -219,39 +346,49 @@ export async function resumeBoxForTurn(
|
|
|
219
346
|
if (!committed.committed || !committed.lease) {
|
|
220
347
|
// A reaper reset our warming row (we were too slow) or a sibling
|
|
221
348
|
// re-established and bumped the epoch. Drop the handle; release our
|
|
222
|
-
// holder; surface supersession.
|
|
349
|
+
// holder; surface supersession. This spawner created the box, so stop it
|
|
350
|
+
// before retrying to avoid an untracked running sandbox.
|
|
351
|
+
await terminateEstablishedSandbox(established);
|
|
223
352
|
await release();
|
|
224
353
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
225
354
|
}
|
|
226
355
|
return { established, leaseEpoch: committed.lease.leaseEpoch, release };
|
|
227
356
|
} catch (error) {
|
|
228
357
|
if (error instanceof SandboxLeaseSupersededError) {
|
|
358
|
+
await terminateEstablishedSandbox(createdEstablished);
|
|
359
|
+
await release();
|
|
229
360
|
throw error;
|
|
230
361
|
}
|
|
231
|
-
|
|
232
|
-
//
|
|
233
|
-
//
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
362
|
+
const terminated = await terminateEstablishedSandbox(createdEstablished);
|
|
363
|
+
// Caught spawn failure: if the just-created sandbox was actually stopped,
|
|
364
|
+
// roll the warming row back to cold so a queued turn can re-acquire and
|
|
365
|
+
// re-spawn. If termination itself failed, keep the recorded instance_id on
|
|
366
|
+
// the warming row; the lease TTL/reaper and Modal orphan sweep are the
|
|
367
|
+
// tracked backstops, and we must not erase the only provider pointer.
|
|
368
|
+
if (terminated) {
|
|
369
|
+
await failWarmingToCold(db, {
|
|
370
|
+
accountId: ids.accountId,
|
|
371
|
+
workspaceId: ids.workspaceId,
|
|
372
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
373
|
+
expectedEpoch,
|
|
374
|
+
});
|
|
375
|
+
}
|
|
240
376
|
await release();
|
|
241
|
-
throw error;
|
|
377
|
+
throw asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
242
378
|
}
|
|
243
379
|
}
|
|
244
380
|
|
|
245
381
|
// ATTACHED / REARMED: the box is live (or a sibling is warming it). Resume it
|
|
246
382
|
// BY ID off the committed lease envelope. For an 'attached'-to-warming lease we
|
|
247
|
-
// first wait for the spawner to commit warm
|
|
248
|
-
//
|
|
249
|
-
|
|
250
|
-
if (acquired.lease.liveness === "warming") {
|
|
251
|
-
leaseEpoch = (await waitForWarmOrReacquire(services, ids, kind, holderId)).leaseEpoch;
|
|
252
|
-
}
|
|
253
|
-
|
|
383
|
+
// first wait for the spawner to commit warm, bounded by the explicit warming
|
|
384
|
+
// budget. A cold reset means the spawner died; requeue the turn instead of
|
|
385
|
+
// trying to enter the spawner create path from the attached branch.
|
|
254
386
|
try {
|
|
387
|
+
let leaseEpoch = acquired.lease.leaseEpoch;
|
|
388
|
+
if (acquired.lease.liveness === "warming") {
|
|
389
|
+
leaseEpoch = (await waitForWarm(services, ids)).leaseEpoch;
|
|
390
|
+
}
|
|
391
|
+
|
|
255
392
|
// Prefer the lease's resume_state (the LIVE box the spawner committed) so we
|
|
256
393
|
// re-attach to the SAME box by id, not cold-restore the original session
|
|
257
394
|
// manifest into a rival. Fall back to the session envelope only when the
|
|
@@ -277,7 +414,7 @@ export async function resumeBoxForTurn(
|
|
|
277
414
|
return { established, leaseEpoch, release };
|
|
278
415
|
} catch (error) {
|
|
279
416
|
await release();
|
|
280
|
-
throw error;
|
|
417
|
+
throw asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
281
418
|
}
|
|
282
419
|
}
|
|
283
420
|
|
|
@@ -378,58 +515,40 @@ export async function acquireSelfhostedLeaseForTurn(
|
|
|
378
515
|
|
|
379
516
|
/**
|
|
380
517
|
* Poll a warming lease until the spawner commits warm. If the warming row is
|
|
381
|
-
* reset to cold (the spawner died and the reaper reset it),
|
|
382
|
-
*
|
|
518
|
+
* reset to cold (the spawner died and the reaper reset it), surface supersession
|
|
519
|
+
* so the turn is re-dispatched and can enter the normal spawner branch from
|
|
520
|
+
* acquireLease. Bounded by OPENGENI_SANDBOX_WARMING_TIMEOUT_MS, not the lease TTL.
|
|
383
521
|
*/
|
|
384
|
-
async function
|
|
522
|
+
async function waitForWarm(
|
|
385
523
|
services: SandboxResumeServices,
|
|
386
524
|
ids: ResumeBoxIds,
|
|
387
|
-
|
|
388
|
-
holderId: string,
|
|
389
|
-
): Promise<{ liveness: string; leaseEpoch: number }> {
|
|
525
|
+
): Promise<{ leaseEpoch: number }> {
|
|
390
526
|
const { db, settings } = services;
|
|
391
|
-
const deadline = Date.now() + settings.
|
|
527
|
+
const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
|
|
392
528
|
while (Date.now() < deadline) {
|
|
393
529
|
await sleep(WARMING_POLL_INTERVAL_MS);
|
|
394
530
|
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
395
531
|
if (!lease) {
|
|
396
|
-
// Lease vanished (cold-reaped). Re-
|
|
397
|
-
|
|
532
|
+
// Lease vanished (cold-reaped). Re-dispatch from scratch.
|
|
533
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, 0);
|
|
534
|
+
}
|
|
535
|
+
if (lease.liveness === "warm") {
|
|
536
|
+
return { leaseEpoch: lease.leaseEpoch };
|
|
398
537
|
}
|
|
399
|
-
if (lease.liveness === "
|
|
400
|
-
|
|
538
|
+
if (lease.liveness === "draining") {
|
|
539
|
+
// The warming attempt either failed into drain or committed+released before
|
|
540
|
+
// this waiter observed it. Re-dispatch so acquireLease can re-arm or spawn
|
|
541
|
+
// through the normal path.
|
|
542
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
401
543
|
}
|
|
402
544
|
if (lease.liveness === "cold") {
|
|
403
|
-
// The spawner died;
|
|
404
|
-
|
|
545
|
+
// The spawner died; re-dispatch so the normal acquireLease path can win
|
|
546
|
+
// cold->warming and run the full spawner branch.
|
|
547
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
405
548
|
}
|
|
406
549
|
// still warming — keep polling.
|
|
407
550
|
}
|
|
408
|
-
|
|
409
|
-
// warm we attach. Either way resumeBoxForTurn's caller already holds a holder
|
|
410
|
-
// row (idempotent), so this re-acquire just re-reads/re-CASes.
|
|
411
|
-
const reacquired = await acquireLease(db, {
|
|
412
|
-
accountId: ids.accountId,
|
|
413
|
-
workspaceId: ids.workspaceId,
|
|
414
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
415
|
-
kind,
|
|
416
|
-
holderId,
|
|
417
|
-
subjectId: ids.sessionId,
|
|
418
|
-
backend: ids.backend,
|
|
419
|
-
os: ids.os ?? "linux",
|
|
420
|
-
leaseTtlMs: settings.sandboxLeaseTtlMs,
|
|
421
|
-
});
|
|
422
|
-
if (reacquired.role === "fenced") {
|
|
423
|
-
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, reacquired.lease.leaseEpoch);
|
|
424
|
-
}
|
|
425
|
-
// For 'spawner' we'd need to run the cold-restore path; to keep resumeBoxForTurn
|
|
426
|
-
// a single critical section we recurse the spawner handling by surfacing it as a
|
|
427
|
-
// re-establish from the (now-cold) envelope. The simplest correct behavior: if
|
|
428
|
-
// we re-won the CAS, establish + commit happens on the NEXT resumeBoxForTurn
|
|
429
|
-
// call (the queued turn re-dispatch); here we just return the lease snapshot so
|
|
430
|
-
// the attached path resumes by id. A cold lease has no box yet, so treat it as
|
|
431
|
-
// warming-resolved only when warm/draining.
|
|
432
|
-
return { liveness: reacquired.lease.liveness, leaseEpoch: reacquired.lease.leaseEpoch };
|
|
551
|
+
throw new SandboxWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
433
552
|
}
|
|
434
553
|
|
|
435
554
|
// ============================================================================
|