@opengeni/worker-bundle 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -0
- package/dist/index.js +215 -40
- package/dist/index.js.map +1 -1
- package/package.json +8 -8
- package/src/activities/agent-turn.ts +4 -0
- package/src/activities/sandbox-lease.ts +97 -3
- package/src/sandbox-resume.ts +178 -59
package/dist/index.d.ts
CHANGED
|
@@ -115,6 +115,8 @@ type ReapSandboxLeasesResult = {
|
|
|
115
115
|
/** Viewer-only boxes force-drained because their workspace is over a limit
|
|
116
116
|
* (0 balance / over the warm cap) — turn-held boxes are never drained (P2.1). */
|
|
117
117
|
forceDrained: number;
|
|
118
|
+
/** Provider-side Modal orphan sandboxes terminated by the defensive sweep. */
|
|
119
|
+
modalOrphansTerminated: number;
|
|
118
120
|
};
|
|
119
121
|
|
|
120
122
|
declare function createActivities(dependencies?: ActivityDependencies): {
|
package/dist/index.js
CHANGED
|
@@ -1170,6 +1170,7 @@ import {
|
|
|
1170
1170
|
failWarmingToCold,
|
|
1171
1171
|
getSandboxSessionEnvelope as getSandboxSessionEnvelope2,
|
|
1172
1172
|
readLease,
|
|
1173
|
+
recordWarmingSandboxCreated,
|
|
1173
1174
|
releaseLeaseHolder,
|
|
1174
1175
|
SandboxLeaseSupersededError
|
|
1175
1176
|
} from "@opengeni/db";
|
|
@@ -1181,13 +1182,84 @@ import {
|
|
|
1181
1182
|
DisplayStackError,
|
|
1182
1183
|
DisplayStackUnsupportedError,
|
|
1183
1184
|
buildStreamUrl,
|
|
1184
|
-
StreamPortUnavailableError
|
|
1185
|
+
StreamPortUnavailableError,
|
|
1186
|
+
modalSandboxAttributionEnvironment,
|
|
1187
|
+
tagModalSandbox
|
|
1185
1188
|
} from "@opengeni/runtime";
|
|
1186
1189
|
import { DESKTOP_STREAM_PORT } from "@opengeni/contracts";
|
|
1190
|
+
var SandboxWarmingTimeoutError = class extends Error {
|
|
1191
|
+
constructor(backend, timeoutMs) {
|
|
1192
|
+
super(
|
|
1193
|
+
`Sandbox backend "${backend}" capacity or creation timed out after ${Math.ceil(timeoutMs / 1e3)}s while warming the sandbox lease. Please try again; if this persists, sandbox capacity may be exhausted.`
|
|
1194
|
+
);
|
|
1195
|
+
this.backend = backend;
|
|
1196
|
+
this.timeoutMs = timeoutMs;
|
|
1197
|
+
this.name = "SandboxWarmingTimeoutError";
|
|
1198
|
+
}
|
|
1199
|
+
backend;
|
|
1200
|
+
timeoutMs;
|
|
1201
|
+
code = "sandbox_warming_timeout";
|
|
1202
|
+
};
|
|
1187
1203
|
var WARMING_POLL_INTERVAL_MS = 250;
|
|
1188
1204
|
async function sleep(ms) {
|
|
1189
1205
|
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
1190
1206
|
}
|
|
1207
|
+
async function terminateEstablishedSandbox(established) {
|
|
1208
|
+
if (!established) {
|
|
1209
|
+
return true;
|
|
1210
|
+
}
|
|
1211
|
+
const client = established.client;
|
|
1212
|
+
if (typeof client.delete === "function" && established.sessionState !== void 0) {
|
|
1213
|
+
try {
|
|
1214
|
+
await client.delete(established.sessionState);
|
|
1215
|
+
return true;
|
|
1216
|
+
} catch {
|
|
1217
|
+
return false;
|
|
1218
|
+
}
|
|
1219
|
+
}
|
|
1220
|
+
const session = established.session;
|
|
1221
|
+
try {
|
|
1222
|
+
if (session.terminate) {
|
|
1223
|
+
await session.terminate();
|
|
1224
|
+
return true;
|
|
1225
|
+
} else if (session.kill) {
|
|
1226
|
+
await session.kill();
|
|
1227
|
+
return true;
|
|
1228
|
+
} else if (session.close) {
|
|
1229
|
+
if (!session.closed) {
|
|
1230
|
+
await session.close();
|
|
1231
|
+
}
|
|
1232
|
+
return true;
|
|
1233
|
+
}
|
|
1234
|
+
return false;
|
|
1235
|
+
} catch {
|
|
1236
|
+
return false;
|
|
1237
|
+
}
|
|
1238
|
+
}
|
|
1239
|
+
function asSandboxWarmingError(error, backend, timeoutMs) {
|
|
1240
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1241
|
+
return /sandbox creation timed out|warming timed out|capacity.*timed out/i.test(message) ? new SandboxWarmingTimeoutError(backend, timeoutMs) : error;
|
|
1242
|
+
}
|
|
1243
|
+
function workspaceArchiveFromEnvelope(envelope) {
|
|
1244
|
+
const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
|
|
1245
|
+
const archive = sessionState?.workspaceArchive;
|
|
1246
|
+
return typeof archive === "string" && archive.length > 0 ? archive : null;
|
|
1247
|
+
}
|
|
1248
|
+
function preserveWorkspaceArchiveOnInterimResumeState(resumeState, archiveSource) {
|
|
1249
|
+
const archive = workspaceArchiveFromEnvelope(archiveSource);
|
|
1250
|
+
if (!archive) {
|
|
1251
|
+
return resumeState;
|
|
1252
|
+
}
|
|
1253
|
+
const existingSessionState = resumeState && typeof resumeState.sessionState === "object" && resumeState.sessionState !== null ? resumeState.sessionState : {};
|
|
1254
|
+
return {
|
|
1255
|
+
...resumeState ?? {},
|
|
1256
|
+
...resumeState?.backendId === void 0 && archiveSource?.backendId !== void 0 ? { backendId: archiveSource.backendId } : {},
|
|
1257
|
+
sessionState: {
|
|
1258
|
+
...existingSessionState,
|
|
1259
|
+
workspaceArchive: archive
|
|
1260
|
+
}
|
|
1261
|
+
};
|
|
1262
|
+
}
|
|
1191
1263
|
async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
1192
1264
|
const { db, settings } = services;
|
|
1193
1265
|
const os = ids.os ?? "linux";
|
|
@@ -1229,14 +1301,50 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
1229
1301
|
}
|
|
1230
1302
|
if (acquired.role === "spawner") {
|
|
1231
1303
|
const expectedEpoch = acquired.lease.leaseEpoch;
|
|
1304
|
+
let createdEstablished = null;
|
|
1305
|
+
if (ids.environment && ids.backend === "modal") {
|
|
1306
|
+
Object.assign(ids.environment, modalSandboxAttributionEnvironment({
|
|
1307
|
+
leaseId: acquired.lease.id,
|
|
1308
|
+
workspaceId: ids.workspaceId,
|
|
1309
|
+
sandboxGroupId: ids.sandboxGroupId
|
|
1310
|
+
}));
|
|
1311
|
+
}
|
|
1232
1312
|
try {
|
|
1233
1313
|
const envelope = await getSandboxSessionEnvelope2(db, ids.workspaceId, ids.sessionId);
|
|
1234
1314
|
const spawnEnvelope = acquired.lease.resumeState ?? envelope;
|
|
1235
1315
|
const established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
|
|
1236
1316
|
sessionId: ids.sessionId,
|
|
1237
1317
|
backendOverride: ids.backend,
|
|
1238
|
-
...ids.environment ? { environment: ids.environment } : {}
|
|
1318
|
+
...ids.environment ? { environment: ids.environment } : {},
|
|
1319
|
+
onSandboxCreated: async (created) => {
|
|
1320
|
+
createdEstablished = created;
|
|
1321
|
+
const resumeEnvelope2 = preserveWorkspaceArchiveOnInterimResumeState(
|
|
1322
|
+
await serializeEstablishedSandboxEnvelope(created) ?? null,
|
|
1323
|
+
spawnEnvelope
|
|
1324
|
+
);
|
|
1325
|
+
const recorded = await recordWarmingSandboxCreated(db, {
|
|
1326
|
+
accountId: ids.accountId,
|
|
1327
|
+
workspaceId: ids.workspaceId,
|
|
1328
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1329
|
+
expectedEpoch,
|
|
1330
|
+
instanceId: created.instanceId,
|
|
1331
|
+
resumeBackendId: created.backendId,
|
|
1332
|
+
resumeState: resumeEnvelope2,
|
|
1333
|
+
leaseTtlMs
|
|
1334
|
+
});
|
|
1335
|
+
if (!recorded.recorded) {
|
|
1336
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
1337
|
+
}
|
|
1338
|
+
if (created.backendId === "modal") {
|
|
1339
|
+
await tagModalSandbox(settings, created.instanceId, {
|
|
1340
|
+
leaseId: acquired.lease.id,
|
|
1341
|
+
workspaceId: ids.workspaceId,
|
|
1342
|
+
sandboxGroupId: ids.sandboxGroupId
|
|
1343
|
+
}).catch(() => void 0);
|
|
1344
|
+
}
|
|
1345
|
+
}
|
|
1239
1346
|
});
|
|
1347
|
+
createdEstablished = established;
|
|
1240
1348
|
await ensureDisplayStack(settings, established);
|
|
1241
1349
|
const endpoint = await exposeStreamPort(settings, established);
|
|
1242
1350
|
const resumeEnvelope = await serializeEstablishedSandboxEnvelope(established) ?? envelope ?? null;
|
|
@@ -1252,29 +1360,35 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
1252
1360
|
leaseTtlMs
|
|
1253
1361
|
});
|
|
1254
1362
|
if (!committed.committed || !committed.lease) {
|
|
1363
|
+
await terminateEstablishedSandbox(established);
|
|
1255
1364
|
await release();
|
|
1256
1365
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
1257
1366
|
}
|
|
1258
1367
|
return { established, leaseEpoch: committed.lease.leaseEpoch, release };
|
|
1259
1368
|
} catch (error) {
|
|
1260
1369
|
if (error instanceof SandboxLeaseSupersededError) {
|
|
1370
|
+
await terminateEstablishedSandbox(createdEstablished);
|
|
1371
|
+
await release();
|
|
1261
1372
|
throw error;
|
|
1262
1373
|
}
|
|
1263
|
-
await
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1374
|
+
const terminated = await terminateEstablishedSandbox(createdEstablished);
|
|
1375
|
+
if (terminated) {
|
|
1376
|
+
await failWarmingToCold(db, {
|
|
1377
|
+
accountId: ids.accountId,
|
|
1378
|
+
workspaceId: ids.workspaceId,
|
|
1379
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1380
|
+
expectedEpoch
|
|
1381
|
+
});
|
|
1382
|
+
}
|
|
1269
1383
|
await release();
|
|
1270
|
-
throw error;
|
|
1384
|
+
throw asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
1271
1385
|
}
|
|
1272
1386
|
}
|
|
1273
|
-
let leaseEpoch = acquired.lease.leaseEpoch;
|
|
1274
|
-
if (acquired.lease.liveness === "warming") {
|
|
1275
|
-
leaseEpoch = (await waitForWarmOrReacquire(services, ids, kind, holderId)).leaseEpoch;
|
|
1276
|
-
}
|
|
1277
1387
|
try {
|
|
1388
|
+
let leaseEpoch = acquired.lease.leaseEpoch;
|
|
1389
|
+
if (acquired.lease.liveness === "warming") {
|
|
1390
|
+
leaseEpoch = (await waitForWarm(services, ids)).leaseEpoch;
|
|
1391
|
+
}
|
|
1278
1392
|
const live = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
1279
1393
|
const envelope = live?.resumeState ?? await getSandboxSessionEnvelope2(db, ids.workspaceId, ids.sessionId);
|
|
1280
1394
|
const established = await establishSandboxSessionFromEnvelope(settings, envelope, {
|
|
@@ -1286,7 +1400,7 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
1286
1400
|
return { established, leaseEpoch, release };
|
|
1287
1401
|
} catch (error) {
|
|
1288
1402
|
await release();
|
|
1289
|
-
throw error;
|
|
1403
|
+
throw asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
1290
1404
|
}
|
|
1291
1405
|
}
|
|
1292
1406
|
async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
|
|
@@ -1343,37 +1457,26 @@ async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
|
|
|
1343
1457
|
}
|
|
1344
1458
|
return { leaseEpoch: acquired.lease.leaseEpoch, release };
|
|
1345
1459
|
}
|
|
1346
|
-
async function
|
|
1460
|
+
async function waitForWarm(services, ids) {
|
|
1347
1461
|
const { db, settings } = services;
|
|
1348
|
-
const deadline = Date.now() + settings.
|
|
1462
|
+
const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
|
|
1349
1463
|
while (Date.now() < deadline) {
|
|
1350
1464
|
await sleep(WARMING_POLL_INTERVAL_MS);
|
|
1351
1465
|
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
1352
1466
|
if (!lease) {
|
|
1353
|
-
|
|
1467
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, 0);
|
|
1468
|
+
}
|
|
1469
|
+
if (lease.liveness === "warm") {
|
|
1470
|
+
return { leaseEpoch: lease.leaseEpoch };
|
|
1354
1471
|
}
|
|
1355
|
-
if (lease.liveness === "
|
|
1356
|
-
|
|
1472
|
+
if (lease.liveness === "draining") {
|
|
1473
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
1357
1474
|
}
|
|
1358
1475
|
if (lease.liveness === "cold") {
|
|
1359
|
-
|
|
1476
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
1360
1477
|
}
|
|
1361
1478
|
}
|
|
1362
|
-
|
|
1363
|
-
accountId: ids.accountId,
|
|
1364
|
-
workspaceId: ids.workspaceId,
|
|
1365
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
1366
|
-
kind,
|
|
1367
|
-
holderId,
|
|
1368
|
-
subjectId: ids.sessionId,
|
|
1369
|
-
backend: ids.backend,
|
|
1370
|
-
os: ids.os ?? "linux",
|
|
1371
|
-
leaseTtlMs: settings.sandboxLeaseTtlMs
|
|
1372
|
-
});
|
|
1373
|
-
if (reacquired.role === "fenced") {
|
|
1374
|
-
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, reacquired.lease.leaseEpoch);
|
|
1375
|
-
}
|
|
1376
|
-
return { liveness: reacquired.lease.liveness, leaseEpoch: reacquired.lease.leaseEpoch };
|
|
1479
|
+
throw new SandboxWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
1377
1480
|
}
|
|
1378
1481
|
async function ensureDisplayStack(settings, established) {
|
|
1379
1482
|
if (!settings.sandboxDesktopEnabled) {
|
|
@@ -2322,6 +2425,10 @@ function createRunAgentTurnActivity(services) {
|
|
|
2322
2425
|
} : { computerToolMode: computerToolModeForTurn(null) },
|
|
2323
2426
|
...packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {},
|
|
2324
2427
|
...workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {},
|
|
2428
|
+
// Per-session persona tier (session > workspace > deployment default).
|
|
2429
|
+
// Composed system-level AFTER the workspace persona so it refines it for
|
|
2430
|
+
// this one session; absent ⇒ byte-identical to today's composition.
|
|
2431
|
+
...session.instructions ? { sessionInstructions: session.instructions } : {},
|
|
2325
2432
|
...workspaceEnvironment ? {
|
|
2326
2433
|
workspaceEnvironment: {
|
|
2327
2434
|
name: workspaceEnvironment.name,
|
|
@@ -3312,6 +3419,7 @@ import {
|
|
|
3312
3419
|
confirmDrainCold,
|
|
3313
3420
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
3314
3421
|
getBillingBalance as getBillingBalance4,
|
|
3422
|
+
listLiveModalSandboxLeaseAttributions,
|
|
3315
3423
|
listMeterableWarmLeases,
|
|
3316
3424
|
persistDrainSnapshot,
|
|
3317
3425
|
readLease as readLease2,
|
|
@@ -3323,14 +3431,17 @@ import {
|
|
|
3323
3431
|
createSandboxClientForBackend,
|
|
3324
3432
|
deletePriorPersistedSnapshot,
|
|
3325
3433
|
deserializeSandboxSessionStateEnvelope,
|
|
3326
|
-
isProviderSandboxNotFoundError
|
|
3434
|
+
isProviderSandboxNotFoundError,
|
|
3435
|
+
sweepModalOrphanSandboxes,
|
|
3436
|
+
terminateModalSandboxById
|
|
3327
3437
|
} from "@opengeni/runtime";
|
|
3328
3438
|
function createSandboxLeaseActivities(services, options = {}) {
|
|
3329
3439
|
const terminateBox = options.terminateBox ?? terminateProviderBox;
|
|
3440
|
+
const sweepModalOrphans = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
3330
3441
|
async function reapSandboxLeases2() {
|
|
3331
3442
|
const { db, settings, observability } = await services();
|
|
3332
3443
|
if (!settings.sandboxOwnershipEnabled) {
|
|
3333
|
-
return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0 };
|
|
3444
|
+
return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
|
|
3334
3445
|
}
|
|
3335
3446
|
const metered = await accrueWarmTick(db, settings, observability);
|
|
3336
3447
|
const forceDrained = await forceDrainOverLimitWorkspaces(db, settings, metered.workspaceIds, observability);
|
|
@@ -3340,6 +3451,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
3340
3451
|
});
|
|
3341
3452
|
let terminated = 0;
|
|
3342
3453
|
let skipped = 0;
|
|
3454
|
+
let modalOrphansTerminated = 0;
|
|
3343
3455
|
for (const row of drainable) {
|
|
3344
3456
|
try {
|
|
3345
3457
|
const drainedCold = await terminateDrainableBox(db, settings, row, observability, terminateBox);
|
|
@@ -3358,16 +3470,24 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
3358
3470
|
});
|
|
3359
3471
|
}
|
|
3360
3472
|
}
|
|
3361
|
-
|
|
3473
|
+
try {
|
|
3474
|
+
modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
|
|
3475
|
+
} catch (error) {
|
|
3476
|
+
observability.warn("sandbox reaper: Modal orphan sweep failed", {
|
|
3477
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3478
|
+
});
|
|
3479
|
+
}
|
|
3480
|
+
if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
|
|
3362
3481
|
observability.info("sandbox reaper swept", {
|
|
3363
3482
|
drainable: drainable.length,
|
|
3364
3483
|
terminated,
|
|
3365
3484
|
skipped,
|
|
3366
3485
|
metered: metered.accrued,
|
|
3367
|
-
forceDrained
|
|
3486
|
+
forceDrained,
|
|
3487
|
+
modalOrphansTerminated
|
|
3368
3488
|
});
|
|
3369
3489
|
}
|
|
3370
|
-
return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained };
|
|
3490
|
+
return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained, modalOrphansTerminated };
|
|
3371
3491
|
}
|
|
3372
3492
|
return { reapSandboxLeases: reapSandboxLeases2 };
|
|
3373
3493
|
}
|
|
@@ -3443,6 +3563,37 @@ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observa
|
|
|
3443
3563
|
}
|
|
3444
3564
|
return forceDrained;
|
|
3445
3565
|
}
|
|
3566
|
+
async function sweepModalOrphansForConfiguredBackend(settings, db, observability) {
|
|
3567
|
+
if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret) {
|
|
3568
|
+
return 0;
|
|
3569
|
+
}
|
|
3570
|
+
let liveLeases;
|
|
3571
|
+
try {
|
|
3572
|
+
liveLeases = await listLiveModalSandboxLeaseAttributions(db);
|
|
3573
|
+
} catch (error) {
|
|
3574
|
+
observability.warn("sandbox reaper: live Modal lease attribution read failed", {
|
|
3575
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3576
|
+
});
|
|
3577
|
+
return 0;
|
|
3578
|
+
}
|
|
3579
|
+
const result = await sweepModalOrphanSandboxes(settings, liveLeases);
|
|
3580
|
+
for (const terminated of result.terminated) {
|
|
3581
|
+
observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
|
|
3582
|
+
sandboxId: terminated.sandboxId,
|
|
3583
|
+
reason: terminated.reason,
|
|
3584
|
+
tags: JSON.stringify(terminated.tags)
|
|
3585
|
+
});
|
|
3586
|
+
}
|
|
3587
|
+
if (result.examined > 0) {
|
|
3588
|
+
observability.info("sandbox reaper: Modal orphan sweep completed", {
|
|
3589
|
+
examined: result.examined,
|
|
3590
|
+
terminated: result.terminated.length,
|
|
3591
|
+
skipped: result.skipped,
|
|
3592
|
+
appName: settings.modalAppName
|
|
3593
|
+
});
|
|
3594
|
+
}
|
|
3595
|
+
return result.terminated.length;
|
|
3596
|
+
}
|
|
3446
3597
|
async function terminateDrainableBox(db, settings, row, observability, terminateBox) {
|
|
3447
3598
|
const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);
|
|
3448
3599
|
const lease = await readLease2(db, row.workspaceId, row.sandboxGroupId);
|
|
@@ -3483,6 +3634,30 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
3483
3634
|
});
|
|
3484
3635
|
return true;
|
|
3485
3636
|
}
|
|
3637
|
+
if (backend === "modal" && !lease.resumeState && lease.instanceId) {
|
|
3638
|
+
const { wrote } = await persistArchive(null);
|
|
3639
|
+
if (!wrote) {
|
|
3640
|
+
observability.info("sandbox reaper: Modal lease re-armed before direct terminate \u2014 leaving sandbox RUNNING", {
|
|
3641
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
3642
|
+
backend,
|
|
3643
|
+
instanceId: lease.instanceId
|
|
3644
|
+
});
|
|
3645
|
+
return false;
|
|
3646
|
+
}
|
|
3647
|
+
try {
|
|
3648
|
+
await terminateModalSandboxById(settings, lease.instanceId);
|
|
3649
|
+
} catch (error) {
|
|
3650
|
+
if (!isProviderSandboxNotFoundError("modal", error)) {
|
|
3651
|
+
throw error;
|
|
3652
|
+
}
|
|
3653
|
+
observability.info("sandbox reaper: Modal sandbox already gone during direct terminate", {
|
|
3654
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
3655
|
+
backend,
|
|
3656
|
+
instanceId: lease.instanceId
|
|
3657
|
+
});
|
|
3658
|
+
}
|
|
3659
|
+
return true;
|
|
3660
|
+
}
|
|
3486
3661
|
if (!lease.resumeState) {
|
|
3487
3662
|
return true;
|
|
3488
3663
|
}
|