@opengeni/worker-bundle 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -115,6 +115,8 @@ type ReapSandboxLeasesResult = {
115
115
  /** Viewer-only boxes force-drained because their workspace is over a limit
116
116
  * (0 balance / over the warm cap) — turn-held boxes are never drained (P2.1). */
117
117
  forceDrained: number;
118
+ /** Provider-side Modal orphan sandboxes terminated by the defensive sweep. */
119
+ modalOrphansTerminated: number;
118
120
  };
119
121
 
120
122
  declare function createActivities(dependencies?: ActivityDependencies): {
package/dist/index.js CHANGED
@@ -1170,6 +1170,7 @@ import {
1170
1170
  failWarmingToCold,
1171
1171
  getSandboxSessionEnvelope as getSandboxSessionEnvelope2,
1172
1172
  readLease,
1173
+ recordWarmingSandboxCreated,
1173
1174
  releaseLeaseHolder,
1174
1175
  SandboxLeaseSupersededError
1175
1176
  } from "@opengeni/db";
@@ -1181,13 +1182,84 @@ import {
1181
1182
  DisplayStackError,
1182
1183
  DisplayStackUnsupportedError,
1183
1184
  buildStreamUrl,
1184
- StreamPortUnavailableError
1185
+ StreamPortUnavailableError,
1186
+ modalSandboxAttributionEnvironment,
1187
+ tagModalSandbox
1185
1188
  } from "@opengeni/runtime";
1186
1189
  import { DESKTOP_STREAM_PORT } from "@opengeni/contracts";
1190
+ var SandboxWarmingTimeoutError = class extends Error {
1191
+ constructor(backend, timeoutMs) {
1192
+ super(
1193
+ `Sandbox backend "${backend}" capacity or creation timed out after ${Math.ceil(timeoutMs / 1e3)}s while warming the sandbox lease. Please try again; if this persists, sandbox capacity may be exhausted.`
1194
+ );
1195
+ this.backend = backend;
1196
+ this.timeoutMs = timeoutMs;
1197
+ this.name = "SandboxWarmingTimeoutError";
1198
+ }
1199
+ backend;
1200
+ timeoutMs;
1201
+ code = "sandbox_warming_timeout";
1202
+ };
1187
1203
  var WARMING_POLL_INTERVAL_MS = 250;
1188
1204
  async function sleep(ms) {
1189
1205
  await new Promise((resolve) => setTimeout(resolve, ms));
1190
1206
  }
1207
+ async function terminateEstablishedSandbox(established) {
1208
+ if (!established) {
1209
+ return true;
1210
+ }
1211
+ const client = established.client;
1212
+ if (typeof client.delete === "function" && established.sessionState !== void 0) {
1213
+ try {
1214
+ await client.delete(established.sessionState);
1215
+ return true;
1216
+ } catch {
1217
+ return false;
1218
+ }
1219
+ }
1220
+ const session = established.session;
1221
+ try {
1222
+ if (session.terminate) {
1223
+ await session.terminate();
1224
+ return true;
1225
+ } else if (session.kill) {
1226
+ await session.kill();
1227
+ return true;
1228
+ } else if (session.close) {
1229
+ if (!session.closed) {
1230
+ await session.close();
1231
+ }
1232
+ return true;
1233
+ }
1234
+ return false;
1235
+ } catch {
1236
+ return false;
1237
+ }
1238
+ }
1239
+ function asSandboxWarmingError(error, backend, timeoutMs) {
1240
+ const message = error instanceof Error ? error.message : String(error);
1241
+ return /sandbox creation timed out|warming timed out|capacity.*timed out/i.test(message) ? new SandboxWarmingTimeoutError(backend, timeoutMs) : error;
1242
+ }
1243
+ function workspaceArchiveFromEnvelope(envelope) {
1244
+ const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
1245
+ const archive = sessionState?.workspaceArchive;
1246
+ return typeof archive === "string" && archive.length > 0 ? archive : null;
1247
+ }
1248
+ function preserveWorkspaceArchiveOnInterimResumeState(resumeState, archiveSource) {
1249
+ const archive = workspaceArchiveFromEnvelope(archiveSource);
1250
+ if (!archive) {
1251
+ return resumeState;
1252
+ }
1253
+ const existingSessionState = resumeState && typeof resumeState.sessionState === "object" && resumeState.sessionState !== null ? resumeState.sessionState : {};
1254
+ return {
1255
+ ...resumeState ?? {},
1256
+ ...resumeState?.backendId === void 0 && archiveSource?.backendId !== void 0 ? { backendId: archiveSource.backendId } : {},
1257
+ sessionState: {
1258
+ ...existingSessionState,
1259
+ workspaceArchive: archive
1260
+ }
1261
+ };
1262
+ }
1191
1263
  async function resumeBoxForTurn(services, ids, kind, holderId) {
1192
1264
  const { db, settings } = services;
1193
1265
  const os = ids.os ?? "linux";
@@ -1229,14 +1301,50 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
1229
1301
  }
1230
1302
  if (acquired.role === "spawner") {
1231
1303
  const expectedEpoch = acquired.lease.leaseEpoch;
1304
+ let createdEstablished = null;
1305
+ if (ids.environment && ids.backend === "modal") {
1306
+ Object.assign(ids.environment, modalSandboxAttributionEnvironment({
1307
+ leaseId: acquired.lease.id,
1308
+ workspaceId: ids.workspaceId,
1309
+ sandboxGroupId: ids.sandboxGroupId
1310
+ }));
1311
+ }
1232
1312
  try {
1233
1313
  const envelope = await getSandboxSessionEnvelope2(db, ids.workspaceId, ids.sessionId);
1234
1314
  const spawnEnvelope = acquired.lease.resumeState ?? envelope;
1235
1315
  const established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
1236
1316
  sessionId: ids.sessionId,
1237
1317
  backendOverride: ids.backend,
1238
- ...ids.environment ? { environment: ids.environment } : {}
1318
+ ...ids.environment ? { environment: ids.environment } : {},
1319
+ onSandboxCreated: async (created) => {
1320
+ createdEstablished = created;
1321
+ const resumeEnvelope2 = preserveWorkspaceArchiveOnInterimResumeState(
1322
+ await serializeEstablishedSandboxEnvelope(created) ?? null,
1323
+ spawnEnvelope
1324
+ );
1325
+ const recorded = await recordWarmingSandboxCreated(db, {
1326
+ accountId: ids.accountId,
1327
+ workspaceId: ids.workspaceId,
1328
+ sandboxGroupId: ids.sandboxGroupId,
1329
+ expectedEpoch,
1330
+ instanceId: created.instanceId,
1331
+ resumeBackendId: created.backendId,
1332
+ resumeState: resumeEnvelope2,
1333
+ leaseTtlMs
1334
+ });
1335
+ if (!recorded.recorded) {
1336
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
1337
+ }
1338
+ if (created.backendId === "modal") {
1339
+ await tagModalSandbox(settings, created.instanceId, {
1340
+ leaseId: acquired.lease.id,
1341
+ workspaceId: ids.workspaceId,
1342
+ sandboxGroupId: ids.sandboxGroupId
1343
+ }).catch(() => void 0);
1344
+ }
1345
+ }
1239
1346
  });
1347
+ createdEstablished = established;
1240
1348
  await ensureDisplayStack(settings, established);
1241
1349
  const endpoint = await exposeStreamPort(settings, established);
1242
1350
  const resumeEnvelope = await serializeEstablishedSandboxEnvelope(established) ?? envelope ?? null;
@@ -1252,29 +1360,35 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
1252
1360
  leaseTtlMs
1253
1361
  });
1254
1362
  if (!committed.committed || !committed.lease) {
1363
+ await terminateEstablishedSandbox(established);
1255
1364
  await release();
1256
1365
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
1257
1366
  }
1258
1367
  return { established, leaseEpoch: committed.lease.leaseEpoch, release };
1259
1368
  } catch (error) {
1260
1369
  if (error instanceof SandboxLeaseSupersededError) {
1370
+ await terminateEstablishedSandbox(createdEstablished);
1371
+ await release();
1261
1372
  throw error;
1262
1373
  }
1263
- await failWarmingToCold(db, {
1264
- accountId: ids.accountId,
1265
- workspaceId: ids.workspaceId,
1266
- sandboxGroupId: ids.sandboxGroupId,
1267
- expectedEpoch
1268
- });
1374
+ const terminated = await terminateEstablishedSandbox(createdEstablished);
1375
+ if (terminated) {
1376
+ await failWarmingToCold(db, {
1377
+ accountId: ids.accountId,
1378
+ workspaceId: ids.workspaceId,
1379
+ sandboxGroupId: ids.sandboxGroupId,
1380
+ expectedEpoch
1381
+ });
1382
+ }
1269
1383
  await release();
1270
- throw error;
1384
+ throw asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
1271
1385
  }
1272
1386
  }
1273
- let leaseEpoch = acquired.lease.leaseEpoch;
1274
- if (acquired.lease.liveness === "warming") {
1275
- leaseEpoch = (await waitForWarmOrReacquire(services, ids, kind, holderId)).leaseEpoch;
1276
- }
1277
1387
  try {
1388
+ let leaseEpoch = acquired.lease.leaseEpoch;
1389
+ if (acquired.lease.liveness === "warming") {
1390
+ leaseEpoch = (await waitForWarm(services, ids)).leaseEpoch;
1391
+ }
1278
1392
  const live = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
1279
1393
  const envelope = live?.resumeState ?? await getSandboxSessionEnvelope2(db, ids.workspaceId, ids.sessionId);
1280
1394
  const established = await establishSandboxSessionFromEnvelope(settings, envelope, {
@@ -1286,7 +1400,7 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
1286
1400
  return { established, leaseEpoch, release };
1287
1401
  } catch (error) {
1288
1402
  await release();
1289
- throw error;
1403
+ throw asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
1290
1404
  }
1291
1405
  }
1292
1406
  async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
@@ -1343,37 +1457,26 @@ async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
1343
1457
  }
1344
1458
  return { leaseEpoch: acquired.lease.leaseEpoch, release };
1345
1459
  }
1346
- async function waitForWarmOrReacquire(services, ids, kind, holderId) {
1460
+ async function waitForWarm(services, ids) {
1347
1461
  const { db, settings } = services;
1348
- const deadline = Date.now() + settings.sandboxLeaseWarmingTtlMs;
1462
+ const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
1349
1463
  while (Date.now() < deadline) {
1350
1464
  await sleep(WARMING_POLL_INTERVAL_MS);
1351
1465
  const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
1352
1466
  if (!lease) {
1353
- break;
1467
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, 0);
1468
+ }
1469
+ if (lease.liveness === "warm") {
1470
+ return { leaseEpoch: lease.leaseEpoch };
1354
1471
  }
1355
- if (lease.liveness === "warm" || lease.liveness === "draining") {
1356
- return { liveness: lease.liveness, leaseEpoch: lease.leaseEpoch };
1472
+ if (lease.liveness === "draining") {
1473
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
1357
1474
  }
1358
1475
  if (lease.liveness === "cold") {
1359
- break;
1476
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
1360
1477
  }
1361
1478
  }
1362
- const reacquired = await acquireLease(db, {
1363
- accountId: ids.accountId,
1364
- workspaceId: ids.workspaceId,
1365
- sandboxGroupId: ids.sandboxGroupId,
1366
- kind,
1367
- holderId,
1368
- subjectId: ids.sessionId,
1369
- backend: ids.backend,
1370
- os: ids.os ?? "linux",
1371
- leaseTtlMs: settings.sandboxLeaseTtlMs
1372
- });
1373
- if (reacquired.role === "fenced") {
1374
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, reacquired.lease.leaseEpoch);
1375
- }
1376
- return { liveness: reacquired.lease.liveness, leaseEpoch: reacquired.lease.leaseEpoch };
1479
+ throw new SandboxWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs);
1377
1480
  }
1378
1481
  async function ensureDisplayStack(settings, established) {
1379
1482
  if (!settings.sandboxDesktopEnabled) {
@@ -2322,6 +2425,10 @@ function createRunAgentTurnActivity(services) {
2322
2425
  } : { computerToolMode: computerToolModeForTurn(null) },
2323
2426
  ...packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {},
2324
2427
  ...workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {},
2428
+ // Per-session persona tier (session > workspace > deployment default).
2429
+ // Composed system-level AFTER the workspace persona so it refines it for
2430
+ // this one session; absent ⇒ byte-identical to today's composition.
2431
+ ...session.instructions ? { sessionInstructions: session.instructions } : {},
2325
2432
  ...workspaceEnvironment ? {
2326
2433
  workspaceEnvironment: {
2327
2434
  name: workspaceEnvironment.name,
@@ -3312,6 +3419,7 @@ import {
3312
3419
  confirmDrainCold,
3313
3420
  forceDrainOverLimitViewerOnlyBoxes,
3314
3421
  getBillingBalance as getBillingBalance4,
3422
+ listLiveModalSandboxLeaseAttributions,
3315
3423
  listMeterableWarmLeases,
3316
3424
  persistDrainSnapshot,
3317
3425
  readLease as readLease2,
@@ -3323,14 +3431,17 @@ import {
3323
3431
  createSandboxClientForBackend,
3324
3432
  deletePriorPersistedSnapshot,
3325
3433
  deserializeSandboxSessionStateEnvelope,
3326
- isProviderSandboxNotFoundError
3434
+ isProviderSandboxNotFoundError,
3435
+ sweepModalOrphanSandboxes,
3436
+ terminateModalSandboxById
3327
3437
  } from "@opengeni/runtime";
3328
3438
  function createSandboxLeaseActivities(services, options = {}) {
3329
3439
  const terminateBox = options.terminateBox ?? terminateProviderBox;
3440
+ const sweepModalOrphans = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
3330
3441
  async function reapSandboxLeases2() {
3331
3442
  const { db, settings, observability } = await services();
3332
3443
  if (!settings.sandboxOwnershipEnabled) {
3333
- return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0 };
3444
+ return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
3334
3445
  }
3335
3446
  const metered = await accrueWarmTick(db, settings, observability);
3336
3447
  const forceDrained = await forceDrainOverLimitWorkspaces(db, settings, metered.workspaceIds, observability);
@@ -3340,6 +3451,7 @@ function createSandboxLeaseActivities(services, options = {}) {
3340
3451
  });
3341
3452
  let terminated = 0;
3342
3453
  let skipped = 0;
3454
+ let modalOrphansTerminated = 0;
3343
3455
  for (const row of drainable) {
3344
3456
  try {
3345
3457
  const drainedCold = await terminateDrainableBox(db, settings, row, observability, terminateBox);
@@ -3358,16 +3470,24 @@ function createSandboxLeaseActivities(services, options = {}) {
3358
3470
  });
3359
3471
  }
3360
3472
  }
3361
- if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0) {
3473
+ try {
3474
+ modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
3475
+ } catch (error) {
3476
+ observability.warn("sandbox reaper: Modal orphan sweep failed", {
3477
+ error: error instanceof Error ? error.message : String(error)
3478
+ });
3479
+ }
3480
+ if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
3362
3481
  observability.info("sandbox reaper swept", {
3363
3482
  drainable: drainable.length,
3364
3483
  terminated,
3365
3484
  skipped,
3366
3485
  metered: metered.accrued,
3367
- forceDrained
3486
+ forceDrained,
3487
+ modalOrphansTerminated
3368
3488
  });
3369
3489
  }
3370
- return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained };
3490
+ return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained, modalOrphansTerminated };
3371
3491
  }
3372
3492
  return { reapSandboxLeases: reapSandboxLeases2 };
3373
3493
  }
@@ -3443,6 +3563,37 @@ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observa
3443
3563
  }
3444
3564
  return forceDrained;
3445
3565
  }
3566
+ async function sweepModalOrphansForConfiguredBackend(settings, db, observability) {
3567
+ if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret) {
3568
+ return 0;
3569
+ }
3570
+ let liveLeases;
3571
+ try {
3572
+ liveLeases = await listLiveModalSandboxLeaseAttributions(db);
3573
+ } catch (error) {
3574
+ observability.warn("sandbox reaper: live Modal lease attribution read failed", {
3575
+ error: error instanceof Error ? error.message : String(error)
3576
+ });
3577
+ return 0;
3578
+ }
3579
+ const result = await sweepModalOrphanSandboxes(settings, liveLeases);
3580
+ for (const terminated of result.terminated) {
3581
+ observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
3582
+ sandboxId: terminated.sandboxId,
3583
+ reason: terminated.reason,
3584
+ tags: JSON.stringify(terminated.tags)
3585
+ });
3586
+ }
3587
+ if (result.examined > 0) {
3588
+ observability.info("sandbox reaper: Modal orphan sweep completed", {
3589
+ examined: result.examined,
3590
+ terminated: result.terminated.length,
3591
+ skipped: result.skipped,
3592
+ appName: settings.modalAppName
3593
+ });
3594
+ }
3595
+ return result.terminated.length;
3596
+ }
3446
3597
  async function terminateDrainableBox(db, settings, row, observability, terminateBox) {
3447
3598
  const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);
3448
3599
  const lease = await readLease2(db, row.workspaceId, row.sandboxGroupId);
@@ -3483,6 +3634,30 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
3483
3634
  });
3484
3635
  return true;
3485
3636
  }
3637
+ if (backend === "modal" && !lease.resumeState && lease.instanceId) {
3638
+ const { wrote } = await persistArchive(null);
3639
+ if (!wrote) {
3640
+ observability.info("sandbox reaper: Modal lease re-armed before direct terminate \u2014 leaving sandbox RUNNING", {
3641
+ sandboxGroupId: lease.sandboxGroupId,
3642
+ backend,
3643
+ instanceId: lease.instanceId
3644
+ });
3645
+ return false;
3646
+ }
3647
+ try {
3648
+ await terminateModalSandboxById(settings, lease.instanceId);
3649
+ } catch (error) {
3650
+ if (!isProviderSandboxNotFoundError("modal", error)) {
3651
+ throw error;
3652
+ }
3653
+ observability.info("sandbox reaper: Modal sandbox already gone during direct terminate", {
3654
+ sandboxGroupId: lease.sandboxGroupId,
3655
+ backend,
3656
+ instanceId: lease.instanceId
3657
+ });
3658
+ }
3659
+ return true;
3660
+ }
3486
3661
  if (!lease.resumeState) {
3487
3662
  return true;
3488
3663
  }