@opengeni/worker-bundle 0.7.1 → 0.7.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dist/index.d.ts +184 -68
  2. package/dist/index.js +7099 -1643
  3. package/dist/index.js.map +1 -1
  4. package/package.json +18 -18
  5. package/src/activities/agent-turn.ts +4332 -1068
  6. package/src/activities/capabilities.ts +43 -8
  7. package/src/activities/codex-capacity.ts +223 -0
  8. package/src/activities/codex-rotation.ts +550 -54
  9. package/src/activities/common.ts +14 -2
  10. package/src/activities/context-compaction.ts +152 -118
  11. package/src/activities/documents.ts +32 -27
  12. package/src/activities/environment.ts +318 -75
  13. package/src/activities/file-upload-reaper.ts +79 -0
  14. package/src/activities/git-credential-renewal.ts +183 -0
  15. package/src/activities/goals.ts +142 -123
  16. package/src/activities/packs.ts +55 -5
  17. package/src/activities/parent-wake.ts +216 -42
  18. package/src/activities/recording.ts +154 -122
  19. package/src/activities/redaction.ts +3 -1
  20. package/src/activities/rig-verification.ts +441 -0
  21. package/src/activities/run-input.ts +110 -122
  22. package/src/activities/sandbox-lease.ts +186 -51
  23. package/src/activities/scheduled-tasks.ts +287 -171
  24. package/src/activities/session-state.ts +165 -157
  25. package/src/activities/streaming.ts +143 -19
  26. package/src/activities/turn-attempt-fenced.ts +14 -0
  27. package/src/activities/types.ts +106 -36
  28. package/src/activities/workflow-wake.ts +44 -0
  29. package/src/activities/workspace-capture.ts +964 -0
  30. package/src/activities.ts +75 -35
  31. package/src/concurrency.ts +11 -0
  32. package/src/http.ts +16 -6
  33. package/src/index.ts +322 -49
  34. package/src/observability-metrics.ts +441 -8
  35. package/src/op-journal.ts +46 -0
  36. package/src/sandbox-resume.ts +431 -201
  37. package/src/sandbox-routing.ts +209 -22
  38. package/src/workflows/activities.ts +65 -8
  39. package/src/workflows/file-upload-reaper.ts +12 -0
  40. package/src/workflows/rig-verification.ts +28 -0
  41. package/src/workflows/scheduled-tasks.ts +12 -26
  42. package/src/workflows/session.ts +330 -192
  43. package/src/workflows/workflow-wake.ts +13 -0
  44. package/src/workflows.ts +9 -7
@@ -33,6 +33,7 @@
33
33
  import {
34
34
  accrueWarmSeconds,
35
35
  confirmDrainCold,
36
+ appendSessionEventToSandboxGroup,
36
37
  countQueuedTurns,
37
38
  countSandboxLeasesByLiveness,
38
39
  forceDrainOverLimitViewerOnlyBoxes,
@@ -42,6 +43,7 @@ import {
42
43
  listMeterableWarmLeases,
43
44
  persistDrainSnapshot,
44
45
  readLease,
46
+ reapExpiredSessionListSnapshots,
45
47
  reapStaleLeaseHoldersGlobal,
46
48
  rlsContextForWorkspace,
47
49
  type MeterableWarmLease,
@@ -63,6 +65,7 @@ import {
63
65
  terminateModalSandboxById,
64
66
  } from "@opengeni/runtime";
65
67
  import type { ActivityServices } from "./types";
68
+ import { reconcilePendingParentSystemUpdates } from "./parent-wake";
66
69
  import {
67
70
  recordCreditBalanceGauges,
68
71
  recordCreditMicros,
@@ -120,16 +123,16 @@ type CreateSandboxClientForBackendFn = typeof createSandboxClientForBackend;
120
123
  * db/accountId/row and delegates to persistDrainSnapshot. Returns:
121
124
  * - wrote:false -> the CAS missed (re-armed / newer epoch / vanished). The box
122
125
  * is wanted again; the seam MUST NOT terminate it.
123
- * - priorArchive -> the superseded archive (if any) to GC the prior snapshot.
126
+ * - priorArchive / priorArchivePrev -> superseded archives to GC at drain.
124
127
  *
125
128
  * Pass null for archiveBase64 to CAS-check WITHOUT writing (re-arm guard for
126
129
  * backends with no persistWorkspace — ensures a re-arm during the snapshot window
127
- * aborts the terminate before client.delete()). priorArchive is always null in
128
- * this case.
130
+ * aborts the terminate before client.delete()). priorArchive values are always
131
+ * null in this case.
129
132
  */
130
133
  export type PersistArchiveFn = (
131
134
  archiveBase64: string | null,
132
- ) => Promise<{ wrote: boolean; priorArchive: string | null }>;
135
+ ) => Promise<{ wrote: boolean; priorArchive: string | null; priorArchivePrev: string | null }>;
133
136
 
134
137
  /** The provider-terminate seam. Production wires the real resume-by-id ->
135
138
  * persistWorkspace -> persist-onto-lease (epoch-fenced) -> snapshot-GC ->
@@ -164,7 +167,8 @@ export function createSandboxLeaseActivities(
164
167
  options: SandboxLeaseActivityOptions = {},
165
168
  ) {
166
169
  const terminateBox: TerminateBoxFn = options.terminateBox ?? terminateProviderBox;
167
- const sweepModalOrphans: SweepModalOrphansFn = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
170
+ const sweepModalOrphans: SweepModalOrphansFn =
171
+ options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
168
172
  /**
169
173
  * The one global reaper sweep. Idempotent; concurrency-safe with itself.
170
174
  * Gated by the caller (the Schedule is only registered when
@@ -172,9 +176,39 @@ export function createSandboxLeaseActivities(
172
176
  * with the flag off can never terminate a box.
173
177
  */
174
178
  async function reapSandboxLeases(): Promise<ReapSandboxLeasesResult> {
175
- const { db, settings, observability } = await services();
179
+ const service = await services();
180
+ const { db, settings, observability } = service;
181
+ const parentUpdates = await reconcilePendingParentSystemUpdates(service, 100).catch((error) => {
182
+ observability.warn("system-update outbox reconciliation failed", {
183
+ error: error instanceof Error ? error.message : String(error),
184
+ });
185
+ return { claimed: 0, delivered: 0, failed: 1 };
186
+ });
187
+ const expiredSessionListSnapshots = await reapExpiredSessionListSnapshots(db, 500).catch(
188
+ (error) => {
189
+ observability.warn("session-list snapshot reconciliation failed", {
190
+ error: error instanceof Error ? error.message : String(error),
191
+ });
192
+ return 0;
193
+ },
194
+ );
195
+ if (expiredSessionListSnapshots > 0) {
196
+ observability.info("expired session-list snapshots reaped", {
197
+ deleted: expiredSessionListSnapshots,
198
+ });
199
+ }
176
200
  if (!settings.sandboxOwnershipEnabled) {
177
- return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
201
+ if (parentUpdates.claimed > 0) {
202
+ observability.info("system-update outbox reconciled", parentUpdates);
203
+ }
204
+ return {
205
+ examined: 0,
206
+ terminated: 0,
207
+ skipped: 0,
208
+ metered: 0,
209
+ forceDrained: 0,
210
+ modalOrphansTerminated: 0,
211
+ };
178
212
  }
179
213
  await refreshQueueLeaseAndCreditGauges(db, observability);
180
214
 
@@ -188,11 +222,26 @@ export function createSandboxLeaseActivities(
188
222
  // workspace at 0 balance / over its warm cap force-drains its VIEWER-ONLY
189
223
  // boxes (guarded turn_holders=0 — a paying turn is NEVER killed). The newly
190
224
  // draining rows are caught by the same sweep's terminate below.
191
- const forceDrained = await forceDrainOverLimitWorkspaces(db, settings, metered.workspaceIds, observability);
225
+ const forceDrained = await forceDrainOverLimitWorkspaces(
226
+ db,
227
+ settings,
228
+ metered.workspaceIds,
229
+ observability,
230
+ );
192
231
 
193
232
  // (1) The DB-only cross-workspace sweep. Returns the drainable rows.
194
233
  const drainable: ReapDrainable[] = await reapStaleLeaseHoldersGlobal(db, {
195
234
  viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
235
+ // Dead-worker turn holders: a live holder is touched every 10s from the
236
+ // moment it is registered (resumeBoxForTurn's holder-liveness loop covers
237
+ // the whole warmup — waitForWarm/establish/display-stack — and the turn
238
+ // heartbeat covers the run), so NO live path is ever silent for more than
239
+ // one tick. The horizon is deliberately generous defense-in-depth (not a
240
+ // tuned guess about path lengths): a killed worker's frozen holder —
241
+ // which would otherwise pin refcount >= 1 FOREVER, so the lease never
242
+ // drains and the box dies at the provider hard-timeout UNPERSISTED —
243
+ // clears within ~12 minutes.
244
+ turnHolderTtlMs: settings.sandboxWarmingTimeoutMs + settings.sandboxLeaseTtlMs,
196
245
  idleGraceMs: settings.sandboxIdleGraceMs,
197
246
  });
198
247
 
@@ -205,7 +254,13 @@ export function createSandboxLeaseActivities(
205
254
  // next sweep retries it; the provider idle-timeout is the backstop).
206
255
  for (const row of drainable) {
207
256
  try {
208
- const drainedCold = await terminateDrainableBox(db, settings, row, observability, terminateBox);
257
+ const drainedCold = await terminateDrainableBox(
258
+ db,
259
+ settings,
260
+ row,
261
+ observability,
262
+ terminateBox,
263
+ );
209
264
  if (drainedCold) {
210
265
  terminated += 1;
211
266
  } else {
@@ -233,7 +288,12 @@ export function createSandboxLeaseActivities(
233
288
 
234
289
  await refreshQueueLeaseAndCreditGauges(db, observability);
235
290
 
236
- if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
291
+ if (
292
+ drainable.length > 0 ||
293
+ metered.accrued > 0 ||
294
+ forceDrained > 0 ||
295
+ modalOrphansTerminated > 0
296
+ ) {
237
297
  observability.info("sandbox reaper swept", {
238
298
  drainable: drainable.length,
239
299
  terminated,
@@ -244,7 +304,14 @@ export function createSandboxLeaseActivities(
244
304
  });
245
305
  }
246
306
 
247
- return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained, modalOrphansTerminated };
307
+ return {
308
+ examined: drainable.length,
309
+ terminated,
310
+ skipped,
311
+ metered: metered.accrued,
312
+ forceDrained,
313
+ modalOrphansTerminated,
314
+ };
248
315
  }
249
316
 
250
317
  return { reapSandboxLeases };
@@ -341,7 +408,8 @@ async function forceDrainOverLimitWorkspaces(
341
408
  workspaceIds: Set<string>,
342
409
  observability: ActivityServices["observability"],
343
410
  ): Promise<number> {
344
- const enforceBalance = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
411
+ const enforceBalance =
412
+ settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
345
413
  const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
346
414
  // Nothing to enforce → skip the whole pass (no lock churn).
347
415
  if (!enforceBalance && cap <= 0) {
@@ -351,7 +419,9 @@ async function forceDrainOverLimitWorkspaces(
351
419
  for (const workspaceId of workspaceIds) {
352
420
  try {
353
421
  const { accountId } = await rlsContextForWorkspace(db, workspaceId);
354
- const balance = enforceBalance ? await getBillingBalance(db, accountId) : { balanceMicros: 1 } as { balanceMicros: number };
422
+ const balance = enforceBalance
423
+ ? await getBillingBalance(db, accountId)
424
+ : ({ balanceMicros: 1 } as { balanceMicros: number });
355
425
  const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
356
426
  workspaceId,
357
427
  balanceMicros: balance.balanceMicros,
@@ -452,9 +522,9 @@ async function terminateDrainableBox(
452
522
  return false;
453
523
  }
454
524
  if (
455
- lease.liveness !== "draining"
456
- || lease.refcount !== 0
457
- || lease.leaseEpoch !== row.leaseEpoch
525
+ lease.liveness !== "draining" ||
526
+ lease.refcount !== 0 ||
527
+ lease.leaseEpoch !== row.leaseEpoch
458
528
  ) {
459
529
  // Re-armed (warm again) / a newer epoch / already drained by a concurrent
460
530
  // sweep. Skip — provider stop() must fire ONLY past the drain grace at
@@ -467,14 +537,23 @@ async function terminateDrainableBox(
467
537
  // Same guard as confirmDrainCold (draining AND refcount=0 AND lease_epoch=
468
538
  // expected): a re-arm or newer epoch that snuck in writes ZERO rows → wrote:false
469
539
  // → the seam leaves the box RUNNING and we skip the cold-commit below.
470
- const persistArchive: PersistArchiveFn = async (archiveBase64: string | null) =>
471
- await persistDrainSnapshot(db, {
540
+ // `persisted` tracks whether a real archive landed on the lease this drain —
541
+ // the durable sandbox.box.terminated event below carries it, so a "terminated
542
+ // with NOTHING persisted" (box already dead at drain) is visible in the DB.
543
+ let persisted = false;
544
+ const persistArchive: PersistArchiveFn = async (archiveBase64: string | null) => {
545
+ const result = await persistDrainSnapshot(db, {
472
546
  accountId,
473
547
  workspaceId: row.workspaceId,
474
548
  sandboxGroupId: row.sandboxGroupId,
475
549
  expectedEpoch: row.leaseEpoch,
476
550
  workspaceArchive: archiveBase64,
477
551
  });
552
+ if (result.wrote && archiveBase64 !== null) {
553
+ persisted = true;
554
+ }
555
+ return result;
556
+ };
478
557
 
479
558
  // Resume-by-id -> persistWorkspace -> persist-onto-lease -> GC prior snapshot ->
480
559
  // provider terminate. A box that is already gone (NotFound) is success (the goal
@@ -497,6 +576,26 @@ async function terminateDrainableBox(
497
576
  sandboxGroupId: row.sandboxGroupId,
498
577
  expectedEpoch: row.leaseEpoch,
499
578
  });
579
+ if (wentCold) {
580
+ // Durable termination record (sandbox-file-persistence observability): who
581
+ // ended this box and whether its /workspace was captured first, appended to
582
+ // every session sharing the group's box. Best-effort: attribution must
583
+ // never affect the drain outcome.
584
+ try {
585
+ await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {
586
+ type: "sandbox.box.terminated",
587
+ payload: { actor: "reaper", persisted, instanceId: lease.instanceId },
588
+ });
589
+ } catch (eventError) {
590
+ observability.warn(
591
+ "sandbox reaper: box-terminated event write failed (drain outcome unaffected)",
592
+ {
593
+ sandboxGroupId: row.sandboxGroupId,
594
+ error: eventError instanceof Error ? eventError.message : String(eventError),
595
+ },
596
+ );
597
+ }
598
+ }
500
599
  return wentCold;
501
600
  }
502
601
 
@@ -557,11 +656,18 @@ export async function terminateProviderBox(
557
656
  // persistence — nothing to snapshot), no delete/kill. The session simply detaches;
558
657
  // the machine stays up under the agent's own process lifetime (§23.0). Checked on
559
658
  // BOTH the lease backend and the resume envelope's backend so neither path leaks.
560
- if (backend === "selfhosted" || lease.backend === "selfhosted" || lease.resumeBackendId === "selfhosted") {
561
- observability.info("sandbox reaper: selfhosted lease drained to cold (NEVER provider-stopped — it is the user's machine)", {
562
- sandboxGroupId: lease.sandboxGroupId,
563
- backend,
564
- });
659
+ if (
660
+ backend === "selfhosted" ||
661
+ lease.backend === "selfhosted" ||
662
+ lease.resumeBackendId === "selfhosted"
663
+ ) {
664
+ observability.info(
665
+ "sandbox reaper: selfhosted lease drained to cold (NEVER provider-stopped — it is the user's machine)",
666
+ {
667
+ sandboxGroupId: lease.sandboxGroupId,
668
+ backend,
669
+ },
670
+ );
565
671
  return true;
566
672
  }
567
673
 
@@ -572,11 +678,14 @@ export async function terminateProviderBox(
572
678
  if (backend === "modal" && !lease.resumeState && lease.instanceId) {
573
679
  const { wrote } = await persistArchive(null);
574
680
  if (!wrote) {
575
- observability.info("sandbox reaper: Modal lease re-armed before direct terminate — leaving sandbox RUNNING", {
576
- sandboxGroupId: lease.sandboxGroupId,
577
- backend,
578
- instanceId: lease.instanceId,
579
- });
681
+ observability.info(
682
+ "sandbox reaper: Modal lease re-armed before direct terminate — leaving sandbox RUNNING",
683
+ {
684
+ sandboxGroupId: lease.sandboxGroupId,
685
+ backend,
686
+ instanceId: lease.instanceId,
687
+ },
688
+ );
580
689
  return false;
581
690
  }
582
691
  try {
@@ -629,7 +738,10 @@ export async function terminateProviderBox(
629
738
  // legacy flat envelope — providerState at the top — resumable too.)
630
739
  const envelopeSessionState =
631
740
  (lease.resumeState as { sessionState?: unknown }).sessionState ?? lease.resumeState;
632
- const resumedState = await deserializeSandboxSessionStateEnvelope(client as never, envelopeSessionState);
741
+ const resumedState = await deserializeSandboxSessionStateEnvelope(
742
+ client as never,
743
+ envelopeSessionState,
744
+ );
633
745
  if (resumedState === undefined) {
634
746
  return true;
635
747
  }
@@ -637,10 +749,13 @@ export async function terminateProviderBox(
637
749
  sessionState = resumedState;
638
750
  } catch (error) {
639
751
  if (isProviderSandboxNotFoundError(client.backendId, error)) {
640
- observability.info("sandbox reaper: drainable box already gone (NotFound on resume) — proceeding to cold", {
641
- sandboxGroupId: lease.sandboxGroupId,
642
- backend,
643
- });
752
+ observability.info(
753
+ "sandbox reaper: drainable box already gone (NotFound on resume) — proceeding to cold",
754
+ {
755
+ sandboxGroupId: lease.sandboxGroupId,
756
+ backend,
757
+ },
758
+ );
644
759
  return true;
645
760
  }
646
761
  // Re-throw a non-NotFound resume failure so the caller SKIPS (the lease stays
@@ -657,17 +772,23 @@ export async function terminateProviderBox(
657
772
  archiveBytes = session?.persistWorkspace ? await session.persistWorkspace() : undefined;
658
773
  } catch (error) {
659
774
  if (isProviderSandboxNotFoundError(client.backendId, error)) {
660
- observability.info("sandbox reaper: box gone during persist (NotFound) — nothing to snapshot, proceeding to cold", {
661
- sandboxGroupId: lease.sandboxGroupId,
662
- backend,
663
- });
775
+ observability.info(
776
+ "sandbox reaper: box gone during persist (NotFound) — nothing to snapshot, proceeding to cold",
777
+ {
778
+ sandboxGroupId: lease.sandboxGroupId,
779
+ backend,
780
+ },
781
+ );
664
782
  return true;
665
783
  }
666
- observability.warn("sandbox reaper: persistWorkspace failed — leaving box draining (files NOT lost)", {
667
- sandboxGroupId: lease.sandboxGroupId,
668
- backend,
669
- error: error instanceof Error ? error.message : String(error),
670
- });
784
+ observability.warn(
785
+ "sandbox reaper: persistWorkspace failed — leaving box draining (files NOT lost)",
786
+ {
787
+ sandboxGroupId: lease.sandboxGroupId,
788
+ backend,
789
+ error: error instanceof Error ? error.message : String(error),
790
+ },
791
+ );
671
792
  // NEVER terminate a box whose snapshot we could not capture.
672
793
  throw error;
673
794
  }
@@ -686,12 +807,15 @@ export async function terminateProviderBox(
686
807
  // refcount/epoch guard, no write. wrote:false → abort the terminate.
687
808
  if (archiveBytes && archiveBytes.length > 0) {
688
809
  const archiveBase64 = Buffer.from(archiveBytes).toString("base64");
689
- const { wrote, priorArchive } = await persistArchive(archiveBase64);
810
+ const { wrote, priorArchive, priorArchivePrev } = await persistArchive(archiveBase64);
690
811
  if (!wrote) {
691
- observability.info("sandbox reaper: lease re-armed during persist — leaving box RUNNING (no terminate)", {
692
- sandboxGroupId: lease.sandboxGroupId,
693
- backend,
694
- });
812
+ observability.info(
813
+ "sandbox reaper: lease re-armed during persist — leaving box RUNNING (no terminate)",
814
+ {
815
+ sandboxGroupId: lease.sandboxGroupId,
816
+ backend,
817
+ },
818
+ );
695
819
  return false;
696
820
  }
697
821
  // Keep-latest-per-lease GC: best-effort delete the prior snapshot image now,
@@ -704,16 +828,27 @@ export async function terminateProviderBox(
704
828
  snapshotId: deleted,
705
829
  });
706
830
  }
831
+ const deletedPrev = await deletePriorPersistedSnapshot(session, priorArchivePrev);
832
+ if (deletedPrev) {
833
+ observability.info("sandbox reaper: GC'd superseded previous workspace snapshot", {
834
+ sandboxGroupId: lease.sandboxGroupId,
835
+ backend,
836
+ snapshotId: deletedPrev,
837
+ });
838
+ }
707
839
  } else {
708
840
  // No archive produced (backend has no persistWorkspace or returned empty).
709
841
  // CAS-check via persistArchive(null): if the lease was re-armed during the
710
842
  // snapshot window, wrote:false → leave the box RUNNING (abort terminate).
711
843
  const { wrote } = await persistArchive(null);
712
844
  if (!wrote) {
713
- observability.info("sandbox reaper: lease re-armed during snapshot window (no-archive path) — leaving box RUNNING (no terminate)", {
714
- sandboxGroupId: lease.sandboxGroupId,
715
- backend,
716
- });
845
+ observability.info(
846
+ "sandbox reaper: lease re-armed during snapshot window (no-archive path) — leaving box RUNNING (no terminate)",
847
+ {
848
+ sandboxGroupId: lease.sandboxGroupId,
849
+ backend,
850
+ },
851
+ );
717
852
  return false;
718
853
  }
719
854
  }