@opengeni/worker-bundle 0.22.8 → 0.23.3-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/dist/activities/agent-turn/agent-build.d.ts +1 -4
  2. package/dist/activities/agent-turn/claim.d.ts +2 -4
  3. package/dist/activities/agent-turn/codex.d.ts +6 -5
  4. package/dist/activities/agent-turn/compaction-prep.d.ts +2 -0
  5. package/dist/activities/agent-turn/errors.d.ts +13 -0
  6. package/dist/activities/agent-turn/finalization.d.ts +1 -3
  7. package/dist/activities/agent-turn/quiescence.d.ts +39 -0
  8. package/dist/activities/agent-turn/sandbox-route.d.ts +14 -24
  9. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -4
  10. package/dist/activities/agent-turn/sandbox-shared-preparation.d.ts +30 -0
  11. package/dist/activities/agent-turn/stream-attempt.d.ts +7 -0
  12. package/dist/activities/agent-turn/tool-policy.d.ts +3 -32
  13. package/dist/activities/agent-turn/turn-context.d.ts +1 -8
  14. package/dist/activities/agent-turn/turn-sandbox-access.d.ts +12 -0
  15. package/dist/activities/agent-turn/video-reference-sandbox.d.ts +7 -0
  16. package/dist/activities/codex-rotation.d.ts +2 -0
  17. package/dist/activities/session-state.d.ts +2 -1
  18. package/dist/{activities-control-VZIIV7RX.js → activities-control-52W5IDWM.js} +125 -13
  19. package/dist/activities-control-52W5IDWM.js.map +1 -0
  20. package/dist/{activities-turn-5RGQDBFV.js → activities-turn-A2G2KLSY.js} +534 -482
  21. package/dist/activities-turn-A2G2KLSY.js.map +1 -0
  22. package/dist/{chunk-NAVSAZ56.js → chunk-GFNXVZVQ.js} +160 -1
  23. package/dist/chunk-GFNXVZVQ.js.map +1 -0
  24. package/dist/{chunk-O4HDX5SC.js → chunk-IPVMCAW5.js} +144 -15
  25. package/dist/chunk-IPVMCAW5.js.map +1 -0
  26. package/dist/index.js +42 -2
  27. package/dist/index.js.map +1 -1
  28. package/dist/observability-metrics.d.ts +28 -0
  29. package/dist/personal-github-git-credentials.d.ts +1 -0
  30. package/dist/unhandled-rejection-boundary.d.ts +29 -0
  31. package/package.json +18 -18
  32. package/src/activities/agent-turn/agent-build.ts +15 -31
  33. package/src/activities/agent-turn/claim.ts +29 -68
  34. package/src/activities/agent-turn/codex.ts +19 -10
  35. package/src/activities/agent-turn/compaction-prep.ts +50 -1
  36. package/src/activities/agent-turn/errors.ts +78 -0
  37. package/src/activities/agent-turn/failure-settlement.ts +30 -19
  38. package/src/activities/agent-turn/finalization.ts +24 -21
  39. package/src/activities/agent-turn/history.ts +5 -1
  40. package/src/activities/agent-turn/quiescence.ts +86 -0
  41. package/src/activities/agent-turn/run-credentials.ts +12 -1
  42. package/src/activities/agent-turn/run.ts +14 -11
  43. package/src/activities/agent-turn/sandbox-establish.ts +46 -11
  44. package/src/activities/agent-turn/sandbox-route.ts +21 -25
  45. package/src/activities/agent-turn/sandbox-runtime.ts +1 -116
  46. package/src/activities/agent-turn/sandbox-shared-preparation.ts +206 -0
  47. package/src/activities/agent-turn/stream-attempt.ts +98 -14
  48. package/src/activities/agent-turn/tool-environment.ts +10 -13
  49. package/src/activities/agent-turn/tool-policy.ts +3 -55
  50. package/src/activities/agent-turn/turn-context.ts +8 -14
  51. package/src/activities/agent-turn/{image-reference-sandbox.ts → turn-sandbox-access.ts} +15 -13
  52. package/src/activities/agent-turn/video-reference-sandbox.ts +23 -0
  53. package/src/activities/codex-capacity.ts +13 -5
  54. package/src/activities/codex-rotation.ts +17 -0
  55. package/src/activities/goals.ts +10 -4
  56. package/src/activities/retained-screenshots.ts +104 -21
  57. package/src/activities/sandbox-lease.ts +7 -4
  58. package/src/activities/scheduled-tasks.ts +128 -4
  59. package/src/activities/session-state.ts +8 -1
  60. package/src/github-rest-mcp.ts +6 -3
  61. package/src/http.ts +3 -0
  62. package/src/index.ts +19 -0
  63. package/src/observability-metrics.ts +200 -0
  64. package/src/personal-github-git-credentials.ts +45 -4
  65. package/src/sandbox-resume.ts +1 -1
  66. package/src/sandbox-routing.ts +7 -4
  67. package/src/unhandled-rejection-boundary.ts +73 -0
  68. package/dist/activities/agent-turn/image-reference-sandbox.d.ts +0 -14
  69. package/dist/activities/recording.d.ts +0 -90
  70. package/dist/activities-control-VZIIV7RX.js.map +0 -1
  71. package/dist/activities-turn-5RGQDBFV.js.map +0 -1
  72. package/dist/chunk-NAVSAZ56.js.map +0 -1
  73. package/dist/chunk-O4HDX5SC.js.map +0 -1
  74. package/src/activities/recording.ts +0 -279
@@ -22,6 +22,7 @@ import {
22
22
  resolveConnectedMachineWorkspaceRoot,
23
23
  } from "@opengeni/runtime";
24
24
  import { type Settings } from "@opengeni/config";
25
+ import { managedSessionGroupBackend, managedSessionGroupOs } from "@opengeni/core";
25
26
  import { mergeResourceRefs } from "../common";
26
27
  import {
27
28
  mintSandboxCodemodeToken,
@@ -42,6 +43,7 @@ import {
42
43
  makeMachineOpObserver,
43
44
  recordSandboxLogicalProvision,
44
45
  recordSandboxProvisionAttempt,
46
+ recordSandboxSharedPreparation,
45
47
  recordTurnStartupPhase,
46
48
  runtimeMetricsHooksForObservability,
47
49
  } from "../../observability-metrics";
@@ -62,11 +64,13 @@ import {
62
64
  sandboxEstablishPolicyDecision,
63
65
  shouldPrefetchManagedSandbox,
64
66
  reconcileActiveSandboxPointer,
67
+ sandboxSettingsForRoute,
65
68
  } from "./sandbox-route";
66
69
  import type { ClaimTurnOk } from "./claim";
67
70
  import type { GovernanceModelOk } from "./governance-model";
68
71
  import type { SandboxTurnRuntime } from "./sandbox-runtime";
69
72
  import type { AttemptIdentityState, EventingState, SandboxRuntimeState } from "./turn-context";
73
+ import { createSharedRigSetupCoordinator } from "./sandbox-shared-preparation";
70
74
 
71
75
  export type SandboxRouteDeps = {
72
76
  input: RunAgentTurnInput;
@@ -249,20 +253,32 @@ export async function resolveSandboxRoute(deps: SandboxRouteDeps): Promise<Sandb
249
253
  // common path this is runSettings.sandboxBackend. A selfhosted home turn
250
254
  // that is NOT machine-primary falls back to the deployment cloud backend
251
255
  // so swap-away / flag-off degrade to a real group box.
252
- const groupBoxBackend: Settings["sandboxBackend"] =
253
- runSettings.sandboxBackend === "selfhosted" && !machinePrimary
254
- ? settings.sandboxBackend
255
- : runSettings.sandboxBackend;
256
+ const groupBoxBackend: Settings["sandboxBackend"] = !machinePrimary
257
+ ? (managedSessionGroupBackend(settings.sandboxBackend, runSettings.sandboxBackend) ??
258
+ runSettings.sandboxBackend)
259
+ : runSettings.sandboxBackend;
260
+ const groupRouteSettings = sandboxSettingsForRoute({
261
+ runSettings,
262
+ machinePrimary,
263
+ groupBoxBackend,
264
+ });
265
+ if (groupRouteSettings !== runSettings) {
266
+ // `modelRunSettings` drives SandboxAgent manifest construction and the
267
+ // legacy SDK-owned client path. Keep its provider/model context overrides,
268
+ // but replace the stale durable machine-home label with the route that will
269
+ // actually execute.
270
+ eventing.modelRunSettings = {
271
+ ...eventing.modelRunSettings,
272
+ sandboxBackend: groupRouteSettings.sandboxBackend,
273
+ };
274
+ }
256
275
  sandboxState.startupMilestoneBackend = activeSandboxBackend ?? groupBoxBackend;
257
276
  media.sandboxFileDownloadBackend = sandboxState.startupMilestoneBackend;
258
277
  // Durable lease identity is always the logical OCI/base image. A provider
259
278
  // image id in runSettings is only a physical cold-create optimization and
260
279
  // must never become cross-surface shared-state fencing.
261
280
  const groupBoxImage = rigProviderImageSourceImage(logicalSandboxSettings, groupBoxBackend);
262
- const sandboxCreationBackend: Settings["sandboxBackend"] =
263
- settings.sandboxOwnershipEnabled && runSettings.sandboxBackend !== "none"
264
- ? groupBoxBackend
265
- : runSettings.sandboxBackend;
281
+ const sandboxCreationBackend: Settings["sandboxBackend"] = groupRouteSettings.sandboxBackend;
266
282
  const effectiveRunCredentialBackend = activeSandboxBackend ?? groupBoxBackend;
267
283
 
268
284
  return {
@@ -315,6 +331,7 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
315
331
  rigVersion,
316
332
  turnResources,
317
333
  } = deps;
334
+ const groupBoxOs = managedSessionGroupOs(runSettings.sandboxBackend, session.sandboxOs);
318
335
  const {
319
336
  releaseLateSandbox,
320
337
  onHomeSandboxRebound,
@@ -577,7 +594,7 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
577
594
  sandboxGroupId: lazyGroupId,
578
595
  sessionId: input.sessionId,
579
596
  backend: groupBoxBackend,
580
- os: session.sandboxOs,
597
+ os: groupBoxOs,
581
598
  environment: sandboxEnvironment,
582
599
  ...(groupBoxImage
583
600
  ? {
@@ -656,7 +673,7 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
656
673
  // deployment default), never a "selfhosted" box (which would throw
657
674
  // for lack of a bound agentId).
658
675
  backend: groupBoxBackend,
659
- os: session.sandboxOs,
676
+ os: groupBoxOs,
660
677
  environment: sandboxEnvironment,
661
678
  // IMAGE IS SHARED STATE (B3): the container image
662
679
  // this run resolves. The lease stamps it + conflicts on a live shared box
@@ -881,7 +898,7 @@ export async function bindLazySandboxProvisioner(
881
898
  // recursion that HANGS the turn — caught live on staging 2026-07-08). The SDK
882
899
  // already holds the proxy directly (injected as lazyOwnedSandbox.session), so it
883
900
  // gets per-op routing; the worker-side handle (resolvedSandbox: release,
884
- // heartbeat, computer-use recording) wants the real box, unproxied.
901
+ // heartbeat, workspace capture) wants the real box, unproxied.
885
902
  return provisioned;
886
903
  },
887
904
  {
@@ -1007,6 +1024,24 @@ export async function bindLazySandboxProvisioner(
1007
1024
  ),
1008
1025
  }
1009
1026
  : {}),
1027
+ ...(sandboxState.sandboxGroupId && sandboxState.sandboxHolderId
1028
+ ? {
1029
+ coordinateSharedRigSetup: createSharedRigSetupCoordinator({
1030
+ db,
1031
+ accountId: input.accountId,
1032
+ workspaceId: input.workspaceId,
1033
+ sandboxGroupId: sandboxState.sandboxGroupId,
1034
+ attemptId: input.attemptId,
1035
+ holderId: sandboxState.sandboxHolderId,
1036
+ sandbox: provisioned,
1037
+ ...(runtimeCancellationSignal
1038
+ ? { signal: runtimeCancellationSignal }
1039
+ : {}),
1040
+ observe: (measurement) =>
1041
+ recordSandboxSharedPreparation(observability, measurement),
1042
+ }),
1043
+ }
1044
+ : {}),
1010
1045
  },
1011
1046
  ),
1012
1047
  (measurement) => {
@@ -64,6 +64,27 @@ export function managedSandboxOwnershipForTurn(
64
64
  };
65
65
  }
66
66
 
67
+ /**
68
+ * Bind backend-aware runtime settings to the route that will actually execute.
69
+ *
70
+ * A machine-home turn keeps `selfhosted` as its durable policy. After an
71
+ * explicit clear-to-default, however, both the ownership-inverted path and the
72
+ * legacy SDK-owned path must construct manifests and clients for the resolved
73
+ * managed group backend. Machine-primary and ordinary session routes preserve
74
+ * their existing settings object.
75
+ */
76
+ export function sandboxSettingsForRoute(input: {
77
+ runSettings: Settings;
78
+ machinePrimary: boolean;
79
+ groupBoxBackend: Settings["sandboxBackend"];
80
+ }): Settings {
81
+ return input.runSettings.sandboxBackend === "selfhosted" &&
82
+ !input.machinePrimary &&
83
+ input.groupBoxBackend !== "selfhosted"
84
+ ? { ...input.runSettings, sandboxBackend: input.groupBoxBackend }
85
+ : input.runSettings;
86
+ }
87
+
67
88
  /** A sandboxless home still establishes an explicitly attached Connected Machine. */
68
89
  export function shouldEstablishSandboxForTurn(
69
90
  sandboxOwnershipEnabled: boolean,
@@ -331,28 +352,3 @@ export function sandboxArtifactRuntimeAdmission(
331
352
  },
332
353
  };
333
354
  }
334
-
335
- /**
336
- * Decide whether the first actual computer action may start a proof recording.
337
- *
338
- * On-turn recording runs ffmpeg/x11grab INSIDE the box and reads the .mp4 back
339
- * out of the box's /tmp — plumbing that exists only for OpenGeni-operated cloud
340
- * boxes (the Modal desktop backend). A turn whose EFFECTIVE backend is a connected
341
- * machine ("selfhosted") runs on the user's REAL computer, which has none of that
342
- * capture plumbing (and the platform must never shell ffmpeg onto a user's machine
343
- * — the same reason the runtime skips its setup hooks for selfhosted). Left ungated
344
- * it films nothing, finds no /tmp file, and emits recording.started followed by
345
- * recording.failed{box-death} on EVERY machine-primary turn — misleading timeline
346
- * noise + wasted work. So gate it off, exactly like a recording-disabled deployment:
347
- * skip silently, emit nothing (no new event shape).
348
- *
349
- * `effectiveBackend` is the resolved ACTIVE backend for the turn
350
- * (resolveActiveSandboxBackend) — NOT the session's home backend. A modal-home
351
- * session actively swapped onto a machine resolves to "selfhosted" here and
352
- * correctly skips; a machine-home turn that degraded back to its cloud group box
353
- * (swap-away / flag-off) resolves to undefined and records as before.
354
- *
355
- * EDGE — mid-turn swap: this is evaluated once when computer-use first runs. A swap
356
- * after recording starts is deliberately ignored; the partial recording already has
357
- * defined failure semantics, so there is no stop/restart machinery.
358
- */
@@ -4,13 +4,11 @@ import {
4
4
  heartbeatLeaseHolder,
5
5
  readLease,
6
6
  accrueWarmSeconds,
7
- abandonRecordingForTurnAttempt,
8
7
  SandboxWorkspaceMutationFencedError,
9
8
  } from "@opengeni/db";
10
9
  import {
11
10
  RoutingMutationOutcomeUnknownError,
12
11
  type EstablishedSandboxSession,
13
- stopRecording as stopRecordingOnBox,
14
12
  } from "@opengeni/runtime";
15
13
  import {
16
14
  sandboxLifecycleTransitionWaitMs,
@@ -26,10 +24,7 @@ import {
26
24
  type ResumedTurnSandbox,
27
25
  } from "../../sandbox-resume";
28
26
  import { recordCreditMicros } from "../../observability-metrics";
29
- import { beginRecording, discardUnpublishedRecording, type ActiveRecording } from "../recording";
30
27
  import { ChannelAPartialMutationError } from "@opengeni/runtime/sandbox";
31
- import { sandboxRunAs } from "@opengeni/runtime";
32
- import { randomUUID } from "node:crypto";
33
28
 
34
29
  import { safeErrorDiagnostic } from "./errors";
35
30
  import {
@@ -38,13 +33,7 @@ import {
38
33
  finalizeDurableTurnOpStreams,
39
34
  } from "./quiescence";
40
35
  import { SandboxDeadlineRotationError } from "./sandbox-provision";
41
- import { shouldStartOnTurnRecording } from "./tool-policy";
42
- import type {
43
- AttemptIdentityState,
44
- EventingState,
45
- RecordingState,
46
- SandboxRuntimeState,
47
- } from "./turn-context";
36
+ import type { AttemptIdentityState, EventingState, SandboxRuntimeState } from "./turn-context";
48
37
 
49
38
  export type SandboxTurnRuntimeDeps = {
50
39
  input: RunAgentTurnInput;
@@ -56,7 +45,6 @@ export type SandboxTurnRuntimeDeps = {
56
45
  activityContext: ReturnType<typeof currentActivityContext>;
57
46
  sandboxRotationController: AbortController;
58
47
  sandboxState: SandboxRuntimeState;
59
- recordingState: RecordingState;
60
48
  eventing: EventingState;
61
49
  attempt: AttemptIdentityState;
62
50
  };
@@ -74,7 +62,6 @@ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
74
62
  activityContext,
75
63
  sandboxRotationController,
76
64
  sandboxState,
77
- recordingState,
78
65
  eventing,
79
66
  attempt,
80
67
  } = deps;
@@ -354,39 +341,6 @@ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
354
341
  // Turn-end capture needs the lease heartbeat to keep its holder alive, but
355
342
  // must prevent that same timer from starting another periodic snapshot
356
343
  // while it reads. This gate separates those two responsibilities.
357
- // Computer-use-only recording. Ordinary shell/filesystem turns leave this
358
- // null; the first actual computer action starts it after :0 is ready.
359
- // P4.3 recording gate: flips true in `onComputerUseReady`, the runtime's
360
- // execution-time callback for the first real computer action. It must flip
361
- // BEFORE awaiting recording startup: the SDK tool-call stream item can arrive
362
- // before ffmpeg has finished starting. A plain text turn ("hey"/"continue")
363
- // never invokes the callback, so settlement performs no storage PUT.
364
- const abandonActiveRecording = async (
365
- reason: string,
366
- disposition: "failed" | "discard" = "failed",
367
- ): Promise<void> => {
368
- const recording = recordingState.activeRecording as ActiveRecording | null;
369
- if (!recording) return;
370
- recordingState.activeRecording = null;
371
- if (sandboxState.resolvedSandbox) {
372
- await stopRecordingOnBox(
373
- sandboxState.resolvedSandbox.established.session,
374
- recording.proc,
375
- ).catch(() => undefined);
376
- }
377
- if (!attempt.turnId || attempt.executionGeneration <= 0) return;
378
- await abandonRecordingForTurnAttempt(db, {
379
- accountId: input.accountId,
380
- workspaceId: input.workspaceId,
381
- sessionId: input.sessionId,
382
- turnId: attempt.turnId,
383
- executionGeneration: attempt.executionGeneration,
384
- attemptId: input.attemptId,
385
- recordingId: recording.recordingId,
386
- disposition,
387
- reason,
388
- }).catch(() => undefined);
389
- };
390
344
  const flushRuntimeBatcher = async () => {
391
345
  const current = eventing.batcher as ReturnType<typeof createRuntimeBatcher> | null;
392
346
  await current?.flush().catch(() => undefined);
@@ -647,73 +601,6 @@ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
647
601
  sandboxState.leaseHeartbeatTimer.unref();
648
602
  }
649
603
  };
650
- const maybeStartOnTurnRecording = async (
651
- sandbox: ResumedTurnSandbox,
652
- effectiveBackend: Settings["sandboxBackend"] | undefined,
653
- ): Promise<void> => {
654
- if (recordingState.activeRecording) {
655
- return;
656
- }
657
- if (recordingState.computerUseRecordingStart) {
658
- await recordingState.computerUseRecordingStart;
659
- return;
660
- }
661
- // Called only by the runtime's first-computer-action hook. Plain sandbox
662
- // operations never start ffmpeg and never boot a display merely to record
663
- // an unused desktop. Recording failure never fails the computer action.
664
- if (
665
- shouldStartOnTurnRecording({
666
- recordingEnabled: settings.recordingEnabled,
667
- desktopEnabled: settings.sandboxDesktopEnabled,
668
- establishedBackendId: sandbox.established.backendId,
669
- // EFFECTIVE (active) backend, not the session home: a machine-primary turn
670
- // resolves to "selfhosted" and skips; a swap back to the cloud group box
671
- // resolves to undefined and records as before.
672
- effectiveBackend,
673
- })
674
- ) {
675
- recordingState.computerUseRecordingStart = (async () => {
676
- let begun: Awaited<ReturnType<typeof beginRecording>> | null = null;
677
- try {
678
- begun = await beginRecording({
679
- settings,
680
- db,
681
- accountId: input.accountId,
682
- workspaceId: input.workspaceId,
683
- sessionId: input.sessionId,
684
- turnId: attempt.turnId!,
685
- recordingId: randomUUID(),
686
- mode: "on-turn",
687
- session: sandbox.established.session,
688
- runAs: sandboxRunAs(settings),
689
- reason: null,
690
- });
691
- if (!eventing.publish) {
692
- throw new Error("recording started before the turn event publisher was ready");
693
- }
694
- await eventing.publish([{ type: "recording.started", payload: begun.started }]);
695
- recordingState.activeRecording = begun.active;
696
- } catch (recordingError) {
697
- recordingState.activeRecording = null;
698
- if (begun) {
699
- await discardUnpublishedRecording({
700
- db,
701
- accountId: input.accountId,
702
- workspaceId: input.workspaceId,
703
- active: begun.active,
704
- session: sandbox.established.session,
705
- });
706
- }
707
- console.error(
708
- "computer-use recording start failed (action outcome unaffected)",
709
- safeErrorDiagnostic(recordingError),
710
- );
711
- }
712
- })();
713
- await recordingState.computerUseRecordingStart;
714
- }
715
- };
716
-
717
604
  return {
718
605
  releaseLateSandbox,
719
606
  requireResolvedSandboxForMutation,
@@ -721,11 +608,9 @@ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
721
608
  onHomeSandboxRebound,
722
609
  runWorkspaceMutationForSandbox,
723
610
  stopLeaseHeartbeat,
724
- abandonActiveRecording,
725
611
  flushRuntimeBatcher,
726
612
  publishSandboxLifecycleEvents,
727
613
  publishSandboxLost,
728
614
  startLeaseHeartbeat,
729
- maybeStartOnTurnRecording,
730
615
  };
731
616
  }
@@ -0,0 +1,206 @@
1
+ import { randomUUID } from "node:crypto";
2
+
3
+ import {
4
+ claimSandboxSharedPreparation,
5
+ readSandboxSharedPreparation,
6
+ SandboxLeaseSupersededError,
7
+ settleSandboxSharedPreparation,
8
+ type Database,
9
+ } from "@opengeni/db";
10
+
11
+ import type { ResumedTurnSandbox } from "../../sandbox-resume";
12
+
13
+ type SharedRigSetupCoordinatorInput = {
14
+ specHash: string;
15
+ timeoutMs: number;
16
+ execute: () => Promise<void>;
17
+ };
18
+
19
+ type SharedRigSetupCoordinatorOptions = {
20
+ db: Database;
21
+ accountId: string;
22
+ workspaceId: string;
23
+ sandboxGroupId: string;
24
+ attemptId: string;
25
+ holderId: string;
26
+ sandbox: ResumedTurnSandbox;
27
+ signal?: AbortSignal;
28
+ observe?: (measurement: {
29
+ path: "owner" | "joined" | "reused";
30
+ outcome: "completed" | "failed";
31
+ durationSeconds: number;
32
+ }) => void;
33
+ };
34
+
35
+ function observeSharedPreparation(
36
+ options: SharedRigSetupCoordinatorOptions,
37
+ measurement: Parameters<NonNullable<SharedRigSetupCoordinatorOptions["observe"]>>[0],
38
+ ): void {
39
+ try {
40
+ options.observe?.(measurement);
41
+ } catch {
42
+ // Telemetry must never alter setup authority or settlement.
43
+ }
44
+ }
45
+
46
+ function waitForPreparationTransition(ms: number, signal?: AbortSignal): Promise<void> {
47
+ return new Promise((resolve, reject) => {
48
+ let settled = false;
49
+ const timer = setTimeout(finish, ms);
50
+ function finish(): void {
51
+ if (settled) return;
52
+ settled = true;
53
+ signal?.removeEventListener("abort", cancel);
54
+ resolve();
55
+ }
56
+ function cancel(): void {
57
+ if (settled) return;
58
+ settled = true;
59
+ clearTimeout(timer);
60
+ signal?.removeEventListener("abort", cancel);
61
+ reject(signal?.reason ?? new Error("Sandbox shared preparation wait cancelled"));
62
+ }
63
+ signal?.addEventListener("abort", cancel, { once: true });
64
+ if (signal?.aborted) cancel();
65
+ });
66
+ }
67
+
68
+ async function settleCompletedWithRetry(
69
+ options: SharedRigSetupCoordinatorOptions,
70
+ input: SharedRigSetupCoordinatorInput,
71
+ claimId: string,
72
+ ): Promise<boolean> {
73
+ let delayMs = 50;
74
+ for (let attempt = 0; attempt < 4; attempt += 1) {
75
+ try {
76
+ if (
77
+ await settleSandboxSharedPreparation(options.db, {
78
+ accountId: options.accountId,
79
+ workspaceId: options.workspaceId,
80
+ sandboxGroupId: options.sandboxGroupId,
81
+ expectedLeaseEpoch: options.sandbox.leaseEpoch,
82
+ expectedInstanceId: options.sandbox.established.instanceId,
83
+ specHash: input.specHash,
84
+ holderId: options.holderId,
85
+ claimId,
86
+ outcome: "completed",
87
+ })
88
+ ) {
89
+ return true;
90
+ }
91
+ } catch {
92
+ // The exact claim remains durable. Retry settlement without replaying the
93
+ // provider operation; a later owner can verify the box-local marker.
94
+ }
95
+ await waitForPreparationTransition(delayMs, options.signal);
96
+ delayMs *= 2;
97
+ }
98
+ return false;
99
+ }
100
+
101
+ /**
102
+ * Coordinate immutable rig setup across workers. Joiners use a revision-aware,
103
+ * exponentially backed-off durable read instead of the old fixed 250 ms lease
104
+ * polling pattern. PostgreSQL remains source of truth; a lost completion write
105
+ * is recovered by the next deadline owner through the hook's box-local marker.
106
+ */
107
+ export function createSharedRigSetupCoordinator(
108
+ options: SharedRigSetupCoordinatorOptions,
109
+ ): (input: SharedRigSetupCoordinatorInput) => Promise<"executed" | "reused"> {
110
+ return async (input) => {
111
+ const startedAt = performance.now();
112
+ const claimId = randomUUID();
113
+ const identity = {
114
+ accountId: options.accountId,
115
+ workspaceId: options.workspaceId,
116
+ sandboxGroupId: options.sandboxGroupId,
117
+ expectedLeaseEpoch: options.sandbox.leaseEpoch,
118
+ expectedInstanceId: options.sandbox.established.instanceId,
119
+ specHash: input.specHash,
120
+ holderId: options.holderId,
121
+ };
122
+ const claimTimeoutMs = Math.min(3_600_000, Math.max(30_000, input.timeoutMs + 30_000));
123
+ let delayMs = 100;
124
+ let observedRevision = -1;
125
+ let path: "owner" | "joined" | "reused" = "owner";
126
+
127
+ try {
128
+ for (;;) {
129
+ if (options.signal?.aborted) {
130
+ throw options.signal.reason ?? new Error("Sandbox shared preparation cancelled");
131
+ }
132
+ const claim = await claimSandboxSharedPreparation(options.db, {
133
+ ...identity,
134
+ claimId,
135
+ ownerAttemptId: options.attemptId,
136
+ timeoutMs: claimTimeoutMs,
137
+ });
138
+ if (claim.role === "fenced") {
139
+ throw new SandboxLeaseSupersededError(options.sandboxGroupId, options.sandbox.leaseEpoch);
140
+ }
141
+ if (claim.role === "reused") {
142
+ if (path !== "joined") path = "reused";
143
+ observeSharedPreparation(options, {
144
+ path,
145
+ outcome: "completed",
146
+ durationSeconds: (performance.now() - startedAt) / 1_000,
147
+ });
148
+ return "reused";
149
+ }
150
+ if (claim.role === "owner") {
151
+ try {
152
+ await input.execute();
153
+ } catch (error) {
154
+ await settleSandboxSharedPreparation(options.db, {
155
+ ...identity,
156
+ claimId,
157
+ outcome: "failed",
158
+ }).catch(() => undefined);
159
+ throw error;
160
+ }
161
+ if (await settleCompletedWithRetry(options, input, claimId)) {
162
+ observeSharedPreparation(options, {
163
+ path,
164
+ outcome: "completed",
165
+ durationSeconds: (performance.now() - startedAt) / 1_000,
166
+ });
167
+ return "executed";
168
+ }
169
+ throw new Error(
170
+ "Sandbox shared preparation completed but its durable settlement remains unavailable",
171
+ );
172
+ }
173
+
174
+ path = "joined";
175
+ observedRevision = claim.preparation.revision;
176
+ const jitterMs = Math.floor(Math.random() * Math.max(1, Math.floor(delayMs / 4)));
177
+ await waitForPreparationTransition(delayMs + jitterMs, options.signal);
178
+ const observed = await readSandboxSharedPreparation(options.db, identity);
179
+ if (observed.status === "fenced") {
180
+ throw new SandboxLeaseSupersededError(options.sandboxGroupId, options.sandbox.leaseEpoch);
181
+ }
182
+ if (observed.preparation.status === "completed") {
183
+ observeSharedPreparation(options, {
184
+ path,
185
+ outcome: "completed",
186
+ durationSeconds: (performance.now() - startedAt) / 1_000,
187
+ });
188
+ return "reused";
189
+ }
190
+ if (observed.preparation.status === "failed") {
191
+ delayMs = 100;
192
+ continue;
193
+ }
194
+ delayMs =
195
+ observed.preparation.revision === observedRevision ? Math.min(2_000, delayMs * 2) : 100;
196
+ }
197
+ } catch (error) {
198
+ observeSharedPreparation(options, {
199
+ path,
200
+ outcome: "failed",
201
+ durationSeconds: (performance.now() - startedAt) / 1_000,
202
+ });
203
+ throw error;
204
+ }
205
+ };
206
+ }