@opengeni/worker-bundle 0.22.7-canary.0 → 0.23.3-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/agent-build.d.ts +1 -4
- package/dist/activities/agent-turn/claim.d.ts +2 -4
- package/dist/activities/agent-turn/codex.d.ts +6 -5
- package/dist/activities/agent-turn/compaction-prep.d.ts +2 -0
- package/dist/activities/agent-turn/errors.d.ts +13 -0
- package/dist/activities/agent-turn/finalization.d.ts +1 -3
- package/dist/activities/agent-turn/quiescence.d.ts +39 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +14 -24
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -4
- package/dist/activities/agent-turn/sandbox-shared-preparation.d.ts +30 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +7 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +3 -32
- package/dist/activities/agent-turn/turn-context.d.ts +1 -8
- package/dist/activities/agent-turn/turn-sandbox-access.d.ts +12 -0
- package/dist/activities/agent-turn/video-reference-sandbox.d.ts +7 -0
- package/dist/activities/codex-rotation.d.ts +2 -0
- package/dist/activities/session-state.d.ts +2 -1
- package/dist/{activities-control-VZIIV7RX.js → activities-control-52W5IDWM.js} +125 -13
- package/dist/activities-control-52W5IDWM.js.map +1 -0
- package/dist/{activities-turn-5RGQDBFV.js → activities-turn-A2G2KLSY.js} +534 -482
- package/dist/activities-turn-A2G2KLSY.js.map +1 -0
- package/dist/{chunk-NAVSAZ56.js → chunk-GFNXVZVQ.js} +160 -1
- package/dist/chunk-GFNXVZVQ.js.map +1 -0
- package/dist/{chunk-O4HDX5SC.js → chunk-IPVMCAW5.js} +144 -15
- package/dist/chunk-IPVMCAW5.js.map +1 -0
- package/dist/index.js +42 -2
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +28 -0
- package/dist/personal-github-git-credentials.d.ts +1 -0
- package/dist/unhandled-rejection-boundary.d.ts +29 -0
- package/package.json +18 -18
- package/src/activities/agent-turn/agent-build.ts +15 -31
- package/src/activities/agent-turn/claim.ts +29 -68
- package/src/activities/agent-turn/codex.ts +19 -10
- package/src/activities/agent-turn/compaction-prep.ts +50 -1
- package/src/activities/agent-turn/errors.ts +78 -0
- package/src/activities/agent-turn/failure-settlement.ts +30 -19
- package/src/activities/agent-turn/finalization.ts +24 -21
- package/src/activities/agent-turn/history.ts +5 -1
- package/src/activities/agent-turn/quiescence.ts +86 -0
- package/src/activities/agent-turn/run-credentials.ts +12 -1
- package/src/activities/agent-turn/run.ts +14 -11
- package/src/activities/agent-turn/sandbox-establish.ts +46 -11
- package/src/activities/agent-turn/sandbox-route.ts +21 -25
- package/src/activities/agent-turn/sandbox-runtime.ts +1 -116
- package/src/activities/agent-turn/sandbox-shared-preparation.ts +206 -0
- package/src/activities/agent-turn/stream-attempt.ts +98 -14
- package/src/activities/agent-turn/tool-environment.ts +10 -13
- package/src/activities/agent-turn/tool-policy.ts +3 -55
- package/src/activities/agent-turn/turn-context.ts +8 -14
- package/src/activities/agent-turn/{image-reference-sandbox.ts → turn-sandbox-access.ts} +15 -13
- package/src/activities/agent-turn/video-reference-sandbox.ts +23 -0
- package/src/activities/codex-capacity.ts +13 -5
- package/src/activities/codex-rotation.ts +17 -0
- package/src/activities/goals.ts +10 -4
- package/src/activities/retained-screenshots.ts +104 -21
- package/src/activities/sandbox-lease.ts +7 -4
- package/src/activities/scheduled-tasks.ts +128 -4
- package/src/activities/session-state.ts +8 -1
- package/src/github-rest-mcp.ts +6 -3
- package/src/http.ts +3 -0
- package/src/index.ts +19 -0
- package/src/observability-metrics.ts +200 -0
- package/src/personal-github-git-credentials.ts +45 -4
- package/src/sandbox-resume.ts +1 -1
- package/src/sandbox-routing.ts +7 -4
- package/src/unhandled-rejection-boundary.ts +73 -0
- package/dist/activities/agent-turn/image-reference-sandbox.d.ts +0 -14
- package/dist/activities/recording.d.ts +0 -90
- package/dist/activities-control-VZIIV7RX.js.map +0 -1
- package/dist/activities-turn-5RGQDBFV.js.map +0 -1
- package/dist/chunk-NAVSAZ56.js.map +0 -1
- package/dist/chunk-O4HDX5SC.js.map +0 -1
- package/src/activities/recording.ts +0 -279
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
resolveConnectedMachineWorkspaceRoot,
|
|
23
23
|
} from "@opengeni/runtime";
|
|
24
24
|
import { type Settings } from "@opengeni/config";
|
|
25
|
+
import { managedSessionGroupBackend, managedSessionGroupOs } from "@opengeni/core";
|
|
25
26
|
import { mergeResourceRefs } from "../common";
|
|
26
27
|
import {
|
|
27
28
|
mintSandboxCodemodeToken,
|
|
@@ -42,6 +43,7 @@ import {
|
|
|
42
43
|
makeMachineOpObserver,
|
|
43
44
|
recordSandboxLogicalProvision,
|
|
44
45
|
recordSandboxProvisionAttempt,
|
|
46
|
+
recordSandboxSharedPreparation,
|
|
45
47
|
recordTurnStartupPhase,
|
|
46
48
|
runtimeMetricsHooksForObservability,
|
|
47
49
|
} from "../../observability-metrics";
|
|
@@ -62,11 +64,13 @@ import {
|
|
|
62
64
|
sandboxEstablishPolicyDecision,
|
|
63
65
|
shouldPrefetchManagedSandbox,
|
|
64
66
|
reconcileActiveSandboxPointer,
|
|
67
|
+
sandboxSettingsForRoute,
|
|
65
68
|
} from "./sandbox-route";
|
|
66
69
|
import type { ClaimTurnOk } from "./claim";
|
|
67
70
|
import type { GovernanceModelOk } from "./governance-model";
|
|
68
71
|
import type { SandboxTurnRuntime } from "./sandbox-runtime";
|
|
69
72
|
import type { AttemptIdentityState, EventingState, SandboxRuntimeState } from "./turn-context";
|
|
73
|
+
import { createSharedRigSetupCoordinator } from "./sandbox-shared-preparation";
|
|
70
74
|
|
|
71
75
|
export type SandboxRouteDeps = {
|
|
72
76
|
input: RunAgentTurnInput;
|
|
@@ -249,20 +253,32 @@ export async function resolveSandboxRoute(deps: SandboxRouteDeps): Promise<Sandb
|
|
|
249
253
|
// common path this is runSettings.sandboxBackend. A selfhosted home turn
|
|
250
254
|
// that is NOT machine-primary falls back to the deployment cloud backend
|
|
251
255
|
// so swap-away / flag-off degrade to a real group box.
|
|
252
|
-
const groupBoxBackend: Settings["sandboxBackend"] =
|
|
253
|
-
runSettings.sandboxBackend
|
|
254
|
-
|
|
255
|
-
|
|
256
|
+
const groupBoxBackend: Settings["sandboxBackend"] = !machinePrimary
|
|
257
|
+
? (managedSessionGroupBackend(settings.sandboxBackend, runSettings.sandboxBackend) ??
|
|
258
|
+
runSettings.sandboxBackend)
|
|
259
|
+
: runSettings.sandboxBackend;
|
|
260
|
+
const groupRouteSettings = sandboxSettingsForRoute({
|
|
261
|
+
runSettings,
|
|
262
|
+
machinePrimary,
|
|
263
|
+
groupBoxBackend,
|
|
264
|
+
});
|
|
265
|
+
if (groupRouteSettings !== runSettings) {
|
|
266
|
+
// `modelRunSettings` drives SandboxAgent manifest construction and the
|
|
267
|
+
// legacy SDK-owned client path. Keep its provider/model context overrides,
|
|
268
|
+
// but replace the stale durable machine-home label with the route that will
|
|
269
|
+
// actually execute.
|
|
270
|
+
eventing.modelRunSettings = {
|
|
271
|
+
...eventing.modelRunSettings,
|
|
272
|
+
sandboxBackend: groupRouteSettings.sandboxBackend,
|
|
273
|
+
};
|
|
274
|
+
}
|
|
256
275
|
sandboxState.startupMilestoneBackend = activeSandboxBackend ?? groupBoxBackend;
|
|
257
276
|
media.sandboxFileDownloadBackend = sandboxState.startupMilestoneBackend;
|
|
258
277
|
// Durable lease identity is always the logical OCI/base image. A provider
|
|
259
278
|
// image id in runSettings is only a physical cold-create optimization and
|
|
260
279
|
// must never become cross-surface shared-state fencing.
|
|
261
280
|
const groupBoxImage = rigProviderImageSourceImage(logicalSandboxSettings, groupBoxBackend);
|
|
262
|
-
const sandboxCreationBackend: Settings["sandboxBackend"] =
|
|
263
|
-
settings.sandboxOwnershipEnabled && runSettings.sandboxBackend !== "none"
|
|
264
|
-
? groupBoxBackend
|
|
265
|
-
: runSettings.sandboxBackend;
|
|
281
|
+
const sandboxCreationBackend: Settings["sandboxBackend"] = groupRouteSettings.sandboxBackend;
|
|
266
282
|
const effectiveRunCredentialBackend = activeSandboxBackend ?? groupBoxBackend;
|
|
267
283
|
|
|
268
284
|
return {
|
|
@@ -315,6 +331,7 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
|
|
|
315
331
|
rigVersion,
|
|
316
332
|
turnResources,
|
|
317
333
|
} = deps;
|
|
334
|
+
const groupBoxOs = managedSessionGroupOs(runSettings.sandboxBackend, session.sandboxOs);
|
|
318
335
|
const {
|
|
319
336
|
releaseLateSandbox,
|
|
320
337
|
onHomeSandboxRebound,
|
|
@@ -577,7 +594,7 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
|
|
|
577
594
|
sandboxGroupId: lazyGroupId,
|
|
578
595
|
sessionId: input.sessionId,
|
|
579
596
|
backend: groupBoxBackend,
|
|
580
|
-
os:
|
|
597
|
+
os: groupBoxOs,
|
|
581
598
|
environment: sandboxEnvironment,
|
|
582
599
|
...(groupBoxImage
|
|
583
600
|
? {
|
|
@@ -656,7 +673,7 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
|
|
|
656
673
|
// deployment default), never a "selfhosted" box (which would throw
|
|
657
674
|
// for lack of a bound agentId).
|
|
658
675
|
backend: groupBoxBackend,
|
|
659
|
-
os:
|
|
676
|
+
os: groupBoxOs,
|
|
660
677
|
environment: sandboxEnvironment,
|
|
661
678
|
// IMAGE IS SHARED STATE (B3): the container image
|
|
662
679
|
// this run resolves. The lease stamps it + conflicts on a live shared box
|
|
@@ -881,7 +898,7 @@ export async function bindLazySandboxProvisioner(
|
|
|
881
898
|
// recursion that HANGS the turn — caught live on staging 2026-07-08). The SDK
|
|
882
899
|
// already holds the proxy directly (injected as lazyOwnedSandbox.session), so it
|
|
883
900
|
// gets per-op routing; the worker-side handle (resolvedSandbox: release,
|
|
884
|
-
// heartbeat,
|
|
901
|
+
// heartbeat, workspace capture) wants the real box, unproxied.
|
|
885
902
|
return provisioned;
|
|
886
903
|
},
|
|
887
904
|
{
|
|
@@ -1007,6 +1024,24 @@ export async function bindLazySandboxProvisioner(
|
|
|
1007
1024
|
),
|
|
1008
1025
|
}
|
|
1009
1026
|
: {}),
|
|
1027
|
+
...(sandboxState.sandboxGroupId && sandboxState.sandboxHolderId
|
|
1028
|
+
? {
|
|
1029
|
+
coordinateSharedRigSetup: createSharedRigSetupCoordinator({
|
|
1030
|
+
db,
|
|
1031
|
+
accountId: input.accountId,
|
|
1032
|
+
workspaceId: input.workspaceId,
|
|
1033
|
+
sandboxGroupId: sandboxState.sandboxGroupId,
|
|
1034
|
+
attemptId: input.attemptId,
|
|
1035
|
+
holderId: sandboxState.sandboxHolderId,
|
|
1036
|
+
sandbox: provisioned,
|
|
1037
|
+
...(runtimeCancellationSignal
|
|
1038
|
+
? { signal: runtimeCancellationSignal }
|
|
1039
|
+
: {}),
|
|
1040
|
+
observe: (measurement) =>
|
|
1041
|
+
recordSandboxSharedPreparation(observability, measurement),
|
|
1042
|
+
}),
|
|
1043
|
+
}
|
|
1044
|
+
: {}),
|
|
1010
1045
|
},
|
|
1011
1046
|
),
|
|
1012
1047
|
(measurement) => {
|
|
@@ -64,6 +64,27 @@ export function managedSandboxOwnershipForTurn(
|
|
|
64
64
|
};
|
|
65
65
|
}
|
|
66
66
|
|
|
67
|
+
/**
|
|
68
|
+
* Bind backend-aware runtime settings to the route that will actually execute.
|
|
69
|
+
*
|
|
70
|
+
* A machine-home turn keeps `selfhosted` as its durable policy. After an
|
|
71
|
+
* explicit clear-to-default, however, both the ownership-inverted path and the
|
|
72
|
+
* legacy SDK-owned path must construct manifests and clients for the resolved
|
|
73
|
+
* managed group backend. Machine-primary and ordinary session routes preserve
|
|
74
|
+
* their existing settings object.
|
|
75
|
+
*/
|
|
76
|
+
export function sandboxSettingsForRoute(input: {
|
|
77
|
+
runSettings: Settings;
|
|
78
|
+
machinePrimary: boolean;
|
|
79
|
+
groupBoxBackend: Settings["sandboxBackend"];
|
|
80
|
+
}): Settings {
|
|
81
|
+
return input.runSettings.sandboxBackend === "selfhosted" &&
|
|
82
|
+
!input.machinePrimary &&
|
|
83
|
+
input.groupBoxBackend !== "selfhosted"
|
|
84
|
+
? { ...input.runSettings, sandboxBackend: input.groupBoxBackend }
|
|
85
|
+
: input.runSettings;
|
|
86
|
+
}
|
|
87
|
+
|
|
67
88
|
/** A sandboxless home still establishes an explicitly attached Connected Machine. */
|
|
68
89
|
export function shouldEstablishSandboxForTurn(
|
|
69
90
|
sandboxOwnershipEnabled: boolean,
|
|
@@ -331,28 +352,3 @@ export function sandboxArtifactRuntimeAdmission(
|
|
|
331
352
|
},
|
|
332
353
|
};
|
|
333
354
|
}
|
|
334
|
-
|
|
335
|
-
/**
|
|
336
|
-
* Decide whether the first actual computer action may start a proof recording.
|
|
337
|
-
*
|
|
338
|
-
* On-turn recording runs ffmpeg/x11grab INSIDE the box and reads the .mp4 back
|
|
339
|
-
* out of the box's /tmp — plumbing that exists only for OpenGeni-operated cloud
|
|
340
|
-
* boxes (the Modal desktop backend). A turn whose EFFECTIVE backend is a connected
|
|
341
|
-
* machine ("selfhosted") runs on the user's REAL computer, which has none of that
|
|
342
|
-
* capture plumbing (and the platform must never shell ffmpeg onto a user's machine
|
|
343
|
-
* — the same reason the runtime skips its setup hooks for selfhosted). Left ungated
|
|
344
|
-
* it films nothing, finds no /tmp file, and emits recording.started followed by
|
|
345
|
-
* recording.failed{box-death} on EVERY machine-primary turn — misleading timeline
|
|
346
|
-
* noise + wasted work. So gate it off, exactly like a recording-disabled deployment:
|
|
347
|
-
* skip silently, emit nothing (no new event shape).
|
|
348
|
-
*
|
|
349
|
-
* `effectiveBackend` is the resolved ACTIVE backend for the turn
|
|
350
|
-
* (resolveActiveSandboxBackend) — NOT the session's home backend. A modal-home
|
|
351
|
-
* session actively swapped onto a machine resolves to "selfhosted" here and
|
|
352
|
-
* correctly skips; a machine-home turn that degraded back to its cloud group box
|
|
353
|
-
* (swap-away / flag-off) resolves to undefined and records as before.
|
|
354
|
-
*
|
|
355
|
-
* EDGE — mid-turn swap: this is evaluated once when computer-use first runs. A swap
|
|
356
|
-
* after recording starts is deliberately ignored; the partial recording already has
|
|
357
|
-
* defined failure semantics, so there is no stop/restart machinery.
|
|
358
|
-
*/
|
|
@@ -4,13 +4,11 @@ import {
|
|
|
4
4
|
heartbeatLeaseHolder,
|
|
5
5
|
readLease,
|
|
6
6
|
accrueWarmSeconds,
|
|
7
|
-
abandonRecordingForTurnAttempt,
|
|
8
7
|
SandboxWorkspaceMutationFencedError,
|
|
9
8
|
} from "@opengeni/db";
|
|
10
9
|
import {
|
|
11
10
|
RoutingMutationOutcomeUnknownError,
|
|
12
11
|
type EstablishedSandboxSession,
|
|
13
|
-
stopRecording as stopRecordingOnBox,
|
|
14
12
|
} from "@opengeni/runtime";
|
|
15
13
|
import {
|
|
16
14
|
sandboxLifecycleTransitionWaitMs,
|
|
@@ -26,10 +24,7 @@ import {
|
|
|
26
24
|
type ResumedTurnSandbox,
|
|
27
25
|
} from "../../sandbox-resume";
|
|
28
26
|
import { recordCreditMicros } from "../../observability-metrics";
|
|
29
|
-
import { beginRecording, discardUnpublishedRecording, type ActiveRecording } from "../recording";
|
|
30
27
|
import { ChannelAPartialMutationError } from "@opengeni/runtime/sandbox";
|
|
31
|
-
import { sandboxRunAs } from "@opengeni/runtime";
|
|
32
|
-
import { randomUUID } from "node:crypto";
|
|
33
28
|
|
|
34
29
|
import { safeErrorDiagnostic } from "./errors";
|
|
35
30
|
import {
|
|
@@ -38,13 +33,7 @@ import {
|
|
|
38
33
|
finalizeDurableTurnOpStreams,
|
|
39
34
|
} from "./quiescence";
|
|
40
35
|
import { SandboxDeadlineRotationError } from "./sandbox-provision";
|
|
41
|
-
import {
|
|
42
|
-
import type {
|
|
43
|
-
AttemptIdentityState,
|
|
44
|
-
EventingState,
|
|
45
|
-
RecordingState,
|
|
46
|
-
SandboxRuntimeState,
|
|
47
|
-
} from "./turn-context";
|
|
36
|
+
import type { AttemptIdentityState, EventingState, SandboxRuntimeState } from "./turn-context";
|
|
48
37
|
|
|
49
38
|
export type SandboxTurnRuntimeDeps = {
|
|
50
39
|
input: RunAgentTurnInput;
|
|
@@ -56,7 +45,6 @@ export type SandboxTurnRuntimeDeps = {
|
|
|
56
45
|
activityContext: ReturnType<typeof currentActivityContext>;
|
|
57
46
|
sandboxRotationController: AbortController;
|
|
58
47
|
sandboxState: SandboxRuntimeState;
|
|
59
|
-
recordingState: RecordingState;
|
|
60
48
|
eventing: EventingState;
|
|
61
49
|
attempt: AttemptIdentityState;
|
|
62
50
|
};
|
|
@@ -74,7 +62,6 @@ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
|
|
|
74
62
|
activityContext,
|
|
75
63
|
sandboxRotationController,
|
|
76
64
|
sandboxState,
|
|
77
|
-
recordingState,
|
|
78
65
|
eventing,
|
|
79
66
|
attempt,
|
|
80
67
|
} = deps;
|
|
@@ -354,39 +341,6 @@ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
|
|
|
354
341
|
// Turn-end capture needs the lease heartbeat to keep its holder alive, but
|
|
355
342
|
// must prevent that same timer from starting another periodic snapshot
|
|
356
343
|
// while it reads. This gate separates those two responsibilities.
|
|
357
|
-
// Computer-use-only recording. Ordinary shell/filesystem turns leave this
|
|
358
|
-
// null; the first actual computer action starts it after :0 is ready.
|
|
359
|
-
// P4.3 recording gate: flips true in `onComputerUseReady`, the runtime's
|
|
360
|
-
// execution-time callback for the first real computer action. It must flip
|
|
361
|
-
// BEFORE awaiting recording startup: the SDK tool-call stream item can arrive
|
|
362
|
-
// before ffmpeg has finished starting. A plain text turn ("hey"/"continue")
|
|
363
|
-
// never invokes the callback, so settlement performs no storage PUT.
|
|
364
|
-
const abandonActiveRecording = async (
|
|
365
|
-
reason: string,
|
|
366
|
-
disposition: "failed" | "discard" = "failed",
|
|
367
|
-
): Promise<void> => {
|
|
368
|
-
const recording = recordingState.activeRecording as ActiveRecording | null;
|
|
369
|
-
if (!recording) return;
|
|
370
|
-
recordingState.activeRecording = null;
|
|
371
|
-
if (sandboxState.resolvedSandbox) {
|
|
372
|
-
await stopRecordingOnBox(
|
|
373
|
-
sandboxState.resolvedSandbox.established.session,
|
|
374
|
-
recording.proc,
|
|
375
|
-
).catch(() => undefined);
|
|
376
|
-
}
|
|
377
|
-
if (!attempt.turnId || attempt.executionGeneration <= 0) return;
|
|
378
|
-
await abandonRecordingForTurnAttempt(db, {
|
|
379
|
-
accountId: input.accountId,
|
|
380
|
-
workspaceId: input.workspaceId,
|
|
381
|
-
sessionId: input.sessionId,
|
|
382
|
-
turnId: attempt.turnId,
|
|
383
|
-
executionGeneration: attempt.executionGeneration,
|
|
384
|
-
attemptId: input.attemptId,
|
|
385
|
-
recordingId: recording.recordingId,
|
|
386
|
-
disposition,
|
|
387
|
-
reason,
|
|
388
|
-
}).catch(() => undefined);
|
|
389
|
-
};
|
|
390
344
|
const flushRuntimeBatcher = async () => {
|
|
391
345
|
const current = eventing.batcher as ReturnType<typeof createRuntimeBatcher> | null;
|
|
392
346
|
await current?.flush().catch(() => undefined);
|
|
@@ -647,73 +601,6 @@ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
|
|
|
647
601
|
sandboxState.leaseHeartbeatTimer.unref();
|
|
648
602
|
}
|
|
649
603
|
};
|
|
650
|
-
const maybeStartOnTurnRecording = async (
|
|
651
|
-
sandbox: ResumedTurnSandbox,
|
|
652
|
-
effectiveBackend: Settings["sandboxBackend"] | undefined,
|
|
653
|
-
): Promise<void> => {
|
|
654
|
-
if (recordingState.activeRecording) {
|
|
655
|
-
return;
|
|
656
|
-
}
|
|
657
|
-
if (recordingState.computerUseRecordingStart) {
|
|
658
|
-
await recordingState.computerUseRecordingStart;
|
|
659
|
-
return;
|
|
660
|
-
}
|
|
661
|
-
// Called only by the runtime's first-computer-action hook. Plain sandbox
|
|
662
|
-
// operations never start ffmpeg and never boot a display merely to record
|
|
663
|
-
// an unused desktop. Recording failure never fails the computer action.
|
|
664
|
-
if (
|
|
665
|
-
shouldStartOnTurnRecording({
|
|
666
|
-
recordingEnabled: settings.recordingEnabled,
|
|
667
|
-
desktopEnabled: settings.sandboxDesktopEnabled,
|
|
668
|
-
establishedBackendId: sandbox.established.backendId,
|
|
669
|
-
// EFFECTIVE (active) backend, not the session home: a machine-primary turn
|
|
670
|
-
// resolves to "selfhosted" and skips; a swap back to the cloud group box
|
|
671
|
-
// resolves to undefined and records as before.
|
|
672
|
-
effectiveBackend,
|
|
673
|
-
})
|
|
674
|
-
) {
|
|
675
|
-
recordingState.computerUseRecordingStart = (async () => {
|
|
676
|
-
let begun: Awaited<ReturnType<typeof beginRecording>> | null = null;
|
|
677
|
-
try {
|
|
678
|
-
begun = await beginRecording({
|
|
679
|
-
settings,
|
|
680
|
-
db,
|
|
681
|
-
accountId: input.accountId,
|
|
682
|
-
workspaceId: input.workspaceId,
|
|
683
|
-
sessionId: input.sessionId,
|
|
684
|
-
turnId: attempt.turnId!,
|
|
685
|
-
recordingId: randomUUID(),
|
|
686
|
-
mode: "on-turn",
|
|
687
|
-
session: sandbox.established.session,
|
|
688
|
-
runAs: sandboxRunAs(settings),
|
|
689
|
-
reason: null,
|
|
690
|
-
});
|
|
691
|
-
if (!eventing.publish) {
|
|
692
|
-
throw new Error("recording started before the turn event publisher was ready");
|
|
693
|
-
}
|
|
694
|
-
await eventing.publish([{ type: "recording.started", payload: begun.started }]);
|
|
695
|
-
recordingState.activeRecording = begun.active;
|
|
696
|
-
} catch (recordingError) {
|
|
697
|
-
recordingState.activeRecording = null;
|
|
698
|
-
if (begun) {
|
|
699
|
-
await discardUnpublishedRecording({
|
|
700
|
-
db,
|
|
701
|
-
accountId: input.accountId,
|
|
702
|
-
workspaceId: input.workspaceId,
|
|
703
|
-
active: begun.active,
|
|
704
|
-
session: sandbox.established.session,
|
|
705
|
-
});
|
|
706
|
-
}
|
|
707
|
-
console.error(
|
|
708
|
-
"computer-use recording start failed (action outcome unaffected)",
|
|
709
|
-
safeErrorDiagnostic(recordingError),
|
|
710
|
-
);
|
|
711
|
-
}
|
|
712
|
-
})();
|
|
713
|
-
await recordingState.computerUseRecordingStart;
|
|
714
|
-
}
|
|
715
|
-
};
|
|
716
|
-
|
|
717
604
|
return {
|
|
718
605
|
releaseLateSandbox,
|
|
719
606
|
requireResolvedSandboxForMutation,
|
|
@@ -721,11 +608,9 @@ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
|
|
|
721
608
|
onHomeSandboxRebound,
|
|
722
609
|
runWorkspaceMutationForSandbox,
|
|
723
610
|
stopLeaseHeartbeat,
|
|
724
|
-
abandonActiveRecording,
|
|
725
611
|
flushRuntimeBatcher,
|
|
726
612
|
publishSandboxLifecycleEvents,
|
|
727
613
|
publishSandboxLost,
|
|
728
614
|
startLeaseHeartbeat,
|
|
729
|
-
maybeStartOnTurnRecording,
|
|
730
615
|
};
|
|
731
616
|
}
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
claimSandboxSharedPreparation,
|
|
5
|
+
readSandboxSharedPreparation,
|
|
6
|
+
SandboxLeaseSupersededError,
|
|
7
|
+
settleSandboxSharedPreparation,
|
|
8
|
+
type Database,
|
|
9
|
+
} from "@opengeni/db";
|
|
10
|
+
|
|
11
|
+
import type { ResumedTurnSandbox } from "../../sandbox-resume";
|
|
12
|
+
|
|
13
|
+
type SharedRigSetupCoordinatorInput = {
|
|
14
|
+
specHash: string;
|
|
15
|
+
timeoutMs: number;
|
|
16
|
+
execute: () => Promise<void>;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
type SharedRigSetupCoordinatorOptions = {
|
|
20
|
+
db: Database;
|
|
21
|
+
accountId: string;
|
|
22
|
+
workspaceId: string;
|
|
23
|
+
sandboxGroupId: string;
|
|
24
|
+
attemptId: string;
|
|
25
|
+
holderId: string;
|
|
26
|
+
sandbox: ResumedTurnSandbox;
|
|
27
|
+
signal?: AbortSignal;
|
|
28
|
+
observe?: (measurement: {
|
|
29
|
+
path: "owner" | "joined" | "reused";
|
|
30
|
+
outcome: "completed" | "failed";
|
|
31
|
+
durationSeconds: number;
|
|
32
|
+
}) => void;
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
function observeSharedPreparation(
|
|
36
|
+
options: SharedRigSetupCoordinatorOptions,
|
|
37
|
+
measurement: Parameters<NonNullable<SharedRigSetupCoordinatorOptions["observe"]>>[0],
|
|
38
|
+
): void {
|
|
39
|
+
try {
|
|
40
|
+
options.observe?.(measurement);
|
|
41
|
+
} catch {
|
|
42
|
+
// Telemetry must never alter setup authority or settlement.
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function waitForPreparationTransition(ms: number, signal?: AbortSignal): Promise<void> {
|
|
47
|
+
return new Promise((resolve, reject) => {
|
|
48
|
+
let settled = false;
|
|
49
|
+
const timer = setTimeout(finish, ms);
|
|
50
|
+
function finish(): void {
|
|
51
|
+
if (settled) return;
|
|
52
|
+
settled = true;
|
|
53
|
+
signal?.removeEventListener("abort", cancel);
|
|
54
|
+
resolve();
|
|
55
|
+
}
|
|
56
|
+
function cancel(): void {
|
|
57
|
+
if (settled) return;
|
|
58
|
+
settled = true;
|
|
59
|
+
clearTimeout(timer);
|
|
60
|
+
signal?.removeEventListener("abort", cancel);
|
|
61
|
+
reject(signal?.reason ?? new Error("Sandbox shared preparation wait cancelled"));
|
|
62
|
+
}
|
|
63
|
+
signal?.addEventListener("abort", cancel, { once: true });
|
|
64
|
+
if (signal?.aborted) cancel();
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async function settleCompletedWithRetry(
|
|
69
|
+
options: SharedRigSetupCoordinatorOptions,
|
|
70
|
+
input: SharedRigSetupCoordinatorInput,
|
|
71
|
+
claimId: string,
|
|
72
|
+
): Promise<boolean> {
|
|
73
|
+
let delayMs = 50;
|
|
74
|
+
for (let attempt = 0; attempt < 4; attempt += 1) {
|
|
75
|
+
try {
|
|
76
|
+
if (
|
|
77
|
+
await settleSandboxSharedPreparation(options.db, {
|
|
78
|
+
accountId: options.accountId,
|
|
79
|
+
workspaceId: options.workspaceId,
|
|
80
|
+
sandboxGroupId: options.sandboxGroupId,
|
|
81
|
+
expectedLeaseEpoch: options.sandbox.leaseEpoch,
|
|
82
|
+
expectedInstanceId: options.sandbox.established.instanceId,
|
|
83
|
+
specHash: input.specHash,
|
|
84
|
+
holderId: options.holderId,
|
|
85
|
+
claimId,
|
|
86
|
+
outcome: "completed",
|
|
87
|
+
})
|
|
88
|
+
) {
|
|
89
|
+
return true;
|
|
90
|
+
}
|
|
91
|
+
} catch {
|
|
92
|
+
// The exact claim remains durable. Retry settlement without replaying the
|
|
93
|
+
// provider operation; a later owner can verify the box-local marker.
|
|
94
|
+
}
|
|
95
|
+
await waitForPreparationTransition(delayMs, options.signal);
|
|
96
|
+
delayMs *= 2;
|
|
97
|
+
}
|
|
98
|
+
return false;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Coordinate immutable rig setup across workers. Joiners use a revision-aware,
|
|
103
|
+
* exponentially backed-off durable read instead of the old fixed 250 ms lease
|
|
104
|
+
* polling pattern. PostgreSQL remains source of truth; a lost completion write
|
|
105
|
+
* is recovered by the next deadline owner through the hook's box-local marker.
|
|
106
|
+
*/
|
|
107
|
+
export function createSharedRigSetupCoordinator(
|
|
108
|
+
options: SharedRigSetupCoordinatorOptions,
|
|
109
|
+
): (input: SharedRigSetupCoordinatorInput) => Promise<"executed" | "reused"> {
|
|
110
|
+
return async (input) => {
|
|
111
|
+
const startedAt = performance.now();
|
|
112
|
+
const claimId = randomUUID();
|
|
113
|
+
const identity = {
|
|
114
|
+
accountId: options.accountId,
|
|
115
|
+
workspaceId: options.workspaceId,
|
|
116
|
+
sandboxGroupId: options.sandboxGroupId,
|
|
117
|
+
expectedLeaseEpoch: options.sandbox.leaseEpoch,
|
|
118
|
+
expectedInstanceId: options.sandbox.established.instanceId,
|
|
119
|
+
specHash: input.specHash,
|
|
120
|
+
holderId: options.holderId,
|
|
121
|
+
};
|
|
122
|
+
const claimTimeoutMs = Math.min(3_600_000, Math.max(30_000, input.timeoutMs + 30_000));
|
|
123
|
+
let delayMs = 100;
|
|
124
|
+
let observedRevision = -1;
|
|
125
|
+
let path: "owner" | "joined" | "reused" = "owner";
|
|
126
|
+
|
|
127
|
+
try {
|
|
128
|
+
for (;;) {
|
|
129
|
+
if (options.signal?.aborted) {
|
|
130
|
+
throw options.signal.reason ?? new Error("Sandbox shared preparation cancelled");
|
|
131
|
+
}
|
|
132
|
+
const claim = await claimSandboxSharedPreparation(options.db, {
|
|
133
|
+
...identity,
|
|
134
|
+
claimId,
|
|
135
|
+
ownerAttemptId: options.attemptId,
|
|
136
|
+
timeoutMs: claimTimeoutMs,
|
|
137
|
+
});
|
|
138
|
+
if (claim.role === "fenced") {
|
|
139
|
+
throw new SandboxLeaseSupersededError(options.sandboxGroupId, options.sandbox.leaseEpoch);
|
|
140
|
+
}
|
|
141
|
+
if (claim.role === "reused") {
|
|
142
|
+
if (path !== "joined") path = "reused";
|
|
143
|
+
observeSharedPreparation(options, {
|
|
144
|
+
path,
|
|
145
|
+
outcome: "completed",
|
|
146
|
+
durationSeconds: (performance.now() - startedAt) / 1_000,
|
|
147
|
+
});
|
|
148
|
+
return "reused";
|
|
149
|
+
}
|
|
150
|
+
if (claim.role === "owner") {
|
|
151
|
+
try {
|
|
152
|
+
await input.execute();
|
|
153
|
+
} catch (error) {
|
|
154
|
+
await settleSandboxSharedPreparation(options.db, {
|
|
155
|
+
...identity,
|
|
156
|
+
claimId,
|
|
157
|
+
outcome: "failed",
|
|
158
|
+
}).catch(() => undefined);
|
|
159
|
+
throw error;
|
|
160
|
+
}
|
|
161
|
+
if (await settleCompletedWithRetry(options, input, claimId)) {
|
|
162
|
+
observeSharedPreparation(options, {
|
|
163
|
+
path,
|
|
164
|
+
outcome: "completed",
|
|
165
|
+
durationSeconds: (performance.now() - startedAt) / 1_000,
|
|
166
|
+
});
|
|
167
|
+
return "executed";
|
|
168
|
+
}
|
|
169
|
+
throw new Error(
|
|
170
|
+
"Sandbox shared preparation completed but its durable settlement remains unavailable",
|
|
171
|
+
);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
path = "joined";
|
|
175
|
+
observedRevision = claim.preparation.revision;
|
|
176
|
+
const jitterMs = Math.floor(Math.random() * Math.max(1, Math.floor(delayMs / 4)));
|
|
177
|
+
await waitForPreparationTransition(delayMs + jitterMs, options.signal);
|
|
178
|
+
const observed = await readSandboxSharedPreparation(options.db, identity);
|
|
179
|
+
if (observed.status === "fenced") {
|
|
180
|
+
throw new SandboxLeaseSupersededError(options.sandboxGroupId, options.sandbox.leaseEpoch);
|
|
181
|
+
}
|
|
182
|
+
if (observed.preparation.status === "completed") {
|
|
183
|
+
observeSharedPreparation(options, {
|
|
184
|
+
path,
|
|
185
|
+
outcome: "completed",
|
|
186
|
+
durationSeconds: (performance.now() - startedAt) / 1_000,
|
|
187
|
+
});
|
|
188
|
+
return "reused";
|
|
189
|
+
}
|
|
190
|
+
if (observed.preparation.status === "failed") {
|
|
191
|
+
delayMs = 100;
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
delayMs =
|
|
195
|
+
observed.preparation.revision === observedRevision ? Math.min(2_000, delayMs * 2) : 100;
|
|
196
|
+
}
|
|
197
|
+
} catch (error) {
|
|
198
|
+
observeSharedPreparation(options, {
|
|
199
|
+
path,
|
|
200
|
+
outcome: "failed",
|
|
201
|
+
durationSeconds: (performance.now() - startedAt) / 1_000,
|
|
202
|
+
});
|
|
203
|
+
throw error;
|
|
204
|
+
}
|
|
205
|
+
};
|
|
206
|
+
}
|