@opengeni/worker-bundle 0.7.5 → 0.10.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dist/chunk-BJUW5LCM.js +325 -0
  2. package/dist/chunk-BJUW5LCM.js.map +1 -0
  3. package/dist/host-export.d.ts +70 -0
  4. package/dist/host-export.js +7 -0
  5. package/dist/host-export.js.map +1 -0
  6. package/dist/index.d.ts +98 -6
  7. package/dist/index.js +4097 -1271
  8. package/dist/index.js.map +1 -1
  9. package/dist/workflow-bundle.js +12360 -0
  10. package/package.json +18 -12
  11. package/src/activities/agent-turn.ts +2674 -828
  12. package/src/activities/capabilities.ts +11 -2
  13. package/src/activities/codex-capacity.ts +6 -11
  14. package/src/activities/codex-rotation.ts +75 -86
  15. package/src/activities/context-compaction.ts +104 -24
  16. package/src/activities/environment.ts +503 -97
  17. package/src/activities/expiring-material-renewal.ts +143 -0
  18. package/src/activities/goals.ts +4 -0
  19. package/src/activities/mcp-credentials.ts +38 -0
  20. package/src/activities/packs.ts +115 -2
  21. package/src/activities/rig-verification.ts +897 -244
  22. package/src/activities/run-credential-renewal.ts +70 -0
  23. package/src/activities/run-credentials.ts +132 -0
  24. package/src/activities/run-input.ts +23 -0
  25. package/src/activities/sandbox-lease.ts +33 -2
  26. package/src/activities/scheduled-tasks.ts +21 -0
  27. package/src/activities/session-state.ts +72 -0
  28. package/src/activities/streaming.ts +1 -0
  29. package/src/activities/toolspace-token-renewal.ts +57 -0
  30. package/src/activities/types.ts +44 -6
  31. package/src/activities/workspace-capture.ts +113 -93
  32. package/src/activities.ts +10 -2
  33. package/src/host-export-pump.ts +416 -0
  34. package/src/host-export.ts +22 -0
  35. package/src/http.ts +77 -28
  36. package/src/index.ts +360 -156
  37. package/src/observability-metrics.ts +83 -15
  38. package/src/sandbox-resume.ts +47 -1
  39. package/src/sandbox-routing.ts +1 -1
  40. package/src/worker-service-lifecycle.ts +104 -0
  41. package/src/workflows/activities.ts +13 -7
  42. package/src/workflows/rig-verification.ts +3 -1
  43. package/src/workflows/session.ts +219 -23
  44. package/src/workflows.ts +1 -0
@@ -5,25 +5,39 @@ import {
5
5
  promoteSetupAppendChange,
6
6
  } from "@opengeni/core";
7
7
  import {
8
+ acquireLease,
8
9
  beginRigChangeVerificationAttempt,
10
+ commitWarmingToWarm,
11
+ failWarmingToCold,
9
12
  getRig,
10
13
  getRigChange,
11
14
  getRigVersionById,
15
+ markWarmLeaseInstanceLost,
16
+ recordWarmingSandboxCreated,
17
+ releaseLeaseHolder,
12
18
  sanitizeEventPayload,
13
19
  sanitizeEventString,
14
20
  sanitizeMemoryText,
21
+ touchLeaseHolder,
15
22
  updateRigChangeStatus,
16
23
  type Database,
17
24
  } from "@opengeni/db";
18
25
  import {
26
+ createTurnToolCancellationController,
19
27
  establishSandboxSessionFromEnvelope,
20
28
  runRigSetupHook,
21
29
  sandboxCommandExitCode,
22
30
  sandboxCommandOutput,
31
+ serializeEstablishedSandboxEnvelope,
32
+ tagModalSandbox,
23
33
  type EstablishedSandboxSession,
34
+ type SandboxLifecycleCommandRunner,
35
+ type TurnSandboxCommandSession,
24
36
  } from "@opengeni/runtime";
37
+ import type { Context } from "@temporalio/activity";
25
38
  import type { ActivityServices } from "./types";
26
39
  import { settingsWithRigImage } from "./packs";
40
+ import { currentActivityContext } from "./streaming";
27
41
 
28
42
  export type RigVerificationWorkflowInput =
29
43
  | { workspaceId: string; changeId: string; versionId?: never }
@@ -34,12 +48,201 @@ type CommandResult = {
34
48
  output: string;
35
49
  };
36
50
 
37
- type CommandSession = {
38
- exec?: (args: Record<string, unknown>) => Promise<unknown>;
39
- execCommand?: (args: Record<string, unknown>) => Promise<unknown>;
51
+ const OUTPUT_TAIL_LIMIT = 64 * 1024;
52
+ const RIG_VERIFICATION_CLEANUP_RESERVE_MAX_MS = 2 * 60_000;
53
+ const RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS = 60_000;
54
+ const RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS = 10_000;
55
+ const RIG_VERIFICATION_HEARTBEAT_MIN_INTERVAL_MS = 250;
56
+ // The Temporal activity has a 15-minute start-to-close timeout. Keep ownership
57
+ // through that complete contract plus five minutes for cancellation delivery
58
+ // and finally cleanup; a dead process then becomes sweep-eligible without any
59
+ // lease-holder fiction or manual recovery.
60
+ export const RIG_VERIFICATION_OWNER_TTL_MS = 20 * 60_000;
61
+ export const RIG_VERIFICATION_OWNERS_DISABLED_MESSAGE =
62
+ "Rig verification lease ownership is disabled; refusing to create an unowned verifier sandbox";
63
+ const RIG_VERIFICATION_HOLDER_TOUCH_INTERVAL_MS = 10_000;
64
+ const RIG_VERIFICATION_RELEASE_GRACE_MS = 1;
65
+
66
+ export class RigVerificationActivityDeadlineError extends Error {
67
+ readonly name = "RigVerificationActivityDeadlineError";
68
+
69
+ constructor(
70
+ readonly startToCloseTimeoutMs: number,
71
+ readonly cleanupReserveMs: number,
72
+ ) {
73
+ super(
74
+ `Rig verification activity-local deadline reached with ${cleanupReserveMs}ms reserved for cleanup before the ${startToCloseTimeoutMs}ms Temporal start-to-close timeout`,
75
+ );
76
+ }
77
+ }
78
+
79
+ export type RigVerificationActivityLifecycle = {
80
+ signal: AbortSignal;
81
+ /** Absolute wall-clock boundary before which immediate cleanup may wait. */
82
+ cleanupDeadlineAtMs: number | null;
83
+ dispose(): void;
40
84
  };
41
85
 
42
- const OUTPUT_TAIL_LIMIT = 64 * 1024;
86
+ function computeCleanupReserveMs(startToCloseTimeoutMs: number): number {
87
+ return Math.min(
88
+ RIG_VERIFICATION_CLEANUP_RESERVE_MAX_MS,
89
+ Math.max(100, Math.floor(startToCloseTimeoutMs / 4)),
90
+ Math.max(1, startToCloseTimeoutMs - 1),
91
+ );
92
+ }
93
+
94
+ /**
95
+ * Start the verifier's real Temporal liveness/cancellation contract.
96
+ *
97
+ * The activity-local deadline is deliberately earlier than start-to-close. In
98
+ * production a 15-minute server deadline becomes a 13-minute work deadline,
99
+ * retaining two minutes for command quiescence, exact deactivation, and
100
+ * provider termination. Short integration-test deadlines use the same bounded
101
+ * one-quarter rule instead of a production-only magic value.
102
+ */
103
+ export function createRigVerificationActivityLifecycle(
104
+ context: Context | null = currentActivityContext(),
105
+ ): RigVerificationActivityLifecycle {
106
+ const controller = new AbortController();
107
+ const temporalSignal = context?.cancellationSignal;
108
+ const forwardTemporalCancellation = (): void => {
109
+ if (!controller.signal.aborted) {
110
+ controller.abort(temporalSignal?.reason ?? new Error("Rig verification cancelled"));
111
+ }
112
+ };
113
+ if (temporalSignal?.aborted) {
114
+ forwardTemporalCancellation();
115
+ } else {
116
+ temporalSignal?.addEventListener("abort", forwardTemporalCancellation, { once: true });
117
+ }
118
+
119
+ const startedAtMs = Date.now();
120
+ const startToCloseTimeoutMs = context?.info.startToCloseTimeoutMs;
121
+ const reserveMs =
122
+ typeof startToCloseTimeoutMs === "number" && startToCloseTimeoutMs > 0
123
+ ? computeCleanupReserveMs(startToCloseTimeoutMs)
124
+ : null;
125
+ const serverDeadlineAtMs =
126
+ reserveMs === null || startToCloseTimeoutMs === undefined
127
+ ? null
128
+ : startedAtMs + startToCloseTimeoutMs;
129
+ // Keep an additional margin after our bounded cleanup waits. If a DB or
130
+ // provider promise remains unresolved, the activity must still report its
131
+ // local deadline before Temporal imposes the hard timeout.
132
+ const serverReportingMarginMs =
133
+ reserveMs === null ? 0 : Math.min(30_000, Math.max(1, Math.floor(reserveMs / 4)));
134
+ const cleanupDeadlineAtMs =
135
+ serverDeadlineAtMs === null ? null : serverDeadlineAtMs - serverReportingMarginMs;
136
+
137
+ let deadlineTimer: ReturnType<typeof setTimeout> | null = null;
138
+ if (reserveMs !== null && startToCloseTimeoutMs !== undefined && !controller.signal.aborted) {
139
+ deadlineTimer = setTimeout(
140
+ () => {
141
+ if (!controller.signal.aborted) {
142
+ controller.abort(
143
+ new RigVerificationActivityDeadlineError(startToCloseTimeoutMs, reserveMs),
144
+ );
145
+ }
146
+ },
147
+ Math.max(1, startToCloseTimeoutMs - reserveMs),
148
+ );
149
+ }
150
+
151
+ const heartbeat = (): void => {
152
+ if (!context) return;
153
+ try {
154
+ context.heartbeat({
155
+ activity: "rig_verification",
156
+ at: new Date().toISOString(),
157
+ cleanup: controller.signal.aborted,
158
+ cleanupDeadlineAt: cleanupDeadlineAtMs ? new Date(cleanupDeadlineAtMs).toISOString() : null,
159
+ });
160
+ } catch (error) {
161
+ // Context.heartbeat reports delivered cancellation synchronously on some
162
+ // SDK paths. Feed that failure into the same cooperative abort fence, but
163
+ // keep the interval alive through bounded cleanup: the abort signal stops
164
+ // verifier work, not the activity's liveness reporting.
165
+ if (!controller.signal.aborted) controller.abort(error);
166
+ }
167
+ };
168
+ heartbeat();
169
+ const heartbeatTimeoutMs = context?.info.heartbeatTimeoutMs;
170
+ const heartbeatIntervalMs =
171
+ typeof heartbeatTimeoutMs === "number" && heartbeatTimeoutMs > 0
172
+ ? Math.min(
173
+ RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS,
174
+ Math.max(RIG_VERIFICATION_HEARTBEAT_MIN_INTERVAL_MS, Math.floor(heartbeatTimeoutMs / 3)),
175
+ )
176
+ : RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS;
177
+ const heartbeatTimer = context ? setInterval(heartbeat, heartbeatIntervalMs) : null;
178
+ if (heartbeatTimer && "unref" in heartbeatTimer && typeof heartbeatTimer.unref === "function") {
179
+ heartbeatTimer.unref();
180
+ }
181
+
182
+ return {
183
+ signal: controller.signal,
184
+ cleanupDeadlineAtMs,
185
+ dispose: () => {
186
+ temporalSignal?.removeEventListener("abort", forwardTemporalCancellation);
187
+ if (deadlineTimer) clearTimeout(deadlineTimer);
188
+ if (heartbeatTimer) clearInterval(heartbeatTimer);
189
+ deadlineTimer = null;
190
+ },
191
+ };
192
+ }
193
+
194
+ function abortReason(signal: AbortSignal): unknown {
195
+ return signal.reason ?? new Error("Rig verification cancelled");
196
+ }
197
+
198
+ function throwIfAborted(signal: AbortSignal): void {
199
+ if (signal.aborted) throw abortReason(signal);
200
+ }
201
+
202
+ async function waitForAbortable<T>(promise: Promise<T>, signal: AbortSignal): Promise<T> {
203
+ throwIfAborted(signal);
204
+ let removeAbortListener = (): void => undefined;
205
+ const cancellation = new Promise<never>((_resolve, reject) => {
206
+ const onAbort = (): void => reject(abortReason(signal));
207
+ removeAbortListener = () => signal.removeEventListener("abort", onAbort);
208
+ signal.addEventListener("abort", onAbort, { once: true });
209
+ });
210
+ try {
211
+ return await Promise.race([promise, cancellation]);
212
+ } finally {
213
+ removeAbortListener();
214
+ }
215
+ }
216
+
217
+ type BoundedSettlement<T> = { status: "completed"; value: T } | { status: "timed_out" };
218
+
219
+ async function settleWithin<T>(
220
+ promise: Promise<T>,
221
+ timeoutMs: number,
222
+ ): Promise<BoundedSettlement<T>> {
223
+ if (timeoutMs <= 0) return { status: "timed_out" };
224
+ let timer: ReturnType<typeof setTimeout> | null = null;
225
+ try {
226
+ return await Promise.race([
227
+ promise.then((value) => ({ status: "completed" as const, value })),
228
+ new Promise<{ status: "timed_out" }>((resolve) => {
229
+ timer = setTimeout(() => resolve({ status: "timed_out" }), timeoutMs);
230
+ }),
231
+ ]);
232
+ } finally {
233
+ if (timer) clearTimeout(timer);
234
+ }
235
+ }
236
+
237
+ function cleanupWaitMs(lifecycle: RigVerificationActivityLifecycle | undefined): number {
238
+ if (lifecycle?.cleanupDeadlineAtMs == null) {
239
+ return RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS;
240
+ }
241
+ return Math.max(
242
+ 0,
243
+ Math.min(RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS, lifecycle.cleanupDeadlineAtMs - Date.now()),
244
+ );
245
+ }
43
246
 
44
247
  function tail(value: string, limit = OUTPUT_TAIL_LIMIT): string {
45
248
  return value.length > limit ? value.slice(-limit) : value;
@@ -62,14 +265,18 @@ function systemGrant(rig: Rig): AccessGrant {
62
265
  };
63
266
  }
64
267
 
65
- async function terminateThrowaway(established: EstablishedSandboxSession | null): Promise<void> {
268
+ async function terminateThrowaway(established: EstablishedSandboxSession | null): Promise<boolean> {
66
269
  if (!established) {
67
- return;
270
+ return true;
68
271
  }
69
272
  const client = established.client as { delete?: (state: unknown) => Promise<unknown> };
70
273
  if (typeof client.delete === "function" && established.sessionState !== undefined) {
71
- await client.delete(established.sessionState).catch(() => undefined);
72
- return;
274
+ try {
275
+ await client.delete(established.sessionState);
276
+ return true;
277
+ } catch {
278
+ return false;
279
+ }
73
280
  }
74
281
  const session = established.session as {
75
282
  terminate?: () => Promise<unknown>;
@@ -77,19 +284,432 @@ async function terminateThrowaway(established: EstablishedSandboxSession | null)
77
284
  close?: () => Promise<unknown>;
78
285
  closed?: boolean;
79
286
  };
80
- if (session.terminate) {
81
- await session.terminate().catch(() => undefined);
82
- } else if (session.kill) {
83
- await session.kill().catch(() => undefined);
84
- } else if (session.close && !session.closed) {
85
- await session.close().catch(() => undefined);
287
+ try {
288
+ if (session.terminate) {
289
+ await session.terminate();
290
+ } else if (session.kill) {
291
+ await session.kill();
292
+ } else if (session.close && !session.closed) {
293
+ await session.close();
294
+ } else {
295
+ return false;
296
+ }
297
+ return true;
298
+ } catch {
299
+ return false;
300
+ }
301
+ }
302
+
303
+ export type RigVerificationOwnershipDependencies = {
304
+ acquire: typeof acquireLease;
305
+ establish: typeof establishSandboxSessionFromEnvelope;
306
+ recordCreated: typeof recordWarmingSandboxCreated;
307
+ commitWarm: typeof commitWarmingToWarm;
308
+ touchHolder: typeof touchLeaseHolder;
309
+ releaseHolder: typeof releaseLeaseHolder;
310
+ failWarming: typeof failWarmingToCold;
311
+ markWarmLost: typeof markWarmLeaseInstanceLost;
312
+ serialize: typeof serializeEstablishedSandboxEnvelope;
313
+ tag: typeof tagModalSandbox;
314
+ terminate: typeof terminateThrowaway;
315
+ createCancellationController: typeof createTurnToolCancellationController;
316
+ randomUUID: () => string;
317
+ };
318
+
319
+ const defaultOwnershipDependencies: RigVerificationOwnershipDependencies = {
320
+ acquire: acquireLease,
321
+ establish: establishSandboxSessionFromEnvelope,
322
+ recordCreated: recordWarmingSandboxCreated,
323
+ commitWarm: commitWarmingToWarm,
324
+ touchHolder: touchLeaseHolder,
325
+ releaseHolder: releaseLeaseHolder,
326
+ failWarming: failWarmingToCold,
327
+ markWarmLost: markWarmLeaseInstanceLost,
328
+ serialize: serializeEstablishedSandboxEnvelope,
329
+ tag: tagModalSandbox,
330
+ terminate: terminateThrowaway,
331
+ createCancellationController: createTurnToolCancellationController,
332
+ randomUUID: () => crypto.randomUUID(),
333
+ };
334
+
335
+ export type RigVerificationSandboxRunContext = {
336
+ signal: AbortSignal;
337
+ commandRunner: SandboxLifecycleCommandRunner;
338
+ };
339
+
340
+ export class RigVerificationLeaseUnavailableError extends Error {
341
+ readonly name = "RigVerificationLeaseUnavailableError";
342
+
343
+ constructor(
344
+ readonly sandboxGroupId: string,
345
+ readonly role: "attached" | "rearmed" | "fenced",
346
+ ) {
347
+ super(`Rig verification requires a new clean sandbox, but lease ${sandboxGroupId} was ${role}`);
348
+ }
349
+ }
350
+
351
+ /**
352
+ * Run one verifier against one exactly attributed standalone sandbox.
353
+ *
354
+ * Ordering is deliberate and source-of-truth safe:
355
+ * acquire spawner -> create callback -> warming instance record -> warm commit
356
+ * -> setup/checks. The canonical sandbox lease, not provider tags, is durable
357
+ * ownership. Every exact create callback is retained before its DB write so an
358
+ * ambiguous acknowledgement can still be terminated in finally. Provider
359
+ * termination precedes clearing that exact durable pointer; when termination
360
+ * fails the pointer is retained and the holder is released with a one-millisecond
361
+ * drain grace so the normal reaper retries. Temporal cancellation first drains
362
+ * every verifier command. Cleanup waits are bounded inside the server reserve;
363
+ * lease expiry and the provider reaper remain the hard-worker-loss backstops.
364
+ */
365
+ export async function runWithOwnedRigVerificationSandbox<T>(
366
+ input: {
367
+ settings: ActivityServices["settings"];
368
+ db: Database;
369
+ observability: ActivityServices["observability"];
370
+ accountId: string;
371
+ workspaceId: string;
372
+ sandboxGroupId: string;
373
+ rigVersionId: string;
374
+ sessionIdPrefix: string;
375
+ lifecycle?: RigVerificationActivityLifecycle;
376
+ },
377
+ run: (
378
+ established: EstablishedSandboxSession,
379
+ context: RigVerificationSandboxRunContext,
380
+ ) => Promise<T>,
381
+ dependencies: RigVerificationOwnershipDependencies = defaultOwnershipDependencies,
382
+ ): Promise<T> {
383
+ // Phase B must remain inert throughout its code rollout. A disabled or
384
+ // omitted flag rejects before establish() can invoke any provider create;
385
+ // falling back to the legacy unowned verifier is never safe.
386
+ if (!input.settings.rigVerificationLeaseOwnershipEnabled) {
387
+ throw new Error(RIG_VERIFICATION_OWNERS_DISABLED_MESSAGE);
388
+ }
389
+
390
+ const holderId = `rig-verification:${dependencies.randomUUID()}`;
391
+ const fallbackController = new AbortController();
392
+ const signal = input.lifecycle?.signal ?? fallbackController.signal;
393
+ const commandController = dependencies.createCancellationController(signal);
394
+ const commandRunner = commandController.runSandboxCommand.bind(commandController);
395
+ let cleanupTarget: EstablishedSandboxSession | null = null;
396
+ let acquired = false;
397
+ let ownsWarmingEpoch = false;
398
+ let expectedEpoch: number | null = null;
399
+ let holderLivenessTimer: ReturnType<typeof setInterval> | null = null;
400
+ let establishmentSettled = false;
401
+ let establishmentPromise: Promise<EstablishedSandboxSession> | null = null;
402
+
403
+ const warnCleanupTimeout = (operation: string, instanceId: string | null): void => {
404
+ input.observability.warn("rig verifier: ownership cleanup timed out", {
405
+ holderId,
406
+ instanceId,
407
+ operation,
408
+ error:
409
+ "cleanup did not settle inside the activity reserve; exact owner expiry and the global provider reaper are the bounded delayed backstop",
410
+ });
411
+ };
412
+
413
+ const warnCleanupFailure = (
414
+ operation: string,
415
+ instanceId: string | null,
416
+ error: unknown,
417
+ ): void => {
418
+ input.observability.warn("rig verifier: ownership cleanup failed", {
419
+ holderId,
420
+ instanceId,
421
+ operation,
422
+ error: error instanceof Error ? error.message : String(error),
423
+ });
424
+ };
425
+
426
+ const clearTerminatedLeasePointer = async (instanceId: string): Promise<void> => {
427
+ if (expectedEpoch === null) return;
428
+ const cleanupEpoch = expectedEpoch;
429
+ // Exactly one transition can match. Running both covers a commit whose DB
430
+ // acknowledgement was lost: expectedEpoch is still warming, while
431
+ // expectedEpoch+1 is the only possible committed warm epoch.
432
+ const results = await Promise.allSettled([
433
+ Promise.resolve().then(() =>
434
+ dependencies.markWarmLost(input.db, {
435
+ accountId: input.accountId,
436
+ workspaceId: input.workspaceId,
437
+ sandboxGroupId: input.sandboxGroupId,
438
+ expectedEpoch: cleanupEpoch + 1,
439
+ expectedInstanceId: instanceId,
440
+ }),
441
+ ),
442
+ Promise.resolve().then(() =>
443
+ dependencies.failWarming(input.db, {
444
+ accountId: input.accountId,
445
+ workspaceId: input.workspaceId,
446
+ sandboxGroupId: input.sandboxGroupId,
447
+ expectedEpoch: cleanupEpoch,
448
+ }),
449
+ ),
450
+ ]);
451
+ for (const [index, result] of results.entries()) {
452
+ if (result.status === "rejected") {
453
+ warnCleanupFailure(
454
+ index === 0 ? "mark_warm_lost" : "fail_warming",
455
+ instanceId,
456
+ result.reason,
457
+ );
458
+ }
459
+ }
460
+ };
461
+
462
+ const terminateAndClear = async (created: EstablishedSandboxSession): Promise<boolean> => {
463
+ let terminated = false;
464
+ try {
465
+ terminated = await dependencies.terminate(created);
466
+ } catch (error) {
467
+ warnCleanupFailure("terminate", created.instanceId, error);
468
+ }
469
+ if (!terminated) {
470
+ warnCleanupFailure(
471
+ "terminate",
472
+ created.instanceId,
473
+ new Error("provider termination was not confirmed; durable lease cleanup remains pending"),
474
+ );
475
+ return false;
476
+ }
477
+ await clearTerminatedLeasePointer(created.instanceId);
478
+ return true;
479
+ };
480
+
481
+ try {
482
+ throwIfAborted(signal);
483
+ const lease = await dependencies.acquire(input.db, {
484
+ accountId: input.accountId,
485
+ workspaceId: input.workspaceId,
486
+ sandboxGroupId: input.sandboxGroupId,
487
+ kind: "turn",
488
+ holderId,
489
+ subjectId: null,
490
+ backend: input.settings.sandboxBackend,
491
+ image: input.settings.modalImageRef ?? input.settings.dockerImage,
492
+ rigVersionId: input.rigVersionId,
493
+ leaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
494
+ warmingLeaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
495
+ });
496
+ acquired = true;
497
+ expectedEpoch = lease.lease.leaseEpoch;
498
+ holderLivenessTimer = setInterval(() => {
499
+ void dependencies
500
+ .touchHolder(input.db, {
501
+ accountId: input.accountId,
502
+ workspaceId: input.workspaceId,
503
+ sandboxGroupId: input.sandboxGroupId,
504
+ kind: "turn",
505
+ holderId,
506
+ })
507
+ .catch(() => undefined);
508
+ }, RIG_VERIFICATION_HOLDER_TOUCH_INTERVAL_MS);
509
+ if ("unref" in holderLivenessTimer && typeof holderLivenessTimer.unref === "function") {
510
+ holderLivenessTimer.unref();
511
+ }
512
+ if (lease.role !== "spawner") {
513
+ throw new RigVerificationLeaseUnavailableError(input.sandboxGroupId, lease.role);
514
+ }
515
+ ownsWarmingEpoch = true;
516
+
517
+ establishmentPromise = Promise.resolve().then(() =>
518
+ dependencies.establish(input.settings, null, {
519
+ sessionId: `${input.sessionIdPrefix}-${holderId}`,
520
+ recovery: "create-or-restore",
521
+ environment: {},
522
+ onSandboxCreated: async (created) => {
523
+ // Retain the exact handle before the durable write. If registration
524
+ // throws after commit, finally still has the provider identity.
525
+ cleanupTarget = created;
526
+ const resumeState =
527
+ created.backendId === "modal" ? null : await dependencies.serialize(created);
528
+ const recorded = await dependencies.recordCreated(input.db, {
529
+ accountId: input.accountId,
530
+ workspaceId: input.workspaceId,
531
+ sandboxGroupId: input.sandboxGroupId,
532
+ expectedEpoch: lease.lease.leaseEpoch,
533
+ instanceId: created.instanceId,
534
+ resumeBackendId: created.backendId,
535
+ resumeState,
536
+ leaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
537
+ warmingLeaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
538
+ });
539
+ if (!recorded.recorded) {
540
+ throw new RigVerificationLeaseUnavailableError(input.sandboxGroupId, "fenced");
541
+ }
542
+
543
+ if (created.backendId === "modal" && !signal.aborted) {
544
+ try {
545
+ await waitForAbortable(
546
+ dependencies.tag(input.settings, created.instanceId, {
547
+ leaseId: lease.lease.id,
548
+ workspaceId: input.workspaceId,
549
+ sandboxGroupId: input.sandboxGroupId,
550
+ }),
551
+ signal,
552
+ );
553
+ } catch (error) {
554
+ if (!signal.aborted) {
555
+ input.observability.warn("rig verifier: Modal ownership tag failed", {
556
+ holderId,
557
+ instanceId: created.instanceId,
558
+ error: error instanceof Error ? error.message : String(error),
559
+ });
560
+ }
561
+ }
562
+ }
563
+
564
+ // Provider create is not abortable. If it returns after cancellation
565
+ // won the outer race, the callback owns an independent exact cleanup
566
+ // path before it throws and triggers the runtime's fail-closed delete.
567
+ if (signal.aborted) {
568
+ await terminateAndClear(created);
569
+ throw abortReason(signal);
570
+ }
571
+ },
572
+ }),
573
+ );
574
+ void establishmentPromise.then(
575
+ () => {
576
+ establishmentSettled = true;
577
+ },
578
+ () => {
579
+ establishmentSettled = true;
580
+ },
581
+ );
582
+ // The losing provider promise retains the callback above and a rejection
583
+ // handler, so a late create can still deactivate/terminate without an
584
+ // unhandled rejection after the activity has reported cancellation.
585
+ void establishmentPromise.catch(() => undefined);
586
+ const established = await waitForAbortable(establishmentPromise, signal);
587
+ cleanupTarget = established;
588
+ const resumeState =
589
+ established.backendId === "modal" ? null : await dependencies.serialize(established);
590
+ const committed = await dependencies.commitWarm(input.db, {
591
+ accountId: input.accountId,
592
+ workspaceId: input.workspaceId,
593
+ sandboxGroupId: input.sandboxGroupId,
594
+ expectedEpoch: lease.lease.leaseEpoch,
595
+ instanceId: established.instanceId,
596
+ dataPlaneUrl: null,
597
+ resumeBackendId: established.backendId,
598
+ resumeState,
599
+ leaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
600
+ });
601
+ if (!committed.committed || !committed.lease) {
602
+ throw new RigVerificationLeaseUnavailableError(input.sandboxGroupId, "fenced");
603
+ }
604
+ const result = await run(established, { signal, commandRunner });
605
+ throwIfAborted(signal);
606
+ return result;
607
+ } catch (error) {
608
+ if (signal.aborted) throw abortReason(signal);
609
+ throw error;
610
+ } finally {
611
+ if (signal.aborted && establishmentPromise && !establishmentSettled) {
612
+ const lateEstablishment = await settleWithin(
613
+ Promise.allSettled([establishmentPromise]),
614
+ cleanupWaitMs(input.lifecycle),
615
+ );
616
+ if (lateEstablishment.status === "timed_out") {
617
+ warnCleanupTimeout("establish_after_abort", cleanupTarget?.instanceId ?? null);
618
+ }
619
+ }
620
+
621
+ if (holderLivenessTimer) {
622
+ clearInterval(holderLivenessTimer);
623
+ holderLivenessTimer = null;
624
+ }
625
+
626
+ // Physical command quiescence precedes provider teardown. If the provider
627
+ // control plane itself hangs, bound the wait so termination still starts
628
+ // inside the server reserve; terminating the throwaway box is then the
629
+ // stronger final process fence.
630
+ commandController.cancel(signal.aborted ? abortReason(signal) : new Error("Verifier complete"));
631
+ const quiescence = commandController.waitForQuiescence();
632
+ void quiescence.catch(() => undefined);
633
+ const quiescenceResult = await settleWithin(quiescence, cleanupWaitMs(input.lifecycle));
634
+ if (quiescenceResult.status === "timed_out") {
635
+ warnCleanupTimeout("command_quiescence", cleanupTarget?.instanceId ?? null);
636
+ }
637
+
638
+ const providerCleanup = Promise.resolve().then(async () => {
639
+ if (!cleanupTarget) {
640
+ // A provider create that rejects before onSandboxCreated leaves no box
641
+ // pointer to terminate. Roll back only our exact warming epoch so the
642
+ // next verification does not wait for the 20-minute death reaper. If a
643
+ // non-abortable create returns after this transition, its callback is
644
+ // fenced and the runtime's create-failure path terminates that box.
645
+ if (ownsWarmingEpoch && expectedEpoch !== null) {
646
+ await dependencies.failWarming(input.db, {
647
+ accountId: input.accountId,
648
+ workspaceId: input.workspaceId,
649
+ sandboxGroupId: input.sandboxGroupId,
650
+ expectedEpoch,
651
+ });
652
+ }
653
+ return true;
654
+ }
655
+ return await terminateAndClear(cleanupTarget);
656
+ });
657
+ void providerCleanup.catch(() => undefined);
658
+ const providerSettlement = await settleWithin(
659
+ Promise.allSettled([providerCleanup]),
660
+ cleanupWaitMs(input.lifecycle),
661
+ );
662
+ if (providerSettlement.status === "timed_out") {
663
+ warnCleanupTimeout("terminate_and_clear", cleanupTarget?.instanceId ?? null);
664
+ } else if (providerSettlement.value[0]?.status === "rejected") {
665
+ warnCleanupFailure(
666
+ "terminate_and_clear",
667
+ cleanupTarget?.instanceId ?? null,
668
+ providerSettlement.value[0].reason,
669
+ );
670
+ }
671
+
672
+ // Release is independent from provider/DB cleanup. A failed direct
673
+ // termination therefore leaves the exact pointer intact and immediately
674
+ // makes the zero-holder lease eligible for the normal drain/reaper path.
675
+ if (acquired) {
676
+ const release = Promise.resolve().then(() =>
677
+ dependencies.releaseHolder(input.db, {
678
+ accountId: input.accountId,
679
+ workspaceId: input.workspaceId,
680
+ sandboxGroupId: input.sandboxGroupId,
681
+ kind: "turn",
682
+ holderId,
683
+ idleGraceMs: RIG_VERIFICATION_RELEASE_GRACE_MS,
684
+ }),
685
+ );
686
+ void release.catch(() => undefined);
687
+ const released = await settleWithin(
688
+ Promise.allSettled([release]),
689
+ cleanupWaitMs(input.lifecycle),
690
+ );
691
+ if (released.status === "timed_out") {
692
+ warnCleanupTimeout("release_holder", cleanupTarget?.instanceId ?? null);
693
+ } else if (released.value[0]?.status === "rejected") {
694
+ warnCleanupFailure(
695
+ "release_holder",
696
+ cleanupTarget?.instanceId ?? null,
697
+ released.value[0].reason,
698
+ );
699
+ }
700
+ }
701
+ // Cancellation may be delivered by a heartbeat while finally is already
702
+ // draining. WAIT_CANCELLATION_COMPLETED must observe cancellation only
703
+ // after cleanup, but it must not turn that late cancellation into success.
704
+ throwIfAborted(signal);
86
705
  }
87
706
  }
88
707
 
89
708
  async function runCommand(
90
- session: CommandSession,
709
+ session: TurnSandboxCommandSession,
91
710
  command: string,
92
711
  timeoutMs: number,
712
+ commandRunner: SandboxLifecycleCommandRunner,
93
713
  ): Promise<CommandResult> {
94
714
  const args = {
95
715
  cmd: command,
@@ -98,13 +718,7 @@ async function runCommand(
98
718
  yieldTimeMs: timeoutMs,
99
719
  maxOutputTokens: 40_000,
100
720
  };
101
- const result = session.exec
102
- ? await session.exec(args)
103
- : session.execCommand
104
- ? await session.execCommand(args)
105
- : (() => {
106
- throw new Error("Sandbox session does not support command execution");
107
- })();
721
+ const result = await commandRunner(session, args);
108
722
  return {
109
723
  exitCode: sandboxCommandExitCode(result),
110
724
  output: scrubVerificationOutput(tail(sandboxCommandOutput(result))),
@@ -190,252 +804,291 @@ async function loadVersionTarget(
190
804
  return { rig, version };
191
805
  }
192
806
 
807
+ async function withRigVerificationActivityLifecycle<T>(
808
+ run: (lifecycle: RigVerificationActivityLifecycle) => Promise<T>,
809
+ ): Promise<T> {
810
+ // Context.current() and the immediate heartbeat happen before lazy service
811
+ // construction, so Temporal can deliver cancellation even while the worker
812
+ // is opening its first database/provider dependencies.
813
+ const lifecycle = createRigVerificationActivityLifecycle();
814
+ try {
815
+ throwIfAborted(lifecycle.signal);
816
+ return await run(lifecycle);
817
+ } finally {
818
+ lifecycle.dispose();
819
+ }
820
+ }
821
+
193
822
  export function createRigVerificationActivities(services: () => Promise<ActivityServices>) {
194
823
  return {
195
- verifyRigChange: async (input: { workspaceId: string; changeId: string }) => {
196
- const { settings, db } = await services();
197
- const { rig, baseVersion, change } = await loadChangeTarget(
198
- db,
199
- input.workspaceId,
200
- input.changeId,
201
- );
202
- const grant = systemGrant(rig);
203
- const startedAt = new Date().toISOString();
204
- await beginRigChangeVerificationAttempt(db, input.workspaceId, change.id, {
205
- startedAt,
206
- allowAlreadyVerifying: true,
207
- });
208
- await recordRigAuditEvent(db, {
209
- grant,
210
- action: "rig.verification.started",
211
- rigId: rig.id,
212
- metadata: { changeId: change.id },
213
- });
214
-
215
- let established: EstablishedSandboxSession | null = null;
216
- const verification: Record<string, unknown> = { startedAt, checkResults: [] };
217
- try {
218
- const candidateVersion = candidateVersionForChange(baseVersion, change);
219
- const runSettings = settingsWithRigImage(settings, candidateVersion.image);
220
- established = await establishSandboxSessionFromEnvelope(runSettings, null, {
221
- sessionId: `rig-verification-${change.id}`,
222
- recovery: "create-or-restore",
223
- environment: {},
824
+ verifyRigChange: (input: { workspaceId: string; changeId: string }) =>
825
+ withRigVerificationActivityLifecycle(async (lifecycle) => {
826
+ const { settings, db, observability } = await services();
827
+ throwIfAborted(lifecycle.signal);
828
+ const { rig, baseVersion, change } = await loadChangeTarget(
829
+ db,
830
+ input.workspaceId,
831
+ input.changeId,
832
+ );
833
+ const grant = systemGrant(rig);
834
+ const startedAt = new Date().toISOString();
835
+ await beginRigChangeVerificationAttempt(db, input.workspaceId, change.id, {
836
+ startedAt,
837
+ allowAlreadyVerifying: true,
838
+ });
839
+ await recordRigAuditEvent(db, {
840
+ grant,
841
+ action: "rig.verification.started",
842
+ rigId: rig.id,
843
+ metadata: { changeId: change.id },
224
844
  });
225
- if ((candidateVersion.setupScript ?? "").trim()) {
226
- await runRigSetupHook(established.session as never, {
227
- environment: {},
228
- runAs: "root",
229
- rigSetup: {
230
- rigId: rig.id,
231
- rigName: rig.name,
232
- versionId: candidateVersion.id,
233
- script: candidateVersion.setupScript ?? "",
234
- timeoutMs: settings.rigSetupTimeoutMs,
845
+
846
+ const verification: Record<string, unknown> = { startedAt, checkResults: [] };
847
+ try {
848
+ const candidateVersion = candidateVersionForChange(baseVersion, change);
849
+ const runSettings = settingsWithRigImage(settings, candidateVersion.image);
850
+ return await runWithOwnedRigVerificationSandbox(
851
+ {
852
+ settings: runSettings,
853
+ db,
854
+ observability,
855
+ accountId: rig.accountId,
856
+ workspaceId: input.workspaceId,
857
+ sandboxGroupId: change.id,
858
+ rigVersionId: candidateVersion.id,
859
+ sessionIdPrefix: `rig-verification-${change.id}`,
860
+ lifecycle,
861
+ },
862
+ async (established, runContext) => {
863
+ if ((candidateVersion.setupScript ?? "").trim()) {
864
+ await runRigSetupHook(established.session as never, {
865
+ environment: {},
866
+ runAs: "root",
867
+ commandRunner: runContext.commandRunner,
868
+ rigSetup: {
869
+ rigId: rig.id,
870
+ rigName: rig.name,
871
+ versionId: candidateVersion.id,
872
+ script: candidateVersion.setupScript ?? "",
873
+ timeoutMs: settings.rigSetupTimeoutMs,
874
+ },
875
+ });
876
+ verification.setupResult = { exitCode: 0, output: "" };
877
+ }
878
+ const command = setupAppendCommand(change);
879
+ if (command) {
880
+ const commandResult = await runCommand(
881
+ established.session as TurnSandboxCommandSession,
882
+ command,
883
+ settings.rigSetupTimeoutMs,
884
+ runContext.commandRunner,
885
+ );
886
+ verification.commandResult = commandResult;
887
+ if (commandResult.exitCode !== 0) {
888
+ verification.finishedAt = new Date().toISOString();
889
+ verification.passed = false;
890
+ const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
891
+ status: "rejected",
892
+ verification: scrubVerificationPayload(verification),
893
+ });
894
+ await recordRigAuditEvent(db, {
895
+ grant,
896
+ action: "rig.verification.failed",
897
+ rigId: rig.id,
898
+ metadata: { changeId: change.id, status: "rejected" },
899
+ });
900
+ await recordRigAuditEvent(db, {
901
+ grant,
902
+ action: "rig.change.rejected",
903
+ rigId: rig.id,
904
+ metadata: { changeId: change.id },
905
+ });
906
+ return updated;
907
+ }
908
+ }
909
+ const checkResults = [];
910
+ for (const check of candidateVersion.checks) {
911
+ const result = await runCommand(
912
+ established.session as TurnSandboxCommandSession,
913
+ check.command,
914
+ settings.rigSetupTimeoutMs,
915
+ runContext.commandRunner,
916
+ );
917
+ checkResults.push({
918
+ name: check.name,
919
+ command: scrubVerificationOutput(check.command),
920
+ ...result,
921
+ });
922
+ }
923
+ verification.checkResults = checkResults;
924
+ const passed = checkResults.every((result) => result.exitCode === 0);
925
+ verification.finishedAt = new Date().toISOString();
926
+ verification.passed = passed;
927
+ const classified = classifyRigVerificationOutcome({ kind: change.kind, passed });
928
+ if (classified.action === "auto_promote") {
929
+ // Keep the change `verifying` (NOT `proposed`) across the write→promote
930
+ // gap: promoteSetupAppendChange accepts `verifying`, and leaving it
931
+ // `verifying` keeps beginRigChangeVerificationAttempt blocking a
932
+ // concurrent /verify — resetting to `proposed` would reopen that race
933
+ // (a second run could reject a change whose first verification passed).
934
+ await updateRigChangeStatus(db, input.workspaceId, change.id, {
935
+ status: "verifying",
936
+ verification: scrubVerificationPayload(verification),
937
+ });
938
+ const { change: merged } = await promoteSetupAppendChange({ db }, grant, rig, {
939
+ ...change,
940
+ verification,
941
+ });
942
+ await recordRigAuditEvent(db, {
943
+ grant,
944
+ action: "rig.verification.passed",
945
+ rigId: rig.id,
946
+ metadata: { changeId: change.id },
947
+ });
948
+ return merged;
949
+ }
950
+ const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
951
+ status: classified.status,
952
+ verification: scrubVerificationPayload(verification),
953
+ });
954
+ await recordRigAuditEvent(db, {
955
+ grant,
956
+ action: passed ? "rig.verification.passed" : "rig.verification.failed",
957
+ rigId: rig.id,
958
+ metadata: { changeId: change.id, status: classified.status },
959
+ });
960
+ if (!passed) {
961
+ await recordRigAuditEvent(db, {
962
+ grant,
963
+ action: "rig.change.rejected",
964
+ rigId: rig.id,
965
+ metadata: { changeId: change.id },
966
+ });
967
+ }
968
+ return updated;
235
969
  },
236
- });
237
- verification.setupResult = { exitCode: 0, output: "" };
238
- }
239
- const command = setupAppendCommand(change);
240
- if (command) {
241
- const commandResult = await runCommand(
242
- established.session as CommandSession,
243
- command,
244
- settings.rigSetupTimeoutMs,
245
970
  );
246
- verification.commandResult = commandResult;
247
- if (commandResult.exitCode !== 0) {
248
- verification.finishedAt = new Date().toISOString();
249
- verification.passed = false;
250
- const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
251
- status: "rejected",
252
- verification: scrubVerificationPayload(verification),
253
- });
254
- await recordRigAuditEvent(db, {
255
- grant,
256
- action: "rig.verification.failed",
257
- rigId: rig.id,
258
- metadata: { changeId: change.id, status: "rejected" },
259
- });
260
- await recordRigAuditEvent(db, {
261
- grant,
262
- action: "rig.change.rejected",
263
- rigId: rig.id,
264
- metadata: { changeId: change.id },
265
- });
266
- return updated;
267
- }
268
- }
269
- const checkResults = [];
270
- for (const check of candidateVersion.checks) {
271
- const result = await runCommand(
272
- established.session as CommandSession,
273
- check.command,
274
- settings.rigSetupTimeoutMs,
971
+ } catch (error) {
972
+ verification.finishedAt = new Date().toISOString();
973
+ verification.passed = false;
974
+ verification.error = scrubVerificationOutput(
975
+ error instanceof Error ? error.message : String(error),
275
976
  );
276
- checkResults.push({
277
- name: check.name,
278
- command: scrubVerificationOutput(check.command),
279
- ...result,
280
- });
281
- }
282
- verification.checkResults = checkResults;
283
- const passed = checkResults.every((result) => result.exitCode === 0);
284
- verification.finishedAt = new Date().toISOString();
285
- verification.passed = passed;
286
- const classified = classifyRigVerificationOutcome({ kind: change.kind, passed });
287
- if (classified.action === "auto_promote") {
288
- // Keep the change `verifying` (NOT `proposed`) across the write→promote
289
- // gap: promoteSetupAppendChange accepts `verifying`, and leaving it
290
- // `verifying` keeps beginRigChangeVerificationAttempt blocking a
291
- // concurrent /verify — resetting to `proposed` would reopen that race
292
- // (a second run could reject a change whose first verification passed).
293
- await updateRigChangeStatus(db, input.workspaceId, change.id, {
294
- status: "verifying",
977
+ const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
978
+ status: "failed",
295
979
  verification: scrubVerificationPayload(verification),
296
980
  });
297
- const { change: merged } = await promoteSetupAppendChange({ db }, grant, rig, {
298
- ...change,
299
- verification,
300
- });
301
981
  await recordRigAuditEvent(db, {
302
982
  grant,
303
- action: "rig.verification.passed",
983
+ action: "rig.verification.failed",
304
984
  rigId: rig.id,
305
- metadata: { changeId: change.id },
985
+ metadata: { changeId: change.id, status: "failed" },
306
986
  });
307
- return merged;
308
- }
309
- const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
310
- status: classified.status,
311
- verification: scrubVerificationPayload(verification),
312
- });
313
- await recordRigAuditEvent(db, {
314
- grant,
315
- action: passed ? "rig.verification.passed" : "rig.verification.failed",
316
- rigId: rig.id,
317
- metadata: { changeId: change.id, status: classified.status },
318
- });
319
- if (!passed) {
320
987
  await recordRigAuditEvent(db, {
321
988
  grant,
322
- action: "rig.change.rejected",
989
+ action: "rig.change.failed",
323
990
  rigId: rig.id,
324
991
  metadata: { changeId: change.id },
325
992
  });
993
+ if (lifecycle.signal.aborted) throw abortReason(lifecycle.signal);
994
+ return updated;
326
995
  }
327
- return updated;
328
- } catch (error) {
329
- verification.finishedAt = new Date().toISOString();
330
- verification.passed = false;
331
- verification.error = scrubVerificationOutput(
332
- error instanceof Error ? error.message : String(error),
333
- );
334
- const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
335
- status: "failed",
336
- verification: scrubVerificationPayload(verification),
337
- });
996
+ }),
997
+
998
+ verifyRigVersion: (input: { workspaceId: string; versionId: string }) =>
999
+ withRigVerificationActivityLifecycle(async (lifecycle) => {
1000
+ const { settings, db, observability } = await services();
1001
+ throwIfAborted(lifecycle.signal);
1002
+ const { rig, version } = await loadVersionTarget(db, input.workspaceId, input.versionId);
1003
+ const grant = systemGrant(rig);
1004
+ const startedAt = new Date().toISOString();
338
1005
  await recordRigAuditEvent(db, {
339
1006
  grant,
340
- action: "rig.verification.failed",
1007
+ action: "rig.verification.started",
341
1008
  rigId: rig.id,
342
- metadata: { changeId: change.id, status: "failed" },
1009
+ metadata: { versionId: version.id },
343
1010
  });
344
- await recordRigAuditEvent(db, {
345
- grant,
346
- action: "rig.change.failed",
347
- rigId: rig.id,
348
- metadata: { changeId: change.id },
349
- });
350
- return updated;
351
- } finally {
352
- await terminateThrowaway(established);
353
- }
354
- },
355
-
356
- verifyRigVersion: async (input: { workspaceId: string; versionId: string }) => {
357
- const { settings, db } = await services();
358
- const { rig, version } = await loadVersionTarget(db, input.workspaceId, input.versionId);
359
- const grant = systemGrant(rig);
360
- const startedAt = new Date().toISOString();
361
- await recordRigAuditEvent(db, {
362
- grant,
363
- action: "rig.verification.started",
364
- rigId: rig.id,
365
- metadata: { versionId: version.id },
366
- });
367
- let established: EstablishedSandboxSession | null = null;
368
- try {
369
- const runSettings = settingsWithRigImage(settings, version.image);
370
- established = await establishSandboxSessionFromEnvelope(runSettings, null, {
371
- sessionId: `rig-version-verification-${version.id}`,
372
- recovery: "create-or-restore",
373
- environment: {},
374
- });
375
- if ((version.setupScript ?? "").trim()) {
376
- await runRigSetupHook(established.session as never, {
377
- environment: {},
378
- runAs: "root",
379
- rigSetup: {
380
- rigId: rig.id,
381
- rigName: rig.name,
382
- versionId: version.id,
383
- script: version.setupScript ?? "",
384
- timeoutMs: settings.rigSetupTimeoutMs,
1011
+ try {
1012
+ const runSettings = settingsWithRigImage(settings, version.image);
1013
+ return await runWithOwnedRigVerificationSandbox(
1014
+ {
1015
+ settings: runSettings,
1016
+ db,
1017
+ observability,
1018
+ accountId: rig.accountId,
1019
+ workspaceId: input.workspaceId,
1020
+ sandboxGroupId: version.id,
1021
+ rigVersionId: version.id,
1022
+ sessionIdPrefix: `rig-version-verification-${version.id}`,
1023
+ lifecycle,
385
1024
  },
1025
+ async (established, runContext) => {
1026
+ if ((version.setupScript ?? "").trim()) {
1027
+ await runRigSetupHook(established.session as never, {
1028
+ environment: {},
1029
+ runAs: "root",
1030
+ commandRunner: runContext.commandRunner,
1031
+ rigSetup: {
1032
+ rigId: rig.id,
1033
+ rigName: rig.name,
1034
+ versionId: version.id,
1035
+ script: version.setupScript ?? "",
1036
+ timeoutMs: settings.rigSetupTimeoutMs,
1037
+ },
1038
+ });
1039
+ }
1040
+ const checkResults = [];
1041
+ for (const check of version.checks) {
1042
+ checkResults.push({
1043
+ name: check.name,
1044
+ command: scrubVerificationOutput(check.command),
1045
+ ...(await runCommand(
1046
+ established.session as TurnSandboxCommandSession,
1047
+ check.command,
1048
+ settings.rigSetupTimeoutMs,
1049
+ runContext.commandRunner,
1050
+ )),
1051
+ });
1052
+ }
1053
+ const passed = checkResults.every((result) => result.exitCode === 0);
1054
+ await recordRigAuditEvent(db, {
1055
+ grant,
1056
+ action: passed ? "rig.verification.passed" : "rig.verification.failed",
1057
+ rigId: rig.id,
1058
+ metadata: scrubVerificationPayload({
1059
+ versionId: version.id,
1060
+ startedAt,
1061
+ finishedAt: new Date().toISOString(),
1062
+ passed,
1063
+ checkResults,
1064
+ }),
1065
+ });
1066
+ return { versionId: version.id, passed, checkResults };
1067
+ },
1068
+ );
1069
+ } catch (error) {
1070
+ // Infra failure (sandbox establish / setup / check exec threw) — record
1071
+ // rig.verification.failed so activeVersionHealth reflects the failed
1072
+ // re-run instead of staying stale, symmetric to verifyRigChange. Then
1073
+ // rethrow so the Temporal activity still surfaces the failure.
1074
+ const detail = tail(
1075
+ scrubVerificationOutput(error instanceof Error ? error.message : String(error)),
1076
+ 4096,
1077
+ );
1078
+ await recordRigAuditEvent(db, {
1079
+ grant,
1080
+ action: "rig.verification.failed",
1081
+ rigId: rig.id,
1082
+ metadata: scrubVerificationPayload({
1083
+ versionId: version.id,
1084
+ startedAt,
1085
+ finishedAt: new Date().toISOString(),
1086
+ passed: false,
1087
+ error: detail,
1088
+ }),
386
1089
  });
1090
+ throw error;
387
1091
  }
388
- const checkResults = [];
389
- for (const check of version.checks) {
390
- checkResults.push({
391
- name: check.name,
392
- command: scrubVerificationOutput(check.command),
393
- ...(await runCommand(
394
- established.session as CommandSession,
395
- check.command,
396
- settings.rigSetupTimeoutMs,
397
- )),
398
- });
399
- }
400
- const passed = checkResults.every((result) => result.exitCode === 0);
401
- await recordRigAuditEvent(db, {
402
- grant,
403
- action: passed ? "rig.verification.passed" : "rig.verification.failed",
404
- rigId: rig.id,
405
- metadata: scrubVerificationPayload({
406
- versionId: version.id,
407
- startedAt,
408
- finishedAt: new Date().toISOString(),
409
- passed,
410
- checkResults,
411
- }),
412
- });
413
- return { versionId: version.id, passed, checkResults };
414
- } catch (error) {
415
- // Infra failure (sandbox establish / setup / check exec threw) — record
416
- // rig.verification.failed so activeVersionHealth reflects the failed
417
- // re-run instead of staying stale, symmetric to verifyRigChange. Then
418
- // rethrow so the Temporal activity still surfaces the failure.
419
- const detail = tail(
420
- scrubVerificationOutput(error instanceof Error ? error.message : String(error)),
421
- 4096,
422
- );
423
- await recordRigAuditEvent(db, {
424
- grant,
425
- action: "rig.verification.failed",
426
- rigId: rig.id,
427
- metadata: scrubVerificationPayload({
428
- versionId: version.id,
429
- startedAt,
430
- finishedAt: new Date().toISOString(),
431
- passed: false,
432
- error: detail,
433
- }),
434
- });
435
- throw error;
436
- } finally {
437
- await terminateThrowaway(established);
438
- }
439
- },
1092
+ }),
440
1093
  };
441
1094
  }