@opengeni/worker-bundle 0.8.3 → 0.10.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1261 -434
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +3 -1
- package/package.json +11 -11
- package/src/activities/agent-turn.ts +639 -293
- package/src/activities/capabilities.ts +3 -1
- package/src/activities/environment.ts +142 -11
- package/src/activities/rig-verification.ts +897 -244
- package/src/activities/sandbox-lease.ts +32 -1
- package/src/observability-metrics.ts +83 -15
- package/src/workflows/rig-verification.ts +3 -1
|
@@ -5,25 +5,39 @@ import {
|
|
|
5
5
|
promoteSetupAppendChange,
|
|
6
6
|
} from "@opengeni/core";
|
|
7
7
|
import {
|
|
8
|
+
acquireLease,
|
|
8
9
|
beginRigChangeVerificationAttempt,
|
|
10
|
+
commitWarmingToWarm,
|
|
11
|
+
failWarmingToCold,
|
|
9
12
|
getRig,
|
|
10
13
|
getRigChange,
|
|
11
14
|
getRigVersionById,
|
|
15
|
+
markWarmLeaseInstanceLost,
|
|
16
|
+
recordWarmingSandboxCreated,
|
|
17
|
+
releaseLeaseHolder,
|
|
12
18
|
sanitizeEventPayload,
|
|
13
19
|
sanitizeEventString,
|
|
14
20
|
sanitizeMemoryText,
|
|
21
|
+
touchLeaseHolder,
|
|
15
22
|
updateRigChangeStatus,
|
|
16
23
|
type Database,
|
|
17
24
|
} from "@opengeni/db";
|
|
18
25
|
import {
|
|
26
|
+
createTurnToolCancellationController,
|
|
19
27
|
establishSandboxSessionFromEnvelope,
|
|
20
28
|
runRigSetupHook,
|
|
21
29
|
sandboxCommandExitCode,
|
|
22
30
|
sandboxCommandOutput,
|
|
31
|
+
serializeEstablishedSandboxEnvelope,
|
|
32
|
+
tagModalSandbox,
|
|
23
33
|
type EstablishedSandboxSession,
|
|
34
|
+
type SandboxLifecycleCommandRunner,
|
|
35
|
+
type TurnSandboxCommandSession,
|
|
24
36
|
} from "@opengeni/runtime";
|
|
37
|
+
import type { Context } from "@temporalio/activity";
|
|
25
38
|
import type { ActivityServices } from "./types";
|
|
26
39
|
import { settingsWithRigImage } from "./packs";
|
|
40
|
+
import { currentActivityContext } from "./streaming";
|
|
27
41
|
|
|
28
42
|
export type RigVerificationWorkflowInput =
|
|
29
43
|
| { workspaceId: string; changeId: string; versionId?: never }
|
|
@@ -34,12 +48,201 @@ type CommandResult = {
|
|
|
34
48
|
output: string;
|
|
35
49
|
};
|
|
36
50
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
51
|
+
const OUTPUT_TAIL_LIMIT = 64 * 1024;
|
|
52
|
+
const RIG_VERIFICATION_CLEANUP_RESERVE_MAX_MS = 2 * 60_000;
|
|
53
|
+
const RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS = 60_000;
|
|
54
|
+
const RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS = 10_000;
|
|
55
|
+
const RIG_VERIFICATION_HEARTBEAT_MIN_INTERVAL_MS = 250;
|
|
56
|
+
// The Temporal activity has a 15-minute start-to-close timeout. Keep ownership
|
|
57
|
+
// through that complete contract plus five minutes for cancellation delivery
|
|
58
|
+
// and finally cleanup; a dead process then becomes sweep-eligible without any
|
|
59
|
+
// lease-holder fiction or manual recovery.
|
|
60
|
+
export const RIG_VERIFICATION_OWNER_TTL_MS = 20 * 60_000;
|
|
61
|
+
export const RIG_VERIFICATION_OWNERS_DISABLED_MESSAGE =
|
|
62
|
+
"Rig verification lease ownership is disabled; refusing to create an unowned verifier sandbox";
|
|
63
|
+
const RIG_VERIFICATION_HOLDER_TOUCH_INTERVAL_MS = 10_000;
|
|
64
|
+
const RIG_VERIFICATION_RELEASE_GRACE_MS = 1;
|
|
65
|
+
|
|
66
|
+
export class RigVerificationActivityDeadlineError extends Error {
|
|
67
|
+
readonly name = "RigVerificationActivityDeadlineError";
|
|
68
|
+
|
|
69
|
+
constructor(
|
|
70
|
+
readonly startToCloseTimeoutMs: number,
|
|
71
|
+
readonly cleanupReserveMs: number,
|
|
72
|
+
) {
|
|
73
|
+
super(
|
|
74
|
+
`Rig verification activity-local deadline reached with ${cleanupReserveMs}ms reserved for cleanup before the ${startToCloseTimeoutMs}ms Temporal start-to-close timeout`,
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export type RigVerificationActivityLifecycle = {
|
|
80
|
+
signal: AbortSignal;
|
|
81
|
+
/** Absolute wall-clock boundary before which immediate cleanup may wait. */
|
|
82
|
+
cleanupDeadlineAtMs: number | null;
|
|
83
|
+
dispose(): void;
|
|
40
84
|
};
|
|
41
85
|
|
|
42
|
-
|
|
86
|
+
function computeCleanupReserveMs(startToCloseTimeoutMs: number): number {
|
|
87
|
+
return Math.min(
|
|
88
|
+
RIG_VERIFICATION_CLEANUP_RESERVE_MAX_MS,
|
|
89
|
+
Math.max(100, Math.floor(startToCloseTimeoutMs / 4)),
|
|
90
|
+
Math.max(1, startToCloseTimeoutMs - 1),
|
|
91
|
+
);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Start the verifier's real Temporal liveness/cancellation contract.
|
|
96
|
+
*
|
|
97
|
+
* The activity-local deadline is deliberately earlier than start-to-close. In
|
|
98
|
+
* production a 15-minute server deadline becomes a 13-minute work deadline,
|
|
99
|
+
* retaining two minutes for command quiescence, exact deactivation, and
|
|
100
|
+
* provider termination. Short integration-test deadlines use the same bounded
|
|
101
|
+
* one-quarter rule instead of a production-only magic value.
|
|
102
|
+
*/
|
|
103
|
+
export function createRigVerificationActivityLifecycle(
|
|
104
|
+
context: Context | null = currentActivityContext(),
|
|
105
|
+
): RigVerificationActivityLifecycle {
|
|
106
|
+
const controller = new AbortController();
|
|
107
|
+
const temporalSignal = context?.cancellationSignal;
|
|
108
|
+
const forwardTemporalCancellation = (): void => {
|
|
109
|
+
if (!controller.signal.aborted) {
|
|
110
|
+
controller.abort(temporalSignal?.reason ?? new Error("Rig verification cancelled"));
|
|
111
|
+
}
|
|
112
|
+
};
|
|
113
|
+
if (temporalSignal?.aborted) {
|
|
114
|
+
forwardTemporalCancellation();
|
|
115
|
+
} else {
|
|
116
|
+
temporalSignal?.addEventListener("abort", forwardTemporalCancellation, { once: true });
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const startedAtMs = Date.now();
|
|
120
|
+
const startToCloseTimeoutMs = context?.info.startToCloseTimeoutMs;
|
|
121
|
+
const reserveMs =
|
|
122
|
+
typeof startToCloseTimeoutMs === "number" && startToCloseTimeoutMs > 0
|
|
123
|
+
? computeCleanupReserveMs(startToCloseTimeoutMs)
|
|
124
|
+
: null;
|
|
125
|
+
const serverDeadlineAtMs =
|
|
126
|
+
reserveMs === null || startToCloseTimeoutMs === undefined
|
|
127
|
+
? null
|
|
128
|
+
: startedAtMs + startToCloseTimeoutMs;
|
|
129
|
+
// Keep an additional margin after our bounded cleanup waits. If a DB or
|
|
130
|
+
// provider promise remains unresolved, the activity must still report its
|
|
131
|
+
// local deadline before Temporal imposes the hard timeout.
|
|
132
|
+
const serverReportingMarginMs =
|
|
133
|
+
reserveMs === null ? 0 : Math.min(30_000, Math.max(1, Math.floor(reserveMs / 4)));
|
|
134
|
+
const cleanupDeadlineAtMs =
|
|
135
|
+
serverDeadlineAtMs === null ? null : serverDeadlineAtMs - serverReportingMarginMs;
|
|
136
|
+
|
|
137
|
+
let deadlineTimer: ReturnType<typeof setTimeout> | null = null;
|
|
138
|
+
if (reserveMs !== null && startToCloseTimeoutMs !== undefined && !controller.signal.aborted) {
|
|
139
|
+
deadlineTimer = setTimeout(
|
|
140
|
+
() => {
|
|
141
|
+
if (!controller.signal.aborted) {
|
|
142
|
+
controller.abort(
|
|
143
|
+
new RigVerificationActivityDeadlineError(startToCloseTimeoutMs, reserveMs),
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
},
|
|
147
|
+
Math.max(1, startToCloseTimeoutMs - reserveMs),
|
|
148
|
+
);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
const heartbeat = (): void => {
|
|
152
|
+
if (!context) return;
|
|
153
|
+
try {
|
|
154
|
+
context.heartbeat({
|
|
155
|
+
activity: "rig_verification",
|
|
156
|
+
at: new Date().toISOString(),
|
|
157
|
+
cleanup: controller.signal.aborted,
|
|
158
|
+
cleanupDeadlineAt: cleanupDeadlineAtMs ? new Date(cleanupDeadlineAtMs).toISOString() : null,
|
|
159
|
+
});
|
|
160
|
+
} catch (error) {
|
|
161
|
+
// Context.heartbeat reports delivered cancellation synchronously on some
|
|
162
|
+
// SDK paths. Feed that failure into the same cooperative abort fence, but
|
|
163
|
+
// keep the interval alive through bounded cleanup: the abort signal stops
|
|
164
|
+
// verifier work, not the activity's liveness reporting.
|
|
165
|
+
if (!controller.signal.aborted) controller.abort(error);
|
|
166
|
+
}
|
|
167
|
+
};
|
|
168
|
+
heartbeat();
|
|
169
|
+
const heartbeatTimeoutMs = context?.info.heartbeatTimeoutMs;
|
|
170
|
+
const heartbeatIntervalMs =
|
|
171
|
+
typeof heartbeatTimeoutMs === "number" && heartbeatTimeoutMs > 0
|
|
172
|
+
? Math.min(
|
|
173
|
+
RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS,
|
|
174
|
+
Math.max(RIG_VERIFICATION_HEARTBEAT_MIN_INTERVAL_MS, Math.floor(heartbeatTimeoutMs / 3)),
|
|
175
|
+
)
|
|
176
|
+
: RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS;
|
|
177
|
+
const heartbeatTimer = context ? setInterval(heartbeat, heartbeatIntervalMs) : null;
|
|
178
|
+
if (heartbeatTimer && "unref" in heartbeatTimer && typeof heartbeatTimer.unref === "function") {
|
|
179
|
+
heartbeatTimer.unref();
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
return {
|
|
183
|
+
signal: controller.signal,
|
|
184
|
+
cleanupDeadlineAtMs,
|
|
185
|
+
dispose: () => {
|
|
186
|
+
temporalSignal?.removeEventListener("abort", forwardTemporalCancellation);
|
|
187
|
+
if (deadlineTimer) clearTimeout(deadlineTimer);
|
|
188
|
+
if (heartbeatTimer) clearInterval(heartbeatTimer);
|
|
189
|
+
deadlineTimer = null;
|
|
190
|
+
},
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function abortReason(signal: AbortSignal): unknown {
|
|
195
|
+
return signal.reason ?? new Error("Rig verification cancelled");
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function throwIfAborted(signal: AbortSignal): void {
|
|
199
|
+
if (signal.aborted) throw abortReason(signal);
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
async function waitForAbortable<T>(promise: Promise<T>, signal: AbortSignal): Promise<T> {
|
|
203
|
+
throwIfAborted(signal);
|
|
204
|
+
let removeAbortListener = (): void => undefined;
|
|
205
|
+
const cancellation = new Promise<never>((_resolve, reject) => {
|
|
206
|
+
const onAbort = (): void => reject(abortReason(signal));
|
|
207
|
+
removeAbortListener = () => signal.removeEventListener("abort", onAbort);
|
|
208
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
209
|
+
});
|
|
210
|
+
try {
|
|
211
|
+
return await Promise.race([promise, cancellation]);
|
|
212
|
+
} finally {
|
|
213
|
+
removeAbortListener();
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
type BoundedSettlement<T> = { status: "completed"; value: T } | { status: "timed_out" };
|
|
218
|
+
|
|
219
|
+
async function settleWithin<T>(
|
|
220
|
+
promise: Promise<T>,
|
|
221
|
+
timeoutMs: number,
|
|
222
|
+
): Promise<BoundedSettlement<T>> {
|
|
223
|
+
if (timeoutMs <= 0) return { status: "timed_out" };
|
|
224
|
+
let timer: ReturnType<typeof setTimeout> | null = null;
|
|
225
|
+
try {
|
|
226
|
+
return await Promise.race([
|
|
227
|
+
promise.then((value) => ({ status: "completed" as const, value })),
|
|
228
|
+
new Promise<{ status: "timed_out" }>((resolve) => {
|
|
229
|
+
timer = setTimeout(() => resolve({ status: "timed_out" }), timeoutMs);
|
|
230
|
+
}),
|
|
231
|
+
]);
|
|
232
|
+
} finally {
|
|
233
|
+
if (timer) clearTimeout(timer);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function cleanupWaitMs(lifecycle: RigVerificationActivityLifecycle | undefined): number {
|
|
238
|
+
if (lifecycle?.cleanupDeadlineAtMs == null) {
|
|
239
|
+
return RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS;
|
|
240
|
+
}
|
|
241
|
+
return Math.max(
|
|
242
|
+
0,
|
|
243
|
+
Math.min(RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS, lifecycle.cleanupDeadlineAtMs - Date.now()),
|
|
244
|
+
);
|
|
245
|
+
}
|
|
43
246
|
|
|
44
247
|
function tail(value: string, limit = OUTPUT_TAIL_LIMIT): string {
|
|
45
248
|
return value.length > limit ? value.slice(-limit) : value;
|
|
@@ -62,14 +265,18 @@ function systemGrant(rig: Rig): AccessGrant {
|
|
|
62
265
|
};
|
|
63
266
|
}
|
|
64
267
|
|
|
65
|
-
async function terminateThrowaway(established: EstablishedSandboxSession | null): Promise<
|
|
268
|
+
async function terminateThrowaway(established: EstablishedSandboxSession | null): Promise<boolean> {
|
|
66
269
|
if (!established) {
|
|
67
|
-
return;
|
|
270
|
+
return true;
|
|
68
271
|
}
|
|
69
272
|
const client = established.client as { delete?: (state: unknown) => Promise<unknown> };
|
|
70
273
|
if (typeof client.delete === "function" && established.sessionState !== undefined) {
|
|
71
|
-
|
|
72
|
-
|
|
274
|
+
try {
|
|
275
|
+
await client.delete(established.sessionState);
|
|
276
|
+
return true;
|
|
277
|
+
} catch {
|
|
278
|
+
return false;
|
|
279
|
+
}
|
|
73
280
|
}
|
|
74
281
|
const session = established.session as {
|
|
75
282
|
terminate?: () => Promise<unknown>;
|
|
@@ -77,19 +284,432 @@ async function terminateThrowaway(established: EstablishedSandboxSession | null)
|
|
|
77
284
|
close?: () => Promise<unknown>;
|
|
78
285
|
closed?: boolean;
|
|
79
286
|
};
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
287
|
+
try {
|
|
288
|
+
if (session.terminate) {
|
|
289
|
+
await session.terminate();
|
|
290
|
+
} else if (session.kill) {
|
|
291
|
+
await session.kill();
|
|
292
|
+
} else if (session.close && !session.closed) {
|
|
293
|
+
await session.close();
|
|
294
|
+
} else {
|
|
295
|
+
return false;
|
|
296
|
+
}
|
|
297
|
+
return true;
|
|
298
|
+
} catch {
|
|
299
|
+
return false;
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
export type RigVerificationOwnershipDependencies = {
|
|
304
|
+
acquire: typeof acquireLease;
|
|
305
|
+
establish: typeof establishSandboxSessionFromEnvelope;
|
|
306
|
+
recordCreated: typeof recordWarmingSandboxCreated;
|
|
307
|
+
commitWarm: typeof commitWarmingToWarm;
|
|
308
|
+
touchHolder: typeof touchLeaseHolder;
|
|
309
|
+
releaseHolder: typeof releaseLeaseHolder;
|
|
310
|
+
failWarming: typeof failWarmingToCold;
|
|
311
|
+
markWarmLost: typeof markWarmLeaseInstanceLost;
|
|
312
|
+
serialize: typeof serializeEstablishedSandboxEnvelope;
|
|
313
|
+
tag: typeof tagModalSandbox;
|
|
314
|
+
terminate: typeof terminateThrowaway;
|
|
315
|
+
createCancellationController: typeof createTurnToolCancellationController;
|
|
316
|
+
randomUUID: () => string;
|
|
317
|
+
};
|
|
318
|
+
|
|
319
|
+
const defaultOwnershipDependencies: RigVerificationOwnershipDependencies = {
|
|
320
|
+
acquire: acquireLease,
|
|
321
|
+
establish: establishSandboxSessionFromEnvelope,
|
|
322
|
+
recordCreated: recordWarmingSandboxCreated,
|
|
323
|
+
commitWarm: commitWarmingToWarm,
|
|
324
|
+
touchHolder: touchLeaseHolder,
|
|
325
|
+
releaseHolder: releaseLeaseHolder,
|
|
326
|
+
failWarming: failWarmingToCold,
|
|
327
|
+
markWarmLost: markWarmLeaseInstanceLost,
|
|
328
|
+
serialize: serializeEstablishedSandboxEnvelope,
|
|
329
|
+
tag: tagModalSandbox,
|
|
330
|
+
terminate: terminateThrowaway,
|
|
331
|
+
createCancellationController: createTurnToolCancellationController,
|
|
332
|
+
randomUUID: () => crypto.randomUUID(),
|
|
333
|
+
};
|
|
334
|
+
|
|
335
|
+
export type RigVerificationSandboxRunContext = {
|
|
336
|
+
signal: AbortSignal;
|
|
337
|
+
commandRunner: SandboxLifecycleCommandRunner;
|
|
338
|
+
};
|
|
339
|
+
|
|
340
|
+
export class RigVerificationLeaseUnavailableError extends Error {
|
|
341
|
+
readonly name = "RigVerificationLeaseUnavailableError";
|
|
342
|
+
|
|
343
|
+
constructor(
|
|
344
|
+
readonly sandboxGroupId: string,
|
|
345
|
+
readonly role: "attached" | "rearmed" | "fenced",
|
|
346
|
+
) {
|
|
347
|
+
super(`Rig verification requires a new clean sandbox, but lease ${sandboxGroupId} was ${role}`);
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/**
|
|
352
|
+
* Run one verifier against one exactly attributed standalone sandbox.
|
|
353
|
+
*
|
|
354
|
+
* Ordering is deliberate and source-of-truth safe:
|
|
355
|
+
* acquire spawner -> create callback -> warming instance record -> warm commit
|
|
356
|
+
* -> setup/checks. The canonical sandbox lease, not provider tags, is durable
|
|
357
|
+
* ownership. Every exact create callback is retained before its DB write so an
|
|
358
|
+
* ambiguous acknowledgement can still be terminated in finally. Provider
|
|
359
|
+
* termination precedes clearing that exact durable pointer; when termination
|
|
360
|
+
* fails the pointer is retained and the holder is released with a one-millisecond
|
|
361
|
+
* drain grace so the normal reaper retries. Temporal cancellation first drains
|
|
362
|
+
* every verifier command. Cleanup waits are bounded inside the server reserve;
|
|
363
|
+
* lease expiry and the provider reaper remain the hard-worker-loss backstops.
|
|
364
|
+
*/
|
|
365
|
+
export async function runWithOwnedRigVerificationSandbox<T>(
|
|
366
|
+
input: {
|
|
367
|
+
settings: ActivityServices["settings"];
|
|
368
|
+
db: Database;
|
|
369
|
+
observability: ActivityServices["observability"];
|
|
370
|
+
accountId: string;
|
|
371
|
+
workspaceId: string;
|
|
372
|
+
sandboxGroupId: string;
|
|
373
|
+
rigVersionId: string;
|
|
374
|
+
sessionIdPrefix: string;
|
|
375
|
+
lifecycle?: RigVerificationActivityLifecycle;
|
|
376
|
+
},
|
|
377
|
+
run: (
|
|
378
|
+
established: EstablishedSandboxSession,
|
|
379
|
+
context: RigVerificationSandboxRunContext,
|
|
380
|
+
) => Promise<T>,
|
|
381
|
+
dependencies: RigVerificationOwnershipDependencies = defaultOwnershipDependencies,
|
|
382
|
+
): Promise<T> {
|
|
383
|
+
// Phase B must remain inert throughout its code rollout. A disabled or
|
|
384
|
+
// omitted flag rejects before establish() can invoke any provider create;
|
|
385
|
+
// falling back to the legacy unowned verifier is never safe.
|
|
386
|
+
if (!input.settings.rigVerificationLeaseOwnershipEnabled) {
|
|
387
|
+
throw new Error(RIG_VERIFICATION_OWNERS_DISABLED_MESSAGE);
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
const holderId = `rig-verification:${dependencies.randomUUID()}`;
|
|
391
|
+
const fallbackController = new AbortController();
|
|
392
|
+
const signal = input.lifecycle?.signal ?? fallbackController.signal;
|
|
393
|
+
const commandController = dependencies.createCancellationController(signal);
|
|
394
|
+
const commandRunner = commandController.runSandboxCommand.bind(commandController);
|
|
395
|
+
let cleanupTarget: EstablishedSandboxSession | null = null;
|
|
396
|
+
let acquired = false;
|
|
397
|
+
let ownsWarmingEpoch = false;
|
|
398
|
+
let expectedEpoch: number | null = null;
|
|
399
|
+
let holderLivenessTimer: ReturnType<typeof setInterval> | null = null;
|
|
400
|
+
let establishmentSettled = false;
|
|
401
|
+
let establishmentPromise: Promise<EstablishedSandboxSession> | null = null;
|
|
402
|
+
|
|
403
|
+
const warnCleanupTimeout = (operation: string, instanceId: string | null): void => {
|
|
404
|
+
input.observability.warn("rig verifier: ownership cleanup timed out", {
|
|
405
|
+
holderId,
|
|
406
|
+
instanceId,
|
|
407
|
+
operation,
|
|
408
|
+
error:
|
|
409
|
+
"cleanup did not settle inside the activity reserve; exact owner expiry and the global provider reaper are the bounded delayed backstop",
|
|
410
|
+
});
|
|
411
|
+
};
|
|
412
|
+
|
|
413
|
+
const warnCleanupFailure = (
|
|
414
|
+
operation: string,
|
|
415
|
+
instanceId: string | null,
|
|
416
|
+
error: unknown,
|
|
417
|
+
): void => {
|
|
418
|
+
input.observability.warn("rig verifier: ownership cleanup failed", {
|
|
419
|
+
holderId,
|
|
420
|
+
instanceId,
|
|
421
|
+
operation,
|
|
422
|
+
error: error instanceof Error ? error.message : String(error),
|
|
423
|
+
});
|
|
424
|
+
};
|
|
425
|
+
|
|
426
|
+
const clearTerminatedLeasePointer = async (instanceId: string): Promise<void> => {
|
|
427
|
+
if (expectedEpoch === null) return;
|
|
428
|
+
const cleanupEpoch = expectedEpoch;
|
|
429
|
+
// Exactly one transition can match. Running both covers a commit whose DB
|
|
430
|
+
// acknowledgement was lost: expectedEpoch is still warming, while
|
|
431
|
+
// expectedEpoch+1 is the only possible committed warm epoch.
|
|
432
|
+
const results = await Promise.allSettled([
|
|
433
|
+
Promise.resolve().then(() =>
|
|
434
|
+
dependencies.markWarmLost(input.db, {
|
|
435
|
+
accountId: input.accountId,
|
|
436
|
+
workspaceId: input.workspaceId,
|
|
437
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
438
|
+
expectedEpoch: cleanupEpoch + 1,
|
|
439
|
+
expectedInstanceId: instanceId,
|
|
440
|
+
}),
|
|
441
|
+
),
|
|
442
|
+
Promise.resolve().then(() =>
|
|
443
|
+
dependencies.failWarming(input.db, {
|
|
444
|
+
accountId: input.accountId,
|
|
445
|
+
workspaceId: input.workspaceId,
|
|
446
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
447
|
+
expectedEpoch: cleanupEpoch,
|
|
448
|
+
}),
|
|
449
|
+
),
|
|
450
|
+
]);
|
|
451
|
+
for (const [index, result] of results.entries()) {
|
|
452
|
+
if (result.status === "rejected") {
|
|
453
|
+
warnCleanupFailure(
|
|
454
|
+
index === 0 ? "mark_warm_lost" : "fail_warming",
|
|
455
|
+
instanceId,
|
|
456
|
+
result.reason,
|
|
457
|
+
);
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
};
|
|
461
|
+
|
|
462
|
+
const terminateAndClear = async (created: EstablishedSandboxSession): Promise<boolean> => {
|
|
463
|
+
let terminated = false;
|
|
464
|
+
try {
|
|
465
|
+
terminated = await dependencies.terminate(created);
|
|
466
|
+
} catch (error) {
|
|
467
|
+
warnCleanupFailure("terminate", created.instanceId, error);
|
|
468
|
+
}
|
|
469
|
+
if (!terminated) {
|
|
470
|
+
warnCleanupFailure(
|
|
471
|
+
"terminate",
|
|
472
|
+
created.instanceId,
|
|
473
|
+
new Error("provider termination was not confirmed; durable lease cleanup remains pending"),
|
|
474
|
+
);
|
|
475
|
+
return false;
|
|
476
|
+
}
|
|
477
|
+
await clearTerminatedLeasePointer(created.instanceId);
|
|
478
|
+
return true;
|
|
479
|
+
};
|
|
480
|
+
|
|
481
|
+
try {
|
|
482
|
+
throwIfAborted(signal);
|
|
483
|
+
const lease = await dependencies.acquire(input.db, {
|
|
484
|
+
accountId: input.accountId,
|
|
485
|
+
workspaceId: input.workspaceId,
|
|
486
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
487
|
+
kind: "turn",
|
|
488
|
+
holderId,
|
|
489
|
+
subjectId: null,
|
|
490
|
+
backend: input.settings.sandboxBackend,
|
|
491
|
+
image: input.settings.modalImageRef ?? input.settings.dockerImage,
|
|
492
|
+
rigVersionId: input.rigVersionId,
|
|
493
|
+
leaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
|
|
494
|
+
warmingLeaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
|
|
495
|
+
});
|
|
496
|
+
acquired = true;
|
|
497
|
+
expectedEpoch = lease.lease.leaseEpoch;
|
|
498
|
+
holderLivenessTimer = setInterval(() => {
|
|
499
|
+
void dependencies
|
|
500
|
+
.touchHolder(input.db, {
|
|
501
|
+
accountId: input.accountId,
|
|
502
|
+
workspaceId: input.workspaceId,
|
|
503
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
504
|
+
kind: "turn",
|
|
505
|
+
holderId,
|
|
506
|
+
})
|
|
507
|
+
.catch(() => undefined);
|
|
508
|
+
}, RIG_VERIFICATION_HOLDER_TOUCH_INTERVAL_MS);
|
|
509
|
+
if ("unref" in holderLivenessTimer && typeof holderLivenessTimer.unref === "function") {
|
|
510
|
+
holderLivenessTimer.unref();
|
|
511
|
+
}
|
|
512
|
+
if (lease.role !== "spawner") {
|
|
513
|
+
throw new RigVerificationLeaseUnavailableError(input.sandboxGroupId, lease.role);
|
|
514
|
+
}
|
|
515
|
+
ownsWarmingEpoch = true;
|
|
516
|
+
|
|
517
|
+
establishmentPromise = Promise.resolve().then(() =>
|
|
518
|
+
dependencies.establish(input.settings, null, {
|
|
519
|
+
sessionId: `${input.sessionIdPrefix}-${holderId}`,
|
|
520
|
+
recovery: "create-or-restore",
|
|
521
|
+
environment: {},
|
|
522
|
+
onSandboxCreated: async (created) => {
|
|
523
|
+
// Retain the exact handle before the durable write. If registration
|
|
524
|
+
// throws after commit, finally still has the provider identity.
|
|
525
|
+
cleanupTarget = created;
|
|
526
|
+
const resumeState =
|
|
527
|
+
created.backendId === "modal" ? null : await dependencies.serialize(created);
|
|
528
|
+
const recorded = await dependencies.recordCreated(input.db, {
|
|
529
|
+
accountId: input.accountId,
|
|
530
|
+
workspaceId: input.workspaceId,
|
|
531
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
532
|
+
expectedEpoch: lease.lease.leaseEpoch,
|
|
533
|
+
instanceId: created.instanceId,
|
|
534
|
+
resumeBackendId: created.backendId,
|
|
535
|
+
resumeState,
|
|
536
|
+
leaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
|
|
537
|
+
warmingLeaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
|
|
538
|
+
});
|
|
539
|
+
if (!recorded.recorded) {
|
|
540
|
+
throw new RigVerificationLeaseUnavailableError(input.sandboxGroupId, "fenced");
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
if (created.backendId === "modal" && !signal.aborted) {
|
|
544
|
+
try {
|
|
545
|
+
await waitForAbortable(
|
|
546
|
+
dependencies.tag(input.settings, created.instanceId, {
|
|
547
|
+
leaseId: lease.lease.id,
|
|
548
|
+
workspaceId: input.workspaceId,
|
|
549
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
550
|
+
}),
|
|
551
|
+
signal,
|
|
552
|
+
);
|
|
553
|
+
} catch (error) {
|
|
554
|
+
if (!signal.aborted) {
|
|
555
|
+
input.observability.warn("rig verifier: Modal ownership tag failed", {
|
|
556
|
+
holderId,
|
|
557
|
+
instanceId: created.instanceId,
|
|
558
|
+
error: error instanceof Error ? error.message : String(error),
|
|
559
|
+
});
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
// Provider create is not abortable. If it returns after cancellation
|
|
565
|
+
// won the outer race, the callback owns an independent exact cleanup
|
|
566
|
+
// path before it throws and triggers the runtime's fail-closed delete.
|
|
567
|
+
if (signal.aborted) {
|
|
568
|
+
await terminateAndClear(created);
|
|
569
|
+
throw abortReason(signal);
|
|
570
|
+
}
|
|
571
|
+
},
|
|
572
|
+
}),
|
|
573
|
+
);
|
|
574
|
+
void establishmentPromise.then(
|
|
575
|
+
() => {
|
|
576
|
+
establishmentSettled = true;
|
|
577
|
+
},
|
|
578
|
+
() => {
|
|
579
|
+
establishmentSettled = true;
|
|
580
|
+
},
|
|
581
|
+
);
|
|
582
|
+
// The losing provider promise retains the callback above and a rejection
|
|
583
|
+
// handler, so a late create can still deactivate/terminate without an
|
|
584
|
+
// unhandled rejection after the activity has reported cancellation.
|
|
585
|
+
void establishmentPromise.catch(() => undefined);
|
|
586
|
+
const established = await waitForAbortable(establishmentPromise, signal);
|
|
587
|
+
cleanupTarget = established;
|
|
588
|
+
const resumeState =
|
|
589
|
+
established.backendId === "modal" ? null : await dependencies.serialize(established);
|
|
590
|
+
const committed = await dependencies.commitWarm(input.db, {
|
|
591
|
+
accountId: input.accountId,
|
|
592
|
+
workspaceId: input.workspaceId,
|
|
593
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
594
|
+
expectedEpoch: lease.lease.leaseEpoch,
|
|
595
|
+
instanceId: established.instanceId,
|
|
596
|
+
dataPlaneUrl: null,
|
|
597
|
+
resumeBackendId: established.backendId,
|
|
598
|
+
resumeState,
|
|
599
|
+
leaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
|
|
600
|
+
});
|
|
601
|
+
if (!committed.committed || !committed.lease) {
|
|
602
|
+
throw new RigVerificationLeaseUnavailableError(input.sandboxGroupId, "fenced");
|
|
603
|
+
}
|
|
604
|
+
const result = await run(established, { signal, commandRunner });
|
|
605
|
+
throwIfAborted(signal);
|
|
606
|
+
return result;
|
|
607
|
+
} catch (error) {
|
|
608
|
+
if (signal.aborted) throw abortReason(signal);
|
|
609
|
+
throw error;
|
|
610
|
+
} finally {
|
|
611
|
+
if (signal.aborted && establishmentPromise && !establishmentSettled) {
|
|
612
|
+
const lateEstablishment = await settleWithin(
|
|
613
|
+
Promise.allSettled([establishmentPromise]),
|
|
614
|
+
cleanupWaitMs(input.lifecycle),
|
|
615
|
+
);
|
|
616
|
+
if (lateEstablishment.status === "timed_out") {
|
|
617
|
+
warnCleanupTimeout("establish_after_abort", cleanupTarget?.instanceId ?? null);
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
if (holderLivenessTimer) {
|
|
622
|
+
clearInterval(holderLivenessTimer);
|
|
623
|
+
holderLivenessTimer = null;
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
// Physical command quiescence precedes provider teardown. If the provider
|
|
627
|
+
// control plane itself hangs, bound the wait so termination still starts
|
|
628
|
+
// inside the server reserve; terminating the throwaway box is then the
|
|
629
|
+
// stronger final process fence.
|
|
630
|
+
commandController.cancel(signal.aborted ? abortReason(signal) : new Error("Verifier complete"));
|
|
631
|
+
const quiescence = commandController.waitForQuiescence();
|
|
632
|
+
void quiescence.catch(() => undefined);
|
|
633
|
+
const quiescenceResult = await settleWithin(quiescence, cleanupWaitMs(input.lifecycle));
|
|
634
|
+
if (quiescenceResult.status === "timed_out") {
|
|
635
|
+
warnCleanupTimeout("command_quiescence", cleanupTarget?.instanceId ?? null);
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
const providerCleanup = Promise.resolve().then(async () => {
|
|
639
|
+
if (!cleanupTarget) {
|
|
640
|
+
// A provider create that rejects before onSandboxCreated leaves no box
|
|
641
|
+
// pointer to terminate. Roll back only our exact warming epoch so the
|
|
642
|
+
// next verification does not wait for the 20-minute death reaper. If a
|
|
643
|
+
// non-abortable create returns after this transition, its callback is
|
|
644
|
+
// fenced and the runtime's create-failure path terminates that box.
|
|
645
|
+
if (ownsWarmingEpoch && expectedEpoch !== null) {
|
|
646
|
+
await dependencies.failWarming(input.db, {
|
|
647
|
+
accountId: input.accountId,
|
|
648
|
+
workspaceId: input.workspaceId,
|
|
649
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
650
|
+
expectedEpoch,
|
|
651
|
+
});
|
|
652
|
+
}
|
|
653
|
+
return true;
|
|
654
|
+
}
|
|
655
|
+
return await terminateAndClear(cleanupTarget);
|
|
656
|
+
});
|
|
657
|
+
void providerCleanup.catch(() => undefined);
|
|
658
|
+
const providerSettlement = await settleWithin(
|
|
659
|
+
Promise.allSettled([providerCleanup]),
|
|
660
|
+
cleanupWaitMs(input.lifecycle),
|
|
661
|
+
);
|
|
662
|
+
if (providerSettlement.status === "timed_out") {
|
|
663
|
+
warnCleanupTimeout("terminate_and_clear", cleanupTarget?.instanceId ?? null);
|
|
664
|
+
} else if (providerSettlement.value[0]?.status === "rejected") {
|
|
665
|
+
warnCleanupFailure(
|
|
666
|
+
"terminate_and_clear",
|
|
667
|
+
cleanupTarget?.instanceId ?? null,
|
|
668
|
+
providerSettlement.value[0].reason,
|
|
669
|
+
);
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
// Release is independent from provider/DB cleanup. A failed direct
|
|
673
|
+
// termination therefore leaves the exact pointer intact and immediately
|
|
674
|
+
// makes the zero-holder lease eligible for the normal drain/reaper path.
|
|
675
|
+
if (acquired) {
|
|
676
|
+
const release = Promise.resolve().then(() =>
|
|
677
|
+
dependencies.releaseHolder(input.db, {
|
|
678
|
+
accountId: input.accountId,
|
|
679
|
+
workspaceId: input.workspaceId,
|
|
680
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
681
|
+
kind: "turn",
|
|
682
|
+
holderId,
|
|
683
|
+
idleGraceMs: RIG_VERIFICATION_RELEASE_GRACE_MS,
|
|
684
|
+
}),
|
|
685
|
+
);
|
|
686
|
+
void release.catch(() => undefined);
|
|
687
|
+
const released = await settleWithin(
|
|
688
|
+
Promise.allSettled([release]),
|
|
689
|
+
cleanupWaitMs(input.lifecycle),
|
|
690
|
+
);
|
|
691
|
+
if (released.status === "timed_out") {
|
|
692
|
+
warnCleanupTimeout("release_holder", cleanupTarget?.instanceId ?? null);
|
|
693
|
+
} else if (released.value[0]?.status === "rejected") {
|
|
694
|
+
warnCleanupFailure(
|
|
695
|
+
"release_holder",
|
|
696
|
+
cleanupTarget?.instanceId ?? null,
|
|
697
|
+
released.value[0].reason,
|
|
698
|
+
);
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
// Cancellation may be delivered by a heartbeat while finally is already
|
|
702
|
+
// draining. WAIT_CANCELLATION_COMPLETED must observe cancellation only
|
|
703
|
+
// after cleanup, but it must not turn that late cancellation into success.
|
|
704
|
+
throwIfAborted(signal);
|
|
86
705
|
}
|
|
87
706
|
}
|
|
88
707
|
|
|
89
708
|
async function runCommand(
|
|
90
|
-
session:
|
|
709
|
+
session: TurnSandboxCommandSession,
|
|
91
710
|
command: string,
|
|
92
711
|
timeoutMs: number,
|
|
712
|
+
commandRunner: SandboxLifecycleCommandRunner,
|
|
93
713
|
): Promise<CommandResult> {
|
|
94
714
|
const args = {
|
|
95
715
|
cmd: command,
|
|
@@ -98,13 +718,7 @@ async function runCommand(
|
|
|
98
718
|
yieldTimeMs: timeoutMs,
|
|
99
719
|
maxOutputTokens: 40_000,
|
|
100
720
|
};
|
|
101
|
-
const result = session
|
|
102
|
-
? await session.exec(args)
|
|
103
|
-
: session.execCommand
|
|
104
|
-
? await session.execCommand(args)
|
|
105
|
-
: (() => {
|
|
106
|
-
throw new Error("Sandbox session does not support command execution");
|
|
107
|
-
})();
|
|
721
|
+
const result = await commandRunner(session, args);
|
|
108
722
|
return {
|
|
109
723
|
exitCode: sandboxCommandExitCode(result),
|
|
110
724
|
output: scrubVerificationOutput(tail(sandboxCommandOutput(result))),
|
|
@@ -190,252 +804,291 @@ async function loadVersionTarget(
|
|
|
190
804
|
return { rig, version };
|
|
191
805
|
}
|
|
192
806
|
|
|
807
|
+
async function withRigVerificationActivityLifecycle<T>(
|
|
808
|
+
run: (lifecycle: RigVerificationActivityLifecycle) => Promise<T>,
|
|
809
|
+
): Promise<T> {
|
|
810
|
+
// Context.current() and the immediate heartbeat happen before lazy service
|
|
811
|
+
// construction, so Temporal can deliver cancellation even while the worker
|
|
812
|
+
// is opening its first database/provider dependencies.
|
|
813
|
+
const lifecycle = createRigVerificationActivityLifecycle();
|
|
814
|
+
try {
|
|
815
|
+
throwIfAborted(lifecycle.signal);
|
|
816
|
+
return await run(lifecycle);
|
|
817
|
+
} finally {
|
|
818
|
+
lifecycle.dispose();
|
|
819
|
+
}
|
|
820
|
+
}
|
|
821
|
+
|
|
193
822
|
export function createRigVerificationActivities(services: () => Promise<ActivityServices>) {
|
|
194
823
|
return {
|
|
195
|
-
verifyRigChange:
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
startedAt
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
let established: EstablishedSandboxSession | null = null;
|
|
216
|
-
const verification: Record<string, unknown> = { startedAt, checkResults: [] };
|
|
217
|
-
try {
|
|
218
|
-
const candidateVersion = candidateVersionForChange(baseVersion, change);
|
|
219
|
-
const runSettings = settingsWithRigImage(settings, candidateVersion.image);
|
|
220
|
-
established = await establishSandboxSessionFromEnvelope(runSettings, null, {
|
|
221
|
-
sessionId: `rig-verification-${change.id}`,
|
|
222
|
-
recovery: "create-or-restore",
|
|
223
|
-
environment: {},
|
|
824
|
+
verifyRigChange: (input: { workspaceId: string; changeId: string }) =>
|
|
825
|
+
withRigVerificationActivityLifecycle(async (lifecycle) => {
|
|
826
|
+
const { settings, db, observability } = await services();
|
|
827
|
+
throwIfAborted(lifecycle.signal);
|
|
828
|
+
const { rig, baseVersion, change } = await loadChangeTarget(
|
|
829
|
+
db,
|
|
830
|
+
input.workspaceId,
|
|
831
|
+
input.changeId,
|
|
832
|
+
);
|
|
833
|
+
const grant = systemGrant(rig);
|
|
834
|
+
const startedAt = new Date().toISOString();
|
|
835
|
+
await beginRigChangeVerificationAttempt(db, input.workspaceId, change.id, {
|
|
836
|
+
startedAt,
|
|
837
|
+
allowAlreadyVerifying: true,
|
|
838
|
+
});
|
|
839
|
+
await recordRigAuditEvent(db, {
|
|
840
|
+
grant,
|
|
841
|
+
action: "rig.verification.started",
|
|
842
|
+
rigId: rig.id,
|
|
843
|
+
metadata: { changeId: change.id },
|
|
224
844
|
});
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
845
|
+
|
|
846
|
+
const verification: Record<string, unknown> = { startedAt, checkResults: [] };
|
|
847
|
+
try {
|
|
848
|
+
const candidateVersion = candidateVersionForChange(baseVersion, change);
|
|
849
|
+
const runSettings = settingsWithRigImage(settings, candidateVersion.image);
|
|
850
|
+
return await runWithOwnedRigVerificationSandbox(
|
|
851
|
+
{
|
|
852
|
+
settings: runSettings,
|
|
853
|
+
db,
|
|
854
|
+
observability,
|
|
855
|
+
accountId: rig.accountId,
|
|
856
|
+
workspaceId: input.workspaceId,
|
|
857
|
+
sandboxGroupId: change.id,
|
|
858
|
+
rigVersionId: candidateVersion.id,
|
|
859
|
+
sessionIdPrefix: `rig-verification-${change.id}`,
|
|
860
|
+
lifecycle,
|
|
861
|
+
},
|
|
862
|
+
async (established, runContext) => {
|
|
863
|
+
if ((candidateVersion.setupScript ?? "").trim()) {
|
|
864
|
+
await runRigSetupHook(established.session as never, {
|
|
865
|
+
environment: {},
|
|
866
|
+
runAs: "root",
|
|
867
|
+
commandRunner: runContext.commandRunner,
|
|
868
|
+
rigSetup: {
|
|
869
|
+
rigId: rig.id,
|
|
870
|
+
rigName: rig.name,
|
|
871
|
+
versionId: candidateVersion.id,
|
|
872
|
+
script: candidateVersion.setupScript ?? "",
|
|
873
|
+
timeoutMs: settings.rigSetupTimeoutMs,
|
|
874
|
+
},
|
|
875
|
+
});
|
|
876
|
+
verification.setupResult = { exitCode: 0, output: "" };
|
|
877
|
+
}
|
|
878
|
+
const command = setupAppendCommand(change);
|
|
879
|
+
if (command) {
|
|
880
|
+
const commandResult = await runCommand(
|
|
881
|
+
established.session as TurnSandboxCommandSession,
|
|
882
|
+
command,
|
|
883
|
+
settings.rigSetupTimeoutMs,
|
|
884
|
+
runContext.commandRunner,
|
|
885
|
+
);
|
|
886
|
+
verification.commandResult = commandResult;
|
|
887
|
+
if (commandResult.exitCode !== 0) {
|
|
888
|
+
verification.finishedAt = new Date().toISOString();
|
|
889
|
+
verification.passed = false;
|
|
890
|
+
const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
891
|
+
status: "rejected",
|
|
892
|
+
verification: scrubVerificationPayload(verification),
|
|
893
|
+
});
|
|
894
|
+
await recordRigAuditEvent(db, {
|
|
895
|
+
grant,
|
|
896
|
+
action: "rig.verification.failed",
|
|
897
|
+
rigId: rig.id,
|
|
898
|
+
metadata: { changeId: change.id, status: "rejected" },
|
|
899
|
+
});
|
|
900
|
+
await recordRigAuditEvent(db, {
|
|
901
|
+
grant,
|
|
902
|
+
action: "rig.change.rejected",
|
|
903
|
+
rigId: rig.id,
|
|
904
|
+
metadata: { changeId: change.id },
|
|
905
|
+
});
|
|
906
|
+
return updated;
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
const checkResults = [];
|
|
910
|
+
for (const check of candidateVersion.checks) {
|
|
911
|
+
const result = await runCommand(
|
|
912
|
+
established.session as TurnSandboxCommandSession,
|
|
913
|
+
check.command,
|
|
914
|
+
settings.rigSetupTimeoutMs,
|
|
915
|
+
runContext.commandRunner,
|
|
916
|
+
);
|
|
917
|
+
checkResults.push({
|
|
918
|
+
name: check.name,
|
|
919
|
+
command: scrubVerificationOutput(check.command),
|
|
920
|
+
...result,
|
|
921
|
+
});
|
|
922
|
+
}
|
|
923
|
+
verification.checkResults = checkResults;
|
|
924
|
+
const passed = checkResults.every((result) => result.exitCode === 0);
|
|
925
|
+
verification.finishedAt = new Date().toISOString();
|
|
926
|
+
verification.passed = passed;
|
|
927
|
+
const classified = classifyRigVerificationOutcome({ kind: change.kind, passed });
|
|
928
|
+
if (classified.action === "auto_promote") {
|
|
929
|
+
// Keep the change `verifying` (NOT `proposed`) across the write→promote
|
|
930
|
+
// gap: promoteSetupAppendChange accepts `verifying`, and leaving it
|
|
931
|
+
// `verifying` keeps beginRigChangeVerificationAttempt blocking a
|
|
932
|
+
// concurrent /verify — resetting to `proposed` would reopen that race
|
|
933
|
+
// (a second run could reject a change whose first verification passed).
|
|
934
|
+
await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
935
|
+
status: "verifying",
|
|
936
|
+
verification: scrubVerificationPayload(verification),
|
|
937
|
+
});
|
|
938
|
+
const { change: merged } = await promoteSetupAppendChange({ db }, grant, rig, {
|
|
939
|
+
...change,
|
|
940
|
+
verification,
|
|
941
|
+
});
|
|
942
|
+
await recordRigAuditEvent(db, {
|
|
943
|
+
grant,
|
|
944
|
+
action: "rig.verification.passed",
|
|
945
|
+
rigId: rig.id,
|
|
946
|
+
metadata: { changeId: change.id },
|
|
947
|
+
});
|
|
948
|
+
return merged;
|
|
949
|
+
}
|
|
950
|
+
const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
951
|
+
status: classified.status,
|
|
952
|
+
verification: scrubVerificationPayload(verification),
|
|
953
|
+
});
|
|
954
|
+
await recordRigAuditEvent(db, {
|
|
955
|
+
grant,
|
|
956
|
+
action: passed ? "rig.verification.passed" : "rig.verification.failed",
|
|
957
|
+
rigId: rig.id,
|
|
958
|
+
metadata: { changeId: change.id, status: classified.status },
|
|
959
|
+
});
|
|
960
|
+
if (!passed) {
|
|
961
|
+
await recordRigAuditEvent(db, {
|
|
962
|
+
grant,
|
|
963
|
+
action: "rig.change.rejected",
|
|
964
|
+
rigId: rig.id,
|
|
965
|
+
metadata: { changeId: change.id },
|
|
966
|
+
});
|
|
967
|
+
}
|
|
968
|
+
return updated;
|
|
235
969
|
},
|
|
236
|
-
});
|
|
237
|
-
verification.setupResult = { exitCode: 0, output: "" };
|
|
238
|
-
}
|
|
239
|
-
const command = setupAppendCommand(change);
|
|
240
|
-
if (command) {
|
|
241
|
-
const commandResult = await runCommand(
|
|
242
|
-
established.session as CommandSession,
|
|
243
|
-
command,
|
|
244
|
-
settings.rigSetupTimeoutMs,
|
|
245
970
|
);
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
status: "rejected",
|
|
252
|
-
verification: scrubVerificationPayload(verification),
|
|
253
|
-
});
|
|
254
|
-
await recordRigAuditEvent(db, {
|
|
255
|
-
grant,
|
|
256
|
-
action: "rig.verification.failed",
|
|
257
|
-
rigId: rig.id,
|
|
258
|
-
metadata: { changeId: change.id, status: "rejected" },
|
|
259
|
-
});
|
|
260
|
-
await recordRigAuditEvent(db, {
|
|
261
|
-
grant,
|
|
262
|
-
action: "rig.change.rejected",
|
|
263
|
-
rigId: rig.id,
|
|
264
|
-
metadata: { changeId: change.id },
|
|
265
|
-
});
|
|
266
|
-
return updated;
|
|
267
|
-
}
|
|
268
|
-
}
|
|
269
|
-
const checkResults = [];
|
|
270
|
-
for (const check of candidateVersion.checks) {
|
|
271
|
-
const result = await runCommand(
|
|
272
|
-
established.session as CommandSession,
|
|
273
|
-
check.command,
|
|
274
|
-
settings.rigSetupTimeoutMs,
|
|
971
|
+
} catch (error) {
|
|
972
|
+
verification.finishedAt = new Date().toISOString();
|
|
973
|
+
verification.passed = false;
|
|
974
|
+
verification.error = scrubVerificationOutput(
|
|
975
|
+
error instanceof Error ? error.message : String(error),
|
|
275
976
|
);
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
command: scrubVerificationOutput(check.command),
|
|
279
|
-
...result,
|
|
280
|
-
});
|
|
281
|
-
}
|
|
282
|
-
verification.checkResults = checkResults;
|
|
283
|
-
const passed = checkResults.every((result) => result.exitCode === 0);
|
|
284
|
-
verification.finishedAt = new Date().toISOString();
|
|
285
|
-
verification.passed = passed;
|
|
286
|
-
const classified = classifyRigVerificationOutcome({ kind: change.kind, passed });
|
|
287
|
-
if (classified.action === "auto_promote") {
|
|
288
|
-
// Keep the change `verifying` (NOT `proposed`) across the write→promote
|
|
289
|
-
// gap: promoteSetupAppendChange accepts `verifying`, and leaving it
|
|
290
|
-
// `verifying` keeps beginRigChangeVerificationAttempt blocking a
|
|
291
|
-
// concurrent /verify — resetting to `proposed` would reopen that race
|
|
292
|
-
// (a second run could reject a change whose first verification passed).
|
|
293
|
-
await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
294
|
-
status: "verifying",
|
|
977
|
+
const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
978
|
+
status: "failed",
|
|
295
979
|
verification: scrubVerificationPayload(verification),
|
|
296
980
|
});
|
|
297
|
-
const { change: merged } = await promoteSetupAppendChange({ db }, grant, rig, {
|
|
298
|
-
...change,
|
|
299
|
-
verification,
|
|
300
|
-
});
|
|
301
981
|
await recordRigAuditEvent(db, {
|
|
302
982
|
grant,
|
|
303
|
-
action: "rig.verification.
|
|
983
|
+
action: "rig.verification.failed",
|
|
304
984
|
rigId: rig.id,
|
|
305
|
-
metadata: { changeId: change.id },
|
|
985
|
+
metadata: { changeId: change.id, status: "failed" },
|
|
306
986
|
});
|
|
307
|
-
return merged;
|
|
308
|
-
}
|
|
309
|
-
const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
310
|
-
status: classified.status,
|
|
311
|
-
verification: scrubVerificationPayload(verification),
|
|
312
|
-
});
|
|
313
|
-
await recordRigAuditEvent(db, {
|
|
314
|
-
grant,
|
|
315
|
-
action: passed ? "rig.verification.passed" : "rig.verification.failed",
|
|
316
|
-
rigId: rig.id,
|
|
317
|
-
metadata: { changeId: change.id, status: classified.status },
|
|
318
|
-
});
|
|
319
|
-
if (!passed) {
|
|
320
987
|
await recordRigAuditEvent(db, {
|
|
321
988
|
grant,
|
|
322
|
-
action: "rig.change.
|
|
989
|
+
action: "rig.change.failed",
|
|
323
990
|
rigId: rig.id,
|
|
324
991
|
metadata: { changeId: change.id },
|
|
325
992
|
});
|
|
993
|
+
if (lifecycle.signal.aborted) throw abortReason(lifecycle.signal);
|
|
994
|
+
return updated;
|
|
326
995
|
}
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
);
|
|
334
|
-
const
|
|
335
|
-
|
|
336
|
-
verification: scrubVerificationPayload(verification),
|
|
337
|
-
});
|
|
996
|
+
}),
|
|
997
|
+
|
|
998
|
+
verifyRigVersion: (input: { workspaceId: string; versionId: string }) =>
|
|
999
|
+
withRigVerificationActivityLifecycle(async (lifecycle) => {
|
|
1000
|
+
const { settings, db, observability } = await services();
|
|
1001
|
+
throwIfAborted(lifecycle.signal);
|
|
1002
|
+
const { rig, version } = await loadVersionTarget(db, input.workspaceId, input.versionId);
|
|
1003
|
+
const grant = systemGrant(rig);
|
|
1004
|
+
const startedAt = new Date().toISOString();
|
|
338
1005
|
await recordRigAuditEvent(db, {
|
|
339
1006
|
grant,
|
|
340
|
-
action: "rig.verification.
|
|
1007
|
+
action: "rig.verification.started",
|
|
341
1008
|
rigId: rig.id,
|
|
342
|
-
metadata: {
|
|
1009
|
+
metadata: { versionId: version.id },
|
|
343
1010
|
});
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
const { settings, db } = await services();
|
|
358
|
-
const { rig, version } = await loadVersionTarget(db, input.workspaceId, input.versionId);
|
|
359
|
-
const grant = systemGrant(rig);
|
|
360
|
-
const startedAt = new Date().toISOString();
|
|
361
|
-
await recordRigAuditEvent(db, {
|
|
362
|
-
grant,
|
|
363
|
-
action: "rig.verification.started",
|
|
364
|
-
rigId: rig.id,
|
|
365
|
-
metadata: { versionId: version.id },
|
|
366
|
-
});
|
|
367
|
-
let established: EstablishedSandboxSession | null = null;
|
|
368
|
-
try {
|
|
369
|
-
const runSettings = settingsWithRigImage(settings, version.image);
|
|
370
|
-
established = await establishSandboxSessionFromEnvelope(runSettings, null, {
|
|
371
|
-
sessionId: `rig-version-verification-${version.id}`,
|
|
372
|
-
recovery: "create-or-restore",
|
|
373
|
-
environment: {},
|
|
374
|
-
});
|
|
375
|
-
if ((version.setupScript ?? "").trim()) {
|
|
376
|
-
await runRigSetupHook(established.session as never, {
|
|
377
|
-
environment: {},
|
|
378
|
-
runAs: "root",
|
|
379
|
-
rigSetup: {
|
|
380
|
-
rigId: rig.id,
|
|
381
|
-
rigName: rig.name,
|
|
382
|
-
versionId: version.id,
|
|
383
|
-
script: version.setupScript ?? "",
|
|
384
|
-
timeoutMs: settings.rigSetupTimeoutMs,
|
|
1011
|
+
try {
|
|
1012
|
+
const runSettings = settingsWithRigImage(settings, version.image);
|
|
1013
|
+
return await runWithOwnedRigVerificationSandbox(
|
|
1014
|
+
{
|
|
1015
|
+
settings: runSettings,
|
|
1016
|
+
db,
|
|
1017
|
+
observability,
|
|
1018
|
+
accountId: rig.accountId,
|
|
1019
|
+
workspaceId: input.workspaceId,
|
|
1020
|
+
sandboxGroupId: version.id,
|
|
1021
|
+
rigVersionId: version.id,
|
|
1022
|
+
sessionIdPrefix: `rig-version-verification-${version.id}`,
|
|
1023
|
+
lifecycle,
|
|
385
1024
|
},
|
|
1025
|
+
async (established, runContext) => {
|
|
1026
|
+
if ((version.setupScript ?? "").trim()) {
|
|
1027
|
+
await runRigSetupHook(established.session as never, {
|
|
1028
|
+
environment: {},
|
|
1029
|
+
runAs: "root",
|
|
1030
|
+
commandRunner: runContext.commandRunner,
|
|
1031
|
+
rigSetup: {
|
|
1032
|
+
rigId: rig.id,
|
|
1033
|
+
rigName: rig.name,
|
|
1034
|
+
versionId: version.id,
|
|
1035
|
+
script: version.setupScript ?? "",
|
|
1036
|
+
timeoutMs: settings.rigSetupTimeoutMs,
|
|
1037
|
+
},
|
|
1038
|
+
});
|
|
1039
|
+
}
|
|
1040
|
+
const checkResults = [];
|
|
1041
|
+
for (const check of version.checks) {
|
|
1042
|
+
checkResults.push({
|
|
1043
|
+
name: check.name,
|
|
1044
|
+
command: scrubVerificationOutput(check.command),
|
|
1045
|
+
...(await runCommand(
|
|
1046
|
+
established.session as TurnSandboxCommandSession,
|
|
1047
|
+
check.command,
|
|
1048
|
+
settings.rigSetupTimeoutMs,
|
|
1049
|
+
runContext.commandRunner,
|
|
1050
|
+
)),
|
|
1051
|
+
});
|
|
1052
|
+
}
|
|
1053
|
+
const passed = checkResults.every((result) => result.exitCode === 0);
|
|
1054
|
+
await recordRigAuditEvent(db, {
|
|
1055
|
+
grant,
|
|
1056
|
+
action: passed ? "rig.verification.passed" : "rig.verification.failed",
|
|
1057
|
+
rigId: rig.id,
|
|
1058
|
+
metadata: scrubVerificationPayload({
|
|
1059
|
+
versionId: version.id,
|
|
1060
|
+
startedAt,
|
|
1061
|
+
finishedAt: new Date().toISOString(),
|
|
1062
|
+
passed,
|
|
1063
|
+
checkResults,
|
|
1064
|
+
}),
|
|
1065
|
+
});
|
|
1066
|
+
return { versionId: version.id, passed, checkResults };
|
|
1067
|
+
},
|
|
1068
|
+
);
|
|
1069
|
+
} catch (error) {
|
|
1070
|
+
// Infra failure (sandbox establish / setup / check exec threw) — record
|
|
1071
|
+
// rig.verification.failed so activeVersionHealth reflects the failed
|
|
1072
|
+
// re-run instead of staying stale, symmetric to verifyRigChange. Then
|
|
1073
|
+
// rethrow so the Temporal activity still surfaces the failure.
|
|
1074
|
+
const detail = tail(
|
|
1075
|
+
scrubVerificationOutput(error instanceof Error ? error.message : String(error)),
|
|
1076
|
+
4096,
|
|
1077
|
+
);
|
|
1078
|
+
await recordRigAuditEvent(db, {
|
|
1079
|
+
grant,
|
|
1080
|
+
action: "rig.verification.failed",
|
|
1081
|
+
rigId: rig.id,
|
|
1082
|
+
metadata: scrubVerificationPayload({
|
|
1083
|
+
versionId: version.id,
|
|
1084
|
+
startedAt,
|
|
1085
|
+
finishedAt: new Date().toISOString(),
|
|
1086
|
+
passed: false,
|
|
1087
|
+
error: detail,
|
|
1088
|
+
}),
|
|
386
1089
|
});
|
|
1090
|
+
throw error;
|
|
387
1091
|
}
|
|
388
|
-
|
|
389
|
-
for (const check of version.checks) {
|
|
390
|
-
checkResults.push({
|
|
391
|
-
name: check.name,
|
|
392
|
-
command: scrubVerificationOutput(check.command),
|
|
393
|
-
...(await runCommand(
|
|
394
|
-
established.session as CommandSession,
|
|
395
|
-
check.command,
|
|
396
|
-
settings.rigSetupTimeoutMs,
|
|
397
|
-
)),
|
|
398
|
-
});
|
|
399
|
-
}
|
|
400
|
-
const passed = checkResults.every((result) => result.exitCode === 0);
|
|
401
|
-
await recordRigAuditEvent(db, {
|
|
402
|
-
grant,
|
|
403
|
-
action: passed ? "rig.verification.passed" : "rig.verification.failed",
|
|
404
|
-
rigId: rig.id,
|
|
405
|
-
metadata: scrubVerificationPayload({
|
|
406
|
-
versionId: version.id,
|
|
407
|
-
startedAt,
|
|
408
|
-
finishedAt: new Date().toISOString(),
|
|
409
|
-
passed,
|
|
410
|
-
checkResults,
|
|
411
|
-
}),
|
|
412
|
-
});
|
|
413
|
-
return { versionId: version.id, passed, checkResults };
|
|
414
|
-
} catch (error) {
|
|
415
|
-
// Infra failure (sandbox establish / setup / check exec threw) — record
|
|
416
|
-
// rig.verification.failed so activeVersionHealth reflects the failed
|
|
417
|
-
// re-run instead of staying stale, symmetric to verifyRigChange. Then
|
|
418
|
-
// rethrow so the Temporal activity still surfaces the failure.
|
|
419
|
-
const detail = tail(
|
|
420
|
-
scrubVerificationOutput(error instanceof Error ? error.message : String(error)),
|
|
421
|
-
4096,
|
|
422
|
-
);
|
|
423
|
-
await recordRigAuditEvent(db, {
|
|
424
|
-
grant,
|
|
425
|
-
action: "rig.verification.failed",
|
|
426
|
-
rigId: rig.id,
|
|
427
|
-
metadata: scrubVerificationPayload({
|
|
428
|
-
versionId: version.id,
|
|
429
|
-
startedAt,
|
|
430
|
-
finishedAt: new Date().toISOString(),
|
|
431
|
-
passed: false,
|
|
432
|
-
error: detail,
|
|
433
|
-
}),
|
|
434
|
-
});
|
|
435
|
-
throw error;
|
|
436
|
-
} finally {
|
|
437
|
-
await terminateThrowaway(established);
|
|
438
|
-
}
|
|
439
|
-
},
|
|
1092
|
+
}),
|
|
440
1093
|
};
|
|
441
1094
|
}
|