@opengeni/worker-bundle 0.16.21 → 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +145 -32
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +5 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +3 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +11 -7
- package/dist/activities/expiring-material-renewal.d.ts +5 -3
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +2 -2
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +13 -26
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
- package/dist/activities/retained-screenshots.d.ts +129 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +2 -2
- package/dist/activities/run-input.d.ts +5 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +4 -4
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +71 -5
- package/dist/activities/video-generation-admission.d.ts +38 -0
- package/dist/activities/video-generation-credential.d.ts +7 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +49 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +11 -1
- package/dist/activities-control-44KBWLSM.js +6885 -0
- package/dist/activities-control-44KBWLSM.js.map +1 -0
- package/dist/activities-control.d.ts +289 -0
- package/dist/activities-turn-K4CEDEGJ.js +14675 -0
- package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +130 -147
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-E3RI3I33.js +828 -0
- package/dist/chunk-E3RI3I33.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-KKCRGAGZ.js +3091 -0
- package/dist/chunk-KKCRGAGZ.js.map +1 -0
- package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
- package/dist/chunk-RAMNYSAW.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/host-export.js +1 -1
- package/dist/index.d.ts +176 -10
- package/dist/index.js +675 -16460
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +22 -0
- package/dist/observability-metrics.d.ts +33 -0
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +40 -7
- package/dist/sandbox-routing.d.ts +13 -6
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1044 -363
- package/dist/workflows/activities.d.ts +7 -1
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +9 -7
- package/package.json +34 -15
- package/src/activities/agent-turn.ts +1905 -559
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +16 -7
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/codex-rotation.ts +19 -5
- package/src/activities/context-compaction.ts +47 -6
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +52 -60
- package/src/activities/expiring-material-renewal.ts +16 -7
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/git-credential-renewal.ts +3 -3
- package/src/activities/goals.ts +6 -2
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +55 -55
- package/src/activities/recording.ts +4 -5
- package/src/activities/retained-screenshot-reaper.ts +202 -0
- package/src/activities/retained-screenshots.ts +1317 -0
- package/src/activities/rig-verification.ts +567 -77
- package/src/activities/run-credential-renewal.ts +9 -2
- package/src/activities/run-input.ts +40 -10
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +96 -7
- package/src/activities/session-state.ts +4 -4
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +87 -5
- package/src/activities/video-generation-admission.ts +270 -0
- package/src/activities/video-generation-credential.ts +34 -0
- package/src/activities/video-generation-reconciliation.ts +578 -0
- package/src/activities/video-media-validation.ts +326 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +263 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +77 -13
- package/src/activities-control.ts +54 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +50 -118
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/host-export-pump.ts +28 -11
- package/src/http.ts +3 -6
- package/src/index.ts +277 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +116 -0
- package/src/observability-metrics.ts +202 -7
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +331 -157
- package/src/sandbox-routing.ts +49 -39
- package/src/worker-service-lifecycle.ts +82 -12
- package/src/workflows/activities.ts +30 -0
- package/src/workflows/file-upload-reaper.ts +6 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +17 -1
- package/dist/activities/redaction.d.ts +0 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/dist/chunk-BJUW5LCM.js.map +0 -1
- package/src/activities/redaction.ts +0 -9
- package/src/activities/toolspace-token-renewal.ts +0 -57
|
@@ -1,45 +1,85 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
|
|
1
|
+
import type {
|
|
2
|
+
AccessGrant,
|
|
3
|
+
Rig,
|
|
4
|
+
RigChange,
|
|
5
|
+
RigProviderImage,
|
|
6
|
+
RigVersion,
|
|
7
|
+
SandboxBackend,
|
|
8
|
+
} from "@opengeni/contracts";
|
|
9
|
+
import { sandboxLifecycleTransitionWaitMs } from "@opengeni/config";
|
|
3
10
|
import {
|
|
11
|
+
appendRigSetupCommand,
|
|
4
12
|
recordRigAuditEvent,
|
|
5
13
|
classifyRigVerificationOutcome,
|
|
6
14
|
promoteSetupAppendChange,
|
|
15
|
+
rigProviderImageBuildRequestId,
|
|
16
|
+
rigProviderImageContentHash,
|
|
17
|
+
rigProviderImageProviderBindingKeyHash,
|
|
18
|
+
rigProviderImageSetupHash,
|
|
19
|
+
type RigProviderImageDefinition,
|
|
7
20
|
} from "@opengeni/core";
|
|
8
21
|
import {
|
|
9
22
|
acquireLease,
|
|
10
23
|
beginRigChangeVerificationAttempt,
|
|
24
|
+
claimRigVersionProviderImageBuild,
|
|
11
25
|
commitWarmingToWarm,
|
|
12
26
|
failWarmingToCold,
|
|
27
|
+
finalizeRigVersionProviderImageBuild,
|
|
13
28
|
getRig,
|
|
14
29
|
getRigChange,
|
|
15
30
|
getRigVersionById,
|
|
16
31
|
markWarmLeaseInstanceLost,
|
|
32
|
+
markSandboxCheckpointArtifactDeletePending,
|
|
17
33
|
recordWarmingSandboxCreated,
|
|
34
|
+
registerSandboxCheckpointArtifact,
|
|
18
35
|
releaseLeaseHolder,
|
|
19
|
-
sanitizeEventPayload,
|
|
20
|
-
sanitizeEventString,
|
|
21
|
-
sanitizeMemoryText,
|
|
22
36
|
touchLeaseHolder,
|
|
23
37
|
updateRigChangeStatus,
|
|
38
|
+
SandboxCheckpointArtifactRegistrationConflictError,
|
|
24
39
|
type Database,
|
|
25
40
|
} from "@opengeni/db";
|
|
26
41
|
import {
|
|
42
|
+
buildImmutableProviderImage,
|
|
27
43
|
createTurnToolCancellationController,
|
|
44
|
+
deleteModalCheckpointSnapshot,
|
|
45
|
+
describeNativeSnapshotArchive,
|
|
46
|
+
encodeNativeSnapshotRef,
|
|
28
47
|
establishSandboxSessionFromEnvelope,
|
|
29
|
-
runRigSetupHook,
|
|
30
48
|
sandboxCommandExitCode,
|
|
31
49
|
sandboxCommandOutput,
|
|
32
50
|
serializeEstablishedSandboxEnvelope,
|
|
51
|
+
providerSupportsImmutableImageBuild,
|
|
33
52
|
tagModalSandbox,
|
|
53
|
+
terminateManagedSandboxSession,
|
|
54
|
+
verifySandboxExecReadiness,
|
|
34
55
|
type EstablishedSandboxSession,
|
|
35
|
-
type
|
|
56
|
+
type TurnSandboxCommandArgs,
|
|
36
57
|
type TurnSandboxCommandSession,
|
|
37
|
-
} from "@opengeni/runtime";
|
|
58
|
+
} from "@opengeni/runtime/sandbox";
|
|
38
59
|
import type { Context } from "@temporalio/activity";
|
|
39
|
-
import type {
|
|
40
|
-
import {
|
|
60
|
+
import type { ControlActivityServices } from "./types";
|
|
61
|
+
import {
|
|
62
|
+
rigProviderImageSourceImage,
|
|
63
|
+
settingsWithPackSandboxImage,
|
|
64
|
+
settingsWithRigImage,
|
|
65
|
+
} from "./sandbox-images";
|
|
66
|
+
import { resolveWorkspacePackRuntime, type WorkspacePackRuntime } from "./packs";
|
|
41
67
|
import { currentActivityContext } from "./streaming";
|
|
42
68
|
|
|
69
|
+
type RigSetupHook = typeof import("@opengeni/runtime").runRigSetupHook;
|
|
70
|
+
let rigSetupHookPromise: Promise<RigSetupHook> | null = null;
|
|
71
|
+
|
|
72
|
+
async function runRigSetupHook(...args: Parameters<RigSetupHook>): Promise<void> {
|
|
73
|
+
rigSetupHookPromise ??= import("@opengeni/runtime").then((runtime) => runtime.runRigSetupHook);
|
|
74
|
+
const hook = await rigSetupHookPromise;
|
|
75
|
+
await hook(...args);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
type SandboxLifecycleCommandRunner = (
|
|
79
|
+
session: TurnSandboxCommandSession,
|
|
80
|
+
args: TurnSandboxCommandArgs,
|
|
81
|
+
) => Promise<unknown>;
|
|
82
|
+
|
|
43
83
|
export type RigVerificationWorkflowInput =
|
|
44
84
|
| { workspaceId: string; changeId: string; versionId?: never }
|
|
45
85
|
| { workspaceId: string; versionId: string; changeId?: never };
|
|
@@ -49,7 +89,6 @@ type CommandResult = {
|
|
|
49
89
|
output: string;
|
|
50
90
|
};
|
|
51
91
|
|
|
52
|
-
const OUTPUT_TAIL_LIMIT = 64 * 1024;
|
|
53
92
|
const RIG_VERIFICATION_CLEANUP_RESERVE_MAX_MS = 2 * 60_000;
|
|
54
93
|
const RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS = 60_000;
|
|
55
94
|
const RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS = 10_000;
|
|
@@ -245,18 +284,6 @@ function cleanupWaitMs(lifecycle: RigVerificationActivityLifecycle | undefined):
|
|
|
245
284
|
);
|
|
246
285
|
}
|
|
247
286
|
|
|
248
|
-
function tail(value: string, limit = OUTPUT_TAIL_LIMIT): string {
|
|
249
|
-
return value.length > limit ? value.slice(-limit) : value;
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
function scrubVerificationOutput(value: string): string {
|
|
253
|
-
return sanitizeMemoryText(sanitizeEventString(value)).text;
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
function scrubVerificationPayload<T>(value: T): T {
|
|
257
|
-
return sanitizeEventPayload(value);
|
|
258
|
-
}
|
|
259
|
-
|
|
260
287
|
function systemGrant(rig: Rig): AccessGrant {
|
|
261
288
|
return {
|
|
262
289
|
accountId: rig.accountId,
|
|
@@ -267,34 +294,13 @@ function systemGrant(rig: Rig): AccessGrant {
|
|
|
267
294
|
}
|
|
268
295
|
|
|
269
296
|
async function terminateThrowaway(established: EstablishedSandboxSession | null): Promise<boolean> {
|
|
270
|
-
if (!established)
|
|
271
|
-
return true;
|
|
272
|
-
}
|
|
273
|
-
const client = established.client as { delete?: (state: unknown) => Promise<unknown> };
|
|
274
|
-
if (typeof client.delete === "function" && established.sessionState !== undefined) {
|
|
275
|
-
try {
|
|
276
|
-
await client.delete(established.sessionState);
|
|
277
|
-
return true;
|
|
278
|
-
} catch {
|
|
279
|
-
return false;
|
|
280
|
-
}
|
|
281
|
-
}
|
|
282
|
-
const session = established.session as {
|
|
283
|
-
terminate?: () => Promise<unknown>;
|
|
284
|
-
kill?: () => Promise<unknown>;
|
|
285
|
-
close?: () => Promise<unknown>;
|
|
286
|
-
closed?: boolean;
|
|
287
|
-
};
|
|
297
|
+
if (!established) return true;
|
|
288
298
|
try {
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
await session.close();
|
|
295
|
-
} else {
|
|
296
|
-
return false;
|
|
297
|
-
}
|
|
299
|
+
await terminateManagedSandboxSession(
|
|
300
|
+
established.client,
|
|
301
|
+
established.sessionState,
|
|
302
|
+
established.session,
|
|
303
|
+
);
|
|
298
304
|
return true;
|
|
299
305
|
} catch {
|
|
300
306
|
return false;
|
|
@@ -336,6 +342,12 @@ const defaultOwnershipDependencies: RigVerificationOwnershipDependencies = {
|
|
|
336
342
|
export type RigVerificationSandboxRunContext = {
|
|
337
343
|
signal: AbortSignal;
|
|
338
344
|
commandRunner: SandboxLifecycleCommandRunner;
|
|
345
|
+
ownership: {
|
|
346
|
+
leaseId: string;
|
|
347
|
+
leaseEpoch: number;
|
|
348
|
+
workspaceGeneration: number;
|
|
349
|
+
instanceId: string;
|
|
350
|
+
};
|
|
339
351
|
};
|
|
340
352
|
|
|
341
353
|
export class RigVerificationLeaseUnavailableError extends Error {
|
|
@@ -365,9 +377,9 @@ export class RigVerificationLeaseUnavailableError extends Error {
|
|
|
365
377
|
*/
|
|
366
378
|
export async function runWithOwnedRigVerificationSandbox<T>(
|
|
367
379
|
input: {
|
|
368
|
-
settings:
|
|
380
|
+
settings: ControlActivityServices["settings"];
|
|
369
381
|
db: Database;
|
|
370
|
-
observability:
|
|
382
|
+
observability: ControlActivityServices["observability"];
|
|
371
383
|
accountId: string;
|
|
372
384
|
workspaceId: string;
|
|
373
385
|
sandboxGroupId: string;
|
|
@@ -493,7 +505,8 @@ export async function runWithOwnedRigVerificationSandbox<T>(
|
|
|
493
505
|
rigVersionId: input.rigVersionId,
|
|
494
506
|
leaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
|
|
495
507
|
warmingLeaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
|
|
496
|
-
captureWaitMs:
|
|
508
|
+
captureWaitMs: sandboxLifecycleTransitionWaitMs(input.settings),
|
|
509
|
+
waitSignal: signal,
|
|
497
510
|
});
|
|
498
511
|
acquired = true;
|
|
499
512
|
expectedEpoch = lease.lease.leaseEpoch;
|
|
@@ -587,6 +600,7 @@ export async function runWithOwnedRigVerificationSandbox<T>(
|
|
|
587
600
|
void establishmentPromise.catch(() => undefined);
|
|
588
601
|
const established = await waitForAbortable(establishmentPromise, signal);
|
|
589
602
|
cleanupTarget = established;
|
|
603
|
+
await waitForAbortable(verifySandboxExecReadiness(established), signal);
|
|
590
604
|
const resumeState =
|
|
591
605
|
established.backendId === "modal" ? null : await dependencies.serialize(established);
|
|
592
606
|
const committed = await dependencies.commitWarm(input.db, {
|
|
@@ -603,7 +617,16 @@ export async function runWithOwnedRigVerificationSandbox<T>(
|
|
|
603
617
|
if (!committed.committed || !committed.lease) {
|
|
604
618
|
throw new RigVerificationLeaseUnavailableError(input.sandboxGroupId, "fenced");
|
|
605
619
|
}
|
|
606
|
-
const result = await run(established, {
|
|
620
|
+
const result = await run(established, {
|
|
621
|
+
signal,
|
|
622
|
+
commandRunner,
|
|
623
|
+
ownership: {
|
|
624
|
+
leaseId: committed.lease.id,
|
|
625
|
+
leaseEpoch: committed.lease.leaseEpoch,
|
|
626
|
+
workspaceGeneration: committed.lease.workspaceGeneration,
|
|
627
|
+
instanceId: established.instanceId,
|
|
628
|
+
},
|
|
629
|
+
});
|
|
607
630
|
throwIfAborted(signal);
|
|
608
631
|
return result;
|
|
609
632
|
} catch (error) {
|
|
@@ -723,7 +746,7 @@ async function runCommand(
|
|
|
723
746
|
const result = await commandRunner(session, args);
|
|
724
747
|
return {
|
|
725
748
|
exitCode: sandboxCommandExitCode(result),
|
|
726
|
-
output:
|
|
749
|
+
output: sandboxCommandOutput(result),
|
|
727
750
|
};
|
|
728
751
|
}
|
|
729
752
|
|
|
@@ -737,7 +760,7 @@ function setupAppendCommand(change: RigChange): string | null {
|
|
|
737
760
|
|
|
738
761
|
function candidateVersionForChange(baseVersion: RigVersion, change: RigChange): RigVersion {
|
|
739
762
|
if (change.kind !== "definition_edit") {
|
|
740
|
-
return baseVersion;
|
|
763
|
+
return { ...baseVersion, providerImages: {} };
|
|
741
764
|
}
|
|
742
765
|
const payload = change.payload as {
|
|
743
766
|
image?: unknown;
|
|
@@ -764,7 +787,402 @@ function candidateVersionForChange(baseVersion: RigVersion, change: RigChange):
|
|
|
764
787
|
? (payload.defaultVariableSetIds as string[])
|
|
765
788
|
: baseVersion.defaultVariableSetIds,
|
|
766
789
|
changelog: typeof payload.changelog === "string" ? payload.changelog : baseVersion.changelog,
|
|
790
|
+
providerImages: {},
|
|
791
|
+
};
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
function providerImageDefinitionForChange(
|
|
795
|
+
baseVersion: RigVersion,
|
|
796
|
+
change: RigChange,
|
|
797
|
+
candidateVersion: RigVersion,
|
|
798
|
+
): RigProviderImageDefinition {
|
|
799
|
+
if (change.kind !== "setup_append") return candidateVersion;
|
|
800
|
+
const command = setupAppendCommand(change);
|
|
801
|
+
return {
|
|
802
|
+
image: baseVersion.image,
|
|
803
|
+
setupScript: command
|
|
804
|
+
? appendRigSetupCommand(baseVersion.setupScript, command)
|
|
805
|
+
: baseVersion.setupScript,
|
|
806
|
+
checks: baseVersion.checks,
|
|
807
|
+
credentialHooks: baseVersion.credentialHooks,
|
|
808
|
+
defaultVariableSetIds: baseVersion.defaultVariableSetIds,
|
|
809
|
+
};
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
export function rigProviderImageContentMarkerCommand(
|
|
813
|
+
contentHash: string,
|
|
814
|
+
markerRoot = "/var/opengeni",
|
|
815
|
+
): string {
|
|
816
|
+
const { marker, normalizedRoot } = rigProviderImageContentMarker(contentHash, markerRoot);
|
|
817
|
+
return `mkdir -p '${normalizedRoot}' && touch '${marker}'`;
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
function rigProviderImageContentMarker(
|
|
821
|
+
contentHash: string,
|
|
822
|
+
markerRoot = "/var/opengeni",
|
|
823
|
+
): { marker: string; normalizedRoot: string } {
|
|
824
|
+
if (!/^sha256:[0-9a-f]{64}$/u.test(contentHash)) {
|
|
825
|
+
throw new Error("Rig provider image content marker requires a canonical SHA-256 value");
|
|
826
|
+
}
|
|
827
|
+
if (!/^\/[A-Za-z0-9._/-]+$/u.test(markerRoot)) {
|
|
828
|
+
throw new Error("Rig provider image marker root must be a safe absolute path");
|
|
829
|
+
}
|
|
830
|
+
const normalizedRoot = markerRoot.replace(/\/+$/u, "") || "/";
|
|
831
|
+
const marker = `${normalizedRoot === "/" ? "" : normalizedRoot}/rig-setup-content-${contentHash.slice("sha256:".length)}.done`;
|
|
832
|
+
return { marker, normalizedRoot };
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
export type RigProviderImageColdBootDependencies = {
|
|
836
|
+
runOwnedSandbox: typeof runWithOwnedRigVerificationSandbox;
|
|
837
|
+
now: () => Date;
|
|
838
|
+
};
|
|
839
|
+
|
|
840
|
+
const defaultRigProviderImageColdBootDependencies: RigProviderImageColdBootDependencies = {
|
|
841
|
+
runOwnedSandbox: runWithOwnedRigVerificationSandbox,
|
|
842
|
+
now: () => new Date(),
|
|
843
|
+
};
|
|
844
|
+
|
|
845
|
+
/** Prove a provider image from a second clean sandbox before runtime selection. */
|
|
846
|
+
export async function verifyRigProviderImageColdBoot(
|
|
847
|
+
input: {
|
|
848
|
+
settings: ControlActivityServices["settings"];
|
|
849
|
+
db: Database;
|
|
850
|
+
observability: ControlActivityServices["observability"];
|
|
851
|
+
accountId: string;
|
|
852
|
+
workspaceId: string;
|
|
853
|
+
buildRequestId: string;
|
|
854
|
+
rigVersionId: string;
|
|
855
|
+
sessionIdPrefix: string;
|
|
856
|
+
imageId: string;
|
|
857
|
+
contentHash: string;
|
|
858
|
+
checks: RigProviderImageDefinition["checks"];
|
|
859
|
+
lifecycle: RigVerificationActivityLifecycle;
|
|
860
|
+
},
|
|
861
|
+
dependencies: RigProviderImageColdBootDependencies = defaultRigProviderImageColdBootDependencies,
|
|
862
|
+
): Promise<string> {
|
|
863
|
+
const { marker } = rigProviderImageContentMarker(input.contentHash);
|
|
864
|
+
await dependencies.runOwnedSandbox(
|
|
865
|
+
{
|
|
866
|
+
settings: {
|
|
867
|
+
...input.settings,
|
|
868
|
+
modalImageId: input.imageId,
|
|
869
|
+
// The full-machine provider image is the immutable rig base. Any future
|
|
870
|
+
// workspace recovery must layer /workspace onto it, never replace it.
|
|
871
|
+
modalWorkspacePersistence: "snapshot_directory",
|
|
872
|
+
},
|
|
873
|
+
db: input.db,
|
|
874
|
+
observability: input.observability,
|
|
875
|
+
accountId: input.accountId,
|
|
876
|
+
workspaceId: input.workspaceId,
|
|
877
|
+
sandboxGroupId: input.buildRequestId,
|
|
878
|
+
rigVersionId: input.rigVersionId,
|
|
879
|
+
sessionIdPrefix: input.sessionIdPrefix,
|
|
880
|
+
lifecycle: input.lifecycle,
|
|
881
|
+
},
|
|
882
|
+
async (established, runContext) => {
|
|
883
|
+
const markerResult = await runCommand(
|
|
884
|
+
established.session as TurnSandboxCommandSession,
|
|
885
|
+
`test -f '${marker}'`,
|
|
886
|
+
input.settings.rigSetupTimeoutMs,
|
|
887
|
+
runContext.commandRunner,
|
|
888
|
+
);
|
|
889
|
+
if (markerResult.exitCode !== 0) {
|
|
890
|
+
throw new Error("built provider image is missing its exact rig-content marker");
|
|
891
|
+
}
|
|
892
|
+
for (const check of input.checks) {
|
|
893
|
+
const result = await runCommand(
|
|
894
|
+
established.session as TurnSandboxCommandSession,
|
|
895
|
+
check.command,
|
|
896
|
+
input.settings.rigSetupTimeoutMs,
|
|
897
|
+
runContext.commandRunner,
|
|
898
|
+
);
|
|
899
|
+
if (result.exitCode !== 0) {
|
|
900
|
+
throw new Error(
|
|
901
|
+
`built provider image failed check ${JSON.stringify(check.name)}: ${result.output.slice(-2000)}`,
|
|
902
|
+
);
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
},
|
|
906
|
+
);
|
|
907
|
+
return dependencies.now().toISOString();
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
export async function buildVerifiedRigProviderImage(input: {
|
|
911
|
+
settings: ControlActivityServices["settings"];
|
|
912
|
+
db: Database;
|
|
913
|
+
observability: ControlActivityServices["observability"];
|
|
914
|
+
accountId: string;
|
|
915
|
+
workspaceId: string;
|
|
916
|
+
existingVersionId?: string;
|
|
917
|
+
definition: RigProviderImageDefinition;
|
|
918
|
+
target: { kind: "change" | "version"; id: string };
|
|
919
|
+
established: EstablishedSandboxSession;
|
|
920
|
+
ownership: RigVerificationSandboxRunContext["ownership"];
|
|
921
|
+
lifecycle: RigVerificationActivityLifecycle;
|
|
922
|
+
signal: AbortSignal;
|
|
923
|
+
}): Promise<RigProviderImage> {
|
|
924
|
+
const backend = input.settings.sandboxBackend as SandboxBackend;
|
|
925
|
+
const providerSupportsBuild = providerSupportsImmutableImageBuild(backend);
|
|
926
|
+
const sourceImage = rigProviderImageSourceImage(input.settings, backend);
|
|
927
|
+
const contentHash = rigProviderImageContentHash({
|
|
928
|
+
backend,
|
|
929
|
+
sourceImage,
|
|
930
|
+
definition: input.definition,
|
|
931
|
+
});
|
|
932
|
+
const startedAt = new Date().toISOString();
|
|
933
|
+
let building: RigProviderImage = {
|
|
934
|
+
backend,
|
|
935
|
+
provider: backend,
|
|
936
|
+
status: "building",
|
|
937
|
+
contentHash,
|
|
938
|
+
setupHash: rigProviderImageSetupHash(input.definition),
|
|
939
|
+
sourceImage,
|
|
940
|
+
buildRequestId: rigProviderImageBuildRequestId({
|
|
941
|
+
targetId: input.target.id,
|
|
942
|
+
backend,
|
|
943
|
+
contentHash,
|
|
944
|
+
}),
|
|
945
|
+
imageId: null,
|
|
946
|
+
imageDigest: null,
|
|
947
|
+
artifactId: null,
|
|
948
|
+
providerBindingKeyHash: null,
|
|
949
|
+
provenance: {
|
|
950
|
+
kind: "rig_verification",
|
|
951
|
+
targetKind: input.target.kind,
|
|
952
|
+
targetId: input.target.id,
|
|
953
|
+
},
|
|
954
|
+
startedAt,
|
|
955
|
+
finishedAt: null,
|
|
956
|
+
error: null,
|
|
957
|
+
};
|
|
958
|
+
|
|
959
|
+
if (input.existingVersionId) {
|
|
960
|
+
const claim = await claimRigVersionProviderImageBuild(input.db, {
|
|
961
|
+
workspaceId: input.workspaceId,
|
|
962
|
+
versionId: input.existingVersionId,
|
|
963
|
+
image: building,
|
|
964
|
+
staleAfterMs: RIG_VERIFICATION_OWNER_TTL_MS,
|
|
965
|
+
retryUnsupported: providerSupportsBuild,
|
|
966
|
+
});
|
|
967
|
+
if (
|
|
968
|
+
claim.status === "ready" ||
|
|
969
|
+
claim.status === "in_progress" ||
|
|
970
|
+
claim.status === "unsupported"
|
|
971
|
+
) {
|
|
972
|
+
return claim.image;
|
|
973
|
+
}
|
|
974
|
+
if (claim.status === "conflict") {
|
|
975
|
+
return {
|
|
976
|
+
...building,
|
|
977
|
+
status: "failed",
|
|
978
|
+
finishedAt: new Date().toISOString(),
|
|
979
|
+
error: {
|
|
980
|
+
code: "provider_image_content_conflict",
|
|
981
|
+
message:
|
|
982
|
+
"this exact rig version already records a provider image for different effective content; mint and verify a new version",
|
|
983
|
+
retryable: false,
|
|
984
|
+
},
|
|
985
|
+
};
|
|
986
|
+
}
|
|
987
|
+
building = claim.image;
|
|
988
|
+
}
|
|
989
|
+
|
|
990
|
+
const finalize = async (image: RigProviderImage): Promise<RigProviderImage> => {
|
|
991
|
+
if (input.existingVersionId) {
|
|
992
|
+
const persisted = await finalizeRigVersionProviderImageBuild(input.db, {
|
|
993
|
+
workspaceId: input.workspaceId,
|
|
994
|
+
versionId: input.existingVersionId,
|
|
995
|
+
image,
|
|
996
|
+
});
|
|
997
|
+
if (!persisted) {
|
|
998
|
+
if (image.artifactId) {
|
|
999
|
+
await markSandboxCheckpointArtifactDeletePending(input.db, {
|
|
1000
|
+
accountId: input.accountId,
|
|
1001
|
+
workspaceId: input.workspaceId,
|
|
1002
|
+
artifactId: image.artifactId,
|
|
1003
|
+
reason: "rig_provider_image_publication_fenced",
|
|
1004
|
+
}).catch(() => false);
|
|
1005
|
+
}
|
|
1006
|
+
return {
|
|
1007
|
+
...image,
|
|
1008
|
+
status: "failed",
|
|
1009
|
+
imageId: null,
|
|
1010
|
+
imageDigest: null,
|
|
1011
|
+
artifactId: null,
|
|
1012
|
+
providerBindingKeyHash: null,
|
|
1013
|
+
finishedAt: new Date().toISOString(),
|
|
1014
|
+
error: {
|
|
1015
|
+
code: "provider_image_publication_fenced",
|
|
1016
|
+
message: "provider image ownership changed before this build could be published",
|
|
1017
|
+
retryable: true,
|
|
1018
|
+
},
|
|
1019
|
+
};
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
return image;
|
|
767
1023
|
};
|
|
1024
|
+
|
|
1025
|
+
if (!providerSupportsBuild) {
|
|
1026
|
+
return await finalize({
|
|
1027
|
+
...building,
|
|
1028
|
+
status: "unsupported",
|
|
1029
|
+
finishedAt: new Date().toISOString(),
|
|
1030
|
+
error: {
|
|
1031
|
+
code: "provider_image_build_unsupported",
|
|
1032
|
+
message: `sandbox backend ${backend} does not support immutable provider image builds`,
|
|
1033
|
+
retryable: false,
|
|
1034
|
+
},
|
|
1035
|
+
});
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
throwIfAborted(input.signal);
|
|
1039
|
+
let builtIdentity: {
|
|
1040
|
+
imageId: string;
|
|
1041
|
+
providerBindingKey: string;
|
|
1042
|
+
} | null = null;
|
|
1043
|
+
let artifactId: string | null = null;
|
|
1044
|
+
let coldBootValidatedAt: string | null = null;
|
|
1045
|
+
try {
|
|
1046
|
+
const built = await buildImmutableProviderImage({
|
|
1047
|
+
backend,
|
|
1048
|
+
settings: input.settings,
|
|
1049
|
+
session: input.established.session,
|
|
1050
|
+
requestId: building.buildRequestId,
|
|
1051
|
+
timeoutMs: input.settings.sandboxSnapshotTimeoutMs,
|
|
1052
|
+
});
|
|
1053
|
+
if (!built || (!built.imageId && !built.imageDigest)) {
|
|
1054
|
+
throw new Error("provider image builder returned no immutable image identity");
|
|
1055
|
+
}
|
|
1056
|
+
if (backend === "modal") {
|
|
1057
|
+
if (!built.imageId || !built.providerBindingKey || !built.providerBinding) {
|
|
1058
|
+
throw new Error("Modal provider image build returned incomplete ownership identity");
|
|
1059
|
+
}
|
|
1060
|
+
builtIdentity = {
|
|
1061
|
+
imageId: built.imageId,
|
|
1062
|
+
providerBindingKey: built.providerBindingKey,
|
|
1063
|
+
};
|
|
1064
|
+
const archiveBytes = encodeNativeSnapshotRef({
|
|
1065
|
+
provider: "modal_snapshot_filesystem",
|
|
1066
|
+
snapshotId: built.imageId,
|
|
1067
|
+
workspacePersistence: "snapshot_filesystem",
|
|
1068
|
+
});
|
|
1069
|
+
const descriptor = describeNativeSnapshotArchive(archiveBytes);
|
|
1070
|
+
if (!descriptor) {
|
|
1071
|
+
throw new Error("Modal provider image build returned an invalid snapshot identity");
|
|
1072
|
+
}
|
|
1073
|
+
const artifact = await registerSandboxCheckpointArtifact(input.db, {
|
|
1074
|
+
accountId: input.accountId,
|
|
1075
|
+
workspaceId: input.workspaceId,
|
|
1076
|
+
sandboxGroupId: input.target.id,
|
|
1077
|
+
sourceLeaseId: input.ownership.leaseId,
|
|
1078
|
+
sourceLeaseEpoch: input.ownership.leaseEpoch,
|
|
1079
|
+
sourceInstanceId: input.ownership.instanceId,
|
|
1080
|
+
sourceWorkspaceGeneration: input.ownership.workspaceGeneration,
|
|
1081
|
+
providerBindingKey: built.providerBindingKey,
|
|
1082
|
+
providerBinding: built.providerBinding,
|
|
1083
|
+
workspaceArchive: Buffer.from(archiveBytes).toString("base64"),
|
|
1084
|
+
workspaceArchiveMeta: descriptor,
|
|
1085
|
+
registrationIdentity: "provider_object",
|
|
1086
|
+
});
|
|
1087
|
+
artifactId = artifact.id;
|
|
1088
|
+
|
|
1089
|
+
// A snapshot receipt is not proof that the artifact can cold-start. Boot
|
|
1090
|
+
// a second, independently owned sandbox from the exact image before it
|
|
1091
|
+
// can become runtime-selectable.
|
|
1092
|
+
coldBootValidatedAt = await verifyRigProviderImageColdBoot({
|
|
1093
|
+
settings: input.settings,
|
|
1094
|
+
db: input.db,
|
|
1095
|
+
observability: input.observability,
|
|
1096
|
+
accountId: input.accountId,
|
|
1097
|
+
workspaceId: input.workspaceId,
|
|
1098
|
+
buildRequestId: building.buildRequestId,
|
|
1099
|
+
rigVersionId: input.target.id,
|
|
1100
|
+
sessionIdPrefix: `rig-provider-image-${input.target.id}`,
|
|
1101
|
+
imageId: built.imageId,
|
|
1102
|
+
contentHash,
|
|
1103
|
+
checks: input.definition.checks,
|
|
1104
|
+
lifecycle: input.lifecycle,
|
|
1105
|
+
});
|
|
1106
|
+
}
|
|
1107
|
+
if (!coldBootValidatedAt) {
|
|
1108
|
+
throw new Error(`${backend} provider image build has no independent cold-boot validator`);
|
|
1109
|
+
}
|
|
1110
|
+
if (input.signal.aborted) {
|
|
1111
|
+
if (artifactId) {
|
|
1112
|
+
await markSandboxCheckpointArtifactDeletePending(input.db, {
|
|
1113
|
+
accountId: input.accountId,
|
|
1114
|
+
workspaceId: input.workspaceId,
|
|
1115
|
+
artifactId,
|
|
1116
|
+
reason: "rig_provider_image_build_cancelled",
|
|
1117
|
+
}).catch(() => false);
|
|
1118
|
+
}
|
|
1119
|
+
throw abortReason(input.signal);
|
|
1120
|
+
}
|
|
1121
|
+
return await finalize({
|
|
1122
|
+
...building,
|
|
1123
|
+
provider: built.provider,
|
|
1124
|
+
status: "ready",
|
|
1125
|
+
imageId: built.imageId,
|
|
1126
|
+
imageDigest: built.imageDigest,
|
|
1127
|
+
artifactId,
|
|
1128
|
+
providerBindingKeyHash: built.providerBindingKey
|
|
1129
|
+
? rigProviderImageProviderBindingKeyHash(built.providerBindingKey)
|
|
1130
|
+
: null,
|
|
1131
|
+
coldBootValidation: {
|
|
1132
|
+
version: 1,
|
|
1133
|
+
checkedAt: coldBootValidatedAt,
|
|
1134
|
+
},
|
|
1135
|
+
finishedAt: new Date().toISOString(),
|
|
1136
|
+
error: null,
|
|
1137
|
+
});
|
|
1138
|
+
} catch (error) {
|
|
1139
|
+
if (input.signal.aborted) throw abortReason(input.signal);
|
|
1140
|
+
if (artifactId) {
|
|
1141
|
+
await markSandboxCheckpointArtifactDeletePending(input.db, {
|
|
1142
|
+
accountId: input.accountId,
|
|
1143
|
+
workspaceId: input.workspaceId,
|
|
1144
|
+
artifactId,
|
|
1145
|
+
reason: "rig_provider_image_build_failed",
|
|
1146
|
+
}).catch(() => false);
|
|
1147
|
+
} else if (
|
|
1148
|
+
backend === "modal" &&
|
|
1149
|
+
builtIdentity &&
|
|
1150
|
+
!(error instanceof SandboxCheckpointArtifactRegistrationConflictError)
|
|
1151
|
+
) {
|
|
1152
|
+
await deleteModalCheckpointSnapshot(
|
|
1153
|
+
input.settings,
|
|
1154
|
+
builtIdentity.providerBindingKey,
|
|
1155
|
+
builtIdentity.imageId,
|
|
1156
|
+
).catch(() => "not_found" as const);
|
|
1157
|
+
}
|
|
1158
|
+
const message = (error instanceof Error ? error.message : String(error)).slice(0, 2000);
|
|
1159
|
+
return await finalize({
|
|
1160
|
+
...building,
|
|
1161
|
+
status: "failed",
|
|
1162
|
+
artifactId: null,
|
|
1163
|
+
finishedAt: new Date().toISOString(),
|
|
1164
|
+
error: {
|
|
1165
|
+
code: "provider_image_build_failed",
|
|
1166
|
+
message: message || "provider image build failed",
|
|
1167
|
+
retryable: true,
|
|
1168
|
+
},
|
|
1169
|
+
});
|
|
1170
|
+
}
|
|
1171
|
+
}
|
|
1172
|
+
|
|
1173
|
+
export function settingsForRigVerification(
|
|
1174
|
+
settings: ControlActivityServices["settings"],
|
|
1175
|
+
packRuntime: WorkspacePackRuntime,
|
|
1176
|
+
rigImage: string | null,
|
|
1177
|
+
): ControlActivityServices["settings"] {
|
|
1178
|
+
return settingsWithRigImage(
|
|
1179
|
+
settingsWithPackSandboxImage(
|
|
1180
|
+
settings,
|
|
1181
|
+
packRuntime.sandboxImage,
|
|
1182
|
+
packRuntime.sandboxProviderImages,
|
|
1183
|
+
),
|
|
1184
|
+
rigImage,
|
|
1185
|
+
);
|
|
768
1186
|
}
|
|
769
1187
|
|
|
770
1188
|
async function loadChangeTarget(
|
|
@@ -821,7 +1239,7 @@ async function withRigVerificationActivityLifecycle<T>(
|
|
|
821
1239
|
}
|
|
822
1240
|
}
|
|
823
1241
|
|
|
824
|
-
export function createRigVerificationActivities(services: () => Promise<
|
|
1242
|
+
export function createRigVerificationActivities(services: () => Promise<ControlActivityServices>) {
|
|
825
1243
|
return {
|
|
826
1244
|
verifyRigChange: (input: { workspaceId: string; changeId: string }) =>
|
|
827
1245
|
withRigVerificationActivityLifecycle(async (lifecycle) => {
|
|
@@ -848,7 +1266,22 @@ export function createRigVerificationActivities(services: () => Promise<Activity
|
|
|
848
1266
|
const verification: Record<string, unknown> = { startedAt, checkResults: [] };
|
|
849
1267
|
try {
|
|
850
1268
|
const candidateVersion = candidateVersionForChange(baseVersion, change);
|
|
851
|
-
const
|
|
1269
|
+
const providerImageDefinition = providerImageDefinitionForChange(
|
|
1270
|
+
baseVersion,
|
|
1271
|
+
change,
|
|
1272
|
+
candidateVersion,
|
|
1273
|
+
);
|
|
1274
|
+
const packRuntime = await resolveWorkspacePackRuntime(db, input.workspaceId);
|
|
1275
|
+
const runSettings = settingsForRigVerification(
|
|
1276
|
+
settings,
|
|
1277
|
+
packRuntime,
|
|
1278
|
+
candidateVersion.image,
|
|
1279
|
+
);
|
|
1280
|
+
const providerImageContentHash = rigProviderImageContentHash({
|
|
1281
|
+
backend: runSettings.sandboxBackend,
|
|
1282
|
+
sourceImage: rigProviderImageSourceImage(runSettings, runSettings.sandboxBackend),
|
|
1283
|
+
definition: providerImageDefinition,
|
|
1284
|
+
});
|
|
852
1285
|
return await runWithOwnedRigVerificationSandbox(
|
|
853
1286
|
{
|
|
854
1287
|
settings: runSettings,
|
|
@@ -873,6 +1306,7 @@ export function createRigVerificationActivities(services: () => Promise<Activity
|
|
|
873
1306
|
versionId: candidateVersion.id,
|
|
874
1307
|
script: candidateVersion.setupScript ?? "",
|
|
875
1308
|
timeoutMs: settings.rigSetupTimeoutMs,
|
|
1309
|
+
contentHash: providerImageContentHash,
|
|
876
1310
|
},
|
|
877
1311
|
});
|
|
878
1312
|
verification.setupResult = { exitCode: 0, output: "" };
|
|
@@ -891,7 +1325,7 @@ export function createRigVerificationActivities(services: () => Promise<Activity
|
|
|
891
1325
|
verification.passed = false;
|
|
892
1326
|
const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
893
1327
|
status: "rejected",
|
|
894
|
-
verification
|
|
1328
|
+
verification,
|
|
895
1329
|
});
|
|
896
1330
|
await recordRigAuditEvent(db, {
|
|
897
1331
|
grant,
|
|
@@ -908,6 +1342,17 @@ export function createRigVerificationActivities(services: () => Promise<Activity
|
|
|
908
1342
|
return updated;
|
|
909
1343
|
}
|
|
910
1344
|
}
|
|
1345
|
+
const markerResult = await runCommand(
|
|
1346
|
+
established.session as TurnSandboxCommandSession,
|
|
1347
|
+
rigProviderImageContentMarkerCommand(providerImageContentHash),
|
|
1348
|
+
settings.rigSetupTimeoutMs,
|
|
1349
|
+
runContext.commandRunner,
|
|
1350
|
+
);
|
|
1351
|
+
if (markerResult.exitCode !== 0) {
|
|
1352
|
+
throw new Error(
|
|
1353
|
+
`Failed to seal rig provider image content marker: ${markerResult.output.slice(-2000)}`,
|
|
1354
|
+
);
|
|
1355
|
+
}
|
|
911
1356
|
const checkResults = [];
|
|
912
1357
|
for (const check of candidateVersion.checks) {
|
|
913
1358
|
const result = await runCommand(
|
|
@@ -918,12 +1363,27 @@ export function createRigVerificationActivities(services: () => Promise<Activity
|
|
|
918
1363
|
);
|
|
919
1364
|
checkResults.push({
|
|
920
1365
|
name: check.name,
|
|
921
|
-
command:
|
|
1366
|
+
command: check.command,
|
|
922
1367
|
...result,
|
|
923
1368
|
});
|
|
924
1369
|
}
|
|
925
1370
|
verification.checkResults = checkResults;
|
|
926
1371
|
const passed = checkResults.every((result) => result.exitCode === 0);
|
|
1372
|
+
if (passed) {
|
|
1373
|
+
verification.providerImage = await buildVerifiedRigProviderImage({
|
|
1374
|
+
settings: runSettings,
|
|
1375
|
+
db,
|
|
1376
|
+
observability,
|
|
1377
|
+
accountId: rig.accountId,
|
|
1378
|
+
workspaceId: input.workspaceId,
|
|
1379
|
+
definition: providerImageDefinition,
|
|
1380
|
+
target: { kind: "change", id: change.id },
|
|
1381
|
+
established,
|
|
1382
|
+
ownership: runContext.ownership,
|
|
1383
|
+
lifecycle,
|
|
1384
|
+
signal: runContext.signal,
|
|
1385
|
+
});
|
|
1386
|
+
}
|
|
927
1387
|
verification.finishedAt = new Date().toISOString();
|
|
928
1388
|
verification.passed = passed;
|
|
929
1389
|
const classified = classifyRigVerificationOutcome({ kind: change.kind, passed });
|
|
@@ -935,7 +1395,7 @@ export function createRigVerificationActivities(services: () => Promise<Activity
|
|
|
935
1395
|
// (a second run could reject a change whose first verification passed).
|
|
936
1396
|
await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
937
1397
|
status: "verifying",
|
|
938
|
-
verification
|
|
1398
|
+
verification,
|
|
939
1399
|
});
|
|
940
1400
|
const { change: merged } = await promoteSetupAppendChange({ db }, grant, rig, {
|
|
941
1401
|
...change,
|
|
@@ -951,7 +1411,7 @@ export function createRigVerificationActivities(services: () => Promise<Activity
|
|
|
951
1411
|
}
|
|
952
1412
|
const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
953
1413
|
status: classified.status,
|
|
954
|
-
verification
|
|
1414
|
+
verification,
|
|
955
1415
|
});
|
|
956
1416
|
await recordRigAuditEvent(db, {
|
|
957
1417
|
grant,
|
|
@@ -973,12 +1433,10 @@ export function createRigVerificationActivities(services: () => Promise<Activity
|
|
|
973
1433
|
} catch (error) {
|
|
974
1434
|
verification.finishedAt = new Date().toISOString();
|
|
975
1435
|
verification.passed = false;
|
|
976
|
-
verification.error =
|
|
977
|
-
error instanceof Error ? error.message : String(error),
|
|
978
|
-
);
|
|
1436
|
+
verification.error = error instanceof Error ? error.message : String(error);
|
|
979
1437
|
const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
980
1438
|
status: "failed",
|
|
981
|
-
verification
|
|
1439
|
+
verification,
|
|
982
1440
|
});
|
|
983
1441
|
await recordRigAuditEvent(db, {
|
|
984
1442
|
grant,
|
|
@@ -1011,7 +1469,13 @@ export function createRigVerificationActivities(services: () => Promise<Activity
|
|
|
1011
1469
|
metadata: { versionId: version.id },
|
|
1012
1470
|
});
|
|
1013
1471
|
try {
|
|
1014
|
-
const
|
|
1472
|
+
const packRuntime = await resolveWorkspacePackRuntime(db, input.workspaceId);
|
|
1473
|
+
const runSettings = settingsForRigVerification(settings, packRuntime, version.image);
|
|
1474
|
+
const providerImageContentHash = rigProviderImageContentHash({
|
|
1475
|
+
backend: runSettings.sandboxBackend,
|
|
1476
|
+
sourceImage: rigProviderImageSourceImage(runSettings, runSettings.sandboxBackend),
|
|
1477
|
+
definition: version,
|
|
1478
|
+
});
|
|
1015
1479
|
return await runWithOwnedRigVerificationSandbox(
|
|
1016
1480
|
{
|
|
1017
1481
|
settings: runSettings,
|
|
@@ -1036,14 +1500,26 @@ export function createRigVerificationActivities(services: () => Promise<Activity
|
|
|
1036
1500
|
versionId: version.id,
|
|
1037
1501
|
script: version.setupScript ?? "",
|
|
1038
1502
|
timeoutMs: settings.rigSetupTimeoutMs,
|
|
1503
|
+
contentHash: providerImageContentHash,
|
|
1039
1504
|
},
|
|
1040
1505
|
});
|
|
1041
1506
|
}
|
|
1507
|
+
const markerResult = await runCommand(
|
|
1508
|
+
established.session as TurnSandboxCommandSession,
|
|
1509
|
+
rigProviderImageContentMarkerCommand(providerImageContentHash),
|
|
1510
|
+
settings.rigSetupTimeoutMs,
|
|
1511
|
+
runContext.commandRunner,
|
|
1512
|
+
);
|
|
1513
|
+
if (markerResult.exitCode !== 0) {
|
|
1514
|
+
throw new Error(
|
|
1515
|
+
`Failed to seal rig provider image content marker: ${markerResult.output.slice(-2000)}`,
|
|
1516
|
+
);
|
|
1517
|
+
}
|
|
1042
1518
|
const checkResults = [];
|
|
1043
1519
|
for (const check of version.checks) {
|
|
1044
1520
|
checkResults.push({
|
|
1045
1521
|
name: check.name,
|
|
1046
|
-
command:
|
|
1522
|
+
command: check.command,
|
|
1047
1523
|
...(await runCommand(
|
|
1048
1524
|
established.session as TurnSandboxCommandSession,
|
|
1049
1525
|
check.command,
|
|
@@ -1053,19 +1529,36 @@ export function createRigVerificationActivities(services: () => Promise<Activity
|
|
|
1053
1529
|
});
|
|
1054
1530
|
}
|
|
1055
1531
|
const passed = checkResults.every((result) => result.exitCode === 0);
|
|
1532
|
+
const providerImage = passed
|
|
1533
|
+
? await buildVerifiedRigProviderImage({
|
|
1534
|
+
settings: runSettings,
|
|
1535
|
+
db,
|
|
1536
|
+
observability,
|
|
1537
|
+
accountId: rig.accountId,
|
|
1538
|
+
workspaceId: input.workspaceId,
|
|
1539
|
+
existingVersionId: version.id,
|
|
1540
|
+
definition: version,
|
|
1541
|
+
target: { kind: "version", id: version.id },
|
|
1542
|
+
established,
|
|
1543
|
+
ownership: runContext.ownership,
|
|
1544
|
+
lifecycle,
|
|
1545
|
+
signal: runContext.signal,
|
|
1546
|
+
})
|
|
1547
|
+
: null;
|
|
1056
1548
|
await recordRigAuditEvent(db, {
|
|
1057
1549
|
grant,
|
|
1058
1550
|
action: passed ? "rig.verification.passed" : "rig.verification.failed",
|
|
1059
1551
|
rigId: rig.id,
|
|
1060
|
-
metadata:
|
|
1552
|
+
metadata: {
|
|
1061
1553
|
versionId: version.id,
|
|
1062
1554
|
startedAt,
|
|
1063
1555
|
finishedAt: new Date().toISOString(),
|
|
1064
1556
|
passed,
|
|
1065
1557
|
checkResults,
|
|
1066
|
-
|
|
1558
|
+
providerImage,
|
|
1559
|
+
},
|
|
1067
1560
|
});
|
|
1068
|
-
return { versionId: version.id, passed, checkResults };
|
|
1561
|
+
return { versionId: version.id, passed, checkResults, providerImage };
|
|
1069
1562
|
},
|
|
1070
1563
|
);
|
|
1071
1564
|
} catch (error) {
|
|
@@ -1073,21 +1566,18 @@ export function createRigVerificationActivities(services: () => Promise<Activity
|
|
|
1073
1566
|
// rig.verification.failed so activeVersionHealth reflects the failed
|
|
1074
1567
|
// re-run instead of staying stale, symmetric to verifyRigChange. Then
|
|
1075
1568
|
// rethrow so the Temporal activity still surfaces the failure.
|
|
1076
|
-
const detail =
|
|
1077
|
-
scrubVerificationOutput(error instanceof Error ? error.message : String(error)),
|
|
1078
|
-
4096,
|
|
1079
|
-
);
|
|
1569
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
1080
1570
|
await recordRigAuditEvent(db, {
|
|
1081
1571
|
grant,
|
|
1082
1572
|
action: "rig.verification.failed",
|
|
1083
1573
|
rigId: rig.id,
|
|
1084
|
-
metadata:
|
|
1574
|
+
metadata: {
|
|
1085
1575
|
versionId: version.id,
|
|
1086
1576
|
startedAt,
|
|
1087
1577
|
finishedAt: new Date().toISOString(),
|
|
1088
1578
|
passed: false,
|
|
1089
1579
|
error: detail,
|
|
1090
|
-
}
|
|
1580
|
+
},
|
|
1091
1581
|
});
|
|
1092
1582
|
throw error;
|
|
1093
1583
|
}
|