@opengeni/worker-bundle 0.16.21 → 0.17.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/dist/activities/agent-turn.d.ts +145 -32
  2. package/dist/activities/api-integrations.d.ts +21 -0
  3. package/dist/activities/codemode-dispatcher.d.ts +35 -0
  4. package/dist/activities/codemode-token-renewal.d.ts +23 -0
  5. package/dist/activities/codex-capacity.d.ts +5 -5
  6. package/dist/activities/codex-image-generation.d.ts +21 -0
  7. package/dist/activities/context-compaction.d.ts +3 -0
  8. package/dist/activities/documents.d.ts +3 -2
  9. package/dist/activities/environment.d.ts +11 -7
  10. package/dist/activities/expiring-material-renewal.d.ts +5 -3
  11. package/dist/activities/file-upload-reaper.d.ts +7 -2
  12. package/dist/activities/gateway-image-generation.d.ts +39 -0
  13. package/dist/activities/gateway-video-generation.d.ts +53 -0
  14. package/dist/activities/generated-images.d.ts +112 -0
  15. package/dist/activities/git-credential-renewal.d.ts +2 -2
  16. package/dist/activities/goals.d.ts +2 -2
  17. package/dist/activities/google-drive-changes.d.ts +99 -0
  18. package/dist/activities/image-generation-operation.d.ts +74 -0
  19. package/dist/activities/image-generation-references.d.ts +25 -0
  20. package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
  21. package/dist/activities/knowledge-source-sync.d.ts +75 -0
  22. package/dist/activities/packs.d.ts +13 -26
  23. package/dist/activities/parent-wake.d.ts +1 -1
  24. package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
  25. package/dist/activities/retained-screenshots.d.ts +129 -0
  26. package/dist/activities/rig-verification.d.ts +82 -5
  27. package/dist/activities/run-credential-renewal.d.ts +2 -2
  28. package/dist/activities/run-input.d.ts +5 -1
  29. package/dist/activities/sandbox-images.d.ts +14 -0
  30. package/dist/activities/sandbox-lease.d.ts +47 -9
  31. package/dist/activities/scheduled-tasks.d.ts +2 -2
  32. package/dist/activities/session-state.d.ts +4 -4
  33. package/dist/activities/streaming.d.ts +1 -1
  34. package/dist/activities/types.d.ts +71 -5
  35. package/dist/activities/video-generation-admission.d.ts +38 -0
  36. package/dist/activities/video-generation-credential.d.ts +7 -0
  37. package/dist/activities/video-generation-reconciliation.d.ts +13 -0
  38. package/dist/activities/video-media-validation.d.ts +34 -0
  39. package/dist/activities/video-output-retention.d.ts +36 -0
  40. package/dist/activities/video-reference-staging.d.ts +49 -0
  41. package/dist/activities/workflow-wake.d.ts +2 -2
  42. package/dist/activities/workspace-capture.d.ts +11 -1
  43. package/dist/activities-control-44KBWLSM.js +6885 -0
  44. package/dist/activities-control-44KBWLSM.js.map +1 -0
  45. package/dist/activities-control.d.ts +289 -0
  46. package/dist/activities-turn-K4CEDEGJ.js +14675 -0
  47. package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
  48. package/dist/activities-turn.d.ts +17 -0
  49. package/dist/activities.d.ts +130 -147
  50. package/dist/activity-services.d.ts +10 -0
  51. package/dist/artifact-materializer-entry.d.ts +6 -0
  52. package/dist/artifact-materializer-entry.js +2662 -0
  53. package/dist/artifact-materializer-entry.js.map +1 -0
  54. package/dist/artifact-outbox-entry.d.ts +2 -0
  55. package/dist/artifact-outbox-entry.js +681 -0
  56. package/dist/artifact-outbox-entry.js.map +1 -0
  57. package/dist/chunk-E3RI3I33.js +828 -0
  58. package/dist/chunk-E3RI3I33.js.map +1 -0
  59. package/dist/chunk-GRFQTTSQ.js +448 -0
  60. package/dist/chunk-GRFQTTSQ.js.map +1 -0
  61. package/dist/chunk-KKCRGAGZ.js +3091 -0
  62. package/dist/chunk-KKCRGAGZ.js.map +1 -0
  63. package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
  64. package/dist/chunk-RAMNYSAW.js.map +1 -0
  65. package/dist/concurrency.d.ts +108 -5
  66. package/dist/editable-artifact-hint-broker.d.ts +49 -0
  67. package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
  68. package/dist/editable-artifact-materializer-service.d.ts +108 -0
  69. package/dist/editable-artifact-materializer-storage.d.ts +12 -0
  70. package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
  71. package/dist/editable-artifact-materializer.d.ts +262 -0
  72. package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
  73. package/dist/editable-artifact-outbox-service.d.ts +70 -0
  74. package/dist/host-export.d.ts +1 -1
  75. package/dist/host-export.js +1 -1
  76. package/dist/index.d.ts +176 -10
  77. package/dist/index.js +675 -16460
  78. package/dist/index.js.map +1 -1
  79. package/dist/memory-pressure-guard.d.ts +57 -0
  80. package/dist/model-checkpoint-memory-collector.d.ts +22 -0
  81. package/dist/observability-metrics.d.ts +33 -0
  82. package/dist/sandbox-reaper-contract.d.ts +40 -0
  83. package/dist/sandbox-reaper-timeout.d.ts +14 -0
  84. package/dist/sandbox-resume.d.ts +40 -7
  85. package/dist/sandbox-routing.d.ts +13 -6
  86. package/dist/worker-service-lifecycle.d.ts +6 -2
  87. package/dist/workflow-bundle.js +1044 -363
  88. package/dist/workflows/activities.d.ts +7 -1
  89. package/dist/workflows/document-index.d.ts +1 -1
  90. package/dist/workflows/knowledge-source-sync.d.ts +13 -0
  91. package/dist/workflows/rig-verification.d.ts +1 -1
  92. package/dist/workflows/sandbox-reaper.d.ts +8 -0
  93. package/dist/workflows/scheduled-tasks.d.ts +1 -1
  94. package/dist/workflows/session.d.ts +2 -2
  95. package/dist/workflows/video-generation.d.ts +10 -0
  96. package/dist/workflows/workflow-wake.d.ts +1 -1
  97. package/dist/workflows.d.ts +9 -7
  98. package/package.json +34 -15
  99. package/src/activities/agent-turn.ts +1905 -559
  100. package/src/activities/api-integrations.ts +138 -0
  101. package/src/activities/codemode-dispatcher.ts +335 -0
  102. package/src/activities/codemode-token-renewal.ts +64 -0
  103. package/src/activities/codex-capacity.ts +16 -7
  104. package/src/activities/codex-image-generation.ts +56 -0
  105. package/src/activities/codex-rotation.ts +19 -5
  106. package/src/activities/context-compaction.ts +47 -6
  107. package/src/activities/documents.ts +14 -4
  108. package/src/activities/environment.ts +52 -60
  109. package/src/activities/expiring-material-renewal.ts +16 -7
  110. package/src/activities/file-upload-reaper.ts +34 -3
  111. package/src/activities/gateway-image-generation.ts +146 -0
  112. package/src/activities/gateway-video-generation.ts +298 -0
  113. package/src/activities/generated-images.ts +860 -0
  114. package/src/activities/git-credential-renewal.ts +3 -3
  115. package/src/activities/goals.ts +6 -2
  116. package/src/activities/google-drive-changes.ts +454 -0
  117. package/src/activities/image-generation-operation.ts +333 -0
  118. package/src/activities/image-generation-references.ts +118 -0
  119. package/src/activities/knowledge-source-sync-driver.ts +27 -0
  120. package/src/activities/knowledge-source-sync.ts +2840 -0
  121. package/src/activities/mcp-credentials.ts +1 -0
  122. package/src/activities/packs.ts +55 -55
  123. package/src/activities/recording.ts +4 -5
  124. package/src/activities/retained-screenshot-reaper.ts +202 -0
  125. package/src/activities/retained-screenshots.ts +1317 -0
  126. package/src/activities/rig-verification.ts +567 -77
  127. package/src/activities/run-credential-renewal.ts +9 -2
  128. package/src/activities/run-input.ts +40 -10
  129. package/src/activities/sandbox-images.ts +167 -0
  130. package/src/activities/sandbox-lease.ts +1065 -483
  131. package/src/activities/scheduled-tasks.ts +96 -7
  132. package/src/activities/session-state.ts +4 -4
  133. package/src/activities/streaming.ts +42 -3
  134. package/src/activities/types.ts +87 -5
  135. package/src/activities/video-generation-admission.ts +270 -0
  136. package/src/activities/video-generation-credential.ts +34 -0
  137. package/src/activities/video-generation-reconciliation.ts +578 -0
  138. package/src/activities/video-media-validation.ts +326 -0
  139. package/src/activities/video-output-retention.ts +314 -0
  140. package/src/activities/video-reference-staging.ts +263 -0
  141. package/src/activities/workflow-wake.ts +2 -2
  142. package/src/activities/workspace-capture.ts +77 -13
  143. package/src/activities-control.ts +54 -0
  144. package/src/activities-turn.ts +40 -0
  145. package/src/activities.ts +50 -118
  146. package/src/activity-services.ts +62 -0
  147. package/src/artifact-materializer-entry.ts +35 -0
  148. package/src/artifact-outbox-entry.ts +29 -0
  149. package/src/concurrency.ts +484 -6
  150. package/src/editable-artifact-hint-broker.ts +235 -0
  151. package/src/editable-artifact-materializer-launcher.ts +515 -0
  152. package/src/editable-artifact-materializer-service.ts +787 -0
  153. package/src/editable-artifact-materializer-storage.ts +413 -0
  154. package/src/editable-artifact-materializer-subprocess.ts +956 -0
  155. package/src/editable-artifact-materializer.ts +1289 -0
  156. package/src/editable-artifact-outbox-dispatcher.ts +244 -0
  157. package/src/editable-artifact-outbox-service.ts +438 -0
  158. package/src/host-export-pump.ts +28 -11
  159. package/src/http.ts +3 -6
  160. package/src/index.ts +277 -72
  161. package/src/memory-pressure-guard.ts +489 -0
  162. package/src/model-checkpoint-memory-collector.ts +116 -0
  163. package/src/observability-metrics.ts +202 -7
  164. package/src/sandbox-reaper-contract.ts +73 -0
  165. package/src/sandbox-reaper-timeout.ts +79 -0
  166. package/src/sandbox-resume.ts +331 -157
  167. package/src/sandbox-routing.ts +49 -39
  168. package/src/worker-service-lifecycle.ts +82 -12
  169. package/src/workflows/activities.ts +30 -0
  170. package/src/workflows/file-upload-reaper.ts +6 -1
  171. package/src/workflows/knowledge-source-sync.ts +83 -0
  172. package/src/workflows/sandbox-reaper.ts +183 -21
  173. package/src/workflows/scheduled-tasks.ts +42 -6
  174. package/src/workflows/video-generation.ts +36 -0
  175. package/src/workflows.ts +17 -1
  176. package/dist/activities/redaction.d.ts +0 -1
  177. package/dist/activities/toolspace-token-renewal.d.ts +0 -23
  178. package/dist/chunk-BJUW5LCM.js.map +0 -1
  179. package/src/activities/redaction.ts +0 -9
  180. package/src/activities/toolspace-token-renewal.ts +0 -57
@@ -1,45 +1,85 @@
1
- import type { AccessGrant, Rig, RigChange, RigVersion } from "@opengeni/contracts";
2
- import { sandboxArchiveCaptureTimeoutMs } from "@opengeni/config";
1
+ import type {
2
+ AccessGrant,
3
+ Rig,
4
+ RigChange,
5
+ RigProviderImage,
6
+ RigVersion,
7
+ SandboxBackend,
8
+ } from "@opengeni/contracts";
9
+ import { sandboxLifecycleTransitionWaitMs } from "@opengeni/config";
3
10
  import {
11
+ appendRigSetupCommand,
4
12
  recordRigAuditEvent,
5
13
  classifyRigVerificationOutcome,
6
14
  promoteSetupAppendChange,
15
+ rigProviderImageBuildRequestId,
16
+ rigProviderImageContentHash,
17
+ rigProviderImageProviderBindingKeyHash,
18
+ rigProviderImageSetupHash,
19
+ type RigProviderImageDefinition,
7
20
  } from "@opengeni/core";
8
21
  import {
9
22
  acquireLease,
10
23
  beginRigChangeVerificationAttempt,
24
+ claimRigVersionProviderImageBuild,
11
25
  commitWarmingToWarm,
12
26
  failWarmingToCold,
27
+ finalizeRigVersionProviderImageBuild,
13
28
  getRig,
14
29
  getRigChange,
15
30
  getRigVersionById,
16
31
  markWarmLeaseInstanceLost,
32
+ markSandboxCheckpointArtifactDeletePending,
17
33
  recordWarmingSandboxCreated,
34
+ registerSandboxCheckpointArtifact,
18
35
  releaseLeaseHolder,
19
- sanitizeEventPayload,
20
- sanitizeEventString,
21
- sanitizeMemoryText,
22
36
  touchLeaseHolder,
23
37
  updateRigChangeStatus,
38
+ SandboxCheckpointArtifactRegistrationConflictError,
24
39
  type Database,
25
40
  } from "@opengeni/db";
26
41
  import {
42
+ buildImmutableProviderImage,
27
43
  createTurnToolCancellationController,
44
+ deleteModalCheckpointSnapshot,
45
+ describeNativeSnapshotArchive,
46
+ encodeNativeSnapshotRef,
28
47
  establishSandboxSessionFromEnvelope,
29
- runRigSetupHook,
30
48
  sandboxCommandExitCode,
31
49
  sandboxCommandOutput,
32
50
  serializeEstablishedSandboxEnvelope,
51
+ providerSupportsImmutableImageBuild,
33
52
  tagModalSandbox,
53
+ terminateManagedSandboxSession,
54
+ verifySandboxExecReadiness,
34
55
  type EstablishedSandboxSession,
35
- type SandboxLifecycleCommandRunner,
56
+ type TurnSandboxCommandArgs,
36
57
  type TurnSandboxCommandSession,
37
- } from "@opengeni/runtime";
58
+ } from "@opengeni/runtime/sandbox";
38
59
  import type { Context } from "@temporalio/activity";
39
- import type { ActivityServices } from "./types";
40
- import { settingsWithRigImage } from "./packs";
60
+ import type { ControlActivityServices } from "./types";
61
+ import {
62
+ rigProviderImageSourceImage,
63
+ settingsWithPackSandboxImage,
64
+ settingsWithRigImage,
65
+ } from "./sandbox-images";
66
+ import { resolveWorkspacePackRuntime, type WorkspacePackRuntime } from "./packs";
41
67
  import { currentActivityContext } from "./streaming";
42
68
 
69
+ type RigSetupHook = typeof import("@opengeni/runtime").runRigSetupHook;
70
+ let rigSetupHookPromise: Promise<RigSetupHook> | null = null;
71
+
72
+ async function runRigSetupHook(...args: Parameters<RigSetupHook>): Promise<void> {
73
+ rigSetupHookPromise ??= import("@opengeni/runtime").then((runtime) => runtime.runRigSetupHook);
74
+ const hook = await rigSetupHookPromise;
75
+ await hook(...args);
76
+ }
77
+
78
+ type SandboxLifecycleCommandRunner = (
79
+ session: TurnSandboxCommandSession,
80
+ args: TurnSandboxCommandArgs,
81
+ ) => Promise<unknown>;
82
+
43
83
  export type RigVerificationWorkflowInput =
44
84
  | { workspaceId: string; changeId: string; versionId?: never }
45
85
  | { workspaceId: string; versionId: string; changeId?: never };
@@ -49,7 +89,6 @@ type CommandResult = {
49
89
  output: string;
50
90
  };
51
91
 
52
- const OUTPUT_TAIL_LIMIT = 64 * 1024;
53
92
  const RIG_VERIFICATION_CLEANUP_RESERVE_MAX_MS = 2 * 60_000;
54
93
  const RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS = 60_000;
55
94
  const RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS = 10_000;
@@ -245,18 +284,6 @@ function cleanupWaitMs(lifecycle: RigVerificationActivityLifecycle | undefined):
245
284
  );
246
285
  }
247
286
 
248
- function tail(value: string, limit = OUTPUT_TAIL_LIMIT): string {
249
- return value.length > limit ? value.slice(-limit) : value;
250
- }
251
-
252
- function scrubVerificationOutput(value: string): string {
253
- return sanitizeMemoryText(sanitizeEventString(value)).text;
254
- }
255
-
256
- function scrubVerificationPayload<T>(value: T): T {
257
- return sanitizeEventPayload(value);
258
- }
259
-
260
287
  function systemGrant(rig: Rig): AccessGrant {
261
288
  return {
262
289
  accountId: rig.accountId,
@@ -267,34 +294,13 @@ function systemGrant(rig: Rig): AccessGrant {
267
294
  }
268
295
 
269
296
  async function terminateThrowaway(established: EstablishedSandboxSession | null): Promise<boolean> {
270
- if (!established) {
271
- return true;
272
- }
273
- const client = established.client as { delete?: (state: unknown) => Promise<unknown> };
274
- if (typeof client.delete === "function" && established.sessionState !== undefined) {
275
- try {
276
- await client.delete(established.sessionState);
277
- return true;
278
- } catch {
279
- return false;
280
- }
281
- }
282
- const session = established.session as {
283
- terminate?: () => Promise<unknown>;
284
- kill?: () => Promise<unknown>;
285
- close?: () => Promise<unknown>;
286
- closed?: boolean;
287
- };
297
+ if (!established) return true;
288
298
  try {
289
- if (session.terminate) {
290
- await session.terminate();
291
- } else if (session.kill) {
292
- await session.kill();
293
- } else if (session.close && !session.closed) {
294
- await session.close();
295
- } else {
296
- return false;
297
- }
299
+ await terminateManagedSandboxSession(
300
+ established.client,
301
+ established.sessionState,
302
+ established.session,
303
+ );
298
304
  return true;
299
305
  } catch {
300
306
  return false;
@@ -336,6 +342,12 @@ const defaultOwnershipDependencies: RigVerificationOwnershipDependencies = {
336
342
  export type RigVerificationSandboxRunContext = {
337
343
  signal: AbortSignal;
338
344
  commandRunner: SandboxLifecycleCommandRunner;
345
+ ownership: {
346
+ leaseId: string;
347
+ leaseEpoch: number;
348
+ workspaceGeneration: number;
349
+ instanceId: string;
350
+ };
339
351
  };
340
352
 
341
353
  export class RigVerificationLeaseUnavailableError extends Error {
@@ -365,9 +377,9 @@ export class RigVerificationLeaseUnavailableError extends Error {
365
377
  */
366
378
  export async function runWithOwnedRigVerificationSandbox<T>(
367
379
  input: {
368
- settings: ActivityServices["settings"];
380
+ settings: ControlActivityServices["settings"];
369
381
  db: Database;
370
- observability: ActivityServices["observability"];
382
+ observability: ControlActivityServices["observability"];
371
383
  accountId: string;
372
384
  workspaceId: string;
373
385
  sandboxGroupId: string;
@@ -493,7 +505,8 @@ export async function runWithOwnedRigVerificationSandbox<T>(
493
505
  rigVersionId: input.rigVersionId,
494
506
  leaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
495
507
  warmingLeaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
496
- captureWaitMs: sandboxArchiveCaptureTimeoutMs(input.settings),
508
+ captureWaitMs: sandboxLifecycleTransitionWaitMs(input.settings),
509
+ waitSignal: signal,
497
510
  });
498
511
  acquired = true;
499
512
  expectedEpoch = lease.lease.leaseEpoch;
@@ -587,6 +600,7 @@ export async function runWithOwnedRigVerificationSandbox<T>(
587
600
  void establishmentPromise.catch(() => undefined);
588
601
  const established = await waitForAbortable(establishmentPromise, signal);
589
602
  cleanupTarget = established;
603
+ await waitForAbortable(verifySandboxExecReadiness(established), signal);
590
604
  const resumeState =
591
605
  established.backendId === "modal" ? null : await dependencies.serialize(established);
592
606
  const committed = await dependencies.commitWarm(input.db, {
@@ -603,7 +617,16 @@ export async function runWithOwnedRigVerificationSandbox<T>(
603
617
  if (!committed.committed || !committed.lease) {
604
618
  throw new RigVerificationLeaseUnavailableError(input.sandboxGroupId, "fenced");
605
619
  }
606
- const result = await run(established, { signal, commandRunner });
620
+ const result = await run(established, {
621
+ signal,
622
+ commandRunner,
623
+ ownership: {
624
+ leaseId: committed.lease.id,
625
+ leaseEpoch: committed.lease.leaseEpoch,
626
+ workspaceGeneration: committed.lease.workspaceGeneration,
627
+ instanceId: established.instanceId,
628
+ },
629
+ });
607
630
  throwIfAborted(signal);
608
631
  return result;
609
632
  } catch (error) {
@@ -723,7 +746,7 @@ async function runCommand(
723
746
  const result = await commandRunner(session, args);
724
747
  return {
725
748
  exitCode: sandboxCommandExitCode(result),
726
- output: scrubVerificationOutput(tail(sandboxCommandOutput(result))),
749
+ output: sandboxCommandOutput(result),
727
750
  };
728
751
  }
729
752
 
@@ -737,7 +760,7 @@ function setupAppendCommand(change: RigChange): string | null {
737
760
 
738
761
  function candidateVersionForChange(baseVersion: RigVersion, change: RigChange): RigVersion {
739
762
  if (change.kind !== "definition_edit") {
740
- return baseVersion;
763
+ return { ...baseVersion, providerImages: {} };
741
764
  }
742
765
  const payload = change.payload as {
743
766
  image?: unknown;
@@ -764,7 +787,402 @@ function candidateVersionForChange(baseVersion: RigVersion, change: RigChange):
764
787
  ? (payload.defaultVariableSetIds as string[])
765
788
  : baseVersion.defaultVariableSetIds,
766
789
  changelog: typeof payload.changelog === "string" ? payload.changelog : baseVersion.changelog,
790
+ providerImages: {},
791
+ };
792
+ }
793
+
794
+ function providerImageDefinitionForChange(
795
+ baseVersion: RigVersion,
796
+ change: RigChange,
797
+ candidateVersion: RigVersion,
798
+ ): RigProviderImageDefinition {
799
+ if (change.kind !== "setup_append") return candidateVersion;
800
+ const command = setupAppendCommand(change);
801
+ return {
802
+ image: baseVersion.image,
803
+ setupScript: command
804
+ ? appendRigSetupCommand(baseVersion.setupScript, command)
805
+ : baseVersion.setupScript,
806
+ checks: baseVersion.checks,
807
+ credentialHooks: baseVersion.credentialHooks,
808
+ defaultVariableSetIds: baseVersion.defaultVariableSetIds,
809
+ };
810
+ }
811
+
812
+ export function rigProviderImageContentMarkerCommand(
813
+ contentHash: string,
814
+ markerRoot = "/var/opengeni",
815
+ ): string {
816
+ const { marker, normalizedRoot } = rigProviderImageContentMarker(contentHash, markerRoot);
817
+ return `mkdir -p '${normalizedRoot}' && touch '${marker}'`;
818
+ }
819
+
820
+ function rigProviderImageContentMarker(
821
+ contentHash: string,
822
+ markerRoot = "/var/opengeni",
823
+ ): { marker: string; normalizedRoot: string } {
824
+ if (!/^sha256:[0-9a-f]{64}$/u.test(contentHash)) {
825
+ throw new Error("Rig provider image content marker requires a canonical SHA-256 value");
826
+ }
827
+ if (!/^\/[A-Za-z0-9._/-]+$/u.test(markerRoot)) {
828
+ throw new Error("Rig provider image marker root must be a safe absolute path");
829
+ }
830
+ const normalizedRoot = markerRoot.replace(/\/+$/u, "") || "/";
831
+ const marker = `${normalizedRoot === "/" ? "" : normalizedRoot}/rig-setup-content-${contentHash.slice("sha256:".length)}.done`;
832
+ return { marker, normalizedRoot };
833
+ }
834
+
835
+ export type RigProviderImageColdBootDependencies = {
836
+ runOwnedSandbox: typeof runWithOwnedRigVerificationSandbox;
837
+ now: () => Date;
838
+ };
839
+
840
+ const defaultRigProviderImageColdBootDependencies: RigProviderImageColdBootDependencies = {
841
+ runOwnedSandbox: runWithOwnedRigVerificationSandbox,
842
+ now: () => new Date(),
843
+ };
844
+
845
+ /** Prove a provider image from a second clean sandbox before runtime selection. */
846
+ export async function verifyRigProviderImageColdBoot(
847
+ input: {
848
+ settings: ControlActivityServices["settings"];
849
+ db: Database;
850
+ observability: ControlActivityServices["observability"];
851
+ accountId: string;
852
+ workspaceId: string;
853
+ buildRequestId: string;
854
+ rigVersionId: string;
855
+ sessionIdPrefix: string;
856
+ imageId: string;
857
+ contentHash: string;
858
+ checks: RigProviderImageDefinition["checks"];
859
+ lifecycle: RigVerificationActivityLifecycle;
860
+ },
861
+ dependencies: RigProviderImageColdBootDependencies = defaultRigProviderImageColdBootDependencies,
862
+ ): Promise<string> {
863
+ const { marker } = rigProviderImageContentMarker(input.contentHash);
864
+ await dependencies.runOwnedSandbox(
865
+ {
866
+ settings: {
867
+ ...input.settings,
868
+ modalImageId: input.imageId,
869
+ // The full-machine provider image is the immutable rig base. Any future
870
+ // workspace recovery must layer /workspace onto it, never replace it.
871
+ modalWorkspacePersistence: "snapshot_directory",
872
+ },
873
+ db: input.db,
874
+ observability: input.observability,
875
+ accountId: input.accountId,
876
+ workspaceId: input.workspaceId,
877
+ sandboxGroupId: input.buildRequestId,
878
+ rigVersionId: input.rigVersionId,
879
+ sessionIdPrefix: input.sessionIdPrefix,
880
+ lifecycle: input.lifecycle,
881
+ },
882
+ async (established, runContext) => {
883
+ const markerResult = await runCommand(
884
+ established.session as TurnSandboxCommandSession,
885
+ `test -f '${marker}'`,
886
+ input.settings.rigSetupTimeoutMs,
887
+ runContext.commandRunner,
888
+ );
889
+ if (markerResult.exitCode !== 0) {
890
+ throw new Error("built provider image is missing its exact rig-content marker");
891
+ }
892
+ for (const check of input.checks) {
893
+ const result = await runCommand(
894
+ established.session as TurnSandboxCommandSession,
895
+ check.command,
896
+ input.settings.rigSetupTimeoutMs,
897
+ runContext.commandRunner,
898
+ );
899
+ if (result.exitCode !== 0) {
900
+ throw new Error(
901
+ `built provider image failed check ${JSON.stringify(check.name)}: ${result.output.slice(-2000)}`,
902
+ );
903
+ }
904
+ }
905
+ },
906
+ );
907
+ return dependencies.now().toISOString();
908
+ }
909
+
910
+ export async function buildVerifiedRigProviderImage(input: {
911
+ settings: ControlActivityServices["settings"];
912
+ db: Database;
913
+ observability: ControlActivityServices["observability"];
914
+ accountId: string;
915
+ workspaceId: string;
916
+ existingVersionId?: string;
917
+ definition: RigProviderImageDefinition;
918
+ target: { kind: "change" | "version"; id: string };
919
+ established: EstablishedSandboxSession;
920
+ ownership: RigVerificationSandboxRunContext["ownership"];
921
+ lifecycle: RigVerificationActivityLifecycle;
922
+ signal: AbortSignal;
923
+ }): Promise<RigProviderImage> {
924
+ const backend = input.settings.sandboxBackend as SandboxBackend;
925
+ const providerSupportsBuild = providerSupportsImmutableImageBuild(backend);
926
+ const sourceImage = rigProviderImageSourceImage(input.settings, backend);
927
+ const contentHash = rigProviderImageContentHash({
928
+ backend,
929
+ sourceImage,
930
+ definition: input.definition,
931
+ });
932
+ const startedAt = new Date().toISOString();
933
+ let building: RigProviderImage = {
934
+ backend,
935
+ provider: backend,
936
+ status: "building",
937
+ contentHash,
938
+ setupHash: rigProviderImageSetupHash(input.definition),
939
+ sourceImage,
940
+ buildRequestId: rigProviderImageBuildRequestId({
941
+ targetId: input.target.id,
942
+ backend,
943
+ contentHash,
944
+ }),
945
+ imageId: null,
946
+ imageDigest: null,
947
+ artifactId: null,
948
+ providerBindingKeyHash: null,
949
+ provenance: {
950
+ kind: "rig_verification",
951
+ targetKind: input.target.kind,
952
+ targetId: input.target.id,
953
+ },
954
+ startedAt,
955
+ finishedAt: null,
956
+ error: null,
957
+ };
958
+
959
+ if (input.existingVersionId) {
960
+ const claim = await claimRigVersionProviderImageBuild(input.db, {
961
+ workspaceId: input.workspaceId,
962
+ versionId: input.existingVersionId,
963
+ image: building,
964
+ staleAfterMs: RIG_VERIFICATION_OWNER_TTL_MS,
965
+ retryUnsupported: providerSupportsBuild,
966
+ });
967
+ if (
968
+ claim.status === "ready" ||
969
+ claim.status === "in_progress" ||
970
+ claim.status === "unsupported"
971
+ ) {
972
+ return claim.image;
973
+ }
974
+ if (claim.status === "conflict") {
975
+ return {
976
+ ...building,
977
+ status: "failed",
978
+ finishedAt: new Date().toISOString(),
979
+ error: {
980
+ code: "provider_image_content_conflict",
981
+ message:
982
+ "this exact rig version already records a provider image for different effective content; mint and verify a new version",
983
+ retryable: false,
984
+ },
985
+ };
986
+ }
987
+ building = claim.image;
988
+ }
989
+
990
+ const finalize = async (image: RigProviderImage): Promise<RigProviderImage> => {
991
+ if (input.existingVersionId) {
992
+ const persisted = await finalizeRigVersionProviderImageBuild(input.db, {
993
+ workspaceId: input.workspaceId,
994
+ versionId: input.existingVersionId,
995
+ image,
996
+ });
997
+ if (!persisted) {
998
+ if (image.artifactId) {
999
+ await markSandboxCheckpointArtifactDeletePending(input.db, {
1000
+ accountId: input.accountId,
1001
+ workspaceId: input.workspaceId,
1002
+ artifactId: image.artifactId,
1003
+ reason: "rig_provider_image_publication_fenced",
1004
+ }).catch(() => false);
1005
+ }
1006
+ return {
1007
+ ...image,
1008
+ status: "failed",
1009
+ imageId: null,
1010
+ imageDigest: null,
1011
+ artifactId: null,
1012
+ providerBindingKeyHash: null,
1013
+ finishedAt: new Date().toISOString(),
1014
+ error: {
1015
+ code: "provider_image_publication_fenced",
1016
+ message: "provider image ownership changed before this build could be published",
1017
+ retryable: true,
1018
+ },
1019
+ };
1020
+ }
1021
+ }
1022
+ return image;
767
1023
  };
1024
+
1025
+ if (!providerSupportsBuild) {
1026
+ return await finalize({
1027
+ ...building,
1028
+ status: "unsupported",
1029
+ finishedAt: new Date().toISOString(),
1030
+ error: {
1031
+ code: "provider_image_build_unsupported",
1032
+ message: `sandbox backend ${backend} does not support immutable provider image builds`,
1033
+ retryable: false,
1034
+ },
1035
+ });
1036
+ }
1037
+
1038
+ throwIfAborted(input.signal);
1039
+ let builtIdentity: {
1040
+ imageId: string;
1041
+ providerBindingKey: string;
1042
+ } | null = null;
1043
+ let artifactId: string | null = null;
1044
+ let coldBootValidatedAt: string | null = null;
1045
+ try {
1046
+ const built = await buildImmutableProviderImage({
1047
+ backend,
1048
+ settings: input.settings,
1049
+ session: input.established.session,
1050
+ requestId: building.buildRequestId,
1051
+ timeoutMs: input.settings.sandboxSnapshotTimeoutMs,
1052
+ });
1053
+ if (!built || (!built.imageId && !built.imageDigest)) {
1054
+ throw new Error("provider image builder returned no immutable image identity");
1055
+ }
1056
+ if (backend === "modal") {
1057
+ if (!built.imageId || !built.providerBindingKey || !built.providerBinding) {
1058
+ throw new Error("Modal provider image build returned incomplete ownership identity");
1059
+ }
1060
+ builtIdentity = {
1061
+ imageId: built.imageId,
1062
+ providerBindingKey: built.providerBindingKey,
1063
+ };
1064
+ const archiveBytes = encodeNativeSnapshotRef({
1065
+ provider: "modal_snapshot_filesystem",
1066
+ snapshotId: built.imageId,
1067
+ workspacePersistence: "snapshot_filesystem",
1068
+ });
1069
+ const descriptor = describeNativeSnapshotArchive(archiveBytes);
1070
+ if (!descriptor) {
1071
+ throw new Error("Modal provider image build returned an invalid snapshot identity");
1072
+ }
1073
+ const artifact = await registerSandboxCheckpointArtifact(input.db, {
1074
+ accountId: input.accountId,
1075
+ workspaceId: input.workspaceId,
1076
+ sandboxGroupId: input.target.id,
1077
+ sourceLeaseId: input.ownership.leaseId,
1078
+ sourceLeaseEpoch: input.ownership.leaseEpoch,
1079
+ sourceInstanceId: input.ownership.instanceId,
1080
+ sourceWorkspaceGeneration: input.ownership.workspaceGeneration,
1081
+ providerBindingKey: built.providerBindingKey,
1082
+ providerBinding: built.providerBinding,
1083
+ workspaceArchive: Buffer.from(archiveBytes).toString("base64"),
1084
+ workspaceArchiveMeta: descriptor,
1085
+ registrationIdentity: "provider_object",
1086
+ });
1087
+ artifactId = artifact.id;
1088
+
1089
+ // A snapshot receipt is not proof that the artifact can cold-start. Boot
1090
+ // a second, independently owned sandbox from the exact image before it
1091
+ // can become runtime-selectable.
1092
+ coldBootValidatedAt = await verifyRigProviderImageColdBoot({
1093
+ settings: input.settings,
1094
+ db: input.db,
1095
+ observability: input.observability,
1096
+ accountId: input.accountId,
1097
+ workspaceId: input.workspaceId,
1098
+ buildRequestId: building.buildRequestId,
1099
+ rigVersionId: input.target.id,
1100
+ sessionIdPrefix: `rig-provider-image-${input.target.id}`,
1101
+ imageId: built.imageId,
1102
+ contentHash,
1103
+ checks: input.definition.checks,
1104
+ lifecycle: input.lifecycle,
1105
+ });
1106
+ }
1107
+ if (!coldBootValidatedAt) {
1108
+ throw new Error(`${backend} provider image build has no independent cold-boot validator`);
1109
+ }
1110
+ if (input.signal.aborted) {
1111
+ if (artifactId) {
1112
+ await markSandboxCheckpointArtifactDeletePending(input.db, {
1113
+ accountId: input.accountId,
1114
+ workspaceId: input.workspaceId,
1115
+ artifactId,
1116
+ reason: "rig_provider_image_build_cancelled",
1117
+ }).catch(() => false);
1118
+ }
1119
+ throw abortReason(input.signal);
1120
+ }
1121
+ return await finalize({
1122
+ ...building,
1123
+ provider: built.provider,
1124
+ status: "ready",
1125
+ imageId: built.imageId,
1126
+ imageDigest: built.imageDigest,
1127
+ artifactId,
1128
+ providerBindingKeyHash: built.providerBindingKey
1129
+ ? rigProviderImageProviderBindingKeyHash(built.providerBindingKey)
1130
+ : null,
1131
+ coldBootValidation: {
1132
+ version: 1,
1133
+ checkedAt: coldBootValidatedAt,
1134
+ },
1135
+ finishedAt: new Date().toISOString(),
1136
+ error: null,
1137
+ });
1138
+ } catch (error) {
1139
+ if (input.signal.aborted) throw abortReason(input.signal);
1140
+ if (artifactId) {
1141
+ await markSandboxCheckpointArtifactDeletePending(input.db, {
1142
+ accountId: input.accountId,
1143
+ workspaceId: input.workspaceId,
1144
+ artifactId,
1145
+ reason: "rig_provider_image_build_failed",
1146
+ }).catch(() => false);
1147
+ } else if (
1148
+ backend === "modal" &&
1149
+ builtIdentity &&
1150
+ !(error instanceof SandboxCheckpointArtifactRegistrationConflictError)
1151
+ ) {
1152
+ await deleteModalCheckpointSnapshot(
1153
+ input.settings,
1154
+ builtIdentity.providerBindingKey,
1155
+ builtIdentity.imageId,
1156
+ ).catch(() => "not_found" as const);
1157
+ }
1158
+ const message = (error instanceof Error ? error.message : String(error)).slice(0, 2000);
1159
+ return await finalize({
1160
+ ...building,
1161
+ status: "failed",
1162
+ artifactId: null,
1163
+ finishedAt: new Date().toISOString(),
1164
+ error: {
1165
+ code: "provider_image_build_failed",
1166
+ message: message || "provider image build failed",
1167
+ retryable: true,
1168
+ },
1169
+ });
1170
+ }
1171
+ }
1172
+
1173
+ export function settingsForRigVerification(
1174
+ settings: ControlActivityServices["settings"],
1175
+ packRuntime: WorkspacePackRuntime,
1176
+ rigImage: string | null,
1177
+ ): ControlActivityServices["settings"] {
1178
+ return settingsWithRigImage(
1179
+ settingsWithPackSandboxImage(
1180
+ settings,
1181
+ packRuntime.sandboxImage,
1182
+ packRuntime.sandboxProviderImages,
1183
+ ),
1184
+ rigImage,
1185
+ );
768
1186
  }
769
1187
 
770
1188
  async function loadChangeTarget(
@@ -821,7 +1239,7 @@ async function withRigVerificationActivityLifecycle<T>(
821
1239
  }
822
1240
  }
823
1241
 
824
- export function createRigVerificationActivities(services: () => Promise<ActivityServices>) {
1242
+ export function createRigVerificationActivities(services: () => Promise<ControlActivityServices>) {
825
1243
  return {
826
1244
  verifyRigChange: (input: { workspaceId: string; changeId: string }) =>
827
1245
  withRigVerificationActivityLifecycle(async (lifecycle) => {
@@ -848,7 +1266,22 @@ export function createRigVerificationActivities(services: () => Promise<Activity
848
1266
  const verification: Record<string, unknown> = { startedAt, checkResults: [] };
849
1267
  try {
850
1268
  const candidateVersion = candidateVersionForChange(baseVersion, change);
851
- const runSettings = settingsWithRigImage(settings, candidateVersion.image);
1269
+ const providerImageDefinition = providerImageDefinitionForChange(
1270
+ baseVersion,
1271
+ change,
1272
+ candidateVersion,
1273
+ );
1274
+ const packRuntime = await resolveWorkspacePackRuntime(db, input.workspaceId);
1275
+ const runSettings = settingsForRigVerification(
1276
+ settings,
1277
+ packRuntime,
1278
+ candidateVersion.image,
1279
+ );
1280
+ const providerImageContentHash = rigProviderImageContentHash({
1281
+ backend: runSettings.sandboxBackend,
1282
+ sourceImage: rigProviderImageSourceImage(runSettings, runSettings.sandboxBackend),
1283
+ definition: providerImageDefinition,
1284
+ });
852
1285
  return await runWithOwnedRigVerificationSandbox(
853
1286
  {
854
1287
  settings: runSettings,
@@ -873,6 +1306,7 @@ export function createRigVerificationActivities(services: () => Promise<Activity
873
1306
  versionId: candidateVersion.id,
874
1307
  script: candidateVersion.setupScript ?? "",
875
1308
  timeoutMs: settings.rigSetupTimeoutMs,
1309
+ contentHash: providerImageContentHash,
876
1310
  },
877
1311
  });
878
1312
  verification.setupResult = { exitCode: 0, output: "" };
@@ -891,7 +1325,7 @@ export function createRigVerificationActivities(services: () => Promise<Activity
891
1325
  verification.passed = false;
892
1326
  const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
893
1327
  status: "rejected",
894
- verification: scrubVerificationPayload(verification),
1328
+ verification,
895
1329
  });
896
1330
  await recordRigAuditEvent(db, {
897
1331
  grant,
@@ -908,6 +1342,17 @@ export function createRigVerificationActivities(services: () => Promise<Activity
908
1342
  return updated;
909
1343
  }
910
1344
  }
1345
+ const markerResult = await runCommand(
1346
+ established.session as TurnSandboxCommandSession,
1347
+ rigProviderImageContentMarkerCommand(providerImageContentHash),
1348
+ settings.rigSetupTimeoutMs,
1349
+ runContext.commandRunner,
1350
+ );
1351
+ if (markerResult.exitCode !== 0) {
1352
+ throw new Error(
1353
+ `Failed to seal rig provider image content marker: ${markerResult.output.slice(-2000)}`,
1354
+ );
1355
+ }
911
1356
  const checkResults = [];
912
1357
  for (const check of candidateVersion.checks) {
913
1358
  const result = await runCommand(
@@ -918,12 +1363,27 @@ export function createRigVerificationActivities(services: () => Promise<Activity
918
1363
  );
919
1364
  checkResults.push({
920
1365
  name: check.name,
921
- command: scrubVerificationOutput(check.command),
1366
+ command: check.command,
922
1367
  ...result,
923
1368
  });
924
1369
  }
925
1370
  verification.checkResults = checkResults;
926
1371
  const passed = checkResults.every((result) => result.exitCode === 0);
1372
+ if (passed) {
1373
+ verification.providerImage = await buildVerifiedRigProviderImage({
1374
+ settings: runSettings,
1375
+ db,
1376
+ observability,
1377
+ accountId: rig.accountId,
1378
+ workspaceId: input.workspaceId,
1379
+ definition: providerImageDefinition,
1380
+ target: { kind: "change", id: change.id },
1381
+ established,
1382
+ ownership: runContext.ownership,
1383
+ lifecycle,
1384
+ signal: runContext.signal,
1385
+ });
1386
+ }
927
1387
  verification.finishedAt = new Date().toISOString();
928
1388
  verification.passed = passed;
929
1389
  const classified = classifyRigVerificationOutcome({ kind: change.kind, passed });
@@ -935,7 +1395,7 @@ export function createRigVerificationActivities(services: () => Promise<Activity
935
1395
  // (a second run could reject a change whose first verification passed).
936
1396
  await updateRigChangeStatus(db, input.workspaceId, change.id, {
937
1397
  status: "verifying",
938
- verification: scrubVerificationPayload(verification),
1398
+ verification,
939
1399
  });
940
1400
  const { change: merged } = await promoteSetupAppendChange({ db }, grant, rig, {
941
1401
  ...change,
@@ -951,7 +1411,7 @@ export function createRigVerificationActivities(services: () => Promise<Activity
951
1411
  }
952
1412
  const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
953
1413
  status: classified.status,
954
- verification: scrubVerificationPayload(verification),
1414
+ verification,
955
1415
  });
956
1416
  await recordRigAuditEvent(db, {
957
1417
  grant,
@@ -973,12 +1433,10 @@ export function createRigVerificationActivities(services: () => Promise<Activity
973
1433
  } catch (error) {
974
1434
  verification.finishedAt = new Date().toISOString();
975
1435
  verification.passed = false;
976
- verification.error = scrubVerificationOutput(
977
- error instanceof Error ? error.message : String(error),
978
- );
1436
+ verification.error = error instanceof Error ? error.message : String(error);
979
1437
  const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
980
1438
  status: "failed",
981
- verification: scrubVerificationPayload(verification),
1439
+ verification,
982
1440
  });
983
1441
  await recordRigAuditEvent(db, {
984
1442
  grant,
@@ -1011,7 +1469,13 @@ export function createRigVerificationActivities(services: () => Promise<Activity
1011
1469
  metadata: { versionId: version.id },
1012
1470
  });
1013
1471
  try {
1014
- const runSettings = settingsWithRigImage(settings, version.image);
1472
+ const packRuntime = await resolveWorkspacePackRuntime(db, input.workspaceId);
1473
+ const runSettings = settingsForRigVerification(settings, packRuntime, version.image);
1474
+ const providerImageContentHash = rigProviderImageContentHash({
1475
+ backend: runSettings.sandboxBackend,
1476
+ sourceImage: rigProviderImageSourceImage(runSettings, runSettings.sandboxBackend),
1477
+ definition: version,
1478
+ });
1015
1479
  return await runWithOwnedRigVerificationSandbox(
1016
1480
  {
1017
1481
  settings: runSettings,
@@ -1036,14 +1500,26 @@ export function createRigVerificationActivities(services: () => Promise<Activity
1036
1500
  versionId: version.id,
1037
1501
  script: version.setupScript ?? "",
1038
1502
  timeoutMs: settings.rigSetupTimeoutMs,
1503
+ contentHash: providerImageContentHash,
1039
1504
  },
1040
1505
  });
1041
1506
  }
1507
+ const markerResult = await runCommand(
1508
+ established.session as TurnSandboxCommandSession,
1509
+ rigProviderImageContentMarkerCommand(providerImageContentHash),
1510
+ settings.rigSetupTimeoutMs,
1511
+ runContext.commandRunner,
1512
+ );
1513
+ if (markerResult.exitCode !== 0) {
1514
+ throw new Error(
1515
+ `Failed to seal rig provider image content marker: ${markerResult.output.slice(-2000)}`,
1516
+ );
1517
+ }
1042
1518
  const checkResults = [];
1043
1519
  for (const check of version.checks) {
1044
1520
  checkResults.push({
1045
1521
  name: check.name,
1046
- command: scrubVerificationOutput(check.command),
1522
+ command: check.command,
1047
1523
  ...(await runCommand(
1048
1524
  established.session as TurnSandboxCommandSession,
1049
1525
  check.command,
@@ -1053,19 +1529,36 @@ export function createRigVerificationActivities(services: () => Promise<Activity
1053
1529
  });
1054
1530
  }
1055
1531
  const passed = checkResults.every((result) => result.exitCode === 0);
1532
+ const providerImage = passed
1533
+ ? await buildVerifiedRigProviderImage({
1534
+ settings: runSettings,
1535
+ db,
1536
+ observability,
1537
+ accountId: rig.accountId,
1538
+ workspaceId: input.workspaceId,
1539
+ existingVersionId: version.id,
1540
+ definition: version,
1541
+ target: { kind: "version", id: version.id },
1542
+ established,
1543
+ ownership: runContext.ownership,
1544
+ lifecycle,
1545
+ signal: runContext.signal,
1546
+ })
1547
+ : null;
1056
1548
  await recordRigAuditEvent(db, {
1057
1549
  grant,
1058
1550
  action: passed ? "rig.verification.passed" : "rig.verification.failed",
1059
1551
  rigId: rig.id,
1060
- metadata: scrubVerificationPayload({
1552
+ metadata: {
1061
1553
  versionId: version.id,
1062
1554
  startedAt,
1063
1555
  finishedAt: new Date().toISOString(),
1064
1556
  passed,
1065
1557
  checkResults,
1066
- }),
1558
+ providerImage,
1559
+ },
1067
1560
  });
1068
- return { versionId: version.id, passed, checkResults };
1561
+ return { versionId: version.id, passed, checkResults, providerImage };
1069
1562
  },
1070
1563
  );
1071
1564
  } catch (error) {
@@ -1073,21 +1566,18 @@ export function createRigVerificationActivities(services: () => Promise<Activity
1073
1566
  // rig.verification.failed so activeVersionHealth reflects the failed
1074
1567
  // re-run instead of staying stale, symmetric to verifyRigChange. Then
1075
1568
  // rethrow so the Temporal activity still surfaces the failure.
1076
- const detail = tail(
1077
- scrubVerificationOutput(error instanceof Error ? error.message : String(error)),
1078
- 4096,
1079
- );
1569
+ const detail = error instanceof Error ? error.message : String(error);
1080
1570
  await recordRigAuditEvent(db, {
1081
1571
  grant,
1082
1572
  action: "rig.verification.failed",
1083
1573
  rigId: rig.id,
1084
- metadata: scrubVerificationPayload({
1574
+ metadata: {
1085
1575
  versionId: version.id,
1086
1576
  startedAt,
1087
1577
  finishedAt: new Date().toISOString(),
1088
1578
  passed: false,
1089
1579
  error: detail,
1090
- }),
1580
+ },
1091
1581
  });
1092
1582
  throw error;
1093
1583
  }