@opengeni/worker-bundle 0.17.2 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/dist/activities/agent-run-admission.d.ts +12 -0
  2. package/dist/activities/agent-turn.d.ts +25 -0
  3. package/dist/activities/browser-state-artifact-reaper.d.ts +29 -0
  4. package/dist/activities/capabilities.d.ts +6 -0
  5. package/dist/activities/codex-capacity.d.ts +1 -0
  6. package/dist/activities/environment.d.ts +7 -1
  7. package/dist/activities/packs.d.ts +8 -10
  8. package/dist/activities/session-state.d.ts +4 -2
  9. package/dist/activities/site-auth-maintenance.d.ts +27 -0
  10. package/dist/activities/types.d.ts +16 -0
  11. package/dist/activities/video-generation-admission.d.ts +9 -4
  12. package/dist/activities/video-generation-credential.d.ts +17 -0
  13. package/dist/activities/video-reference-staging.d.ts +7 -1
  14. package/dist/activities/xai-auth.d.ts +23 -0
  15. package/dist/activities/xai-image-generation.d.ts +26 -0
  16. package/dist/activities/xai-video-generation.d.ts +18 -0
  17. package/dist/{activities-control-44KBWLSM.js → activities-control-XURE4VKN.js} +387 -51
  18. package/dist/activities-control-XURE4VKN.js.map +1 -0
  19. package/dist/activities-control.d.ts +8 -0
  20. package/dist/{activities-turn-K4CEDEGJ.js → activities-turn-6EBXGGLS.js} +5312 -3951
  21. package/dist/activities-turn-6EBXGGLS.js.map +1 -0
  22. package/dist/activities.d.ts +9 -1
  23. package/dist/{chunk-KKCRGAGZ.js → chunk-5GRJIP5H.js} +99 -98
  24. package/dist/chunk-5GRJIP5H.js.map +1 -0
  25. package/dist/{chunk-E3RI3I33.js → chunk-TYNPCG4U.js} +88 -10
  26. package/dist/chunk-TYNPCG4U.js.map +1 -0
  27. package/dist/index.d.ts +13 -0
  28. package/dist/index.js +49 -2
  29. package/dist/index.js.map +1 -1
  30. package/dist/model-checkpoint-memory-collector.d.ts +1 -0
  31. package/dist/model-context-contributions.d.ts +40 -0
  32. package/dist/observability-metrics.d.ts +20 -1
  33. package/dist/sandbox-routing.d.ts +4 -0
  34. package/dist/workflow-bundle.js +59 -15
  35. package/dist/workflows/activities.d.ts +1 -1
  36. package/dist/workflows/site-auth-maintenance.d.ts +2 -0
  37. package/dist/workflows.d.ts +1 -0
  38. package/package.json +16 -15
  39. package/src/activities/agent-run-admission.ts +76 -0
  40. package/src/activities/agent-turn.ts +1178 -142
  41. package/src/activities/browser-state-artifact-reaper.ts +120 -0
  42. package/src/activities/capabilities.ts +10 -0
  43. package/src/activities/codex-capacity.ts +48 -1
  44. package/src/activities/environment.ts +14 -8
  45. package/src/activities/goals.ts +2 -1
  46. package/src/activities/packs.ts +45 -122
  47. package/src/activities/parent-wake.ts +13 -1
  48. package/src/activities/scheduled-tasks.ts +30 -73
  49. package/src/activities/session-state.ts +18 -0
  50. package/src/activities/site-auth-maintenance.ts +269 -0
  51. package/src/activities/types.ts +17 -0
  52. package/src/activities/video-generation-admission.ts +37 -4
  53. package/src/activities/video-generation-credential.ts +84 -6
  54. package/src/activities/video-generation-reconciliation.ts +212 -19
  55. package/src/activities/video-media-validation.ts +8 -2
  56. package/src/activities/video-reference-staging.ts +121 -34
  57. package/src/activities/xai-auth.ts +122 -0
  58. package/src/activities/xai-image-generation.ts +86 -0
  59. package/src/activities/xai-video-generation.ts +69 -0
  60. package/src/activities-control.ts +4 -0
  61. package/src/activities.ts +5 -0
  62. package/src/index.ts +51 -0
  63. package/src/model-checkpoint-memory-collector.ts +8 -1
  64. package/src/model-context-contributions.ts +159 -0
  65. package/src/observability-metrics.ts +98 -9
  66. package/src/sandbox-routing.ts +13 -0
  67. package/src/workflows/activities.ts +1 -0
  68. package/src/workflows/file-upload-reaper.ts +8 -2
  69. package/src/workflows/session.ts +20 -6
  70. package/src/workflows/site-auth-maintenance.ts +14 -0
  71. package/src/workflows.ts +1 -0
  72. package/dist/activities-control-44KBWLSM.js.map +0 -1
  73. package/dist/activities-turn-K4CEDEGJ.js.map +0 -1
  74. package/dist/chunk-E3RI3I33.js.map +0 -1
  75. package/dist/chunk-KKCRGAGZ.js.map +0 -1
@@ -1,16 +1,51 @@
1
1
  import { decryptEnvironmentValue, encryptEnvironmentValue } from "@opengeni/db";
2
+ import type { XaiProviderAccountAuthoritySnapshotV1 } from "@opengeni/contracts";
3
+
4
+ export type VideoGenerationProviderCredential =
5
+ | Readonly<{ kind: "api-key"; apiKey: string }>
6
+ | Readonly<{
7
+ kind: "xai-subscription";
8
+ accessToken: string;
9
+ refreshToken: string;
10
+ userId: string;
11
+ credentialId: string;
12
+ subjectId: string;
13
+ authoritySnapshot: XaiProviderAccountAuthoritySnapshotV1;
14
+ }>;
2
15
 
3
16
  /** One frozen, provider-neutral credential envelope for every video funding route. */
4
17
  export function encryptVideoGenerationApiKey(key: Uint8Array, apiKey: string): string {
5
18
  if (!apiKey.trim()) throw new Error("Video provider credential is empty");
6
- return encryptEnvironmentValue(key, JSON.stringify({ apiKey }));
19
+ return encryptEnvironmentValue(key, JSON.stringify({ kind: "api-key", apiKey }));
20
+ }
21
+
22
+ export function encryptVideoGenerationXaiCredential(
23
+ key: Uint8Array,
24
+ credential: Omit<
25
+ Extract<VideoGenerationProviderCredential, { kind: "xai-subscription" }>,
26
+ "kind"
27
+ >,
28
+ ): string {
29
+ if (
30
+ !credential.accessToken.trim() ||
31
+ !credential.refreshToken.trim() ||
32
+ !credential.userId.trim() ||
33
+ !credential.credentialId.trim() ||
34
+ !credential.subjectId.trim()
35
+ ) {
36
+ throw new Error("SuperGrok video credential is incomplete");
37
+ }
38
+ return encryptEnvironmentValue(key, JSON.stringify({ kind: "xai-subscription", ...credential }));
7
39
  }
8
40
 
9
41
  /**
10
42
  * Decrypt without ever attaching parser/decryption errors: JSON parser errors
11
43
  * may quote plaintext and must not enter Temporal failure payloads or logs.
12
44
  */
13
- export function decryptVideoGenerationApiKey(key: Uint8Array, stored: string): string {
45
+ export function decryptVideoGenerationCredential(
46
+ key: Uint8Array,
47
+ stored: string,
48
+ ): VideoGenerationProviderCredential {
14
49
  let plaintext: string;
15
50
  try {
16
51
  plaintext = decryptEnvironmentValue(key, stored);
@@ -23,12 +58,55 @@ export function decryptVideoGenerationApiKey(key: Uint8Array, stored: string): s
23
58
  } catch {
24
59
  throw new Error("Video provider credential lease is malformed");
25
60
  }
26
- const apiKey =
61
+ const row =
27
62
  decoded && typeof decoded === "object" && !Array.isArray(decoded)
28
- ? (decoded as Record<string, unknown>).apiKey
63
+ ? (decoded as Record<string, unknown>)
29
64
  : null;
30
- if (typeof apiKey !== "string" || !apiKey.trim()) {
65
+ if (row?.kind === "api-key" || (row?.kind === undefined && typeof row?.apiKey === "string")) {
66
+ if (typeof row.apiKey !== "string" || !row.apiKey.trim()) {
67
+ throw new Error("Video provider credential lease is malformed");
68
+ }
69
+ return Object.freeze({ kind: "api-key", apiKey: row.apiKey });
70
+ }
71
+ const authority =
72
+ row?.authoritySnapshot &&
73
+ typeof row.authoritySnapshot === "object" &&
74
+ !Array.isArray(row.authoritySnapshot)
75
+ ? (row.authoritySnapshot as Record<string, unknown>)
76
+ : null;
77
+ if (
78
+ row?.kind !== "xai-subscription" ||
79
+ typeof row.accessToken !== "string" ||
80
+ !row.accessToken.trim() ||
81
+ typeof row.refreshToken !== "string" ||
82
+ !row.refreshToken.trim() ||
83
+ typeof row.userId !== "string" ||
84
+ !row.userId.trim() ||
85
+ typeof row.credentialId !== "string" ||
86
+ !row.credentialId.trim() ||
87
+ typeof row.subjectId !== "string" ||
88
+ !row.subjectId.trim() ||
89
+ authority?.version !== 1 ||
90
+ (authority.scope !== "workspace" && authority.scope !== "user") ||
91
+ (authority.scope === "user" && !Number.isSafeInteger(authority.authorityGeneration))
92
+ ) {
31
93
  throw new Error("Video provider credential lease is malformed");
32
94
  }
33
- return apiKey;
95
+ return Object.freeze({
96
+ kind: "xai-subscription",
97
+ accessToken: row.accessToken,
98
+ refreshToken: row.refreshToken,
99
+ userId: row.userId,
100
+ credentialId: row.credentialId,
101
+ subjectId: row.subjectId,
102
+ authoritySnapshot: authority as XaiProviderAccountAuthoritySnapshotV1,
103
+ });
104
+ }
105
+
106
+ export function decryptVideoGenerationApiKey(key: Uint8Array, stored: string): string {
107
+ const credential = decryptVideoGenerationCredential(key, stored);
108
+ if (credential.kind !== "api-key") {
109
+ throw new Error("Video provider credential is not an API key");
110
+ }
111
+ return credential.apiKey;
34
112
  }
@@ -11,12 +11,16 @@ import {
11
11
  markVideoGenerationRetaining,
12
12
  markVideoGenerationSubmissionIntent,
13
13
  markVideoGenerationSubmissionUncertain,
14
+ materializeXaiCredentialForRun,
15
+ refreshXaiSubscriptionCredentialSerialized,
16
+ rotateVideoGenerationCredential,
14
17
  markVideoGenerationTerminalUpdate,
15
18
  markVideoGenerationTerminalUpdateInTransaction,
16
19
  mediaGenerationResultForStoredOperation,
17
20
  rescheduleVideoGenerationOperation,
18
21
  settleVideoGenerationFailure,
19
22
  settleVideoGenerationReady,
23
+ type XaiCredentialForRun,
20
24
  type VideoGenerationOperationWithReferences,
21
25
  } from "@opengeni/db";
22
26
  import { publishDurableSessionEvents } from "@opengeni/events";
@@ -32,7 +36,22 @@ import {
32
36
  downloadGeneratedVideoToVerifiedTemp,
33
37
  retainVerifiedGeneratedVideo,
34
38
  } from "./video-output-retention";
35
- import { decryptVideoGenerationApiKey } from "./video-generation-credential";
39
+ import {
40
+ decryptVideoGenerationCredential,
41
+ encryptVideoGenerationXaiCredential,
42
+ type VideoGenerationProviderCredential,
43
+ } from "./video-generation-credential";
44
+ import {
45
+ refreshXaiToken,
46
+ XaiSubscriptionError,
47
+ xaiAccessTokenExpiry,
48
+ type XaiSubscriptionRequestContext,
49
+ } from "@opengeni/xai-subscription";
50
+ import {
51
+ buildXaiVideoStartBody,
52
+ getXaiVideoGenerationStatus,
53
+ startXaiVideoGeneration,
54
+ } from "./xai-video-generation";
36
55
  import type {
37
56
  TurnActivityServices,
38
57
  VideoGenerationReconcileResult,
@@ -72,7 +91,17 @@ export async function reconcileVideoGenerationOperation(
72
91
  if (!operation.credentialEncrypted) {
73
92
  throw new Error("Video provider credential lease was erased too early");
74
93
  }
75
- const apiKey = decryptVideoGenerationApiKey(key, operation.credentialEncrypted);
94
+ const providerCredential = decryptVideoGenerationCredential(key, operation.credentialEncrypted);
95
+ const usesSuperGrok = operation.fundingSource === "supergrok_subscription";
96
+ if (usesSuperGrok !== (providerCredential.kind === "xai-subscription")) {
97
+ throw new Error("Video provider credential does not match its funding source");
98
+ }
99
+ const apiKey = providerCredential.kind === "api-key" ? providerCredential.apiKey : null;
100
+ const xaiAuth =
101
+ providerCredential.kind === "xai-subscription"
102
+ ? await durableXaiVideoAuth(service, operation, key, providerCredential)
103
+ : null;
104
+ let preparedSubmissionThisRun = false;
76
105
 
77
106
  if (operation.status === "accepted") {
78
107
  const canonical = decryptCanonicalRequest(operation, key);
@@ -96,7 +125,9 @@ export async function reconcileVideoGenerationOperation(
96
125
  const providerRequestExpiresAt = expirations.length
97
126
  ? new Date(Math.min(...expirations.map((value) => value.getTime())))
98
127
  : operation.recoveryDeadlineAt;
99
- const body = buildGatewayVideoStartBody(canonical, grants);
128
+ const body = usesSuperGrok
129
+ ? buildXaiVideoStartBody(canonical, grants)
130
+ : buildGatewayVideoStartBody(canonical, grants);
100
131
  operation = retainReferences(
101
132
  operation,
102
133
  await markVideoGenerationSubmissionIntent(service.db, {
@@ -109,9 +140,26 @@ export async function reconcileVideoGenerationOperation(
109
140
  nextReconcileAt: new Date(),
110
141
  }),
111
142
  );
143
+ preparedSubmissionThisRun = true;
112
144
  }
113
145
 
114
146
  if (operation.status === "submission_uncertain") {
147
+ if (usesSuperGrok && !preparedSubmissionThisRun) {
148
+ operation = retainReferences(
149
+ operation,
150
+ await settleVideoGenerationFailure(service.db, {
151
+ accountId: input.accountId,
152
+ workspaceId: input.workspaceId,
153
+ operationId: operation.id,
154
+ requestDigest: operation.requestDigest,
155
+ status: "outcome_unknown",
156
+ publicReason:
157
+ "The video provider did not confirm whether generation started. It was not replayed because xAI provides no start idempotency key.",
158
+ }),
159
+ );
160
+ await deliverTerminalResult(service, operation);
161
+ return { action: "terminal", status: terminalStatus(operation.status) };
162
+ }
115
163
  const deadline = Math.min(
116
164
  operation.recoveryDeadlineAt.getTime(),
117
165
  operation.providerRequestExpiresAt?.getTime() ?? 0,
@@ -134,12 +182,18 @@ export async function reconcileVideoGenerationOperation(
134
182
  }
135
183
  const body = decryptProviderBody(operation, key);
136
184
  try {
137
- const started = await startGatewayVideoGenerationWithBody({
138
- apiKey,
139
- modelId: operation.modelId,
140
- body,
141
- idempotencyKey: operation.providerIdempotencyKey,
142
- });
185
+ const started = usesSuperGrok
186
+ ? await startXaiVideoGeneration({
187
+ body,
188
+ sessionId: operation.sessionId,
189
+ auth: requireXaiAuth(xaiAuth),
190
+ })
191
+ : await startGatewayVideoGenerationWithBody({
192
+ apiKey: requireApiKey(apiKey),
193
+ modelId: operation.modelId,
194
+ body,
195
+ idempotencyKey: operation.providerIdempotencyKey,
196
+ });
143
197
  operation = retainReferences(
144
198
  operation,
145
199
  await markVideoGenerationProviderStarted(service.db, {
@@ -153,7 +207,7 @@ export async function reconcileVideoGenerationOperation(
153
207
  );
154
208
  return waitResult(service);
155
209
  } catch (error) {
156
- if (isPermanentGatewayFailure(error)) {
210
+ if (isPermanentProviderFailure(error)) {
157
211
  operation = retainReferences(
158
212
  operation,
159
213
  await settleVideoGenerationFailure(service.db, {
@@ -187,13 +241,19 @@ export async function reconcileVideoGenerationOperation(
187
241
  if (!operation.providerJobId) throw new Error("Video provider job identity is missing");
188
242
  let status;
189
243
  try {
190
- status = await getGatewayVideoGenerationStatus({
191
- apiKey,
192
- modelId: operation.modelId,
193
- providerJobId: operation.providerJobId,
194
- });
244
+ status = usesSuperGrok
245
+ ? await getXaiVideoGenerationStatus({
246
+ providerJobId: operation.providerJobId,
247
+ sessionId: operation.sessionId,
248
+ auth: requireXaiAuth(xaiAuth),
249
+ })
250
+ : await getGatewayVideoGenerationStatus({
251
+ apiKey: requireApiKey(apiKey),
252
+ modelId: operation.modelId,
253
+ providerJobId: operation.providerJobId,
254
+ });
195
255
  } catch (error) {
196
- if (Date.now() < operation.recoveryDeadlineAt.getTime() && !isPermanentGatewayFailure(error)) {
256
+ if (Date.now() < operation.recoveryDeadlineAt.getTime() && !isPermanentProviderFailure(error)) {
197
257
  await rescheduleVideoGenerationOperation(service.db, {
198
258
  accountId: input.accountId,
199
259
  workspaceId: input.workspaceId,
@@ -536,11 +596,144 @@ function nextPoll(service: TurnActivityServices): Date {
536
596
  }
537
597
 
538
598
  function waitResult(service: TurnActivityServices): VideoGenerationReconcileResult {
539
- return { action: "waiting", delayMs: service.settings.videoGenerationPollIntervalMs };
599
+ return {
600
+ action: "waiting",
601
+ delayMs: service.settings.videoGenerationPollIntervalMs,
602
+ };
603
+ }
604
+
605
+ function requireApiKey(value: string | null): string {
606
+ if (!value) throw new Error("Video provider API key is unavailable");
607
+ return value;
540
608
  }
541
609
 
542
- function isPermanentGatewayFailure(error: unknown): boolean {
543
- return error instanceof GatewayVideoApiError && !error.retryable;
610
+ function requireXaiAuth(
611
+ value: Pick<XaiSubscriptionRequestContext, "getToken" | "refresh"> | null,
612
+ ): Pick<XaiSubscriptionRequestContext, "getToken" | "refresh"> {
613
+ if (!value) throw new Error("SuperGrok video authorization is unavailable");
614
+ return value;
615
+ }
616
+
617
+ /**
618
+ * A paid video job outlives the connected-account row. Keep its frozen OAuth
619
+ * lease usable after disconnect, and durably replace rotated refresh tokens
620
+ * before exposing them to another provider call. The shared account row is a
621
+ * best-effort mirror only; it is never the operation's recovery authority.
622
+ */
623
+ async function durableXaiVideoAuth(
624
+ service: TurnActivityServices,
625
+ operation: VideoGenerationOperationWithReferences,
626
+ key: Uint8Array,
627
+ credential: Extract<VideoGenerationProviderCredential, { kind: "xai-subscription" }>,
628
+ ): Promise<Pick<XaiSubscriptionRequestContext, "getToken" | "refresh">> {
629
+ let current = credential;
630
+ const frozenEncrypted = operation.credentialEncrypted;
631
+ if (!frozenEncrypted) throw new Error("SuperGrok video credential lease is unavailable");
632
+ let currentEncrypted: string = frozenEncrypted;
633
+
634
+ let connectedAccount: XaiCredentialForRun | null = null;
635
+ try {
636
+ connectedAccount = await materializeXaiCredentialForRun(service.db, {
637
+ workspaceId: operation.workspaceId,
638
+ subjectId: current.subjectId,
639
+ credentialId: current.credentialId,
640
+ authoritySnapshot: current.authoritySnapshot,
641
+ encryptionKey: key,
642
+ });
643
+ } catch {
644
+ // Disconnect/revocation must not cancel a previously accepted paid job.
645
+ }
646
+
647
+ return {
648
+ getToken: async () => ({
649
+ accessToken: current.accessToken,
650
+ userId: current.userId,
651
+ }),
652
+ refresh: async () => {
653
+ let next: typeof current | null = null;
654
+ if (connectedAccount) {
655
+ try {
656
+ const result = await refreshXaiSubscriptionCredentialSerialized(service.db, {
657
+ accountId: operation.accountId,
658
+ workspaceId: operation.workspaceId,
659
+ subjectId: current.subjectId,
660
+ credentialId: current.credentialId,
661
+ authoritySnapshot: current.authoritySnapshot,
662
+ encryptionKey: key,
663
+ observedAccessToken: current.accessToken,
664
+ observedRefreshToken: current.refreshToken,
665
+ refresh: async (shared) => {
666
+ const refreshToken = shared.secret.refreshToken;
667
+ if (!refreshToken) {
668
+ throw new Error("The connected SuperGrok account cannot be refreshed");
669
+ }
670
+ const tokens = await refreshXaiToken(refreshToken);
671
+ return {
672
+ secret: {
673
+ version: 1,
674
+ accessToken: tokens.accessToken,
675
+ refreshToken: tokens.refreshToken,
676
+ },
677
+ expiresAt:
678
+ xaiAccessTokenExpiry(tokens.accessToken) ??
679
+ new Date(Date.now() + tokens.expiresInSeconds * 1_000),
680
+ };
681
+ },
682
+ });
683
+ connectedAccount = result.credential;
684
+ if (
685
+ connectedAccount.secret.accessToken &&
686
+ connectedAccount.secret.refreshToken &&
687
+ connectedAccount.providerAccountId
688
+ ) {
689
+ next = Object.freeze({
690
+ ...current,
691
+ accessToken: connectedAccount.secret.accessToken,
692
+ refreshToken: connectedAccount.secret.refreshToken,
693
+ userId: connectedAccount.providerAccountId,
694
+ });
695
+ }
696
+ } catch {
697
+ // Disconnect/revocation must not cancel a previously accepted paid job.
698
+ connectedAccount = null;
699
+ }
700
+ }
701
+ if (!next) {
702
+ const tokens = await refreshXaiToken(current.refreshToken);
703
+ next = Object.freeze({
704
+ ...current,
705
+ accessToken: tokens.accessToken,
706
+ refreshToken: tokens.refreshToken,
707
+ });
708
+ }
709
+ const nextEncrypted = encryptVideoGenerationXaiCredential(key, next);
710
+ await rotateVideoGenerationCredential(service.db, {
711
+ accountId: operation.accountId,
712
+ workspaceId: operation.workspaceId,
713
+ operationId: operation.id,
714
+ requestDigest: operation.requestDigest,
715
+ expectedCredentialEncrypted: currentEncrypted,
716
+ credentialEncrypted: nextEncrypted,
717
+ });
718
+ current = next;
719
+ currentEncrypted = nextEncrypted;
720
+ return { accessToken: current.accessToken, userId: current.userId };
721
+ },
722
+ };
723
+ }
724
+
725
+ function isPermanentProviderFailure(error: unknown): boolean {
726
+ if (error instanceof GatewayVideoApiError) return !error.retryable;
727
+ if (!(error instanceof XaiSubscriptionError)) return false;
728
+ if (error.kind === "relogin_required") return true;
729
+ if (error.kind !== "provider_rejected") return false;
730
+ return (
731
+ typeof error.status === "number" &&
732
+ error.status >= 400 &&
733
+ error.status < 500 &&
734
+ error.status !== 408 &&
735
+ error.status !== 429
736
+ );
544
737
  }
545
738
 
546
739
  function privateError(error: unknown): string {
@@ -172,7 +172,7 @@ async function probeBrowserCompatibleMp4(
172
172
  "-analyzeduration",
173
173
  "5000000",
174
174
  "-show_entries",
175
- "format=format_name,duration:stream=codec_type,codec_name,width,height,avg_frame_rate",
175
+ "format=format_name,duration:stream=codec_type,codec_name,width,height,avg_frame_rate:stream_disposition=attached_pic",
176
176
  "-of",
177
177
  "json",
178
178
  "--",
@@ -187,7 +187,9 @@ async function probeBrowserCompatibleMp4(
187
187
  const root = record(decoded);
188
188
  const format = record(root?.format);
189
189
  const streams = Array.isArray(root?.streams) ? root.streams.map(record).filter(Boolean) : [];
190
- const video = streams.filter((stream) => stream?.codec_type === "video");
190
+ const video = streams.filter(
191
+ (stream) => stream?.codec_type === "video" && !isAttachedPicture(stream),
192
+ );
191
193
  const audio = streams.filter((stream) => stream?.codec_type === "audio");
192
194
  const durationSeconds = finitePositive(format?.duration);
193
195
  const width = positiveInteger(video[0]?.width);
@@ -223,6 +225,10 @@ async function probeBrowserCompatibleMp4(
223
225
  };
224
226
  }
225
227
 
228
+ function isAttachedPicture(stream: Record<string, unknown>): boolean {
229
+ return Number(record(stream.disposition)?.attached_pic) === 1;
230
+ }
231
+
226
232
  async function spawnBounded(
227
233
  command: string,
228
234
  args: string[],
@@ -1,4 +1,8 @@
1
- import { GENERATED_VIDEO_MAX_BYTES, type GenerateVideoToolInput } from "@opengeni/contracts";
1
+ import {
2
+ GENERATED_VIDEO_MAX_BYTES,
3
+ type GenerateVideoToolInput,
4
+ type VideoGenerationRejectedCode,
5
+ } from "@opengeni/contracts";
2
6
  import type { SealedVideoReference, SealedVideoReferenceRole } from "@opengeni/core";
3
7
  import type { ObjectStorage } from "@opengeni/storage";
4
8
  import {
@@ -26,6 +30,19 @@ export type InspectedSandboxVideoReference = Readonly<{
26
30
  sha256: string;
27
31
  }>;
28
32
 
33
+ export type VideoReferenceInputErrorCode = VideoGenerationRejectedCode;
34
+
35
+ /** A deterministic, pre-admission tool-input error. No video operation exists yet. */
36
+ export class VideoReferenceInputError extends Error {
37
+ readonly code: VideoReferenceInputErrorCode;
38
+
39
+ constructor(code: VideoReferenceInputErrorCode, message: string) {
40
+ super(message);
41
+ this.name = "VideoReferenceInputError";
42
+ this.code = code;
43
+ }
44
+ }
45
+
29
46
  export function videoReferencePaths(
30
47
  request: GenerateVideoToolInput,
31
48
  ): Array<{ role: SealedVideoReferenceRole; path: string }> {
@@ -56,7 +73,12 @@ export async function inspectSandboxVideoReferences(input: {
56
73
  workdir: "/workspace",
57
74
  maxOutputTokens: 4_096,
58
75
  });
59
- if (result.exitCode !== 0) throw new Error("Video reference is not a stable workspace file");
76
+ if (result.exitCode !== 0) {
77
+ throw new VideoReferenceInputError(
78
+ "reference_not_stable",
79
+ "The video reference must be an existing regular file at the exact /workspace path returned by the preceding tool.",
80
+ );
81
+ }
60
82
  const [sizeText, sha256, prefixBase64] = result.stdout.trim().split("\t");
61
83
  const sizeBytes = Number(sizeText);
62
84
  if (
@@ -64,18 +86,29 @@ export async function inspectSandboxVideoReferences(input: {
64
86
  sizeBytes <= 0 ||
65
87
  !/^[0-9a-f]{64}$/u.test(sha256 ?? "")
66
88
  ) {
67
- throw new Error("Video reference inspection returned invalid metadata");
89
+ throw new VideoReferenceInputError(
90
+ "reference_not_stable",
91
+ "The video reference changed or could not be read safely. Use the exact current /workspace path and try again.",
92
+ );
68
93
  }
69
94
  const prefix = Buffer.from(prefixBase64 ?? "", "base64");
70
95
  const contentType = sniffContentType(prefix);
71
96
  const maxBytes =
72
97
  reference.role === "video_reference" ? MAX_VIDEO_REFERENCE_BYTES : MAX_IMAGE_REFERENCE_BYTES;
73
- if (sizeBytes > maxBytes) throw new Error("Video reference exceeds the supported size");
98
+ if (sizeBytes > maxBytes) {
99
+ throw new VideoReferenceInputError(
100
+ "reference_too_large",
101
+ "The video reference exceeds the supported size for this source mode.",
102
+ );
103
+ }
74
104
  if (
75
105
  (reference.role === "video_reference" && contentType !== "video/mp4") ||
76
106
  (reference.role !== "video_reference" && !contentType.startsWith("image/"))
77
107
  ) {
78
- throw new Error("Video reference media type does not match its semantic role");
108
+ throw new VideoReferenceInputError(
109
+ "reference_media_type_mismatch",
110
+ "The video reference media type does not match the selected source mode.",
111
+ );
79
112
  }
80
113
  inspected.push({
81
114
  ordinal,
@@ -136,7 +169,10 @@ export async function uploadAndVerifyVideoReferences(input: {
136
169
  });
137
170
  try {
138
171
  if (reference.role === "video_reference") {
139
- await validateVideoReference({ path: verified.path, ffprobePath: input.ffprobePath });
172
+ await validateVideoReference({
173
+ path: verified.path,
174
+ ffprobePath: input.ffprobePath,
175
+ });
140
176
  } else {
141
177
  const image = await validateImageReference(verified.path);
142
178
  if (image.contentType !== reference.contentType) {
@@ -169,25 +205,16 @@ export function videoReferenceStagingKey(input: {
169
205
  function inspectCommand(path: string): string {
170
206
  return [
171
207
  "set -euo pipefail",
172
- `requested=${shellQuote(path)}`,
173
- 'case "$requested" in /workspace/*) ;; *) exit 41 ;; esac',
174
- 'case "$requested" in *"/../"*|*"/./"*|*/..|*/.) exit 42 ;; esac',
175
- 'resolved=$(realpath -e -- "$requested")',
176
- '[ "$resolved" = "$requested" ] || exit 43',
177
- 'exec 3<"$resolved"',
178
- 'fd_path=$(realpath -e -- "/proc/$$/fd/3" 2>/dev/null || realpath -e -- "/dev/fd/3")',
179
- 'case "$fd_path" in /workspace/*) ;; *) exit 44 ;; esac',
180
- '[ -f "/proc/$$/fd/3" ] 2>/dev/null || [ -f "/dev/fd/3" ] || exit 45',
181
- 'fd=/proc/$$/fd/3; [ -e "$fd" ] || fd=/dev/fd/3',
182
- 'before=$(stat -Lc "%d:%i:%s:%Y" -- "$fd" 2>/dev/null || stat -f "%d:%i:%z:%m" "$fd")',
183
- 'size=$(stat -Lc "%s" -- "$fd" 2>/dev/null || stat -f "%z" "$fd")',
184
- "sha=$(sha256sum -- \"$fd\" 2>/dev/null | awk '{print $1}' || shasum -a 256 \"$fd\" | awk '{print $1}')",
208
+ ...stableWorkspaceFilePrelude(path, { nonCanonical: 43, invalidFile: 45 }),
209
+ 'sha=$(video_reference_sha256 "$fd")',
185
210
  'exec 3<&-; exec 3<"$resolved"',
186
211
  'fd=/proc/$$/fd/3; [ -e "$fd" ] || fd=/dev/fd/3',
187
- 'reopened=$(stat -Lc "%d:%i:%s:%Y" -- "$fd" 2>/dev/null || stat -f "%d:%i:%z:%m" "$fd")',
212
+ 'reopened=$(video_reference_stat_identity "$fd")',
188
213
  '[ "$before" = "$reopened" ] || exit 46',
214
+ '[ "$(video_reference_stat_content_identity "$resolved")" = "$(video_reference_stat_content_identity "$fd")" ] || exit 46',
215
+ '[ "$(realpath -- "$requested")" = "$expected" ] || exit 46',
189
216
  'prefix=$(dd if="$fd" bs=32 count=1 2>/dev/null | base64 | tr -d "\\r\\n")',
190
- 'after=$(stat -Lc "%d:%i:%s:%Y" -- "$fd" 2>/dev/null || stat -f "%d:%i:%z:%m" "$fd")',
217
+ 'after=$(video_reference_stat_identity "$fd")',
191
218
  '[ "$before" = "$after" ] || exit 46',
192
219
  'printf "%s\\t%s\\t%s\\n" "$size" "$sha" "$prefix"',
193
220
  ].join("\n");
@@ -203,28 +230,85 @@ function uploadCommand(
203
230
  .join(" ");
204
231
  return [
205
232
  "set -euo pipefail",
206
- `requested=${shellQuote(reference.path)}`,
207
- 'resolved=$(realpath -e -- "$requested")',
208
- '[ "$resolved" = "$requested" ] || exit 51',
209
- 'exec 3<"$resolved"',
210
- 'fd_path=$(realpath -e -- "/proc/$$/fd/3" 2>/dev/null || realpath -e -- "/dev/fd/3")',
211
- 'case "$fd_path" in /workspace/*) ;; *) exit 52 ;; esac',
212
- 'fd=/proc/$$/fd/3; [ -e "$fd" ] || fd=/dev/fd/3',
213
- 'before=$(stat -Lc "%d:%i:%s:%Y" -- "$fd" 2>/dev/null || stat -f "%d:%i:%z:%m" "$fd")',
214
- 'size=$(stat -Lc "%s" -- "$fd" 2>/dev/null || stat -f "%z" "$fd")',
233
+ ...stableWorkspaceFilePrelude(reference.path, {
234
+ nonCanonical: 51,
235
+ invalidFile: 52,
236
+ }),
215
237
  `[ "$size" = ${shellQuote(String(reference.sizeBytes))} ] || exit 53`,
216
- "sha=$(sha256sum -- \"$fd\" 2>/dev/null | awk '{print $1}' || shasum -a 256 \"$fd\" | awk '{print $1}')",
238
+ 'sha=$(video_reference_sha256 "$fd")',
217
239
  `[ "$sha" = ${shellQuote(reference.sha256)} ] || exit 54`,
218
240
  'exec 3<&-; exec 3<"$resolved"',
219
241
  'fd=/proc/$$/fd/3; [ -e "$fd" ] || fd=/dev/fd/3',
220
- 'reopened=$(stat -Lc "%d:%i:%s:%Y" -- "$fd" 2>/dev/null || stat -f "%d:%i:%z:%m" "$fd")',
242
+ 'reopened=$(video_reference_stat_identity "$fd")',
221
243
  '[ "$before" = "$reopened" ] || exit 55',
244
+ '[ "$(video_reference_stat_content_identity "$resolved")" = "$(video_reference_stat_content_identity "$fd")" ] || exit 55',
245
+ '[ "$(realpath -- "$requested")" = "$expected" ] || exit 55',
222
246
  `curl --silent --show-error --fail --connect-timeout 20 --max-time ${UPLOAD_TIMEOUT_SECONDS} --request PUT ${headers} --upload-file "$fd" --output /dev/null ${shellQuote(upload.url)}`,
223
- 'after=$(stat -Lc "%d:%i:%s:%Y" -- "$fd" 2>/dev/null || stat -f "%d:%i:%z:%m" "$fd")',
247
+ 'after=$(video_reference_stat_identity "$fd")',
224
248
  '[ "$before" = "$after" ] || exit 56',
225
249
  ].join("\n");
226
250
  }
227
251
 
252
+ function stableWorkspaceFilePrelude(
253
+ path: string,
254
+ exitCodes: { nonCanonical: number; invalidFile: number },
255
+ ): string[] {
256
+ const requested = workspaceRelativeReferencePath(path);
257
+ return [
258
+ `requested=${shellQuote(requested)}`,
259
+ "root=$(pwd -P)",
260
+ 'expected="$root/${requested#./}"',
261
+ 'resolved=$(realpath -- "$requested")',
262
+ `[ "$resolved" = "$expected" ] || exit ${exitCodes.nonCanonical}`,
263
+ `[ -f "$resolved" ] || exit ${exitCodes.invalidFile}`,
264
+ 'exec 3<"$resolved"',
265
+ 'fd=/proc/$$/fd/3; [ -e "$fd" ] || fd=/dev/fd/3',
266
+ `[ -f "$fd" ] || exit ${exitCodes.invalidFile}`,
267
+ "video_reference_platform=$(uname -s)",
268
+ 'if [ "$video_reference_platform" = Darwin ]; then',
269
+ ' video_reference_stat_identity() { stat -f "%d:%i:%z:%m" "$1"; }',
270
+ ' video_reference_stat_content_identity() { stat -f "%i:%z:%m" "$1"; }',
271
+ ' video_reference_stat_size() { stat -f "%z" "$1"; }',
272
+ "else",
273
+ ' video_reference_stat_identity() { stat -Lc "%d:%i:%s:%Y" -- "$1"; }',
274
+ ' video_reference_stat_content_identity() { stat -Lc "%d:%i:%s:%Y" -- "$1"; }',
275
+ ' video_reference_stat_size() { stat -Lc "%s" -- "$1"; }',
276
+ "fi",
277
+ "video_reference_sha256() {",
278
+ " if command -v sha256sum >/dev/null 2>&1; then",
279
+ " sha256sum \"$1\" | awk '{print $1}'",
280
+ " else",
281
+ " shasum -a 256 \"$1\" | awk '{print $1}'",
282
+ " fi",
283
+ "}",
284
+ 'if [ "$video_reference_platform" != Darwin ]; then',
285
+ ' fd_path=$(realpath -- "$fd")',
286
+ ` [ "$fd_path" = "$expected" ] || exit ${exitCodes.invalidFile}`,
287
+ "fi",
288
+ `[ "$(video_reference_stat_content_identity "$resolved")" = "$(video_reference_stat_content_identity "$fd")" ] || exit ${exitCodes.invalidFile}`,
289
+ `[ "$(realpath -- "$requested")" = "$expected" ] || exit ${exitCodes.invalidFile}`,
290
+ 'before=$(video_reference_stat_identity "$fd")',
291
+ 'size=$(video_reference_stat_size "$fd")',
292
+ ];
293
+ }
294
+
295
+ function workspaceRelativeReferencePath(path: string): string {
296
+ if (!path.startsWith("/workspace/") || /[\0\r\n]/u.test(path)) {
297
+ throw new VideoReferenceInputError(
298
+ "invalid_reference_path",
299
+ "Video references must use an exact file path under /workspace.",
300
+ );
301
+ }
302
+ const segments = path.slice("/workspace/".length).split("/");
303
+ if (segments.some((segment) => segment.length === 0 || segment === "." || segment === "..")) {
304
+ throw new VideoReferenceInputError(
305
+ "invalid_reference_path",
306
+ "Video references must use a canonical file path under /workspace.",
307
+ );
308
+ }
309
+ return `./${segments.join("/")}`;
310
+ }
311
+
228
312
  function sniffContentType(prefix: Uint8Array): string {
229
313
  if (
230
314
  prefix.byteLength >= 8 &&
@@ -245,7 +329,10 @@ function sniffContentType(prefix: Uint8Array): string {
245
329
  if (prefix.byteLength >= 12 && Buffer.from(prefix.subarray(4, 8)).toString("ascii") === "ftyp") {
246
330
  return "video/mp4";
247
331
  }
248
- throw new Error("Video reference has an unsupported media signature");
332
+ throw new VideoReferenceInputError(
333
+ "unsupported_reference_media",
334
+ "The video reference is not a supported PNG, JPEG, WebP, or MP4 file.",
335
+ );
249
336
  }
250
337
 
251
338
  function shellQuote(value: string): string {