@opengeni/worker-bundle 0.17.2 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +12 -0
- package/dist/activities/agent-turn.d.ts +49 -1
- package/dist/activities/browser-state-artifact-reaper.d.ts +29 -0
- package/dist/activities/capabilities.d.ts +6 -0
- package/dist/activities/codex-capacity.d.ts +1 -0
- package/dist/activities/environment.d.ts +7 -1
- package/dist/activities/packs.d.ts +8 -10
- package/dist/activities/run-input.d.ts +8 -0
- package/dist/activities/session-state.d.ts +7 -3
- package/dist/activities/site-auth-maintenance.d.ts +27 -0
- package/dist/activities/types.d.ts +43 -0
- package/dist/activities/video-generation-admission.d.ts +9 -4
- package/dist/activities/video-generation-credential.d.ts +17 -0
- package/dist/activities/video-reference-staging.d.ts +7 -1
- package/dist/activities/xai-auth.d.ts +23 -0
- package/dist/activities/xai-image-generation.d.ts +26 -0
- package/dist/activities/xai-video-generation.d.ts +18 -0
- package/dist/{activities-control-44KBWLSM.js → activities-control-SMKF2OBE.js} +438 -55
- package/dist/activities-control-SMKF2OBE.js.map +1 -0
- package/dist/activities-control.d.ts +10 -2
- package/dist/{activities-turn-K4CEDEGJ.js → activities-turn-IDS3WEHA.js} +6704 -4847
- package/dist/activities-turn-IDS3WEHA.js.map +1 -0
- package/dist/activities.d.ts +11 -3
- package/dist/{chunk-E3RI3I33.js → chunk-N5TJIFCE.js} +156 -10
- package/dist/chunk-N5TJIFCE.js.map +1 -0
- package/dist/{chunk-KKCRGAGZ.js → chunk-P6Z2SLAA.js} +99 -98
- package/dist/chunk-P6Z2SLAA.js.map +1 -0
- package/dist/index.d.ts +14 -1
- package/dist/index.js +49 -2
- package/dist/index.js.map +1 -1
- package/dist/model-checkpoint-memory-collector.d.ts +1 -0
- package/dist/model-context-contributions.d.ts +40 -0
- package/dist/observability-metrics.d.ts +57 -1
- package/dist/sandbox-routing.d.ts +4 -0
- package/dist/workflow-bundle.js +232 -18
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/session.d.ts +17 -1
- package/dist/workflows/site-auth-maintenance.d.ts +2 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +16 -15
- package/src/activities/agent-run-admission.ts +76 -0
- package/src/activities/agent-turn.ts +2777 -1241
- package/src/activities/browser-state-artifact-reaper.ts +120 -0
- package/src/activities/capabilities.ts +10 -0
- package/src/activities/codex-capacity.ts +48 -1
- package/src/activities/environment.ts +14 -8
- package/src/activities/goals.ts +2 -1
- package/src/activities/packs.ts +45 -122
- package/src/activities/parent-wake.ts +13 -1
- package/src/activities/run-input.ts +137 -45
- package/src/activities/scheduled-tasks.ts +30 -73
- package/src/activities/session-state.ts +87 -5
- package/src/activities/site-auth-maintenance.ts +269 -0
- package/src/activities/types.ts +44 -0
- package/src/activities/video-generation-admission.ts +37 -4
- package/src/activities/video-generation-credential.ts +84 -6
- package/src/activities/video-generation-reconciliation.ts +212 -19
- package/src/activities/video-media-validation.ts +8 -2
- package/src/activities/video-reference-staging.ts +121 -34
- package/src/activities/xai-auth.ts +122 -0
- package/src/activities/xai-image-generation.ts +86 -0
- package/src/activities/xai-video-generation.ts +69 -0
- package/src/activities-control.ts +4 -0
- package/src/activities.ts +7 -0
- package/src/index.ts +51 -0
- package/src/model-checkpoint-memory-collector.ts +8 -1
- package/src/model-context-contributions.ts +159 -0
- package/src/observability-metrics.ts +282 -12
- package/src/sandbox-routing.ts +13 -0
- package/src/workflows/activities.ts +1 -0
- package/src/workflows/file-upload-reaper.ts +8 -2
- package/src/workflows/session.ts +219 -14
- package/src/workflows/site-auth-maintenance.ts +14 -0
- package/src/workflows.ts +1 -0
- package/dist/activities-control-44KBWLSM.js.map +0 -1
- package/dist/activities-turn-K4CEDEGJ.js.map +0 -1
- package/dist/chunk-E3RI3I33.js.map +0 -1
- package/dist/chunk-KKCRGAGZ.js.map +0 -1
|
@@ -11,12 +11,16 @@ import {
|
|
|
11
11
|
markVideoGenerationRetaining,
|
|
12
12
|
markVideoGenerationSubmissionIntent,
|
|
13
13
|
markVideoGenerationSubmissionUncertain,
|
|
14
|
+
materializeXaiCredentialForRun,
|
|
15
|
+
refreshXaiSubscriptionCredentialSerialized,
|
|
16
|
+
rotateVideoGenerationCredential,
|
|
14
17
|
markVideoGenerationTerminalUpdate,
|
|
15
18
|
markVideoGenerationTerminalUpdateInTransaction,
|
|
16
19
|
mediaGenerationResultForStoredOperation,
|
|
17
20
|
rescheduleVideoGenerationOperation,
|
|
18
21
|
settleVideoGenerationFailure,
|
|
19
22
|
settleVideoGenerationReady,
|
|
23
|
+
type XaiCredentialForRun,
|
|
20
24
|
type VideoGenerationOperationWithReferences,
|
|
21
25
|
} from "@opengeni/db";
|
|
22
26
|
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
@@ -32,7 +36,22 @@ import {
|
|
|
32
36
|
downloadGeneratedVideoToVerifiedTemp,
|
|
33
37
|
retainVerifiedGeneratedVideo,
|
|
34
38
|
} from "./video-output-retention";
|
|
35
|
-
import {
|
|
39
|
+
import {
|
|
40
|
+
decryptVideoGenerationCredential,
|
|
41
|
+
encryptVideoGenerationXaiCredential,
|
|
42
|
+
type VideoGenerationProviderCredential,
|
|
43
|
+
} from "./video-generation-credential";
|
|
44
|
+
import {
|
|
45
|
+
refreshXaiToken,
|
|
46
|
+
XaiSubscriptionError,
|
|
47
|
+
xaiAccessTokenExpiry,
|
|
48
|
+
type XaiSubscriptionRequestContext,
|
|
49
|
+
} from "@opengeni/xai-subscription";
|
|
50
|
+
import {
|
|
51
|
+
buildXaiVideoStartBody,
|
|
52
|
+
getXaiVideoGenerationStatus,
|
|
53
|
+
startXaiVideoGeneration,
|
|
54
|
+
} from "./xai-video-generation";
|
|
36
55
|
import type {
|
|
37
56
|
TurnActivityServices,
|
|
38
57
|
VideoGenerationReconcileResult,
|
|
@@ -72,7 +91,17 @@ export async function reconcileVideoGenerationOperation(
|
|
|
72
91
|
if (!operation.credentialEncrypted) {
|
|
73
92
|
throw new Error("Video provider credential lease was erased too early");
|
|
74
93
|
}
|
|
75
|
-
const
|
|
94
|
+
const providerCredential = decryptVideoGenerationCredential(key, operation.credentialEncrypted);
|
|
95
|
+
const usesSuperGrok = operation.fundingSource === "supergrok_subscription";
|
|
96
|
+
if (usesSuperGrok !== (providerCredential.kind === "xai-subscription")) {
|
|
97
|
+
throw new Error("Video provider credential does not match its funding source");
|
|
98
|
+
}
|
|
99
|
+
const apiKey = providerCredential.kind === "api-key" ? providerCredential.apiKey : null;
|
|
100
|
+
const xaiAuth =
|
|
101
|
+
providerCredential.kind === "xai-subscription"
|
|
102
|
+
? await durableXaiVideoAuth(service, operation, key, providerCredential)
|
|
103
|
+
: null;
|
|
104
|
+
let preparedSubmissionThisRun = false;
|
|
76
105
|
|
|
77
106
|
if (operation.status === "accepted") {
|
|
78
107
|
const canonical = decryptCanonicalRequest(operation, key);
|
|
@@ -96,7 +125,9 @@ export async function reconcileVideoGenerationOperation(
|
|
|
96
125
|
const providerRequestExpiresAt = expirations.length
|
|
97
126
|
? new Date(Math.min(...expirations.map((value) => value.getTime())))
|
|
98
127
|
: operation.recoveryDeadlineAt;
|
|
99
|
-
const body =
|
|
128
|
+
const body = usesSuperGrok
|
|
129
|
+
? buildXaiVideoStartBody(canonical, grants)
|
|
130
|
+
: buildGatewayVideoStartBody(canonical, grants);
|
|
100
131
|
operation = retainReferences(
|
|
101
132
|
operation,
|
|
102
133
|
await markVideoGenerationSubmissionIntent(service.db, {
|
|
@@ -109,9 +140,26 @@ export async function reconcileVideoGenerationOperation(
|
|
|
109
140
|
nextReconcileAt: new Date(),
|
|
110
141
|
}),
|
|
111
142
|
);
|
|
143
|
+
preparedSubmissionThisRun = true;
|
|
112
144
|
}
|
|
113
145
|
|
|
114
146
|
if (operation.status === "submission_uncertain") {
|
|
147
|
+
if (usesSuperGrok && !preparedSubmissionThisRun) {
|
|
148
|
+
operation = retainReferences(
|
|
149
|
+
operation,
|
|
150
|
+
await settleVideoGenerationFailure(service.db, {
|
|
151
|
+
accountId: input.accountId,
|
|
152
|
+
workspaceId: input.workspaceId,
|
|
153
|
+
operationId: operation.id,
|
|
154
|
+
requestDigest: operation.requestDigest,
|
|
155
|
+
status: "outcome_unknown",
|
|
156
|
+
publicReason:
|
|
157
|
+
"The video provider did not confirm whether generation started. It was not replayed because xAI provides no start idempotency key.",
|
|
158
|
+
}),
|
|
159
|
+
);
|
|
160
|
+
await deliverTerminalResult(service, operation);
|
|
161
|
+
return { action: "terminal", status: terminalStatus(operation.status) };
|
|
162
|
+
}
|
|
115
163
|
const deadline = Math.min(
|
|
116
164
|
operation.recoveryDeadlineAt.getTime(),
|
|
117
165
|
operation.providerRequestExpiresAt?.getTime() ?? 0,
|
|
@@ -134,12 +182,18 @@ export async function reconcileVideoGenerationOperation(
|
|
|
134
182
|
}
|
|
135
183
|
const body = decryptProviderBody(operation, key);
|
|
136
184
|
try {
|
|
137
|
-
const started =
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
185
|
+
const started = usesSuperGrok
|
|
186
|
+
? await startXaiVideoGeneration({
|
|
187
|
+
body,
|
|
188
|
+
sessionId: operation.sessionId,
|
|
189
|
+
auth: requireXaiAuth(xaiAuth),
|
|
190
|
+
})
|
|
191
|
+
: await startGatewayVideoGenerationWithBody({
|
|
192
|
+
apiKey: requireApiKey(apiKey),
|
|
193
|
+
modelId: operation.modelId,
|
|
194
|
+
body,
|
|
195
|
+
idempotencyKey: operation.providerIdempotencyKey,
|
|
196
|
+
});
|
|
143
197
|
operation = retainReferences(
|
|
144
198
|
operation,
|
|
145
199
|
await markVideoGenerationProviderStarted(service.db, {
|
|
@@ -153,7 +207,7 @@ export async function reconcileVideoGenerationOperation(
|
|
|
153
207
|
);
|
|
154
208
|
return waitResult(service);
|
|
155
209
|
} catch (error) {
|
|
156
|
-
if (
|
|
210
|
+
if (isPermanentProviderFailure(error)) {
|
|
157
211
|
operation = retainReferences(
|
|
158
212
|
operation,
|
|
159
213
|
await settleVideoGenerationFailure(service.db, {
|
|
@@ -187,13 +241,19 @@ export async function reconcileVideoGenerationOperation(
|
|
|
187
241
|
if (!operation.providerJobId) throw new Error("Video provider job identity is missing");
|
|
188
242
|
let status;
|
|
189
243
|
try {
|
|
190
|
-
status =
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
244
|
+
status = usesSuperGrok
|
|
245
|
+
? await getXaiVideoGenerationStatus({
|
|
246
|
+
providerJobId: operation.providerJobId,
|
|
247
|
+
sessionId: operation.sessionId,
|
|
248
|
+
auth: requireXaiAuth(xaiAuth),
|
|
249
|
+
})
|
|
250
|
+
: await getGatewayVideoGenerationStatus({
|
|
251
|
+
apiKey: requireApiKey(apiKey),
|
|
252
|
+
modelId: operation.modelId,
|
|
253
|
+
providerJobId: operation.providerJobId,
|
|
254
|
+
});
|
|
195
255
|
} catch (error) {
|
|
196
|
-
if (Date.now() < operation.recoveryDeadlineAt.getTime() && !
|
|
256
|
+
if (Date.now() < operation.recoveryDeadlineAt.getTime() && !isPermanentProviderFailure(error)) {
|
|
197
257
|
await rescheduleVideoGenerationOperation(service.db, {
|
|
198
258
|
accountId: input.accountId,
|
|
199
259
|
workspaceId: input.workspaceId,
|
|
@@ -536,11 +596,144 @@ function nextPoll(service: TurnActivityServices): Date {
|
|
|
536
596
|
}
|
|
537
597
|
|
|
538
598
|
function waitResult(service: TurnActivityServices): VideoGenerationReconcileResult {
|
|
539
|
-
return {
|
|
599
|
+
return {
|
|
600
|
+
action: "waiting",
|
|
601
|
+
delayMs: service.settings.videoGenerationPollIntervalMs,
|
|
602
|
+
};
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
function requireApiKey(value: string | null): string {
|
|
606
|
+
if (!value) throw new Error("Video provider API key is unavailable");
|
|
607
|
+
return value;
|
|
540
608
|
}
|
|
541
609
|
|
|
542
|
-
function
|
|
543
|
-
|
|
610
|
+
function requireXaiAuth(
|
|
611
|
+
value: Pick<XaiSubscriptionRequestContext, "getToken" | "refresh"> | null,
|
|
612
|
+
): Pick<XaiSubscriptionRequestContext, "getToken" | "refresh"> {
|
|
613
|
+
if (!value) throw new Error("SuperGrok video authorization is unavailable");
|
|
614
|
+
return value;
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
/**
|
|
618
|
+
* A paid video job outlives the connected-account row. Keep its frozen OAuth
|
|
619
|
+
* lease usable after disconnect, and durably replace rotated refresh tokens
|
|
620
|
+
* before exposing them to another provider call. The shared account row is a
|
|
621
|
+
* best-effort mirror only; it is never the operation's recovery authority.
|
|
622
|
+
*/
|
|
623
|
+
async function durableXaiVideoAuth(
|
|
624
|
+
service: TurnActivityServices,
|
|
625
|
+
operation: VideoGenerationOperationWithReferences,
|
|
626
|
+
key: Uint8Array,
|
|
627
|
+
credential: Extract<VideoGenerationProviderCredential, { kind: "xai-subscription" }>,
|
|
628
|
+
): Promise<Pick<XaiSubscriptionRequestContext, "getToken" | "refresh">> {
|
|
629
|
+
let current = credential;
|
|
630
|
+
const frozenEncrypted = operation.credentialEncrypted;
|
|
631
|
+
if (!frozenEncrypted) throw new Error("SuperGrok video credential lease is unavailable");
|
|
632
|
+
let currentEncrypted: string = frozenEncrypted;
|
|
633
|
+
|
|
634
|
+
let connectedAccount: XaiCredentialForRun | null = null;
|
|
635
|
+
try {
|
|
636
|
+
connectedAccount = await materializeXaiCredentialForRun(service.db, {
|
|
637
|
+
workspaceId: operation.workspaceId,
|
|
638
|
+
subjectId: current.subjectId,
|
|
639
|
+
credentialId: current.credentialId,
|
|
640
|
+
authoritySnapshot: current.authoritySnapshot,
|
|
641
|
+
encryptionKey: key,
|
|
642
|
+
});
|
|
643
|
+
} catch {
|
|
644
|
+
// Disconnect/revocation must not cancel a previously accepted paid job.
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
return {
|
|
648
|
+
getToken: async () => ({
|
|
649
|
+
accessToken: current.accessToken,
|
|
650
|
+
userId: current.userId,
|
|
651
|
+
}),
|
|
652
|
+
refresh: async () => {
|
|
653
|
+
let next: typeof current | null = null;
|
|
654
|
+
if (connectedAccount) {
|
|
655
|
+
try {
|
|
656
|
+
const result = await refreshXaiSubscriptionCredentialSerialized(service.db, {
|
|
657
|
+
accountId: operation.accountId,
|
|
658
|
+
workspaceId: operation.workspaceId,
|
|
659
|
+
subjectId: current.subjectId,
|
|
660
|
+
credentialId: current.credentialId,
|
|
661
|
+
authoritySnapshot: current.authoritySnapshot,
|
|
662
|
+
encryptionKey: key,
|
|
663
|
+
observedAccessToken: current.accessToken,
|
|
664
|
+
observedRefreshToken: current.refreshToken,
|
|
665
|
+
refresh: async (shared) => {
|
|
666
|
+
const refreshToken = shared.secret.refreshToken;
|
|
667
|
+
if (!refreshToken) {
|
|
668
|
+
throw new Error("The connected SuperGrok account cannot be refreshed");
|
|
669
|
+
}
|
|
670
|
+
const tokens = await refreshXaiToken(refreshToken);
|
|
671
|
+
return {
|
|
672
|
+
secret: {
|
|
673
|
+
version: 1,
|
|
674
|
+
accessToken: tokens.accessToken,
|
|
675
|
+
refreshToken: tokens.refreshToken,
|
|
676
|
+
},
|
|
677
|
+
expiresAt:
|
|
678
|
+
xaiAccessTokenExpiry(tokens.accessToken) ??
|
|
679
|
+
new Date(Date.now() + tokens.expiresInSeconds * 1_000),
|
|
680
|
+
};
|
|
681
|
+
},
|
|
682
|
+
});
|
|
683
|
+
connectedAccount = result.credential;
|
|
684
|
+
if (
|
|
685
|
+
connectedAccount.secret.accessToken &&
|
|
686
|
+
connectedAccount.secret.refreshToken &&
|
|
687
|
+
connectedAccount.providerAccountId
|
|
688
|
+
) {
|
|
689
|
+
next = Object.freeze({
|
|
690
|
+
...current,
|
|
691
|
+
accessToken: connectedAccount.secret.accessToken,
|
|
692
|
+
refreshToken: connectedAccount.secret.refreshToken,
|
|
693
|
+
userId: connectedAccount.providerAccountId,
|
|
694
|
+
});
|
|
695
|
+
}
|
|
696
|
+
} catch {
|
|
697
|
+
// Disconnect/revocation must not cancel a previously accepted paid job.
|
|
698
|
+
connectedAccount = null;
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
if (!next) {
|
|
702
|
+
const tokens = await refreshXaiToken(current.refreshToken);
|
|
703
|
+
next = Object.freeze({
|
|
704
|
+
...current,
|
|
705
|
+
accessToken: tokens.accessToken,
|
|
706
|
+
refreshToken: tokens.refreshToken,
|
|
707
|
+
});
|
|
708
|
+
}
|
|
709
|
+
const nextEncrypted = encryptVideoGenerationXaiCredential(key, next);
|
|
710
|
+
await rotateVideoGenerationCredential(service.db, {
|
|
711
|
+
accountId: operation.accountId,
|
|
712
|
+
workspaceId: operation.workspaceId,
|
|
713
|
+
operationId: operation.id,
|
|
714
|
+
requestDigest: operation.requestDigest,
|
|
715
|
+
expectedCredentialEncrypted: currentEncrypted,
|
|
716
|
+
credentialEncrypted: nextEncrypted,
|
|
717
|
+
});
|
|
718
|
+
current = next;
|
|
719
|
+
currentEncrypted = nextEncrypted;
|
|
720
|
+
return { accessToken: current.accessToken, userId: current.userId };
|
|
721
|
+
},
|
|
722
|
+
};
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
function isPermanentProviderFailure(error: unknown): boolean {
|
|
726
|
+
if (error instanceof GatewayVideoApiError) return !error.retryable;
|
|
727
|
+
if (!(error instanceof XaiSubscriptionError)) return false;
|
|
728
|
+
if (error.kind === "relogin_required") return true;
|
|
729
|
+
if (error.kind !== "provider_rejected") return false;
|
|
730
|
+
return (
|
|
731
|
+
typeof error.status === "number" &&
|
|
732
|
+
error.status >= 400 &&
|
|
733
|
+
error.status < 500 &&
|
|
734
|
+
error.status !== 408 &&
|
|
735
|
+
error.status !== 429
|
|
736
|
+
);
|
|
544
737
|
}
|
|
545
738
|
|
|
546
739
|
function privateError(error: unknown): string {
|
|
@@ -172,7 +172,7 @@ async function probeBrowserCompatibleMp4(
|
|
|
172
172
|
"-analyzeduration",
|
|
173
173
|
"5000000",
|
|
174
174
|
"-show_entries",
|
|
175
|
-
"format=format_name,duration:stream=codec_type,codec_name,width,height,avg_frame_rate",
|
|
175
|
+
"format=format_name,duration:stream=codec_type,codec_name,width,height,avg_frame_rate:stream_disposition=attached_pic",
|
|
176
176
|
"-of",
|
|
177
177
|
"json",
|
|
178
178
|
"--",
|
|
@@ -187,7 +187,9 @@ async function probeBrowserCompatibleMp4(
|
|
|
187
187
|
const root = record(decoded);
|
|
188
188
|
const format = record(root?.format);
|
|
189
189
|
const streams = Array.isArray(root?.streams) ? root.streams.map(record).filter(Boolean) : [];
|
|
190
|
-
const video = streams.filter(
|
|
190
|
+
const video = streams.filter(
|
|
191
|
+
(stream) => stream?.codec_type === "video" && !isAttachedPicture(stream),
|
|
192
|
+
);
|
|
191
193
|
const audio = streams.filter((stream) => stream?.codec_type === "audio");
|
|
192
194
|
const durationSeconds = finitePositive(format?.duration);
|
|
193
195
|
const width = positiveInteger(video[0]?.width);
|
|
@@ -223,6 +225,10 @@ async function probeBrowserCompatibleMp4(
|
|
|
223
225
|
};
|
|
224
226
|
}
|
|
225
227
|
|
|
228
|
+
function isAttachedPicture(stream: Record<string, unknown>): boolean {
|
|
229
|
+
return Number(record(stream.disposition)?.attached_pic) === 1;
|
|
230
|
+
}
|
|
231
|
+
|
|
226
232
|
async function spawnBounded(
|
|
227
233
|
command: string,
|
|
228
234
|
args: string[],
|
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
GENERATED_VIDEO_MAX_BYTES,
|
|
3
|
+
type GenerateVideoToolInput,
|
|
4
|
+
type VideoGenerationRejectedCode,
|
|
5
|
+
} from "@opengeni/contracts";
|
|
2
6
|
import type { SealedVideoReference, SealedVideoReferenceRole } from "@opengeni/core";
|
|
3
7
|
import type { ObjectStorage } from "@opengeni/storage";
|
|
4
8
|
import {
|
|
@@ -26,6 +30,19 @@ export type InspectedSandboxVideoReference = Readonly<{
|
|
|
26
30
|
sha256: string;
|
|
27
31
|
}>;
|
|
28
32
|
|
|
33
|
+
export type VideoReferenceInputErrorCode = VideoGenerationRejectedCode;
|
|
34
|
+
|
|
35
|
+
/** A deterministic, pre-admission tool-input error. No video operation exists yet. */
|
|
36
|
+
export class VideoReferenceInputError extends Error {
|
|
37
|
+
readonly code: VideoReferenceInputErrorCode;
|
|
38
|
+
|
|
39
|
+
constructor(code: VideoReferenceInputErrorCode, message: string) {
|
|
40
|
+
super(message);
|
|
41
|
+
this.name = "VideoReferenceInputError";
|
|
42
|
+
this.code = code;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
29
46
|
export function videoReferencePaths(
|
|
30
47
|
request: GenerateVideoToolInput,
|
|
31
48
|
): Array<{ role: SealedVideoReferenceRole; path: string }> {
|
|
@@ -56,7 +73,12 @@ export async function inspectSandboxVideoReferences(input: {
|
|
|
56
73
|
workdir: "/workspace",
|
|
57
74
|
maxOutputTokens: 4_096,
|
|
58
75
|
});
|
|
59
|
-
if (result.exitCode !== 0)
|
|
76
|
+
if (result.exitCode !== 0) {
|
|
77
|
+
throw new VideoReferenceInputError(
|
|
78
|
+
"reference_not_stable",
|
|
79
|
+
"The video reference must be an existing regular file at the exact /workspace path returned by the preceding tool.",
|
|
80
|
+
);
|
|
81
|
+
}
|
|
60
82
|
const [sizeText, sha256, prefixBase64] = result.stdout.trim().split("\t");
|
|
61
83
|
const sizeBytes = Number(sizeText);
|
|
62
84
|
if (
|
|
@@ -64,18 +86,29 @@ export async function inspectSandboxVideoReferences(input: {
|
|
|
64
86
|
sizeBytes <= 0 ||
|
|
65
87
|
!/^[0-9a-f]{64}$/u.test(sha256 ?? "")
|
|
66
88
|
) {
|
|
67
|
-
throw new
|
|
89
|
+
throw new VideoReferenceInputError(
|
|
90
|
+
"reference_not_stable",
|
|
91
|
+
"The video reference changed or could not be read safely. Use the exact current /workspace path and try again.",
|
|
92
|
+
);
|
|
68
93
|
}
|
|
69
94
|
const prefix = Buffer.from(prefixBase64 ?? "", "base64");
|
|
70
95
|
const contentType = sniffContentType(prefix);
|
|
71
96
|
const maxBytes =
|
|
72
97
|
reference.role === "video_reference" ? MAX_VIDEO_REFERENCE_BYTES : MAX_IMAGE_REFERENCE_BYTES;
|
|
73
|
-
if (sizeBytes > maxBytes)
|
|
98
|
+
if (sizeBytes > maxBytes) {
|
|
99
|
+
throw new VideoReferenceInputError(
|
|
100
|
+
"reference_too_large",
|
|
101
|
+
"The video reference exceeds the supported size for this source mode.",
|
|
102
|
+
);
|
|
103
|
+
}
|
|
74
104
|
if (
|
|
75
105
|
(reference.role === "video_reference" && contentType !== "video/mp4") ||
|
|
76
106
|
(reference.role !== "video_reference" && !contentType.startsWith("image/"))
|
|
77
107
|
) {
|
|
78
|
-
throw new
|
|
108
|
+
throw new VideoReferenceInputError(
|
|
109
|
+
"reference_media_type_mismatch",
|
|
110
|
+
"The video reference media type does not match the selected source mode.",
|
|
111
|
+
);
|
|
79
112
|
}
|
|
80
113
|
inspected.push({
|
|
81
114
|
ordinal,
|
|
@@ -136,7 +169,10 @@ export async function uploadAndVerifyVideoReferences(input: {
|
|
|
136
169
|
});
|
|
137
170
|
try {
|
|
138
171
|
if (reference.role === "video_reference") {
|
|
139
|
-
await validateVideoReference({
|
|
172
|
+
await validateVideoReference({
|
|
173
|
+
path: verified.path,
|
|
174
|
+
ffprobePath: input.ffprobePath,
|
|
175
|
+
});
|
|
140
176
|
} else {
|
|
141
177
|
const image = await validateImageReference(verified.path);
|
|
142
178
|
if (image.contentType !== reference.contentType) {
|
|
@@ -169,25 +205,16 @@ export function videoReferenceStagingKey(input: {
|
|
|
169
205
|
function inspectCommand(path: string): string {
|
|
170
206
|
return [
|
|
171
207
|
"set -euo pipefail",
|
|
172
|
-
|
|
173
|
-
'
|
|
174
|
-
'case "$requested" in *"/../"*|*"/./"*|*/..|*/.) exit 42 ;; esac',
|
|
175
|
-
'resolved=$(realpath -e -- "$requested")',
|
|
176
|
-
'[ "$resolved" = "$requested" ] || exit 43',
|
|
177
|
-
'exec 3<"$resolved"',
|
|
178
|
-
'fd_path=$(realpath -e -- "/proc/$$/fd/3" 2>/dev/null || realpath -e -- "/dev/fd/3")',
|
|
179
|
-
'case "$fd_path" in /workspace/*) ;; *) exit 44 ;; esac',
|
|
180
|
-
'[ -f "/proc/$$/fd/3" ] 2>/dev/null || [ -f "/dev/fd/3" ] || exit 45',
|
|
181
|
-
'fd=/proc/$$/fd/3; [ -e "$fd" ] || fd=/dev/fd/3',
|
|
182
|
-
'before=$(stat -Lc "%d:%i:%s:%Y" -- "$fd" 2>/dev/null || stat -f "%d:%i:%z:%m" "$fd")',
|
|
183
|
-
'size=$(stat -Lc "%s" -- "$fd" 2>/dev/null || stat -f "%z" "$fd")',
|
|
184
|
-
"sha=$(sha256sum -- \"$fd\" 2>/dev/null | awk '{print $1}' || shasum -a 256 \"$fd\" | awk '{print $1}')",
|
|
208
|
+
...stableWorkspaceFilePrelude(path, { nonCanonical: 43, invalidFile: 45 }),
|
|
209
|
+
'sha=$(video_reference_sha256 "$fd")',
|
|
185
210
|
'exec 3<&-; exec 3<"$resolved"',
|
|
186
211
|
'fd=/proc/$$/fd/3; [ -e "$fd" ] || fd=/dev/fd/3',
|
|
187
|
-
'reopened=$(
|
|
212
|
+
'reopened=$(video_reference_stat_identity "$fd")',
|
|
188
213
|
'[ "$before" = "$reopened" ] || exit 46',
|
|
214
|
+
'[ "$(video_reference_stat_content_identity "$resolved")" = "$(video_reference_stat_content_identity "$fd")" ] || exit 46',
|
|
215
|
+
'[ "$(realpath -- "$requested")" = "$expected" ] || exit 46',
|
|
189
216
|
'prefix=$(dd if="$fd" bs=32 count=1 2>/dev/null | base64 | tr -d "\\r\\n")',
|
|
190
|
-
'after=$(
|
|
217
|
+
'after=$(video_reference_stat_identity "$fd")',
|
|
191
218
|
'[ "$before" = "$after" ] || exit 46',
|
|
192
219
|
'printf "%s\\t%s\\t%s\\n" "$size" "$sha" "$prefix"',
|
|
193
220
|
].join("\n");
|
|
@@ -203,28 +230,85 @@ function uploadCommand(
|
|
|
203
230
|
.join(" ");
|
|
204
231
|
return [
|
|
205
232
|
"set -euo pipefail",
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
'fd_path=$(realpath -e -- "/proc/$$/fd/3" 2>/dev/null || realpath -e -- "/dev/fd/3")',
|
|
211
|
-
'case "$fd_path" in /workspace/*) ;; *) exit 52 ;; esac',
|
|
212
|
-
'fd=/proc/$$/fd/3; [ -e "$fd" ] || fd=/dev/fd/3',
|
|
213
|
-
'before=$(stat -Lc "%d:%i:%s:%Y" -- "$fd" 2>/dev/null || stat -f "%d:%i:%z:%m" "$fd")',
|
|
214
|
-
'size=$(stat -Lc "%s" -- "$fd" 2>/dev/null || stat -f "%z" "$fd")',
|
|
233
|
+
...stableWorkspaceFilePrelude(reference.path, {
|
|
234
|
+
nonCanonical: 51,
|
|
235
|
+
invalidFile: 52,
|
|
236
|
+
}),
|
|
215
237
|
`[ "$size" = ${shellQuote(String(reference.sizeBytes))} ] || exit 53`,
|
|
216
|
-
|
|
238
|
+
'sha=$(video_reference_sha256 "$fd")',
|
|
217
239
|
`[ "$sha" = ${shellQuote(reference.sha256)} ] || exit 54`,
|
|
218
240
|
'exec 3<&-; exec 3<"$resolved"',
|
|
219
241
|
'fd=/proc/$$/fd/3; [ -e "$fd" ] || fd=/dev/fd/3',
|
|
220
|
-
'reopened=$(
|
|
242
|
+
'reopened=$(video_reference_stat_identity "$fd")',
|
|
221
243
|
'[ "$before" = "$reopened" ] || exit 55',
|
|
244
|
+
'[ "$(video_reference_stat_content_identity "$resolved")" = "$(video_reference_stat_content_identity "$fd")" ] || exit 55',
|
|
245
|
+
'[ "$(realpath -- "$requested")" = "$expected" ] || exit 55',
|
|
222
246
|
`curl --silent --show-error --fail --connect-timeout 20 --max-time ${UPLOAD_TIMEOUT_SECONDS} --request PUT ${headers} --upload-file "$fd" --output /dev/null ${shellQuote(upload.url)}`,
|
|
223
|
-
'after=$(
|
|
247
|
+
'after=$(video_reference_stat_identity "$fd")',
|
|
224
248
|
'[ "$before" = "$after" ] || exit 56',
|
|
225
249
|
].join("\n");
|
|
226
250
|
}
|
|
227
251
|
|
|
252
|
+
function stableWorkspaceFilePrelude(
|
|
253
|
+
path: string,
|
|
254
|
+
exitCodes: { nonCanonical: number; invalidFile: number },
|
|
255
|
+
): string[] {
|
|
256
|
+
const requested = workspaceRelativeReferencePath(path);
|
|
257
|
+
return [
|
|
258
|
+
`requested=${shellQuote(requested)}`,
|
|
259
|
+
"root=$(pwd -P)",
|
|
260
|
+
'expected="$root/${requested#./}"',
|
|
261
|
+
'resolved=$(realpath -- "$requested")',
|
|
262
|
+
`[ "$resolved" = "$expected" ] || exit ${exitCodes.nonCanonical}`,
|
|
263
|
+
`[ -f "$resolved" ] || exit ${exitCodes.invalidFile}`,
|
|
264
|
+
'exec 3<"$resolved"',
|
|
265
|
+
'fd=/proc/$$/fd/3; [ -e "$fd" ] || fd=/dev/fd/3',
|
|
266
|
+
`[ -f "$fd" ] || exit ${exitCodes.invalidFile}`,
|
|
267
|
+
"video_reference_platform=$(uname -s)",
|
|
268
|
+
'if [ "$video_reference_platform" = Darwin ]; then',
|
|
269
|
+
' video_reference_stat_identity() { stat -f "%d:%i:%z:%m" "$1"; }',
|
|
270
|
+
' video_reference_stat_content_identity() { stat -f "%i:%z:%m" "$1"; }',
|
|
271
|
+
' video_reference_stat_size() { stat -f "%z" "$1"; }',
|
|
272
|
+
"else",
|
|
273
|
+
' video_reference_stat_identity() { stat -Lc "%d:%i:%s:%Y" -- "$1"; }',
|
|
274
|
+
' video_reference_stat_content_identity() { stat -Lc "%d:%i:%s:%Y" -- "$1"; }',
|
|
275
|
+
' video_reference_stat_size() { stat -Lc "%s" -- "$1"; }',
|
|
276
|
+
"fi",
|
|
277
|
+
"video_reference_sha256() {",
|
|
278
|
+
" if command -v sha256sum >/dev/null 2>&1; then",
|
|
279
|
+
" sha256sum \"$1\" | awk '{print $1}'",
|
|
280
|
+
" else",
|
|
281
|
+
" shasum -a 256 \"$1\" | awk '{print $1}'",
|
|
282
|
+
" fi",
|
|
283
|
+
"}",
|
|
284
|
+
'if [ "$video_reference_platform" != Darwin ]; then',
|
|
285
|
+
' fd_path=$(realpath -- "$fd")',
|
|
286
|
+
` [ "$fd_path" = "$expected" ] || exit ${exitCodes.invalidFile}`,
|
|
287
|
+
"fi",
|
|
288
|
+
`[ "$(video_reference_stat_content_identity "$resolved")" = "$(video_reference_stat_content_identity "$fd")" ] || exit ${exitCodes.invalidFile}`,
|
|
289
|
+
`[ "$(realpath -- "$requested")" = "$expected" ] || exit ${exitCodes.invalidFile}`,
|
|
290
|
+
'before=$(video_reference_stat_identity "$fd")',
|
|
291
|
+
'size=$(video_reference_stat_size "$fd")',
|
|
292
|
+
];
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
function workspaceRelativeReferencePath(path: string): string {
|
|
296
|
+
if (!path.startsWith("/workspace/") || /[\0\r\n]/u.test(path)) {
|
|
297
|
+
throw new VideoReferenceInputError(
|
|
298
|
+
"invalid_reference_path",
|
|
299
|
+
"Video references must use an exact file path under /workspace.",
|
|
300
|
+
);
|
|
301
|
+
}
|
|
302
|
+
const segments = path.slice("/workspace/".length).split("/");
|
|
303
|
+
if (segments.some((segment) => segment.length === 0 || segment === "." || segment === "..")) {
|
|
304
|
+
throw new VideoReferenceInputError(
|
|
305
|
+
"invalid_reference_path",
|
|
306
|
+
"Video references must use a canonical file path under /workspace.",
|
|
307
|
+
);
|
|
308
|
+
}
|
|
309
|
+
return `./${segments.join("/")}`;
|
|
310
|
+
}
|
|
311
|
+
|
|
228
312
|
function sniffContentType(prefix: Uint8Array): string {
|
|
229
313
|
if (
|
|
230
314
|
prefix.byteLength >= 8 &&
|
|
@@ -245,7 +329,10 @@ function sniffContentType(prefix: Uint8Array): string {
|
|
|
245
329
|
if (prefix.byteLength >= 12 && Buffer.from(prefix.subarray(4, 8)).toString("ascii") === "ftyp") {
|
|
246
330
|
return "video/mp4";
|
|
247
331
|
}
|
|
248
|
-
throw new
|
|
332
|
+
throw new VideoReferenceInputError(
|
|
333
|
+
"unsupported_reference_media",
|
|
334
|
+
"The video reference is not a supported PNG, JPEG, WebP, or MP4 file.",
|
|
335
|
+
);
|
|
249
336
|
}
|
|
250
337
|
|
|
251
338
|
function shellQuote(value: string): string {
|