@opengeni/worker-bundle 0.16.21 → 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +145 -32
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +5 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +3 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +11 -7
- package/dist/activities/expiring-material-renewal.d.ts +5 -3
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +2 -2
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +13 -26
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
- package/dist/activities/retained-screenshots.d.ts +129 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +2 -2
- package/dist/activities/run-input.d.ts +5 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +4 -4
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +71 -5
- package/dist/activities/video-generation-admission.d.ts +38 -0
- package/dist/activities/video-generation-credential.d.ts +7 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +49 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +11 -1
- package/dist/activities-control-44KBWLSM.js +6885 -0
- package/dist/activities-control-44KBWLSM.js.map +1 -0
- package/dist/activities-control.d.ts +289 -0
- package/dist/activities-turn-K4CEDEGJ.js +14675 -0
- package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +130 -147
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-E3RI3I33.js +828 -0
- package/dist/chunk-E3RI3I33.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-KKCRGAGZ.js +3091 -0
- package/dist/chunk-KKCRGAGZ.js.map +1 -0
- package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
- package/dist/chunk-RAMNYSAW.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/host-export.js +1 -1
- package/dist/index.d.ts +176 -10
- package/dist/index.js +675 -16460
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +22 -0
- package/dist/observability-metrics.d.ts +33 -0
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +40 -7
- package/dist/sandbox-routing.d.ts +13 -6
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1044 -363
- package/dist/workflows/activities.d.ts +7 -1
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +9 -7
- package/package.json +34 -15
- package/src/activities/agent-turn.ts +1905 -559
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +16 -7
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/codex-rotation.ts +19 -5
- package/src/activities/context-compaction.ts +47 -6
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +52 -60
- package/src/activities/expiring-material-renewal.ts +16 -7
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/git-credential-renewal.ts +3 -3
- package/src/activities/goals.ts +6 -2
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +55 -55
- package/src/activities/recording.ts +4 -5
- package/src/activities/retained-screenshot-reaper.ts +202 -0
- package/src/activities/retained-screenshots.ts +1317 -0
- package/src/activities/rig-verification.ts +567 -77
- package/src/activities/run-credential-renewal.ts +9 -2
- package/src/activities/run-input.ts +40 -10
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +96 -7
- package/src/activities/session-state.ts +4 -4
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +87 -5
- package/src/activities/video-generation-admission.ts +270 -0
- package/src/activities/video-generation-credential.ts +34 -0
- package/src/activities/video-generation-reconciliation.ts +578 -0
- package/src/activities/video-media-validation.ts +326 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +263 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +77 -13
- package/src/activities-control.ts +54 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +50 -118
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/host-export-pump.ts +28 -11
- package/src/http.ts +3 -6
- package/src/index.ts +277 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +116 -0
- package/src/observability-metrics.ts +202 -7
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +331 -157
- package/src/sandbox-routing.ts +49 -39
- package/src/worker-service-lifecycle.ts +82 -12
- package/src/workflows/activities.ts +30 -0
- package/src/workflows/file-upload-reaper.ts +6 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +17 -1
- package/dist/activities/redaction.d.ts +0 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/dist/chunk-BJUW5LCM.js.map +0 -1
- package/src/activities/redaction.ts +0 -9
- package/src/activities/toolspace-token-renewal.ts +0 -57
|
@@ -0,0 +1,578 @@
|
|
|
1
|
+
import { environmentsEncryptionKeyBytes } from "@opengeni/config";
|
|
2
|
+
import type { MediaGenerationResult } from "@opengeni/contracts";
|
|
3
|
+
import { type CanonicalVideoGenerationRequest, videoGenerationRequestDigest } from "@opengeni/core";
|
|
4
|
+
import {
|
|
5
|
+
addSessionSystemUpdateWithSourceMutation,
|
|
6
|
+
decryptEnvironmentValue,
|
|
7
|
+
encryptEnvironmentValue,
|
|
8
|
+
getVideoGenerationOperation,
|
|
9
|
+
markVideoGenerationProviderStarted,
|
|
10
|
+
markVideoGenerationReferenceCleaned,
|
|
11
|
+
markVideoGenerationRetaining,
|
|
12
|
+
markVideoGenerationSubmissionIntent,
|
|
13
|
+
markVideoGenerationSubmissionUncertain,
|
|
14
|
+
markVideoGenerationTerminalUpdate,
|
|
15
|
+
markVideoGenerationTerminalUpdateInTransaction,
|
|
16
|
+
mediaGenerationResultForStoredOperation,
|
|
17
|
+
rescheduleVideoGenerationOperation,
|
|
18
|
+
settleVideoGenerationFailure,
|
|
19
|
+
settleVideoGenerationReady,
|
|
20
|
+
type VideoGenerationOperationWithReferences,
|
|
21
|
+
} from "@opengeni/db";
|
|
22
|
+
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
23
|
+
import type { ObjectStorage } from "@opengeni/storage";
|
|
24
|
+
import {
|
|
25
|
+
buildGatewayVideoStartBody,
|
|
26
|
+
GatewayVideoApiError,
|
|
27
|
+
getGatewayVideoGenerationStatus,
|
|
28
|
+
startGatewayVideoGenerationWithBody,
|
|
29
|
+
type GatewayVideoReferenceGrant,
|
|
30
|
+
} from "./gateway-video-generation";
|
|
31
|
+
import {
|
|
32
|
+
downloadGeneratedVideoToVerifiedTemp,
|
|
33
|
+
retainVerifiedGeneratedVideo,
|
|
34
|
+
} from "./video-output-retention";
|
|
35
|
+
import { decryptVideoGenerationApiKey } from "./video-generation-credential";
|
|
36
|
+
import type {
|
|
37
|
+
TurnActivityServices,
|
|
38
|
+
VideoGenerationReconcileResult,
|
|
39
|
+
VideoGenerationTerminalStatus,
|
|
40
|
+
} from "./types";
|
|
41
|
+
|
|
42
|
+
const RECONCILE_ERROR_MAX = 1_000;
|
|
43
|
+
|
|
44
|
+
export function createVideoGenerationActivities(services: () => Promise<TurnActivityServices>) {
|
|
45
|
+
return {
|
|
46
|
+
reconcileVideoGenerationOperation: async (input: {
|
|
47
|
+
accountId: string;
|
|
48
|
+
workspaceId: string;
|
|
49
|
+
operationId: string;
|
|
50
|
+
}): Promise<VideoGenerationReconcileResult> => {
|
|
51
|
+
const service = await services();
|
|
52
|
+
if (!service.objectStorage) {
|
|
53
|
+
throw new Error("Video generation reconciliation requires object storage");
|
|
54
|
+
}
|
|
55
|
+
return await reconcileVideoGenerationOperation(service, input);
|
|
56
|
+
},
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export async function reconcileVideoGenerationOperation(
|
|
61
|
+
service: TurnActivityServices,
|
|
62
|
+
input: { accountId: string; workspaceId: string; operationId: string },
|
|
63
|
+
): Promise<VideoGenerationReconcileResult> {
|
|
64
|
+
const storage = requireStorage(service.objectStorage);
|
|
65
|
+
let operation = await requireOperation(service, input);
|
|
66
|
+
if (isTerminal(operation.status)) {
|
|
67
|
+
await deliverTerminalResult(service, operation);
|
|
68
|
+
return { action: "terminal", status: operation.status };
|
|
69
|
+
}
|
|
70
|
+
const key = environmentsEncryptionKeyBytes(service.settings);
|
|
71
|
+
if (!key) throw new Error("Video generation recovery encryption key is unavailable");
|
|
72
|
+
if (!operation.credentialEncrypted) {
|
|
73
|
+
throw new Error("Video provider credential lease was erased too early");
|
|
74
|
+
}
|
|
75
|
+
const apiKey = decryptVideoGenerationApiKey(key, operation.credentialEncrypted);
|
|
76
|
+
|
|
77
|
+
if (operation.status === "accepted") {
|
|
78
|
+
const canonical = decryptCanonicalRequest(operation, key);
|
|
79
|
+
const grants: GatewayVideoReferenceGrant[] = [];
|
|
80
|
+
const expirations: Date[] = [];
|
|
81
|
+
for (const reference of operation.references) {
|
|
82
|
+
if (!reference.stagingObjectKey) {
|
|
83
|
+
throw new Error("Video generation reference staging object is missing");
|
|
84
|
+
}
|
|
85
|
+
const signed = await storage.createGetUrl({
|
|
86
|
+
key: reference.stagingObjectKey,
|
|
87
|
+
expiresInSeconds: service.settings.videoGenerationReferenceUrlTtlSeconds,
|
|
88
|
+
});
|
|
89
|
+
grants.push({
|
|
90
|
+
role: parseReferenceRole(reference.role),
|
|
91
|
+
url: signed.url,
|
|
92
|
+
mediaType: reference.contentType,
|
|
93
|
+
});
|
|
94
|
+
expirations.push(signed.expiresAt);
|
|
95
|
+
}
|
|
96
|
+
const providerRequestExpiresAt = expirations.length
|
|
97
|
+
? new Date(Math.min(...expirations.map((value) => value.getTime())))
|
|
98
|
+
: operation.recoveryDeadlineAt;
|
|
99
|
+
const body = buildGatewayVideoStartBody(canonical, grants);
|
|
100
|
+
operation = retainReferences(
|
|
101
|
+
operation,
|
|
102
|
+
await markVideoGenerationSubmissionIntent(service.db, {
|
|
103
|
+
accountId: input.accountId,
|
|
104
|
+
workspaceId: input.workspaceId,
|
|
105
|
+
operationId: operation.id,
|
|
106
|
+
requestDigest: operation.requestDigest,
|
|
107
|
+
encryptedProviderRequest: encryptEnvironmentValue(key, JSON.stringify(body)),
|
|
108
|
+
providerRequestExpiresAt,
|
|
109
|
+
nextReconcileAt: new Date(),
|
|
110
|
+
}),
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
if (operation.status === "submission_uncertain") {
|
|
115
|
+
const deadline = Math.min(
|
|
116
|
+
operation.recoveryDeadlineAt.getTime(),
|
|
117
|
+
operation.providerRequestExpiresAt?.getTime() ?? 0,
|
|
118
|
+
);
|
|
119
|
+
if (Date.now() >= deadline) {
|
|
120
|
+
operation = retainReferences(
|
|
121
|
+
operation,
|
|
122
|
+
await settleVideoGenerationFailure(service.db, {
|
|
123
|
+
accountId: input.accountId,
|
|
124
|
+
workspaceId: input.workspaceId,
|
|
125
|
+
operationId: operation.id,
|
|
126
|
+
requestDigest: operation.requestDigest,
|
|
127
|
+
status: "outcome_unknown",
|
|
128
|
+
publicReason:
|
|
129
|
+
"The video provider did not confirm whether generation started. It was not retried with a new request.",
|
|
130
|
+
}),
|
|
131
|
+
);
|
|
132
|
+
await deliverTerminalResult(service, operation);
|
|
133
|
+
return { action: "terminal", status: terminalStatus(operation.status) };
|
|
134
|
+
}
|
|
135
|
+
const body = decryptProviderBody(operation, key);
|
|
136
|
+
try {
|
|
137
|
+
const started = await startGatewayVideoGenerationWithBody({
|
|
138
|
+
apiKey,
|
|
139
|
+
modelId: operation.modelId,
|
|
140
|
+
body,
|
|
141
|
+
idempotencyKey: operation.providerIdempotencyKey,
|
|
142
|
+
});
|
|
143
|
+
operation = retainReferences(
|
|
144
|
+
operation,
|
|
145
|
+
await markVideoGenerationProviderStarted(service.db, {
|
|
146
|
+
accountId: input.accountId,
|
|
147
|
+
workspaceId: input.workspaceId,
|
|
148
|
+
operationId: operation.id,
|
|
149
|
+
requestDigest: operation.requestDigest,
|
|
150
|
+
providerJobId: started.providerJobId,
|
|
151
|
+
nextReconcileAt: nextPoll(service),
|
|
152
|
+
}),
|
|
153
|
+
);
|
|
154
|
+
return waitResult(service);
|
|
155
|
+
} catch (error) {
|
|
156
|
+
if (isPermanentGatewayFailure(error)) {
|
|
157
|
+
operation = retainReferences(
|
|
158
|
+
operation,
|
|
159
|
+
await settleVideoGenerationFailure(service.db, {
|
|
160
|
+
accountId: input.accountId,
|
|
161
|
+
workspaceId: input.workspaceId,
|
|
162
|
+
operationId: operation.id,
|
|
163
|
+
requestDigest: operation.requestDigest,
|
|
164
|
+
status: "provider_failed",
|
|
165
|
+
publicReason: "The video provider rejected the generation request.",
|
|
166
|
+
privateError: privateError(error),
|
|
167
|
+
}),
|
|
168
|
+
);
|
|
169
|
+
await deliverTerminalResult(service, operation);
|
|
170
|
+
return { action: "terminal", status: terminalStatus(operation.status) };
|
|
171
|
+
}
|
|
172
|
+
await markVideoGenerationSubmissionUncertain(service.db, {
|
|
173
|
+
accountId: input.accountId,
|
|
174
|
+
workspaceId: input.workspaceId,
|
|
175
|
+
operationId: operation.id,
|
|
176
|
+
requestDigest: operation.requestDigest,
|
|
177
|
+
nextReconcileAt: nextPoll(service),
|
|
178
|
+
error: privateError(error),
|
|
179
|
+
});
|
|
180
|
+
return waitResult(service);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
if (operation.status !== "provider_started" && operation.status !== "retaining") {
|
|
185
|
+
throw new Error(`Video operation cannot be reconciled from ${operation.status}`);
|
|
186
|
+
}
|
|
187
|
+
if (!operation.providerJobId) throw new Error("Video provider job identity is missing");
|
|
188
|
+
let status;
|
|
189
|
+
try {
|
|
190
|
+
status = await getGatewayVideoGenerationStatus({
|
|
191
|
+
apiKey,
|
|
192
|
+
modelId: operation.modelId,
|
|
193
|
+
providerJobId: operation.providerJobId,
|
|
194
|
+
});
|
|
195
|
+
} catch (error) {
|
|
196
|
+
if (Date.now() < operation.recoveryDeadlineAt.getTime() && !isPermanentGatewayFailure(error)) {
|
|
197
|
+
await rescheduleVideoGenerationOperation(service.db, {
|
|
198
|
+
accountId: input.accountId,
|
|
199
|
+
workspaceId: input.workspaceId,
|
|
200
|
+
operationId: operation.id,
|
|
201
|
+
requestDigest: operation.requestDigest,
|
|
202
|
+
nextReconcileAt: nextPoll(service),
|
|
203
|
+
error: privateError(error),
|
|
204
|
+
});
|
|
205
|
+
return waitResult(service);
|
|
206
|
+
}
|
|
207
|
+
const failureStatus = operation.status === "retaining" ? "retention_failed" : "outcome_unknown";
|
|
208
|
+
operation = retainReferences(
|
|
209
|
+
operation,
|
|
210
|
+
await settleVideoGenerationFailure(service.db, {
|
|
211
|
+
accountId: input.accountId,
|
|
212
|
+
workspaceId: input.workspaceId,
|
|
213
|
+
operationId: operation.id,
|
|
214
|
+
requestDigest: operation.requestDigest,
|
|
215
|
+
status: failureStatus,
|
|
216
|
+
publicReason:
|
|
217
|
+
failureStatus === "retention_failed"
|
|
218
|
+
? "The generated video could not be retained."
|
|
219
|
+
: "The video provider stopped reporting the accepted generation.",
|
|
220
|
+
privateError: privateError(error),
|
|
221
|
+
}),
|
|
222
|
+
);
|
|
223
|
+
await deliverTerminalResult(service, operation);
|
|
224
|
+
return { action: "terminal", status: terminalStatus(operation.status) };
|
|
225
|
+
}
|
|
226
|
+
if (status.status === "pending") {
|
|
227
|
+
await rescheduleVideoGenerationOperation(service.db, {
|
|
228
|
+
accountId: input.accountId,
|
|
229
|
+
workspaceId: input.workspaceId,
|
|
230
|
+
operationId: operation.id,
|
|
231
|
+
requestDigest: operation.requestDigest,
|
|
232
|
+
nextReconcileAt: nextPoll(service),
|
|
233
|
+
});
|
|
234
|
+
return waitResult(service);
|
|
235
|
+
}
|
|
236
|
+
if (status.status === "error") {
|
|
237
|
+
const failureStatus = operation.status === "retaining" ? "retention_failed" : "provider_failed";
|
|
238
|
+
operation = retainReferences(
|
|
239
|
+
operation,
|
|
240
|
+
await settleVideoGenerationFailure(service.db, {
|
|
241
|
+
accountId: input.accountId,
|
|
242
|
+
workspaceId: input.workspaceId,
|
|
243
|
+
operationId: operation.id,
|
|
244
|
+
requestDigest: operation.requestDigest,
|
|
245
|
+
status: failureStatus,
|
|
246
|
+
publicReason: status.publicReason,
|
|
247
|
+
}),
|
|
248
|
+
);
|
|
249
|
+
await deliverTerminalResult(service, operation);
|
|
250
|
+
return { action: "terminal", status: terminalStatus(operation.status) };
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
if (operation.status === "provider_started") {
|
|
254
|
+
operation = retainReferences(
|
|
255
|
+
operation,
|
|
256
|
+
await markVideoGenerationRetaining(service.db, {
|
|
257
|
+
accountId: input.accountId,
|
|
258
|
+
workspaceId: input.workspaceId,
|
|
259
|
+
operationId: operation.id,
|
|
260
|
+
requestDigest: operation.requestDigest,
|
|
261
|
+
}),
|
|
262
|
+
);
|
|
263
|
+
}
|
|
264
|
+
try {
|
|
265
|
+
const canonical = decryptCanonicalRequest(operation, key);
|
|
266
|
+
const downloaded = await downloadGeneratedVideoToVerifiedTemp({
|
|
267
|
+
url: status.outputUrl,
|
|
268
|
+
mediaType: status.mediaType,
|
|
269
|
+
settings: service.settings,
|
|
270
|
+
expectedDurationSeconds: canonical.durationSeconds,
|
|
271
|
+
});
|
|
272
|
+
try {
|
|
273
|
+
const retained = await retainVerifiedGeneratedVideo({
|
|
274
|
+
storage,
|
|
275
|
+
temp: downloaded.temp,
|
|
276
|
+
facts: downloaded.facts,
|
|
277
|
+
});
|
|
278
|
+
const settled = await settleVideoGenerationReady(service.db, {
|
|
279
|
+
accountId: input.accountId,
|
|
280
|
+
workspaceId: input.workspaceId,
|
|
281
|
+
operationId: operation.id,
|
|
282
|
+
requestDigest: operation.requestDigest,
|
|
283
|
+
fileId: operation.expectedFileId,
|
|
284
|
+
bucket: retained.bucket,
|
|
285
|
+
objectKey: retained.objectKey,
|
|
286
|
+
sizeBytes: retained.sizeBytes,
|
|
287
|
+
sha256: retained.sha256,
|
|
288
|
+
facts: retained.facts,
|
|
289
|
+
});
|
|
290
|
+
operation = { ...settled.operation, references: operation.references };
|
|
291
|
+
} finally {
|
|
292
|
+
await downloaded.temp.cleanup();
|
|
293
|
+
}
|
|
294
|
+
} catch (error) {
|
|
295
|
+
if (Date.now() < operation.recoveryDeadlineAt.getTime()) {
|
|
296
|
+
await rescheduleVideoGenerationOperation(service.db, {
|
|
297
|
+
accountId: input.accountId,
|
|
298
|
+
workspaceId: input.workspaceId,
|
|
299
|
+
operationId: operation.id,
|
|
300
|
+
requestDigest: operation.requestDigest,
|
|
301
|
+
nextReconcileAt: nextPoll(service),
|
|
302
|
+
error: privateError(error),
|
|
303
|
+
});
|
|
304
|
+
return waitResult(service);
|
|
305
|
+
}
|
|
306
|
+
operation = retainReferences(
|
|
307
|
+
operation,
|
|
308
|
+
await settleVideoGenerationFailure(service.db, {
|
|
309
|
+
accountId: input.accountId,
|
|
310
|
+
workspaceId: input.workspaceId,
|
|
311
|
+
operationId: operation.id,
|
|
312
|
+
requestDigest: operation.requestDigest,
|
|
313
|
+
status: "retention_failed",
|
|
314
|
+
publicReason: "The generated video could not be retained.",
|
|
315
|
+
privateError: privateError(error),
|
|
316
|
+
}),
|
|
317
|
+
);
|
|
318
|
+
}
|
|
319
|
+
await deliverTerminalResult(service, operation);
|
|
320
|
+
return { action: "terminal", status: terminalStatus(operation.status) };
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
async function deliverTerminalResult(
|
|
324
|
+
service: TurnActivityServices,
|
|
325
|
+
operation: VideoGenerationOperationWithReferences,
|
|
326
|
+
): Promise<void> {
|
|
327
|
+
if (
|
|
328
|
+
operation.terminalUpdateState === "delivered" ||
|
|
329
|
+
operation.terminalUpdateState === "suppressed"
|
|
330
|
+
) {
|
|
331
|
+
await cleanupTerminalReferences(service, operation);
|
|
332
|
+
return;
|
|
333
|
+
}
|
|
334
|
+
if (!operation.sessionId) {
|
|
335
|
+
await markVideoGenerationTerminalUpdate(service.db, {
|
|
336
|
+
accountId: operation.accountId,
|
|
337
|
+
workspaceId: operation.workspaceId,
|
|
338
|
+
operationId: operation.id,
|
|
339
|
+
state: "suppressed",
|
|
340
|
+
});
|
|
341
|
+
await cleanupTerminalReferences(service, operation);
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
344
|
+
const payload = await terminalPayload(service, operation);
|
|
345
|
+
const result = await addSessionSystemUpdateWithSourceMutation(
|
|
346
|
+
service.db,
|
|
347
|
+
{
|
|
348
|
+
accountId: operation.accountId,
|
|
349
|
+
workspaceId: operation.workspaceId,
|
|
350
|
+
sessionId: operation.sessionId,
|
|
351
|
+
kind: "media_generation_result",
|
|
352
|
+
classification: payload.status === "ready" ? "success" : "failure",
|
|
353
|
+
sourceId: operation.id,
|
|
354
|
+
dedupeKey: `video-generation-terminal:${operation.id}`,
|
|
355
|
+
summary:
|
|
356
|
+
payload.status === "ready"
|
|
357
|
+
? "The requested video is ready and available in the workspace."
|
|
358
|
+
: payload.boundedPublicReason,
|
|
359
|
+
payload,
|
|
360
|
+
lineage: { operationId: operation.id },
|
|
361
|
+
},
|
|
362
|
+
async (tx, _wakeEventId, updateId) => {
|
|
363
|
+
const marked = await markVideoGenerationTerminalUpdateInTransaction(tx, {
|
|
364
|
+
workspaceId: operation.workspaceId,
|
|
365
|
+
operationId: operation.id,
|
|
366
|
+
state: updateId ? "leased" : "suppressed",
|
|
367
|
+
...(updateId ? { updateId } : {}),
|
|
368
|
+
});
|
|
369
|
+
if (!marked && operation.terminalUpdateState !== "leased") {
|
|
370
|
+
throw new Error("Video terminal result lost its disposition fence");
|
|
371
|
+
}
|
|
372
|
+
},
|
|
373
|
+
);
|
|
374
|
+
if (result.reason === "session_cancelled") {
|
|
375
|
+
await cleanupTerminalReferences(service, operation);
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
if (result.added && result.events.length > 0) {
|
|
379
|
+
await publishDurableSessionEvents(
|
|
380
|
+
service.bus,
|
|
381
|
+
operation.workspaceId,
|
|
382
|
+
operation.sessionId,
|
|
383
|
+
result.events,
|
|
384
|
+
);
|
|
385
|
+
}
|
|
386
|
+
if (result.shouldWake && service.wakeSessionWorkflow) {
|
|
387
|
+
if (result.workflowWakeRevision === null) {
|
|
388
|
+
throw new Error("Video terminal result wake has no revision");
|
|
389
|
+
}
|
|
390
|
+
await service.wakeSessionWorkflow({
|
|
391
|
+
accountId: operation.accountId,
|
|
392
|
+
workspaceId: operation.workspaceId,
|
|
393
|
+
sessionId: operation.sessionId,
|
|
394
|
+
workflowId: result.temporalWorkflowId ?? `session-${operation.sessionId}`,
|
|
395
|
+
wakeRevision: result.workflowWakeRevision,
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
await cleanupTerminalReferences(service, operation);
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
async function cleanupTerminalReferences(
|
|
402
|
+
service: TurnActivityServices,
|
|
403
|
+
operation: VideoGenerationOperationWithReferences,
|
|
404
|
+
): Promise<void> {
|
|
405
|
+
const storage = requireStorage(service.objectStorage);
|
|
406
|
+
for (const reference of operation.references) {
|
|
407
|
+
if (!reference.stagingObjectKey || reference.cleanedAt) continue;
|
|
408
|
+
// `outcome_unknown` is only terminal after the frozen provider grant and
|
|
409
|
+
// recovery deadline have elapsed. Other terminal outcomes have observed a
|
|
410
|
+
// provider terminal state, so their immutable staging inputs are no longer
|
|
411
|
+
// needed. Delete first, then CAS the row; a crash repeats an idempotent delete.
|
|
412
|
+
if (
|
|
413
|
+
operation.status === "outcome_unknown" &&
|
|
414
|
+
Date.now() <
|
|
415
|
+
Math.max(
|
|
416
|
+
operation.recoveryDeadlineAt.getTime(),
|
|
417
|
+
reference.grantExpiresAt?.getTime() ?? 0,
|
|
418
|
+
reference.cleanupAfter?.getTime() ?? 0,
|
|
419
|
+
)
|
|
420
|
+
) {
|
|
421
|
+
continue;
|
|
422
|
+
}
|
|
423
|
+
const stagingObjectKey = reference.stagingObjectKey;
|
|
424
|
+
await storage.deleteObject(stagingObjectKey);
|
|
425
|
+
const marked = await markVideoGenerationReferenceCleaned(service.db, {
|
|
426
|
+
accountId: operation.accountId,
|
|
427
|
+
workspaceId: operation.workspaceId,
|
|
428
|
+
operationId: operation.id,
|
|
429
|
+
ordinal: reference.ordinal,
|
|
430
|
+
stagingObjectKey,
|
|
431
|
+
});
|
|
432
|
+
if (!marked) throw new Error("Video reference cleanup lost its durable identity");
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
async function terminalPayload(
|
|
437
|
+
service: TurnActivityServices,
|
|
438
|
+
operation: VideoGenerationOperationWithReferences,
|
|
439
|
+
): Promise<MediaGenerationResult> {
|
|
440
|
+
return await mediaGenerationResultForStoredOperation(service.db, operation);
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
async function requireOperation(
|
|
444
|
+
service: TurnActivityServices,
|
|
445
|
+
input: { workspaceId: string; operationId: string },
|
|
446
|
+
): Promise<VideoGenerationOperationWithReferences> {
|
|
447
|
+
const operation = await getVideoGenerationOperation(
|
|
448
|
+
service.db,
|
|
449
|
+
input.workspaceId,
|
|
450
|
+
input.operationId,
|
|
451
|
+
);
|
|
452
|
+
if (!operation) throw new Error("Video generation operation was not found");
|
|
453
|
+
return operation;
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
function decryptCanonicalRequest(
|
|
457
|
+
operation: VideoGenerationOperationWithReferences,
|
|
458
|
+
key: Uint8Array,
|
|
459
|
+
): CanonicalVideoGenerationRequest {
|
|
460
|
+
if (!operation.requestEncrypted) throw new Error("Video generation request was erased too early");
|
|
461
|
+
const parsed = JSON.parse(decryptEnvironmentValue(key, operation.requestEncrypted)) as unknown;
|
|
462
|
+
const request = validateCanonicalRequest(parsed);
|
|
463
|
+
if (videoGenerationRequestDigest(request) !== operation.requestDigest) {
|
|
464
|
+
throw new Error("Video generation request digest is inconsistent");
|
|
465
|
+
}
|
|
466
|
+
return request;
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
function validateCanonicalRequest(value: unknown): CanonicalVideoGenerationRequest {
|
|
470
|
+
const row = record(value);
|
|
471
|
+
const references = Array.isArray(row?.references) ? row.references.map(record) : null;
|
|
472
|
+
if (
|
|
473
|
+
row?.schemaVersion !== 1 ||
|
|
474
|
+
typeof row.modelId !== "string" ||
|
|
475
|
+
typeof row.prompt !== "string" ||
|
|
476
|
+
![
|
|
477
|
+
"text",
|
|
478
|
+
"first_frame",
|
|
479
|
+
"first_and_last_frames",
|
|
480
|
+
"image_reference",
|
|
481
|
+
"video_reference",
|
|
482
|
+
].includes(String(row.sourceMode)) ||
|
|
483
|
+
!references ||
|
|
484
|
+
!Number.isInteger(row.durationSeconds) ||
|
|
485
|
+
!["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"].includes(String(row.aspectRatio)) ||
|
|
486
|
+
!["480p", "720p"].includes(String(row.resolution)) ||
|
|
487
|
+
typeof row.generateAudio !== "boolean" ||
|
|
488
|
+
references.some(
|
|
489
|
+
(reference) =>
|
|
490
|
+
!reference ||
|
|
491
|
+
!["first_frame", "last_frame", "image_reference", "video_reference"].includes(
|
|
492
|
+
String(reference.role),
|
|
493
|
+
) ||
|
|
494
|
+
typeof reference.contentSha256 !== "string" ||
|
|
495
|
+
!/^[0-9a-f]{64}$/u.test(reference.contentSha256) ||
|
|
496
|
+
typeof reference.contentType !== "string" ||
|
|
497
|
+
!Number.isSafeInteger(reference.byteSize),
|
|
498
|
+
)
|
|
499
|
+
) {
|
|
500
|
+
throw new Error("Encrypted video generation request is malformed");
|
|
501
|
+
}
|
|
502
|
+
return value as CanonicalVideoGenerationRequest;
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
function decryptProviderBody(
|
|
506
|
+
operation: VideoGenerationOperationWithReferences,
|
|
507
|
+
key: Uint8Array,
|
|
508
|
+
): Record<string, unknown> {
|
|
509
|
+
if (!operation.providerRequestEncrypted) throw new Error("Frozen provider request is missing");
|
|
510
|
+
const body = record(JSON.parse(decryptEnvironmentValue(key, operation.providerRequestEncrypted)));
|
|
511
|
+
if (!body || Buffer.byteLength(JSON.stringify(body)) > 256 * 1024) {
|
|
512
|
+
throw new Error("Frozen provider request is malformed");
|
|
513
|
+
}
|
|
514
|
+
return body;
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
function parseReferenceRole(value: string): GatewayVideoReferenceGrant["role"] {
|
|
518
|
+
if (
|
|
519
|
+
value !== "first_frame" &&
|
|
520
|
+
value !== "last_frame" &&
|
|
521
|
+
value !== "image_reference" &&
|
|
522
|
+
value !== "video_reference"
|
|
523
|
+
) {
|
|
524
|
+
throw new Error("Video reference role is invalid");
|
|
525
|
+
}
|
|
526
|
+
return value;
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
function requireStorage(storage: ObjectStorage | null): ObjectStorage {
|
|
530
|
+
if (!storage) throw new Error("Video generation requires object storage");
|
|
531
|
+
return storage;
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
function nextPoll(service: TurnActivityServices): Date {
|
|
535
|
+
return new Date(Date.now() + service.settings.videoGenerationPollIntervalMs);
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
function waitResult(service: TurnActivityServices): VideoGenerationReconcileResult {
|
|
539
|
+
return { action: "waiting", delayMs: service.settings.videoGenerationPollIntervalMs };
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
function isPermanentGatewayFailure(error: unknown): boolean {
|
|
543
|
+
return error instanceof GatewayVideoApiError && !error.retryable;
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
function privateError(error: unknown): string {
|
|
547
|
+
return (error instanceof Error ? `${error.name}: ${error.message}` : String(error))
|
|
548
|
+
.replace(/https?:\/\/\S+/gu, "[url omitted]")
|
|
549
|
+
.slice(0, RECONCILE_ERROR_MAX);
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
function record(value: unknown): Record<string, unknown> | null {
|
|
553
|
+
return value && typeof value === "object" && !Array.isArray(value)
|
|
554
|
+
? (value as Record<string, unknown>)
|
|
555
|
+
: null;
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
function isTerminal(status: string): status is VideoGenerationTerminalStatus {
|
|
559
|
+
return [
|
|
560
|
+
"completed",
|
|
561
|
+
"provider_failed",
|
|
562
|
+
"cancelled_before_submit",
|
|
563
|
+
"outcome_unknown",
|
|
564
|
+
"retention_failed",
|
|
565
|
+
].includes(status);
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
function terminalStatus(status: string): VideoGenerationTerminalStatus {
|
|
569
|
+
if (!isTerminal(status)) throw new Error(`Video generation status ${status} is not terminal`);
|
|
570
|
+
return status;
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
function retainReferences(
|
|
574
|
+
current: VideoGenerationOperationWithReferences,
|
|
575
|
+
next: Omit<VideoGenerationOperationWithReferences, "references">,
|
|
576
|
+
): VideoGenerationOperationWithReferences {
|
|
577
|
+
return { ...next, references: current.references };
|
|
578
|
+
}
|