@mux/ai 0.40.0 → 0.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{index-UhRs9t1h.d.ts → index-NkSOF92U.d.ts} +211 -78
- package/dist/index.d.ts +3 -2
- package/dist/index.js +656 -167
- package/dist/index.js.map +1 -1
- package/dist/primitives/index.js +1 -0
- package/dist/primitives/index.js.map +1 -1
- package/dist/prompts/index.d.ts +335 -0
- package/dist/prompts/index.js +490 -0
- package/dist/prompts/index.js.map +1 -0
- package/dist/workflows/index.d.ts +2 -1
- package/dist/workflows/index.js +618 -142
- package/dist/workflows/index.js.map +1 -1
- package/package.json +7 -1
package/dist/index.js
CHANGED
|
@@ -5,7 +5,7 @@ var __export = (target, all) => {
|
|
|
5
5
|
};
|
|
6
6
|
|
|
7
7
|
// package.json
|
|
8
|
-
var version = "0.
|
|
8
|
+
var version = "0.42.0";
|
|
9
9
|
|
|
10
10
|
// src/lib/language-codes.ts
|
|
11
11
|
var ISO639_1_TO_3 = {
|
|
@@ -446,6 +446,7 @@ var PROMPT_TAG_NAMES = [
|
|
|
446
446
|
"answer_guidelines",
|
|
447
447
|
"relevance_filtering",
|
|
448
448
|
"transcript_guidance",
|
|
449
|
+
"scene_guidance",
|
|
449
450
|
"capabilities",
|
|
450
451
|
"language_guidelines",
|
|
451
452
|
"tone_guidance",
|
|
@@ -1113,6 +1114,234 @@ async function resolveMuxSigningContext(credentials) {
|
|
|
1113
1114
|
return { keyId, keySecret };
|
|
1114
1115
|
}
|
|
1115
1116
|
|
|
1117
|
+
// src/lib/mux-tracks.ts
|
|
1118
|
+
var MUX_TRACK_PASSTHROUGH_MAX_CHARS = 255;
|
|
1119
|
+
function normalizeTrackLanguageCode(code) {
|
|
1120
|
+
if (!code) {
|
|
1121
|
+
return void 0;
|
|
1122
|
+
}
|
|
1123
|
+
const primarySubtag = code.trim().toLowerCase().split(/[-_]/)[0];
|
|
1124
|
+
if (!primarySubtag || primarySubtag === "auto" || isUndeterminedLanguageCode(primarySubtag)) {
|
|
1125
|
+
return void 0;
|
|
1126
|
+
}
|
|
1127
|
+
return toISO639_1(primarySubtag);
|
|
1128
|
+
}
|
|
1129
|
+
function normalizeTrackName(name) {
|
|
1130
|
+
const normalized = name?.trim().toLowerCase();
|
|
1131
|
+
return normalized || void 0;
|
|
1132
|
+
}
|
|
1133
|
+
function isMuxGeneratedTextTrack(track) {
|
|
1134
|
+
return track.text_source === "generated_vod" || track.text_source === "generated_live" || track.text_source === "generated_live_final";
|
|
1135
|
+
}
|
|
1136
|
+
function targetType(target) {
|
|
1137
|
+
return target.type ?? "text";
|
|
1138
|
+
}
|
|
1139
|
+
function listTracks(asset, type) {
|
|
1140
|
+
return (asset.tracks ?? []).filter((track) => track.type === type && track.status !== "deleted" && !!track.id);
|
|
1141
|
+
}
|
|
1142
|
+
function summarizeTextTrack(track) {
|
|
1143
|
+
return {
|
|
1144
|
+
id: track.id,
|
|
1145
|
+
type: track.type === "audio" ? "audio" : "text",
|
|
1146
|
+
name: track.name,
|
|
1147
|
+
languageCode: track.language_code,
|
|
1148
|
+
status: track.status,
|
|
1149
|
+
textSource: track.text_source,
|
|
1150
|
+
passthrough: track.passthrough,
|
|
1151
|
+
...track.type === "audio" ? { primary: track.primary === true } : {}
|
|
1152
|
+
};
|
|
1153
|
+
}
|
|
1154
|
+
function findSameLanguageTextTracks(asset, target) {
|
|
1155
|
+
const targetLanguage = normalizeTrackLanguageCode(target.languageCode);
|
|
1156
|
+
if (!targetLanguage) {
|
|
1157
|
+
return [];
|
|
1158
|
+
}
|
|
1159
|
+
const type = targetType(target);
|
|
1160
|
+
return listTracks(asset, type).filter(
|
|
1161
|
+
(track) => (type === "audio" || track.text_type === "subtitles") && normalizeTrackLanguageCode(track.language_code) === targetLanguage
|
|
1162
|
+
);
|
|
1163
|
+
}
|
|
1164
|
+
function findNameCollisionTextTracks(asset, target) {
|
|
1165
|
+
const targetName = normalizeTrackName(target.name);
|
|
1166
|
+
if (!targetName) {
|
|
1167
|
+
return [];
|
|
1168
|
+
}
|
|
1169
|
+
return listTracks(asset, targetType(target)).filter((track) => normalizeTrackName(track.name) === targetName);
|
|
1170
|
+
}
|
|
1171
|
+
function describeTracks(tracks) {
|
|
1172
|
+
return tracks.map((track) => `${track.name ?? "(unnamed)"} [${track.language_code ?? "?"}, ${track.type === "audio" ? track.primary ? "primary audio" : "audio" : track.text_source ?? "unknown source"}, ${track.status ?? "ready"}]`).join("; ");
|
|
1173
|
+
}
|
|
1174
|
+
function planTextTrackReplacement(asset, target, policy, options = {}) {
|
|
1175
|
+
const label = targetType(target) === "audio" ? "Audio" : "Text";
|
|
1176
|
+
const keep = new Set(options.keepTrackIds ?? []);
|
|
1177
|
+
const conflicts = /* @__PURE__ */ new Map();
|
|
1178
|
+
for (const track of [...findSameLanguageTextTracks(asset, target), ...findNameCollisionTextTracks(asset, target)]) {
|
|
1179
|
+
if (!keep.has(track.id)) {
|
|
1180
|
+
conflicts.set(track.id, track);
|
|
1181
|
+
}
|
|
1182
|
+
}
|
|
1183
|
+
const conflicting = [...conflicts.values()];
|
|
1184
|
+
if (conflicting.length === 0) {
|
|
1185
|
+
return { kind: "clear" };
|
|
1186
|
+
}
|
|
1187
|
+
if (policy === "fail") {
|
|
1188
|
+
return {
|
|
1189
|
+
kind: "blocked",
|
|
1190
|
+
reason: `${label} track(s) already exist for language '${target.languageCode}' or name '${target.name}': ${describeTracks(conflicting)}. Set replaceExistingTracks to replace them.`,
|
|
1191
|
+
tracks: conflicting.map(summarizeTextTrack)
|
|
1192
|
+
};
|
|
1193
|
+
}
|
|
1194
|
+
const primary = conflicting.filter((track) => track.type === "audio" && track.primary === true);
|
|
1195
|
+
if (primary.length > 0) {
|
|
1196
|
+
return {
|
|
1197
|
+
kind: "blocked",
|
|
1198
|
+
reason: `The asset's primary audio track is in the way for language '${target.languageCode}' or name '${target.name}': ${describeTracks(primary)}. Primary audio cannot be replaced; choose a different trackName.`,
|
|
1199
|
+
tracks: primary.map(summarizeTextTrack)
|
|
1200
|
+
};
|
|
1201
|
+
}
|
|
1202
|
+
if (policy === "replace_generated") {
|
|
1203
|
+
const notGenerated = conflicting.filter((track) => track.type !== "text" || !isMuxGeneratedTextTrack(track));
|
|
1204
|
+
if (notGenerated.length > 0) {
|
|
1205
|
+
return {
|
|
1206
|
+
kind: "blocked",
|
|
1207
|
+
reason: `${label} track(s) that are not Mux-generated exist for language '${target.languageCode}' or name '${target.name}': ${describeTracks(notGenerated)}. Use replaceExistingTracks: "replace_all" to replace them.`,
|
|
1208
|
+
tracks: notGenerated.map(summarizeTextTrack)
|
|
1209
|
+
};
|
|
1210
|
+
}
|
|
1211
|
+
}
|
|
1212
|
+
return { kind: "replace", toDelete: conflicting.map(summarizeTextTrack) };
|
|
1213
|
+
}
|
|
1214
|
+
function isDuplicateTrackNameError(error) {
|
|
1215
|
+
if (typeof error !== "object" || error === null) {
|
|
1216
|
+
return false;
|
|
1217
|
+
}
|
|
1218
|
+
const record = error;
|
|
1219
|
+
if (record.status !== 400) {
|
|
1220
|
+
return false;
|
|
1221
|
+
}
|
|
1222
|
+
const messages = Array.isArray(record.error?.messages) ? record.error.messages : [];
|
|
1223
|
+
const haystack = [...messages, record.message].filter((value) => typeof value === "string").map((value) => value.toLowerCase());
|
|
1224
|
+
return haystack.some((value) => value.includes("is not unique"));
|
|
1225
|
+
}
|
|
1226
|
+
function isNotFoundError(error) {
|
|
1227
|
+
return typeof error === "object" && error !== null && error.status === 404;
|
|
1228
|
+
}
|
|
1229
|
+
function buildMuxAiTrackPassthrough(workflow) {
|
|
1230
|
+
return JSON.stringify({ mux_ai: { workflow } });
|
|
1231
|
+
}
|
|
1232
|
+
function validateTrackPassthrough(passthrough) {
|
|
1233
|
+
if (passthrough !== void 0 && passthrough.length > MUX_TRACK_PASSTHROUGH_MAX_CHARS) {
|
|
1234
|
+
throw new MuxAiError(
|
|
1235
|
+
`Track passthrough must be at most ${MUX_TRACK_PASSTHROUGH_MAX_CHARS} characters (received ${passthrough.length}).`,
|
|
1236
|
+
{ type: "validation_error" }
|
|
1237
|
+
);
|
|
1238
|
+
}
|
|
1239
|
+
return passthrough;
|
|
1240
|
+
}
|
|
1241
|
+
async function fetchVttFromMux(vttUrl) {
|
|
1242
|
+
"use step";
|
|
1243
|
+
const vttResponse = await fetch(vttUrl);
|
|
1244
|
+
if (!vttResponse.ok) {
|
|
1245
|
+
throw new Error(`Failed to fetch VTT file: ${vttResponse.statusText}`);
|
|
1246
|
+
}
|
|
1247
|
+
return vttResponse.text();
|
|
1248
|
+
}
|
|
1249
|
+
async function createTextTrackOnMux(assetId, languageCode, trackName, presignedUrl, credentials, options = {}) {
|
|
1250
|
+
"use step";
|
|
1251
|
+
const muxClient = await resolveMuxClient(credentials);
|
|
1252
|
+
const mux = await muxClient.createClient();
|
|
1253
|
+
const trackResponse = await mux.video.assets.createTrack(assetId, {
|
|
1254
|
+
type: "text",
|
|
1255
|
+
text_type: "subtitles",
|
|
1256
|
+
language_code: languageCode,
|
|
1257
|
+
name: trackName,
|
|
1258
|
+
url: presignedUrl,
|
|
1259
|
+
closed_captions: options.closedCaptions,
|
|
1260
|
+
passthrough: options.passthrough
|
|
1261
|
+
});
|
|
1262
|
+
if (!trackResponse.id) {
|
|
1263
|
+
throw new Error("Failed to create text track: no track ID returned from Mux");
|
|
1264
|
+
}
|
|
1265
|
+
return trackResponse.id;
|
|
1266
|
+
}
|
|
1267
|
+
async function replaceAndCreateTrack(input) {
|
|
1268
|
+
"use step";
|
|
1269
|
+
const muxClient = await resolveMuxClient(input.credentials);
|
|
1270
|
+
const mux = await muxClient.createClient();
|
|
1271
|
+
const type = targetType(input.target);
|
|
1272
|
+
const deleted = [];
|
|
1273
|
+
const failed = (error) => ({
|
|
1274
|
+
kind: "create_failed",
|
|
1275
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
1276
|
+
deleted
|
|
1277
|
+
});
|
|
1278
|
+
const clearConflicts = async () => {
|
|
1279
|
+
let asset;
|
|
1280
|
+
try {
|
|
1281
|
+
asset = await mux.video.assets.retrieve(input.assetId);
|
|
1282
|
+
} catch (error) {
|
|
1283
|
+
if (deleted.length === 0) {
|
|
1284
|
+
throw error;
|
|
1285
|
+
}
|
|
1286
|
+
return failed(error);
|
|
1287
|
+
}
|
|
1288
|
+
const plan = planTextTrackReplacement(asset, input.target, input.policy, { keepTrackIds: input.keepTrackIds });
|
|
1289
|
+
if (plan.kind === "blocked") {
|
|
1290
|
+
return { ...plan, deleted };
|
|
1291
|
+
}
|
|
1292
|
+
if (plan.kind === "replace") {
|
|
1293
|
+
for (const track of plan.toDelete) {
|
|
1294
|
+
try {
|
|
1295
|
+
await mux.video.assets.deleteTrack(input.assetId, track.id);
|
|
1296
|
+
deleted.push(track);
|
|
1297
|
+
} catch (error) {
|
|
1298
|
+
if (!isNotFoundError(error)) {
|
|
1299
|
+
return failed(error);
|
|
1300
|
+
}
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
}
|
|
1304
|
+
return void 0;
|
|
1305
|
+
};
|
|
1306
|
+
const create = async () => {
|
|
1307
|
+
const track = await mux.video.assets.createTrack(input.assetId, {
|
|
1308
|
+
type,
|
|
1309
|
+
...type === "text" ? { text_type: "subtitles", closed_captions: input.closedCaptions } : {},
|
|
1310
|
+
language_code: input.target.languageCode,
|
|
1311
|
+
name: input.target.name.trim(),
|
|
1312
|
+
url: input.presignedUrl,
|
|
1313
|
+
passthrough: input.passthrough
|
|
1314
|
+
});
|
|
1315
|
+
if (!track.id) {
|
|
1316
|
+
throw new Error(`Failed to create ${type} track: no track ID returned from Mux`);
|
|
1317
|
+
}
|
|
1318
|
+
return track.id;
|
|
1319
|
+
};
|
|
1320
|
+
const stopped = await clearConflicts();
|
|
1321
|
+
if (stopped) {
|
|
1322
|
+
return stopped;
|
|
1323
|
+
}
|
|
1324
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
1325
|
+
try {
|
|
1326
|
+
return { kind: "created", trackId: await create(), deleted };
|
|
1327
|
+
} catch (error) {
|
|
1328
|
+
if (isDuplicateTrackNameError(error) && attempt === 0) {
|
|
1329
|
+
const stoppedOnRetry = await clearConflicts();
|
|
1330
|
+
if (stoppedOnRetry) {
|
|
1331
|
+
return stoppedOnRetry;
|
|
1332
|
+
}
|
|
1333
|
+
continue;
|
|
1334
|
+
}
|
|
1335
|
+
return failed(error);
|
|
1336
|
+
}
|
|
1337
|
+
}
|
|
1338
|
+
return { kind: "create_failed", reason: "Mux rejected the track name as not unique after retrying.", deleted };
|
|
1339
|
+
}
|
|
1340
|
+
async function replaceAndCreateTextTrack(input) {
|
|
1341
|
+
"use step";
|
|
1342
|
+
return replaceAndCreateTrack({ ...input, target: { ...input.target, type: "text" } });
|
|
1343
|
+
}
|
|
1344
|
+
|
|
1116
1345
|
// node_modules/@workflow/serde/dist/index.js
|
|
1117
1346
|
var WORKFLOW_SERIALIZE = /* @__PURE__ */ Symbol.for("workflow-serialize");
|
|
1118
1347
|
var WORKFLOW_DESERIALIZE = /* @__PURE__ */ Symbol.for("workflow-deserialize");
|
|
@@ -4540,6 +4769,17 @@ var AUDIO_ONLY_SYSTEM_PROMPT_OVERRIDES = {
|
|
|
4540
4769
|
You receive a timestamped transcript from audio-only content with lines in the form "[12s] Transcript text".
|
|
4541
4770
|
Use those timestamps as anchors to determine chapter start times in seconds.`
|
|
4542
4771
|
};
|
|
4772
|
+
var SCENE_CONTEXT_SYSTEM_PROMPT_OVERRIDES = {
|
|
4773
|
+
context: dedent4`
|
|
4774
|
+
You receive a timestamped transcript with lines in the form "[12s] Caption text" and an ordered scene context.
|
|
4775
|
+
Transcript timestamps are in seconds. Scene start and end timestamps are in milliseconds.
|
|
4776
|
+
Use the scene progression to choose logical chapter groupings and scene starts as chapter boundaries.`,
|
|
4777
|
+
constraints: dedent4`
|
|
4778
|
+
- Only use information present in the transcript and scene context
|
|
4779
|
+
- Return structured data that matches the requested JSON schema
|
|
4780
|
+
- Do not add commentary or extra text outside the JSON
|
|
4781
|
+
- When a <language> section is provided, all chapter titles MUST be written in that language`
|
|
4782
|
+
};
|
|
4543
4783
|
var chaptersPromptBuilder = createPromptBuilder({
|
|
4544
4784
|
template: {
|
|
4545
4785
|
task: {
|
|
@@ -4577,6 +4817,7 @@ var chaptersPromptBuilder = createPromptBuilder({
|
|
|
4577
4817
|
});
|
|
4578
4818
|
function buildUserPrompt3({
|
|
4579
4819
|
timestampedTranscript,
|
|
4820
|
+
sceneContext,
|
|
4580
4821
|
promptOverrides,
|
|
4581
4822
|
minChaptersPerHour = 3,
|
|
4582
4823
|
maxChaptersPerHour = 8,
|
|
@@ -4590,14 +4831,31 @@ function buildUserPrompt3({
|
|
|
4590
4831
|
attributes: { format: "seconds" }
|
|
4591
4832
|
}
|
|
4592
4833
|
];
|
|
4834
|
+
if (sceneContext?.length) {
|
|
4835
|
+
contextSections.push({
|
|
4836
|
+
tag: "scene_context",
|
|
4837
|
+
content: JSON.stringify(sceneContext, null, 2),
|
|
4838
|
+
attributes: { timestamps: "milliseconds", trust: "untrusted_evidence" }
|
|
4839
|
+
});
|
|
4840
|
+
}
|
|
4593
4841
|
if (languageName) {
|
|
4594
4842
|
contextSections.push(createLanguageSection(languageName));
|
|
4595
4843
|
}
|
|
4844
|
+
const sceneBoundarySeconds = getSceneBoundarySeconds(sceneContext).filter(
|
|
4845
|
+
(boundary) => scope ? boundary >= scope.startTime && boundary < scope.endTime : true
|
|
4846
|
+
);
|
|
4847
|
+
let firstChapterGuideline = "";
|
|
4848
|
+
if (sceneBoundarySeconds.length > 0) {
|
|
4849
|
+
firstChapterGuideline = `- The first chapter must start at ${sceneBoundarySeconds[0]}s, the first scene boundary inside the analyzed range`;
|
|
4850
|
+
} else if (scope) {
|
|
4851
|
+
firstChapterGuideline = `- The first chapter must start at ${scope.startTime}s`;
|
|
4852
|
+
}
|
|
4596
4853
|
const dynamicChapterGuidelines = dedent4`
|
|
4597
4854
|
- Create at least ${minChaptersPerHour} and at most ${maxChaptersPerHour} chapters per hour of content
|
|
4598
4855
|
- Use start times in seconds (not HH:MM:SS)
|
|
4599
|
-
${scope ? `- Analyze only the range from ${scope.startTime}s (inclusive) to ${scope.endTime}s (exclusive)
|
|
4600
|
-
|
|
4856
|
+
${scope ? `- Analyze only the range from ${scope.startTime}s (inclusive) to ${scope.endTime}s (exclusive)` : ""}
|
|
4857
|
+
${firstChapterGuideline}
|
|
4858
|
+
${sceneContext?.length ? "- Use the ordered scene context to identify chapter-sized runs of consecutive scenes\n- Every chapter startTime must exactly equal a scene start_ms divided by 1000; never split a scene" : ""}
|
|
4601
4859
|
- Chapter start times should be non-decreasing
|
|
4602
4860
|
- Do not include text before or after the JSON`;
|
|
4603
4861
|
const mergedOverrides = {
|
|
@@ -4606,6 +4864,32 @@ function buildUserPrompt3({
|
|
|
4606
4864
|
};
|
|
4607
4865
|
return chaptersPromptBuilder.buildWithContext(mergedOverrides, contextSections);
|
|
4608
4866
|
}
|
|
4867
|
+
function getSceneBoundarySeconds(sceneContext) {
|
|
4868
|
+
if (!sceneContext?.length) {
|
|
4869
|
+
return [];
|
|
4870
|
+
}
|
|
4871
|
+
return [...new Set(sceneContext.map((scene) => scene.start_ms / 1e3))].sort((a, b) => a - b);
|
|
4872
|
+
}
|
|
4873
|
+
function alignChaptersToSceneBoundaries(chapters, sceneBoundarySeconds) {
|
|
4874
|
+
if (sceneBoundarySeconds.length === 0) {
|
|
4875
|
+
return chapters;
|
|
4876
|
+
}
|
|
4877
|
+
const chapterByBoundary = /* @__PURE__ */ new Map();
|
|
4878
|
+
for (const chapter of chapters) {
|
|
4879
|
+
const startTime = sceneBoundarySeconds.reduce(
|
|
4880
|
+
(closest, boundary) => Math.abs(boundary - chapter.startTime) < Math.abs(closest - chapter.startTime) ? boundary : closest
|
|
4881
|
+
);
|
|
4882
|
+
const distance = Math.abs(startTime - chapter.startTime);
|
|
4883
|
+
const existing = chapterByBoundary.get(startTime);
|
|
4884
|
+
if (!existing || distance === 0 && existing.distance !== 0) {
|
|
4885
|
+
chapterByBoundary.set(startTime, {
|
|
4886
|
+
chapter: { ...chapter, startTime },
|
|
4887
|
+
distance
|
|
4888
|
+
});
|
|
4889
|
+
}
|
|
4890
|
+
}
|
|
4891
|
+
return [...chapterByBoundary.values()].map(({ chapter }) => chapter).sort((a, b) => a.startTime - b.startTime);
|
|
4892
|
+
}
|
|
4609
4893
|
async function generateChapters(assetId, options = {}) {
|
|
4610
4894
|
"use workflow";
|
|
4611
4895
|
const collectedUsage = [];
|
|
@@ -4625,7 +4909,8 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
|
4625
4909
|
maxChaptersPerHour,
|
|
4626
4910
|
credentials,
|
|
4627
4911
|
outputLanguageCode,
|
|
4628
|
-
scope
|
|
4912
|
+
scope,
|
|
4913
|
+
sceneContext
|
|
4629
4914
|
} = options;
|
|
4630
4915
|
const modelConfig = resolveLanguageModelConfig({
|
|
4631
4916
|
...options,
|
|
@@ -4667,8 +4952,18 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
|
4667
4952
|
}
|
|
4668
4953
|
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : getReliableLanguageCode(transcriptResult.track) ?? languageCode;
|
|
4669
4954
|
const languageName = resolvedLanguageCode ? getLanguageName(resolvedLanguageCode) : void 0;
|
|
4955
|
+
const sceneBoundarySeconds = getSceneBoundarySeconds(sceneContext).filter(
|
|
4956
|
+
(boundary) => resolvedScope ? boundary >= resolvedScope.startTime && boundary < resolvedScope.endTime : true
|
|
4957
|
+
);
|
|
4958
|
+
if (sceneContext?.length && sceneBoundarySeconds.length === 0) {
|
|
4959
|
+
throw new MuxAiError(
|
|
4960
|
+
"Scene context has no chapter boundaries within the requested scope.",
|
|
4961
|
+
{ type: "validation_error" }
|
|
4962
|
+
);
|
|
4963
|
+
}
|
|
4670
4964
|
const userPrompt = buildUserPrompt3({
|
|
4671
4965
|
timestampedTranscript,
|
|
4966
|
+
sceneContext,
|
|
4672
4967
|
promptOverrides,
|
|
4673
4968
|
minChaptersPerHour,
|
|
4674
4969
|
maxChaptersPerHour,
|
|
@@ -4677,7 +4972,11 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
|
4677
4972
|
});
|
|
4678
4973
|
let chaptersData = null;
|
|
4679
4974
|
try {
|
|
4680
|
-
const
|
|
4975
|
+
const systemPromptOverrides = {
|
|
4976
|
+
...isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT_OVERRIDES : {},
|
|
4977
|
+
...sceneContext?.length ? SCENE_CONTEXT_SYSTEM_PROMPT_OVERRIDES : {}
|
|
4978
|
+
};
|
|
4979
|
+
const systemPrompt = chapterSystemPromptBuilder.build(systemPromptOverrides);
|
|
4681
4980
|
chaptersData = await generateChaptersWithAI({
|
|
4682
4981
|
provider: modelConfig.provider,
|
|
4683
4982
|
modelId: modelConfig.modelId,
|
|
@@ -4695,9 +4994,13 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
|
4695
4994
|
throw new MuxAiError(`Failed to generate chapters for asset ${assetId}.`);
|
|
4696
4995
|
}
|
|
4697
4996
|
const { chapters: chaptersPayload, usage } = chaptersData;
|
|
4698
|
-
const
|
|
4997
|
+
const scopedChapters = chaptersPayload.chapters.filter((chapter) => typeof chapter.startTime === "number" && typeof chapter.title === "string").filter(
|
|
4699
4998
|
(chapter) => resolvedScope ? chapter.startTime >= resolvedScope.startTime && chapter.startTime < resolvedScope.endTime : true
|
|
4700
4999
|
).sort((a, b) => a.startTime - b.startTime);
|
|
5000
|
+
const validChapters = alignChaptersToSceneBoundaries(
|
|
5001
|
+
scopedChapters,
|
|
5002
|
+
sceneBoundarySeconds
|
|
5003
|
+
);
|
|
4701
5004
|
if (validChapters.length === 0) {
|
|
4702
5005
|
throw new MuxAiError(`Failed to generate valid chapters for asset ${assetId}.`);
|
|
4703
5006
|
}
|
|
@@ -4723,9 +5026,11 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
|
4723
5026
|
if (scrubbedChapters.length === 0) {
|
|
4724
5027
|
throw new MuxAiError(`Failed to generate valid chapters for asset ${assetId}.`);
|
|
4725
5028
|
}
|
|
4726
|
-
|
|
4727
|
-
|
|
4728
|
-
scrubbedChapters[0].startTime
|
|
5029
|
+
if (sceneBoundarySeconds.length === 0) {
|
|
5030
|
+
const firstChapterStartTime = resolvedScope?.startTime ?? 0;
|
|
5031
|
+
if (scrubbedChapters[0].startTime !== firstChapterStartTime) {
|
|
5032
|
+
scrubbedChapters[0].startTime = firstChapterStartTime;
|
|
5033
|
+
}
|
|
4729
5034
|
}
|
|
4730
5035
|
const usageWithMetadata = {
|
|
4731
5036
|
...usage,
|
|
@@ -4747,32 +5052,6 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
|
4747
5052
|
import { generateText as generateText4, Output as Output4 } from "ai";
|
|
4748
5053
|
import { z as z6 } from "zod";
|
|
4749
5054
|
|
|
4750
|
-
// src/lib/mux-tracks.ts
|
|
4751
|
-
async function fetchVttFromMux(vttUrl) {
|
|
4752
|
-
"use step";
|
|
4753
|
-
const vttResponse = await fetch(vttUrl);
|
|
4754
|
-
if (!vttResponse.ok) {
|
|
4755
|
-
throw new Error(`Failed to fetch VTT file: ${vttResponse.statusText}`);
|
|
4756
|
-
}
|
|
4757
|
-
return vttResponse.text();
|
|
4758
|
-
}
|
|
4759
|
-
async function createTextTrackOnMux(assetId, languageCode, trackName, presignedUrl, credentials) {
|
|
4760
|
-
"use step";
|
|
4761
|
-
const muxClient = await resolveMuxClient(credentials);
|
|
4762
|
-
const mux = await muxClient.createClient();
|
|
4763
|
-
const trackResponse = await mux.video.assets.createTrack(assetId, {
|
|
4764
|
-
type: "text",
|
|
4765
|
-
text_type: "subtitles",
|
|
4766
|
-
language_code: languageCode,
|
|
4767
|
-
name: trackName,
|
|
4768
|
-
url: presignedUrl
|
|
4769
|
-
});
|
|
4770
|
-
if (!trackResponse.id) {
|
|
4771
|
-
throw new Error("Failed to create text track: no track ID returned from Mux");
|
|
4772
|
-
}
|
|
4773
|
-
return trackResponse.id;
|
|
4774
|
-
}
|
|
4775
|
-
|
|
4776
5055
|
// src/lib/storage-adapter.ts
|
|
4777
5056
|
function requireCredentials(accessKeyId, secretAccessKey) {
|
|
4778
5057
|
if (!accessKeyId || !secretAccessKey) {
|
|
@@ -5051,6 +5330,7 @@ async function uploadEditedVttToS3({
|
|
|
5051
5330
|
editedVtt,
|
|
5052
5331
|
assetId,
|
|
5053
5332
|
trackId,
|
|
5333
|
+
variant = "edited",
|
|
5054
5334
|
s3Endpoint,
|
|
5055
5335
|
s3Region,
|
|
5056
5336
|
s3Bucket,
|
|
@@ -5060,7 +5340,7 @@ async function uploadEditedVttToS3({
|
|
|
5060
5340
|
"use step";
|
|
5061
5341
|
const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
|
|
5062
5342
|
const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
|
|
5063
|
-
const vttKey = `edited/${assetId}/${trackId}
|
|
5343
|
+
const vttKey = `edited/${assetId}/${trackId}-${variant}-${Date.now()}.vtt`;
|
|
5064
5344
|
await putObjectWithStorageAdapter({
|
|
5065
5345
|
accessKeyId: s3AccessKeyId,
|
|
5066
5346
|
secretAccessKey: s3SecretAccessKey,
|
|
@@ -5110,6 +5390,9 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
|
|
|
5110
5390
|
s3Region: providedS3Region,
|
|
5111
5391
|
s3Bucket: providedS3Bucket,
|
|
5112
5392
|
trackNameSuffix,
|
|
5393
|
+
replaceExistingTracks: replaceExistingTracksOption,
|
|
5394
|
+
trackName: providedTrackName,
|
|
5395
|
+
trackPassthrough: providedTrackPassthrough,
|
|
5113
5396
|
storageAdapter,
|
|
5114
5397
|
credentials
|
|
5115
5398
|
} = options;
|
|
@@ -5122,6 +5405,21 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
|
|
|
5122
5405
|
if (autoCensorOption && !provider) {
|
|
5123
5406
|
throw new MuxAiError("provider is required when using autoCensorProfanity.", { type: "validation_error" });
|
|
5124
5407
|
}
|
|
5408
|
+
const legacyTrackNaming = deleteOriginalTrack !== void 0 || trackNameSuffix !== void 0;
|
|
5409
|
+
if (legacyTrackNaming && (replaceExistingTracksOption !== void 0 || providedTrackName !== void 0)) {
|
|
5410
|
+
throw new MuxAiError(
|
|
5411
|
+
"deleteOriginalTrack and trackNameSuffix are deprecated and cannot be combined with replaceExistingTracks or trackName.",
|
|
5412
|
+
{ type: "validation_error" }
|
|
5413
|
+
);
|
|
5414
|
+
}
|
|
5415
|
+
const replaceExistingTracks = replaceExistingTracksOption ?? "replace_all";
|
|
5416
|
+
if (!legacyTrackNaming && replaceExistingTracks === "fail" && !providedTrackName) {
|
|
5417
|
+
throw new MuxAiError(
|
|
5418
|
+
`trackName is required when replaceExistingTracks is "fail": the edited track cannot reuse the source track's name while the source is kept.`,
|
|
5419
|
+
{ type: "validation_error" }
|
|
5420
|
+
);
|
|
5421
|
+
}
|
|
5422
|
+
const trackPassthrough = validateTrackPassthrough(providedTrackPassthrough) ?? buildMuxAiTrackPassthrough("edit-captions");
|
|
5125
5423
|
const deleteOriginal = deleteOriginalTrack !== false;
|
|
5126
5424
|
const uploadToMux = uploadToMuxOption !== false;
|
|
5127
5425
|
const uploadToS3 = uploadToS3Option || uploadToMux;
|
|
@@ -5164,6 +5462,25 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
|
|
|
5164
5462
|
{ type: "validation_error" }
|
|
5165
5463
|
);
|
|
5166
5464
|
}
|
|
5465
|
+
const sourceLanguageCode = sourceTrack.language_code || "en";
|
|
5466
|
+
const sourceName = sourceTrack.name || "Subtitles";
|
|
5467
|
+
const outputTrack = {
|
|
5468
|
+
languageCode: sourceLanguageCode,
|
|
5469
|
+
name: providedTrackName ?? sourceName
|
|
5470
|
+
};
|
|
5471
|
+
const keepTrackIds = replaceExistingTracks === "fail" ? [trackId] : [];
|
|
5472
|
+
if (replaceExistingTracks === "fail" && normalizeTrackName(outputTrack.name) === normalizeTrackName(sourceName)) {
|
|
5473
|
+
throw new MuxAiError(
|
|
5474
|
+
`trackName "${outputTrack.name}" matches the source track's name. Mux track names must be unique; choose a different trackName or use replaceExistingTracks: "replace_all" to replace the source.`,
|
|
5475
|
+
{ type: "validation_error" }
|
|
5476
|
+
);
|
|
5477
|
+
}
|
|
5478
|
+
if (uploadToMux && !legacyTrackNaming) {
|
|
5479
|
+
const plan = planTextTrackReplacement(assetData, outputTrack, replaceExistingTracks, { keepTrackIds });
|
|
5480
|
+
if (plan.kind === "blocked") {
|
|
5481
|
+
throw new MuxAiError(plan.reason, { type: "validation_error" });
|
|
5482
|
+
}
|
|
5483
|
+
}
|
|
5167
5484
|
const vttUrl = await buildTranscriptUrl(playbackId, trackId, policy === "signed", credentials);
|
|
5168
5485
|
let vttContent;
|
|
5169
5486
|
try {
|
|
@@ -5241,32 +5558,32 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
|
|
|
5241
5558
|
} : void 0;
|
|
5242
5559
|
let presignedUrl;
|
|
5243
5560
|
let uploadedTrackId;
|
|
5561
|
+
let replacedTracks;
|
|
5244
5562
|
if (uploadToS3) {
|
|
5563
|
+
const s3Config = {
|
|
5564
|
+
assetId,
|
|
5565
|
+
trackId,
|
|
5566
|
+
s3Endpoint,
|
|
5567
|
+
s3Region,
|
|
5568
|
+
s3Bucket,
|
|
5569
|
+
storageAdapter,
|
|
5570
|
+
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
5571
|
+
};
|
|
5245
5572
|
try {
|
|
5246
|
-
presignedUrl = await uploadEditedVttToS3({
|
|
5247
|
-
editedVtt,
|
|
5248
|
-
assetId,
|
|
5249
|
-
trackId,
|
|
5250
|
-
s3Endpoint,
|
|
5251
|
-
s3Region,
|
|
5252
|
-
s3Bucket,
|
|
5253
|
-
storageAdapter,
|
|
5254
|
-
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
5255
|
-
});
|
|
5573
|
+
presignedUrl = await uploadEditedVttToS3({ editedVtt, ...s3Config });
|
|
5256
5574
|
} catch (error) {
|
|
5257
5575
|
wrapError(error, "Failed to upload VTT to S3");
|
|
5258
5576
|
}
|
|
5259
|
-
if (uploadToMux) {
|
|
5577
|
+
if (uploadToMux && legacyTrackNaming) {
|
|
5260
5578
|
try {
|
|
5261
|
-
const
|
|
5262
|
-
const suffix = trackNameSuffix ?? "edited";
|
|
5263
|
-
const trackName = `${sourceTrack.name || "Subtitles"} (${suffix})`;
|
|
5579
|
+
const trackName = `${sourceName} (${trackNameSuffix ?? "edited"})`;
|
|
5264
5580
|
uploadedTrackId = await createTextTrackOnMux(
|
|
5265
5581
|
assetId,
|
|
5266
|
-
|
|
5582
|
+
sourceLanguageCode,
|
|
5267
5583
|
trackName,
|
|
5268
5584
|
presignedUrl,
|
|
5269
|
-
credentials
|
|
5585
|
+
credentials,
|
|
5586
|
+
{ closedCaptions: sourceTrack.closed_captions, passthrough: trackPassthrough }
|
|
5270
5587
|
);
|
|
5271
5588
|
} catch (error) {
|
|
5272
5589
|
wrapError(error, "Failed to add edited track to Mux asset");
|
|
@@ -5278,6 +5595,48 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
|
|
|
5278
5595
|
wrapError(error, "Failed to delete original track");
|
|
5279
5596
|
}
|
|
5280
5597
|
}
|
|
5598
|
+
} else if (uploadToMux) {
|
|
5599
|
+
let outcome;
|
|
5600
|
+
try {
|
|
5601
|
+
outcome = await replaceAndCreateTextTrack({
|
|
5602
|
+
assetId,
|
|
5603
|
+
target: outputTrack,
|
|
5604
|
+
policy: replaceExistingTracks,
|
|
5605
|
+
presignedUrl,
|
|
5606
|
+
closedCaptions: sourceTrack.closed_captions,
|
|
5607
|
+
passthrough: trackPassthrough,
|
|
5608
|
+
keepTrackIds,
|
|
5609
|
+
credentials
|
|
5610
|
+
});
|
|
5611
|
+
} catch (error) {
|
|
5612
|
+
wrapError(error, "Failed to add edited track to Mux asset");
|
|
5613
|
+
}
|
|
5614
|
+
if (outcome.kind !== "created") {
|
|
5615
|
+
const sourceWasDeleted = outcome.deleted.some((track) => track.id === trackId);
|
|
5616
|
+
let restoreNote = "";
|
|
5617
|
+
if (sourceWasDeleted) {
|
|
5618
|
+
try {
|
|
5619
|
+
const originalUrl = await uploadEditedVttToS3({ editedVtt: vttContent, variant: "original", ...s3Config });
|
|
5620
|
+
const restoredTrackId = await createTextTrackOnMux(
|
|
5621
|
+
assetId,
|
|
5622
|
+
sourceLanguageCode,
|
|
5623
|
+
sourceName,
|
|
5624
|
+
originalUrl,
|
|
5625
|
+
credentials,
|
|
5626
|
+
{ closedCaptions: sourceTrack.closed_captions, passthrough: sourceTrack.passthrough }
|
|
5627
|
+
);
|
|
5628
|
+
restoreNote = ` The source track was restored as ${restoredTrackId}.`;
|
|
5629
|
+
} catch (restoreError) {
|
|
5630
|
+
restoreNote = ` Restoring the source track also failed: ${restoreError instanceof Error ? restoreError.message : String(restoreError)}.`;
|
|
5631
|
+
}
|
|
5632
|
+
}
|
|
5633
|
+
if (outcome.kind === "blocked") {
|
|
5634
|
+
throw new MuxAiError(`${outcome.reason}${restoreNote}`, { type: "validation_error" });
|
|
5635
|
+
}
|
|
5636
|
+
throw new Error(`Failed to add edited track to Mux asset: ${outcome.reason}.${restoreNote}`);
|
|
5637
|
+
}
|
|
5638
|
+
uploadedTrackId = outcome.trackId;
|
|
5639
|
+
replacedTracks = outcome.deleted;
|
|
5281
5640
|
}
|
|
5282
5641
|
}
|
|
5283
5642
|
return {
|
|
@@ -5290,6 +5649,7 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
|
|
|
5290
5649
|
replacements: replacementsResult,
|
|
5291
5650
|
speakerReplacements: speakerReplacementsResult,
|
|
5292
5651
|
uploadedTrackId,
|
|
5652
|
+
replacedTracks,
|
|
5293
5653
|
presignedUrl,
|
|
5294
5654
|
usage: usageWithMetadata,
|
|
5295
5655
|
safety: safety.report()
|
|
@@ -6777,14 +7137,44 @@ function buildDescriptionGuidance(wordCount, contentType) {
|
|
|
6777
7137
|
Write in present tense. Be specific about what is discussed or presented rather than making assumptions.
|
|
6778
7138
|
Focus on the spoken content and any key insights, dialogue, or narrative elements.`;
|
|
6779
7139
|
}
|
|
6780
|
-
function
|
|
7140
|
+
function buildSceneAwareDescriptionGuidance(wordCount) {
|
|
7141
|
+
if (wordCount < DESCRIPTION_LENGTH_THRESHOLD_SMALL) {
|
|
7142
|
+
return dedent6`A brief summary of the video in no more than ${wordCount} words. Shorter is fine.
|
|
7143
|
+
Focus on the single most important subject or action across the ordered scenes.
|
|
7144
|
+
Use the storyboard as direct visual evidence and the scene context to preserve progression.
|
|
7145
|
+
Write in present tense.`;
|
|
7146
|
+
}
|
|
7147
|
+
if (wordCount > DESCRIPTION_LENGTH_THRESHOLD_LARGE) {
|
|
7148
|
+
return dedent6`A detailed summary that describes what happens across the ordered scenes.
|
|
7149
|
+
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
7150
|
+
Be thorough: cover subjects, actions, setting, progression, and notable details supported across scenes.
|
|
7151
|
+
Use the storyboard as direct visual evidence and the scene context to preserve progression.
|
|
7152
|
+
Write in present tense. If the transcript provides dialogue or narration, incorporate key points while preserving scene progression.`;
|
|
7153
|
+
}
|
|
7154
|
+
return dedent6`A summary that describes what happens across the ordered scenes.
|
|
7155
|
+
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
7156
|
+
Cover the main subjects, actions, setting, and notable progression supported across scenes.
|
|
7157
|
+
Use the storyboard as direct visual evidence and the scene context to preserve progression.
|
|
7158
|
+
Write in present tense. If the transcript provides dialogue or narration, incorporate key points while preserving scene progression.`;
|
|
7159
|
+
}
|
|
7160
|
+
function createSummarizationBuilder({ titleLength, descriptionLength, tagCount } = {}, { hasSceneContext = false } = {}) {
|
|
6781
7161
|
const titleLimit = titleLength ?? DEFAULT_TITLE_LENGTH;
|
|
6782
7162
|
const keywordLimit = tagCount ?? DEFAULT_SUMMARY_KEYWORD_LIMIT;
|
|
7163
|
+
const task = hasSceneContext ? "Analyze the storyboard frames and ordered scene context together, then generate metadata that captures the full progression of the video content." : "Analyze the storyboard frames and generate metadata that captures the essence of the video content.";
|
|
7164
|
+
const qualityGuidelines = hasSceneContext ? dedent6`
|
|
7165
|
+
- Follow scene_index order to understand what begins, develops, and concludes
|
|
7166
|
+
- Use scene narratives and concepts as grounded evidence, preserving consistent terminology across the output
|
|
7167
|
+
- Use the storyboard as direct visual evidence for whole-asset details
|
|
7168
|
+
- Balance brevity with informativeness` : dedent6`
|
|
7169
|
+
- Examine all frames to understand the full context and progression
|
|
7170
|
+
- Be precise: "golden retriever" is better than "dog" when identifiable
|
|
7171
|
+
- Capture the narrative: what begins, develops, and concludes
|
|
7172
|
+
- Balance brevity with informativeness`;
|
|
6783
7173
|
return createPromptBuilder({
|
|
6784
7174
|
template: {
|
|
6785
7175
|
task: {
|
|
6786
7176
|
tag: "task",
|
|
6787
|
-
content:
|
|
7177
|
+
content: task
|
|
6788
7178
|
},
|
|
6789
7179
|
title: {
|
|
6790
7180
|
tag: "title_requirements",
|
|
@@ -6797,7 +7187,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
6797
7187
|
},
|
|
6798
7188
|
description: {
|
|
6799
7189
|
tag: "description_requirements",
|
|
6800
|
-
content: buildDescriptionGuidance(descriptionLength ?? DEFAULT_DESCRIPTION_LENGTH, "video")
|
|
7190
|
+
content: hasSceneContext ? buildSceneAwareDescriptionGuidance(descriptionLength ?? DEFAULT_DESCRIPTION_LENGTH) : buildDescriptionGuidance(descriptionLength ?? DEFAULT_DESCRIPTION_LENGTH, "video")
|
|
6801
7191
|
},
|
|
6802
7192
|
keywords: {
|
|
6803
7193
|
tag: "keywords_requirements",
|
|
@@ -6813,11 +7203,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
6813
7203
|
},
|
|
6814
7204
|
qualityGuidelines: {
|
|
6815
7205
|
tag: "quality_guidelines",
|
|
6816
|
-
content:
|
|
6817
|
-
- Examine all frames to understand the full context and progression
|
|
6818
|
-
- Be precise: "golden retriever" is better than "dog" when identifiable
|
|
6819
|
-
- Capture the narrative: what begins, develops, and concludes
|
|
6820
|
-
- Balance brevity with informativeness`
|
|
7206
|
+
content: qualityGuidelines
|
|
6821
7207
|
}
|
|
6822
7208
|
},
|
|
6823
7209
|
sectionOrder: ["task", "title", "description", "keywords", "qualityGuidelines"]
|
|
@@ -6869,59 +7255,83 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
6869
7255
|
sectionOrder: ["task", "title", "description", "keywords", "qualityGuidelines"]
|
|
6870
7256
|
});
|
|
6871
7257
|
}
|
|
6872
|
-
|
|
6873
|
-
|
|
6874
|
-
|
|
6875
|
-
|
|
6876
|
-
|
|
6877
|
-
|
|
6878
|
-
|
|
6879
|
-
|
|
6880
|
-
|
|
6881
|
-
|
|
6882
|
-
|
|
6883
|
-
|
|
6884
|
-
|
|
6885
|
-
|
|
6886
|
-
|
|
6887
|
-
|
|
6888
|
-
|
|
6889
|
-
|
|
6890
|
-
|
|
6891
|
-
|
|
6892
|
-
|
|
6893
|
-
|
|
6894
|
-
|
|
6895
|
-
|
|
6896
|
-
|
|
6897
|
-
|
|
6898
|
-
|
|
6899
|
-
|
|
6900
|
-
|
|
6901
|
-
|
|
6902
|
-
|
|
6903
|
-
|
|
6904
|
-
|
|
6905
|
-
|
|
6906
|
-
|
|
6907
|
-
|
|
6908
|
-
|
|
6909
|
-
|
|
6910
|
-
|
|
6911
|
-
|
|
6912
|
-
|
|
6913
|
-
|
|
6914
|
-
|
|
6915
|
-
|
|
6916
|
-
|
|
6917
|
-
|
|
6918
|
-
|
|
6919
|
-
|
|
6920
|
-
|
|
6921
|
-
|
|
6922
|
-
|
|
6923
|
-
|
|
6924
|
-
|
|
7258
|
+
function createVideoSystemPrompt(hasSceneContext) {
|
|
7259
|
+
let sceneContext = "";
|
|
7260
|
+
let sceneGuidance = "";
|
|
7261
|
+
if (hasSceneContext) {
|
|
7262
|
+
sceneContext = promptDedent`
|
|
7263
|
+
Ordered scene context is also provided as structured evidence of the video's progression.
|
|
7264
|
+
Entries use stable, zero-based scene_index values and absolute millisecond timestamps.
|
|
7265
|
+
Read scenes in scene_index order to understand what begins, develops, and concludes.`;
|
|
7266
|
+
sceneGuidance = promptDedent`
|
|
7267
|
+
<scene_guidance>
|
|
7268
|
+
- Use scene titles, narratives, and concepts to understand temporal progression
|
|
7269
|
+
- Use storyboard frames as direct visual evidence for subjects, actions, settings, and details
|
|
7270
|
+
- Preserve consistent terminology from the scene context across the title, description, and keywords
|
|
7271
|
+
- Treat optional scene fields as evidence only when present; do not infer omitted details
|
|
7272
|
+
- Consider all provided evidence together; if derived scene context conflicts with storyboard visuals, trust the visuals
|
|
7273
|
+
</scene_guidance>`;
|
|
7274
|
+
}
|
|
7275
|
+
const transcriptEvidence = hasSceneContext ? "storyboard frames and ordered scenes" : "visual frames";
|
|
7276
|
+
const descriptionPriority = hasSceneContext ? "Use the scene context for temporal structure and the storyboard for direct visual details, enriching both with transcript insights" : "Prioritize visual content for the description, but enrich it with transcript insights";
|
|
7277
|
+
const groundingConstraint = hasSceneContext ? "Use all provided evidence together. Ground each claim in the storyboard, scene context, or transcript" : "Only describe what is clearly observable in the frames or explicitly stated in the transcript";
|
|
7278
|
+
return promptDedent`
|
|
7279
|
+
<role>
|
|
7280
|
+
You are a video content analyst specializing in storyboard interpretation and multimodal analysis.
|
|
7281
|
+
</role>
|
|
7282
|
+
|
|
7283
|
+
<context>
|
|
7284
|
+
You receive storyboard images containing multiple sequential frames extracted from a video.
|
|
7285
|
+
${STORYBOARD_FRAME_INSTRUCTIONS}
|
|
7286
|
+
${sceneContext}
|
|
7287
|
+
</context>
|
|
7288
|
+
|
|
7289
|
+
${sceneGuidance}
|
|
7290
|
+
|
|
7291
|
+
<transcript_guidance>
|
|
7292
|
+
When a transcript is provided alongside the storyboard:
|
|
7293
|
+
- Use it to understand spoken content, dialogue, narration, and audio context
|
|
7294
|
+
- Correlate transcript content with ${transcriptEvidence} to build a complete picture
|
|
7295
|
+
- Extract key terminology, names, and specific language used by speakers
|
|
7296
|
+
- Let the transcript inform keyword selection, especially for topics not visually obvious
|
|
7297
|
+
- ${descriptionPriority}
|
|
7298
|
+
- If transcript and visuals conflict, trust the visual evidence
|
|
7299
|
+
</transcript_guidance>
|
|
7300
|
+
|
|
7301
|
+
<capabilities>
|
|
7302
|
+
- Extract meaning from visual sequences
|
|
7303
|
+
- Identify subjects, actions, settings, and narrative arcs
|
|
7304
|
+
- Generate accurate, searchable metadata
|
|
7305
|
+
- Synthesize visual and transcript information when provided
|
|
7306
|
+
</capabilities>
|
|
7307
|
+
|
|
7308
|
+
<security>
|
|
7309
|
+
${NON_DISCLOSURE_CONSTRAINT}
|
|
7310
|
+
|
|
7311
|
+
${UNTRUSTED_USER_INPUT_NOTICE}
|
|
7312
|
+
|
|
7313
|
+
${VISUAL_TEXT_AS_CONTENT}
|
|
7314
|
+
|
|
7315
|
+
${CANARY_TRIPWIRE}
|
|
7316
|
+
</security>
|
|
7317
|
+
|
|
7318
|
+
<constraints>
|
|
7319
|
+
- ${groundingConstraint}
|
|
7320
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
7321
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
7322
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
7323
|
+
- Output only the JSON object; no markdown or extra text
|
|
7324
|
+
- When a <language> section is provided, all output text MUST be written in that language
|
|
7325
|
+
</constraints>
|
|
7326
|
+
|
|
7327
|
+
<tone_guidance>
|
|
7328
|
+
${TONE_GUIDANCE}
|
|
7329
|
+
</tone_guidance>
|
|
7330
|
+
|
|
7331
|
+
<language_guidelines>
|
|
7332
|
+
${createLanguageGuidelines("video")}
|
|
7333
|
+
</language_guidelines>`;
|
|
7334
|
+
}
|
|
6925
7335
|
var AUDIO_ONLY_SYSTEM_PROMPT3 = promptDedent`
|
|
6926
7336
|
<role>
|
|
6927
7337
|
You are an audio content analyst specializing in transcript analysis and metadata generation.
|
|
@@ -6972,6 +7382,16 @@ var AUDIO_ONLY_SYSTEM_PROMPT3 = promptDedent`
|
|
|
6972
7382
|
<language_guidelines>
|
|
6973
7383
|
${createLanguageGuidelines("audio")}
|
|
6974
7384
|
</language_guidelines>`;
|
|
7385
|
+
function createSceneContextSection(sceneContext) {
|
|
7386
|
+
return {
|
|
7387
|
+
tag: "scene_context",
|
|
7388
|
+
content: JSON.stringify(sceneContext),
|
|
7389
|
+
attributes: {
|
|
7390
|
+
format: "json",
|
|
7391
|
+
order: "scene_index_ascending"
|
|
7392
|
+
}
|
|
7393
|
+
};
|
|
7394
|
+
}
|
|
6975
7395
|
function buildUserPrompt5({
|
|
6976
7396
|
tone,
|
|
6977
7397
|
transcriptText,
|
|
@@ -6981,7 +7401,8 @@ function buildUserPrompt5({
|
|
|
6981
7401
|
titleLength,
|
|
6982
7402
|
descriptionLength,
|
|
6983
7403
|
tagCount,
|
|
6984
|
-
languageName
|
|
7404
|
+
languageName,
|
|
7405
|
+
sceneContext
|
|
6985
7406
|
}) {
|
|
6986
7407
|
const contextSections = [createToneSection(TONE_INSTRUCTIONS[tone])];
|
|
6987
7408
|
if (languageName) {
|
|
@@ -6996,11 +7417,15 @@ function buildUserPrompt5({
|
|
|
6996
7417
|
const format = isCleanTranscript ? "plain text" : "WebVTT";
|
|
6997
7418
|
contextSections.push(createTranscriptSection(transcriptText, format));
|
|
6998
7419
|
}
|
|
7420
|
+
const hasSceneContext = !isAudioOnly && Boolean(sceneContext?.length);
|
|
7421
|
+
if (hasSceneContext && sceneContext) {
|
|
7422
|
+
contextSections.push(createSceneContextSection(sceneContext));
|
|
7423
|
+
}
|
|
6999
7424
|
const constraints = { titleLength, descriptionLength, tagCount };
|
|
7000
|
-
const promptBuilder = isAudioOnly ? createAudioOnlyBuilder(constraints) : createSummarizationBuilder(constraints);
|
|
7425
|
+
const promptBuilder = isAudioOnly ? createAudioOnlyBuilder(constraints) : createSummarizationBuilder(constraints, { hasSceneContext });
|
|
7001
7426
|
return promptBuilder.buildWithContext(promptOverrides, contextSections);
|
|
7002
7427
|
}
|
|
7003
|
-
async function
|
|
7428
|
+
async function analyzeVideo(imageDataUrl, provider, modelId, userPrompt, systemPrompt, descriptionLength, credentials) {
|
|
7004
7429
|
"use step";
|
|
7005
7430
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
7006
7431
|
const schema = buildSummarySchema(descriptionLength);
|
|
@@ -7016,10 +7441,10 @@ async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, s
|
|
|
7016
7441
|
messages: [
|
|
7017
7442
|
{
|
|
7018
7443
|
role: "user",
|
|
7019
|
-
content: [
|
|
7444
|
+
content: imageDataUrl ? [
|
|
7020
7445
|
{ type: "text", text: userPrompt },
|
|
7021
7446
|
{ type: "image", image: imageDataUrl }
|
|
7022
|
-
]
|
|
7447
|
+
] : userPrompt
|
|
7023
7448
|
}
|
|
7024
7449
|
]
|
|
7025
7450
|
}));
|
|
@@ -7129,6 +7554,7 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
|
|
|
7129
7554
|
cleanTranscript = true,
|
|
7130
7555
|
imageSubmissionMode = "url",
|
|
7131
7556
|
imageDownloadOptions,
|
|
7557
|
+
sceneContext,
|
|
7132
7558
|
promptOverrides,
|
|
7133
7559
|
credentials,
|
|
7134
7560
|
titleLength,
|
|
@@ -7152,6 +7578,7 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
|
|
|
7152
7578
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, workflowCredentials, options?.assetSnapshot);
|
|
7153
7579
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
7154
7580
|
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
7581
|
+
const hasSceneContext = !isAudioOnly && Boolean(sceneContext?.length);
|
|
7155
7582
|
const effectiveScope = hasWorkflowScopeBoundaries(scope) ? scope : void 0;
|
|
7156
7583
|
const storyboardScope = isAudioOnly ? void 0 : resolveRenderableVideoScope(
|
|
7157
7584
|
effectiveScope,
|
|
@@ -7168,7 +7595,8 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
|
|
|
7168
7595
|
);
|
|
7169
7596
|
}
|
|
7170
7597
|
const signingContext = await resolveMuxSigningContext(workflowCredentials);
|
|
7171
|
-
|
|
7598
|
+
const requiresSignedPlaybackAccess = includeTranscript || !isAudioOnly;
|
|
7599
|
+
if (policy === "signed" && requiresSignedPlaybackAccess && !signingContext) {
|
|
7172
7600
|
throw new MuxAiError(
|
|
7173
7601
|
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
|
|
7174
7602
|
{ type: "validation_error" }
|
|
@@ -7194,11 +7622,15 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
|
|
|
7194
7622
|
titleLength,
|
|
7195
7623
|
descriptionLength,
|
|
7196
7624
|
tagCount,
|
|
7197
|
-
languageName
|
|
7625
|
+
languageName,
|
|
7626
|
+
sceneContext: hasSceneContext ? sceneContext : void 0
|
|
7198
7627
|
});
|
|
7199
7628
|
let analysisResponse;
|
|
7200
7629
|
let imageUrl;
|
|
7201
|
-
|
|
7630
|
+
let systemPrompt = AUDIO_ONLY_SYSTEM_PROMPT3;
|
|
7631
|
+
if (!isAudioOnly) {
|
|
7632
|
+
systemPrompt = createVideoSystemPrompt(hasSceneContext);
|
|
7633
|
+
}
|
|
7202
7634
|
const effectiveDescriptionLength = descriptionLength ?? DEFAULT_DESCRIPTION_LENGTH;
|
|
7203
7635
|
try {
|
|
7204
7636
|
if (isAudioOnly) {
|
|
@@ -7221,7 +7653,7 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
|
|
|
7221
7653
|
imageUrl = storyboardUrl;
|
|
7222
7654
|
if (imageSubmissionMode === "base64") {
|
|
7223
7655
|
const downloadResult = await downloadImageAsBase64(storyboardUrl, imageDownloadOptions);
|
|
7224
|
-
analysisResponse = await
|
|
7656
|
+
analysisResponse = await analyzeVideo(
|
|
7225
7657
|
downloadResult.base64Data,
|
|
7226
7658
|
modelConfig.provider,
|
|
7227
7659
|
modelConfig.modelId,
|
|
@@ -7231,7 +7663,7 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
|
|
|
7231
7663
|
workflowCredentials
|
|
7232
7664
|
);
|
|
7233
7665
|
} else {
|
|
7234
|
-
analysisResponse = await
|
|
7666
|
+
analysisResponse = await analyzeVideo(
|
|
7235
7667
|
storyboardUrl,
|
|
7236
7668
|
modelConfig.provider,
|
|
7237
7669
|
modelConfig.modelId,
|
|
@@ -7550,23 +7982,6 @@ async function downloadAndUploadDubTranscript({
|
|
|
7550
7982
|
console.warn(`\u2705 Dub transcript uploaded successfully to: ${vttKey}`);
|
|
7551
7983
|
return presignedUrl;
|
|
7552
7984
|
}
|
|
7553
|
-
async function createAudioTrackOnMux(assetId, languageCode, presignedUrl, credentials) {
|
|
7554
|
-
"use step";
|
|
7555
|
-
const muxClient = await resolveMuxClient(credentials);
|
|
7556
|
-
const mux = await muxClient.createClient();
|
|
7557
|
-
const languageName = new Intl.DisplayNames(["en"], { type: "language" }).of(languageCode) || languageCode.toUpperCase();
|
|
7558
|
-
const trackName = `${languageName} (auto-dubbed)`;
|
|
7559
|
-
const trackResponse = await mux.video.assets.createTrack(assetId, {
|
|
7560
|
-
type: "audio",
|
|
7561
|
-
language_code: languageCode,
|
|
7562
|
-
name: trackName,
|
|
7563
|
-
url: presignedUrl
|
|
7564
|
-
});
|
|
7565
|
-
if (!trackResponse.id) {
|
|
7566
|
-
throw new Error("Failed to create audio track: no track ID returned from Mux");
|
|
7567
|
-
}
|
|
7568
|
-
return trackResponse.id;
|
|
7569
|
-
}
|
|
7570
7985
|
async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
7571
7986
|
"use workflow";
|
|
7572
7987
|
const {
|
|
@@ -7579,15 +7994,23 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
7579
7994
|
uploadCaptionsToMux = false,
|
|
7580
7995
|
staticRenditionCleanup: staticRenditionCleanupOption = "delete",
|
|
7581
7996
|
storageAdapter,
|
|
7582
|
-
credentials: providedCredentials
|
|
7997
|
+
credentials: providedCredentials,
|
|
7998
|
+
replaceExistingTracks = "fail",
|
|
7999
|
+
trackName: providedTrackName,
|
|
8000
|
+
trackPassthrough: providedTrackPassthrough
|
|
7583
8001
|
} = options;
|
|
7584
8002
|
if (provider !== "elevenlabs") {
|
|
7585
8003
|
throw new MuxAiError("Only ElevenLabs provider is currently supported for audio translation.", { type: "validation_error" });
|
|
7586
8004
|
}
|
|
7587
8005
|
const credentials = providedCredentials;
|
|
7588
8006
|
const effectiveStorageAdapter = storageAdapter;
|
|
8007
|
+
const trackPassthrough = validateTrackPassthrough(providedTrackPassthrough) ?? buildMuxAiTrackPassthrough("translate-audio");
|
|
7589
8008
|
const uploadToMux = uploadToMuxOption !== false;
|
|
7590
8009
|
const uploadToS3 = uploadToS3Option || uploadToMux || uploadCaptionsToMux;
|
|
8010
|
+
const muxLangCode = toISO639_1(toLanguageCode);
|
|
8011
|
+
const trackName = providedTrackName ?? `${getLanguageName(muxLangCode) ?? muxLangCode.toUpperCase()} (Auto-dubbed)`;
|
|
8012
|
+
const audioTrackTarget = { type: "audio", languageCode: muxLangCode, name: trackName };
|
|
8013
|
+
const captionsTrackTarget = { type: "text", languageCode: muxLangCode, name: trackName };
|
|
7591
8014
|
const s3Endpoint = options.s3Endpoint ?? env_default.S3_ENDPOINT;
|
|
7592
8015
|
const s3Region = options.s3Region ?? env_default.S3_REGION ?? "auto";
|
|
7593
8016
|
const s3Bucket = options.s3Bucket ?? env_default.S3_BUCKET;
|
|
@@ -7598,6 +8021,15 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
7598
8021
|
}
|
|
7599
8022
|
const { asset: initialAsset, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials, options.assetSnapshot);
|
|
7600
8023
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(initialAsset);
|
|
8024
|
+
for (const target of [
|
|
8025
|
+
...uploadToMux ? [audioTrackTarget] : [],
|
|
8026
|
+
...uploadCaptionsToMux ? [captionsTrackTarget] : []
|
|
8027
|
+
]) {
|
|
8028
|
+
const plan = planTextTrackReplacement(initialAsset, target, replaceExistingTracks);
|
|
8029
|
+
if (plan.kind === "blocked") {
|
|
8030
|
+
throw new MuxAiError(plan.reason, { type: "validation_error" });
|
|
8031
|
+
}
|
|
8032
|
+
}
|
|
7601
8033
|
let currentAsset = initialAsset;
|
|
7602
8034
|
let createdStaticRenditionId;
|
|
7603
8035
|
if (!hasReadyAudioStaticRendition(currentAsset)) {
|
|
@@ -7613,6 +8045,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
7613
8045
|
let dubbingId;
|
|
7614
8046
|
let presignedUrl;
|
|
7615
8047
|
let uploadedTrackId;
|
|
8048
|
+
let replacedTracks;
|
|
7616
8049
|
let captionsPresignedUrl;
|
|
7617
8050
|
let captionsTrackId;
|
|
7618
8051
|
try {
|
|
@@ -7703,18 +8136,29 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
7703
8136
|
}
|
|
7704
8137
|
if (uploadToMux) {
|
|
7705
8138
|
console.warn("\u{1F4F9} Adding dubbed audio track to Mux asset...");
|
|
7706
|
-
|
|
8139
|
+
let outcome;
|
|
7707
8140
|
try {
|
|
7708
|
-
|
|
7709
|
-
|
|
7710
|
-
|
|
7711
|
-
|
|
7712
|
-
|
|
8141
|
+
outcome = await replaceAndCreateTrack({
|
|
8142
|
+
assetId,
|
|
8143
|
+
target: audioTrackTarget,
|
|
8144
|
+
policy: replaceExistingTracks,
|
|
8145
|
+
presignedUrl,
|
|
8146
|
+
passthrough: trackPassthrough,
|
|
8147
|
+
credentials
|
|
8148
|
+
});
|
|
7713
8149
|
} catch (error) {
|
|
7714
|
-
|
|
7715
|
-
|
|
7716
|
-
|
|
8150
|
+
wrapError(error, `Failed to add dubbed audio track to Mux asset (dubbed audio is staged at: ${presignedUrl})`);
|
|
8151
|
+
}
|
|
8152
|
+
if (outcome.kind === "blocked") {
|
|
8153
|
+
throw new MuxAiError(`${outcome.reason} Dubbed audio is staged at: ${presignedUrl}`, { type: "validation_error" });
|
|
7717
8154
|
}
|
|
8155
|
+
if (outcome.kind === "create_failed") {
|
|
8156
|
+
throw new Error(`Failed to add dubbed audio track to Mux asset: ${outcome.reason}. Dubbed audio is staged at: ${presignedUrl}`);
|
|
8157
|
+
}
|
|
8158
|
+
uploadedTrackId = outcome.trackId;
|
|
8159
|
+
replacedTracks = outcome.deleted;
|
|
8160
|
+
console.warn(`\u2705 Track added to Mux asset with ID: ${uploadedTrackId}`);
|
|
8161
|
+
console.warn(`\u{1F4CB} Track name: "${trackName}"`);
|
|
7718
8162
|
}
|
|
7719
8163
|
try {
|
|
7720
8164
|
console.warn("\u{1F4E5} Downloading dub transcript from ElevenLabs and staging to S3...");
|
|
@@ -7732,15 +8176,21 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
7732
8176
|
});
|
|
7733
8177
|
if (uploadCaptionsToMux) {
|
|
7734
8178
|
console.warn("\u{1F4F9} Adding dubbed captions track to Mux asset...");
|
|
7735
|
-
const
|
|
7736
|
-
const languageName = new Intl.DisplayNames(["en"], { type: "language" }).of(muxLangCode) || muxLangCode.toUpperCase();
|
|
7737
|
-
captionsTrackId = await createTextTrackOnMux(
|
|
8179
|
+
const captionsOutcome = await replaceAndCreateTrack({
|
|
7738
8180
|
assetId,
|
|
7739
|
-
|
|
7740
|
-
|
|
7741
|
-
captionsPresignedUrl,
|
|
8181
|
+
target: captionsTrackTarget,
|
|
8182
|
+
policy: replaceExistingTracks,
|
|
8183
|
+
presignedUrl: captionsPresignedUrl,
|
|
8184
|
+
passthrough: trackPassthrough,
|
|
7742
8185
|
credentials
|
|
7743
|
-
);
|
|
8186
|
+
});
|
|
8187
|
+
if (captionsOutcome.deleted.length > 0) {
|
|
8188
|
+
replacedTracks = [...replacedTracks ?? [], ...captionsOutcome.deleted];
|
|
8189
|
+
}
|
|
8190
|
+
if (captionsOutcome.kind !== "created") {
|
|
8191
|
+
throw new Error(captionsOutcome.reason);
|
|
8192
|
+
}
|
|
8193
|
+
captionsTrackId = captionsOutcome.trackId;
|
|
7744
8194
|
console.warn(`\u2705 Captions track added to Mux asset with ID: ${captionsTrackId}`);
|
|
7745
8195
|
}
|
|
7746
8196
|
} catch (error) {
|
|
@@ -7771,6 +8221,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
7771
8221
|
dubbingId,
|
|
7772
8222
|
uploadedTrackId,
|
|
7773
8223
|
presignedUrl,
|
|
8224
|
+
replacedTracks,
|
|
7774
8225
|
captionsTrackId,
|
|
7775
8226
|
captionsPresignedUrl,
|
|
7776
8227
|
createdStaticRenditionId,
|
|
@@ -7797,7 +8248,7 @@ import { z as z9 } from "zod";
|
|
|
7797
8248
|
var translationSchema = z9.object({
|
|
7798
8249
|
translation: z9.string()
|
|
7799
8250
|
});
|
|
7800
|
-
var
|
|
8251
|
+
var SYSTEM_PROMPT5 = promptDedent`
|
|
7801
8252
|
You are a subtitle translation expert. Translate VTT subtitle files to the target language specified by the user.
|
|
7802
8253
|
You may receive either a full VTT file or a chunk from a larger VTT.
|
|
7803
8254
|
Preserve all timestamps, cue ordering, and VTT formatting exactly as they appear.
|
|
@@ -8122,7 +8573,7 @@ async function translateVttWithAI({
|
|
|
8122
8573
|
model,
|
|
8123
8574
|
maxRetries: 0,
|
|
8124
8575
|
output: Output7.object({ schema: translationSchema }),
|
|
8125
|
-
system:
|
|
8576
|
+
system: SYSTEM_PROMPT5,
|
|
8126
8577
|
messages: [
|
|
8127
8578
|
{
|
|
8128
8579
|
role: "user",
|
|
@@ -8440,11 +8891,15 @@ async function translateCaptionsInternal(assetId, trackId, toLanguageCode, optio
|
|
|
8440
8891
|
storageAdapter,
|
|
8441
8892
|
credentials: providedCredentials,
|
|
8442
8893
|
chunking,
|
|
8443
|
-
neverTranslate: neverTranslateOption
|
|
8894
|
+
neverTranslate: neverTranslateOption,
|
|
8895
|
+
replaceExistingTracks = "fail",
|
|
8896
|
+
trackName: providedTrackName,
|
|
8897
|
+
trackPassthrough: providedTrackPassthrough
|
|
8444
8898
|
} = options;
|
|
8445
8899
|
const credentials = providedCredentials;
|
|
8446
8900
|
const effectiveStorageAdapter = storageAdapter;
|
|
8447
8901
|
const neverTranslateTerms = neverTranslateOption ? validateNeverTranslateTerms(neverTranslateOption) : [];
|
|
8902
|
+
const trackPassthrough = validateTrackPassthrough(providedTrackPassthrough) ?? buildMuxAiTrackPassthrough("translate-captions");
|
|
8448
8903
|
const s3Endpoint = providedS3Endpoint ?? env_default.S3_ENDPOINT;
|
|
8449
8904
|
const s3Region = providedS3Region ?? env_default.S3_REGION ?? "auto";
|
|
8450
8905
|
const s3Bucket = providedS3Bucket ?? env_default.S3_BUCKET;
|
|
@@ -8485,6 +8940,16 @@ async function translateCaptionsInternal(assetId, trackId, toLanguageCode, optio
|
|
|
8485
8940
|
{ type: "validation_error" }
|
|
8486
8941
|
);
|
|
8487
8942
|
}
|
|
8943
|
+
const outputTrack = {
|
|
8944
|
+
languageCode: toLanguageCode,
|
|
8945
|
+
name: providedTrackName ?? `${getLanguageName(toLanguageCode) ?? toLanguageCode.toUpperCase()} (Auto-translated)`
|
|
8946
|
+
};
|
|
8947
|
+
if (uploadToMux) {
|
|
8948
|
+
const plan = planTextTrackReplacement(assetData, outputTrack, replaceExistingTracks);
|
|
8949
|
+
if (plan.kind === "blocked") {
|
|
8950
|
+
throw new MuxAiError(plan.reason, { type: "validation_error" });
|
|
8951
|
+
}
|
|
8952
|
+
}
|
|
8488
8953
|
const vttUrl = await buildTranscriptUrl(playbackId, trackId, policy === "signed", credentials);
|
|
8489
8954
|
let vttContent;
|
|
8490
8955
|
try {
|
|
@@ -8552,6 +9017,7 @@ async function translateCaptionsInternal(assetId, trackId, toLanguageCode, optio
|
|
|
8552
9017
|
const targetLanguage = getLanguageCodePair(toLanguageCode);
|
|
8553
9018
|
let presignedUrl;
|
|
8554
9019
|
let uploadedTrackId;
|
|
9020
|
+
let replacedTracks;
|
|
8555
9021
|
if (uploadToS3) {
|
|
8556
9022
|
try {
|
|
8557
9023
|
presignedUrl = await uploadVttToS3({
|
|
@@ -8569,19 +9035,28 @@ async function translateCaptionsInternal(assetId, trackId, toLanguageCode, optio
|
|
|
8569
9035
|
wrapError(error, "Failed to upload VTT to S3");
|
|
8570
9036
|
}
|
|
8571
9037
|
if (uploadToMux) {
|
|
9038
|
+
let outcome;
|
|
8572
9039
|
try {
|
|
8573
|
-
|
|
8574
|
-
const trackName = `${languageName} (auto-translated)`;
|
|
8575
|
-
uploadedTrackId = await createTextTrackOnMux(
|
|
9040
|
+
outcome = await replaceAndCreateTextTrack({
|
|
8576
9041
|
assetId,
|
|
8577
|
-
|
|
8578
|
-
|
|
9042
|
+
target: outputTrack,
|
|
9043
|
+
policy: replaceExistingTracks,
|
|
8579
9044
|
presignedUrl,
|
|
9045
|
+
passthrough: trackPassthrough,
|
|
8580
9046
|
credentials
|
|
8581
|
-
);
|
|
9047
|
+
});
|
|
8582
9048
|
} catch (error) {
|
|
8583
|
-
|
|
9049
|
+
wrapError(error, "Failed to add translated track to Mux asset");
|
|
9050
|
+
}
|
|
9051
|
+
if (outcome.kind !== "created") {
|
|
9052
|
+
const deletedNote = outcome.deleted.length > 0 ? ` Tracks already deleted before the failure: ${outcome.deleted.map((track) => `${track.name ?? track.id} (${track.id})`).join(", ")}.` : "";
|
|
9053
|
+
if (outcome.kind === "blocked") {
|
|
9054
|
+
throw new MuxAiError(`${outcome.reason}${deletedNote}`, { type: "validation_error" });
|
|
9055
|
+
}
|
|
9056
|
+
throw new Error(`Failed to add translated track to Mux asset: ${outcome.reason}.${deletedNote}`);
|
|
8584
9057
|
}
|
|
9058
|
+
uploadedTrackId = outcome.trackId;
|
|
9059
|
+
replacedTracks = outcome.deleted;
|
|
8585
9060
|
}
|
|
8586
9061
|
}
|
|
8587
9062
|
return {
|
|
@@ -8594,6 +9069,7 @@ async function translateCaptionsInternal(assetId, trackId, toLanguageCode, optio
|
|
|
8594
9069
|
originalVtt: vttContent,
|
|
8595
9070
|
translatedVtt,
|
|
8596
9071
|
uploadedTrackId,
|
|
9072
|
+
replacedTracks,
|
|
8597
9073
|
presignedUrl,
|
|
8598
9074
|
usage: usageWithMetadata,
|
|
8599
9075
|
safety,
|
|
@@ -8601,20 +9077,33 @@ async function translateCaptionsInternal(assetId, trackId, toLanguageCode, optio
|
|
|
8601
9077
|
};
|
|
8602
9078
|
}
|
|
8603
9079
|
export {
|
|
9080
|
+
MUX_TRACK_PASSTHROUGH_MAX_CHARS,
|
|
8604
9081
|
MuxAiError,
|
|
8605
9082
|
WorkflowStorageClient,
|
|
9083
|
+
buildMuxAiTrackPassthrough,
|
|
8606
9084
|
createWorkflowStorageClient,
|
|
8607
9085
|
decryptFromWorkflow,
|
|
8608
9086
|
encryptForWorkflow,
|
|
9087
|
+
findNameCollisionTextTracks,
|
|
9088
|
+
findSameLanguageTextTracks,
|
|
8609
9089
|
getLanguageCodePair,
|
|
8610
9090
|
getLanguageName,
|
|
9091
|
+
isDuplicateTrackNameError,
|
|
9092
|
+
isMuxGeneratedTextTrack,
|
|
8611
9093
|
isUndeterminedLanguageCode,
|
|
8612
9094
|
isValidISO639_1,
|
|
8613
9095
|
isValidISO639_3,
|
|
9096
|
+
normalizeTrackLanguageCode,
|
|
9097
|
+
normalizeTrackName,
|
|
9098
|
+
planTextTrackReplacement,
|
|
8614
9099
|
primitives_exports as primitives,
|
|
9100
|
+
replaceAndCreateTextTrack,
|
|
9101
|
+
replaceAndCreateTrack,
|
|
8615
9102
|
setWorkflowCredentialsProvider,
|
|
9103
|
+
summarizeTextTrack,
|
|
8616
9104
|
toISO639_1,
|
|
8617
9105
|
toISO639_3,
|
|
9106
|
+
validateTrackPassthrough,
|
|
8618
9107
|
version,
|
|
8619
9108
|
workflows_exports as workflows,
|
|
8620
9109
|
wrapError
|