@mux/ai 0.40.0 → 0.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -5,7 +5,7 @@ var __export = (target, all) => {
5
5
  };
6
6
 
7
7
  // package.json
8
- var version = "0.40.0";
8
+ var version = "0.42.0";
9
9
 
10
10
  // src/lib/language-codes.ts
11
11
  var ISO639_1_TO_3 = {
@@ -446,6 +446,7 @@ var PROMPT_TAG_NAMES = [
446
446
  "answer_guidelines",
447
447
  "relevance_filtering",
448
448
  "transcript_guidance",
449
+ "scene_guidance",
449
450
  "capabilities",
450
451
  "language_guidelines",
451
452
  "tone_guidance",
@@ -1113,6 +1114,234 @@ async function resolveMuxSigningContext(credentials) {
1113
1114
  return { keyId, keySecret };
1114
1115
  }
1115
1116
 
1117
+ // src/lib/mux-tracks.ts
1118
+ var MUX_TRACK_PASSTHROUGH_MAX_CHARS = 255;
1119
+ function normalizeTrackLanguageCode(code) {
1120
+ if (!code) {
1121
+ return void 0;
1122
+ }
1123
+ const primarySubtag = code.trim().toLowerCase().split(/[-_]/)[0];
1124
+ if (!primarySubtag || primarySubtag === "auto" || isUndeterminedLanguageCode(primarySubtag)) {
1125
+ return void 0;
1126
+ }
1127
+ return toISO639_1(primarySubtag);
1128
+ }
1129
+ function normalizeTrackName(name) {
1130
+ const normalized = name?.trim().toLowerCase();
1131
+ return normalized || void 0;
1132
+ }
1133
+ function isMuxGeneratedTextTrack(track) {
1134
+ return track.text_source === "generated_vod" || track.text_source === "generated_live" || track.text_source === "generated_live_final";
1135
+ }
1136
+ function targetType(target) {
1137
+ return target.type ?? "text";
1138
+ }
1139
+ function listTracks(asset, type) {
1140
+ return (asset.tracks ?? []).filter((track) => track.type === type && track.status !== "deleted" && !!track.id);
1141
+ }
1142
+ function summarizeTextTrack(track) {
1143
+ return {
1144
+ id: track.id,
1145
+ type: track.type === "audio" ? "audio" : "text",
1146
+ name: track.name,
1147
+ languageCode: track.language_code,
1148
+ status: track.status,
1149
+ textSource: track.text_source,
1150
+ passthrough: track.passthrough,
1151
+ ...track.type === "audio" ? { primary: track.primary === true } : {}
1152
+ };
1153
+ }
1154
+ function findSameLanguageTextTracks(asset, target) {
1155
+ const targetLanguage = normalizeTrackLanguageCode(target.languageCode);
1156
+ if (!targetLanguage) {
1157
+ return [];
1158
+ }
1159
+ const type = targetType(target);
1160
+ return listTracks(asset, type).filter(
1161
+ (track) => (type === "audio" || track.text_type === "subtitles") && normalizeTrackLanguageCode(track.language_code) === targetLanguage
1162
+ );
1163
+ }
1164
+ function findNameCollisionTextTracks(asset, target) {
1165
+ const targetName = normalizeTrackName(target.name);
1166
+ if (!targetName) {
1167
+ return [];
1168
+ }
1169
+ return listTracks(asset, targetType(target)).filter((track) => normalizeTrackName(track.name) === targetName);
1170
+ }
1171
+ function describeTracks(tracks) {
1172
+ return tracks.map((track) => `${track.name ?? "(unnamed)"} [${track.language_code ?? "?"}, ${track.type === "audio" ? track.primary ? "primary audio" : "audio" : track.text_source ?? "unknown source"}, ${track.status ?? "ready"}]`).join("; ");
1173
+ }
1174
+ function planTextTrackReplacement(asset, target, policy, options = {}) {
1175
+ const label = targetType(target) === "audio" ? "Audio" : "Text";
1176
+ const keep = new Set(options.keepTrackIds ?? []);
1177
+ const conflicts = /* @__PURE__ */ new Map();
1178
+ for (const track of [...findSameLanguageTextTracks(asset, target), ...findNameCollisionTextTracks(asset, target)]) {
1179
+ if (!keep.has(track.id)) {
1180
+ conflicts.set(track.id, track);
1181
+ }
1182
+ }
1183
+ const conflicting = [...conflicts.values()];
1184
+ if (conflicting.length === 0) {
1185
+ return { kind: "clear" };
1186
+ }
1187
+ if (policy === "fail") {
1188
+ return {
1189
+ kind: "blocked",
1190
+ reason: `${label} track(s) already exist for language '${target.languageCode}' or name '${target.name}': ${describeTracks(conflicting)}. Set replaceExistingTracks to replace them.`,
1191
+ tracks: conflicting.map(summarizeTextTrack)
1192
+ };
1193
+ }
1194
+ const primary = conflicting.filter((track) => track.type === "audio" && track.primary === true);
1195
+ if (primary.length > 0) {
1196
+ return {
1197
+ kind: "blocked",
1198
+ reason: `The asset's primary audio track is in the way for language '${target.languageCode}' or name '${target.name}': ${describeTracks(primary)}. Primary audio cannot be replaced; choose a different trackName.`,
1199
+ tracks: primary.map(summarizeTextTrack)
1200
+ };
1201
+ }
1202
+ if (policy === "replace_generated") {
1203
+ const notGenerated = conflicting.filter((track) => track.type !== "text" || !isMuxGeneratedTextTrack(track));
1204
+ if (notGenerated.length > 0) {
1205
+ return {
1206
+ kind: "blocked",
1207
+ reason: `${label} track(s) that are not Mux-generated exist for language '${target.languageCode}' or name '${target.name}': ${describeTracks(notGenerated)}. Use replaceExistingTracks: "replace_all" to replace them.`,
1208
+ tracks: notGenerated.map(summarizeTextTrack)
1209
+ };
1210
+ }
1211
+ }
1212
+ return { kind: "replace", toDelete: conflicting.map(summarizeTextTrack) };
1213
+ }
1214
+ function isDuplicateTrackNameError(error) {
1215
+ if (typeof error !== "object" || error === null) {
1216
+ return false;
1217
+ }
1218
+ const record = error;
1219
+ if (record.status !== 400) {
1220
+ return false;
1221
+ }
1222
+ const messages = Array.isArray(record.error?.messages) ? record.error.messages : [];
1223
+ const haystack = [...messages, record.message].filter((value) => typeof value === "string").map((value) => value.toLowerCase());
1224
+ return haystack.some((value) => value.includes("is not unique"));
1225
+ }
1226
+ function isNotFoundError(error) {
1227
+ return typeof error === "object" && error !== null && error.status === 404;
1228
+ }
1229
+ function buildMuxAiTrackPassthrough(workflow) {
1230
+ return JSON.stringify({ mux_ai: { workflow } });
1231
+ }
1232
+ function validateTrackPassthrough(passthrough) {
1233
+ if (passthrough !== void 0 && passthrough.length > MUX_TRACK_PASSTHROUGH_MAX_CHARS) {
1234
+ throw new MuxAiError(
1235
+ `Track passthrough must be at most ${MUX_TRACK_PASSTHROUGH_MAX_CHARS} characters (received ${passthrough.length}).`,
1236
+ { type: "validation_error" }
1237
+ );
1238
+ }
1239
+ return passthrough;
1240
+ }
1241
+ async function fetchVttFromMux(vttUrl) {
1242
+ "use step";
1243
+ const vttResponse = await fetch(vttUrl);
1244
+ if (!vttResponse.ok) {
1245
+ throw new Error(`Failed to fetch VTT file: ${vttResponse.statusText}`);
1246
+ }
1247
+ return vttResponse.text();
1248
+ }
1249
+ async function createTextTrackOnMux(assetId, languageCode, trackName, presignedUrl, credentials, options = {}) {
1250
+ "use step";
1251
+ const muxClient = await resolveMuxClient(credentials);
1252
+ const mux = await muxClient.createClient();
1253
+ const trackResponse = await mux.video.assets.createTrack(assetId, {
1254
+ type: "text",
1255
+ text_type: "subtitles",
1256
+ language_code: languageCode,
1257
+ name: trackName,
1258
+ url: presignedUrl,
1259
+ closed_captions: options.closedCaptions,
1260
+ passthrough: options.passthrough
1261
+ });
1262
+ if (!trackResponse.id) {
1263
+ throw new Error("Failed to create text track: no track ID returned from Mux");
1264
+ }
1265
+ return trackResponse.id;
1266
+ }
1267
+ async function replaceAndCreateTrack(input) {
1268
+ "use step";
1269
+ const muxClient = await resolveMuxClient(input.credentials);
1270
+ const mux = await muxClient.createClient();
1271
+ const type = targetType(input.target);
1272
+ const deleted = [];
1273
+ const failed = (error) => ({
1274
+ kind: "create_failed",
1275
+ reason: error instanceof Error ? error.message : String(error),
1276
+ deleted
1277
+ });
1278
+ const clearConflicts = async () => {
1279
+ let asset;
1280
+ try {
1281
+ asset = await mux.video.assets.retrieve(input.assetId);
1282
+ } catch (error) {
1283
+ if (deleted.length === 0) {
1284
+ throw error;
1285
+ }
1286
+ return failed(error);
1287
+ }
1288
+ const plan = planTextTrackReplacement(asset, input.target, input.policy, { keepTrackIds: input.keepTrackIds });
1289
+ if (plan.kind === "blocked") {
1290
+ return { ...plan, deleted };
1291
+ }
1292
+ if (plan.kind === "replace") {
1293
+ for (const track of plan.toDelete) {
1294
+ try {
1295
+ await mux.video.assets.deleteTrack(input.assetId, track.id);
1296
+ deleted.push(track);
1297
+ } catch (error) {
1298
+ if (!isNotFoundError(error)) {
1299
+ return failed(error);
1300
+ }
1301
+ }
1302
+ }
1303
+ }
1304
+ return void 0;
1305
+ };
1306
+ const create = async () => {
1307
+ const track = await mux.video.assets.createTrack(input.assetId, {
1308
+ type,
1309
+ ...type === "text" ? { text_type: "subtitles", closed_captions: input.closedCaptions } : {},
1310
+ language_code: input.target.languageCode,
1311
+ name: input.target.name.trim(),
1312
+ url: input.presignedUrl,
1313
+ passthrough: input.passthrough
1314
+ });
1315
+ if (!track.id) {
1316
+ throw new Error(`Failed to create ${type} track: no track ID returned from Mux`);
1317
+ }
1318
+ return track.id;
1319
+ };
1320
+ const stopped = await clearConflicts();
1321
+ if (stopped) {
1322
+ return stopped;
1323
+ }
1324
+ for (let attempt = 0; attempt < 2; attempt++) {
1325
+ try {
1326
+ return { kind: "created", trackId: await create(), deleted };
1327
+ } catch (error) {
1328
+ if (isDuplicateTrackNameError(error) && attempt === 0) {
1329
+ const stoppedOnRetry = await clearConflicts();
1330
+ if (stoppedOnRetry) {
1331
+ return stoppedOnRetry;
1332
+ }
1333
+ continue;
1334
+ }
1335
+ return failed(error);
1336
+ }
1337
+ }
1338
+ return { kind: "create_failed", reason: "Mux rejected the track name as not unique after retrying.", deleted };
1339
+ }
1340
+ async function replaceAndCreateTextTrack(input) {
1341
+ "use step";
1342
+ return replaceAndCreateTrack({ ...input, target: { ...input.target, type: "text" } });
1343
+ }
1344
+
1116
1345
  // node_modules/@workflow/serde/dist/index.js
1117
1346
  var WORKFLOW_SERIALIZE = /* @__PURE__ */ Symbol.for("workflow-serialize");
1118
1347
  var WORKFLOW_DESERIALIZE = /* @__PURE__ */ Symbol.for("workflow-deserialize");
@@ -4540,6 +4769,17 @@ var AUDIO_ONLY_SYSTEM_PROMPT_OVERRIDES = {
4540
4769
  You receive a timestamped transcript from audio-only content with lines in the form "[12s] Transcript text".
4541
4770
  Use those timestamps as anchors to determine chapter start times in seconds.`
4542
4771
  };
4772
+ var SCENE_CONTEXT_SYSTEM_PROMPT_OVERRIDES = {
4773
+ context: dedent4`
4774
+ You receive a timestamped transcript with lines in the form "[12s] Caption text" and an ordered scene context.
4775
+ Transcript timestamps are in seconds. Scene start and end timestamps are in milliseconds.
4776
+ Use the scene progression to choose logical chapter groupings and scene starts as chapter boundaries.`,
4777
+ constraints: dedent4`
4778
+ - Only use information present in the transcript and scene context
4779
+ - Return structured data that matches the requested JSON schema
4780
+ - Do not add commentary or extra text outside the JSON
4781
+ - When a <language> section is provided, all chapter titles MUST be written in that language`
4782
+ };
4543
4783
  var chaptersPromptBuilder = createPromptBuilder({
4544
4784
  template: {
4545
4785
  task: {
@@ -4577,6 +4817,7 @@ var chaptersPromptBuilder = createPromptBuilder({
4577
4817
  });
4578
4818
  function buildUserPrompt3({
4579
4819
  timestampedTranscript,
4820
+ sceneContext,
4580
4821
  promptOverrides,
4581
4822
  minChaptersPerHour = 3,
4582
4823
  maxChaptersPerHour = 8,
@@ -4590,14 +4831,31 @@ function buildUserPrompt3({
4590
4831
  attributes: { format: "seconds" }
4591
4832
  }
4592
4833
  ];
4834
+ if (sceneContext?.length) {
4835
+ contextSections.push({
4836
+ tag: "scene_context",
4837
+ content: JSON.stringify(sceneContext, null, 2),
4838
+ attributes: { timestamps: "milliseconds", trust: "untrusted_evidence" }
4839
+ });
4840
+ }
4593
4841
  if (languageName) {
4594
4842
  contextSections.push(createLanguageSection(languageName));
4595
4843
  }
4844
+ const sceneBoundarySeconds = getSceneBoundarySeconds(sceneContext).filter(
4845
+ (boundary) => scope ? boundary >= scope.startTime && boundary < scope.endTime : true
4846
+ );
4847
+ let firstChapterGuideline = "";
4848
+ if (sceneBoundarySeconds.length > 0) {
4849
+ firstChapterGuideline = `- The first chapter must start at ${sceneBoundarySeconds[0]}s, the first scene boundary inside the analyzed range`;
4850
+ } else if (scope) {
4851
+ firstChapterGuideline = `- The first chapter must start at ${scope.startTime}s`;
4852
+ }
4596
4853
  const dynamicChapterGuidelines = dedent4`
4597
4854
  - Create at least ${minChaptersPerHour} and at most ${maxChaptersPerHour} chapters per hour of content
4598
4855
  - Use start times in seconds (not HH:MM:SS)
4599
- ${scope ? `- Analyze only the range from ${scope.startTime}s (inclusive) to ${scope.endTime}s (exclusive)
4600
- - The first chapter must start at ${scope.startTime}s` : ""}
4856
+ ${scope ? `- Analyze only the range from ${scope.startTime}s (inclusive) to ${scope.endTime}s (exclusive)` : ""}
4857
+ ${firstChapterGuideline}
4858
+ ${sceneContext?.length ? "- Use the ordered scene context to identify chapter-sized runs of consecutive scenes\n- Every chapter startTime must exactly equal a scene start_ms divided by 1000; never split a scene" : ""}
4601
4859
  - Chapter start times should be non-decreasing
4602
4860
  - Do not include text before or after the JSON`;
4603
4861
  const mergedOverrides = {
@@ -4606,6 +4864,32 @@ function buildUserPrompt3({
4606
4864
  };
4607
4865
  return chaptersPromptBuilder.buildWithContext(mergedOverrides, contextSections);
4608
4866
  }
4867
+ function getSceneBoundarySeconds(sceneContext) {
4868
+ if (!sceneContext?.length) {
4869
+ return [];
4870
+ }
4871
+ return [...new Set(sceneContext.map((scene) => scene.start_ms / 1e3))].sort((a, b) => a - b);
4872
+ }
4873
+ function alignChaptersToSceneBoundaries(chapters, sceneBoundarySeconds) {
4874
+ if (sceneBoundarySeconds.length === 0) {
4875
+ return chapters;
4876
+ }
4877
+ const chapterByBoundary = /* @__PURE__ */ new Map();
4878
+ for (const chapter of chapters) {
4879
+ const startTime = sceneBoundarySeconds.reduce(
4880
+ (closest, boundary) => Math.abs(boundary - chapter.startTime) < Math.abs(closest - chapter.startTime) ? boundary : closest
4881
+ );
4882
+ const distance = Math.abs(startTime - chapter.startTime);
4883
+ const existing = chapterByBoundary.get(startTime);
4884
+ if (!existing || distance === 0 && existing.distance !== 0) {
4885
+ chapterByBoundary.set(startTime, {
4886
+ chapter: { ...chapter, startTime },
4887
+ distance
4888
+ });
4889
+ }
4890
+ }
4891
+ return [...chapterByBoundary.values()].map(({ chapter }) => chapter).sort((a, b) => a.startTime - b.startTime);
4892
+ }
4609
4893
  async function generateChapters(assetId, options = {}) {
4610
4894
  "use workflow";
4611
4895
  const collectedUsage = [];
@@ -4625,7 +4909,8 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
4625
4909
  maxChaptersPerHour,
4626
4910
  credentials,
4627
4911
  outputLanguageCode,
4628
- scope
4912
+ scope,
4913
+ sceneContext
4629
4914
  } = options;
4630
4915
  const modelConfig = resolveLanguageModelConfig({
4631
4916
  ...options,
@@ -4667,8 +4952,18 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
4667
4952
  }
4668
4953
  const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : getReliableLanguageCode(transcriptResult.track) ?? languageCode;
4669
4954
  const languageName = resolvedLanguageCode ? getLanguageName(resolvedLanguageCode) : void 0;
4955
+ const sceneBoundarySeconds = getSceneBoundarySeconds(sceneContext).filter(
4956
+ (boundary) => resolvedScope ? boundary >= resolvedScope.startTime && boundary < resolvedScope.endTime : true
4957
+ );
4958
+ if (sceneContext?.length && sceneBoundarySeconds.length === 0) {
4959
+ throw new MuxAiError(
4960
+ "Scene context has no chapter boundaries within the requested scope.",
4961
+ { type: "validation_error" }
4962
+ );
4963
+ }
4670
4964
  const userPrompt = buildUserPrompt3({
4671
4965
  timestampedTranscript,
4966
+ sceneContext,
4672
4967
  promptOverrides,
4673
4968
  minChaptersPerHour,
4674
4969
  maxChaptersPerHour,
@@ -4677,7 +4972,11 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
4677
4972
  });
4678
4973
  let chaptersData = null;
4679
4974
  try {
4680
- const systemPrompt = isAudioOnly ? chapterSystemPromptBuilder.build(AUDIO_ONLY_SYSTEM_PROMPT_OVERRIDES) : chapterSystemPromptBuilder.build();
4975
+ const systemPromptOverrides = {
4976
+ ...isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT_OVERRIDES : {},
4977
+ ...sceneContext?.length ? SCENE_CONTEXT_SYSTEM_PROMPT_OVERRIDES : {}
4978
+ };
4979
+ const systemPrompt = chapterSystemPromptBuilder.build(systemPromptOverrides);
4681
4980
  chaptersData = await generateChaptersWithAI({
4682
4981
  provider: modelConfig.provider,
4683
4982
  modelId: modelConfig.modelId,
@@ -4695,9 +4994,13 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
4695
4994
  throw new MuxAiError(`Failed to generate chapters for asset ${assetId}.`);
4696
4995
  }
4697
4996
  const { chapters: chaptersPayload, usage } = chaptersData;
4698
- const validChapters = chaptersPayload.chapters.filter((chapter) => typeof chapter.startTime === "number" && typeof chapter.title === "string").filter(
4997
+ const scopedChapters = chaptersPayload.chapters.filter((chapter) => typeof chapter.startTime === "number" && typeof chapter.title === "string").filter(
4699
4998
  (chapter) => resolvedScope ? chapter.startTime >= resolvedScope.startTime && chapter.startTime < resolvedScope.endTime : true
4700
4999
  ).sort((a, b) => a.startTime - b.startTime);
5000
+ const validChapters = alignChaptersToSceneBoundaries(
5001
+ scopedChapters,
5002
+ sceneBoundarySeconds
5003
+ );
4701
5004
  if (validChapters.length === 0) {
4702
5005
  throw new MuxAiError(`Failed to generate valid chapters for asset ${assetId}.`);
4703
5006
  }
@@ -4723,9 +5026,11 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
4723
5026
  if (scrubbedChapters.length === 0) {
4724
5027
  throw new MuxAiError(`Failed to generate valid chapters for asset ${assetId}.`);
4725
5028
  }
4726
- const firstChapterStartTime = resolvedScope?.startTime ?? 0;
4727
- if (scrubbedChapters[0].startTime !== firstChapterStartTime) {
4728
- scrubbedChapters[0].startTime = firstChapterStartTime;
5029
+ if (sceneBoundarySeconds.length === 0) {
5030
+ const firstChapterStartTime = resolvedScope?.startTime ?? 0;
5031
+ if (scrubbedChapters[0].startTime !== firstChapterStartTime) {
5032
+ scrubbedChapters[0].startTime = firstChapterStartTime;
5033
+ }
4729
5034
  }
4730
5035
  const usageWithMetadata = {
4731
5036
  ...usage,
@@ -4747,32 +5052,6 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
4747
5052
  import { generateText as generateText4, Output as Output4 } from "ai";
4748
5053
  import { z as z6 } from "zod";
4749
5054
 
4750
- // src/lib/mux-tracks.ts
4751
- async function fetchVttFromMux(vttUrl) {
4752
- "use step";
4753
- const vttResponse = await fetch(vttUrl);
4754
- if (!vttResponse.ok) {
4755
- throw new Error(`Failed to fetch VTT file: ${vttResponse.statusText}`);
4756
- }
4757
- return vttResponse.text();
4758
- }
4759
- async function createTextTrackOnMux(assetId, languageCode, trackName, presignedUrl, credentials) {
4760
- "use step";
4761
- const muxClient = await resolveMuxClient(credentials);
4762
- const mux = await muxClient.createClient();
4763
- const trackResponse = await mux.video.assets.createTrack(assetId, {
4764
- type: "text",
4765
- text_type: "subtitles",
4766
- language_code: languageCode,
4767
- name: trackName,
4768
- url: presignedUrl
4769
- });
4770
- if (!trackResponse.id) {
4771
- throw new Error("Failed to create text track: no track ID returned from Mux");
4772
- }
4773
- return trackResponse.id;
4774
- }
4775
-
4776
5055
  // src/lib/storage-adapter.ts
4777
5056
  function requireCredentials(accessKeyId, secretAccessKey) {
4778
5057
  if (!accessKeyId || !secretAccessKey) {
@@ -5051,6 +5330,7 @@ async function uploadEditedVttToS3({
5051
5330
  editedVtt,
5052
5331
  assetId,
5053
5332
  trackId,
5333
+ variant = "edited",
5054
5334
  s3Endpoint,
5055
5335
  s3Region,
5056
5336
  s3Bucket,
@@ -5060,7 +5340,7 @@ async function uploadEditedVttToS3({
5060
5340
  "use step";
5061
5341
  const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
5062
5342
  const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
5063
- const vttKey = `edited/${assetId}/${trackId}-edited-${Date.now()}.vtt`;
5343
+ const vttKey = `edited/${assetId}/${trackId}-${variant}-${Date.now()}.vtt`;
5064
5344
  await putObjectWithStorageAdapter({
5065
5345
  accessKeyId: s3AccessKeyId,
5066
5346
  secretAccessKey: s3SecretAccessKey,
@@ -5110,6 +5390,9 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
5110
5390
  s3Region: providedS3Region,
5111
5391
  s3Bucket: providedS3Bucket,
5112
5392
  trackNameSuffix,
5393
+ replaceExistingTracks: replaceExistingTracksOption,
5394
+ trackName: providedTrackName,
5395
+ trackPassthrough: providedTrackPassthrough,
5113
5396
  storageAdapter,
5114
5397
  credentials
5115
5398
  } = options;
@@ -5122,6 +5405,21 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
5122
5405
  if (autoCensorOption && !provider) {
5123
5406
  throw new MuxAiError("provider is required when using autoCensorProfanity.", { type: "validation_error" });
5124
5407
  }
5408
+ const legacyTrackNaming = deleteOriginalTrack !== void 0 || trackNameSuffix !== void 0;
5409
+ if (legacyTrackNaming && (replaceExistingTracksOption !== void 0 || providedTrackName !== void 0)) {
5410
+ throw new MuxAiError(
5411
+ "deleteOriginalTrack and trackNameSuffix are deprecated and cannot be combined with replaceExistingTracks or trackName.",
5412
+ { type: "validation_error" }
5413
+ );
5414
+ }
5415
+ const replaceExistingTracks = replaceExistingTracksOption ?? "replace_all";
5416
+ if (!legacyTrackNaming && replaceExistingTracks === "fail" && !providedTrackName) {
5417
+ throw new MuxAiError(
5418
+ `trackName is required when replaceExistingTracks is "fail": the edited track cannot reuse the source track's name while the source is kept.`,
5419
+ { type: "validation_error" }
5420
+ );
5421
+ }
5422
+ const trackPassthrough = validateTrackPassthrough(providedTrackPassthrough) ?? buildMuxAiTrackPassthrough("edit-captions");
5125
5423
  const deleteOriginal = deleteOriginalTrack !== false;
5126
5424
  const uploadToMux = uploadToMuxOption !== false;
5127
5425
  const uploadToS3 = uploadToS3Option || uploadToMux;
@@ -5164,6 +5462,25 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
5164
5462
  { type: "validation_error" }
5165
5463
  );
5166
5464
  }
5465
+ const sourceLanguageCode = sourceTrack.language_code || "en";
5466
+ const sourceName = sourceTrack.name || "Subtitles";
5467
+ const outputTrack = {
5468
+ languageCode: sourceLanguageCode,
5469
+ name: providedTrackName ?? sourceName
5470
+ };
5471
+ const keepTrackIds = replaceExistingTracks === "fail" ? [trackId] : [];
5472
+ if (replaceExistingTracks === "fail" && normalizeTrackName(outputTrack.name) === normalizeTrackName(sourceName)) {
5473
+ throw new MuxAiError(
5474
+ `trackName "${outputTrack.name}" matches the source track's name. Mux track names must be unique; choose a different trackName or use replaceExistingTracks: "replace_all" to replace the source.`,
5475
+ { type: "validation_error" }
5476
+ );
5477
+ }
5478
+ if (uploadToMux && !legacyTrackNaming) {
5479
+ const plan = planTextTrackReplacement(assetData, outputTrack, replaceExistingTracks, { keepTrackIds });
5480
+ if (plan.kind === "blocked") {
5481
+ throw new MuxAiError(plan.reason, { type: "validation_error" });
5482
+ }
5483
+ }
5167
5484
  const vttUrl = await buildTranscriptUrl(playbackId, trackId, policy === "signed", credentials);
5168
5485
  let vttContent;
5169
5486
  try {
@@ -5241,32 +5558,32 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
5241
5558
  } : void 0;
5242
5559
  let presignedUrl;
5243
5560
  let uploadedTrackId;
5561
+ let replacedTracks;
5244
5562
  if (uploadToS3) {
5563
+ const s3Config = {
5564
+ assetId,
5565
+ trackId,
5566
+ s3Endpoint,
5567
+ s3Region,
5568
+ s3Bucket,
5569
+ storageAdapter,
5570
+ s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
5571
+ };
5245
5572
  try {
5246
- presignedUrl = await uploadEditedVttToS3({
5247
- editedVtt,
5248
- assetId,
5249
- trackId,
5250
- s3Endpoint,
5251
- s3Region,
5252
- s3Bucket,
5253
- storageAdapter,
5254
- s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
5255
- });
5573
+ presignedUrl = await uploadEditedVttToS3({ editedVtt, ...s3Config });
5256
5574
  } catch (error) {
5257
5575
  wrapError(error, "Failed to upload VTT to S3");
5258
5576
  }
5259
- if (uploadToMux) {
5577
+ if (uploadToMux && legacyTrackNaming) {
5260
5578
  try {
5261
- const languageCode = sourceTrack.language_code || "en";
5262
- const suffix = trackNameSuffix ?? "edited";
5263
- const trackName = `${sourceTrack.name || "Subtitles"} (${suffix})`;
5579
+ const trackName = `${sourceName} (${trackNameSuffix ?? "edited"})`;
5264
5580
  uploadedTrackId = await createTextTrackOnMux(
5265
5581
  assetId,
5266
- languageCode,
5582
+ sourceLanguageCode,
5267
5583
  trackName,
5268
5584
  presignedUrl,
5269
- credentials
5585
+ credentials,
5586
+ { closedCaptions: sourceTrack.closed_captions, passthrough: trackPassthrough }
5270
5587
  );
5271
5588
  } catch (error) {
5272
5589
  wrapError(error, "Failed to add edited track to Mux asset");
@@ -5278,6 +5595,48 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
5278
5595
  wrapError(error, "Failed to delete original track");
5279
5596
  }
5280
5597
  }
5598
+ } else if (uploadToMux) {
5599
+ let outcome;
5600
+ try {
5601
+ outcome = await replaceAndCreateTextTrack({
5602
+ assetId,
5603
+ target: outputTrack,
5604
+ policy: replaceExistingTracks,
5605
+ presignedUrl,
5606
+ closedCaptions: sourceTrack.closed_captions,
5607
+ passthrough: trackPassthrough,
5608
+ keepTrackIds,
5609
+ credentials
5610
+ });
5611
+ } catch (error) {
5612
+ wrapError(error, "Failed to add edited track to Mux asset");
5613
+ }
5614
+ if (outcome.kind !== "created") {
5615
+ const sourceWasDeleted = outcome.deleted.some((track) => track.id === trackId);
5616
+ let restoreNote = "";
5617
+ if (sourceWasDeleted) {
5618
+ try {
5619
+ const originalUrl = await uploadEditedVttToS3({ editedVtt: vttContent, variant: "original", ...s3Config });
5620
+ const restoredTrackId = await createTextTrackOnMux(
5621
+ assetId,
5622
+ sourceLanguageCode,
5623
+ sourceName,
5624
+ originalUrl,
5625
+ credentials,
5626
+ { closedCaptions: sourceTrack.closed_captions, passthrough: sourceTrack.passthrough }
5627
+ );
5628
+ restoreNote = ` The source track was restored as ${restoredTrackId}.`;
5629
+ } catch (restoreError) {
5630
+ restoreNote = ` Restoring the source track also failed: ${restoreError instanceof Error ? restoreError.message : String(restoreError)}.`;
5631
+ }
5632
+ }
5633
+ if (outcome.kind === "blocked") {
5634
+ throw new MuxAiError(`${outcome.reason}${restoreNote}`, { type: "validation_error" });
5635
+ }
5636
+ throw new Error(`Failed to add edited track to Mux asset: ${outcome.reason}.${restoreNote}`);
5637
+ }
5638
+ uploadedTrackId = outcome.trackId;
5639
+ replacedTracks = outcome.deleted;
5281
5640
  }
5282
5641
  }
5283
5642
  return {
@@ -5290,6 +5649,7 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
5290
5649
  replacements: replacementsResult,
5291
5650
  speakerReplacements: speakerReplacementsResult,
5292
5651
  uploadedTrackId,
5652
+ replacedTracks,
5293
5653
  presignedUrl,
5294
5654
  usage: usageWithMetadata,
5295
5655
  safety: safety.report()
@@ -6777,14 +7137,44 @@ function buildDescriptionGuidance(wordCount, contentType) {
6777
7137
  Write in present tense. Be specific about what is discussed or presented rather than making assumptions.
6778
7138
  Focus on the spoken content and any key insights, dialogue, or narrative elements.`;
6779
7139
  }
6780
- function createSummarizationBuilder({ titleLength, descriptionLength, tagCount } = {}) {
7140
+ function buildSceneAwareDescriptionGuidance(wordCount) {
7141
+ if (wordCount < DESCRIPTION_LENGTH_THRESHOLD_SMALL) {
7142
+ return dedent6`A brief summary of the video in no more than ${wordCount} words. Shorter is fine.
7143
+ Focus on the single most important subject or action across the ordered scenes.
7144
+ Use the storyboard as direct visual evidence and the scene context to preserve progression.
7145
+ Write in present tense.`;
7146
+ }
7147
+ if (wordCount > DESCRIPTION_LENGTH_THRESHOLD_LARGE) {
7148
+ return dedent6`A detailed summary that describes what happens across the ordered scenes.
7149
+ Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
7150
+ Be thorough: cover subjects, actions, setting, progression, and notable details supported across scenes.
7151
+ Use the storyboard as direct visual evidence and the scene context to preserve progression.
7152
+ Write in present tense. If the transcript provides dialogue or narration, incorporate key points while preserving scene progression.`;
7153
+ }
7154
+ return dedent6`A summary that describes what happens across the ordered scenes.
7155
+ Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
7156
+ Cover the main subjects, actions, setting, and notable progression supported across scenes.
7157
+ Use the storyboard as direct visual evidence and the scene context to preserve progression.
7158
+ Write in present tense. If the transcript provides dialogue or narration, incorporate key points while preserving scene progression.`;
7159
+ }
7160
+ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount } = {}, { hasSceneContext = false } = {}) {
6781
7161
  const titleLimit = titleLength ?? DEFAULT_TITLE_LENGTH;
6782
7162
  const keywordLimit = tagCount ?? DEFAULT_SUMMARY_KEYWORD_LIMIT;
7163
+ const task = hasSceneContext ? "Analyze the storyboard frames and ordered scene context together, then generate metadata that captures the full progression of the video content." : "Analyze the storyboard frames and generate metadata that captures the essence of the video content.";
7164
+ const qualityGuidelines = hasSceneContext ? dedent6`
7165
+ - Follow scene_index order to understand what begins, develops, and concludes
7166
+ - Use scene narratives and concepts as grounded evidence, preserving consistent terminology across the output
7167
+ - Use the storyboard as direct visual evidence for whole-asset details
7168
+ - Balance brevity with informativeness` : dedent6`
7169
+ - Examine all frames to understand the full context and progression
7170
+ - Be precise: "golden retriever" is better than "dog" when identifiable
7171
+ - Capture the narrative: what begins, develops, and concludes
7172
+ - Balance brevity with informativeness`;
6783
7173
  return createPromptBuilder({
6784
7174
  template: {
6785
7175
  task: {
6786
7176
  tag: "task",
6787
- content: "Analyze the storyboard frames and generate metadata that captures the essence of the video content."
7177
+ content: task
6788
7178
  },
6789
7179
  title: {
6790
7180
  tag: "title_requirements",
@@ -6797,7 +7187,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
6797
7187
  },
6798
7188
  description: {
6799
7189
  tag: "description_requirements",
6800
- content: buildDescriptionGuidance(descriptionLength ?? DEFAULT_DESCRIPTION_LENGTH, "video")
7190
+ content: hasSceneContext ? buildSceneAwareDescriptionGuidance(descriptionLength ?? DEFAULT_DESCRIPTION_LENGTH) : buildDescriptionGuidance(descriptionLength ?? DEFAULT_DESCRIPTION_LENGTH, "video")
6801
7191
  },
6802
7192
  keywords: {
6803
7193
  tag: "keywords_requirements",
@@ -6813,11 +7203,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
6813
7203
  },
6814
7204
  qualityGuidelines: {
6815
7205
  tag: "quality_guidelines",
6816
- content: dedent6`
6817
- - Examine all frames to understand the full context and progression
6818
- - Be precise: "golden retriever" is better than "dog" when identifiable
6819
- - Capture the narrative: what begins, develops, and concludes
6820
- - Balance brevity with informativeness`
7206
+ content: qualityGuidelines
6821
7207
  }
6822
7208
  },
6823
7209
  sectionOrder: ["task", "title", "description", "keywords", "qualityGuidelines"]
@@ -6869,59 +7255,83 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
6869
7255
  sectionOrder: ["task", "title", "description", "keywords", "qualityGuidelines"]
6870
7256
  });
6871
7257
  }
6872
- var SYSTEM_PROMPT5 = promptDedent`
6873
- <role>
6874
- You are a video content analyst specializing in storyboard interpretation and multimodal analysis.
6875
- </role>
6876
-
6877
- <context>
6878
- You receive storyboard images containing multiple sequential frames extracted from a video.
6879
- ${STORYBOARD_FRAME_INSTRUCTIONS}
6880
- </context>
6881
-
6882
- <transcript_guidance>
6883
- When a transcript is provided alongside the storyboard:
6884
- - Use it to understand spoken content, dialogue, narration, and audio context
6885
- - Correlate transcript content with visual frames to build a complete picture
6886
- - Extract key terminology, names, and specific language used by speakers
6887
- - Let the transcript inform keyword selection, especially for topics not visually obvious
6888
- - Prioritize visual content for the description, but enrich it with transcript insights
6889
- - If transcript and visuals conflict, trust the visual evidence
6890
- </transcript_guidance>
6891
-
6892
- <capabilities>
6893
- - Extract meaning from visual sequences
6894
- - Identify subjects, actions, settings, and narrative arcs
6895
- - Generate accurate, searchable metadata
6896
- - Synthesize visual and transcript information when provided
6897
- </capabilities>
6898
-
6899
- <security>
6900
- ${NON_DISCLOSURE_CONSTRAINT}
6901
-
6902
- ${UNTRUSTED_USER_INPUT_NOTICE}
6903
-
6904
- ${VISUAL_TEXT_AS_CONTENT}
6905
-
6906
- ${CANARY_TRIPWIRE}
6907
- </security>
6908
-
6909
- <constraints>
6910
- - Only describe what is clearly observable in the frames or explicitly stated in the transcript
6911
- - ${NO_FABRICATION_CONSTRAINT}
6912
- - ${METADATA_BOUNDARY_WARNING}
6913
- - ${STRUCTURED_DATA_CONSTRAINT}
6914
- - Output only the JSON object; no markdown or extra text
6915
- - When a <language> section is provided, all output text MUST be written in that language
6916
- </constraints>
6917
-
6918
- <tone_guidance>
6919
- ${TONE_GUIDANCE}
6920
- </tone_guidance>
6921
-
6922
- <language_guidelines>
6923
- ${createLanguageGuidelines("video")}
6924
- </language_guidelines>`;
7258
+ function createVideoSystemPrompt(hasSceneContext) {
7259
+ let sceneContext = "";
7260
+ let sceneGuidance = "";
7261
+ if (hasSceneContext) {
7262
+ sceneContext = promptDedent`
7263
+ Ordered scene context is also provided as structured evidence of the video's progression.
7264
+ Entries use stable, zero-based scene_index values and absolute millisecond timestamps.
7265
+ Read scenes in scene_index order to understand what begins, develops, and concludes.`;
7266
+ sceneGuidance = promptDedent`
7267
+ <scene_guidance>
7268
+ - Use scene titles, narratives, and concepts to understand temporal progression
7269
+ - Use storyboard frames as direct visual evidence for subjects, actions, settings, and details
7270
+ - Preserve consistent terminology from the scene context across the title, description, and keywords
7271
+ - Treat optional scene fields as evidence only when present; do not infer omitted details
7272
+ - Consider all provided evidence together; if derived scene context conflicts with storyboard visuals, trust the visuals
7273
+ </scene_guidance>`;
7274
+ }
7275
+ const transcriptEvidence = hasSceneContext ? "storyboard frames and ordered scenes" : "visual frames";
7276
+ const descriptionPriority = hasSceneContext ? "Use the scene context for temporal structure and the storyboard for direct visual details, enriching both with transcript insights" : "Prioritize visual content for the description, but enrich it with transcript insights";
7277
+ const groundingConstraint = hasSceneContext ? "Use all provided evidence together. Ground each claim in the storyboard, scene context, or transcript" : "Only describe what is clearly observable in the frames or explicitly stated in the transcript";
7278
+ return promptDedent`
7279
+ <role>
7280
+ You are a video content analyst specializing in storyboard interpretation and multimodal analysis.
7281
+ </role>
7282
+
7283
+ <context>
7284
+ You receive storyboard images containing multiple sequential frames extracted from a video.
7285
+ ${STORYBOARD_FRAME_INSTRUCTIONS}
7286
+ ${sceneContext}
7287
+ </context>
7288
+
7289
+ ${sceneGuidance}
7290
+
7291
+ <transcript_guidance>
7292
+ When a transcript is provided alongside the storyboard:
7293
+ - Use it to understand spoken content, dialogue, narration, and audio context
7294
+ - Correlate transcript content with ${transcriptEvidence} to build a complete picture
7295
+ - Extract key terminology, names, and specific language used by speakers
7296
+ - Let the transcript inform keyword selection, especially for topics not visually obvious
7297
+ - ${descriptionPriority}
7298
+ - If transcript and visuals conflict, trust the visual evidence
7299
+ </transcript_guidance>
7300
+
7301
+ <capabilities>
7302
+ - Extract meaning from visual sequences
7303
+ - Identify subjects, actions, settings, and narrative arcs
7304
+ - Generate accurate, searchable metadata
7305
+ - Synthesize visual and transcript information when provided
7306
+ </capabilities>
7307
+
7308
+ <security>
7309
+ ${NON_DISCLOSURE_CONSTRAINT}
7310
+
7311
+ ${UNTRUSTED_USER_INPUT_NOTICE}
7312
+
7313
+ ${VISUAL_TEXT_AS_CONTENT}
7314
+
7315
+ ${CANARY_TRIPWIRE}
7316
+ </security>
7317
+
7318
+ <constraints>
7319
+ - ${groundingConstraint}
7320
+ - ${NO_FABRICATION_CONSTRAINT}
7321
+ - ${METADATA_BOUNDARY_WARNING}
7322
+ - ${STRUCTURED_DATA_CONSTRAINT}
7323
+ - Output only the JSON object; no markdown or extra text
7324
+ - When a <language> section is provided, all output text MUST be written in that language
7325
+ </constraints>
7326
+
7327
+ <tone_guidance>
7328
+ ${TONE_GUIDANCE}
7329
+ </tone_guidance>
7330
+
7331
+ <language_guidelines>
7332
+ ${createLanguageGuidelines("video")}
7333
+ </language_guidelines>`;
7334
+ }
6925
7335
  var AUDIO_ONLY_SYSTEM_PROMPT3 = promptDedent`
6926
7336
  <role>
6927
7337
  You are an audio content analyst specializing in transcript analysis and metadata generation.
@@ -6972,6 +7382,16 @@ var AUDIO_ONLY_SYSTEM_PROMPT3 = promptDedent`
6972
7382
  <language_guidelines>
6973
7383
  ${createLanguageGuidelines("audio")}
6974
7384
  </language_guidelines>`;
7385
+ function createSceneContextSection(sceneContext) {
7386
+ return {
7387
+ tag: "scene_context",
7388
+ content: JSON.stringify(sceneContext),
7389
+ attributes: {
7390
+ format: "json",
7391
+ order: "scene_index_ascending"
7392
+ }
7393
+ };
7394
+ }
6975
7395
  function buildUserPrompt5({
6976
7396
  tone,
6977
7397
  transcriptText,
@@ -6981,7 +7401,8 @@ function buildUserPrompt5({
6981
7401
  titleLength,
6982
7402
  descriptionLength,
6983
7403
  tagCount,
6984
- languageName
7404
+ languageName,
7405
+ sceneContext
6985
7406
  }) {
6986
7407
  const contextSections = [createToneSection(TONE_INSTRUCTIONS[tone])];
6987
7408
  if (languageName) {
@@ -6996,11 +7417,15 @@ function buildUserPrompt5({
6996
7417
  const format = isCleanTranscript ? "plain text" : "WebVTT";
6997
7418
  contextSections.push(createTranscriptSection(transcriptText, format));
6998
7419
  }
7420
+ const hasSceneContext = !isAudioOnly && Boolean(sceneContext?.length);
7421
+ if (hasSceneContext && sceneContext) {
7422
+ contextSections.push(createSceneContextSection(sceneContext));
7423
+ }
6999
7424
  const constraints = { titleLength, descriptionLength, tagCount };
7000
- const promptBuilder = isAudioOnly ? createAudioOnlyBuilder(constraints) : createSummarizationBuilder(constraints);
7425
+ const promptBuilder = isAudioOnly ? createAudioOnlyBuilder(constraints) : createSummarizationBuilder(constraints, { hasSceneContext });
7001
7426
  return promptBuilder.buildWithContext(promptOverrides, contextSections);
7002
7427
  }
7003
- async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, systemPrompt, descriptionLength, credentials) {
7428
+ async function analyzeVideo(imageDataUrl, provider, modelId, userPrompt, systemPrompt, descriptionLength, credentials) {
7004
7429
  "use step";
7005
7430
  const model = await createLanguageModelFromConfig(provider, modelId, credentials);
7006
7431
  const schema = buildSummarySchema(descriptionLength);
@@ -7016,10 +7441,10 @@ async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, s
7016
7441
  messages: [
7017
7442
  {
7018
7443
  role: "user",
7019
- content: [
7444
+ content: imageDataUrl ? [
7020
7445
  { type: "text", text: userPrompt },
7021
7446
  { type: "image", image: imageDataUrl }
7022
- ]
7447
+ ] : userPrompt
7023
7448
  }
7024
7449
  ]
7025
7450
  }));
@@ -7129,6 +7554,7 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
7129
7554
  cleanTranscript = true,
7130
7555
  imageSubmissionMode = "url",
7131
7556
  imageDownloadOptions,
7557
+ sceneContext,
7132
7558
  promptOverrides,
7133
7559
  credentials,
7134
7560
  titleLength,
@@ -7152,6 +7578,7 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
7152
7578
  const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, workflowCredentials, options?.assetSnapshot);
7153
7579
  const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
7154
7580
  const isAudioOnly = isAudioOnlyAsset(assetData);
7581
+ const hasSceneContext = !isAudioOnly && Boolean(sceneContext?.length);
7155
7582
  const effectiveScope = hasWorkflowScopeBoundaries(scope) ? scope : void 0;
7156
7583
  const storyboardScope = isAudioOnly ? void 0 : resolveRenderableVideoScope(
7157
7584
  effectiveScope,
@@ -7168,7 +7595,8 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
7168
7595
  );
7169
7596
  }
7170
7597
  const signingContext = await resolveMuxSigningContext(workflowCredentials);
7171
- if (policy === "signed" && !signingContext) {
7598
+ const requiresSignedPlaybackAccess = includeTranscript || !isAudioOnly;
7599
+ if (policy === "signed" && requiresSignedPlaybackAccess && !signingContext) {
7172
7600
  throw new MuxAiError(
7173
7601
  "Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
7174
7602
  { type: "validation_error" }
@@ -7194,11 +7622,15 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
7194
7622
  titleLength,
7195
7623
  descriptionLength,
7196
7624
  tagCount,
7197
- languageName
7625
+ languageName,
7626
+ sceneContext: hasSceneContext ? sceneContext : void 0
7198
7627
  });
7199
7628
  let analysisResponse;
7200
7629
  let imageUrl;
7201
- const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT3 : SYSTEM_PROMPT5;
7630
+ let systemPrompt = AUDIO_ONLY_SYSTEM_PROMPT3;
7631
+ if (!isAudioOnly) {
7632
+ systemPrompt = createVideoSystemPrompt(hasSceneContext);
7633
+ }
7202
7634
  const effectiveDescriptionLength = descriptionLength ?? DEFAULT_DESCRIPTION_LENGTH;
7203
7635
  try {
7204
7636
  if (isAudioOnly) {
@@ -7221,7 +7653,7 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
7221
7653
  imageUrl = storyboardUrl;
7222
7654
  if (imageSubmissionMode === "base64") {
7223
7655
  const downloadResult = await downloadImageAsBase64(storyboardUrl, imageDownloadOptions);
7224
- analysisResponse = await analyzeStoryboard2(
7656
+ analysisResponse = await analyzeVideo(
7225
7657
  downloadResult.base64Data,
7226
7658
  modelConfig.provider,
7227
7659
  modelConfig.modelId,
@@ -7231,7 +7663,7 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
7231
7663
  workflowCredentials
7232
7664
  );
7233
7665
  } else {
7234
- analysisResponse = await analyzeStoryboard2(
7666
+ analysisResponse = await analyzeVideo(
7235
7667
  storyboardUrl,
7236
7668
  modelConfig.provider,
7237
7669
  modelConfig.modelId,
@@ -7550,23 +7982,6 @@ async function downloadAndUploadDubTranscript({
7550
7982
  console.warn(`\u2705 Dub transcript uploaded successfully to: ${vttKey}`);
7551
7983
  return presignedUrl;
7552
7984
  }
7553
- async function createAudioTrackOnMux(assetId, languageCode, presignedUrl, credentials) {
7554
- "use step";
7555
- const muxClient = await resolveMuxClient(credentials);
7556
- const mux = await muxClient.createClient();
7557
- const languageName = new Intl.DisplayNames(["en"], { type: "language" }).of(languageCode) || languageCode.toUpperCase();
7558
- const trackName = `${languageName} (auto-dubbed)`;
7559
- const trackResponse = await mux.video.assets.createTrack(assetId, {
7560
- type: "audio",
7561
- language_code: languageCode,
7562
- name: trackName,
7563
- url: presignedUrl
7564
- });
7565
- if (!trackResponse.id) {
7566
- throw new Error("Failed to create audio track: no track ID returned from Mux");
7567
- }
7568
- return trackResponse.id;
7569
- }
7570
7985
  async function translateAudio(assetId, toLanguageCode, options = {}) {
7571
7986
  "use workflow";
7572
7987
  const {
@@ -7579,15 +7994,23 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
7579
7994
  uploadCaptionsToMux = false,
7580
7995
  staticRenditionCleanup: staticRenditionCleanupOption = "delete",
7581
7996
  storageAdapter,
7582
- credentials: providedCredentials
7997
+ credentials: providedCredentials,
7998
+ replaceExistingTracks = "fail",
7999
+ trackName: providedTrackName,
8000
+ trackPassthrough: providedTrackPassthrough
7583
8001
  } = options;
7584
8002
  if (provider !== "elevenlabs") {
7585
8003
  throw new MuxAiError("Only ElevenLabs provider is currently supported for audio translation.", { type: "validation_error" });
7586
8004
  }
7587
8005
  const credentials = providedCredentials;
7588
8006
  const effectiveStorageAdapter = storageAdapter;
8007
+ const trackPassthrough = validateTrackPassthrough(providedTrackPassthrough) ?? buildMuxAiTrackPassthrough("translate-audio");
7589
8008
  const uploadToMux = uploadToMuxOption !== false;
7590
8009
  const uploadToS3 = uploadToS3Option || uploadToMux || uploadCaptionsToMux;
8010
+ const muxLangCode = toISO639_1(toLanguageCode);
8011
+ const trackName = providedTrackName ?? `${getLanguageName(muxLangCode) ?? muxLangCode.toUpperCase()} (Auto-dubbed)`;
8012
+ const audioTrackTarget = { type: "audio", languageCode: muxLangCode, name: trackName };
8013
+ const captionsTrackTarget = { type: "text", languageCode: muxLangCode, name: trackName };
7591
8014
  const s3Endpoint = options.s3Endpoint ?? env_default.S3_ENDPOINT;
7592
8015
  const s3Region = options.s3Region ?? env_default.S3_REGION ?? "auto";
7593
8016
  const s3Bucket = options.s3Bucket ?? env_default.S3_BUCKET;
@@ -7598,6 +8021,15 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
7598
8021
  }
7599
8022
  const { asset: initialAsset, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials, options.assetSnapshot);
7600
8023
  const assetDurationSeconds = getAssetDurationSecondsFromAsset(initialAsset);
8024
+ for (const target of [
8025
+ ...uploadToMux ? [audioTrackTarget] : [],
8026
+ ...uploadCaptionsToMux ? [captionsTrackTarget] : []
8027
+ ]) {
8028
+ const plan = planTextTrackReplacement(initialAsset, target, replaceExistingTracks);
8029
+ if (plan.kind === "blocked") {
8030
+ throw new MuxAiError(plan.reason, { type: "validation_error" });
8031
+ }
8032
+ }
7601
8033
  let currentAsset = initialAsset;
7602
8034
  let createdStaticRenditionId;
7603
8035
  if (!hasReadyAudioStaticRendition(currentAsset)) {
@@ -7613,6 +8045,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
7613
8045
  let dubbingId;
7614
8046
  let presignedUrl;
7615
8047
  let uploadedTrackId;
8048
+ let replacedTracks;
7616
8049
  let captionsPresignedUrl;
7617
8050
  let captionsTrackId;
7618
8051
  try {
@@ -7703,18 +8136,29 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
7703
8136
  }
7704
8137
  if (uploadToMux) {
7705
8138
  console.warn("\u{1F4F9} Adding dubbed audio track to Mux asset...");
7706
- const muxLangCode = toISO639_1(toLanguageCode);
8139
+ let outcome;
7707
8140
  try {
7708
- uploadedTrackId = await createAudioTrackOnMux(assetId, muxLangCode, presignedUrl, credentials);
7709
- const languageName = new Intl.DisplayNames(["en"], { type: "language" }).of(muxLangCode) || muxLangCode.toUpperCase();
7710
- const trackName = `${languageName} (auto-dubbed)`;
7711
- console.warn(`\u2705 Track added to Mux asset with ID: ${uploadedTrackId}`);
7712
- console.warn(`\u{1F4CB} Track name: "${trackName}"`);
8141
+ outcome = await replaceAndCreateTrack({
8142
+ assetId,
8143
+ target: audioTrackTarget,
8144
+ policy: replaceExistingTracks,
8145
+ presignedUrl,
8146
+ passthrough: trackPassthrough,
8147
+ credentials
8148
+ });
7713
8149
  } catch (error) {
7714
- console.warn(`\u26A0\uFE0F Failed to add audio track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
7715
- console.warn("\u{1F517} You can manually add the track using this presigned URL:");
7716
- console.warn(presignedUrl);
8150
+ wrapError(error, `Failed to add dubbed audio track to Mux asset (dubbed audio is staged at: ${presignedUrl})`);
8151
+ }
8152
+ if (outcome.kind === "blocked") {
8153
+ throw new MuxAiError(`${outcome.reason} Dubbed audio is staged at: ${presignedUrl}`, { type: "validation_error" });
7717
8154
  }
8155
+ if (outcome.kind === "create_failed") {
8156
+ throw new Error(`Failed to add dubbed audio track to Mux asset: ${outcome.reason}. Dubbed audio is staged at: ${presignedUrl}`);
8157
+ }
8158
+ uploadedTrackId = outcome.trackId;
8159
+ replacedTracks = outcome.deleted;
8160
+ console.warn(`\u2705 Track added to Mux asset with ID: ${uploadedTrackId}`);
8161
+ console.warn(`\u{1F4CB} Track name: "${trackName}"`);
7718
8162
  }
7719
8163
  try {
7720
8164
  console.warn("\u{1F4E5} Downloading dub transcript from ElevenLabs and staging to S3...");
@@ -7732,15 +8176,21 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
7732
8176
  });
7733
8177
  if (uploadCaptionsToMux) {
7734
8178
  console.warn("\u{1F4F9} Adding dubbed captions track to Mux asset...");
7735
- const muxLangCode = toISO639_1(toLanguageCode);
7736
- const languageName = new Intl.DisplayNames(["en"], { type: "language" }).of(muxLangCode) || muxLangCode.toUpperCase();
7737
- captionsTrackId = await createTextTrackOnMux(
8179
+ const captionsOutcome = await replaceAndCreateTrack({
7738
8180
  assetId,
7739
- muxLangCode,
7740
- `${languageName} (auto-dubbed)`,
7741
- captionsPresignedUrl,
8181
+ target: captionsTrackTarget,
8182
+ policy: replaceExistingTracks,
8183
+ presignedUrl: captionsPresignedUrl,
8184
+ passthrough: trackPassthrough,
7742
8185
  credentials
7743
- );
8186
+ });
8187
+ if (captionsOutcome.deleted.length > 0) {
8188
+ replacedTracks = [...replacedTracks ?? [], ...captionsOutcome.deleted];
8189
+ }
8190
+ if (captionsOutcome.kind !== "created") {
8191
+ throw new Error(captionsOutcome.reason);
8192
+ }
8193
+ captionsTrackId = captionsOutcome.trackId;
7744
8194
  console.warn(`\u2705 Captions track added to Mux asset with ID: ${captionsTrackId}`);
7745
8195
  }
7746
8196
  } catch (error) {
@@ -7771,6 +8221,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
7771
8221
  dubbingId,
7772
8222
  uploadedTrackId,
7773
8223
  presignedUrl,
8224
+ replacedTracks,
7774
8225
  captionsTrackId,
7775
8226
  captionsPresignedUrl,
7776
8227
  createdStaticRenditionId,
@@ -7797,7 +8248,7 @@ import { z as z9 } from "zod";
7797
8248
  var translationSchema = z9.object({
7798
8249
  translation: z9.string()
7799
8250
  });
7800
- var SYSTEM_PROMPT6 = promptDedent`
8251
+ var SYSTEM_PROMPT5 = promptDedent`
7801
8252
  You are a subtitle translation expert. Translate VTT subtitle files to the target language specified by the user.
7802
8253
  You may receive either a full VTT file or a chunk from a larger VTT.
7803
8254
  Preserve all timestamps, cue ordering, and VTT formatting exactly as they appear.
@@ -8122,7 +8573,7 @@ async function translateVttWithAI({
8122
8573
  model,
8123
8574
  maxRetries: 0,
8124
8575
  output: Output7.object({ schema: translationSchema }),
8125
- system: SYSTEM_PROMPT6,
8576
+ system: SYSTEM_PROMPT5,
8126
8577
  messages: [
8127
8578
  {
8128
8579
  role: "user",
@@ -8440,11 +8891,15 @@ async function translateCaptionsInternal(assetId, trackId, toLanguageCode, optio
8440
8891
  storageAdapter,
8441
8892
  credentials: providedCredentials,
8442
8893
  chunking,
8443
- neverTranslate: neverTranslateOption
8894
+ neverTranslate: neverTranslateOption,
8895
+ replaceExistingTracks = "fail",
8896
+ trackName: providedTrackName,
8897
+ trackPassthrough: providedTrackPassthrough
8444
8898
  } = options;
8445
8899
  const credentials = providedCredentials;
8446
8900
  const effectiveStorageAdapter = storageAdapter;
8447
8901
  const neverTranslateTerms = neverTranslateOption ? validateNeverTranslateTerms(neverTranslateOption) : [];
8902
+ const trackPassthrough = validateTrackPassthrough(providedTrackPassthrough) ?? buildMuxAiTrackPassthrough("translate-captions");
8448
8903
  const s3Endpoint = providedS3Endpoint ?? env_default.S3_ENDPOINT;
8449
8904
  const s3Region = providedS3Region ?? env_default.S3_REGION ?? "auto";
8450
8905
  const s3Bucket = providedS3Bucket ?? env_default.S3_BUCKET;
@@ -8485,6 +8940,16 @@ async function translateCaptionsInternal(assetId, trackId, toLanguageCode, optio
8485
8940
  { type: "validation_error" }
8486
8941
  );
8487
8942
  }
8943
+ const outputTrack = {
8944
+ languageCode: toLanguageCode,
8945
+ name: providedTrackName ?? `${getLanguageName(toLanguageCode) ?? toLanguageCode.toUpperCase()} (Auto-translated)`
8946
+ };
8947
+ if (uploadToMux) {
8948
+ const plan = planTextTrackReplacement(assetData, outputTrack, replaceExistingTracks);
8949
+ if (plan.kind === "blocked") {
8950
+ throw new MuxAiError(plan.reason, { type: "validation_error" });
8951
+ }
8952
+ }
8488
8953
  const vttUrl = await buildTranscriptUrl(playbackId, trackId, policy === "signed", credentials);
8489
8954
  let vttContent;
8490
8955
  try {
@@ -8552,6 +9017,7 @@ async function translateCaptionsInternal(assetId, trackId, toLanguageCode, optio
8552
9017
  const targetLanguage = getLanguageCodePair(toLanguageCode);
8553
9018
  let presignedUrl;
8554
9019
  let uploadedTrackId;
9020
+ let replacedTracks;
8555
9021
  if (uploadToS3) {
8556
9022
  try {
8557
9023
  presignedUrl = await uploadVttToS3({
@@ -8569,19 +9035,28 @@ async function translateCaptionsInternal(assetId, trackId, toLanguageCode, optio
8569
9035
  wrapError(error, "Failed to upload VTT to S3");
8570
9036
  }
8571
9037
  if (uploadToMux) {
9038
+ let outcome;
8572
9039
  try {
8573
- const languageName = getLanguageName(toLanguageCode) ?? toLanguageCode.toUpperCase();
8574
- const trackName = `${languageName} (auto-translated)`;
8575
- uploadedTrackId = await createTextTrackOnMux(
9040
+ outcome = await replaceAndCreateTextTrack({
8576
9041
  assetId,
8577
- toLanguageCode,
8578
- trackName,
9042
+ target: outputTrack,
9043
+ policy: replaceExistingTracks,
8579
9044
  presignedUrl,
9045
+ passthrough: trackPassthrough,
8580
9046
  credentials
8581
- );
9047
+ });
8582
9048
  } catch (error) {
8583
- console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
9049
+ wrapError(error, "Failed to add translated track to Mux asset");
9050
+ }
9051
+ if (outcome.kind !== "created") {
9052
+ const deletedNote = outcome.deleted.length > 0 ? ` Tracks already deleted before the failure: ${outcome.deleted.map((track) => `${track.name ?? track.id} (${track.id})`).join(", ")}.` : "";
9053
+ if (outcome.kind === "blocked") {
9054
+ throw new MuxAiError(`${outcome.reason}${deletedNote}`, { type: "validation_error" });
9055
+ }
9056
+ throw new Error(`Failed to add translated track to Mux asset: ${outcome.reason}.${deletedNote}`);
8584
9057
  }
9058
+ uploadedTrackId = outcome.trackId;
9059
+ replacedTracks = outcome.deleted;
8585
9060
  }
8586
9061
  }
8587
9062
  return {
@@ -8594,6 +9069,7 @@ async function translateCaptionsInternal(assetId, trackId, toLanguageCode, optio
8594
9069
  originalVtt: vttContent,
8595
9070
  translatedVtt,
8596
9071
  uploadedTrackId,
9072
+ replacedTracks,
8597
9073
  presignedUrl,
8598
9074
  usage: usageWithMetadata,
8599
9075
  safety,
@@ -8601,20 +9077,33 @@ async function translateCaptionsInternal(assetId, trackId, toLanguageCode, optio
8601
9077
  };
8602
9078
  }
8603
9079
  export {
9080
+ MUX_TRACK_PASSTHROUGH_MAX_CHARS,
8604
9081
  MuxAiError,
8605
9082
  WorkflowStorageClient,
9083
+ buildMuxAiTrackPassthrough,
8606
9084
  createWorkflowStorageClient,
8607
9085
  decryptFromWorkflow,
8608
9086
  encryptForWorkflow,
9087
+ findNameCollisionTextTracks,
9088
+ findSameLanguageTextTracks,
8609
9089
  getLanguageCodePair,
8610
9090
  getLanguageName,
9091
+ isDuplicateTrackNameError,
9092
+ isMuxGeneratedTextTrack,
8611
9093
  isUndeterminedLanguageCode,
8612
9094
  isValidISO639_1,
8613
9095
  isValidISO639_3,
9096
+ normalizeTrackLanguageCode,
9097
+ normalizeTrackName,
9098
+ planTextTrackReplacement,
8614
9099
  primitives_exports as primitives,
9100
+ replaceAndCreateTextTrack,
9101
+ replaceAndCreateTrack,
8615
9102
  setWorkflowCredentialsProvider,
9103
+ summarizeTextTrack,
8616
9104
  toISO639_1,
8617
9105
  toISO639_3,
9106
+ validateTrackPassthrough,
8618
9107
  version,
8619
9108
  workflows_exports as workflows,
8620
9109
  wrapError