@mux/ai 0.40.0 → 0.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -275,6 +275,7 @@ var PROMPT_TAG_NAMES = [
275
275
  "answer_guidelines",
276
276
  "relevance_filtering",
277
277
  "transcript_guidance",
278
+ "scene_guidance",
278
279
  "capabilities",
279
280
  "language_guidelines",
280
281
  "tone_guidance",
@@ -3571,6 +3572,17 @@ var AUDIO_ONLY_SYSTEM_PROMPT_OVERRIDES = {
3571
3572
  You receive a timestamped transcript from audio-only content with lines in the form "[12s] Transcript text".
3572
3573
  Use those timestamps as anchors to determine chapter start times in seconds.`
3573
3574
  };
3575
+ var SCENE_CONTEXT_SYSTEM_PROMPT_OVERRIDES = {
3576
+ context: dedent4`
3577
+ You receive a timestamped transcript with lines in the form "[12s] Caption text" and an ordered scene context.
3578
+ Transcript timestamps are in seconds. Scene start and end timestamps are in milliseconds.
3579
+ Use the scene progression to choose logical chapter groupings and scene starts as chapter boundaries.`,
3580
+ constraints: dedent4`
3581
+ - Only use information present in the transcript and scene context
3582
+ - Return structured data that matches the requested JSON schema
3583
+ - Do not add commentary or extra text outside the JSON
3584
+ - When a <language> section is provided, all chapter titles MUST be written in that language`
3585
+ };
3574
3586
  var chaptersPromptBuilder = createPromptBuilder({
3575
3587
  template: {
3576
3588
  task: {
@@ -3608,6 +3620,7 @@ var chaptersPromptBuilder = createPromptBuilder({
3608
3620
  });
3609
3621
  function buildUserPrompt3({
3610
3622
  timestampedTranscript,
3623
+ sceneContext,
3611
3624
  promptOverrides,
3612
3625
  minChaptersPerHour = 3,
3613
3626
  maxChaptersPerHour = 8,
@@ -3621,14 +3634,31 @@ function buildUserPrompt3({
3621
3634
  attributes: { format: "seconds" }
3622
3635
  }
3623
3636
  ];
3637
+ if (sceneContext?.length) {
3638
+ contextSections.push({
3639
+ tag: "scene_context",
3640
+ content: JSON.stringify(sceneContext, null, 2),
3641
+ attributes: { timestamps: "milliseconds", trust: "untrusted_evidence" }
3642
+ });
3643
+ }
3624
3644
  if (languageName) {
3625
3645
  contextSections.push(createLanguageSection(languageName));
3626
3646
  }
3647
+ const sceneBoundarySeconds = getSceneBoundarySeconds(sceneContext).filter(
3648
+ (boundary) => scope ? boundary >= scope.startTime && boundary < scope.endTime : true
3649
+ );
3650
+ let firstChapterGuideline = "";
3651
+ if (sceneBoundarySeconds.length > 0) {
3652
+ firstChapterGuideline = `- The first chapter must start at ${sceneBoundarySeconds[0]}s, the first scene boundary inside the analyzed range`;
3653
+ } else if (scope) {
3654
+ firstChapterGuideline = `- The first chapter must start at ${scope.startTime}s`;
3655
+ }
3627
3656
  const dynamicChapterGuidelines = dedent4`
3628
3657
  - Create at least ${minChaptersPerHour} and at most ${maxChaptersPerHour} chapters per hour of content
3629
3658
  - Use start times in seconds (not HH:MM:SS)
3630
- ${scope ? `- Analyze only the range from ${scope.startTime}s (inclusive) to ${scope.endTime}s (exclusive)
3631
- - The first chapter must start at ${scope.startTime}s` : ""}
3659
+ ${scope ? `- Analyze only the range from ${scope.startTime}s (inclusive) to ${scope.endTime}s (exclusive)` : ""}
3660
+ ${firstChapterGuideline}
3661
+ ${sceneContext?.length ? "- Use the ordered scene context to identify chapter-sized runs of consecutive scenes\n- Every chapter startTime must exactly equal a scene start_ms divided by 1000; never split a scene" : ""}
3632
3662
  - Chapter start times should be non-decreasing
3633
3663
  - Do not include text before or after the JSON`;
3634
3664
  const mergedOverrides = {
@@ -3637,6 +3667,32 @@ function buildUserPrompt3({
3637
3667
  };
3638
3668
  return chaptersPromptBuilder.buildWithContext(mergedOverrides, contextSections);
3639
3669
  }
3670
+ function getSceneBoundarySeconds(sceneContext) {
3671
+ if (!sceneContext?.length) {
3672
+ return [];
3673
+ }
3674
+ return [...new Set(sceneContext.map((scene) => scene.start_ms / 1e3))].sort((a, b) => a - b);
3675
+ }
3676
+ function alignChaptersToSceneBoundaries(chapters, sceneBoundarySeconds) {
3677
+ if (sceneBoundarySeconds.length === 0) {
3678
+ return chapters;
3679
+ }
3680
+ const chapterByBoundary = /* @__PURE__ */ new Map();
3681
+ for (const chapter of chapters) {
3682
+ const startTime = sceneBoundarySeconds.reduce(
3683
+ (closest, boundary) => Math.abs(boundary - chapter.startTime) < Math.abs(closest - chapter.startTime) ? boundary : closest
3684
+ );
3685
+ const distance = Math.abs(startTime - chapter.startTime);
3686
+ const existing = chapterByBoundary.get(startTime);
3687
+ if (!existing || distance === 0 && existing.distance !== 0) {
3688
+ chapterByBoundary.set(startTime, {
3689
+ chapter: { ...chapter, startTime },
3690
+ distance
3691
+ });
3692
+ }
3693
+ }
3694
+ return [...chapterByBoundary.values()].map(({ chapter }) => chapter).sort((a, b) => a.startTime - b.startTime);
3695
+ }
3640
3696
  async function generateChapters(assetId, options = {}) {
3641
3697
  "use workflow";
3642
3698
  const collectedUsage = [];
@@ -3656,7 +3712,8 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
3656
3712
  maxChaptersPerHour,
3657
3713
  credentials,
3658
3714
  outputLanguageCode,
3659
- scope
3715
+ scope,
3716
+ sceneContext
3660
3717
  } = options;
3661
3718
  const modelConfig = resolveLanguageModelConfig({
3662
3719
  ...options,
@@ -3698,8 +3755,18 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
3698
3755
  }
3699
3756
  const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : getReliableLanguageCode(transcriptResult.track) ?? languageCode;
3700
3757
  const languageName = resolvedLanguageCode ? getLanguageName(resolvedLanguageCode) : void 0;
3758
+ const sceneBoundarySeconds = getSceneBoundarySeconds(sceneContext).filter(
3759
+ (boundary) => resolvedScope ? boundary >= resolvedScope.startTime && boundary < resolvedScope.endTime : true
3760
+ );
3761
+ if (sceneContext?.length && sceneBoundarySeconds.length === 0) {
3762
+ throw new MuxAiError(
3763
+ "Scene context has no chapter boundaries within the requested scope.",
3764
+ { type: "validation_error" }
3765
+ );
3766
+ }
3701
3767
  const userPrompt = buildUserPrompt3({
3702
3768
  timestampedTranscript,
3769
+ sceneContext,
3703
3770
  promptOverrides,
3704
3771
  minChaptersPerHour,
3705
3772
  maxChaptersPerHour,
@@ -3708,7 +3775,11 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
3708
3775
  });
3709
3776
  let chaptersData = null;
3710
3777
  try {
3711
- const systemPrompt = isAudioOnly ? chapterSystemPromptBuilder.build(AUDIO_ONLY_SYSTEM_PROMPT_OVERRIDES) : chapterSystemPromptBuilder.build();
3778
+ const systemPromptOverrides = {
3779
+ ...isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT_OVERRIDES : {},
3780
+ ...sceneContext?.length ? SCENE_CONTEXT_SYSTEM_PROMPT_OVERRIDES : {}
3781
+ };
3782
+ const systemPrompt = chapterSystemPromptBuilder.build(systemPromptOverrides);
3712
3783
  chaptersData = await generateChaptersWithAI({
3713
3784
  provider: modelConfig.provider,
3714
3785
  modelId: modelConfig.modelId,
@@ -3726,9 +3797,13 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
3726
3797
  throw new MuxAiError(`Failed to generate chapters for asset ${assetId}.`);
3727
3798
  }
3728
3799
  const { chapters: chaptersPayload, usage } = chaptersData;
3729
- const validChapters = chaptersPayload.chapters.filter((chapter) => typeof chapter.startTime === "number" && typeof chapter.title === "string").filter(
3800
+ const scopedChapters = chaptersPayload.chapters.filter((chapter) => typeof chapter.startTime === "number" && typeof chapter.title === "string").filter(
3730
3801
  (chapter) => resolvedScope ? chapter.startTime >= resolvedScope.startTime && chapter.startTime < resolvedScope.endTime : true
3731
3802
  ).sort((a, b) => a.startTime - b.startTime);
3803
+ const validChapters = alignChaptersToSceneBoundaries(
3804
+ scopedChapters,
3805
+ sceneBoundarySeconds
3806
+ );
3732
3807
  if (validChapters.length === 0) {
3733
3808
  throw new MuxAiError(`Failed to generate valid chapters for asset ${assetId}.`);
3734
3809
  }
@@ -3754,9 +3829,11 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
3754
3829
  if (scrubbedChapters.length === 0) {
3755
3830
  throw new MuxAiError(`Failed to generate valid chapters for asset ${assetId}.`);
3756
3831
  }
3757
- const firstChapterStartTime = resolvedScope?.startTime ?? 0;
3758
- if (scrubbedChapters[0].startTime !== firstChapterStartTime) {
3759
- scrubbedChapters[0].startTime = firstChapterStartTime;
3832
+ if (sceneBoundarySeconds.length === 0) {
3833
+ const firstChapterStartTime = resolvedScope?.startTime ?? 0;
3834
+ if (scrubbedChapters[0].startTime !== firstChapterStartTime) {
3835
+ scrubbedChapters[0].startTime = firstChapterStartTime;
3836
+ }
3760
3837
  }
3761
3838
  const usageWithMetadata = {
3762
3839
  ...usage,
@@ -3779,6 +3856,129 @@ import { generateText as generateText4, Output as Output4 } from "ai";
3779
3856
  import { z as z6 } from "zod";
3780
3857
 
3781
3858
  // src/lib/mux-tracks.ts
3859
+ var MUX_TRACK_PASSTHROUGH_MAX_CHARS = 255;
3860
+ function normalizeTrackLanguageCode(code) {
3861
+ if (!code) {
3862
+ return void 0;
3863
+ }
3864
+ const primarySubtag = code.trim().toLowerCase().split(/[-_]/)[0];
3865
+ if (!primarySubtag || primarySubtag === "auto" || isUndeterminedLanguageCode(primarySubtag)) {
3866
+ return void 0;
3867
+ }
3868
+ return toISO639_1(primarySubtag);
3869
+ }
3870
+ function normalizeTrackName(name) {
3871
+ const normalized = name?.trim().toLowerCase();
3872
+ return normalized || void 0;
3873
+ }
3874
+ function isMuxGeneratedTextTrack(track) {
3875
+ return track.text_source === "generated_vod" || track.text_source === "generated_live" || track.text_source === "generated_live_final";
3876
+ }
3877
+ function targetType(target) {
3878
+ return target.type ?? "text";
3879
+ }
3880
+ function listTracks(asset, type) {
3881
+ return (asset.tracks ?? []).filter((track) => track.type === type && track.status !== "deleted" && !!track.id);
3882
+ }
3883
+ function summarizeTextTrack(track) {
3884
+ return {
3885
+ id: track.id,
3886
+ type: track.type === "audio" ? "audio" : "text",
3887
+ name: track.name,
3888
+ languageCode: track.language_code,
3889
+ status: track.status,
3890
+ textSource: track.text_source,
3891
+ passthrough: track.passthrough,
3892
+ ...track.type === "audio" ? { primary: track.primary === true } : {}
3893
+ };
3894
+ }
3895
+ function findSameLanguageTextTracks(asset, target) {
3896
+ const targetLanguage = normalizeTrackLanguageCode(target.languageCode);
3897
+ if (!targetLanguage) {
3898
+ return [];
3899
+ }
3900
+ const type = targetType(target);
3901
+ return listTracks(asset, type).filter(
3902
+ (track) => (type === "audio" || track.text_type === "subtitles") && normalizeTrackLanguageCode(track.language_code) === targetLanguage
3903
+ );
3904
+ }
3905
+ function findNameCollisionTextTracks(asset, target) {
3906
+ const targetName = normalizeTrackName(target.name);
3907
+ if (!targetName) {
3908
+ return [];
3909
+ }
3910
+ return listTracks(asset, targetType(target)).filter((track) => normalizeTrackName(track.name) === targetName);
3911
+ }
3912
+ function describeTracks(tracks) {
3913
+ return tracks.map((track) => `${track.name ?? "(unnamed)"} [${track.language_code ?? "?"}, ${track.type === "audio" ? track.primary ? "primary audio" : "audio" : track.text_source ?? "unknown source"}, ${track.status ?? "ready"}]`).join("; ");
3914
+ }
3915
+ function planTextTrackReplacement(asset, target, policy, options = {}) {
3916
+ const label = targetType(target) === "audio" ? "Audio" : "Text";
3917
+ const keep = new Set(options.keepTrackIds ?? []);
3918
+ const conflicts = /* @__PURE__ */ new Map();
3919
+ for (const track of [...findSameLanguageTextTracks(asset, target), ...findNameCollisionTextTracks(asset, target)]) {
3920
+ if (!keep.has(track.id)) {
3921
+ conflicts.set(track.id, track);
3922
+ }
3923
+ }
3924
+ const conflicting = [...conflicts.values()];
3925
+ if (conflicting.length === 0) {
3926
+ return { kind: "clear" };
3927
+ }
3928
+ if (policy === "fail") {
3929
+ return {
3930
+ kind: "blocked",
3931
+ reason: `${label} track(s) already exist for language '${target.languageCode}' or name '${target.name}': ${describeTracks(conflicting)}. Set replaceExistingTracks to replace them.`,
3932
+ tracks: conflicting.map(summarizeTextTrack)
3933
+ };
3934
+ }
3935
+ const primary = conflicting.filter((track) => track.type === "audio" && track.primary === true);
3936
+ if (primary.length > 0) {
3937
+ return {
3938
+ kind: "blocked",
3939
+ reason: `The asset's primary audio track is in the way for language '${target.languageCode}' or name '${target.name}': ${describeTracks(primary)}. Primary audio cannot be replaced; choose a different trackName.`,
3940
+ tracks: primary.map(summarizeTextTrack)
3941
+ };
3942
+ }
3943
+ if (policy === "replace_generated") {
3944
+ const notGenerated = conflicting.filter((track) => track.type !== "text" || !isMuxGeneratedTextTrack(track));
3945
+ if (notGenerated.length > 0) {
3946
+ return {
3947
+ kind: "blocked",
3948
+ reason: `${label} track(s) that are not Mux-generated exist for language '${target.languageCode}' or name '${target.name}': ${describeTracks(notGenerated)}. Use replaceExistingTracks: "replace_all" to replace them.`,
3949
+ tracks: notGenerated.map(summarizeTextTrack)
3950
+ };
3951
+ }
3952
+ }
3953
+ return { kind: "replace", toDelete: conflicting.map(summarizeTextTrack) };
3954
+ }
3955
+ function isDuplicateTrackNameError(error) {
3956
+ if (typeof error !== "object" || error === null) {
3957
+ return false;
3958
+ }
3959
+ const record = error;
3960
+ if (record.status !== 400) {
3961
+ return false;
3962
+ }
3963
+ const messages = Array.isArray(record.error?.messages) ? record.error.messages : [];
3964
+ const haystack = [...messages, record.message].filter((value) => typeof value === "string").map((value) => value.toLowerCase());
3965
+ return haystack.some((value) => value.includes("is not unique"));
3966
+ }
3967
+ function isNotFoundError(error) {
3968
+ return typeof error === "object" && error !== null && error.status === 404;
3969
+ }
3970
+ function buildMuxAiTrackPassthrough(workflow) {
3971
+ return JSON.stringify({ mux_ai: { workflow } });
3972
+ }
3973
+ function validateTrackPassthrough(passthrough) {
3974
+ if (passthrough !== void 0 && passthrough.length > MUX_TRACK_PASSTHROUGH_MAX_CHARS) {
3975
+ throw new MuxAiError(
3976
+ `Track passthrough must be at most ${MUX_TRACK_PASSTHROUGH_MAX_CHARS} characters (received ${passthrough.length}).`,
3977
+ { type: "validation_error" }
3978
+ );
3979
+ }
3980
+ return passthrough;
3981
+ }
3782
3982
  async function fetchVttFromMux(vttUrl) {
3783
3983
  "use step";
3784
3984
  const vttResponse = await fetch(vttUrl);
@@ -3787,7 +3987,7 @@ async function fetchVttFromMux(vttUrl) {
3787
3987
  }
3788
3988
  return vttResponse.text();
3789
3989
  }
3790
- async function createTextTrackOnMux(assetId, languageCode, trackName, presignedUrl, credentials) {
3990
+ async function createTextTrackOnMux(assetId, languageCode, trackName, presignedUrl, credentials, options = {}) {
3791
3991
  "use step";
3792
3992
  const muxClient = await resolveMuxClient(credentials);
3793
3993
  const mux = await muxClient.createClient();
@@ -3796,13 +3996,92 @@ async function createTextTrackOnMux(assetId, languageCode, trackName, presignedU
3796
3996
  text_type: "subtitles",
3797
3997
  language_code: languageCode,
3798
3998
  name: trackName,
3799
- url: presignedUrl
3999
+ url: presignedUrl,
4000
+ closed_captions: options.closedCaptions,
4001
+ passthrough: options.passthrough
3800
4002
  });
3801
4003
  if (!trackResponse.id) {
3802
4004
  throw new Error("Failed to create text track: no track ID returned from Mux");
3803
4005
  }
3804
4006
  return trackResponse.id;
3805
4007
  }
4008
+ async function replaceAndCreateTrack(input) {
4009
+ "use step";
4010
+ const muxClient = await resolveMuxClient(input.credentials);
4011
+ const mux = await muxClient.createClient();
4012
+ const type = targetType(input.target);
4013
+ const deleted = [];
4014
+ const failed = (error) => ({
4015
+ kind: "create_failed",
4016
+ reason: error instanceof Error ? error.message : String(error),
4017
+ deleted
4018
+ });
4019
+ const clearConflicts = async () => {
4020
+ let asset;
4021
+ try {
4022
+ asset = await mux.video.assets.retrieve(input.assetId);
4023
+ } catch (error) {
4024
+ if (deleted.length === 0) {
4025
+ throw error;
4026
+ }
4027
+ return failed(error);
4028
+ }
4029
+ const plan = planTextTrackReplacement(asset, input.target, input.policy, { keepTrackIds: input.keepTrackIds });
4030
+ if (plan.kind === "blocked") {
4031
+ return { ...plan, deleted };
4032
+ }
4033
+ if (plan.kind === "replace") {
4034
+ for (const track of plan.toDelete) {
4035
+ try {
4036
+ await mux.video.assets.deleteTrack(input.assetId, track.id);
4037
+ deleted.push(track);
4038
+ } catch (error) {
4039
+ if (!isNotFoundError(error)) {
4040
+ return failed(error);
4041
+ }
4042
+ }
4043
+ }
4044
+ }
4045
+ return void 0;
4046
+ };
4047
+ const create = async () => {
4048
+ const track = await mux.video.assets.createTrack(input.assetId, {
4049
+ type,
4050
+ ...type === "text" ? { text_type: "subtitles", closed_captions: input.closedCaptions } : {},
4051
+ language_code: input.target.languageCode,
4052
+ name: input.target.name.trim(),
4053
+ url: input.presignedUrl,
4054
+ passthrough: input.passthrough
4055
+ });
4056
+ if (!track.id) {
4057
+ throw new Error(`Failed to create ${type} track: no track ID returned from Mux`);
4058
+ }
4059
+ return track.id;
4060
+ };
4061
+ const stopped = await clearConflicts();
4062
+ if (stopped) {
4063
+ return stopped;
4064
+ }
4065
+ for (let attempt = 0; attempt < 2; attempt++) {
4066
+ try {
4067
+ return { kind: "created", trackId: await create(), deleted };
4068
+ } catch (error) {
4069
+ if (isDuplicateTrackNameError(error) && attempt === 0) {
4070
+ const stoppedOnRetry = await clearConflicts();
4071
+ if (stoppedOnRetry) {
4072
+ return stoppedOnRetry;
4073
+ }
4074
+ continue;
4075
+ }
4076
+ return failed(error);
4077
+ }
4078
+ }
4079
+ return { kind: "create_failed", reason: "Mux rejected the track name as not unique after retrying.", deleted };
4080
+ }
4081
+ async function replaceAndCreateTextTrack(input) {
4082
+ "use step";
4083
+ return replaceAndCreateTrack({ ...input, target: { ...input.target, type: "text" } });
4084
+ }
3806
4085
 
3807
4086
  // src/lib/s3-sigv4.ts
3808
4087
  var AWS4_ALGORITHM = "AWS4-HMAC-SHA256";
@@ -4294,6 +4573,7 @@ async function uploadEditedVttToS3({
4294
4573
  editedVtt,
4295
4574
  assetId,
4296
4575
  trackId,
4576
+ variant = "edited",
4297
4577
  s3Endpoint,
4298
4578
  s3Region,
4299
4579
  s3Bucket,
@@ -4303,7 +4583,7 @@ async function uploadEditedVttToS3({
4303
4583
  "use step";
4304
4584
  const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
4305
4585
  const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
4306
- const vttKey = `edited/${assetId}/${trackId}-edited-${Date.now()}.vtt`;
4586
+ const vttKey = `edited/${assetId}/${trackId}-${variant}-${Date.now()}.vtt`;
4307
4587
  await putObjectWithStorageAdapter({
4308
4588
  accessKeyId: s3AccessKeyId,
4309
4589
  secretAccessKey: s3SecretAccessKey,
@@ -4353,6 +4633,9 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
4353
4633
  s3Region: providedS3Region,
4354
4634
  s3Bucket: providedS3Bucket,
4355
4635
  trackNameSuffix,
4636
+ replaceExistingTracks: replaceExistingTracksOption,
4637
+ trackName: providedTrackName,
4638
+ trackPassthrough: providedTrackPassthrough,
4356
4639
  storageAdapter,
4357
4640
  credentials
4358
4641
  } = options;
@@ -4365,6 +4648,21 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
4365
4648
  if (autoCensorOption && !provider) {
4366
4649
  throw new MuxAiError("provider is required when using autoCensorProfanity.", { type: "validation_error" });
4367
4650
  }
4651
+ const legacyTrackNaming = deleteOriginalTrack !== void 0 || trackNameSuffix !== void 0;
4652
+ if (legacyTrackNaming && (replaceExistingTracksOption !== void 0 || providedTrackName !== void 0)) {
4653
+ throw new MuxAiError(
4654
+ "deleteOriginalTrack and trackNameSuffix are deprecated and cannot be combined with replaceExistingTracks or trackName.",
4655
+ { type: "validation_error" }
4656
+ );
4657
+ }
4658
+ const replaceExistingTracks = replaceExistingTracksOption ?? "replace_all";
4659
+ if (!legacyTrackNaming && replaceExistingTracks === "fail" && !providedTrackName) {
4660
+ throw new MuxAiError(
4661
+ `trackName is required when replaceExistingTracks is "fail": the edited track cannot reuse the source track's name while the source is kept.`,
4662
+ { type: "validation_error" }
4663
+ );
4664
+ }
4665
+ const trackPassthrough = validateTrackPassthrough(providedTrackPassthrough) ?? buildMuxAiTrackPassthrough("edit-captions");
4368
4666
  const deleteOriginal = deleteOriginalTrack !== false;
4369
4667
  const uploadToMux = uploadToMuxOption !== false;
4370
4668
  const uploadToS3 = uploadToS3Option || uploadToMux;
@@ -4407,6 +4705,25 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
4407
4705
  { type: "validation_error" }
4408
4706
  );
4409
4707
  }
4708
+ const sourceLanguageCode = sourceTrack.language_code || "en";
4709
+ const sourceName = sourceTrack.name || "Subtitles";
4710
+ const outputTrack = {
4711
+ languageCode: sourceLanguageCode,
4712
+ name: providedTrackName ?? sourceName
4713
+ };
4714
+ const keepTrackIds = replaceExistingTracks === "fail" ? [trackId] : [];
4715
+ if (replaceExistingTracks === "fail" && normalizeTrackName(outputTrack.name) === normalizeTrackName(sourceName)) {
4716
+ throw new MuxAiError(
4717
+ `trackName "${outputTrack.name}" matches the source track's name. Mux track names must be unique; choose a different trackName or use replaceExistingTracks: "replace_all" to replace the source.`,
4718
+ { type: "validation_error" }
4719
+ );
4720
+ }
4721
+ if (uploadToMux && !legacyTrackNaming) {
4722
+ const plan = planTextTrackReplacement(assetData, outputTrack, replaceExistingTracks, { keepTrackIds });
4723
+ if (plan.kind === "blocked") {
4724
+ throw new MuxAiError(plan.reason, { type: "validation_error" });
4725
+ }
4726
+ }
4410
4727
  const vttUrl = await buildTranscriptUrl(playbackId, trackId, policy === "signed", credentials);
4411
4728
  let vttContent;
4412
4729
  try {
@@ -4484,32 +4801,32 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
4484
4801
  } : void 0;
4485
4802
  let presignedUrl;
4486
4803
  let uploadedTrackId;
4804
+ let replacedTracks;
4487
4805
  if (uploadToS3) {
4806
+ const s3Config = {
4807
+ assetId,
4808
+ trackId,
4809
+ s3Endpoint,
4810
+ s3Region,
4811
+ s3Bucket,
4812
+ storageAdapter,
4813
+ s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
4814
+ };
4488
4815
  try {
4489
- presignedUrl = await uploadEditedVttToS3({
4490
- editedVtt,
4491
- assetId,
4492
- trackId,
4493
- s3Endpoint,
4494
- s3Region,
4495
- s3Bucket,
4496
- storageAdapter,
4497
- s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
4498
- });
4816
+ presignedUrl = await uploadEditedVttToS3({ editedVtt, ...s3Config });
4499
4817
  } catch (error) {
4500
4818
  wrapError(error, "Failed to upload VTT to S3");
4501
4819
  }
4502
- if (uploadToMux) {
4820
+ if (uploadToMux && legacyTrackNaming) {
4503
4821
  try {
4504
- const languageCode = sourceTrack.language_code || "en";
4505
- const suffix = trackNameSuffix ?? "edited";
4506
- const trackName = `${sourceTrack.name || "Subtitles"} (${suffix})`;
4822
+ const trackName = `${sourceName} (${trackNameSuffix ?? "edited"})`;
4507
4823
  uploadedTrackId = await createTextTrackOnMux(
4508
4824
  assetId,
4509
- languageCode,
4825
+ sourceLanguageCode,
4510
4826
  trackName,
4511
4827
  presignedUrl,
4512
- credentials
4828
+ credentials,
4829
+ { closedCaptions: sourceTrack.closed_captions, passthrough: trackPassthrough }
4513
4830
  );
4514
4831
  } catch (error) {
4515
4832
  wrapError(error, "Failed to add edited track to Mux asset");
@@ -4521,6 +4838,48 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
4521
4838
  wrapError(error, "Failed to delete original track");
4522
4839
  }
4523
4840
  }
4841
+ } else if (uploadToMux) {
4842
+ let outcome;
4843
+ try {
4844
+ outcome = await replaceAndCreateTextTrack({
4845
+ assetId,
4846
+ target: outputTrack,
4847
+ policy: replaceExistingTracks,
4848
+ presignedUrl,
4849
+ closedCaptions: sourceTrack.closed_captions,
4850
+ passthrough: trackPassthrough,
4851
+ keepTrackIds,
4852
+ credentials
4853
+ });
4854
+ } catch (error) {
4855
+ wrapError(error, "Failed to add edited track to Mux asset");
4856
+ }
4857
+ if (outcome.kind !== "created") {
4858
+ const sourceWasDeleted = outcome.deleted.some((track) => track.id === trackId);
4859
+ let restoreNote = "";
4860
+ if (sourceWasDeleted) {
4861
+ try {
4862
+ const originalUrl = await uploadEditedVttToS3({ editedVtt: vttContent, variant: "original", ...s3Config });
4863
+ const restoredTrackId = await createTextTrackOnMux(
4864
+ assetId,
4865
+ sourceLanguageCode,
4866
+ sourceName,
4867
+ originalUrl,
4868
+ credentials,
4869
+ { closedCaptions: sourceTrack.closed_captions, passthrough: sourceTrack.passthrough }
4870
+ );
4871
+ restoreNote = ` The source track was restored as ${restoredTrackId}.`;
4872
+ } catch (restoreError) {
4873
+ restoreNote = ` Restoring the source track also failed: ${restoreError instanceof Error ? restoreError.message : String(restoreError)}.`;
4874
+ }
4875
+ }
4876
+ if (outcome.kind === "blocked") {
4877
+ throw new MuxAiError(`${outcome.reason}${restoreNote}`, { type: "validation_error" });
4878
+ }
4879
+ throw new Error(`Failed to add edited track to Mux asset: ${outcome.reason}.${restoreNote}`);
4880
+ }
4881
+ uploadedTrackId = outcome.trackId;
4882
+ replacedTracks = outcome.deleted;
4524
4883
  }
4525
4884
  }
4526
4885
  return {
@@ -4533,6 +4892,7 @@ async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
4533
4892
  replacements: replacementsResult,
4534
4893
  speakerReplacements: speakerReplacementsResult,
4535
4894
  uploadedTrackId,
4895
+ replacedTracks,
4536
4896
  presignedUrl,
4537
4897
  usage: usageWithMetadata,
4538
4898
  safety: safety.report()
@@ -6447,14 +6807,44 @@ function buildDescriptionGuidance(wordCount, contentType) {
6447
6807
  Write in present tense. Be specific about what is discussed or presented rather than making assumptions.
6448
6808
  Focus on the spoken content and any key insights, dialogue, or narrative elements.`;
6449
6809
  }
6450
- function createSummarizationBuilder({ titleLength, descriptionLength, tagCount } = {}) {
6810
+ function buildSceneAwareDescriptionGuidance(wordCount) {
6811
+ if (wordCount < DESCRIPTION_LENGTH_THRESHOLD_SMALL) {
6812
+ return dedent6`A brief summary of the video in no more than ${wordCount} words. Shorter is fine.
6813
+ Focus on the single most important subject or action across the ordered scenes.
6814
+ Use the storyboard as direct visual evidence and the scene context to preserve progression.
6815
+ Write in present tense.`;
6816
+ }
6817
+ if (wordCount > DESCRIPTION_LENGTH_THRESHOLD_LARGE) {
6818
+ return dedent6`A detailed summary that describes what happens across the ordered scenes.
6819
+ Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
6820
+ Be thorough: cover subjects, actions, setting, progression, and notable details supported across scenes.
6821
+ Use the storyboard as direct visual evidence and the scene context to preserve progression.
6822
+ Write in present tense. If the transcript provides dialogue or narration, incorporate key points while preserving scene progression.`;
6823
+ }
6824
+ return dedent6`A summary that describes what happens across the ordered scenes.
6825
+ Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
6826
+ Cover the main subjects, actions, setting, and notable progression supported across scenes.
6827
+ Use the storyboard as direct visual evidence and the scene context to preserve progression.
6828
+ Write in present tense. If the transcript provides dialogue or narration, incorporate key points while preserving scene progression.`;
6829
+ }
6830
+ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount } = {}, { hasSceneContext = false } = {}) {
6451
6831
  const titleLimit = titleLength ?? DEFAULT_TITLE_LENGTH;
6452
6832
  const keywordLimit = tagCount ?? DEFAULT_SUMMARY_KEYWORD_LIMIT;
6833
+ const task = hasSceneContext ? "Analyze the storyboard frames and ordered scene context together, then generate metadata that captures the full progression of the video content." : "Analyze the storyboard frames and generate metadata that captures the essence of the video content.";
6834
+ const qualityGuidelines = hasSceneContext ? dedent6`
6835
+ - Follow scene_index order to understand what begins, develops, and concludes
6836
+ - Use scene narratives and concepts as grounded evidence, preserving consistent terminology across the output
6837
+ - Use the storyboard as direct visual evidence for whole-asset details
6838
+ - Balance brevity with informativeness` : dedent6`
6839
+ - Examine all frames to understand the full context and progression
6840
+ - Be precise: "golden retriever" is better than "dog" when identifiable
6841
+ - Capture the narrative: what begins, develops, and concludes
6842
+ - Balance brevity with informativeness`;
6453
6843
  return createPromptBuilder({
6454
6844
  template: {
6455
6845
  task: {
6456
6846
  tag: "task",
6457
- content: "Analyze the storyboard frames and generate metadata that captures the essence of the video content."
6847
+ content: task
6458
6848
  },
6459
6849
  title: {
6460
6850
  tag: "title_requirements",
@@ -6467,7 +6857,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
6467
6857
  },
6468
6858
  description: {
6469
6859
  tag: "description_requirements",
6470
- content: buildDescriptionGuidance(descriptionLength ?? DEFAULT_DESCRIPTION_LENGTH, "video")
6860
+ content: hasSceneContext ? buildSceneAwareDescriptionGuidance(descriptionLength ?? DEFAULT_DESCRIPTION_LENGTH) : buildDescriptionGuidance(descriptionLength ?? DEFAULT_DESCRIPTION_LENGTH, "video")
6471
6861
  },
6472
6862
  keywords: {
6473
6863
  tag: "keywords_requirements",
@@ -6483,11 +6873,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
6483
6873
  },
6484
6874
  qualityGuidelines: {
6485
6875
  tag: "quality_guidelines",
6486
- content: dedent6`
6487
- - Examine all frames to understand the full context and progression
6488
- - Be precise: "golden retriever" is better than "dog" when identifiable
6489
- - Capture the narrative: what begins, develops, and concludes
6490
- - Balance brevity with informativeness`
6876
+ content: qualityGuidelines
6491
6877
  }
6492
6878
  },
6493
6879
  sectionOrder: ["task", "title", "description", "keywords", "qualityGuidelines"]
@@ -6539,59 +6925,83 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
6539
6925
  sectionOrder: ["task", "title", "description", "keywords", "qualityGuidelines"]
6540
6926
  });
6541
6927
  }
6542
- var SYSTEM_PROMPT5 = promptDedent`
6543
- <role>
6544
- You are a video content analyst specializing in storyboard interpretation and multimodal analysis.
6545
- </role>
6546
-
6547
- <context>
6548
- You receive storyboard images containing multiple sequential frames extracted from a video.
6549
- ${STORYBOARD_FRAME_INSTRUCTIONS}
6550
- </context>
6551
-
6552
- <transcript_guidance>
6553
- When a transcript is provided alongside the storyboard:
6554
- - Use it to understand spoken content, dialogue, narration, and audio context
6555
- - Correlate transcript content with visual frames to build a complete picture
6556
- - Extract key terminology, names, and specific language used by speakers
6557
- - Let the transcript inform keyword selection, especially for topics not visually obvious
6558
- - Prioritize visual content for the description, but enrich it with transcript insights
6559
- - If transcript and visuals conflict, trust the visual evidence
6560
- </transcript_guidance>
6561
-
6562
- <capabilities>
6563
- - Extract meaning from visual sequences
6564
- - Identify subjects, actions, settings, and narrative arcs
6565
- - Generate accurate, searchable metadata
6566
- - Synthesize visual and transcript information when provided
6567
- </capabilities>
6568
-
6569
- <security>
6570
- ${NON_DISCLOSURE_CONSTRAINT}
6571
-
6572
- ${UNTRUSTED_USER_INPUT_NOTICE}
6573
-
6574
- ${VISUAL_TEXT_AS_CONTENT}
6575
-
6576
- ${CANARY_TRIPWIRE}
6577
- </security>
6578
-
6579
- <constraints>
6580
- - Only describe what is clearly observable in the frames or explicitly stated in the transcript
6581
- - ${NO_FABRICATION_CONSTRAINT}
6582
- - ${METADATA_BOUNDARY_WARNING}
6583
- - ${STRUCTURED_DATA_CONSTRAINT}
6584
- - Output only the JSON object; no markdown or extra text
6585
- - When a <language> section is provided, all output text MUST be written in that language
6586
- </constraints>
6587
-
6588
- <tone_guidance>
6589
- ${TONE_GUIDANCE}
6590
- </tone_guidance>
6591
-
6592
- <language_guidelines>
6593
- ${createLanguageGuidelines("video")}
6594
- </language_guidelines>`;
6928
+ function createVideoSystemPrompt(hasSceneContext) {
6929
+ let sceneContext = "";
6930
+ let sceneGuidance = "";
6931
+ if (hasSceneContext) {
6932
+ sceneContext = promptDedent`
6933
+ Ordered scene context is also provided as structured evidence of the video's progression.
6934
+ Entries use stable, zero-based scene_index values and absolute millisecond timestamps.
6935
+ Read scenes in scene_index order to understand what begins, develops, and concludes.`;
6936
+ sceneGuidance = promptDedent`
6937
+ <scene_guidance>
6938
+ - Use scene titles, narratives, and concepts to understand temporal progression
6939
+ - Use storyboard frames as direct visual evidence for subjects, actions, settings, and details
6940
+ - Preserve consistent terminology from the scene context across the title, description, and keywords
6941
+ - Treat optional scene fields as evidence only when present; do not infer omitted details
6942
+ - Consider all provided evidence together; if derived scene context conflicts with storyboard visuals, trust the visuals
6943
+ </scene_guidance>`;
6944
+ }
6945
+ const transcriptEvidence = hasSceneContext ? "storyboard frames and ordered scenes" : "visual frames";
6946
+ const descriptionPriority = hasSceneContext ? "Use the scene context for temporal structure and the storyboard for direct visual details, enriching both with transcript insights" : "Prioritize visual content for the description, but enrich it with transcript insights";
6947
+ const groundingConstraint = hasSceneContext ? "Use all provided evidence together. Ground each claim in the storyboard, scene context, or transcript" : "Only describe what is clearly observable in the frames or explicitly stated in the transcript";
6948
+ return promptDedent`
6949
+ <role>
6950
+ You are a video content analyst specializing in storyboard interpretation and multimodal analysis.
6951
+ </role>
6952
+
6953
+ <context>
6954
+ You receive storyboard images containing multiple sequential frames extracted from a video.
6955
+ ${STORYBOARD_FRAME_INSTRUCTIONS}
6956
+ ${sceneContext}
6957
+ </context>
6958
+
6959
+ ${sceneGuidance}
6960
+
6961
+ <transcript_guidance>
6962
+ When a transcript is provided alongside the storyboard:
6963
+ - Use it to understand spoken content, dialogue, narration, and audio context
6964
+ - Correlate transcript content with ${transcriptEvidence} to build a complete picture
6965
+ - Extract key terminology, names, and specific language used by speakers
6966
+ - Let the transcript inform keyword selection, especially for topics not visually obvious
6967
+ - ${descriptionPriority}
6968
+ - If transcript and visuals conflict, trust the visual evidence
6969
+ </transcript_guidance>
6970
+
6971
+ <capabilities>
6972
+ - Extract meaning from visual sequences
6973
+ - Identify subjects, actions, settings, and narrative arcs
6974
+ - Generate accurate, searchable metadata
6975
+ - Synthesize visual and transcript information when provided
6976
+ </capabilities>
6977
+
6978
+ <security>
6979
+ ${NON_DISCLOSURE_CONSTRAINT}
6980
+
6981
+ ${UNTRUSTED_USER_INPUT_NOTICE}
6982
+
6983
+ ${VISUAL_TEXT_AS_CONTENT}
6984
+
6985
+ ${CANARY_TRIPWIRE}
6986
+ </security>
6987
+
6988
+ <constraints>
6989
+ - ${groundingConstraint}
6990
+ - ${NO_FABRICATION_CONSTRAINT}
6991
+ - ${METADATA_BOUNDARY_WARNING}
6992
+ - ${STRUCTURED_DATA_CONSTRAINT}
6993
+ - Output only the JSON object; no markdown or extra text
6994
+ - When a <language> section is provided, all output text MUST be written in that language
6995
+ </constraints>
6996
+
6997
+ <tone_guidance>
6998
+ ${TONE_GUIDANCE}
6999
+ </tone_guidance>
7000
+
7001
+ <language_guidelines>
7002
+ ${createLanguageGuidelines("video")}
7003
+ </language_guidelines>`;
7004
+ }
6595
7005
  var AUDIO_ONLY_SYSTEM_PROMPT3 = promptDedent`
6596
7006
  <role>
6597
7007
  You are an audio content analyst specializing in transcript analysis and metadata generation.
@@ -6642,6 +7052,16 @@ var AUDIO_ONLY_SYSTEM_PROMPT3 = promptDedent`
6642
7052
  <language_guidelines>
6643
7053
  ${createLanguageGuidelines("audio")}
6644
7054
  </language_guidelines>`;
7055
+ function createSceneContextSection(sceneContext) {
7056
+ return {
7057
+ tag: "scene_context",
7058
+ content: JSON.stringify(sceneContext),
7059
+ attributes: {
7060
+ format: "json",
7061
+ order: "scene_index_ascending"
7062
+ }
7063
+ };
7064
+ }
6645
7065
  function buildUserPrompt5({
6646
7066
  tone,
6647
7067
  transcriptText,
@@ -6651,7 +7071,8 @@ function buildUserPrompt5({
6651
7071
  titleLength,
6652
7072
  descriptionLength,
6653
7073
  tagCount,
6654
- languageName
7074
+ languageName,
7075
+ sceneContext
6655
7076
  }) {
6656
7077
  const contextSections = [createToneSection(TONE_INSTRUCTIONS[tone])];
6657
7078
  if (languageName) {
@@ -6666,11 +7087,15 @@ function buildUserPrompt5({
6666
7087
  const format = isCleanTranscript ? "plain text" : "WebVTT";
6667
7088
  contextSections.push(createTranscriptSection(transcriptText, format));
6668
7089
  }
7090
+ const hasSceneContext = !isAudioOnly && Boolean(sceneContext?.length);
7091
+ if (hasSceneContext && sceneContext) {
7092
+ contextSections.push(createSceneContextSection(sceneContext));
7093
+ }
6669
7094
  const constraints = { titleLength, descriptionLength, tagCount };
6670
- const promptBuilder = isAudioOnly ? createAudioOnlyBuilder(constraints) : createSummarizationBuilder(constraints);
7095
+ const promptBuilder = isAudioOnly ? createAudioOnlyBuilder(constraints) : createSummarizationBuilder(constraints, { hasSceneContext });
6671
7096
  return promptBuilder.buildWithContext(promptOverrides, contextSections);
6672
7097
  }
6673
- async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, systemPrompt, descriptionLength, credentials) {
7098
+ async function analyzeVideo(imageDataUrl, provider, modelId, userPrompt, systemPrompt, descriptionLength, credentials) {
6674
7099
  "use step";
6675
7100
  const model = await createLanguageModelFromConfig(provider, modelId, credentials);
6676
7101
  const schema = buildSummarySchema(descriptionLength);
@@ -6686,10 +7111,10 @@ async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, s
6686
7111
  messages: [
6687
7112
  {
6688
7113
  role: "user",
6689
- content: [
7114
+ content: imageDataUrl ? [
6690
7115
  { type: "text", text: userPrompt },
6691
7116
  { type: "image", image: imageDataUrl }
6692
- ]
7117
+ ] : userPrompt
6693
7118
  }
6694
7119
  ]
6695
7120
  }));
@@ -6799,6 +7224,7 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
6799
7224
  cleanTranscript = true,
6800
7225
  imageSubmissionMode = "url",
6801
7226
  imageDownloadOptions,
7227
+ sceneContext,
6802
7228
  promptOverrides,
6803
7229
  credentials,
6804
7230
  titleLength,
@@ -6822,6 +7248,7 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
6822
7248
  const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, workflowCredentials, options?.assetSnapshot);
6823
7249
  const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
6824
7250
  const isAudioOnly = isAudioOnlyAsset(assetData);
7251
+ const hasSceneContext = !isAudioOnly && Boolean(sceneContext?.length);
6825
7252
  const effectiveScope = hasWorkflowScopeBoundaries(scope) ? scope : void 0;
6826
7253
  const storyboardScope = isAudioOnly ? void 0 : resolveRenderableVideoScope(
6827
7254
  effectiveScope,
@@ -6838,7 +7265,8 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
6838
7265
  );
6839
7266
  }
6840
7267
  const signingContext = await resolveMuxSigningContext(workflowCredentials);
6841
- if (policy === "signed" && !signingContext) {
7268
+ const requiresSignedPlaybackAccess = includeTranscript || !isAudioOnly;
7269
+ if (policy === "signed" && requiresSignedPlaybackAccess && !signingContext) {
6842
7270
  throw new MuxAiError(
6843
7271
  "Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
6844
7272
  { type: "validation_error" }
@@ -6864,11 +7292,15 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
6864
7292
  titleLength,
6865
7293
  descriptionLength,
6866
7294
  tagCount,
6867
- languageName
7295
+ languageName,
7296
+ sceneContext: hasSceneContext ? sceneContext : void 0
6868
7297
  });
6869
7298
  let analysisResponse;
6870
7299
  let imageUrl;
6871
- const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT3 : SYSTEM_PROMPT5;
7300
+ let systemPrompt = AUDIO_ONLY_SYSTEM_PROMPT3;
7301
+ if (!isAudioOnly) {
7302
+ systemPrompt = createVideoSystemPrompt(hasSceneContext);
7303
+ }
6872
7304
  const effectiveDescriptionLength = descriptionLength ?? DEFAULT_DESCRIPTION_LENGTH;
6873
7305
  try {
6874
7306
  if (isAudioOnly) {
@@ -6891,7 +7323,7 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
6891
7323
  imageUrl = storyboardUrl;
6892
7324
  if (imageSubmissionMode === "base64") {
6893
7325
  const downloadResult = await downloadImageAsBase64(storyboardUrl, imageDownloadOptions);
6894
- analysisResponse = await analyzeStoryboard2(
7326
+ analysisResponse = await analyzeVideo(
6895
7327
  downloadResult.base64Data,
6896
7328
  modelConfig.provider,
6897
7329
  modelConfig.modelId,
@@ -6901,7 +7333,7 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
6901
7333
  workflowCredentials
6902
7334
  );
6903
7335
  } else {
6904
- analysisResponse = await analyzeStoryboard2(
7336
+ analysisResponse = await analyzeVideo(
6905
7337
  storyboardUrl,
6906
7338
  modelConfig.provider,
6907
7339
  modelConfig.modelId,
@@ -7220,23 +7652,6 @@ async function downloadAndUploadDubTranscript({
7220
7652
  console.warn(`\u2705 Dub transcript uploaded successfully to: ${vttKey}`);
7221
7653
  return presignedUrl;
7222
7654
  }
7223
- async function createAudioTrackOnMux(assetId, languageCode, presignedUrl, credentials) {
7224
- "use step";
7225
- const muxClient = await resolveMuxClient(credentials);
7226
- const mux = await muxClient.createClient();
7227
- const languageName = new Intl.DisplayNames(["en"], { type: "language" }).of(languageCode) || languageCode.toUpperCase();
7228
- const trackName = `${languageName} (auto-dubbed)`;
7229
- const trackResponse = await mux.video.assets.createTrack(assetId, {
7230
- type: "audio",
7231
- language_code: languageCode,
7232
- name: trackName,
7233
- url: presignedUrl
7234
- });
7235
- if (!trackResponse.id) {
7236
- throw new Error("Failed to create audio track: no track ID returned from Mux");
7237
- }
7238
- return trackResponse.id;
7239
- }
7240
7655
  async function translateAudio(assetId, toLanguageCode, options = {}) {
7241
7656
  "use workflow";
7242
7657
  const {
@@ -7249,15 +7664,23 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
7249
7664
  uploadCaptionsToMux = false,
7250
7665
  staticRenditionCleanup: staticRenditionCleanupOption = "delete",
7251
7666
  storageAdapter,
7252
- credentials: providedCredentials
7667
+ credentials: providedCredentials,
7668
+ replaceExistingTracks = "fail",
7669
+ trackName: providedTrackName,
7670
+ trackPassthrough: providedTrackPassthrough
7253
7671
  } = options;
7254
7672
  if (provider !== "elevenlabs") {
7255
7673
  throw new MuxAiError("Only ElevenLabs provider is currently supported for audio translation.", { type: "validation_error" });
7256
7674
  }
7257
7675
  const credentials = providedCredentials;
7258
7676
  const effectiveStorageAdapter = storageAdapter;
7677
+ const trackPassthrough = validateTrackPassthrough(providedTrackPassthrough) ?? buildMuxAiTrackPassthrough("translate-audio");
7259
7678
  const uploadToMux = uploadToMuxOption !== false;
7260
7679
  const uploadToS3 = uploadToS3Option || uploadToMux || uploadCaptionsToMux;
7680
+ const muxLangCode = toISO639_1(toLanguageCode);
7681
+ const trackName = providedTrackName ?? `${getLanguageName(muxLangCode) ?? muxLangCode.toUpperCase()} (Auto-dubbed)`;
7682
+ const audioTrackTarget = { type: "audio", languageCode: muxLangCode, name: trackName };
7683
+ const captionsTrackTarget = { type: "text", languageCode: muxLangCode, name: trackName };
7261
7684
  const s3Endpoint = options.s3Endpoint ?? env_default.S3_ENDPOINT;
7262
7685
  const s3Region = options.s3Region ?? env_default.S3_REGION ?? "auto";
7263
7686
  const s3Bucket = options.s3Bucket ?? env_default.S3_BUCKET;
@@ -7268,6 +7691,15 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
7268
7691
  }
7269
7692
  const { asset: initialAsset, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials, options.assetSnapshot);
7270
7693
  const assetDurationSeconds = getAssetDurationSecondsFromAsset(initialAsset);
7694
+ for (const target of [
7695
+ ...uploadToMux ? [audioTrackTarget] : [],
7696
+ ...uploadCaptionsToMux ? [captionsTrackTarget] : []
7697
+ ]) {
7698
+ const plan = planTextTrackReplacement(initialAsset, target, replaceExistingTracks);
7699
+ if (plan.kind === "blocked") {
7700
+ throw new MuxAiError(plan.reason, { type: "validation_error" });
7701
+ }
7702
+ }
7271
7703
  let currentAsset = initialAsset;
7272
7704
  let createdStaticRenditionId;
7273
7705
  if (!hasReadyAudioStaticRendition(currentAsset)) {
@@ -7283,6 +7715,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
7283
7715
  let dubbingId;
7284
7716
  let presignedUrl;
7285
7717
  let uploadedTrackId;
7718
+ let replacedTracks;
7286
7719
  let captionsPresignedUrl;
7287
7720
  let captionsTrackId;
7288
7721
  try {
@@ -7373,18 +7806,29 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
7373
7806
  }
7374
7807
  if (uploadToMux) {
7375
7808
  console.warn("\u{1F4F9} Adding dubbed audio track to Mux asset...");
7376
- const muxLangCode = toISO639_1(toLanguageCode);
7809
+ let outcome;
7377
7810
  try {
7378
- uploadedTrackId = await createAudioTrackOnMux(assetId, muxLangCode, presignedUrl, credentials);
7379
- const languageName = new Intl.DisplayNames(["en"], { type: "language" }).of(muxLangCode) || muxLangCode.toUpperCase();
7380
- const trackName = `${languageName} (auto-dubbed)`;
7381
- console.warn(`\u2705 Track added to Mux asset with ID: ${uploadedTrackId}`);
7382
- console.warn(`\u{1F4CB} Track name: "${trackName}"`);
7811
+ outcome = await replaceAndCreateTrack({
7812
+ assetId,
7813
+ target: audioTrackTarget,
7814
+ policy: replaceExistingTracks,
7815
+ presignedUrl,
7816
+ passthrough: trackPassthrough,
7817
+ credentials
7818
+ });
7383
7819
  } catch (error) {
7384
- console.warn(`\u26A0\uFE0F Failed to add audio track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
7385
- console.warn("\u{1F517} You can manually add the track using this presigned URL:");
7386
- console.warn(presignedUrl);
7820
+ wrapError(error, `Failed to add dubbed audio track to Mux asset (dubbed audio is staged at: ${presignedUrl})`);
7821
+ }
7822
+ if (outcome.kind === "blocked") {
7823
+ throw new MuxAiError(`${outcome.reason} Dubbed audio is staged at: ${presignedUrl}`, { type: "validation_error" });
7387
7824
  }
7825
+ if (outcome.kind === "create_failed") {
7826
+ throw new Error(`Failed to add dubbed audio track to Mux asset: ${outcome.reason}. Dubbed audio is staged at: ${presignedUrl}`);
7827
+ }
7828
+ uploadedTrackId = outcome.trackId;
7829
+ replacedTracks = outcome.deleted;
7830
+ console.warn(`\u2705 Track added to Mux asset with ID: ${uploadedTrackId}`);
7831
+ console.warn(`\u{1F4CB} Track name: "${trackName}"`);
7388
7832
  }
7389
7833
  try {
7390
7834
  console.warn("\u{1F4E5} Downloading dub transcript from ElevenLabs and staging to S3...");
@@ -7402,15 +7846,21 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
7402
7846
  });
7403
7847
  if (uploadCaptionsToMux) {
7404
7848
  console.warn("\u{1F4F9} Adding dubbed captions track to Mux asset...");
7405
- const muxLangCode = toISO639_1(toLanguageCode);
7406
- const languageName = new Intl.DisplayNames(["en"], { type: "language" }).of(muxLangCode) || muxLangCode.toUpperCase();
7407
- captionsTrackId = await createTextTrackOnMux(
7849
+ const captionsOutcome = await replaceAndCreateTrack({
7408
7850
  assetId,
7409
- muxLangCode,
7410
- `${languageName} (auto-dubbed)`,
7411
- captionsPresignedUrl,
7851
+ target: captionsTrackTarget,
7852
+ policy: replaceExistingTracks,
7853
+ presignedUrl: captionsPresignedUrl,
7854
+ passthrough: trackPassthrough,
7412
7855
  credentials
7413
- );
7856
+ });
7857
+ if (captionsOutcome.deleted.length > 0) {
7858
+ replacedTracks = [...replacedTracks ?? [], ...captionsOutcome.deleted];
7859
+ }
7860
+ if (captionsOutcome.kind !== "created") {
7861
+ throw new Error(captionsOutcome.reason);
7862
+ }
7863
+ captionsTrackId = captionsOutcome.trackId;
7414
7864
  console.warn(`\u2705 Captions track added to Mux asset with ID: ${captionsTrackId}`);
7415
7865
  }
7416
7866
  } catch (error) {
@@ -7441,6 +7891,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
7441
7891
  dubbingId,
7442
7892
  uploadedTrackId,
7443
7893
  presignedUrl,
7894
+ replacedTracks,
7444
7895
  captionsTrackId,
7445
7896
  captionsPresignedUrl,
7446
7897
  createdStaticRenditionId,
@@ -7467,7 +7918,7 @@ import { z as z9 } from "zod";
7467
7918
  var translationSchema = z9.object({
7468
7919
  translation: z9.string()
7469
7920
  });
7470
- var SYSTEM_PROMPT6 = promptDedent`
7921
+ var SYSTEM_PROMPT5 = promptDedent`
7471
7922
  You are a subtitle translation expert. Translate VTT subtitle files to the target language specified by the user.
7472
7923
  You may receive either a full VTT file or a chunk from a larger VTT.
7473
7924
  Preserve all timestamps, cue ordering, and VTT formatting exactly as they appear.
@@ -7792,7 +8243,7 @@ async function translateVttWithAI({
7792
8243
  model,
7793
8244
  maxRetries: 0,
7794
8245
  output: Output7.object({ schema: translationSchema }),
7795
- system: SYSTEM_PROMPT6,
8246
+ system: SYSTEM_PROMPT5,
7796
8247
  messages: [
7797
8248
  {
7798
8249
  role: "user",
@@ -8110,11 +8561,15 @@ async function translateCaptionsInternal(assetId, trackId, toLanguageCode, optio
8110
8561
  storageAdapter,
8111
8562
  credentials: providedCredentials,
8112
8563
  chunking,
8113
- neverTranslate: neverTranslateOption
8564
+ neverTranslate: neverTranslateOption,
8565
+ replaceExistingTracks = "fail",
8566
+ trackName: providedTrackName,
8567
+ trackPassthrough: providedTrackPassthrough
8114
8568
  } = options;
8115
8569
  const credentials = providedCredentials;
8116
8570
  const effectiveStorageAdapter = storageAdapter;
8117
8571
  const neverTranslateTerms = neverTranslateOption ? validateNeverTranslateTerms(neverTranslateOption) : [];
8572
+ const trackPassthrough = validateTrackPassthrough(providedTrackPassthrough) ?? buildMuxAiTrackPassthrough("translate-captions");
8118
8573
  const s3Endpoint = providedS3Endpoint ?? env_default.S3_ENDPOINT;
8119
8574
  const s3Region = providedS3Region ?? env_default.S3_REGION ?? "auto";
8120
8575
  const s3Bucket = providedS3Bucket ?? env_default.S3_BUCKET;
@@ -8155,6 +8610,16 @@ async function translateCaptionsInternal(assetId, trackId, toLanguageCode, optio
8155
8610
  { type: "validation_error" }
8156
8611
  );
8157
8612
  }
8613
+ const outputTrack = {
8614
+ languageCode: toLanguageCode,
8615
+ name: providedTrackName ?? `${getLanguageName(toLanguageCode) ?? toLanguageCode.toUpperCase()} (Auto-translated)`
8616
+ };
8617
+ if (uploadToMux) {
8618
+ const plan = planTextTrackReplacement(assetData, outputTrack, replaceExistingTracks);
8619
+ if (plan.kind === "blocked") {
8620
+ throw new MuxAiError(plan.reason, { type: "validation_error" });
8621
+ }
8622
+ }
8158
8623
  const vttUrl = await buildTranscriptUrl(playbackId, trackId, policy === "signed", credentials);
8159
8624
  let vttContent;
8160
8625
  try {
@@ -8222,6 +8687,7 @@ async function translateCaptionsInternal(assetId, trackId, toLanguageCode, optio
8222
8687
  const targetLanguage = getLanguageCodePair(toLanguageCode);
8223
8688
  let presignedUrl;
8224
8689
  let uploadedTrackId;
8690
+ let replacedTracks;
8225
8691
  if (uploadToS3) {
8226
8692
  try {
8227
8693
  presignedUrl = await uploadVttToS3({
@@ -8239,19 +8705,28 @@ async function translateCaptionsInternal(assetId, trackId, toLanguageCode, optio
8239
8705
  wrapError(error, "Failed to upload VTT to S3");
8240
8706
  }
8241
8707
  if (uploadToMux) {
8708
+ let outcome;
8242
8709
  try {
8243
- const languageName = getLanguageName(toLanguageCode) ?? toLanguageCode.toUpperCase();
8244
- const trackName = `${languageName} (auto-translated)`;
8245
- uploadedTrackId = await createTextTrackOnMux(
8710
+ outcome = await replaceAndCreateTextTrack({
8246
8711
  assetId,
8247
- toLanguageCode,
8248
- trackName,
8712
+ target: outputTrack,
8713
+ policy: replaceExistingTracks,
8249
8714
  presignedUrl,
8715
+ passthrough: trackPassthrough,
8250
8716
  credentials
8251
- );
8717
+ });
8252
8718
  } catch (error) {
8253
- console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
8719
+ wrapError(error, "Failed to add translated track to Mux asset");
8720
+ }
8721
+ if (outcome.kind !== "created") {
8722
+ const deletedNote = outcome.deleted.length > 0 ? ` Tracks already deleted before the failure: ${outcome.deleted.map((track) => `${track.name ?? track.id} (${track.id})`).join(", ")}.` : "";
8723
+ if (outcome.kind === "blocked") {
8724
+ throw new MuxAiError(`${outcome.reason}${deletedNote}`, { type: "validation_error" });
8725
+ }
8726
+ throw new Error(`Failed to add translated track to Mux asset: ${outcome.reason}.${deletedNote}`);
8254
8727
  }
8728
+ uploadedTrackId = outcome.trackId;
8729
+ replacedTracks = outcome.deleted;
8255
8730
  }
8256
8731
  }
8257
8732
  return {
@@ -8264,6 +8739,7 @@ async function translateCaptionsInternal(assetId, trackId, toLanguageCode, optio
8264
8739
  originalVtt: vttContent,
8265
8740
  translatedVtt,
8266
8741
  uploadedTrackId,
8742
+ replacedTracks,
8267
8743
  presignedUrl,
8268
8744
  usage: usageWithMetadata,
8269
8745
  safety,