@mux/ai 0.30.0 → 0.31.0

package/dist/workflows/index.d.ts

@@ -1,4 +1,4 @@
-export { A as ASK_QUESTIONS_DEFAULT_MAX_ANSWER_OPTION_LENGTH, j as ASK_QUESTIONS_DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH, k as ASK_QUESTIONS_MAX_QUESTION_LENGTH, l as AskQuestionsOptions, m as AskQuestionsResult, n as AskQuestionsType, o as AudioTranslationOptions, p as AudioTranslationResult, q as AutoCensorProfanityOptions, B as BurnedInCaptionsAnalysis, r as BurnedInCaptionsOptions, s as BurnedInCaptionsPromptOverrides, u as BurnedInCaptionsPromptSections, v as BurnedInCaptionsResult, C as CaptionReplacement, w as CensorMode, x as Chapter, y as ChapterSystemPromptSections, z as ChaptersOptions, D as ChaptersPromptOverrides, E as ChaptersPromptSections, F as ChaptersResult, G as ChaptersType, H as DEFAULT_DESCRIPTION_LENGTH, J as DEFAULT_SUMMARY_KEYWORD_LIMIT, K as DEFAULT_TITLE_LENGTH, M as EditCaptionsOptions, N as EditCaptionsResult, O as EmbeddingsOptions, P as EmbeddingsResult, Q as EngagementInsightsOptions, R as EngagementInsightsResult, T as GOOGLE_VISION_LIKELIHOOD_TO_SCORE, U as HIVE_SEXUAL_CATEGORIES, V as HIVE_VIOLENCE_CATEGORIES, W as HiveModerationOutput, X as HiveModerationSource, Y as ModerationOptions, Z as ModerationProvider, _ as ModerationResult, $ as MomentInsight, a0 as OverallInsight, a1 as ProfanityDetectionPayload, a2 as Question, a3 as QuestionAnswer, a4 as QuestionAnswerType, a5 as ReplacementRecord, a6 as SummarizationOptions, a7 as SummarizationPromptOverrides, a8 as SummarizationPromptSections, a9 as SummaryAndTagsResult, aa as SummaryType, ab as ThumbnailModerationScore, ac as TranslationChunkingOptions, ad as TranslationOptions, ae as TranslationPayload, af as TranslationResult, ag as aggregateTokenUsage, ah as applyOverrideLists, ai as applyReplacements, aj as askQuestions, ak as buildReplacementRegex, al as burnedInCaptionsSchema, am as censorVttContent, an as chapterSchema, ao as chaptersSchema, ap as createReplacer, aq as editCaptions, ar as extractTranscriptSegmentsForHotspots, as as generateChapters, at as generateEmbeddings, au as generateEngagementInsights, av as generateVideoEmbeddings, aw as getModerationScores, ax as getSummaryAndTags, ay as hasBurnedInCaptions, az as mapShotsToHotspots, aA as normalizeTranslatedVtt, aB as profanityDetectionSchema, aC as questionAnswerSchema, aD as shouldSplitChunkTranslationError, aE as summarySchema, aF as transformCueText, aG as translateAudio, aH as translateCaptions, aI as translationSchema } from '../index-D3cB2ltG.js';
+export { A as ASK_QUESTIONS_DEFAULT_MAX_ANSWER_OPTION_LENGTH, j as ASK_QUESTIONS_DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH, k as ASK_QUESTIONS_MAX_QUESTION_LENGTH, l as AskQuestionsOptions, m as AskQuestionsResult, n as AskQuestionsType, o as AudioTranslationOptions, p as AudioTranslationResult, q as AutoCensorProfanityOptions, B as BurnedInCaptionsAnalysis, r as BurnedInCaptionsOptions, s as BurnedInCaptionsPromptOverrides, u as BurnedInCaptionsPromptSections, v as BurnedInCaptionsResult, C as CaptionReplacement, w as CensorMode, x as Chapter, y as ChapterSystemPromptSections, z as ChaptersOptions, D as ChaptersPromptOverrides, E as ChaptersPromptSections, F as ChaptersResult, G as ChaptersType, H as DEFAULT_DESCRIPTION_LENGTH, J as DEFAULT_SUMMARY_KEYWORD_LIMIT, K as DEFAULT_TITLE_LENGTH, M as EditCaptionsOptions, N as EditCaptionsResult, O as EmbeddingsOptions, P as EmbeddingsResult, Q as EngagementInsightsOptions, R as EngagementInsightsResult, T as GOOGLE_VISION_LIKELIHOOD_TO_SCORE, U as HIVE_SEXUAL_CATEGORIES, V as HIVE_VIOLENCE_CATEGORIES, W as HiveModerationOutput, X as HiveModerationSource, Y as ModerationOptions, Z as ModerationProvider, _ as ModerationResult, $ as MomentInsight, a0 as OverallInsight, a1 as ProfanityDetectionPayload, a2 as Question, a3 as QuestionAnswer, a4 as QuestionAnswerType, a5 as ReplacementRecord, a6 as SummarizationOptions, a7 as SummarizationPromptOverrides, a8 as SummarizationPromptSections, a9 as SummaryAndTagsResult, aa as SummaryType, ab as ThumbnailModerationScore, ac as TranslationChunkingOptions, ad as TranslationOptions, ae as TranslationPayload, af as TranslationResult, ag as aggregateTokenUsage, ah as applyOverrideLists, ai as applyReplacements, aj as askQuestions, ak as buildReplacementRegex, al as burnedInCaptionsSchema, am as censorVttContent, an as chapterSchema, ao as chaptersSchema, ap as createReplacer, aq as editCaptions, ar as extractTranscriptSegmentsForHotspots, as as generateChapters, at as generateEmbeddings, au as generateEngagementInsights, av as generateVideoEmbeddings, aw as getModerationScores, ax as getSummaryAndTags, ay as hasBurnedInCaptions, az as mapShotsToHotspots, aA as normalizeTranslatedVtt, aB as profanityDetectionSchema, aC as questionAnswerSchema, aD as shouldSplitChunkTranslationError, aE as summarySchema, aF as transformCueText, aG as translateAudio, aH as translateCaptions, aI as translationSchema } from '../index-ooikuED7.js';
 import 'zod';
 import '@ai-sdk/anthropic';
 import '@ai-sdk/google';

package/dist/workflows/index.js

@@ -1430,13 +1430,23 @@ function createLanguageSection(languageName) {
 }
 
 // src/lib/retry.ts
+import { DownloadError } from "ai";
 var DEFAULT_RETRY_OPTIONS = {
   maxRetries: 3,
   baseDelay: 2e3,
   maxDelay: 1e4
 };
 function defaultShouldRetry(error, _attempt) {
-  return Boolean(error.message && error.message.includes("Timeout while downloading"));
+  if (error.message.includes("Timeout while downloading")) {
+    return true;
+  }
+  const isDownloadError = DownloadError.isInstance(error);
+  const isSerializedDownloadError = error.name === "AI_DownloadError";
+  if (!isDownloadError && !isSerializedDownloadError) {
+    return false;
+  }
+  const statusCode = error.statusCode;
+  return statusCode === void 0 || statusCode === 408 || statusCode === 425 || statusCode === 429 || statusCode >= 500;
 }
 function calculateDelay(attempt, baseDelay, maxDelay) {
   const exponentialDelay = baseDelay * 2 ** (attempt - 1);
@@ -3136,14 +3146,14 @@ async function hasBurnedInCaptionsInternal(assetId, options, collectedUsage) {
       credentials
     });
   } else {
-    analysisResponse = await analyzeStoryboard({
+    analysisResponse = await withRetry(() => analyzeStoryboard({
       imageDataUrl: imageUrl,
       provider: modelConfig.provider,
       modelId: modelConfig.modelId,
       userPrompt,
       systemPrompt: SYSTEM_PROMPT2,
       credentials
-    });
+    }));
   }
   collectedUsage.push(analysisResponse.usage);
   if (!analysisResponse.result) {
@@ -6834,6 +6844,59 @@ async function downloadAndUploadDubbedAudio({
   console.warn(`\u{1F517} Generated presigned URL (expires in ${expiryHours} hour${expiryHours === 1 ? "" : "s"})`);
   return presignedUrl;
 }
+async function downloadAndUploadDubTranscript({
+  dubbingId,
+  languageCode,
+  assetId,
+  toLanguageCode,
+  s3Endpoint,
+  s3Region,
+  s3Bucket,
+  storageAdapter,
+  s3SignedUrlExpirySeconds,
+  credentials
+}) {
+  "use step";
+  const elevenLabsApiKey = await getApiKeyFromEnv("elevenlabs", credentials);
+  const transcriptUrl = `https://api.elevenlabs.io/v1/dubbing/${dubbingId}/transcripts/${languageCode}/format/webvtt`;
+  const transcriptResponse = await fetch(transcriptUrl, {
+    headers: {
+      "xi-api-key": elevenLabsApiKey
+    }
+  });
+  if (!transcriptResponse.ok) {
+    throw new Error(`Failed to fetch dub transcript: ${transcriptResponse.statusText}`);
+  }
+  const transcriptBody = await transcriptResponse.json();
+  const transcriptVtt = transcriptBody.webvtt;
+  if (!transcriptVtt) {
+    throw new Error("Dub transcript response did not include webvtt content");
+  }
+  const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
+  const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
+  const vttKey = `audio-translations/${assetId}/auto-to-${toLanguageCode}-${Date.now()}.vtt`;
+  await putObjectWithStorageAdapter({
+    accessKeyId: s3AccessKeyId,
+    secretAccessKey: s3SecretAccessKey,
+    endpoint: s3Endpoint,
+    region: s3Region,
+    bucket: s3Bucket,
+    key: vttKey,
+    body: transcriptVtt,
+    contentType: "text/vtt"
+  }, storageAdapter);
+  const presignedUrl = await createPresignedGetUrlWithStorageAdapter({
+    accessKeyId: s3AccessKeyId,
+    secretAccessKey: s3SecretAccessKey,
+    endpoint: s3Endpoint,
+    region: s3Region,
+    bucket: s3Bucket,
+    key: vttKey,
+    expiresInSeconds: s3SignedUrlExpirySeconds ?? 86400
+  }, storageAdapter);
+  console.warn(`\u2705 Dub transcript uploaded successfully to: ${vttKey}`);
+  return presignedUrl;
+}
 async function createAudioTrackOnMux(assetId, languageCode, presignedUrl, credentials) {
   "use step";
   const muxClient = await resolveMuxClient(credentials);
@@ -6860,6 +6923,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
     // 0 = auto-detect
     uploadToS3: uploadToS3Option,
     uploadToMux: uploadToMuxOption,
+    uploadCaptionsToMux = false,
     storageAdapter,
     credentials: providedCredentials
   } = options;
@@ -6869,7 +6933,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
   const credentials = providedCredentials;
   const effectiveStorageAdapter = storageAdapter;
   const uploadToMux = uploadToMuxOption !== false;
-  const uploadToS3 = uploadToS3Option || uploadToMux;
+  const uploadToS3 = uploadToS3Option || uploadToMux || uploadCaptionsToMux;
   const s3Endpoint = options.s3Endpoint ?? env_default.S3_ENDPOINT;
   const s3Region = options.s3Region ?? env_default.S3_REGION ?? "auto";
   const s3Bucket = options.s3Bucket ?? env_default.S3_BUCKET;
@@ -6950,10 +7014,12 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
   console.warn("\u2705 Dubbing completed successfully!");
   let presignedUrl;
   let uploadedTrackId;
+  let captionsPresignedUrl;
+  let captionsTrackId;
   if (uploadToS3) {
     console.warn("\u{1F4E5} Downloading dubbed audio from ElevenLabs and staging to S3...");
+    const downloadLangCode = targetLanguages[0] ?? toISO639_1(toLanguageCode);
     try {
-      const downloadLangCode = targetLanguages[0] ?? toISO639_1(toLanguageCode);
       presignedUrl = await downloadAndUploadDubbedAudio({
         dubbingId,
         languageCode: downloadLangCode,
@@ -6985,6 +7051,40 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
         console.warn(presignedUrl);
       }
     }
+    try {
+      console.warn("\u{1F4E5} Downloading dub transcript from ElevenLabs and staging to S3...");
+      captionsPresignedUrl = await downloadAndUploadDubTranscript({
+        dubbingId,
+        languageCode: downloadLangCode,
+        assetId,
+        toLanguageCode,
+        s3Endpoint,
+        s3Region,
+        s3Bucket,
+        storageAdapter: effectiveStorageAdapter,
+        s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds,
+        credentials
+      });
+      if (uploadCaptionsToMux) {
+        console.warn("\u{1F4F9} Adding dubbed captions track to Mux asset...");
+        const muxLangCode = toISO639_1(toLanguageCode);
+        const languageName = new Intl.DisplayNames(["en"], { type: "language" }).of(muxLangCode) || muxLangCode.toUpperCase();
+        captionsTrackId = await createTextTrackOnMux(
+          assetId,
+          muxLangCode,
+          `${languageName} (auto-dubbed)`,
+          captionsPresignedUrl,
+          credentials
+        );
+        console.warn(`\u2705 Captions track added to Mux asset with ID: ${captionsTrackId}`);
+      }
+    } catch (error) {
+      console.warn(`\u26A0\uFE0F Failed to attach dubbed captions: ${error instanceof Error ? error.message : "Unknown error"}`);
+      if (captionsPresignedUrl) {
+        console.warn("\u{1F517} You can manually add the captions track using this presigned URL:");
+        console.warn(captionsPresignedUrl);
+      }
+    }
   }
   const targetLanguage = getLanguageCodePair(toLanguageCode);
   return {
@@ -6994,6 +7094,8 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
     dubbingId,
     uploadedTrackId,
     presignedUrl,
+    captionsTrackId,
+    captionsPresignedUrl,
     usage: {
       metadata: {
         assetDurationSeconds