@mux/ai 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -981,11 +981,20 @@ function findCaptionTrack(asset, languageCode) {
981
981
  if (!tracks.length)
982
982
  return void 0;
983
983
  if (!languageCode) {
984
- return tracks[0];
984
+ const englishTrack = tracks.find(
985
+ (track) => track.text_type === "subtitles" && track.language_code === "en"
986
+ );
987
+ return englishTrack ?? tracks[0];
985
988
  }
986
- return tracks.find(
989
+ const languageMatch = tracks.find(
987
990
  (track) => track.text_type === "subtitles" && track.language_code === languageCode
988
991
  );
992
+ if (languageMatch)
993
+ return languageMatch;
994
+ if (isAudioOnlyAsset(asset) && tracks.length === 1) {
995
+ return tracks[0];
996
+ }
997
+ return void 0;
989
998
  }
990
999
  function normalizeLineEndings(value) {
991
1000
  return value.replace(/\r\n/g, "\n");
@@ -1422,19 +1431,102 @@ var SYSTEM_PROMPT = dedent`
1422
1431
  - GOOD: "A person runs through a park"
1423
1432
  - Be specific and evidence-based
1424
1433
  </language_guidelines>`;
1434
+ var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
1435
+ <role>
1436
+ You are an audio content analyst specializing in classification tasks.
1437
+ Your job is to answer questions about audio content based on transcript data.
1438
+ </role>
1439
+
1440
+ <context>
1441
+ You will receive:
1442
+ - A transcript of the audio/dialogue
1443
+ - A list of questions about the asset content
1444
+ </context>
1445
+
1446
+ <transcript_guidance>
1447
+ - Use the transcript to understand spoken content, dialogue, and audio context
1448
+ - Consider only transcript evidence when answering questions
1449
+ </transcript_guidance>
1450
+
1451
+ <task>
1452
+ For each question provided, you must:
1453
+ 1. Analyze the transcript
1454
+ 2. Answer with ONLY the allowed response options - no other values are acceptable
1455
+ 3. Provide a confidence score between 0 and 1 reflecting your certainty
1456
+ 4. Explain your reasoning based on observable evidence
1457
+ </task>
1458
+
1459
+ <answer_guidelines>
1460
+ - Choose the affirmative option only if you have clear evidence supporting it
1461
+ - Choose the negative/contradicting option if evidence contradicts or if insufficient evidence exists
1462
+ - Confidence should reflect the clarity and strength of evidence:
1463
+ * 0.9-1.0: Clear, unambiguous evidence
1464
+ * 0.7-0.9: Strong evidence with minor ambiguity
1465
+ * 0.5-0.7: Moderate evidence or some conflicting signals
1466
+ * 0.3-0.5: Weak evidence or significant ambiguity
1467
+ * 0.0-0.3: Very uncertain, minimal relevant evidence
1468
+ - Reasoning should cite specific transcript evidence
1469
+ - Be precise: cite specific quotes or passages from transcript text
1470
+ </answer_guidelines>
1471
+
1472
+ <relevance_filtering>
1473
+ Before answering each question, assess whether it can be meaningfully
1474
+ answered based on the transcript. A question is relevant if it asks about
1475
+ something observable or inferable from spoken/audio content.
1476
+
1477
+ Mark a question as skipped (skipped: true) if it:
1478
+ - Is completely unrelated to transcript/audio content (e.g., math, trivia, personal questions)
1479
+ - Asks about information that cannot be determined from transcript content
1480
+ - Is a general knowledge question with no connection to what is said in the transcript
1481
+ - Attempts to use the system for non-content-analysis purposes
1482
+
1483
+ For skipped questions:
1484
+ - Set skipped to true
1485
+ - Set answer to "${SKIP_SENTINEL}"
1486
+ - Set confidence to 0
1487
+ - Use the reasoning field to explain why the question is not answerable
1488
+ from transcript content
1489
+
1490
+ For borderline questions that are loosely related to transcript content,
1491
+ still answer them but use a lower confidence score to reflect uncertainty.
1492
+ </relevance_filtering>
1493
+
1494
+ <constraints>
1495
+ - You MUST answer every relevant question with one of the allowed response options
1496
+ - Skip irrelevant questions as described in relevance_filtering
1497
+ - Only describe observable evidence from transcript content
1498
+ - Do not fabricate details or make unsupported assumptions
1499
+ - Return structured data matching the requested schema exactly
1500
+ - Provide reasoning in the same language as the question
1501
+ </constraints>
1502
+
1503
+ <language_guidelines>
1504
+ When explaining reasoning:
1505
+ - Describe content directly, not the medium
1506
+ - BAD: "The audio says someone is running"
1507
+ - GOOD: "The speaker describes running through a park"
1508
+ - Be specific and evidence-based
1509
+ </language_guidelines>`;
1425
1510
  var askQuestionsPromptBuilder = createPromptBuilder({
1426
1511
  template: {
1427
1512
  questions: {
1428
1513
  tag: "questions",
1429
- content: "Please answer the following yes/no questions about this video:"
1514
+ content: "Please answer the following questions about this content:"
1430
1515
  }
1431
1516
  },
1432
1517
  sectionOrder: ["questions"]
1433
1518
  });
1434
- function buildUserPrompt(questions, allowedAnswers, transcriptText, isCleanTranscript = true) {
1519
+ function buildUserPrompt({
1520
+ questions,
1521
+ allowedAnswers,
1522
+ transcriptText,
1523
+ isCleanTranscript = true,
1524
+ isAudioOnly = false
1525
+ }) {
1435
1526
  const questionsList = questions.map((q, idx) => `${idx + 1}. ${q.question}`).join("\n");
1527
+ const questionsIntro = isAudioOnly ? "Please answer the following questions about this audio content using transcript evidence:" : "Please answer the following questions about this content:";
1436
1528
  const questionsContent = dedent`
1437
- Please answer the following yes/no questions about this video:
1529
+ ${questionsIntro}
1438
1530
 
1439
1531
  ${questionsList}`;
1440
1532
  const answerList = allowedAnswers.map((answer) => `"${answer}"`).join(", ");
@@ -1461,7 +1553,15 @@ async function fetchImageAsBase64(imageUrl, imageDownloadOptions) {
1461
1553
  const downloadResult = await downloadImageAsBase64(imageUrl, imageDownloadOptions);
1462
1554
  return downloadResult.base64Data;
1463
1555
  }
1464
- async function analyzeQuestionsWithStoryboard(imageDataUrl, provider, modelId, userPrompt, systemPrompt, allowedAnswers, credentials) {
1556
+ async function analyzeQuestions({
1557
+ provider,
1558
+ modelId,
1559
+ userPrompt,
1560
+ systemPrompt,
1561
+ allowedAnswers,
1562
+ imageDataUrl,
1563
+ credentials
1564
+ }) {
1465
1565
  "use step";
1466
1566
  const model = await createLanguageModelFromConfig(provider, modelId, credentials);
1467
1567
  const responseSchema = createAskQuestionsSchema(allowedAnswers);
@@ -1476,15 +1576,19 @@ async function analyzeQuestionsWithStoryboard(imageDataUrl, provider, modelId, u
1476
1576
  },
1477
1577
  {
1478
1578
  role: "user",
1479
- content: [
1579
+ content: imageDataUrl ? [
1480
1580
  { type: "text", text: userPrompt },
1481
1581
  { type: "image", image: imageDataUrl }
1482
- ]
1582
+ ] : userPrompt
1483
1583
  }
1484
1584
  ]
1485
1585
  });
1586
+ if (!response.output) {
1587
+ throw new Error("Ask-questions output missing");
1588
+ }
1589
+ const parsed = responseSchema.parse(response.output);
1486
1590
  return {
1487
- result: response.output,
1591
+ result: parsed,
1488
1592
  usage: {
1489
1593
  inputTokens: response.usage.inputTokens,
1490
1594
  outputTokens: response.usage.outputTokens,
@@ -1509,6 +1613,7 @@ async function askQuestions(assetId, questions, options) {
1509
1613
  const {
1510
1614
  provider = "openai",
1511
1615
  model,
1616
+ languageCode,
1512
1617
  answerOptions,
1513
1618
  includeTranscript = true,
1514
1619
  cleanTranscript = true,
@@ -1533,53 +1638,83 @@ async function askQuestions(assetId, questions, options) {
1533
1638
  });
1534
1639
  const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
1535
1640
  const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
1641
+ const isAudioOnly = isAudioOnlyAsset(assetData);
1642
+ if (isAudioOnly && !includeTranscript) {
1643
+ throw new Error(
1644
+ "Audio-only assets require a transcript. Set includeTranscript: true and ensure the asset has a ready text track (captions/subtitles)."
1645
+ );
1646
+ }
1536
1647
  const signingContext = await resolveMuxSigningContext(credentials);
1537
1648
  if (policy === "signed" && !signingContext) {
1538
1649
  throw new Error(
1539
1650
  "Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
1540
1651
  );
1541
1652
  }
1542
- const transcriptText = includeTranscript ? (await fetchTranscriptForAsset(assetData, playbackId, {
1653
+ const transcriptResult = includeTranscript ? await fetchTranscriptForAsset(assetData, playbackId, {
1654
+ languageCode,
1543
1655
  cleanTranscript,
1544
- shouldSign: policy === "signed"
1545
- })).transcriptText : "";
1546
- const userPrompt = buildUserPrompt(questions, allowedAnswers, transcriptText, cleanTranscript);
1547
- const systemPrompt = SYSTEM_PROMPT;
1548
- const imageUrl = await getStoryboardUrl(
1549
- playbackId,
1550
- storyboardWidth,
1551
- policy === "signed",
1552
- credentials
1553
- );
1656
+ shouldSign: policy === "signed",
1657
+ credentials,
1658
+ required: isAudioOnly
1659
+ }) : void 0;
1660
+ const transcriptText = transcriptResult?.transcriptText ?? "";
1661
+ const userPrompt = buildUserPrompt({
1662
+ questions,
1663
+ allowedAnswers,
1664
+ transcriptText,
1665
+ isCleanTranscript: cleanTranscript,
1666
+ isAudioOnly
1667
+ });
1668
+ const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT : SYSTEM_PROMPT;
1554
1669
  let analysisResponse;
1670
+ let imageUrl;
1555
1671
  try {
1556
- if (imageSubmissionMode === "base64") {
1557
- const base64Data = await fetchImageAsBase64(imageUrl, imageDownloadOptions);
1558
- analysisResponse = await analyzeQuestionsWithStoryboard(
1559
- base64Data,
1560
- modelConfig.provider,
1561
- modelConfig.modelId,
1672
+ if (isAudioOnly) {
1673
+ analysisResponse = await analyzeQuestions({
1674
+ provider: modelConfig.provider,
1675
+ modelId: modelConfig.modelId,
1562
1676
  userPrompt,
1563
1677
  systemPrompt,
1564
1678
  allowedAnswers,
1565
1679
  credentials
1566
- );
1680
+ });
1567
1681
  } else {
1568
- analysisResponse = await withRetry(
1569
- () => analyzeQuestionsWithStoryboard(
1570
- imageUrl,
1571
- modelConfig.provider,
1572
- modelConfig.modelId,
1682
+ const storyboardUrl = await getStoryboardUrl(
1683
+ playbackId,
1684
+ storyboardWidth,
1685
+ policy === "signed",
1686
+ credentials
1687
+ );
1688
+ imageUrl = storyboardUrl;
1689
+ if (imageSubmissionMode === "base64") {
1690
+ const base64Data = await fetchImageAsBase64(storyboardUrl, imageDownloadOptions);
1691
+ analysisResponse = await analyzeQuestions({
1692
+ provider: modelConfig.provider,
1693
+ modelId: modelConfig.modelId,
1573
1694
  userPrompt,
1574
1695
  systemPrompt,
1575
1696
  allowedAnswers,
1697
+ imageDataUrl: base64Data,
1576
1698
  credentials
1577
- )
1578
- );
1699
+ });
1700
+ } else {
1701
+ analysisResponse = await withRetry(
1702
+ () => analyzeQuestions({
1703
+ provider: modelConfig.provider,
1704
+ modelId: modelConfig.modelId,
1705
+ userPrompt,
1706
+ systemPrompt,
1707
+ allowedAnswers,
1708
+ imageDataUrl: storyboardUrl,
1709
+ credentials
1710
+ })
1711
+ );
1712
+ }
1579
1713
  }
1580
1714
  } catch (error) {
1715
+ const contentType = isAudioOnly ? "audio" : "video";
1581
1716
  throw new Error(
1582
- `Failed to analyze questions with ${provider}: ${error instanceof Error ? error.message : "Unknown error"}`
1717
+ `Failed to analyze ${contentType} questions with ${provider}: ${error instanceof Error ? error.message : "Unknown error"}`
1583
1718
  );
1584
1719
  }
1585
1720
  if (!analysisResponse.result?.answers) {
@@ -2119,9 +2254,10 @@ function buildUserPrompt3({
2119
2254
  };
2120
2255
  return chaptersPromptBuilder.buildWithContext(mergedOverrides, contextSections);
2121
2256
  }
2122
- async function generateChapters(assetId, languageCode, options = {}) {
2257
+ async function generateChapters(assetId, options = {}) {
2123
2258
  "use workflow";
2124
2259
  const {
2260
+ languageCode,
2125
2261
  provider = "openai",
2126
2262
  model,
2127
2263
  promptOverrides,
@@ -2145,26 +2281,17 @@ async function generateChapters(assetId, languageCode, options = {}) {
2145
2281
  );
2146
2282
  }
2147
2283
  const readyTextTracks = getReadyTextTracks(assetData);
2148
- let transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
2284
+ const transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
2149
2285
  languageCode,
2150
2286
  cleanTranscript: false,
2151
2287
  // keep timestamps for chapter segmentation
2152
2288
  shouldSign: policy === "signed",
2153
2289
  credentials
2154
2290
  });
2155
- if (isAudioOnly && !transcriptResult.track && readyTextTracks.length === 1) {
2156
- transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
2157
- cleanTranscript: false,
2158
- // keep timestamps for chapter segmentation
2159
- shouldSign: policy === "signed",
2160
- credentials,
2161
- required: true
2162
- });
2163
- }
2164
2291
  if (!transcriptResult.track || !transcriptResult.transcriptText) {
2165
2292
  const availableLanguages = readyTextTracks.map((t) => t.language_code).filter(Boolean).join(", ");
2166
2293
  throw new Error(
2167
- `No caption track found for language '${languageCode}'. Available languages: ${availableLanguages || "none"}`
2294
+ `No caption track found${languageCode ? ` for language '${languageCode}'` : ""}. Available languages: ${availableLanguages || "none"}`
2168
2295
  );
2169
2296
  }
2170
2297
  const timestampedTranscript = extractTimestampedTranscript(transcriptResult.transcriptText);
@@ -2216,7 +2343,7 @@ async function generateChapters(assetId, languageCode, options = {}) {
2216
2343
  };
2217
2344
  return {
2218
2345
  assetId,
2219
- languageCode,
2346
+ languageCode: languageCode ?? transcriptResult.track?.language_code ?? "en",
2220
2347
  chapters: validChapters,
2221
2348
  usage: usageWithMetadata
2222
2349
  };
@@ -2696,6 +2823,7 @@ async function editCaptions(assetId, trackId, options) {
2696
2823
  autoCensorProfanity: autoCensorOption,
2697
2824
  replacements: replacementsOption,
2698
2825
  deleteOriginalTrack,
2826
+ uploadToS3: uploadToS3Option,
2699
2827
  uploadToMux: uploadToMuxOption,
2700
2828
  s3Endpoint: providedS3Endpoint,
2701
2829
  s3Region: providedS3Region,
@@ -2714,14 +2842,15 @@ async function editCaptions(assetId, trackId, options) {
2714
2842
  }
2715
2843
  const deleteOriginal = deleteOriginalTrack !== false;
2716
2844
  const uploadToMux = uploadToMuxOption !== false;
2845
+ const uploadToS3 = uploadToS3Option || uploadToMux;
2717
2846
  const s3Endpoint = providedS3Endpoint ?? env_default.S3_ENDPOINT;
2718
2847
  const s3Region = providedS3Region ?? env_default.S3_REGION ?? "auto";
2719
2848
  const s3Bucket = providedS3Bucket ?? env_default.S3_BUCKET;
2720
2849
  const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
2721
2850
  const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
2722
- if (uploadToMux && (!s3Endpoint || !s3Bucket || !storageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
2851
+ if (uploadToS3 && (!s3Endpoint || !s3Bucket || !storageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
2723
2852
  throw new Error(
2724
- "Storage configuration is required for uploading to Mux. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables."
2853
+ "Storage configuration is required for uploading. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables."
2725
2854
  );
2726
2855
  }
2727
2856
  const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
@@ -2794,53 +2923,45 @@ async function editCaptions(assetId, trackId, options) {
2794
2923
  assetDurationSeconds
2795
2924
  }
2796
2925
  } : void 0;
2797
- if (!uploadToMux) {
2798
- return {
2799
- assetId,
2800
- trackId,
2801
- originalVtt: vttContent,
2802
- editedVtt,
2803
- totalReplacementCount,
2804
- autoCensorProfanity: autoCensorResult,
2805
- replacements: replacementsResult,
2806
- usage: usageWithMetadata
2807
- };
2808
- }
2809
2926
  let presignedUrl;
2810
- try {
2811
- presignedUrl = await uploadEditedVttToS3({
2812
- editedVtt,
2813
- assetId,
2814
- trackId,
2815
- s3Endpoint,
2816
- s3Region,
2817
- s3Bucket,
2818
- storageAdapter,
2819
- s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
2820
- });
2821
- } catch (error) {
2822
- throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
2823
- }
2824
2927
  let uploadedTrackId;
2825
- try {
2826
- const languageCode = sourceTrack.language_code || "en";
2827
- const suffix = trackNameSuffix ?? "edited";
2828
- const trackName = `${sourceTrack.name || "Subtitles"} (${suffix})`;
2829
- uploadedTrackId = await createTextTrackOnMux(
2830
- assetId,
2831
- languageCode,
2832
- trackName,
2833
- presignedUrl,
2834
- credentials
2835
- );
2836
- } catch (error) {
2837
- console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
2838
- }
2839
- if (deleteOriginal && uploadedTrackId) {
2928
+ if (uploadToS3) {
2840
2929
  try {
2841
- await deleteTrackOnMux(assetId, trackId, credentials);
2930
+ presignedUrl = await uploadEditedVttToS3({
2931
+ editedVtt,
2932
+ assetId,
2933
+ trackId,
2934
+ s3Endpoint,
2935
+ s3Region,
2936
+ s3Bucket,
2937
+ storageAdapter,
2938
+ s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
2939
+ });
2842
2940
  } catch (error) {
2843
- console.warn(`Failed to delete original track: ${error instanceof Error ? error.message : "Unknown error"}`);
2941
+ throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
2942
+ }
2943
+ if (uploadToMux) {
2944
+ try {
2945
+ const languageCode = sourceTrack.language_code || "en";
2946
+ const suffix = trackNameSuffix ?? "edited";
2947
+ const trackName = `${sourceTrack.name || "Subtitles"} (${suffix})`;
2948
+ uploadedTrackId = await createTextTrackOnMux(
2949
+ assetId,
2950
+ languageCode,
2951
+ trackName,
2952
+ presignedUrl,
2953
+ credentials
2954
+ );
2955
+ } catch (error) {
2956
+ console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
2957
+ }
2958
+ if (deleteOriginal && uploadedTrackId) {
2959
+ try {
2960
+ await deleteTrackOnMux(assetId, trackId, credentials);
2961
+ } catch (error) {
2962
+ console.warn(`Failed to delete original track: ${error instanceof Error ? error.message : "Unknown error"}`);
2963
+ }
2964
+ }
2844
2965
  }
2845
2966
  }
2846
2967
  return {
@@ -3151,43 +3272,21 @@ async function generateEmbeddingsInternal(assetId, options = {}) {
3151
3272
  const embeddingModel = resolveEmbeddingModelConfig({ ...options, provider, model });
3152
3273
  const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
3153
3274
  const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
3154
- const isAudioOnly = isAudioOnlyAsset(assetData);
3155
3275
  const signingContext = await resolveMuxSigningContext(credentials);
3156
3276
  if (policy === "signed" && !signingContext) {
3157
3277
  throw new Error(
3158
3278
  "Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
3159
3279
  );
3160
3280
  }
3161
- const readyTextTracks = getReadyTextTracks(assetData);
3162
3281
  const useVttChunking = chunkingStrategy.type === "vtt";
3163
- let transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
3282
+ const transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
3164
3283
  languageCode,
3165
3284
  cleanTranscript: !useVttChunking,
3166
3285
  shouldSign: policy === "signed",
3167
- credentials
3286
+ credentials,
3287
+ required: true
3168
3288
  });
3169
- if (isAudioOnly && !transcriptResult.track && readyTextTracks.length === 1) {
3170
- transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
3171
- cleanTranscript: !useVttChunking,
3172
- shouldSign: policy === "signed",
3173
- credentials
3174
- });
3175
- }
3176
- if (!transcriptResult.track || !transcriptResult.transcriptText) {
3177
- const availableLanguages = readyTextTracks.map((t) => t.language_code).filter(Boolean).join(", ");
3178
- if (isAudioOnly) {
3179
- throw new Error(
3180
- `No transcript track found${languageCode ? ` for language '${languageCode}'` : ""}. Audio-only assets require a transcript. Available languages: ${availableLanguages || "none"}`
3181
- );
3182
- }
3183
- throw new Error(
3184
- `No caption track found${languageCode ? ` for language '${languageCode}'` : ""}. Available languages: ${availableLanguages || "none"}`
3185
- );
3186
- }
3187
3289
  const transcriptText = transcriptResult.transcriptText;
3188
- if (!transcriptText.trim()) {
3189
- throw new Error("Transcript is empty");
3190
- }
3191
3290
  const chunks = useVttChunking ? chunkVTTCues(
3192
3291
  parseVTTCues(transcriptText),
3193
3292
  chunkingStrategy.maxTokens,
@@ -3448,7 +3547,7 @@ var SYSTEM_PROMPT4 = dedent5`
3448
3547
  - Connect engagement data to content features
3449
3548
  </quality_guidelines>
3450
3549
  `;
3451
- var AUDIO_ONLY_SYSTEM_PROMPT = dedent5`
3550
+ var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent5`
3452
3551
  <role>
3453
3552
  You are a video engagement analyst specializing in understanding viewer behavior for audio content.
3454
3553
  Your job is to explain why specific moments have high or low engagement based on audio/dialogue and viewer watch patterns.
@@ -3736,9 +3835,14 @@ async function generateEngagementInsights(assetId, options = {}) {
3736
3835
  }
3737
3836
  const { hotspots, heatmap } = engagementResult.value;
3738
3837
  if (hotspots.length === 0) {
3739
- throw new Error(
3740
- `No engagement data available for asset ${assetId} in timeframe ${timeframe}. Video may not have been viewed yet or engagement tracking is not enabled.`
3741
- );
3838
+ return {
3839
+ assetId,
3840
+ momentInsights: [],
3841
+ overallInsight: {
3842
+ summary: `No engagement data available for asset ${assetId} in timeframe ${timeframe}. Video may not have been viewed yet, not contain notable engagement patterns, or engagement tracking is not enabled.`,
3843
+ trends: []
3844
+ }
3845
+ };
3742
3846
  }
3743
3847
  let transcriptText = "";
3744
3848
  if (transcriptResult.status === "fulfilled") {
@@ -3781,7 +3885,7 @@ async function generateEngagementInsights(assetId, options = {}) {
3781
3885
  imageUrls.push(storyboardResult.value);
3782
3886
  }
3783
3887
  }
3784
- const systemPrompt = audioOnly ? AUDIO_ONLY_SYSTEM_PROMPT : SYSTEM_PROMPT4;
3888
+ const systemPrompt = audioOnly ? AUDIO_ONLY_SYSTEM_PROMPT2 : SYSTEM_PROMPT4;
3785
3889
  const userPrompt = buildUserPrompt4(
3786
3890
  hotspots,
3787
3891
  transcriptSegments,
@@ -3941,6 +4045,12 @@ var DEFAULT_THRESHOLDS = {
3941
4045
  violence: 0.8
3942
4046
  };
3943
4047
  var DEFAULT_PROVIDER2 = "openai";
4048
+ var OPENAI_MODERATION_RETRIABLE_STATUS_CODES = /* @__PURE__ */ new Set([408, 429, 500, 502, 503, 504]);
4049
+ var OPENAI_MODERATION_MAX_RETRIES = 2;
4050
+ var OPENAI_MODERATION_BASE_DELAY_MS = 750;
4051
+ var OPENAI_MODERATION_MAX_DELAY_MS = 3e3;
4052
+ var MIN_SAMPLE_COVERAGE_FOR_CONFIDENT_THRESHOLDING = 0.5;
4053
+ var MIN_SUCCESSFUL_THUMBNAILS_FOR_CONFIDENT_THRESHOLDING = 3;
3944
4054
  var HIVE_ENDPOINT = "https://api.thehive.ai/api/v2/task/sync";
3945
4055
  var HIVE_SEXUAL_CATEGORIES = [
3946
4056
  "general_nsfw",
@@ -3962,6 +4072,92 @@ var HIVE_VIOLENCE_CATEGORIES = [
3962
4072
  "yes_self_harm",
3963
4073
  "garm_death_injury_or_military_conflict"
3964
4074
  ];
4075
+ var OpenAIModerationRequestError = class extends Error {
4076
+ constructor(message, {
4077
+ status,
4078
+ retriable
4079
+ }) {
4080
+ super(message);
4081
+ this.name = "OpenAIModerationRequestError";
4082
+ this.status = status;
4083
+ this.retriable = retriable;
4084
+ }
4085
+ };
4086
+ function isRetriableOpenAIModerationStatus(status) {
4087
+ return OPENAI_MODERATION_RETRIABLE_STATUS_CODES.has(status);
4088
+ }
4089
+ function compactBodySnippet(body, maxLength = 180) {
4090
+ const compact = body.replace(/\s+/g, " ").trim();
4091
+ if (!compact) {
4092
+ return "(empty response body)";
4093
+ }
4094
+ return compact.length > maxLength ? `${compact.slice(0, maxLength)}...` : compact;
4095
+ }
4096
+ function parseOpenAIModerationResponse(bodyText, status, statusText) {
4097
+ if (!bodyText.trim()) {
4098
+ throw new OpenAIModerationRequestError(
4099
+ `OpenAI moderation returned an empty response (${status} ${statusText}).`,
4100
+ { status, retriable: isRetriableOpenAIModerationStatus(status) }
4101
+ );
4102
+ }
4103
+ try {
4104
+ return JSON.parse(bodyText);
4105
+ } catch {
4106
+ throw new OpenAIModerationRequestError(
4107
+ `OpenAI moderation returned non-JSON response (${status} ${statusText}): ${compactBodySnippet(bodyText)}`,
4108
+ { status, retriable: true }
4109
+ );
4110
+ }
4111
+ }
4112
+ async function callOpenAIModerationApi({
4113
+ model,
4114
+ input,
4115
+ credentials
4116
+ }) {
4117
+ "use step";
4118
+ const apiKey = await getApiKeyFromEnv("openai", credentials);
4119
+ return withRetry(
4120
+ async () => {
4121
+ let res;
4122
+ try {
4123
+ res = await fetch("https://api.openai.com/v1/moderations", {
4124
+ method: "POST",
4125
+ headers: {
4126
+ "Content-Type": "application/json",
4127
+ "Authorization": `Bearer ${apiKey}`
4128
+ },
4129
+ body: JSON.stringify({
4130
+ model,
4131
+ input
4132
+ })
4133
+ });
4134
+ } catch (error) {
4135
+ throw new OpenAIModerationRequestError(
4136
+ `OpenAI moderation request failed: ${error instanceof Error ? error.message : String(error)}`,
4137
+ { retriable: true }
4138
+ );
4139
+ }
4140
+ const bodyText = await res.text();
4141
+ const json = parseOpenAIModerationResponse(bodyText, res.status, res.statusText);
4142
+ if (!res.ok) {
4143
+ throw new OpenAIModerationRequestError(
4144
+ `OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`,
4145
+ {
4146
+ status: res.status,
4147
+ retriable: isRetriableOpenAIModerationStatus(res.status)
4148
+ }
4149
+ );
4150
+ }
4151
+ return json;
4152
+ },
4153
+ {
4154
+ maxRetries: OPENAI_MODERATION_MAX_RETRIES,
4155
+ baseDelay: OPENAI_MODERATION_BASE_DELAY_MS,
4156
+ maxDelay: OPENAI_MODERATION_MAX_DELAY_MS,
4157
+ shouldRetry: (error) => error instanceof OpenAIModerationRequestError ? error.retriable : true
4158
+ }
4159
+ );
4160
+ }
3965
4161
  async function processConcurrently(items, processor, maxConcurrent = 5) {
3966
4162
  "use step";
3967
4163
  const results = [];
@@ -3975,32 +4171,19 @@ async function processConcurrently(items, processor, maxConcurrent = 5) {
3975
4171
  }
3976
4172
  async function moderateImageWithOpenAI(entry) {
3977
4173
  "use step";
3978
- const apiKey = await getApiKeyFromEnv("openai", entry.credentials);
3979
4174
  try {
3980
- const res = await fetch("https://api.openai.com/v1/moderations", {
3981
- method: "POST",
3982
- headers: {
3983
- "Content-Type": "application/json",
3984
- "Authorization": `Bearer ${apiKey}`
3985
- },
3986
- body: JSON.stringify({
3987
- model: entry.model,
3988
- input: [
3989
- {
3990
- type: "image_url",
3991
- image_url: {
3992
- url: entry.image
3993
- }
4175
+ const json = await callOpenAIModerationApi({
4176
+ model: entry.model,
4177
+ input: [
4178
+ {
4179
+ type: "image_url",
4180
+ image_url: {
4181
+ url: entry.image
3994
4182
  }
3995
- ]
3996
- })
4183
+ }
4184
+ ],
4185
+ credentials: entry.credentials
3997
4186
  });
3998
- const json = await res.json();
3999
- if (!res.ok) {
4000
- throw new Error(
4001
- `OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`
4002
- );
4003
- }
4004
4187
  const categoryScores = json.results?.[0]?.category_scores || {};
4005
4188
  return {
4006
4189
  url: entry.url,
@@ -4032,25 +4215,12 @@ async function requestOpenAIModeration(images, model, maxConcurrent = 5, submiss
4032
4215
  }
4033
4216
  async function requestOpenAITextModeration(text, model, url, credentials) {
4034
4217
  "use step";
4035
- const apiKey = await getApiKeyFromEnv("openai", credentials);
4036
4218
  try {
4037
- const res = await fetch("https://api.openai.com/v1/moderations", {
4038
- method: "POST",
4039
- headers: {
4040
- "Content-Type": "application/json",
4041
- "Authorization": `Bearer ${apiKey}`
4042
- },
4043
- body: JSON.stringify({
4044
- model,
4045
- input: text
4046
- })
4219
+ const json = await callOpenAIModerationApi({
4220
+ model,
4221
+ input: text,
4222
+ credentials
4047
4223
  });
4048
- const json = await res.json();
4049
- if (!res.ok) {
4050
- throw new Error(
4051
- `OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`
4052
- );
4053
- }
4054
4224
  const categoryScores = json.results?.[0]?.category_scores || {};
4055
4225
  return {
4056
4226
  url,
@@ -4256,22 +4426,13 @@ async function getModerationScores(assetId, options = {}) {
4256
4426
  let thumbnailCount;
4257
4427
  if (isAudioOnly) {
4258
4428
  mode = "transcript";
4259
- const readyTextTracks = getReadyTextTracks(asset);
4260
- let transcriptResult = await fetchTranscriptForAsset(asset, playbackId, {
4429
+ const transcriptResult = await fetchTranscriptForAsset(asset, playbackId, {
4261
4430
  languageCode,
4262
4431
  cleanTranscript: true,
4263
4432
  shouldSign: policy === "signed",
4264
4433
  credentials,
4265
4434
  required: true
4266
4435
  });
4267
- if (!transcriptResult.track && readyTextTracks.length === 1) {
4268
- transcriptResult = await fetchTranscriptForAsset(asset, playbackId, {
4269
- cleanTranscript: true,
4270
- shouldSign: policy === "signed",
4271
- credentials,
4272
- required: true
4273
- });
4274
- }
4275
4436
  if (provider === "openai") {
4276
4437
  thumbnailScores = await requestOpenAITranscriptModeration(
4277
4438
  transcriptResult.transcriptText,
@@ -4285,35 +4446,28 @@ async function getModerationScores(assetId, options = {}) {
4285
4446
  throw new Error(`Unsupported moderation provider: ${provider}`);
4286
4447
  }
4287
4448
  } else {
4288
- const thumbnailUrls = maxSamples === void 0 ? (
4289
- // Generate thumbnail URLs (signed if needed) using existing interval-based logic.
4290
- await getThumbnailUrls(playbackId, duration, {
4291
- interval: thumbnailInterval,
4449
+ const estimatedIntervalCount = duration <= 50 ? 5 : Math.ceil(duration / thumbnailInterval);
4450
+ const thumbnailUrls = maxSamples !== void 0 && estimatedIntervalCount > maxSamples ? await getThumbnailUrlsFromTimestamps(
4451
+ playbackId,
4452
+ planSamplingTimestamps({
4453
+ duration_sec: duration,
4454
+ max_candidates: maxSamples,
4455
+ trim_start_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
4456
+ trim_end_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
4457
+ fps: videoTrackFps,
4458
+ base_cadence_hz: thumbnailInterval > 0 ? 1 / thumbnailInterval : void 0
4459
+ }),
4460
+ {
4292
4461
  width: thumbnailWidth,
4293
4462
  shouldSign: policy === "signed",
4294
4463
  credentials
4295
- })
4296
- ) : (
4297
- // In maxSamples mode, sample valid timestamps over the trimmed usable span.
4298
- // Use proportional trims (≈ duration/6, capped at 5s) to stay well inside the
4299
- // renderable range — Mux can't always serve thumbnails at the very edges.
4300
- await getThumbnailUrlsFromTimestamps(
4301
- playbackId,
4302
- planSamplingTimestamps({
4303
- duration_sec: duration,
4304
- max_candidates: maxSamples,
4305
- trim_start_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
4306
- trim_end_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
4307
- fps: videoTrackFps,
4308
- base_cadence_hz: thumbnailInterval > 0 ? 1 / thumbnailInterval : void 0
4309
- }),
4310
- {
4311
- width: thumbnailWidth,
4312
- shouldSign: policy === "signed",
4313
- credentials
4314
- }
4315
- )
4316
- );
4464
+ }
4465
+ ) : await getThumbnailUrls(playbackId, duration, {
4466
+ interval: thumbnailInterval,
4467
+ width: thumbnailWidth,
4468
+ shouldSign: policy === "signed",
4469
+ credentials
4470
+ });
4317
4471
  thumbnailCount = thumbnailUrls.length;
4318
4472
  if (provider === "openai") {
4319
4473
  thumbnailScores = await requestOpenAIModeration(
@@ -4337,20 +4491,40 @@ async function getModerationScores(assetId, options = {}) {
4337
4491
  }
4338
4492
  }
4339
4493
  const failed = thumbnailScores.filter((s) => s.error);
4340
- if (failed.length > 0) {
4494
+ const successful = thumbnailScores.filter((s) => !s.error);
4495
+ if (successful.length === 0) {
4341
4496
  const details = failed.map((s) => `${s.url}: ${s.errorMessage || "Unknown error"}`).join("; ");
4342
4497
  throw new Error(
4343
- `Moderation failed for ${failed.length}/${thumbnailScores.length} thumbnail(s): ${details}`
4498
+ `Moderation failed for all ${thumbnailScores.length} sample(s): ${details}`
4499
+ );
4500
+ }
4501
+ if (failed.length > 0) {
4502
+ console.warn(
4503
+ `Moderation had partial failures (${failed.length}/${thumbnailScores.length}); continuing with successful samples.`
4344
4504
  );
4345
4505
  }
4346
- const maxSexual = Math.max(...thumbnailScores.map((s) => s.sexual));
4347
- const maxViolence = Math.max(...thumbnailScores.map((s) => s.violence));
4506
+ const maxSexual = Math.max(...successful.map((s) => s.sexual));
4507
+ const maxViolence = Math.max(...successful.map((s) => s.violence));
4348
4508
  const finalThresholds = { ...DEFAULT_THRESHOLDS, ...thresholds };
4509
+ const requestedSampleCount = thumbnailScores.length;
4510
+ const successfulSampleCount = successful.length;
4511
+ const failedSampleCount = failed.length;
4512
+ const sampleCoverage = successfulSampleCount / requestedSampleCount;
4513
+ const hasEnoughSuccessfulSamples = mode === "transcript" || successfulSampleCount >= MIN_SUCCESSFUL_THUMBNAILS_FOR_CONFIDENT_THRESHOLDING;
4514
+ const isLowConfidence = sampleCoverage < MIN_SAMPLE_COVERAGE_FOR_CONFIDENT_THRESHOLDING || !hasEnoughSuccessfulSamples;
4349
4515
  return {
4350
4516
  assetId,
4351
4517
  mode,
4352
4518
  isAudioOnly,
4353
4519
  thumbnailScores,
4520
+ coverage: {
4521
+ requestedSampleCount,
4522
+ successfulSampleCount,
4523
+ failedSampleCount,
4524
+ sampleCoverage,
4525
+ isPartial: failedSampleCount > 0,
4526
+ isLowConfidence
4527
+ },
4354
4528
  usage: {
4355
4529
  metadata: {
4356
4530
  assetDurationSeconds: duration,
@@ -4578,7 +4752,7 @@ var SYSTEM_PROMPT5 = dedent6`
4578
4752
 
4579
4753
  Write as if describing reality, not describing a recording of reality.
4580
4754
  </language_guidelines>`;
4581
- var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent6`
4755
+ var AUDIO_ONLY_SYSTEM_PROMPT3 = dedent6`
4582
4756
  <role>
4583
4757
  You are an audio content analyst specializing in transcript analysis and metadata generation.
4584
4758
  </role>
@@ -4755,6 +4929,7 @@ async function getSummaryAndTags(assetId, options) {
4755
4929
  const {
4756
4930
  provider = "openai",
4757
4931
  model,
4932
+ languageCode,
4758
4933
  tone = "neutral",
4759
4934
  includeTranscript = true,
4760
4935
  cleanTranscript = true,
@@ -4793,6 +4968,7 @@ async function getSummaryAndTags(assetId, options) {
4793
4968
  );
4794
4969
  }
4795
4970
  const transcriptResult = includeTranscript ? await fetchTranscriptForAsset(assetData, playbackId, {
4971
+ languageCode,
4796
4972
  cleanTranscript,
4797
4973
  shouldSign: policy === "signed",
4798
4974
  credentials: workflowCredentials,
@@ -4814,7 +4990,7 @@ async function getSummaryAndTags(assetId, options) {
4814
4990
  });
4815
4991
  let analysisResponse;
4816
4992
  let imageUrl;
4817
- const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT2 : SYSTEM_PROMPT5;
4993
+ const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT3 : SYSTEM_PROMPT5;
4818
4994
  try {
4819
4995
  if (isAudioOnly) {
4820
4996
  analysisResponse = await analyzeAudioOnly(
@@ -5111,7 +5287,8 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
5111
5287
  fromLanguageCode,
5112
5288
  numSpeakers = 0,
5113
5289
  // 0 = auto-detect
5114
- uploadToMux = true,
5290
+ uploadToS3: uploadToS3Option,
5291
+ uploadToMux: uploadToMuxOption,
5115
5292
  storageAdapter,
5116
5293
  credentials: providedCredentials
5117
5294
  } = options;
@@ -5120,13 +5297,15 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
5120
5297
  }
5121
5298
  const credentials = providedCredentials;
5122
5299
  const effectiveStorageAdapter = storageAdapter;
5300
+ const uploadToMux = uploadToMuxOption !== false;
5301
+ const uploadToS3 = uploadToS3Option || uploadToMux;
5123
5302
  const s3Endpoint = options.s3Endpoint ?? env_default.S3_ENDPOINT;
5124
5303
  const s3Region = options.s3Region ?? env_default.S3_REGION ?? "auto";
5125
5304
  const s3Bucket = options.s3Bucket ?? env_default.S3_BUCKET;
5126
5305
  const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
5127
5306
  const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
5128
- if (uploadToMux && (!s3Endpoint || !s3Bucket || !effectiveStorageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
5129
- throw new Error("Storage configuration is required for uploading to Mux. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.");
5307
+ if (uploadToS3 && (!s3Endpoint || !s3Bucket || !effectiveStorageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
5308
+ throw new Error("Storage configuration is required for uploading. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.");
5130
5309
  }
5131
5310
  const { asset: initialAsset, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
5132
5311
  const assetDurationSeconds = getAssetDurationSecondsFromAsset(initialAsset);
@@ -5203,70 +5382,65 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
5203
5382
  throw new Error(`Dubbing job timed out or failed. Final status: ${dubbingStatus}`);
5204
5383
  }
5205
5384
  console.warn("\u2705 Dubbing completed successfully!");
5206
- if (!uploadToMux) {
5207
- const targetLanguage2 = getLanguageCodePair(toLanguageCode);
5208
- return {
5209
- assetId,
5210
- targetLanguageCode: targetLanguage2.iso639_1,
5211
- targetLanguage: targetLanguage2,
5212
- dubbingId
5213
- };
5214
- }
5215
- console.warn("\u{1F4E5} Downloading dubbed audio from ElevenLabs...");
5216
- let dubbedAudioBuffer;
5217
- try {
5218
- const requestedLangCode = toISO639_3(toLanguageCode);
5219
- let downloadLangCode = targetLanguages.find(
5220
- (lang) => lang === requestedLangCode
5221
- ) ?? targetLanguages.find(
5222
- (lang) => lang.toLowerCase() === requestedLangCode.toLowerCase()
5223
- );
5224
- if (!downloadLangCode && targetLanguages.length > 0) {
5225
- downloadLangCode = targetLanguages[0];
5226
- console.warn(`\u26A0\uFE0F Requested language "${requestedLangCode}" not found in target_languages. Using "${downloadLangCode}" instead.`);
5227
- }
5228
- if (!downloadLangCode) {
5229
- downloadLangCode = requestedLangCode;
5230
- console.warn(`\u26A0\uFE0F No target_languages available from ElevenLabs status. Using requested language code: ${requestedLangCode}`);
5231
- }
5232
- dubbedAudioBuffer = await downloadDubbedAudioFromElevenLabs({
5233
- dubbingId,
5234
- languageCode: downloadLangCode,
5235
- credentials
5236
- });
5237
- console.warn("\u2705 Dubbed audio downloaded successfully!");
5238
- } catch (error) {
5239
- throw new Error(`Failed to download dubbed audio: ${error instanceof Error ? error.message : "Unknown error"}`);
5240
- }
5241
- console.warn("\u{1F4E4} Uploading dubbed audio to S3-compatible storage...");
5242
5385
  let presignedUrl;
5243
- try {
5244
- presignedUrl = await uploadDubbedAudioToS3({
5245
- dubbedAudioBuffer,
5246
- assetId,
5247
- toLanguageCode,
5248
- s3Endpoint,
5249
- s3Region,
5250
- s3Bucket,
5251
- storageAdapter: effectiveStorageAdapter,
5252
- s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
5253
- });
5254
- } catch (error) {
5255
- throw new Error(`Failed to upload audio to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
5256
- }
5257
- console.warn("\u{1F4F9} Adding dubbed audio track to Mux asset...");
5258
5386
  let uploadedTrackId;
5259
- const muxLangCode = toISO639_1(toLanguageCode);
5260
- try {
5261
- uploadedTrackId = await createAudioTrackOnMux(assetId, muxLangCode, presignedUrl, credentials);
5262
- const languageName = new Intl.DisplayNames(["en"], { type: "language" }).of(muxLangCode) || muxLangCode.toUpperCase();
5263
- const trackName = `${languageName} (auto-dubbed)`;
5264
- console.warn(`\u2705 Track added to Mux asset with ID: ${uploadedTrackId}`);
5265
- console.warn(`\u{1F4CB} Track name: "${trackName}"`);
5266
- } catch (error) {
5267
- console.warn(`\u26A0\uFE0F Failed to add audio track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
5268
- console.warn("\u{1F517} You can manually add the track using this presigned URL:");
5269
- console.warn(presignedUrl);
5387
+ if (uploadToS3) {
5388
+ console.warn("\u{1F4E5} Downloading dubbed audio from ElevenLabs...");
5389
+ let dubbedAudioBuffer;
5390
+ try {
5391
+ const requestedLangCode = toISO639_3(toLanguageCode);
5392
+ let downloadLangCode = targetLanguages.find(
5393
+ (lang) => lang === requestedLangCode
5394
+ ) ?? targetLanguages.find(
5395
+ (lang) => lang.toLowerCase() === requestedLangCode.toLowerCase()
5396
+ );
5397
+ if (!downloadLangCode && targetLanguages.length > 0) {
5398
+ downloadLangCode = targetLanguages[0];
5399
+ console.warn(`\u26A0\uFE0F Requested language "${requestedLangCode}" not found in target_languages. Using "${downloadLangCode}" instead.`);
5400
+ }
5401
+ if (!downloadLangCode) {
5402
+ downloadLangCode = requestedLangCode;
5403
+ console.warn(`\u26A0\uFE0F No target_languages available from ElevenLabs status. Using requested language code: ${requestedLangCode}`);
5404
+ }
5405
+ dubbedAudioBuffer = await downloadDubbedAudioFromElevenLabs({
5406
+ dubbingId,
5407
+ languageCode: downloadLangCode,
5408
+ credentials
5409
+ });
5410
+ console.warn("\u2705 Dubbed audio downloaded successfully!");
5411
+ } catch (error) {
5412
+ throw new Error(`Failed to download dubbed audio: ${error instanceof Error ? error.message : "Unknown error"}`);
5413
+ }
5414
+ console.warn("\u{1F4E4} Uploading dubbed audio to S3-compatible storage...");
5415
+ try {
5416
+ presignedUrl = await uploadDubbedAudioToS3({
5417
+ dubbedAudioBuffer,
5418
+ assetId,
5419
+ toLanguageCode,
5420
+ s3Endpoint,
5421
+ s3Region,
5422
+ s3Bucket,
5423
+ storageAdapter: effectiveStorageAdapter,
5424
+ s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
5425
+ });
5426
+ } catch (error) {
5427
+ throw new Error(`Failed to upload audio to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
5428
+ }
5429
+ if (uploadToMux) {
5430
+ console.warn("\u{1F4F9} Adding dubbed audio track to Mux asset...");
5431
+ const muxLangCode = toISO639_1(toLanguageCode);
5432
+ try {
5433
+ uploadedTrackId = await createAudioTrackOnMux(assetId, muxLangCode, presignedUrl, credentials);
5434
+ const languageName = new Intl.DisplayNames(["en"], { type: "language" }).of(muxLangCode) || muxLangCode.toUpperCase();
5435
+ const trackName = `${languageName} (auto-dubbed)`;
5436
+ console.warn(`\u2705 Track added to Mux asset with ID: ${uploadedTrackId}`);
5437
+ console.warn(`\u{1F4CB} Track name: "${trackName}"`);
5438
+ } catch (error) {
5439
+ console.warn(`\u26A0\uFE0F Failed to add audio track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
5440
+ console.warn("\u{1F517} You can manually add the track using this presigned URL:");
5441
+ console.warn(presignedUrl);
5442
+ }
5443
+ }
5270
5444
  }
5271
5445
  const targetLanguage = getLanguageCodePair(toLanguageCode);
5272
5446
  return {
@@ -5708,7 +5882,7 @@ async function uploadVttToS3({
5708
5882
  expiresInSeconds: s3SignedUrlExpirySeconds ?? 86400
5709
5883
  }, storageAdapter);
5710
5884
  }
5711
- async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, options) {
5885
+ async function translateCaptions(assetId, trackId, toLanguageCode, options) {
5712
5886
  "use workflow";
5713
5887
  const {
5714
5888
  provider = "openai",
@@ -5716,6 +5890,7 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
5716
5890
  s3Endpoint: providedS3Endpoint,
5717
5891
  s3Region: providedS3Region,
5718
5892
  s3Bucket: providedS3Bucket,
5893
+ uploadToS3: uploadToS3Option,
5719
5894
  uploadToMux: uploadToMuxOption,
5720
5895
  storageAdapter,
5721
5896
  credentials: providedCredentials,
@@ -5729,17 +5904,17 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
5729
5904
  const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
5730
5905
  const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
5731
5906
  const uploadToMux = uploadToMuxOption !== false;
5907
+ const uploadToS3 = uploadToS3Option || uploadToMux;
5732
5908
  const modelConfig = resolveLanguageModelConfig({
5733
5909
  ...options,
5734
5910
  model,
5735
5911
  provider
5736
5912
  });
5737
- if (uploadToMux && (!s3Endpoint || !s3Bucket || !effectiveStorageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
5738
- throw new Error("Storage configuration is required for uploading to Mux. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.");
5913
+ if (uploadToS3 && (!s3Endpoint || !s3Bucket || !effectiveStorageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
5914
+ throw new Error("Storage configuration is required for uploading. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.");
5739
5915
  }
5740
5916
  const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
5741
5917
  const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
5742
- const isAudioOnly = isAudioOnlyAsset(assetData);
5743
5918
  const signingContext = await resolveMuxSigningContext(credentials);
5744
5919
  if (policy === "signed" && !signingContext) {
5745
5920
  throw new Error(
@@ -5747,30 +5922,20 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
5747
5922
  );
5748
5923
  }
5749
5924
  const readyTextTracks = getReadyTextTracks(assetData);
5750
- if (!readyTextTracks.length) {
5751
- throw new Error("No ready text tracks found for this asset");
5752
- }
5753
- let sourceTextTrack = readyTextTracks.find(
5754
- (track) => track.text_type === "subtitles" && track.language_code === fromLanguageCode
5755
- );
5756
- if (!sourceTextTrack && isAudioOnly && readyTextTracks.length === 1) {
5757
- sourceTextTrack = readyTextTracks[0];
5758
- }
5925
+ const sourceTextTrack = readyTextTracks.find((t) => t.id === trackId);
5759
5926
  if (!sourceTextTrack) {
5760
- const availableLanguages = readyTextTracks.map((t) => t.language_code).filter(Boolean).join(", ");
5761
- if (isAudioOnly) {
5762
- throw new Error(
5763
- `No transcript track found with language code '${fromLanguageCode}' for this asset. Audio-only assets require a transcript. Available languages: ${availableLanguages || "none"}`
5764
- );
5765
- }
5927
+ const availableTrackIds = readyTextTracks.map((t) => t.id).filter(Boolean).join(", ");
5766
5928
  throw new Error(
5767
- `No ready text track found with language code '${fromLanguageCode}' for this asset. Available languages: ${availableLanguages || "none"}`
5929
+ `Track '${trackId}' not found or not ready on asset '${assetId}'. Available track IDs: ${availableTrackIds || "none"}`
5768
5930
  );
5769
5931
  }
5770
- if (!sourceTextTrack.id) {
5771
- throw new Error("Transcript track is missing an id");
5932
+ const fromLanguageCode = sourceTextTrack.language_code;
5933
+ if (!fromLanguageCode) {
5934
+ throw new Error(
5935
+ `Track '${trackId}' is missing language metadata (language_code). Cannot determine source language for translation.`
5936
+ );
5772
5937
  }
5773
- const vttUrl = await buildTranscriptUrl(playbackId, sourceTextTrack.id, policy === "signed", credentials);
5938
+ const vttUrl = await buildTranscriptUrl(playbackId, trackId, policy === "signed", credentials);
5774
5939
  let vttContent;
5775
5940
  try {
5776
5941
  vttContent = await fetchVttFromMux(vttUrl);
@@ -5803,50 +5968,43 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
5803
5968
  } : void 0;
5804
5969
  const sourceLanguage = getLanguageCodePair(fromLanguageCode);
5805
5970
  const targetLanguage = getLanguageCodePair(toLanguageCode);
5806
- if (!uploadToMux) {
5807
- return {
5808
- assetId,
5809
- sourceLanguageCode: fromLanguageCode,
5810
- targetLanguageCode: toLanguageCode,
5811
- sourceLanguage,
5812
- targetLanguage,
5813
- originalVtt: vttContent,
5814
- translatedVtt,
5815
- usage: usageWithMetadata
5816
- };
5817
- }
5818
5971
  let presignedUrl;
5819
- try {
5820
- presignedUrl = await uploadVttToS3({
5821
- translatedVtt,
5822
- assetId,
5823
- fromLanguageCode,
5824
- toLanguageCode,
5825
- s3Endpoint,
5826
- s3Region,
5827
- s3Bucket,
5828
- storageAdapter: effectiveStorageAdapter,
5829
- s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
5830
- });
5831
- } catch (error) {
5832
- throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
5833
- }
5834
5972
  let uploadedTrackId;
5835
- try {
5836
- const languageName = getLanguageName(toLanguageCode);
5837
- const trackName = `${languageName} (auto-translated)`;
5838
- uploadedTrackId = await createTextTrackOnMux(
5839
- assetId,
5840
- toLanguageCode,
5841
- trackName,
5842
- presignedUrl,
5843
- credentials
5844
- );
5845
- } catch (error) {
5846
- console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
5973
+ if (uploadToS3) {
5974
+ try {
5975
+ presignedUrl = await uploadVttToS3({
5976
+ translatedVtt,
5977
+ assetId,
5978
+ fromLanguageCode,
5979
+ toLanguageCode,
5980
+ s3Endpoint,
5981
+ s3Region,
5982
+ s3Bucket,
5983
+ storageAdapter: effectiveStorageAdapter,
5984
+ s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
5985
+ });
5986
+ } catch (error) {
5987
+ throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
5988
+ }
5989
+ if (uploadToMux) {
5990
+ try {
5991
+ const languageName = getLanguageName(toLanguageCode);
5992
+ const trackName = `${languageName} (auto-translated)`;
5993
+ uploadedTrackId = await createTextTrackOnMux(
5994
+ assetId,
5995
+ toLanguageCode,
5996
+ trackName,
5997
+ presignedUrl,
5998
+ credentials
5999
+ );
6000
+ } catch (error) {
6001
+ console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
6002
+ }
6003
+ }
5847
6004
  }
5848
6005
  return {
5849
6006
  assetId,
6007
+ trackId,
5850
6008
  sourceLanguageCode: fromLanguageCode,
5851
6009
  targetLanguageCode: toLanguageCode,
5852
6010
  sourceLanguage,