@mux/ai 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -5,7 +5,7 @@ var __export = (target, all) => {
5
5
  };
6
6
 
7
7
  // package.json
8
- var version = "0.13.0";
8
+ var version = "0.14.0";
9
9
 
10
10
  // src/env.ts
11
11
  import { z } from "zod";
@@ -1580,6 +1580,56 @@ async function getThumbnailUrls(playbackId, duration, options = {}) {
1580
1580
  return Promise.all(urlPromises);
1581
1581
  }
1582
1582
 
1583
+ // src/lib/mux-assets.ts
1584
+ function getPlaybackId(asset) {
1585
+ const playbackIds = asset.playback_ids || [];
1586
+ const publicPlaybackId = playbackIds.find((pid) => pid.policy === "public");
1587
+ if (publicPlaybackId?.id) {
1588
+ return { id: publicPlaybackId.id, policy: "public" };
1589
+ }
1590
+ const signedPlaybackId = playbackIds.find((pid) => pid.policy === "signed");
1591
+ if (signedPlaybackId?.id) {
1592
+ return { id: signedPlaybackId.id, policy: "signed" };
1593
+ }
1594
+ throw new Error(
1595
+ "No public or signed playback ID found for this asset. A public or signed playback ID is required. DRM playback IDs are not currently supported."
1596
+ );
1597
+ }
1598
+ function isAudioOnlyAsset(asset) {
1599
+ const hasAudioTrack = asset.tracks?.some((track) => track.type === "audio") ?? false;
1600
+ const hasVideoTrack = asset.tracks?.some((track) => track.type === "video") ?? false;
1601
+ return hasAudioTrack && !hasVideoTrack;
1602
+ }
1603
+ function toPlaybackAsset(asset) {
1604
+ const { id: playbackId, policy } = getPlaybackId(asset);
1605
+ return { asset, playbackId, policy };
1606
+ }
1607
+ async function getPlaybackIdForAsset(assetId, credentials) {
1608
+ "use step";
1609
+ const asset = await getMuxAsset(assetId, credentials);
1610
+ return toPlaybackAsset(asset);
1611
+ }
1612
+ async function getMuxAsset(assetId, credentials) {
1613
+ "use step";
1614
+ const muxClient = await getMuxClientFromEnv(credentials);
1615
+ const mux = await muxClient.createClient();
1616
+ return mux.video.assets.retrieve(assetId);
1617
+ }
1618
+ function getAssetDurationSecondsFromAsset(asset) {
1619
+ const duration = asset.duration;
1620
+ return typeof duration === "number" && Number.isFinite(duration) ? duration : void 0;
1621
+ }
1622
+ function getVideoTrackDurationSecondsFromAsset(asset) {
1623
+ const videoTrack = asset.tracks?.find((track) => track.type === "video");
1624
+ const duration = videoTrack?.duration;
1625
+ return typeof duration === "number" && Number.isFinite(duration) ? duration : void 0;
1626
+ }
1627
+ function getVideoTrackMaxFrameRateFromAsset(asset) {
1628
+ const videoTrack = asset.tracks?.find((track) => track.type === "video");
1629
+ const maxFrameRate = videoTrack?.max_frame_rate;
1630
+ return typeof maxFrameRate === "number" && Number.isFinite(maxFrameRate) && maxFrameRate > 0 ? maxFrameRate : void 0;
1631
+ }
1632
+
1583
1633
  // src/primitives/transcripts.ts
1584
1634
  function getReadyTextTracks(asset) {
1585
1635
  return (asset.tracks || []).filter(
@@ -1591,11 +1641,20 @@ function findCaptionTrack(asset, languageCode) {
1591
1641
  if (!tracks.length)
1592
1642
  return void 0;
1593
1643
  if (!languageCode) {
1594
- return tracks[0];
1644
+ const englishTrack = tracks.find(
1645
+ (track) => track.text_type === "subtitles" && track.language_code === "en"
1646
+ );
1647
+ return englishTrack ?? tracks[0];
1595
1648
  }
1596
- return tracks.find(
1649
+ const languageMatch = tracks.find(
1597
1650
  (track) => track.text_type === "subtitles" && track.language_code === languageCode
1598
1651
  );
1652
+ if (languageMatch)
1653
+ return languageMatch;
1654
+ if (isAudioOnlyAsset(asset) && tracks.length === 1) {
1655
+ return tracks[0];
1656
+ }
1657
+ return void 0;
1599
1658
  }
1600
1659
  function normalizeLineEndings(value) {
1601
1660
  return value.replace(/\r\n/g, "\n");
@@ -2071,56 +2130,6 @@ async function downloadImagesAsBase64(urls, options = {}, maxConcurrent = 5) {
2071
2130
  return results;
2072
2131
  }
2073
2132
 
2074
- // src/lib/mux-assets.ts
2075
- function getPlaybackId(asset) {
2076
- const playbackIds = asset.playback_ids || [];
2077
- const publicPlaybackId = playbackIds.find((pid) => pid.policy === "public");
2078
- if (publicPlaybackId?.id) {
2079
- return { id: publicPlaybackId.id, policy: "public" };
2080
- }
2081
- const signedPlaybackId = playbackIds.find((pid) => pid.policy === "signed");
2082
- if (signedPlaybackId?.id) {
2083
- return { id: signedPlaybackId.id, policy: "signed" };
2084
- }
2085
- throw new Error(
2086
- "No public or signed playback ID found for this asset. A public or signed playback ID is required. DRM playback IDs are not currently supported."
2087
- );
2088
- }
2089
- function isAudioOnlyAsset(asset) {
2090
- const hasAudioTrack = asset.tracks?.some((track) => track.type === "audio") ?? false;
2091
- const hasVideoTrack = asset.tracks?.some((track) => track.type === "video") ?? false;
2092
- return hasAudioTrack && !hasVideoTrack;
2093
- }
2094
- function toPlaybackAsset(asset) {
2095
- const { id: playbackId, policy } = getPlaybackId(asset);
2096
- return { asset, playbackId, policy };
2097
- }
2098
- async function getPlaybackIdForAsset(assetId, credentials) {
2099
- "use step";
2100
- const asset = await getMuxAsset(assetId, credentials);
2101
- return toPlaybackAsset(asset);
2102
- }
2103
- async function getMuxAsset(assetId, credentials) {
2104
- "use step";
2105
- const muxClient = await getMuxClientFromEnv(credentials);
2106
- const mux = await muxClient.createClient();
2107
- return mux.video.assets.retrieve(assetId);
2108
- }
2109
- function getAssetDurationSecondsFromAsset(asset) {
2110
- const duration = asset.duration;
2111
- return typeof duration === "number" && Number.isFinite(duration) ? duration : void 0;
2112
- }
2113
- function getVideoTrackDurationSecondsFromAsset(asset) {
2114
- const videoTrack = asset.tracks?.find((track) => track.type === "video");
2115
- const duration = videoTrack?.duration;
2116
- return typeof duration === "number" && Number.isFinite(duration) ? duration : void 0;
2117
- }
2118
- function getVideoTrackMaxFrameRateFromAsset(asset) {
2119
- const videoTrack = asset.tracks?.find((track) => track.type === "video");
2120
- const maxFrameRate = videoTrack?.max_frame_rate;
2121
- return typeof maxFrameRate === "number" && Number.isFinite(maxFrameRate) && maxFrameRate > 0 ? maxFrameRate : void 0;
2122
- }
2123
-
2124
2133
  // src/lib/prompt-builder.ts
2125
2134
  function renderSection(section) {
2126
2135
  const { tag, content, attributes } = section;
@@ -2352,19 +2361,102 @@ var SYSTEM_PROMPT = dedent`
2352
2361
  - GOOD: "A person runs through a park"
2353
2362
  - Be specific and evidence-based
2354
2363
  </language_guidelines>`;
2364
+ var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
2365
+ <role>
2366
+ You are an audio content analyst specializing in classification tasks.
2367
+ Your job is to answer questions about audio content based on transcript data.
2368
+ </role>
2369
+
2370
+ <context>
2371
+ You will receive:
2372
+ - A transcript of the audio/dialogue
2373
+ - A list of questions about the asset content
2374
+ </context>
2375
+
2376
+ <transcript_guidance>
2377
+ - Use the transcript to understand spoken content, dialogue, and audio context
2378
+ - Consider only transcript evidence when answering questions
2379
+ </transcript_guidance>
2380
+
2381
+ <task>
2382
+ For each question provided, you must:
2383
+ 1. Analyze the transcript
2384
+ 2. Answer with ONLY the allowed response options - no other values are acceptable
2385
+ 3. Provide a confidence score between 0 and 1 reflecting your certainty
2386
+ 4. Explain your reasoning based on observable evidence
2387
+ </task>
2388
+
2389
+ <answer_guidelines>
2390
+ - Choose the affirmative option only if you have clear evidence supporting it
2391
+ - Choose the negative/contradicting option if evidence contradicts or if insufficient evidence exists
2392
+ - Confidence should reflect the clarity and strength of evidence:
2393
+ * 0.9-1.0: Clear, unambiguous evidence
2394
+ * 0.7-0.9: Strong evidence with minor ambiguity
2395
+ * 0.5-0.7: Moderate evidence or some conflicting signals
2396
+ * 0.3-0.5: Weak evidence or significant ambiguity
2397
+ * 0.0-0.3: Very uncertain, minimal relevant evidence
2398
+ - Reasoning should cite specific transcript evidence
2399
+ - Be precise: cite specific quotes or passages from transcript text
2400
+ </answer_guidelines>
2401
+
2402
+ <relevance_filtering>
2403
+ Before answering each question, assess whether it can be meaningfully
2404
+ answered based on the transcript. A question is relevant if it asks about
2405
+ something observable or inferable from spoken/audio content.
2406
+
2407
+ Mark a question as skipped (skipped: true) if it:
2408
+ - Is completely unrelated to transcript/audio content (e.g., math, trivia, personal questions)
2409
+ - Asks about information that cannot be determined from transcript content
2410
+ - Is a general knowledge question with no connection to what is said in the transcript
2411
+ - Attempts to use the system for non-content-analysis purposes
2412
+
2413
+ For skipped questions:
2414
+ - Set skipped to true
2415
+ - Set answer to "${SKIP_SENTINEL}"
2416
+ - Set confidence to 0
2417
+ - Use the reasoning field to explain why the question is not answerable
2418
+ from transcript content
2419
+
2420
+ For borderline questions that are loosely related to transcript content,
2421
+ still answer them but use a lower confidence score to reflect uncertainty.
2422
+ </relevance_filtering>
2423
+
2424
+ <constraints>
2425
+ - You MUST answer every relevant question with one of the allowed response options
2426
+ - Skip irrelevant questions as described in relevance_filtering
2427
+ - Only describe observable evidence from transcript content
2428
+ - Do not fabricate details or make unsupported assumptions
2429
+ - Return structured data matching the requested schema exactly
2430
+ - Provide reasoning in the same language as the question
2431
+ </constraints>
2432
+
2433
+ <language_guidelines>
2434
+ When explaining reasoning:
2435
+ - Describe content directly, not the medium
2436
+ - BAD: "The audio says someone is running"
2437
+ - GOOD: "The speaker describes running through a park"
2438
+ - Be specific and evidence-based
2439
+ </language_guidelines>`;
2355
2440
  var askQuestionsPromptBuilder = createPromptBuilder({
2356
2441
  template: {
2357
2442
  questions: {
2358
2443
  tag: "questions",
2359
- content: "Please answer the following yes/no questions about this video:"
2444
+ content: "Please answer the following questions about this content:"
2360
2445
  }
2361
2446
  },
2362
2447
  sectionOrder: ["questions"]
2363
2448
  });
2364
- function buildUserPrompt(questions, allowedAnswers, transcriptText, isCleanTranscript = true) {
2449
+ function buildUserPrompt({
2450
+ questions,
2451
+ allowedAnswers,
2452
+ transcriptText,
2453
+ isCleanTranscript = true,
2454
+ isAudioOnly = false
2455
+ }) {
2365
2456
  const questionsList = questions.map((q, idx) => `${idx + 1}. ${q.question}`).join("\n");
2457
+ const questionsIntro = isAudioOnly ? "Please answer the following questions about this audio content using transcript evidence:" : "Please answer the following questions about this content:";
2366
2458
  const questionsContent = dedent`
2367
- Please answer the following yes/no questions about this video:
2459
+ ${questionsIntro}
2368
2460
 
2369
2461
  ${questionsList}`;
2370
2462
  const answerList = allowedAnswers.map((answer) => `"${answer}"`).join(", ");
@@ -2391,7 +2483,15 @@ async function fetchImageAsBase64(imageUrl, imageDownloadOptions) {
2391
2483
  const downloadResult = await downloadImageAsBase64(imageUrl, imageDownloadOptions);
2392
2484
  return downloadResult.base64Data;
2393
2485
  }
2394
- async function analyzeQuestionsWithStoryboard(imageDataUrl, provider, modelId, userPrompt, systemPrompt, allowedAnswers, credentials) {
2486
+ async function analyzeQuestions({
2487
+ provider,
2488
+ modelId,
2489
+ userPrompt,
2490
+ systemPrompt,
2491
+ allowedAnswers,
2492
+ imageDataUrl,
2493
+ credentials
2494
+ }) {
2395
2495
  "use step";
2396
2496
  const model = await createLanguageModelFromConfig(provider, modelId, credentials);
2397
2497
  const responseSchema = createAskQuestionsSchema(allowedAnswers);
@@ -2406,15 +2506,19 @@ async function analyzeQuestionsWithStoryboard(imageDataUrl, provider, modelId, u
2406
2506
  },
2407
2507
  {
2408
2508
  role: "user",
2409
- content: [
2509
+ content: imageDataUrl ? [
2410
2510
  { type: "text", text: userPrompt },
2411
2511
  { type: "image", image: imageDataUrl }
2412
- ]
2512
+ ] : userPrompt
2413
2513
  }
2414
2514
  ]
2415
2515
  });
2516
+ if (!response.output) {
2517
+ throw new Error("Ask-questions output missing");
2518
+ }
2519
+ const parsed = responseSchema.parse(response.output);
2416
2520
  return {
2417
- result: response.output,
2521
+ result: parsed,
2418
2522
  usage: {
2419
2523
  inputTokens: response.usage.inputTokens,
2420
2524
  outputTokens: response.usage.outputTokens,
@@ -2439,6 +2543,7 @@ async function askQuestions(assetId, questions, options) {
2439
2543
  const {
2440
2544
  provider = "openai",
2441
2545
  model,
2546
+ languageCode,
2442
2547
  answerOptions,
2443
2548
  includeTranscript = true,
2444
2549
  cleanTranscript = true,
@@ -2463,53 +2568,83 @@ async function askQuestions(assetId, questions, options) {
2463
2568
  });
2464
2569
  const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
2465
2570
  const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
2571
+ const isAudioOnly = isAudioOnlyAsset(assetData);
2572
+ if (isAudioOnly && !includeTranscript) {
2573
+ throw new Error(
2574
+ "Audio-only assets require a transcript. Set includeTranscript: true and ensure the asset has a ready text track (captions/subtitles)."
2575
+ );
2576
+ }
2466
2577
  const signingContext = await resolveMuxSigningContext(credentials);
2467
2578
  if (policy === "signed" && !signingContext) {
2468
2579
  throw new Error(
2469
2580
  "Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
2470
2581
  );
2471
2582
  }
2472
- const transcriptText = includeTranscript ? (await fetchTranscriptForAsset(assetData, playbackId, {
2583
+ const transcriptResult = includeTranscript ? await fetchTranscriptForAsset(assetData, playbackId, {
2584
+ languageCode,
2473
2585
  cleanTranscript,
2474
- shouldSign: policy === "signed"
2475
- })).transcriptText : "";
2476
- const userPrompt = buildUserPrompt(questions, allowedAnswers, transcriptText, cleanTranscript);
2477
- const systemPrompt = SYSTEM_PROMPT;
2478
- const imageUrl = await getStoryboardUrl(
2479
- playbackId,
2480
- storyboardWidth,
2481
- policy === "signed",
2482
- credentials
2483
- );
2586
+ shouldSign: policy === "signed",
2587
+ credentials,
2588
+ required: isAudioOnly
2589
+ }) : void 0;
2590
+ const transcriptText = transcriptResult?.transcriptText ?? "";
2591
+ const userPrompt = buildUserPrompt({
2592
+ questions,
2593
+ allowedAnswers,
2594
+ transcriptText,
2595
+ isCleanTranscript: cleanTranscript,
2596
+ isAudioOnly
2597
+ });
2598
+ const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT : SYSTEM_PROMPT;
2484
2599
  let analysisResponse;
2600
+ let imageUrl;
2485
2601
  try {
2486
- if (imageSubmissionMode === "base64") {
2487
- const base64Data = await fetchImageAsBase64(imageUrl, imageDownloadOptions);
2488
- analysisResponse = await analyzeQuestionsWithStoryboard(
2489
- base64Data,
2490
- modelConfig.provider,
2491
- modelConfig.modelId,
2602
+ if (isAudioOnly) {
2603
+ analysisResponse = await analyzeQuestions({
2604
+ provider: modelConfig.provider,
2605
+ modelId: modelConfig.modelId,
2492
2606
  userPrompt,
2493
2607
  systemPrompt,
2494
2608
  allowedAnswers,
2495
2609
  credentials
2496
- );
2610
+ });
2497
2611
  } else {
2498
- analysisResponse = await withRetry(
2499
- () => analyzeQuestionsWithStoryboard(
2500
- imageUrl,
2501
- modelConfig.provider,
2502
- modelConfig.modelId,
2612
+ const storyboardUrl = await getStoryboardUrl(
2613
+ playbackId,
2614
+ storyboardWidth,
2615
+ policy === "signed",
2616
+ credentials
2617
+ );
2618
+ imageUrl = storyboardUrl;
2619
+ if (imageSubmissionMode === "base64") {
2620
+ const base64Data = await fetchImageAsBase64(storyboardUrl, imageDownloadOptions);
2621
+ analysisResponse = await analyzeQuestions({
2622
+ provider: modelConfig.provider,
2623
+ modelId: modelConfig.modelId,
2503
2624
  userPrompt,
2504
2625
  systemPrompt,
2505
2626
  allowedAnswers,
2627
+ imageDataUrl: base64Data,
2506
2628
  credentials
2507
- )
2508
- );
2629
+ });
2630
+ } else {
2631
+ analysisResponse = await withRetry(
2632
+ () => analyzeQuestions({
2633
+ provider: modelConfig.provider,
2634
+ modelId: modelConfig.modelId,
2635
+ userPrompt,
2636
+ systemPrompt,
2637
+ allowedAnswers,
2638
+ imageDataUrl: storyboardUrl,
2639
+ credentials
2640
+ })
2641
+ );
2642
+ }
2509
2643
  }
2510
2644
  } catch (error) {
2645
+ const contentType = isAudioOnly ? "audio" : "video";
2511
2646
  throw new Error(
2512
- `Failed to analyze questions with ${provider}: ${error instanceof Error ? error.message : "Unknown error"}`
2647
+ `Failed to analyze ${contentType} questions with ${provider}: ${error instanceof Error ? error.message : "Unknown error"}`
2513
2648
  );
2514
2649
  }
2515
2650
  if (!analysisResponse.result?.answers) {
@@ -3049,9 +3184,10 @@ function buildUserPrompt3({
3049
3184
  };
3050
3185
  return chaptersPromptBuilder.buildWithContext(mergedOverrides, contextSections);
3051
3186
  }
3052
- async function generateChapters(assetId, languageCode, options = {}) {
3187
+ async function generateChapters(assetId, options = {}) {
3053
3188
  "use workflow";
3054
3189
  const {
3190
+ languageCode,
3055
3191
  provider = "openai",
3056
3192
  model,
3057
3193
  promptOverrides,
@@ -3075,26 +3211,17 @@ async function generateChapters(assetId, languageCode, options = {}) {
3075
3211
  );
3076
3212
  }
3077
3213
  const readyTextTracks = getReadyTextTracks(assetData);
3078
- let transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
3214
+ const transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
3079
3215
  languageCode,
3080
3216
  cleanTranscript: false,
3081
3217
  // keep timestamps for chapter segmentation
3082
3218
  shouldSign: policy === "signed",
3083
3219
  credentials
3084
3220
  });
3085
- if (isAudioOnly && !transcriptResult.track && readyTextTracks.length === 1) {
3086
- transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
3087
- cleanTranscript: false,
3088
- // keep timestamps for chapter segmentation
3089
- shouldSign: policy === "signed",
3090
- credentials,
3091
- required: true
3092
- });
3093
- }
3094
3221
  if (!transcriptResult.track || !transcriptResult.transcriptText) {
3095
3222
  const availableLanguages = readyTextTracks.map((t) => t.language_code).filter(Boolean).join(", ");
3096
3223
  throw new Error(
3097
- `No caption track found for language '${languageCode}'. Available languages: ${availableLanguages || "none"}`
3224
+ `No caption track found${languageCode ? ` for language '${languageCode}'` : ""}. Available languages: ${availableLanguages || "none"}`
3098
3225
  );
3099
3226
  }
3100
3227
  const timestampedTranscript = extractTimestampedTranscript(transcriptResult.transcriptText);
@@ -3146,7 +3273,7 @@ async function generateChapters(assetId, languageCode, options = {}) {
3146
3273
  };
3147
3274
  return {
3148
3275
  assetId,
3149
- languageCode,
3276
+ languageCode: languageCode ?? transcriptResult.track?.language_code ?? "en",
3150
3277
  chapters: validChapters,
3151
3278
  usage: usageWithMetadata
3152
3279
  };
@@ -3414,6 +3541,7 @@ async function editCaptions(assetId, trackId, options) {
3414
3541
  autoCensorProfanity: autoCensorOption,
3415
3542
  replacements: replacementsOption,
3416
3543
  deleteOriginalTrack,
3544
+ uploadToS3: uploadToS3Option,
3417
3545
  uploadToMux: uploadToMuxOption,
3418
3546
  s3Endpoint: providedS3Endpoint,
3419
3547
  s3Region: providedS3Region,
@@ -3432,14 +3560,15 @@ async function editCaptions(assetId, trackId, options) {
3432
3560
  }
3433
3561
  const deleteOriginal = deleteOriginalTrack !== false;
3434
3562
  const uploadToMux = uploadToMuxOption !== false;
3563
+ const uploadToS3 = uploadToS3Option || uploadToMux;
3435
3564
  const s3Endpoint = providedS3Endpoint ?? env_default.S3_ENDPOINT;
3436
3565
  const s3Region = providedS3Region ?? env_default.S3_REGION ?? "auto";
3437
3566
  const s3Bucket = providedS3Bucket ?? env_default.S3_BUCKET;
3438
3567
  const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
3439
3568
  const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
3440
- if (uploadToMux && (!s3Endpoint || !s3Bucket || !storageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
3569
+ if (uploadToS3 && (!s3Endpoint || !s3Bucket || !storageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
3441
3570
  throw new Error(
3442
- "Storage configuration is required for uploading to Mux. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables."
3571
+ "Storage configuration is required for uploading. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables."
3443
3572
  );
3444
3573
  }
3445
3574
  const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
@@ -3512,53 +3641,45 @@ async function editCaptions(assetId, trackId, options) {
3512
3641
  assetDurationSeconds
3513
3642
  }
3514
3643
  } : void 0;
3515
- if (!uploadToMux) {
3516
- return {
3517
- assetId,
3518
- trackId,
3519
- originalVtt: vttContent,
3520
- editedVtt,
3521
- totalReplacementCount,
3522
- autoCensorProfanity: autoCensorResult,
3523
- replacements: replacementsResult,
3524
- usage: usageWithMetadata
3525
- };
3526
- }
3527
3644
  let presignedUrl;
3528
- try {
3529
- presignedUrl = await uploadEditedVttToS3({
3530
- editedVtt,
3531
- assetId,
3532
- trackId,
3533
- s3Endpoint,
3534
- s3Region,
3535
- s3Bucket,
3536
- storageAdapter,
3537
- s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
3538
- });
3539
- } catch (error) {
3540
- throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
3541
- }
3542
3645
  let uploadedTrackId;
3543
- try {
3544
- const languageCode = sourceTrack.language_code || "en";
3545
- const suffix = trackNameSuffix ?? "edited";
3546
- const trackName = `${sourceTrack.name || "Subtitles"} (${suffix})`;
3547
- uploadedTrackId = await createTextTrackOnMux(
3548
- assetId,
3549
- languageCode,
3550
- trackName,
3551
- presignedUrl,
3552
- credentials
3553
- );
3554
- } catch (error) {
3555
- console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
3556
- }
3557
- if (deleteOriginal && uploadedTrackId) {
3646
+ if (uploadToS3) {
3558
3647
  try {
3559
- await deleteTrackOnMux(assetId, trackId, credentials);
3648
+ presignedUrl = await uploadEditedVttToS3({
3649
+ editedVtt,
3650
+ assetId,
3651
+ trackId,
3652
+ s3Endpoint,
3653
+ s3Region,
3654
+ s3Bucket,
3655
+ storageAdapter,
3656
+ s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
3657
+ });
3560
3658
  } catch (error) {
3561
- console.warn(`Failed to delete original track: ${error instanceof Error ? error.message : "Unknown error"}`);
3659
+ throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
3660
+ }
3661
+ if (uploadToMux) {
3662
+ try {
3663
+ const languageCode = sourceTrack.language_code || "en";
3664
+ const suffix = trackNameSuffix ?? "edited";
3665
+ const trackName = `${sourceTrack.name || "Subtitles"} (${suffix})`;
3666
+ uploadedTrackId = await createTextTrackOnMux(
3667
+ assetId,
3668
+ languageCode,
3669
+ trackName,
3670
+ presignedUrl,
3671
+ credentials
3672
+ );
3673
+ } catch (error) {
3674
+ console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
3675
+ }
3676
+ if (deleteOriginal && uploadedTrackId) {
3677
+ try {
3678
+ await deleteTrackOnMux(assetId, trackId, credentials);
3679
+ } catch (error) {
3680
+ console.warn(`Failed to delete original track: ${error instanceof Error ? error.message : "Unknown error"}`);
3681
+ }
3682
+ }
3562
3683
  }
3563
3684
  }
3564
3685
  return {
@@ -3629,43 +3750,21 @@ async function generateEmbeddingsInternal(assetId, options = {}) {
3629
3750
  const embeddingModel = resolveEmbeddingModelConfig({ ...options, provider, model });
3630
3751
  const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
3631
3752
  const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
3632
- const isAudioOnly = isAudioOnlyAsset(assetData);
3633
3753
  const signingContext = await resolveMuxSigningContext(credentials);
3634
3754
  if (policy === "signed" && !signingContext) {
3635
3755
  throw new Error(
3636
3756
  "Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
3637
3757
  );
3638
3758
  }
3639
- const readyTextTracks = getReadyTextTracks(assetData);
3640
3759
  const useVttChunking = chunkingStrategy.type === "vtt";
3641
- let transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
3760
+ const transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
3642
3761
  languageCode,
3643
3762
  cleanTranscript: !useVttChunking,
3644
3763
  shouldSign: policy === "signed",
3645
- credentials
3764
+ credentials,
3765
+ required: true
3646
3766
  });
3647
- if (isAudioOnly && !transcriptResult.track && readyTextTracks.length === 1) {
3648
- transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
3649
- cleanTranscript: !useVttChunking,
3650
- shouldSign: policy === "signed",
3651
- credentials
3652
- });
3653
- }
3654
- if (!transcriptResult.track || !transcriptResult.transcriptText) {
3655
- const availableLanguages = readyTextTracks.map((t) => t.language_code).filter(Boolean).join(", ");
3656
- if (isAudioOnly) {
3657
- throw new Error(
3658
- `No transcript track found${languageCode ? ` for language '${languageCode}'` : ""}. Audio-only assets require a transcript. Available languages: ${availableLanguages || "none"}`
3659
- );
3660
- }
3661
- throw new Error(
3662
- `No caption track found${languageCode ? ` for language '${languageCode}'` : ""}. Available languages: ${availableLanguages || "none"}`
3663
- );
3664
- }
3665
3767
  const transcriptText = transcriptResult.transcriptText;
3666
- if (!transcriptText.trim()) {
3667
- throw new Error("Transcript is empty");
3668
- }
3669
3768
  const chunks = useVttChunking ? chunkVTTCues(
3670
3769
  parseVTTCues(transcriptText),
3671
3770
  chunkingStrategy.maxTokens,
@@ -3791,7 +3890,7 @@ var SYSTEM_PROMPT4 = dedent5`
3791
3890
  - Connect engagement data to content features
3792
3891
  </quality_guidelines>
3793
3892
  `;
3794
- var AUDIO_ONLY_SYSTEM_PROMPT = dedent5`
3893
+ var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent5`
3795
3894
  <role>
3796
3895
  You are a video engagement analyst specializing in understanding viewer behavior for audio content.
3797
3896
  Your job is to explain why specific moments have high or low engagement based on audio/dialogue and viewer watch patterns.
@@ -4079,9 +4178,14 @@ async function generateEngagementInsights(assetId, options = {}) {
4079
4178
  }
4080
4179
  const { hotspots, heatmap } = engagementResult.value;
4081
4180
  if (hotspots.length === 0) {
4082
- throw new Error(
4083
- `No engagement data available for asset ${assetId} in timeframe ${timeframe}. Video may not have been viewed yet or engagement tracking is not enabled.`
4084
- );
4181
+ return {
4182
+ assetId,
4183
+ momentInsights: [],
4184
+ overallInsight: {
4185
+ summary: `No engagement data available for asset ${assetId} in timeframe ${timeframe}. Video may not have been viewed yet, not contain notable engagement patterns, or engagement tracking is not enabled.`,
4186
+ trends: []
4187
+ }
4188
+ };
4085
4189
  }
4086
4190
  let transcriptText = "";
4087
4191
  if (transcriptResult.status === "fulfilled") {
@@ -4124,7 +4228,7 @@ async function generateEngagementInsights(assetId, options = {}) {
4124
4228
  imageUrls.push(storyboardResult.value);
4125
4229
  }
4126
4230
  }
4127
- const systemPrompt = audioOnly ? AUDIO_ONLY_SYSTEM_PROMPT : SYSTEM_PROMPT4;
4231
+ const systemPrompt = audioOnly ? AUDIO_ONLY_SYSTEM_PROMPT2 : SYSTEM_PROMPT4;
4128
4232
  const userPrompt = buildUserPrompt4(
4129
4233
  hotspots,
4130
4234
  transcriptSegments,
@@ -4249,6 +4353,12 @@ var DEFAULT_THRESHOLDS = {
4249
4353
  violence: 0.8
4250
4354
  };
4251
4355
  var DEFAULT_PROVIDER2 = "openai";
4356
+ var OPENAI_MODERATION_RETRIABLE_STATUS_CODES = /* @__PURE__ */ new Set([408, 429, 500, 502, 503, 504]);
4357
+ var OPENAI_MODERATION_MAX_RETRIES = 2;
4358
+ var OPENAI_MODERATION_BASE_DELAY_MS = 750;
4359
+ var OPENAI_MODERATION_MAX_DELAY_MS = 3e3;
4360
+ var MIN_SAMPLE_COVERAGE_FOR_CONFIDENT_THRESHOLDING = 0.5;
4361
+ var MIN_SUCCESSFUL_THUMBNAILS_FOR_CONFIDENT_THRESHOLDING = 3;
4252
4362
  var HIVE_ENDPOINT = "https://api.thehive.ai/api/v2/task/sync";
4253
4363
  var HIVE_SEXUAL_CATEGORIES = [
4254
4364
  "general_nsfw",
@@ -4270,6 +4380,92 @@ var HIVE_VIOLENCE_CATEGORIES = [
4270
4380
  "yes_self_harm",
4271
4381
  "garm_death_injury_or_military_conflict"
4272
4382
  ];
4383
+ var OpenAIModerationRequestError = class extends Error {
4384
+ constructor(message, {
4385
+ status,
4386
+ retriable
4387
+ }) {
4388
+ super(message);
4389
+ this.name = "OpenAIModerationRequestError";
4390
+ this.status = status;
4391
+ this.retriable = retriable;
4392
+ }
4393
+ };
4394
+ function isRetriableOpenAIModerationStatus(status) {
4395
+ return OPENAI_MODERATION_RETRIABLE_STATUS_CODES.has(status);
4396
+ }
4397
+ function compactBodySnippet(body, maxLength = 180) {
4398
+ const compact = body.replace(/\s+/g, " ").trim();
4399
+ if (!compact) {
4400
+ return "(empty response body)";
4401
+ }
4402
+ return compact.length > maxLength ? `${compact.slice(0, maxLength)}...` : compact;
4403
+ }
4404
+ function parseOpenAIModerationResponse(bodyText, status, statusText) {
4405
+ if (!bodyText.trim()) {
4406
+ throw new OpenAIModerationRequestError(
4407
+ `OpenAI moderation returned an empty response (${status} ${statusText}).`,
4408
+ { status, retriable: isRetriableOpenAIModerationStatus(status) }
4409
+ );
4410
+ }
4411
+ try {
4412
+ return JSON.parse(bodyText);
4413
+ } catch {
4414
+ throw new OpenAIModerationRequestError(
4415
+ `OpenAI moderation returned non-JSON response (${status} ${statusText}): ${compactBodySnippet(bodyText)}`,
4416
+ { status, retriable: true }
4417
+ );
4418
+ }
4419
+ }
4420
+ async function callOpenAIModerationApi({
4421
+ model,
4422
+ input,
4423
+ credentials
4424
+ }) {
4425
+ "use step";
4426
+ const apiKey = await getApiKeyFromEnv("openai", credentials);
4427
+ return withRetry(
4428
+ async () => {
4429
+ let res;
4430
+ try {
4431
+ res = await fetch("https://api.openai.com/v1/moderations", {
4432
+ method: "POST",
4433
+ headers: {
4434
+ "Content-Type": "application/json",
4435
+ "Authorization": `Bearer ${apiKey}`
4436
+ },
4437
+ body: JSON.stringify({
4438
+ model,
4439
+ input
4440
+ })
4441
+ });
4442
+ } catch (error) {
4443
+ throw new OpenAIModerationRequestError(
4444
+ `OpenAI moderation request failed: ${error instanceof Error ? error.message : String(error)}`,
4445
+ { retriable: true }
4446
+ );
4447
+ }
4448
+ const bodyText = await res.text();
4449
+ const json = parseOpenAIModerationResponse(bodyText, res.status, res.statusText);
4450
+ if (!res.ok) {
4451
+ throw new OpenAIModerationRequestError(
4452
+ `OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`,
4453
+ {
4454
+ status: res.status,
4455
+ retriable: isRetriableOpenAIModerationStatus(res.status)
4456
+ }
4457
+ );
4458
+ }
4459
+ return json;
4460
+ },
4461
+ {
4462
+ maxRetries: OPENAI_MODERATION_MAX_RETRIES,
4463
+ baseDelay: OPENAI_MODERATION_BASE_DELAY_MS,
4464
+ maxDelay: OPENAI_MODERATION_MAX_DELAY_MS,
4465
+ shouldRetry: (error) => error instanceof OpenAIModerationRequestError ? error.retriable : true
4466
+ }
4467
+ );
4468
+ }
4273
4469
  async function processConcurrently(items, processor, maxConcurrent = 5) {
4274
4470
  "use step";
4275
4471
  const results = [];
@@ -4283,32 +4479,19 @@ async function processConcurrently(items, processor, maxConcurrent = 5) {
4283
4479
  }
4284
4480
  async function moderateImageWithOpenAI(entry) {
4285
4481
  "use step";
4286
- const apiKey = await getApiKeyFromEnv("openai", entry.credentials);
4287
4482
  try {
4288
- const res = await fetch("https://api.openai.com/v1/moderations", {
4289
- method: "POST",
4290
- headers: {
4291
- "Content-Type": "application/json",
4292
- "Authorization": `Bearer ${apiKey}`
4293
- },
4294
- body: JSON.stringify({
4295
- model: entry.model,
4296
- input: [
4297
- {
4298
- type: "image_url",
4299
- image_url: {
4300
- url: entry.image
4301
- }
4483
+ const json = await callOpenAIModerationApi({
4484
+ model: entry.model,
4485
+ input: [
4486
+ {
4487
+ type: "image_url",
4488
+ image_url: {
4489
+ url: entry.image
4302
4490
  }
4303
- ]
4304
- })
4491
+ }
4492
+ ],
4493
+ credentials: entry.credentials
4305
4494
  });
4306
- const json = await res.json();
4307
- if (!res.ok) {
4308
- throw new Error(
4309
- `OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`
4310
- );
4311
- }
4312
4495
  const categoryScores = json.results?.[0]?.category_scores || {};
4313
4496
  return {
4314
4497
  url: entry.url,
@@ -4340,25 +4523,12 @@ async function requestOpenAIModeration(images, model, maxConcurrent = 5, submiss
4340
4523
  }
4341
4524
  async function requestOpenAITextModeration(text, model, url, credentials) {
4342
4525
  "use step";
4343
- const apiKey = await getApiKeyFromEnv("openai", credentials);
4344
4526
  try {
4345
- const res = await fetch("https://api.openai.com/v1/moderations", {
4346
- method: "POST",
4347
- headers: {
4348
- "Content-Type": "application/json",
4349
- "Authorization": `Bearer ${apiKey}`
4350
- },
4351
- body: JSON.stringify({
4352
- model,
4353
- input: text
4354
- })
4527
+ const json = await callOpenAIModerationApi({
4528
+ model,
4529
+ input: text,
4530
+ credentials
4355
4531
  });
4356
- const json = await res.json();
4357
- if (!res.ok) {
4358
- throw new Error(
4359
- `OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`
4360
- );
4361
- }
4362
4532
  const categoryScores = json.results?.[0]?.category_scores || {};
4363
4533
  return {
4364
4534
  url,
@@ -4564,22 +4734,13 @@ async function getModerationScores(assetId, options = {}) {
4564
4734
  let thumbnailCount;
4565
4735
  if (isAudioOnly) {
4566
4736
  mode = "transcript";
4567
- const readyTextTracks = getReadyTextTracks(asset);
4568
- let transcriptResult = await fetchTranscriptForAsset(asset, playbackId, {
4737
+ const transcriptResult = await fetchTranscriptForAsset(asset, playbackId, {
4569
4738
  languageCode,
4570
4739
  cleanTranscript: true,
4571
4740
  shouldSign: policy === "signed",
4572
4741
  credentials,
4573
4742
  required: true
4574
4743
  });
4575
- if (!transcriptResult.track && readyTextTracks.length === 1) {
4576
- transcriptResult = await fetchTranscriptForAsset(asset, playbackId, {
4577
- cleanTranscript: true,
4578
- shouldSign: policy === "signed",
4579
- credentials,
4580
- required: true
4581
- });
4582
- }
4583
4744
  if (provider === "openai") {
4584
4745
  thumbnailScores = await requestOpenAITranscriptModeration(
4585
4746
  transcriptResult.transcriptText,
@@ -4593,35 +4754,28 @@ async function getModerationScores(assetId, options = {}) {
4593
4754
  throw new Error(`Unsupported moderation provider: ${provider}`);
4594
4755
  }
4595
4756
  } else {
4596
- const thumbnailUrls = maxSamples === void 0 ? (
4597
- // Generate thumbnail URLs (signed if needed) using existing interval-based logic.
4598
- await getThumbnailUrls(playbackId, duration, {
4599
- interval: thumbnailInterval,
4757
+ const estimatedIntervalCount = duration <= 50 ? 5 : Math.ceil(duration / thumbnailInterval);
4758
+ const thumbnailUrls = maxSamples !== void 0 && estimatedIntervalCount > maxSamples ? await getThumbnailUrlsFromTimestamps(
4759
+ playbackId,
4760
+ planSamplingTimestamps({
4761
+ duration_sec: duration,
4762
+ max_candidates: maxSamples,
4763
+ trim_start_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
4764
+ trim_end_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
4765
+ fps: videoTrackFps,
4766
+ base_cadence_hz: thumbnailInterval > 0 ? 1 / thumbnailInterval : void 0
4767
+ }),
4768
+ {
4600
4769
  width: thumbnailWidth,
4601
4770
  shouldSign: policy === "signed",
4602
4771
  credentials
4603
- })
4604
- ) : (
4605
- // In maxSamples mode, sample valid timestamps over the trimmed usable span.
4606
- // Use proportional trims (≈ duration/6, capped at 5s) to stay well inside the
4607
- // renderable range — Mux can't always serve thumbnails at the very edges.
4608
- await getThumbnailUrlsFromTimestamps(
4609
- playbackId,
4610
- planSamplingTimestamps({
4611
- duration_sec: duration,
4612
- max_candidates: maxSamples,
4613
- trim_start_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
4614
- trim_end_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
4615
- fps: videoTrackFps,
4616
- base_cadence_hz: thumbnailInterval > 0 ? 1 / thumbnailInterval : void 0
4617
- }),
4618
- {
4619
- width: thumbnailWidth,
4620
- shouldSign: policy === "signed",
4621
- credentials
4622
- }
4623
- )
4624
- );
4772
+ }
4773
+ ) : await getThumbnailUrls(playbackId, duration, {
4774
+ interval: thumbnailInterval,
4775
+ width: thumbnailWidth,
4776
+ shouldSign: policy === "signed",
4777
+ credentials
4778
+ });
4625
4779
  thumbnailCount = thumbnailUrls.length;
4626
4780
  if (provider === "openai") {
4627
4781
  thumbnailScores = await requestOpenAIModeration(
@@ -4645,20 +4799,40 @@ async function getModerationScores(assetId, options = {}) {
4645
4799
  }
4646
4800
  }
4647
4801
  const failed = thumbnailScores.filter((s) => s.error);
4648
- if (failed.length > 0) {
4802
+ const successful = thumbnailScores.filter((s) => !s.error);
4803
+ if (successful.length === 0) {
4649
4804
  const details = failed.map((s) => `${s.url}: ${s.errorMessage || "Unknown error"}`).join("; ");
4650
4805
  throw new Error(
4651
- `Moderation failed for ${failed.length}/${thumbnailScores.length} thumbnail(s): ${details}`
4806
+ `Moderation failed for all ${thumbnailScores.length} sample(s): ${details}`
4807
+ );
4808
+ }
4809
+ if (failed.length > 0) {
4810
+ console.warn(
4811
+ `Moderation had partial failures (${failed.length}/${thumbnailScores.length}); continuing with successful samples.`
4652
4812
  );
4653
4813
  }
4654
- const maxSexual = Math.max(...thumbnailScores.map((s) => s.sexual));
4655
- const maxViolence = Math.max(...thumbnailScores.map((s) => s.violence));
4814
+ const maxSexual = Math.max(...successful.map((s) => s.sexual));
4815
+ const maxViolence = Math.max(...successful.map((s) => s.violence));
4656
4816
  const finalThresholds = { ...DEFAULT_THRESHOLDS, ...thresholds };
4817
+ const requestedSampleCount = thumbnailScores.length;
4818
+ const successfulSampleCount = successful.length;
4819
+ const failedSampleCount = failed.length;
4820
+ const sampleCoverage = successfulSampleCount / requestedSampleCount;
4821
+ const hasEnoughSuccessfulSamples = mode === "transcript" || successfulSampleCount >= MIN_SUCCESSFUL_THUMBNAILS_FOR_CONFIDENT_THRESHOLDING;
4822
+ const isLowConfidence = sampleCoverage < MIN_SAMPLE_COVERAGE_FOR_CONFIDENT_THRESHOLDING || !hasEnoughSuccessfulSamples;
4657
4823
  return {
4658
4824
  assetId,
4659
4825
  mode,
4660
4826
  isAudioOnly,
4661
4827
  thumbnailScores,
4828
+ coverage: {
4829
+ requestedSampleCount,
4830
+ successfulSampleCount,
4831
+ failedSampleCount,
4832
+ sampleCoverage,
4833
+ isPartial: failedSampleCount > 0,
4834
+ isLowConfidence
4835
+ },
4662
4836
  usage: {
4663
4837
  metadata: {
4664
4838
  assetDurationSeconds: duration,
@@ -4886,7 +5060,7 @@ var SYSTEM_PROMPT5 = dedent6`
4886
5060
 
4887
5061
  Write as if describing reality, not describing a recording of reality.
4888
5062
  </language_guidelines>`;
4889
- var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent6`
5063
+ var AUDIO_ONLY_SYSTEM_PROMPT3 = dedent6`
4890
5064
  <role>
4891
5065
  You are an audio content analyst specializing in transcript analysis and metadata generation.
4892
5066
  </role>
@@ -5063,6 +5237,7 @@ async function getSummaryAndTags(assetId, options) {
5063
5237
  const {
5064
5238
  provider = "openai",
5065
5239
  model,
5240
+ languageCode,
5066
5241
  tone = "neutral",
5067
5242
  includeTranscript = true,
5068
5243
  cleanTranscript = true,
@@ -5101,6 +5276,7 @@ async function getSummaryAndTags(assetId, options) {
5101
5276
  );
5102
5277
  }
5103
5278
  const transcriptResult = includeTranscript ? await fetchTranscriptForAsset(assetData, playbackId, {
5279
+ languageCode,
5104
5280
  cleanTranscript,
5105
5281
  shouldSign: policy === "signed",
5106
5282
  credentials: workflowCredentials,
@@ -5122,7 +5298,7 @@ async function getSummaryAndTags(assetId, options) {
5122
5298
  });
5123
5299
  let analysisResponse;
5124
5300
  let imageUrl;
5125
- const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT2 : SYSTEM_PROMPT5;
5301
+ const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT3 : SYSTEM_PROMPT5;
5126
5302
  try {
5127
5303
  if (isAudioOnly) {
5128
5304
  analysisResponse = await analyzeAudioOnly(
@@ -5419,7 +5595,8 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
5419
5595
  fromLanguageCode,
5420
5596
  numSpeakers = 0,
5421
5597
  // 0 = auto-detect
5422
- uploadToMux = true,
5598
+ uploadToS3: uploadToS3Option,
5599
+ uploadToMux: uploadToMuxOption,
5423
5600
  storageAdapter,
5424
5601
  credentials: providedCredentials
5425
5602
  } = options;
@@ -5428,13 +5605,15 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
5428
5605
  }
5429
5606
  const credentials = providedCredentials;
5430
5607
  const effectiveStorageAdapter = storageAdapter;
5608
+ const uploadToMux = uploadToMuxOption !== false;
5609
+ const uploadToS3 = uploadToS3Option || uploadToMux;
5431
5610
  const s3Endpoint = options.s3Endpoint ?? env_default.S3_ENDPOINT;
5432
5611
  const s3Region = options.s3Region ?? env_default.S3_REGION ?? "auto";
5433
5612
  const s3Bucket = options.s3Bucket ?? env_default.S3_BUCKET;
5434
5613
  const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
5435
5614
  const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
5436
- if (uploadToMux && (!s3Endpoint || !s3Bucket || !effectiveStorageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
5437
- throw new Error("Storage configuration is required for uploading to Mux. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.");
5615
+ if (uploadToS3 && (!s3Endpoint || !s3Bucket || !effectiveStorageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
5616
+ throw new Error("Storage configuration is required for uploading. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.");
5438
5617
  }
5439
5618
  const { asset: initialAsset, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
5440
5619
  const assetDurationSeconds = getAssetDurationSecondsFromAsset(initialAsset);
@@ -5511,70 +5690,65 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
5511
5690
  throw new Error(`Dubbing job timed out or failed. Final status: ${dubbingStatus}`);
5512
5691
  }
5513
5692
  console.warn("\u2705 Dubbing completed successfully!");
5514
- if (!uploadToMux) {
5515
- const targetLanguage2 = getLanguageCodePair(toLanguageCode);
5516
- return {
5517
- assetId,
5518
- targetLanguageCode: targetLanguage2.iso639_1,
5519
- targetLanguage: targetLanguage2,
5520
- dubbingId
5521
- };
5522
- }
5523
- console.warn("\u{1F4E5} Downloading dubbed audio from ElevenLabs...");
5524
- let dubbedAudioBuffer;
5525
- try {
5526
- const requestedLangCode = toISO639_3(toLanguageCode);
5527
- let downloadLangCode = targetLanguages.find(
5528
- (lang) => lang === requestedLangCode
5529
- ) ?? targetLanguages.find(
5530
- (lang) => lang.toLowerCase() === requestedLangCode.toLowerCase()
5531
- );
5532
- if (!downloadLangCode && targetLanguages.length > 0) {
5533
- downloadLangCode = targetLanguages[0];
5534
- console.warn(`\u26A0\uFE0F Requested language "${requestedLangCode}" not found in target_languages. Using "${downloadLangCode}" instead.`);
5535
- }
5536
- if (!downloadLangCode) {
5537
- downloadLangCode = requestedLangCode;
5538
- console.warn(`\u26A0\uFE0F No target_languages available from ElevenLabs status. Using requested language code: ${requestedLangCode}`);
5539
- }
5540
- dubbedAudioBuffer = await downloadDubbedAudioFromElevenLabs({
5541
- dubbingId,
5542
- languageCode: downloadLangCode,
5543
- credentials
5544
- });
5545
- console.warn("\u2705 Dubbed audio downloaded successfully!");
5546
- } catch (error) {
5547
- throw new Error(`Failed to download dubbed audio: ${error instanceof Error ? error.message : "Unknown error"}`);
5548
- }
5549
- console.warn("\u{1F4E4} Uploading dubbed audio to S3-compatible storage...");
5550
5693
  let presignedUrl;
5551
- try {
5552
- presignedUrl = await uploadDubbedAudioToS3({
5553
- dubbedAudioBuffer,
5554
- assetId,
5555
- toLanguageCode,
5556
- s3Endpoint,
5557
- s3Region,
5558
- s3Bucket,
5559
- storageAdapter: effectiveStorageAdapter,
5560
- s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
5561
- });
5562
- } catch (error) {
5563
- throw new Error(`Failed to upload audio to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
5564
- }
5565
- console.warn("\u{1F4F9} Adding dubbed audio track to Mux asset...");
5566
5694
  let uploadedTrackId;
5567
- const muxLangCode = toISO639_1(toLanguageCode);
5568
- try {
5569
- uploadedTrackId = await createAudioTrackOnMux(assetId, muxLangCode, presignedUrl, credentials);
5570
- const languageName = new Intl.DisplayNames(["en"], { type: "language" }).of(muxLangCode) || muxLangCode.toUpperCase();
5571
- const trackName = `${languageName} (auto-dubbed)`;
5572
- console.warn(`\u2705 Track added to Mux asset with ID: ${uploadedTrackId}`);
5573
- console.warn(`\u{1F4CB} Track name: "${trackName}"`);
5574
- } catch (error) {
5575
- console.warn(`\u26A0\uFE0F Failed to add audio track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
5576
- console.warn("\u{1F517} You can manually add the track using this presigned URL:");
5577
- console.warn(presignedUrl);
5695
+ if (uploadToS3) {
5696
+ console.warn("\u{1F4E5} Downloading dubbed audio from ElevenLabs...");
5697
+ let dubbedAudioBuffer;
5698
+ try {
5699
+ const requestedLangCode = toISO639_3(toLanguageCode);
5700
+ let downloadLangCode = targetLanguages.find(
5701
+ (lang) => lang === requestedLangCode
5702
+ ) ?? targetLanguages.find(
5703
+ (lang) => lang.toLowerCase() === requestedLangCode.toLowerCase()
5704
+ );
5705
+ if (!downloadLangCode && targetLanguages.length > 0) {
5706
+ downloadLangCode = targetLanguages[0];
5707
+ console.warn(`\u26A0\uFE0F Requested language "${requestedLangCode}" not found in target_languages. Using "${downloadLangCode}" instead.`);
5708
+ }
5709
+ if (!downloadLangCode) {
5710
+ downloadLangCode = requestedLangCode;
5711
+ console.warn(`\u26A0\uFE0F No target_languages available from ElevenLabs status. Using requested language code: ${requestedLangCode}`);
5712
+ }
5713
+ dubbedAudioBuffer = await downloadDubbedAudioFromElevenLabs({
5714
+ dubbingId,
5715
+ languageCode: downloadLangCode,
5716
+ credentials
5717
+ });
5718
+ console.warn("\u2705 Dubbed audio downloaded successfully!");
5719
+ } catch (error) {
5720
+ throw new Error(`Failed to download dubbed audio: ${error instanceof Error ? error.message : "Unknown error"}`);
5721
+ }
5722
+ console.warn("\u{1F4E4} Uploading dubbed audio to S3-compatible storage...");
5723
+ try {
5724
+ presignedUrl = await uploadDubbedAudioToS3({
5725
+ dubbedAudioBuffer,
5726
+ assetId,
5727
+ toLanguageCode,
5728
+ s3Endpoint,
5729
+ s3Region,
5730
+ s3Bucket,
5731
+ storageAdapter: effectiveStorageAdapter,
5732
+ s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
5733
+ });
5734
+ } catch (error) {
5735
+ throw new Error(`Failed to upload audio to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
5736
+ }
5737
+ if (uploadToMux) {
5738
+ console.warn("\u{1F4F9} Adding dubbed audio track to Mux asset...");
5739
+ const muxLangCode = toISO639_1(toLanguageCode);
5740
+ try {
5741
+ uploadedTrackId = await createAudioTrackOnMux(assetId, muxLangCode, presignedUrl, credentials);
5742
+ const languageName = new Intl.DisplayNames(["en"], { type: "language" }).of(muxLangCode) || muxLangCode.toUpperCase();
5743
+ const trackName = `${languageName} (auto-dubbed)`;
5744
+ console.warn(`\u2705 Track added to Mux asset with ID: ${uploadedTrackId}`);
5745
+ console.warn(`\u{1F4CB} Track name: "${trackName}"`);
5746
+ } catch (error) {
5747
+ console.warn(`\u26A0\uFE0F Failed to add audio track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
5748
+ console.warn("\u{1F517} You can manually add the track using this presigned URL:");
5749
+ console.warn(presignedUrl);
5750
+ }
5751
+ }
5578
5752
  }
5579
5753
  const targetLanguage = getLanguageCodePair(toLanguageCode);
5580
5754
  return {
@@ -6016,7 +6190,7 @@ async function uploadVttToS3({
6016
6190
  expiresInSeconds: s3SignedUrlExpirySeconds ?? 86400
6017
6191
  }, storageAdapter);
6018
6192
  }
6019
- async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, options) {
6193
+ async function translateCaptions(assetId, trackId, toLanguageCode, options) {
6020
6194
  "use workflow";
6021
6195
  const {
6022
6196
  provider = "openai",
@@ -6024,6 +6198,7 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
6024
6198
  s3Endpoint: providedS3Endpoint,
6025
6199
  s3Region: providedS3Region,
6026
6200
  s3Bucket: providedS3Bucket,
6201
+ uploadToS3: uploadToS3Option,
6027
6202
  uploadToMux: uploadToMuxOption,
6028
6203
  storageAdapter,
6029
6204
  credentials: providedCredentials,
@@ -6037,17 +6212,17 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
6037
6212
  const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
6038
6213
  const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
6039
6214
  const uploadToMux = uploadToMuxOption !== false;
6215
+ const uploadToS3 = uploadToS3Option || uploadToMux;
6040
6216
  const modelConfig = resolveLanguageModelConfig({
6041
6217
  ...options,
6042
6218
  model,
6043
6219
  provider
6044
6220
  });
6045
- if (uploadToMux && (!s3Endpoint || !s3Bucket || !effectiveStorageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
6046
- throw new Error("Storage configuration is required for uploading to Mux. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.");
6221
+ if (uploadToS3 && (!s3Endpoint || !s3Bucket || !effectiveStorageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
6222
+ throw new Error("Storage configuration is required for uploading. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.");
6047
6223
  }
6048
6224
  const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
6049
6225
  const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
6050
- const isAudioOnly = isAudioOnlyAsset(assetData);
6051
6226
  const signingContext = await resolveMuxSigningContext(credentials);
6052
6227
  if (policy === "signed" && !signingContext) {
6053
6228
  throw new Error(
@@ -6055,30 +6230,20 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
6055
6230
  );
6056
6231
  }
6057
6232
  const readyTextTracks = getReadyTextTracks(assetData);
6058
- if (!readyTextTracks.length) {
6059
- throw new Error("No ready text tracks found for this asset");
6060
- }
6061
- let sourceTextTrack = readyTextTracks.find(
6062
- (track) => track.text_type === "subtitles" && track.language_code === fromLanguageCode
6063
- );
6064
- if (!sourceTextTrack && isAudioOnly && readyTextTracks.length === 1) {
6065
- sourceTextTrack = readyTextTracks[0];
6066
- }
6233
+ const sourceTextTrack = readyTextTracks.find((t) => t.id === trackId);
6067
6234
  if (!sourceTextTrack) {
6068
- const availableLanguages = readyTextTracks.map((t) => t.language_code).filter(Boolean).join(", ");
6069
- if (isAudioOnly) {
6070
- throw new Error(
6071
- `No transcript track found with language code '${fromLanguageCode}' for this asset. Audio-only assets require a transcript. Available languages: ${availableLanguages || "none"}`
6072
- );
6073
- }
6235
+ const availableTrackIds = readyTextTracks.map((t) => t.id).filter(Boolean).join(", ");
6074
6236
  throw new Error(
6075
- `No ready text track found with language code '${fromLanguageCode}' for this asset. Available languages: ${availableLanguages || "none"}`
6237
+ `Track '${trackId}' not found or not ready on asset '${assetId}'. Available track IDs: ${availableTrackIds || "none"}`
6076
6238
  );
6077
6239
  }
6078
- if (!sourceTextTrack.id) {
6079
- throw new Error("Transcript track is missing an id");
6240
+ const fromLanguageCode = sourceTextTrack.language_code;
6241
+ if (!fromLanguageCode) {
6242
+ throw new Error(
6243
+ `Track '${trackId}' is missing language metadata (language_code). Cannot determine source language for translation.`
6244
+ );
6080
6245
  }
6081
- const vttUrl = await buildTranscriptUrl(playbackId, sourceTextTrack.id, policy === "signed", credentials);
6246
+ const vttUrl = await buildTranscriptUrl(playbackId, trackId, policy === "signed", credentials);
6082
6247
  let vttContent;
6083
6248
  try {
6084
6249
  vttContent = await fetchVttFromMux(vttUrl);
@@ -6111,50 +6276,43 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
6111
6276
  } : void 0;
6112
6277
  const sourceLanguage = getLanguageCodePair(fromLanguageCode);
6113
6278
  const targetLanguage = getLanguageCodePair(toLanguageCode);
6114
- if (!uploadToMux) {
6115
- return {
6116
- assetId,
6117
- sourceLanguageCode: fromLanguageCode,
6118
- targetLanguageCode: toLanguageCode,
6119
- sourceLanguage,
6120
- targetLanguage,
6121
- originalVtt: vttContent,
6122
- translatedVtt,
6123
- usage: usageWithMetadata
6124
- };
6125
- }
6126
6279
  let presignedUrl;
6127
- try {
6128
- presignedUrl = await uploadVttToS3({
6129
- translatedVtt,
6130
- assetId,
6131
- fromLanguageCode,
6132
- toLanguageCode,
6133
- s3Endpoint,
6134
- s3Region,
6135
- s3Bucket,
6136
- storageAdapter: effectiveStorageAdapter,
6137
- s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
6138
- });
6139
- } catch (error) {
6140
- throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
6141
- }
6142
6280
  let uploadedTrackId;
6143
- try {
6144
- const languageName = getLanguageName(toLanguageCode);
6145
- const trackName = `${languageName} (auto-translated)`;
6146
- uploadedTrackId = await createTextTrackOnMux(
6147
- assetId,
6148
- toLanguageCode,
6149
- trackName,
6150
- presignedUrl,
6151
- credentials
6152
- );
6153
- } catch (error) {
6154
- console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
6281
+ if (uploadToS3) {
6282
+ try {
6283
+ presignedUrl = await uploadVttToS3({
6284
+ translatedVtt,
6285
+ assetId,
6286
+ fromLanguageCode,
6287
+ toLanguageCode,
6288
+ s3Endpoint,
6289
+ s3Region,
6290
+ s3Bucket,
6291
+ storageAdapter: effectiveStorageAdapter,
6292
+ s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
6293
+ });
6294
+ } catch (error) {
6295
+ throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
6296
+ }
6297
+ if (uploadToMux) {
6298
+ try {
6299
+ const languageName = getLanguageName(toLanguageCode);
6300
+ const trackName = `${languageName} (auto-translated)`;
6301
+ uploadedTrackId = await createTextTrackOnMux(
6302
+ assetId,
6303
+ toLanguageCode,
6304
+ trackName,
6305
+ presignedUrl,
6306
+ credentials
6307
+ );
6308
+ } catch (error) {
6309
+ console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
6310
+ }
6311
+ }
6155
6312
  }
6156
6313
  return {
6157
6314
  assetId,
6315
+ trackId,
6158
6316
  sourceLanguageCode: fromLanguageCode,
6159
6317
  targetLanguageCode: toLanguageCode,
6160
6318
  sourceLanguage,