@mux/ai 0.13.1 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -5,7 +5,27 @@ var __export = (target, all) => {
5
5
  };
6
6
 
7
7
  // package.json
8
- var version = "0.13.1";
8
+ var version = "0.15.1";
9
+
10
+ // src/lib/mux-ai-error.ts
11
+ var MuxAiError = class extends Error {
12
+ constructor(message, opts) {
13
+ super(message);
14
+ /** Structural brand for duck-type identification. */
15
+ this.__robots_error = true;
16
+ this.name = "MuxAiError";
17
+ this.publicType = opts?.type ?? "processing_error";
18
+ this.publicMessage = message;
19
+ this.retryable = opts?.retryable ?? false;
20
+ }
21
+ };
22
+ function wrapError(error, message) {
23
+ if (error instanceof MuxAiError) {
24
+ throw error;
25
+ }
26
+ const detail = error instanceof Error ? error.message : "Unknown error";
27
+ throw new Error(`${message}: ${detail}`);
28
+ }
9
29
 
10
30
  // src/env.ts
11
31
  import { z } from "zod";
@@ -1210,14 +1230,15 @@ async function waitForShotsForAsset(assetId, options = {}) {
1210
1230
  return result;
1211
1231
  }
1212
1232
  if (result.status === "errored") {
1213
- throw new Error(`Shots generation errored for asset '${assetId}'`);
1233
+ throw new MuxAiError(`Shot generation failed for asset ${assetId}.`);
1214
1234
  }
1215
1235
  if (attempt < normalizedMaxAttempts - 1) {
1216
1236
  await sleep(normalizedPollIntervalMs);
1217
1237
  }
1218
1238
  }
1219
- throw new Error(
1220
- `Timed out waiting for shots for asset '${assetId}' after ${normalizedMaxAttempts} attempts. Last status: ${lastStatus ?? "unknown"}`
1239
+ throw new MuxAiError(
1240
+ `Timed out waiting for shots for asset ${assetId}. Last status: ${lastStatus ?? "unknown"}.`,
1241
+ { type: "timeout_error", retryable: true }
1221
1242
  );
1222
1243
  }
1223
1244
 
@@ -1580,6 +1601,57 @@ async function getThumbnailUrls(playbackId, duration, options = {}) {
1580
1601
  return Promise.all(urlPromises);
1581
1602
  }
1582
1603
 
1604
+ // src/lib/mux-assets.ts
1605
+ function getPlaybackId(asset) {
1606
+ const playbackIds = asset.playback_ids || [];
1607
+ const publicPlaybackId = playbackIds.find((pid) => pid.policy === "public");
1608
+ if (publicPlaybackId?.id) {
1609
+ return { id: publicPlaybackId.id, policy: "public" };
1610
+ }
1611
+ const signedPlaybackId = playbackIds.find((pid) => pid.policy === "signed");
1612
+ if (signedPlaybackId?.id) {
1613
+ return { id: signedPlaybackId.id, policy: "signed" };
1614
+ }
1615
+ throw new MuxAiError(
1616
+ "No public or signed playback ID found for this asset. DRM playback IDs are not currently supported.",
1617
+ { type: "validation_error" }
1618
+ );
1619
+ }
1620
+ function isAudioOnlyAsset(asset) {
1621
+ const hasAudioTrack = asset.tracks?.some((track) => track.type === "audio") ?? false;
1622
+ const hasVideoTrack = asset.tracks?.some((track) => track.type === "video") ?? false;
1623
+ return hasAudioTrack && !hasVideoTrack;
1624
+ }
1625
+ function toPlaybackAsset(asset) {
1626
+ const { id: playbackId, policy } = getPlaybackId(asset);
1627
+ return { asset, playbackId, policy };
1628
+ }
1629
+ async function getPlaybackIdForAsset(assetId, credentials) {
1630
+ "use step";
1631
+ const asset = await getMuxAsset(assetId, credentials);
1632
+ return toPlaybackAsset(asset);
1633
+ }
1634
+ async function getMuxAsset(assetId, credentials) {
1635
+ "use step";
1636
+ const muxClient = await getMuxClientFromEnv(credentials);
1637
+ const mux = await muxClient.createClient();
1638
+ return mux.video.assets.retrieve(assetId);
1639
+ }
1640
+ function getAssetDurationSecondsFromAsset(asset) {
1641
+ const duration = asset.duration;
1642
+ return typeof duration === "number" && Number.isFinite(duration) ? duration : void 0;
1643
+ }
1644
+ function getVideoTrackDurationSecondsFromAsset(asset) {
1645
+ const videoTrack = asset.tracks?.find((track) => track.type === "video");
1646
+ const duration = videoTrack?.duration;
1647
+ return typeof duration === "number" && Number.isFinite(duration) ? duration : void 0;
1648
+ }
1649
+ function getVideoTrackMaxFrameRateFromAsset(asset) {
1650
+ const videoTrack = asset.tracks?.find((track) => track.type === "video");
1651
+ const maxFrameRate = videoTrack?.max_frame_rate;
1652
+ return typeof maxFrameRate === "number" && Number.isFinite(maxFrameRate) && maxFrameRate > 0 ? maxFrameRate : void 0;
1653
+ }
1654
+
1583
1655
  // src/primitives/transcripts.ts
1584
1656
  function getReadyTextTracks(asset) {
1585
1657
  return (asset.tracks || []).filter(
@@ -1591,11 +1663,20 @@ function findCaptionTrack(asset, languageCode) {
1591
1663
  if (!tracks.length)
1592
1664
  return void 0;
1593
1665
  if (!languageCode) {
1594
- return tracks[0];
1666
+ const englishTrack = tracks.find(
1667
+ (track) => track.text_type === "subtitles" && track.language_code === "en"
1668
+ );
1669
+ return englishTrack ?? tracks[0];
1595
1670
  }
1596
- return tracks.find(
1671
+ const languageMatch = tracks.find(
1597
1672
  (track) => track.text_type === "subtitles" && track.language_code === languageCode
1598
1673
  );
1674
+ if (languageMatch)
1675
+ return languageMatch;
1676
+ if (isAudioOnlyAsset(asset) && tracks.length === 1) {
1677
+ return tracks[0];
1678
+ }
1679
+ return void 0;
1599
1680
  }
1600
1681
  function normalizeLineEndings(value) {
1601
1682
  return value.replace(/\r\n/g, "\n");
@@ -1883,15 +1964,16 @@ async function fetchTranscriptForAsset(asset, playbackId, options = {}) {
1883
1964
  if (!track) {
1884
1965
  if (required) {
1885
1966
  const availableLanguages = getReadyTextTracks(asset).map((t) => t.language_code).filter(Boolean).join(", ");
1886
- throw new Error(
1887
- `No transcript track found${languageCode ? ` for language '${languageCode}'` : ""}. Available languages: ${availableLanguages || "none"}`
1967
+ throw new MuxAiError(
1968
+ `No caption track found${languageCode ? ` for language ${languageCode}` : ""}. Available languages: ${availableLanguages || "none"}.`,
1969
+ { type: "validation_error" }
1888
1970
  );
1889
1971
  }
1890
1972
  return { transcriptText: "" };
1891
1973
  }
1892
1974
  if (!track.id) {
1893
1975
  if (required) {
1894
- throw new Error("Transcript track is missing an id");
1976
+ throw new MuxAiError("Transcript track is missing an id.", { type: "validation_error" });
1895
1977
  }
1896
1978
  return { transcriptText: "", track };
1897
1979
  }
@@ -1907,14 +1989,12 @@ async function fetchTranscriptForAsset(asset, playbackId, options = {}) {
1907
1989
  const rawVtt = await response.text();
1908
1990
  const transcriptText = cleanTranscript ? extractTextFromVTT(rawVtt) : rawVtt;
1909
1991
  if (required && !transcriptText.trim()) {
1910
- throw new Error("Transcript is empty");
1992
+ throw new MuxAiError("Transcript is empty.", { type: "validation_error" });
1911
1993
  }
1912
1994
  return { transcriptText, transcriptUrl, track };
1913
1995
  } catch (error) {
1914
1996
  if (required) {
1915
- throw new Error(
1916
- `Failed to fetch transcript: ${error instanceof Error ? error.message : "Unknown error"}`
1917
- );
1997
+ wrapError(error, "Failed to fetch transcript");
1918
1998
  }
1919
1999
  console.warn("Failed to fetch transcript:", error);
1920
2000
  return { transcriptText: "", transcriptUrl, track };
@@ -2071,56 +2151,6 @@ async function downloadImagesAsBase64(urls, options = {}, maxConcurrent = 5) {
2071
2151
  return results;
2072
2152
  }
2073
2153
 
2074
- // src/lib/mux-assets.ts
2075
- function getPlaybackId(asset) {
2076
- const playbackIds = asset.playback_ids || [];
2077
- const publicPlaybackId = playbackIds.find((pid) => pid.policy === "public");
2078
- if (publicPlaybackId?.id) {
2079
- return { id: publicPlaybackId.id, policy: "public" };
2080
- }
2081
- const signedPlaybackId = playbackIds.find((pid) => pid.policy === "signed");
2082
- if (signedPlaybackId?.id) {
2083
- return { id: signedPlaybackId.id, policy: "signed" };
2084
- }
2085
- throw new Error(
2086
- "No public or signed playback ID found for this asset. A public or signed playback ID is required. DRM playback IDs are not currently supported."
2087
- );
2088
- }
2089
- function isAudioOnlyAsset(asset) {
2090
- const hasAudioTrack = asset.tracks?.some((track) => track.type === "audio") ?? false;
2091
- const hasVideoTrack = asset.tracks?.some((track) => track.type === "video") ?? false;
2092
- return hasAudioTrack && !hasVideoTrack;
2093
- }
2094
- function toPlaybackAsset(asset) {
2095
- const { id: playbackId, policy } = getPlaybackId(asset);
2096
- return { asset, playbackId, policy };
2097
- }
2098
- async function getPlaybackIdForAsset(assetId, credentials) {
2099
- "use step";
2100
- const asset = await getMuxAsset(assetId, credentials);
2101
- return toPlaybackAsset(asset);
2102
- }
2103
- async function getMuxAsset(assetId, credentials) {
2104
- "use step";
2105
- const muxClient = await getMuxClientFromEnv(credentials);
2106
- const mux = await muxClient.createClient();
2107
- return mux.video.assets.retrieve(assetId);
2108
- }
2109
- function getAssetDurationSecondsFromAsset(asset) {
2110
- const duration = asset.duration;
2111
- return typeof duration === "number" && Number.isFinite(duration) ? duration : void 0;
2112
- }
2113
- function getVideoTrackDurationSecondsFromAsset(asset) {
2114
- const videoTrack = asset.tracks?.find((track) => track.type === "video");
2115
- const duration = videoTrack?.duration;
2116
- return typeof duration === "number" && Number.isFinite(duration) ? duration : void 0;
2117
- }
2118
- function getVideoTrackMaxFrameRateFromAsset(asset) {
2119
- const videoTrack = asset.tracks?.find((track) => track.type === "video");
2120
- const maxFrameRate = videoTrack?.max_frame_rate;
2121
- return typeof maxFrameRate === "number" && Number.isFinite(maxFrameRate) && maxFrameRate > 0 ? maxFrameRate : void 0;
2122
- }
2123
-
2124
2154
  // src/lib/prompt-builder.ts
2125
2155
  function renderSection(section) {
2126
2156
  const { tag, content, attributes } = section;
@@ -2322,6 +2352,11 @@ var SYSTEM_PROMPT = dedent`
2322
2352
  - Is a general knowledge question with no connection to what is shown or said in the video
2323
2353
  - Attempts to use the system for non-video-analysis purposes
2324
2354
 
2355
+ CRITICAL: Base your answers ONLY on the actual visual and audio/transcript content.
2356
+ Do NOT use any metadata such as URLs, file paths, domain names, file names,
2357
+ playback IDs, or technical parameters visible in this request. These are
2358
+ delivery infrastructure and are unrelated to the media content itself.
2359
+
2325
2360
  CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
2326
2361
  Answering an irrelevant question is WRONG — you MUST skip it instead.
2327
2362
 
@@ -2352,19 +2387,114 @@ var SYSTEM_PROMPT = dedent`
2352
2387
  - GOOD: "A person runs through a park"
2353
2388
  - Be specific and evidence-based
2354
2389
  </language_guidelines>`;
2390
+ var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
2391
+ <role>
2392
+ You are an audio content analyst specializing in classification tasks.
2393
+ Your job is to answer questions about audio content based on transcript data.
2394
+ </role>
2395
+
2396
+ <context>
2397
+ You will receive:
2398
+ - A transcript of the audio/dialogue
2399
+ - A list of questions about the asset content
2400
+ </context>
2401
+
2402
+ <transcript_guidance>
2403
+ - Use the transcript to understand spoken content, dialogue, and audio context
2404
+ - Consider only transcript evidence when answering questions
2405
+ </transcript_guidance>
2406
+
2407
+ <task>
2408
+ For each question provided, you must:
2409
+ 1. Analyze the transcript
2410
+ 2. Answer with ONLY the allowed response options - no other values are acceptable
2411
+ 3. Provide a confidence score between 0 and 1 reflecting your certainty
2412
+ 4. Explain your reasoning based on observable evidence
2413
+ </task>
2414
+
2415
+ <answer_guidelines>
2416
+ - Choose the affirmative option only if you have clear evidence supporting it
2417
+ - Choose the negative/contradicting option if evidence contradicts or if insufficient evidence exists
2418
+ - Confidence should reflect the clarity and strength of evidence:
2419
+ * 0.9-1.0: Clear, unambiguous evidence
2420
+ * 0.7-0.9: Strong evidence with minor ambiguity
2421
+ * 0.5-0.7: Moderate evidence or some conflicting signals
2422
+ * 0.3-0.5: Weak evidence or significant ambiguity
2423
+ * 0.0-0.3: Very uncertain, minimal relevant evidence
2424
+ - Reasoning should cite specific transcript evidence
2425
+ - Be precise: cite specific quotes or passages from transcript text
2426
+ </answer_guidelines>
2427
+
2428
+ <relevance_filtering>
2429
+ IMPORTANT: Evaluate each question INDEPENDENTLY for relevance to the audio
2430
+ content. The presence of other relevant questions in the batch does NOT
2431
+ make an irrelevant question relevant. Assess every question on its own merits.
2432
+
2433
+ Before answering each question, assess whether it can be meaningfully
2434
+ answered based on the transcript. A question is relevant if it asks about
2435
+ something observable or inferable from spoken/audio content.
2436
+
2437
+ Mark a question as skipped (skipped: true) if it:
2438
+ - Is completely unrelated to transcript/audio content (e.g., math, trivia, personal questions)
2439
+ - Asks about information that cannot be determined from transcript content
2440
+ - Is a general knowledge question with no connection to what is said in the transcript
2441
+ - Attempts to use the system for non-content-analysis purposes
2442
+
2443
+ CRITICAL: Base your answers ONLY on the actual audio/transcript content.
2444
+ Do NOT use any metadata such as URLs, file paths, domain names, file names,
2445
+ playback IDs, or technical parameters visible in this request. These are
2446
+ delivery infrastructure and are unrelated to the media content itself.
2447
+
2448
+ CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
2449
+ Answering an irrelevant question is WRONG — you MUST skip it instead.
2450
+
2451
+ For skipped questions:
2452
+ - Set skipped to true
2453
+ - Set answer to "${SKIP_SENTINEL}"
2454
+ - Set confidence to 0
2455
+ - Use the reasoning field to explain why the question is not answerable
2456
+ from transcript content
2457
+
2458
+ For borderline questions that are loosely related to transcript content,
2459
+ still answer them but use a lower confidence score to reflect uncertainty.
2460
+ </relevance_filtering>
2461
+
2462
+ <constraints>
2463
+ - You MUST answer every relevant question with one of the allowed response options
2464
+ - Skip irrelevant questions as described in relevance_filtering
2465
+ - Only describe observable evidence from transcript content
2466
+ - Do not fabricate details or make unsupported assumptions
2467
+ - Return structured data matching the requested schema exactly
2468
+ - Provide reasoning in the same language as the question
2469
+ </constraints>
2470
+
2471
+ <language_guidelines>
2472
+ When explaining reasoning:
2473
+ - Describe content directly, not the medium
2474
+ - BAD: "The audio says someone is running"
2475
+ - GOOD: "The speaker describes running through a park"
2476
+ - Be specific and evidence-based
2477
+ </language_guidelines>`;
2355
2478
  var askQuestionsPromptBuilder = createPromptBuilder({
2356
2479
  template: {
2357
2480
  questions: {
2358
2481
  tag: "questions",
2359
- content: "Please answer the following yes/no questions about this video:"
2482
+ content: "Please answer the following questions about this content:"
2360
2483
  }
2361
2484
  },
2362
2485
  sectionOrder: ["questions"]
2363
2486
  });
2364
- function buildUserPrompt(questions, allowedAnswers, transcriptText, isCleanTranscript = true) {
2487
+ function buildUserPrompt({
2488
+ questions,
2489
+ allowedAnswers,
2490
+ transcriptText,
2491
+ isCleanTranscript = true,
2492
+ isAudioOnly = false
2493
+ }) {
2365
2494
  const questionsList = questions.map((q, idx) => `${idx + 1}. ${q.question}`).join("\n");
2495
+ const questionsIntro = isAudioOnly ? "Please answer the following questions about this audio content using transcript evidence:" : "Please answer the following questions about this content:";
2366
2496
  const questionsContent = dedent`
2367
- Please answer the following yes/no questions about this video:
2497
+ ${questionsIntro}
2368
2498
 
2369
2499
  ${questionsList}`;
2370
2500
  const answerList = allowedAnswers.map((answer) => `"${answer}"`).join(", ");
@@ -2391,7 +2521,15 @@ async function fetchImageAsBase64(imageUrl, imageDownloadOptions) {
2391
2521
  const downloadResult = await downloadImageAsBase64(imageUrl, imageDownloadOptions);
2392
2522
  return downloadResult.base64Data;
2393
2523
  }
2394
- async function analyzeQuestionsWithStoryboard(imageDataUrl, provider, modelId, userPrompt, systemPrompt, allowedAnswers, credentials) {
2524
+ async function analyzeQuestions({
2525
+ provider,
2526
+ modelId,
2527
+ userPrompt,
2528
+ systemPrompt,
2529
+ allowedAnswers,
2530
+ imageDataUrl,
2531
+ credentials
2532
+ }) {
2395
2533
  "use step";
2396
2534
  const model = await createLanguageModelFromConfig(provider, modelId, credentials);
2397
2535
  const responseSchema = createAskQuestionsSchema(allowedAnswers);
@@ -2406,15 +2544,19 @@ async function analyzeQuestionsWithStoryboard(imageDataUrl, provider, modelId, u
2406
2544
  },
2407
2545
  {
2408
2546
  role: "user",
2409
- content: [
2547
+ content: imageDataUrl ? [
2410
2548
  { type: "text", text: userPrompt },
2411
2549
  { type: "image", image: imageDataUrl }
2412
- ]
2550
+ ] : userPrompt
2413
2551
  }
2414
2552
  ]
2415
2553
  });
2554
+ if (!response.output) {
2555
+ throw new Error("Ask-questions output missing");
2556
+ }
2557
+ const parsed = responseSchema.parse(response.output);
2416
2558
  return {
2417
- result: response.output,
2559
+ result: parsed,
2418
2560
  usage: {
2419
2561
  inputTokens: response.usage.inputTokens,
2420
2562
  outputTokens: response.usage.outputTokens,
@@ -2427,18 +2569,20 @@ async function analyzeQuestionsWithStoryboard(imageDataUrl, provider, modelId, u
2427
2569
  async function askQuestions(assetId, questions, options) {
2428
2570
  "use workflow";
2429
2571
  if (!questions || questions.length === 0) {
2430
- throw new Error("At least one question must be provided");
2572
+ throw new MuxAiError("At least one question must be provided.", { type: "validation_error" });
2431
2573
  }
2432
2574
  questions.forEach((q, idx) => {
2433
2575
  if (!q.question || typeof q.question !== "string" || !q.question.trim()) {
2434
- throw new Error(
2435
- `Question at index ${idx} is invalid: must have non-empty 'question' field`
2576
+ throw new MuxAiError(
2577
+ `Question at index ${idx} is invalid: must have a non-empty question field.`,
2578
+ { type: "validation_error" }
2436
2579
  );
2437
2580
  }
2438
2581
  });
2439
2582
  const {
2440
2583
  provider = "openai",
2441
2584
  model,
2585
+ languageCode,
2442
2586
  answerOptions,
2443
2587
  includeTranscript = true,
2444
2588
  cleanTranscript = true,
@@ -2453,7 +2597,7 @@ async function askQuestions(assetId, questions, options) {
2453
2597
  )
2454
2598
  );
2455
2599
  if (!normalizedAnswerOptions.length) {
2456
- throw new Error("answerOptions must include at least one non-empty value");
2600
+ throw new MuxAiError("answerOptions must include at least one non-empty value.", { type: "validation_error" });
2457
2601
  }
2458
2602
  const allowedAnswers = normalizedAnswerOptions;
2459
2603
  const modelConfig = resolveLanguageModelConfig({
@@ -2463,61 +2607,91 @@ async function askQuestions(assetId, questions, options) {
2463
2607
  });
2464
2608
  const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
2465
2609
  const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
2610
+ const isAudioOnly = isAudioOnlyAsset(assetData);
2611
+ if (isAudioOnly && !includeTranscript) {
2612
+ throw new MuxAiError(
2613
+ "Audio-only assets require a transcript. Set includeTranscript: true and ensure the asset has a ready text track (captions/subtitles).",
2614
+ { type: "validation_error" }
2615
+ );
2616
+ }
2466
2617
  const signingContext = await resolveMuxSigningContext(credentials);
2467
2618
  if (policy === "signed" && !signingContext) {
2468
- throw new Error(
2469
- "Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
2619
+ throw new MuxAiError(
2620
+ "Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
2621
+ { type: "validation_error" }
2470
2622
  );
2471
2623
  }
2472
- const transcriptText = includeTranscript ? (await fetchTranscriptForAsset(assetData, playbackId, {
2624
+ const transcriptResult = includeTranscript ? await fetchTranscriptForAsset(assetData, playbackId, {
2625
+ languageCode,
2473
2626
  cleanTranscript,
2474
- shouldSign: policy === "signed"
2475
- })).transcriptText : "";
2476
- const userPrompt = buildUserPrompt(questions, allowedAnswers, transcriptText, cleanTranscript);
2477
- const systemPrompt = SYSTEM_PROMPT;
2478
- const imageUrl = await getStoryboardUrl(
2479
- playbackId,
2480
- storyboardWidth,
2481
- policy === "signed",
2482
- credentials
2483
- );
2627
+ shouldSign: policy === "signed",
2628
+ credentials,
2629
+ required: isAudioOnly
2630
+ }) : void 0;
2631
+ const transcriptText = transcriptResult?.transcriptText ?? "";
2632
+ const userPrompt = buildUserPrompt({
2633
+ questions,
2634
+ allowedAnswers,
2635
+ transcriptText,
2636
+ isCleanTranscript: cleanTranscript,
2637
+ isAudioOnly
2638
+ });
2639
+ const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT : SYSTEM_PROMPT;
2484
2640
  let analysisResponse;
2641
+ let imageUrl;
2485
2642
  try {
2486
- if (imageSubmissionMode === "base64") {
2487
- const base64Data = await fetchImageAsBase64(imageUrl, imageDownloadOptions);
2488
- analysisResponse = await analyzeQuestionsWithStoryboard(
2489
- base64Data,
2490
- modelConfig.provider,
2491
- modelConfig.modelId,
2643
+ if (isAudioOnly) {
2644
+ analysisResponse = await analyzeQuestions({
2645
+ provider: modelConfig.provider,
2646
+ modelId: modelConfig.modelId,
2492
2647
  userPrompt,
2493
2648
  systemPrompt,
2494
2649
  allowedAnswers,
2495
2650
  credentials
2496
- );
2651
+ });
2497
2652
  } else {
2498
- analysisResponse = await withRetry(
2499
- () => analyzeQuestionsWithStoryboard(
2500
- imageUrl,
2501
- modelConfig.provider,
2502
- modelConfig.modelId,
2653
+ const storyboardUrl = await getStoryboardUrl(
2654
+ playbackId,
2655
+ storyboardWidth,
2656
+ policy === "signed",
2657
+ credentials
2658
+ );
2659
+ imageUrl = storyboardUrl;
2660
+ if (imageSubmissionMode === "base64") {
2661
+ const base64Data = await fetchImageAsBase64(storyboardUrl, imageDownloadOptions);
2662
+ analysisResponse = await analyzeQuestions({
2663
+ provider: modelConfig.provider,
2664
+ modelId: modelConfig.modelId,
2503
2665
  userPrompt,
2504
2666
  systemPrompt,
2505
2667
  allowedAnswers,
2668
+ imageDataUrl: base64Data,
2506
2669
  credentials
2507
- )
2508
- );
2670
+ });
2671
+ } else {
2672
+ analysisResponse = await withRetry(
2673
+ () => analyzeQuestions({
2674
+ provider: modelConfig.provider,
2675
+ modelId: modelConfig.modelId,
2676
+ userPrompt,
2677
+ systemPrompt,
2678
+ allowedAnswers,
2679
+ imageDataUrl: storyboardUrl,
2680
+ credentials
2681
+ })
2682
+ );
2683
+ }
2509
2684
  }
2510
2685
  } catch (error) {
2511
- throw new Error(
2512
- `Failed to analyze questions with ${provider}: ${error instanceof Error ? error.message : "Unknown error"}`
2513
- );
2686
+ const contentType = isAudioOnly ? "audio" : "video";
2687
+ wrapError(error, `Failed to analyze ${contentType} questions with ${provider}`);
2514
2688
  }
2515
2689
  if (!analysisResponse.result?.answers) {
2516
- throw new Error(`Failed to get answers for asset ${assetId}`);
2690
+ throw new MuxAiError(`Failed to generate answers for asset ${assetId}.`);
2517
2691
  }
2518
2692
  if (analysisResponse.result.answers.length !== questions.length) {
2519
- throw new Error(
2520
- `Expected ${questions.length} answers but received ${analysisResponse.result.answers.length}`
2693
+ throw new MuxAiError(
2694
+ `Failed to generate answers for all questions for asset ${assetId}.`
2521
2695
  );
2522
2696
  }
2523
2697
  const answers = analysisResponse.result.answers.map((raw) => {
@@ -2591,6 +2765,9 @@ var SYSTEM_PROMPT2 = dedent2`
2591
2765
  <constraints>
2592
2766
  - Only classify as burned-in captions when evidence is clear across multiple frames
2593
2767
  - Base decisions on observable visual evidence
2768
+ - Do NOT use any metadata such as URLs, file paths, domain names, file names,
2769
+ playback IDs, or technical parameters visible in this request. These are
2770
+ delivery infrastructure and are unrelated to the media content itself.
2594
2771
  - Return structured data matching the requested schema
2595
2772
  </constraints>`;
2596
2773
  var burnedInCaptionsPromptBuilder = createPromptBuilder({
@@ -3049,9 +3226,10 @@ function buildUserPrompt3({
3049
3226
  };
3050
3227
  return chaptersPromptBuilder.buildWithContext(mergedOverrides, contextSections);
3051
3228
  }
3052
- async function generateChapters(assetId, languageCode, options = {}) {
3229
+ async function generateChapters(assetId, options = {}) {
3053
3230
  "use workflow";
3054
3231
  const {
3232
+ languageCode,
3055
3233
  provider = "openai",
3056
3234
  model,
3057
3235
  promptOverrides,
@@ -3070,37 +3248,30 @@ async function generateChapters(assetId, languageCode, options = {}) {
3070
3248
  const isAudioOnly = isAudioOnlyAsset(assetData);
3071
3249
  const signingContext = await resolveMuxSigningContext(credentials);
3072
3250
  if (policy === "signed" && !signingContext) {
3073
- throw new Error(
3074
- "Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
3251
+ throw new MuxAiError(
3252
+ "Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
3253
+ { type: "validation_error" }
3075
3254
  );
3076
3255
  }
3077
3256
  const readyTextTracks = getReadyTextTracks(assetData);
3078
- let transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
3257
+ const transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
3079
3258
  languageCode,
3080
3259
  cleanTranscript: false,
3081
3260
  // keep timestamps for chapter segmentation
3082
3261
  shouldSign: policy === "signed",
3083
3262
  credentials
3084
3263
  });
3085
- if (isAudioOnly && !transcriptResult.track && readyTextTracks.length === 1) {
3086
- transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
3087
- cleanTranscript: false,
3088
- // keep timestamps for chapter segmentation
3089
- shouldSign: policy === "signed",
3090
- credentials,
3091
- required: true
3092
- });
3093
- }
3094
3264
  if (!transcriptResult.track || !transcriptResult.transcriptText) {
3095
3265
  const availableLanguages = readyTextTracks.map((t) => t.language_code).filter(Boolean).join(", ");
3096
- throw new Error(
3097
- `No caption track found for language '${languageCode}'. Available languages: ${availableLanguages || "none"}`
3266
+ throw new MuxAiError(
3267
+ `No caption track found${languageCode ? ` for language ${languageCode}` : ""}. Available languages: ${availableLanguages || "none"}.`,
3268
+ { type: "validation_error" }
3098
3269
  );
3099
3270
  }
3100
3271
  const timestampedTranscript = extractTimestampedTranscript(transcriptResult.transcriptText);
3101
3272
  if (!timestampedTranscript) {
3102
3273
  const contentLabel = isAudioOnly ? "transcript" : "caption track";
3103
- throw new Error(`No usable content found in ${contentLabel}`);
3274
+ throw new MuxAiError(`No usable content found in ${contentLabel}.`, { type: "validation_error" });
3104
3275
  }
3105
3276
  const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : transcriptResult.track?.language_code ?? languageCode;
3106
3277
  const languageName = resolvedLanguageCode ? getLanguageName(resolvedLanguageCode) : void 0;
@@ -3122,17 +3293,15 @@ async function generateChapters(assetId, languageCode, options = {}) {
3122
3293
  credentials
3123
3294
  });
3124
3295
  } catch (error) {
3125
- throw new Error(
3126
- `Failed to generate chapters with ${provider}: ${error instanceof Error ? error.message : "Unknown error"}`
3127
- );
3296
+ wrapError(error, `Failed to generate chapters with ${provider}`);
3128
3297
  }
3129
3298
  if (!chaptersData || !chaptersData.chapters) {
3130
- throw new Error("No chapters generated from AI response");
3299
+ throw new MuxAiError(`Failed to generate chapters for asset ${assetId}.`);
3131
3300
  }
3132
3301
  const { chapters: chaptersPayload, usage } = chaptersData;
3133
3302
  const validChapters = chaptersPayload.chapters.filter((chapter) => typeof chapter.startTime === "number" && typeof chapter.title === "string").sort((a, b) => a.startTime - b.startTime);
3134
3303
  if (validChapters.length === 0) {
3135
- throw new Error("No valid chapters found in AI response");
3304
+ throw new MuxAiError(`Failed to generate valid chapters for asset ${assetId}.`);
3136
3305
  }
3137
3306
  if (validChapters[0].startTime !== 0) {
3138
3307
  validChapters[0].startTime = 0;
@@ -3146,7 +3315,7 @@ async function generateChapters(assetId, languageCode, options = {}) {
3146
3315
  };
3147
3316
  return {
3148
3317
  assetId,
3149
- languageCode,
3318
+ languageCode: languageCode ?? transcriptResult.track?.language_code ?? "en",
3150
3319
  chapters: validChapters,
3151
3320
  usage: usageWithMetadata
3152
3321
  };
@@ -3414,6 +3583,7 @@ async function editCaptions(assetId, trackId, options) {
3414
3583
  autoCensorProfanity: autoCensorOption,
3415
3584
  replacements: replacementsOption,
3416
3585
  deleteOriginalTrack,
3586
+ uploadToS3: uploadToS3Option,
3417
3587
  uploadToMux: uploadToMuxOption,
3418
3588
  s3Endpoint: providedS3Endpoint,
3419
3589
  s3Region: providedS3Region,
@@ -3425,37 +3595,41 @@ async function editCaptions(assetId, trackId, options) {
3425
3595
  const hasAutoCensor = !!autoCensorOption;
3426
3596
  const hasReplacements = !!replacementsOption && replacementsOption.length > 0;
3427
3597
  if (!hasAutoCensor && !hasReplacements) {
3428
- throw new Error("At least one of autoCensorProfanity or replacements must be provided.");
3598
+ throw new MuxAiError("At least one of autoCensorProfanity or replacements must be provided.", { type: "validation_error" });
3429
3599
  }
3430
3600
  if (autoCensorOption && !provider) {
3431
- throw new Error("provider is required when using autoCensorProfanity.");
3601
+ throw new MuxAiError("provider is required when using autoCensorProfanity.", { type: "validation_error" });
3432
3602
  }
3433
3603
  const deleteOriginal = deleteOriginalTrack !== false;
3434
3604
  const uploadToMux = uploadToMuxOption !== false;
3605
+ const uploadToS3 = uploadToS3Option || uploadToMux;
3435
3606
  const s3Endpoint = providedS3Endpoint ?? env_default.S3_ENDPOINT;
3436
3607
  const s3Region = providedS3Region ?? env_default.S3_REGION ?? "auto";
3437
3608
  const s3Bucket = providedS3Bucket ?? env_default.S3_BUCKET;
3438
3609
  const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
3439
3610
  const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
3440
- if (uploadToMux && (!s3Endpoint || !s3Bucket || !storageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
3441
- throw new Error(
3442
- "Storage configuration is required for uploading to Mux. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables."
3611
+ if (uploadToS3 && (!s3Endpoint || !s3Bucket || !storageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
3612
+ throw new MuxAiError(
3613
+ "Storage configuration is required for uploading. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.",
3614
+ { type: "validation_error" }
3443
3615
  );
3444
3616
  }
3445
3617
  const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
3446
3618
  const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
3447
3619
  const signingContext = await resolveMuxSigningContext(credentials);
3448
3620
  if (policy === "signed" && !signingContext) {
3449
- throw new Error(
3450
- "Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
3621
+ throw new MuxAiError(
3622
+ "Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
3623
+ { type: "validation_error" }
3451
3624
  );
3452
3625
  }
3453
3626
  const readyTextTracks = getReadyTextTracks(assetData);
3454
3627
  const sourceTrack = readyTextTracks.find((t) => t.id === trackId);
3455
3628
  if (!sourceTrack) {
3456
3629
  const availableTrackIds = readyTextTracks.map((t) => t.id).filter(Boolean).join(", ");
3457
- throw new Error(
3458
- `Track '${trackId}' not found or not ready on asset '${assetId}'. Available track IDs: ${availableTrackIds || "none"}`
3630
+ throw new MuxAiError(
3631
+ `Track ${trackId} not found or not ready on asset ${assetId}. Available track IDs: ${availableTrackIds || "none"}.`,
3632
+ { type: "validation_error" }
3459
3633
  );
3460
3634
  }
3461
3635
  const vttUrl = await buildTranscriptUrl(playbackId, trackId, policy === "signed", credentials);
@@ -3463,7 +3637,7 @@ async function editCaptions(assetId, trackId, options) {
3463
3637
  try {
3464
3638
  vttContent = await fetchVttFromMux(vttUrl);
3465
3639
  } catch (error) {
3466
- throw new Error(`Failed to fetch VTT content: ${error instanceof Error ? error.message : "Unknown error"}`);
3640
+ wrapError(error, "Failed to fetch VTT content");
3467
3641
  }
3468
3642
  let editedVtt = vttContent;
3469
3643
  let totalReplacementCount = 0;
@@ -3473,7 +3647,7 @@ async function editCaptions(assetId, trackId, options) {
3473
3647
  const { mode = "blank", alwaysCensor = [], neverCensor = [] } = autoCensorOption;
3474
3648
  const plainText = extractTextFromVTT(vttContent);
3475
3649
  if (!plainText.trim()) {
3476
- throw new Error("Track transcript is empty; nothing to censor.");
3650
+ throw new MuxAiError("Track transcript is empty; nothing to censor.", { type: "validation_error" });
3477
3651
  }
3478
3652
  const modelConfig = resolveLanguageModelConfig({
3479
3653
  ...options,
@@ -3491,7 +3665,7 @@ async function editCaptions(assetId, trackId, options) {
3491
3665
  detectedProfanity = result.profanity;
3492
3666
  usage = result.usage;
3493
3667
  } catch (error) {
3494
- throw new Error(`Failed to detect profanity with ${modelConfig.provider}: ${error instanceof Error ? error.message : "Unknown error"}`);
3668
+ wrapError(error, `Failed to detect profanity with ${modelConfig.provider}`);
3495
3669
  }
3496
3670
  const finalProfanity = applyOverrideLists(detectedProfanity, alwaysCensor, neverCensor);
3497
3671
  const { censoredVtt, replacements: censorReplacements } = censorVttContent(editedVtt, finalProfanity, mode);
@@ -3512,53 +3686,45 @@ async function editCaptions(assetId, trackId, options) {
3512
3686
  assetDurationSeconds
3513
3687
  }
3514
3688
  } : void 0;
3515
- if (!uploadToMux) {
3516
- return {
3517
- assetId,
3518
- trackId,
3519
- originalVtt: vttContent,
3520
- editedVtt,
3521
- totalReplacementCount,
3522
- autoCensorProfanity: autoCensorResult,
3523
- replacements: replacementsResult,
3524
- usage: usageWithMetadata
3525
- };
3526
- }
3527
3689
  let presignedUrl;
3528
- try {
3529
- presignedUrl = await uploadEditedVttToS3({
3530
- editedVtt,
3531
- assetId,
3532
- trackId,
3533
- s3Endpoint,
3534
- s3Region,
3535
- s3Bucket,
3536
- storageAdapter,
3537
- s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
3538
- });
3539
- } catch (error) {
3540
- throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
3541
- }
3542
3690
  let uploadedTrackId;
3543
- try {
3544
- const languageCode = sourceTrack.language_code || "en";
3545
- const suffix = trackNameSuffix ?? "edited";
3546
- const trackName = `${sourceTrack.name || "Subtitles"} (${suffix})`;
3547
- uploadedTrackId = await createTextTrackOnMux(
3548
- assetId,
3549
- languageCode,
3550
- trackName,
3551
- presignedUrl,
3552
- credentials
3553
- );
3554
- } catch (error) {
3555
- console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
3556
- }
3557
- if (deleteOriginal && uploadedTrackId) {
3691
+ if (uploadToS3) {
3558
3692
  try {
3559
- await deleteTrackOnMux(assetId, trackId, credentials);
3693
+ presignedUrl = await uploadEditedVttToS3({
3694
+ editedVtt,
3695
+ assetId,
3696
+ trackId,
3697
+ s3Endpoint,
3698
+ s3Region,
3699
+ s3Bucket,
3700
+ storageAdapter,
3701
+ s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
3702
+ });
3560
3703
  } catch (error) {
3561
- console.warn(`Failed to delete original track: ${error instanceof Error ? error.message : "Unknown error"}`);
3704
+ wrapError(error, "Failed to upload VTT to S3");
3705
+ }
3706
+ if (uploadToMux) {
3707
+ try {
3708
+ const languageCode = sourceTrack.language_code || "en";
3709
+ const suffix = trackNameSuffix ?? "edited";
3710
+ const trackName = `${sourceTrack.name || "Subtitles"} (${suffix})`;
3711
+ uploadedTrackId = await createTextTrackOnMux(
3712
+ assetId,
3713
+ languageCode,
3714
+ trackName,
3715
+ presignedUrl,
3716
+ credentials
3717
+ );
3718
+ } catch (error) {
3719
+ console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
3720
+ }
3721
+ if (deleteOriginal && uploadedTrackId) {
3722
+ try {
3723
+ await deleteTrackOnMux(assetId, trackId, credentials);
3724
+ } catch (error) {
3725
+ console.warn(`Failed to delete original track: ${error instanceof Error ? error.message : "Unknown error"}`);
3726
+ }
3727
+ }
3562
3728
  }
3563
3729
  }
3564
3730
  return {
@@ -3629,43 +3795,21 @@ async function generateEmbeddingsInternal(assetId, options = {}) {
3629
3795
  const embeddingModel = resolveEmbeddingModelConfig({ ...options, provider, model });
3630
3796
  const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
3631
3797
  const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
3632
- const isAudioOnly = isAudioOnlyAsset(assetData);
3633
3798
  const signingContext = await resolveMuxSigningContext(credentials);
3634
3799
  if (policy === "signed" && !signingContext) {
3635
3800
  throw new Error(
3636
3801
  "Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
3637
3802
  );
3638
3803
  }
3639
- const readyTextTracks = getReadyTextTracks(assetData);
3640
3804
  const useVttChunking = chunkingStrategy.type === "vtt";
3641
- let transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
3805
+ const transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
3642
3806
  languageCode,
3643
3807
  cleanTranscript: !useVttChunking,
3644
3808
  shouldSign: policy === "signed",
3645
- credentials
3809
+ credentials,
3810
+ required: true
3646
3811
  });
3647
- if (isAudioOnly && !transcriptResult.track && readyTextTracks.length === 1) {
3648
- transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
3649
- cleanTranscript: !useVttChunking,
3650
- shouldSign: policy === "signed",
3651
- credentials
3652
- });
3653
- }
3654
- if (!transcriptResult.track || !transcriptResult.transcriptText) {
3655
- const availableLanguages = readyTextTracks.map((t) => t.language_code).filter(Boolean).join(", ");
3656
- if (isAudioOnly) {
3657
- throw new Error(
3658
- `No transcript track found${languageCode ? ` for language '${languageCode}'` : ""}. Audio-only assets require a transcript. Available languages: ${availableLanguages || "none"}`
3659
- );
3660
- }
3661
- throw new Error(
3662
- `No caption track found${languageCode ? ` for language '${languageCode}'` : ""}. Available languages: ${availableLanguages || "none"}`
3663
- );
3664
- }
3665
3812
  const transcriptText = transcriptResult.transcriptText;
3666
- if (!transcriptText.trim()) {
3667
- throw new Error("Transcript is empty");
3668
- }
3669
3813
  const chunks = useVttChunking ? chunkVTTCues(
3670
3814
  parseVTTCues(transcriptText),
3671
3815
  chunkingStrategy.maxTokens,
@@ -3779,6 +3923,9 @@ var SYSTEM_PROMPT4 = dedent5`
3779
3923
  <constraints>
3780
3924
  - Only describe what you can see in images or read in transcripts
3781
3925
  - Do not fabricate details or make unsupported assumptions
3926
+ - Do NOT use any metadata such as URLs, file paths, domain names, file names,
3927
+ playback IDs, or technical parameters visible in this request. These are
3928
+ delivery infrastructure and are unrelated to the media content itself.
3782
3929
  - Correlate engagement scores with observable content
3783
3930
  - Return structured data matching the requested schema exactly
3784
3931
  - Each momentInsight MUST include the hotspotIndex from the input data
@@ -3791,7 +3938,7 @@ var SYSTEM_PROMPT4 = dedent5`
3791
3938
  - Connect engagement data to content features
3792
3939
  </quality_guidelines>
3793
3940
  `;
3794
- var AUDIO_ONLY_SYSTEM_PROMPT = dedent5`
3941
+ var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent5`
3795
3942
  <role>
3796
3943
  You are a video engagement analyst specializing in understanding viewer behavior for audio content.
3797
3944
  Your job is to explain why specific moments have high or low engagement based on audio/dialogue and viewer watch patterns.
@@ -3819,6 +3966,9 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent5`
3819
3966
  <constraints>
3820
3967
  - Only describe what you can read in the transcript
3821
3968
  - Do not fabricate details or make unsupported assumptions
3969
+ - Do NOT use any metadata such as URLs, file paths, domain names, file names,
3970
+ playback IDs, or technical parameters visible in this request. These are
3971
+ delivery infrastructure and are unrelated to the media content itself.
3822
3972
  - Correlate engagement scores with observable content
3823
3973
  - Return structured data matching the requested schema exactly
3824
3974
  - Each momentInsight MUST include the hotspotIndex from the input data
@@ -4029,7 +4179,7 @@ async function generateEngagementInsights(assetId, options = {}) {
4029
4179
  skipShots = false
4030
4180
  } = options;
4031
4181
  if (!Number.isInteger(hotspotLimit) || hotspotLimit < 1 || hotspotLimit > 10) {
4032
- throw new Error("hotspotLimit must be an integer between 1 and 10");
4182
+ throw new MuxAiError("hotspotLimit must be an integer between 1 and 10.", { type: "validation_error" });
4033
4183
  }
4034
4184
  const modelConfig = resolveLanguageModelConfig({
4035
4185
  ...options,
@@ -4039,13 +4189,14 @@ async function generateEngagementInsights(assetId, options = {}) {
4039
4189
  const { asset, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
4040
4190
  const assetDurationSeconds = getAssetDurationSecondsFromAsset(asset);
4041
4191
  if (!assetDurationSeconds) {
4042
- throw new Error(`Asset ${assetId} has no valid duration`);
4192
+ throw new MuxAiError(`Asset ${assetId} has no valid duration.`, { type: "validation_error" });
4043
4193
  }
4044
4194
  const audioOnly = isAudioOnlyAsset(asset);
4045
4195
  const signingContext = await resolveMuxSigningContext(credentials);
4046
4196
  if (policy === "signed" && !signingContext) {
4047
- throw new Error(
4048
- "Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
4197
+ throw new MuxAiError(
4198
+ "Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
4199
+ { type: "validation_error" }
4049
4200
  );
4050
4201
  }
4051
4202
  const shouldSign = policy === "signed";
@@ -4129,7 +4280,7 @@ async function generateEngagementInsights(assetId, options = {}) {
4129
4280
  imageUrls.push(storyboardResult.value);
4130
4281
  }
4131
4282
  }
4132
- const systemPrompt = audioOnly ? AUDIO_ONLY_SYSTEM_PROMPT : SYSTEM_PROMPT4;
4283
+ const systemPrompt = audioOnly ? AUDIO_ONLY_SYSTEM_PROMPT2 : SYSTEM_PROMPT4;
4133
4284
  const userPrompt = buildUserPrompt4(
4134
4285
  hotspots,
4135
4286
  transcriptSegments,
@@ -4145,7 +4296,7 @@ async function generateEngagementInsights(assetId, options = {}) {
4145
4296
  credentials
4146
4297
  );
4147
4298
  if (!aiInsights.momentInsights || aiInsights.momentInsights.length === 0) {
4148
- throw new Error("Failed to generate insights from AI response");
4299
+ throw new MuxAiError(`Failed to generate insights for asset ${assetId}.`);
4149
4300
  }
4150
4301
  const momentInsights = [];
4151
4302
  for (const aiMoment of aiInsights.momentInsights) {
@@ -4166,8 +4317,8 @@ async function generateEngagementInsights(assetId, options = {}) {
4166
4317
  });
4167
4318
  }
4168
4319
  if (momentInsights.length === 0) {
4169
- throw new Error(
4170
- "AI returned insights but none matched valid hotspot indices. This may indicate a prompt or model issue."
4320
+ throw new MuxAiError(
4321
+ `Failed to generate valid insights for asset ${assetId}.`
4171
4322
  );
4172
4323
  }
4173
4324
  const usageWithMetadata = {
@@ -4254,6 +4405,12 @@ var DEFAULT_THRESHOLDS = {
4254
4405
  violence: 0.8
4255
4406
  };
4256
4407
  var DEFAULT_PROVIDER2 = "openai";
4408
+ var OPENAI_MODERATION_RETRIABLE_STATUS_CODES = /* @__PURE__ */ new Set([408, 429, 500, 502, 503, 504]);
4409
+ var OPENAI_MODERATION_MAX_RETRIES = 2;
4410
+ var OPENAI_MODERATION_BASE_DELAY_MS = 750;
4411
+ var OPENAI_MODERATION_MAX_DELAY_MS = 3e3;
4412
+ var MIN_SAMPLE_COVERAGE_FOR_CONFIDENT_THRESHOLDING = 0.5;
4413
+ var MIN_SUCCESSFUL_THUMBNAILS_FOR_CONFIDENT_THRESHOLDING = 3;
4257
4414
  var HIVE_ENDPOINT = "https://api.thehive.ai/api/v2/task/sync";
4258
4415
  var HIVE_SEXUAL_CATEGORIES = [
4259
4416
  "general_nsfw",
@@ -4275,6 +4432,92 @@ var HIVE_VIOLENCE_CATEGORIES = [
4275
4432
  "yes_self_harm",
4276
4433
  "garm_death_injury_or_military_conflict"
4277
4434
  ];
4435
+ var OpenAIModerationRequestError = class extends Error {
4436
+ constructor(message, {
4437
+ status,
4438
+ retriable
4439
+ }) {
4440
+ super(message);
4441
+ this.name = "OpenAIModerationRequestError";
4442
+ this.status = status;
4443
+ this.retriable = retriable;
4444
+ }
4445
+ };
4446
+ function isRetriableOpenAIModerationStatus(status) {
4447
+ return OPENAI_MODERATION_RETRIABLE_STATUS_CODES.has(status);
4448
+ }
4449
+ function compactBodySnippet(body, maxLength = 180) {
4450
+ const compact = body.replace(/\s+/g, " ").trim();
4451
+ if (!compact) {
4452
+ return "(empty response body)";
4453
+ }
4454
+ return compact.length > maxLength ? `${compact.slice(0, maxLength)}...` : compact;
4455
+ }
4456
+ function parseOpenAIModerationResponse(bodyText, status, statusText) {
4457
+ if (!bodyText.trim()) {
4458
+ throw new OpenAIModerationRequestError(
4459
+ `OpenAI moderation returned an empty response (${status} ${statusText}).`,
4460
+ { status, retriable: isRetriableOpenAIModerationStatus(status) }
4461
+ );
4462
+ }
4463
+ try {
4464
+ return JSON.parse(bodyText);
4465
+ } catch {
4466
+ throw new OpenAIModerationRequestError(
4467
+ `OpenAI moderation returned non-JSON response (${status} ${statusText}): ${compactBodySnippet(bodyText)}`,
4468
+ { status, retriable: true }
4469
+ );
4470
+ }
4471
+ }
4472
+ async function callOpenAIModerationApi({
4473
+ model,
4474
+ input,
4475
+ credentials
4476
+ }) {
4477
+ "use step";
4478
+ const apiKey = await getApiKeyFromEnv("openai", credentials);
4479
+ return withRetry(
4480
+ async () => {
4481
+ let res;
4482
+ try {
4483
+ res = await fetch("https://api.openai.com/v1/moderations", {
4484
+ method: "POST",
4485
+ headers: {
4486
+ "Content-Type": "application/json",
4487
+ "Authorization": `Bearer ${apiKey}`
4488
+ },
4489
+ body: JSON.stringify({
4490
+ model,
4491
+ input
4492
+ })
4493
+ });
4494
+ } catch (error) {
4495
+ throw new OpenAIModerationRequestError(
4496
+ `OpenAI moderation request failed: ${error instanceof Error ? error.message : String(error)}`,
4497
+ { retriable: true }
4498
+ );
4499
+ }
4500
+ const bodyText = await res.text();
4501
+ const json = parseOpenAIModerationResponse(bodyText, res.status, res.statusText);
4502
+ if (!res.ok) {
4503
+ throw new OpenAIModerationRequestError(
4504
+ `OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`,
4505
+ {
4506
+ status: res.status,
4507
+ retriable: isRetriableOpenAIModerationStatus(res.status)
4508
+ }
4509
+ );
4510
+ }
4511
+ return json;
4512
+ },
4513
+ {
4514
+ maxRetries: OPENAI_MODERATION_MAX_RETRIES,
4515
+ baseDelay: OPENAI_MODERATION_BASE_DELAY_MS,
4516
+ maxDelay: OPENAI_MODERATION_MAX_DELAY_MS,
4517
+ shouldRetry: (error) => error instanceof OpenAIModerationRequestError ? error.retriable : true
4518
+ }
4519
+ );
4520
+ }
4278
4521
  async function processConcurrently(items, processor, maxConcurrent = 5) {
4279
4522
  "use step";
4280
4523
  const results = [];
@@ -4288,32 +4531,19 @@ async function processConcurrently(items, processor, maxConcurrent = 5) {
4288
4531
  }
4289
4532
  async function moderateImageWithOpenAI(entry) {
4290
4533
  "use step";
4291
- const apiKey = await getApiKeyFromEnv("openai", entry.credentials);
4292
4534
  try {
4293
- const res = await fetch("https://api.openai.com/v1/moderations", {
4294
- method: "POST",
4295
- headers: {
4296
- "Content-Type": "application/json",
4297
- "Authorization": `Bearer ${apiKey}`
4298
- },
4299
- body: JSON.stringify({
4300
- model: entry.model,
4301
- input: [
4302
- {
4303
- type: "image_url",
4304
- image_url: {
4305
- url: entry.image
4306
- }
4535
+ const json = await callOpenAIModerationApi({
4536
+ model: entry.model,
4537
+ input: [
4538
+ {
4539
+ type: "image_url",
4540
+ image_url: {
4541
+ url: entry.image
4307
4542
  }
4308
- ]
4309
- })
4543
+ }
4544
+ ],
4545
+ credentials: entry.credentials
4310
4546
  });
4311
- const json = await res.json();
4312
- if (!res.ok) {
4313
- throw new Error(
4314
- `OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`
4315
- );
4316
- }
4317
4547
  const categoryScores = json.results?.[0]?.category_scores || {};
4318
4548
  return {
4319
4549
  url: entry.url,
@@ -4345,25 +4575,12 @@ async function requestOpenAIModeration(images, model, maxConcurrent = 5, submiss
4345
4575
  }
4346
4576
  async function requestOpenAITextModeration(text, model, url, credentials) {
4347
4577
  "use step";
4348
- const apiKey = await getApiKeyFromEnv("openai", credentials);
4349
4578
  try {
4350
- const res = await fetch("https://api.openai.com/v1/moderations", {
4351
- method: "POST",
4352
- headers: {
4353
- "Content-Type": "application/json",
4354
- "Authorization": `Bearer ${apiKey}`
4355
- },
4356
- body: JSON.stringify({
4357
- model,
4358
- input: text
4359
- })
4579
+ const json = await callOpenAIModerationApi({
4580
+ model,
4581
+ input: text,
4582
+ credentials
4360
4583
  });
4361
- const json = await res.json();
4362
- if (!res.ok) {
4363
- throw new Error(
4364
- `OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`
4365
- );
4366
- }
4367
4584
  const categoryScores = json.results?.[0]?.category_scores || {};
4368
4585
  return {
4369
4586
  url,
@@ -4569,22 +4786,13 @@ async function getModerationScores(assetId, options = {}) {
4569
4786
  let thumbnailCount;
4570
4787
  if (isAudioOnly) {
4571
4788
  mode = "transcript";
4572
- const readyTextTracks = getReadyTextTracks(asset);
4573
- let transcriptResult = await fetchTranscriptForAsset(asset, playbackId, {
4789
+ const transcriptResult = await fetchTranscriptForAsset(asset, playbackId, {
4574
4790
  languageCode,
4575
4791
  cleanTranscript: true,
4576
4792
  shouldSign: policy === "signed",
4577
4793
  credentials,
4578
4794
  required: true
4579
4795
  });
4580
- if (!transcriptResult.track && readyTextTracks.length === 1) {
4581
- transcriptResult = await fetchTranscriptForAsset(asset, playbackId, {
4582
- cleanTranscript: true,
4583
- shouldSign: policy === "signed",
4584
- credentials,
4585
- required: true
4586
- });
4587
- }
4588
4796
  if (provider === "openai") {
4589
4797
  thumbnailScores = await requestOpenAITranscriptModeration(
4590
4798
  transcriptResult.transcriptText,
@@ -4598,35 +4806,28 @@ async function getModerationScores(assetId, options = {}) {
4598
4806
  throw new Error(`Unsupported moderation provider: ${provider}`);
4599
4807
  }
4600
4808
  } else {
4601
- const thumbnailUrls = maxSamples === void 0 ? (
4602
- // Generate thumbnail URLs (signed if needed) using existing interval-based logic.
4603
- await getThumbnailUrls(playbackId, duration, {
4604
- interval: thumbnailInterval,
4809
+ const estimatedIntervalCount = duration <= 50 ? 5 : Math.ceil(duration / thumbnailInterval);
4810
+ const thumbnailUrls = maxSamples !== void 0 && estimatedIntervalCount > maxSamples ? await getThumbnailUrlsFromTimestamps(
4811
+ playbackId,
4812
+ planSamplingTimestamps({
4813
+ duration_sec: duration,
4814
+ max_candidates: maxSamples,
4815
+ trim_start_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
4816
+ trim_end_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
4817
+ fps: videoTrackFps,
4818
+ base_cadence_hz: thumbnailInterval > 0 ? 1 / thumbnailInterval : void 0
4819
+ }),
4820
+ {
4605
4821
  width: thumbnailWidth,
4606
4822
  shouldSign: policy === "signed",
4607
4823
  credentials
4608
- })
4609
- ) : (
4610
- // In maxSamples mode, sample valid timestamps over the trimmed usable span.
4611
- // Use proportional trims (≈ duration/6, capped at 5s) to stay well inside the
4612
- // renderable range — Mux can't always serve thumbnails at the very edges.
4613
- await getThumbnailUrlsFromTimestamps(
4614
- playbackId,
4615
- planSamplingTimestamps({
4616
- duration_sec: duration,
4617
- max_candidates: maxSamples,
4618
- trim_start_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
4619
- trim_end_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
4620
- fps: videoTrackFps,
4621
- base_cadence_hz: thumbnailInterval > 0 ? 1 / thumbnailInterval : void 0
4622
- }),
4623
- {
4624
- width: thumbnailWidth,
4625
- shouldSign: policy === "signed",
4626
- credentials
4627
- }
4628
- )
4629
- );
4824
+ }
4825
+ ) : await getThumbnailUrls(playbackId, duration, {
4826
+ interval: thumbnailInterval,
4827
+ width: thumbnailWidth,
4828
+ shouldSign: policy === "signed",
4829
+ credentials
4830
+ });
4630
4831
  thumbnailCount = thumbnailUrls.length;
4631
4832
  if (provider === "openai") {
4632
4833
  thumbnailScores = await requestOpenAIModeration(
@@ -4650,20 +4851,40 @@ async function getModerationScores(assetId, options = {}) {
4650
4851
  }
4651
4852
  }
4652
4853
  const failed = thumbnailScores.filter((s) => s.error);
4653
- if (failed.length > 0) {
4854
+ const successful = thumbnailScores.filter((s) => !s.error);
4855
+ if (successful.length === 0) {
4654
4856
  const details = failed.map((s) => `${s.url}: ${s.errorMessage || "Unknown error"}`).join("; ");
4655
4857
  throw new Error(
4656
- `Moderation failed for ${failed.length}/${thumbnailScores.length} thumbnail(s): ${details}`
4858
+ `Moderation failed for all ${thumbnailScores.length} sample(s): ${details}`
4859
+ );
4860
+ }
4861
+ if (failed.length > 0) {
4862
+ console.warn(
4863
+ `Moderation had partial failures (${failed.length}/${thumbnailScores.length}); continuing with successful samples.`
4657
4864
  );
4658
4865
  }
4659
- const maxSexual = Math.max(...thumbnailScores.map((s) => s.sexual));
4660
- const maxViolence = Math.max(...thumbnailScores.map((s) => s.violence));
4866
+ const maxSexual = Math.max(...successful.map((s) => s.sexual));
4867
+ const maxViolence = Math.max(...successful.map((s) => s.violence));
4661
4868
  const finalThresholds = { ...DEFAULT_THRESHOLDS, ...thresholds };
4869
+ const requestedSampleCount = thumbnailScores.length;
4870
+ const successfulSampleCount = successful.length;
4871
+ const failedSampleCount = failed.length;
4872
+ const sampleCoverage = successfulSampleCount / requestedSampleCount;
4873
+ const hasEnoughSuccessfulSamples = mode === "transcript" || successfulSampleCount >= MIN_SUCCESSFUL_THUMBNAILS_FOR_CONFIDENT_THRESHOLDING;
4874
+ const isLowConfidence = sampleCoverage < MIN_SAMPLE_COVERAGE_FOR_CONFIDENT_THRESHOLDING || !hasEnoughSuccessfulSamples;
4662
4875
  return {
4663
4876
  assetId,
4664
4877
  mode,
4665
4878
  isAudioOnly,
4666
4879
  thumbnailScores,
4880
+ coverage: {
4881
+ requestedSampleCount,
4882
+ successfulSampleCount,
4883
+ failedSampleCount,
4884
+ sampleCoverage,
4885
+ isPartial: failedSampleCount > 0,
4886
+ isLowConfidence
4887
+ },
4667
4888
  usage: {
4668
4889
  metadata: {
4669
4890
  assetDurationSeconds: duration,
@@ -4865,6 +5086,9 @@ var SYSTEM_PROMPT5 = dedent6`
4865
5086
  <constraints>
4866
5087
  - Only describe what is clearly observable in the frames or explicitly stated in the transcript
4867
5088
  - Do not fabricate details or make unsupported assumptions
5089
+ - Do NOT use any metadata such as URLs, file paths, domain names, file names,
5090
+ playback IDs, or technical parameters visible in this request. These are
5091
+ delivery infrastructure and are unrelated to the media content itself.
4868
5092
  - Return structured data matching the requested schema
4869
5093
  - Output only the JSON object; no markdown or extra text
4870
5094
  - When a <language> section is provided, all output text MUST be written in that language
@@ -4891,7 +5115,7 @@ var SYSTEM_PROMPT5 = dedent6`
4891
5115
 
4892
5116
  Write as if describing reality, not describing a recording of reality.
4893
5117
  </language_guidelines>`;
4894
- var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent6`
5118
+ var AUDIO_ONLY_SYSTEM_PROMPT3 = dedent6`
4895
5119
  <role>
4896
5120
  You are an audio content analyst specializing in transcript analysis and metadata generation.
4897
5121
  </role>
@@ -4919,6 +5143,9 @@ var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent6`
4919
5143
  <constraints>
4920
5144
  - Only describe what is explicitly stated or strongly implied in the transcript
4921
5145
  - Do not fabricate details or make unsupported assumptions
5146
+ - Do NOT use any metadata such as URLs, file paths, domain names, file names,
5147
+ playback IDs, or technical parameters visible in this request. These are
5148
+ delivery infrastructure and are unrelated to the media content itself.
4922
5149
  - Return structured data matching the requested schema
4923
5150
  - Focus entirely on audio/spoken content - there are no visual elements
4924
5151
  - Output only the JSON object; no markdown or extra text
@@ -5068,6 +5295,7 @@ async function getSummaryAndTags(assetId, options) {
5068
5295
  const {
5069
5296
  provider = "openai",
5070
5297
  model,
5298
+ languageCode,
5071
5299
  tone = "neutral",
5072
5300
  includeTranscript = true,
5073
5301
  cleanTranscript = true,
@@ -5081,8 +5309,9 @@ async function getSummaryAndTags(assetId, options) {
5081
5309
  outputLanguageCode
5082
5310
  } = options ?? {};
5083
5311
  if (!VALID_TONES.includes(tone)) {
5084
- throw new Error(
5085
- `Invalid tone "${tone}". Valid tones are: ${VALID_TONES.join(", ")}`
5312
+ throw new MuxAiError(
5313
+ `Invalid tone "${tone}". Valid tones are: ${VALID_TONES.join(", ")}.`,
5314
+ { type: "validation_error" }
5086
5315
  );
5087
5316
  }
5088
5317
  const modelConfig = resolveLanguageModelConfig({
@@ -5095,17 +5324,20 @@ async function getSummaryAndTags(assetId, options) {
5095
5324
  const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
5096
5325
  const isAudioOnly = isAudioOnlyAsset(assetData);
5097
5326
  if (isAudioOnly && !includeTranscript) {
5098
- throw new Error(
5099
- "Audio-only assets require a transcript. Set includeTranscript: true and ensure the asset has a ready text track (captions/subtitles)."
5327
+ throw new MuxAiError(
5328
+ "Audio-only assets require a transcript. Set includeTranscript: true and ensure the asset has a ready text track (captions/subtitles).",
5329
+ { type: "validation_error" }
5100
5330
  );
5101
5331
  }
5102
5332
  const signingContext = await resolveMuxSigningContext(workflowCredentials);
5103
5333
  if (policy === "signed" && !signingContext) {
5104
- throw new Error(
5105
- "Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
5334
+ throw new MuxAiError(
5335
+ "Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
5336
+ { type: "validation_error" }
5106
5337
  );
5107
5338
  }
5108
5339
  const transcriptResult = includeTranscript ? await fetchTranscriptForAsset(assetData, playbackId, {
5340
+ languageCode,
5109
5341
  cleanTranscript,
5110
5342
  shouldSign: policy === "signed",
5111
5343
  credentials: workflowCredentials,
@@ -5127,7 +5359,7 @@ async function getSummaryAndTags(assetId, options) {
5127
5359
  });
5128
5360
  let analysisResponse;
5129
5361
  let imageUrl;
5130
- const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT2 : SYSTEM_PROMPT5;
5362
+ const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT3 : SYSTEM_PROMPT5;
5131
5363
  try {
5132
5364
  if (isAudioOnly) {
5133
5365
  analysisResponse = await analyzeAudioOnly(
@@ -5163,18 +5395,17 @@ async function getSummaryAndTags(assetId, options) {
5163
5395
  }
5164
5396
  } catch (error) {
5165
5397
  const contentType = isAudioOnly ? "audio" : "video";
5166
- throw new Error(
5167
- `Failed to analyze ${contentType} content with ${provider}: ${error instanceof Error ? error.message : "Unknown error"}`
5168
- );
5398
+ wrapError(error, `Failed to analyze ${contentType} content with ${provider}`);
5169
5399
  }
5170
5400
  if (!analysisResponse.result) {
5171
- throw new Error(`Failed to analyze video content for asset ${assetId}`);
5401
+ const contentType = isAudioOnly ? "audio" : "video";
5402
+ throw new MuxAiError(`Failed to analyze ${contentType} content for asset ${assetId}.`);
5172
5403
  }
5173
5404
  if (!analysisResponse.result.title) {
5174
- throw new Error(`Failed to generate title for asset ${assetId}`);
5405
+ throw new MuxAiError(`Failed to generate title for asset ${assetId}.`);
5175
5406
  }
5176
5407
  if (!analysisResponse.result.description) {
5177
- throw new Error(`Failed to generate description for asset ${assetId}`);
5408
+ throw new MuxAiError(`Failed to generate description for asset ${assetId}.`);
5178
5409
  }
5179
5410
  return {
5180
5411
  assetId,
@@ -5233,12 +5464,11 @@ async function requestStaticRenditionCreation(assetId, credentials) {
5233
5464
  } catch (error) {
5234
5465
  const statusCode = error?.status ?? error?.statusCode;
5235
5466
  const messages = error?.error?.messages;
5236
- const alreadyDefined = messages?.some((message2) => message2.toLowerCase().includes("already defined")) ?? error?.message?.toLowerCase().includes("already defined");
5467
+ const alreadyDefined = messages?.some((message) => message.toLowerCase().includes("already defined")) ?? error?.message?.toLowerCase().includes("already defined");
5237
5468
  if (statusCode === 409 || alreadyDefined) {
5238
5469
  return;
5239
5470
  }
5240
- const message = error instanceof Error ? error.message : "Unknown error";
5241
- throw new Error(`Failed to request static rendition from Mux: ${message}`);
5471
+ wrapError(error, "Failed to request static rendition from Mux");
5242
5472
  }
5243
5473
  }
5244
5474
  async function waitForAudioStaticRendition({
@@ -5272,13 +5502,14 @@ async function waitForAudioStaticRendition({
5272
5502
  `\u231B Waiting for static rendition (attempt ${attempt}/${STATIC_RENDITION_MAX_ATTEMPTS}) \u2192 ${currentStatus}`
5273
5503
  );
5274
5504
  if (currentStatus === "errored") {
5275
- throw new Error(
5505
+ throw new MuxAiError(
5276
5506
  "Mux failed to create the static rendition for this asset. Please check the asset in the Mux dashboard."
5277
5507
  );
5278
5508
  }
5279
5509
  }
5280
- throw new Error(
5281
- "Timed out waiting for the static rendition to become ready. Please try again in a moment."
5510
+ throw new MuxAiError(
5511
+ "Timed out waiting for the static rendition to become ready. Please try again in a moment.",
5512
+ { type: "timeout_error", retryable: true }
5282
5513
  );
5283
5514
  }
5284
5515
  async function fetchAudioFromMux(audioUrl) {
@@ -5424,22 +5655,25 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
5424
5655
  fromLanguageCode,
5425
5656
  numSpeakers = 0,
5426
5657
  // 0 = auto-detect
5427
- uploadToMux = true,
5658
+ uploadToS3: uploadToS3Option,
5659
+ uploadToMux: uploadToMuxOption,
5428
5660
  storageAdapter,
5429
5661
  credentials: providedCredentials
5430
5662
  } = options;
5431
5663
  if (provider !== "elevenlabs") {
5432
- throw new Error("Only ElevenLabs provider is currently supported for audio translation");
5664
+ throw new MuxAiError("Only ElevenLabs provider is currently supported for audio translation.", { type: "validation_error" });
5433
5665
  }
5434
5666
  const credentials = providedCredentials;
5435
5667
  const effectiveStorageAdapter = storageAdapter;
5668
+ const uploadToMux = uploadToMuxOption !== false;
5669
+ const uploadToS3 = uploadToS3Option || uploadToMux;
5436
5670
  const s3Endpoint = options.s3Endpoint ?? env_default.S3_ENDPOINT;
5437
5671
  const s3Region = options.s3Region ?? env_default.S3_REGION ?? "auto";
5438
5672
  const s3Bucket = options.s3Bucket ?? env_default.S3_BUCKET;
5439
5673
  const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
5440
5674
  const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
5441
- if (uploadToMux && (!s3Endpoint || !s3Bucket || !effectiveStorageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
5442
- throw new Error("Storage configuration is required for uploading to Mux. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.");
5675
+ if (uploadToS3 && (!s3Endpoint || !s3Bucket || !effectiveStorageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
5676
+ throw new MuxAiError("Storage configuration is required for uploading. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.", { type: "validation_error" });
5443
5677
  }
5444
5678
  const { asset: initialAsset, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
5445
5679
  const assetDurationSeconds = getAssetDurationSecondsFromAsset(initialAsset);
@@ -5454,8 +5688,9 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
5454
5688
  }
5455
5689
  const audioRendition = getReadyAudioStaticRendition(currentAsset);
5456
5690
  if (!audioRendition) {
5457
- throw new Error(
5458
- "Unable to obtain an audio-only static rendition for this asset. Please verify static renditions are enabled in Mux."
5691
+ throw new MuxAiError(
5692
+ "Unable to obtain an audio-only static rendition for this asset. Please verify static renditions are enabled in Mux.",
5693
+ { type: "validation_error" }
5459
5694
  );
5460
5695
  }
5461
5696
  let audioUrl = `https://stream.mux.com/${playbackId}/audio.m4a`;
@@ -5467,7 +5702,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
5467
5702
  try {
5468
5703
  audioBuffer = await fetchAudioFromMux(audioUrl);
5469
5704
  } catch (error) {
5470
- throw new Error(`Failed to fetch audio from Mux: ${error instanceof Error ? error.message : "Unknown error"}`);
5705
+ wrapError(error, "Failed to fetch audio from Mux");
5471
5706
  }
5472
5707
  console.warn("\u{1F399}\uFE0F Creating dubbing job in ElevenLabs...");
5473
5708
  const elevenLabsLangCode = toISO639_3(toLanguageCode);
@@ -5488,7 +5723,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
5488
5723
  });
5489
5724
  console.warn(`\u2705 Dubbing job created with ID: ${dubbingId}`);
5490
5725
  } catch (error) {
5491
- throw new Error(`Failed to create ElevenLabs dubbing job: ${error instanceof Error ? error.message : "Unknown error"}`);
5726
+ wrapError(error, "Failed to create ElevenLabs dubbing job");
5492
5727
  }
5493
5728
  console.warn("\u23F3 Waiting for dubbing to complete...");
5494
5729
  let dubbingStatus = "dubbing";
@@ -5509,77 +5744,72 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
5509
5744
  throw new Error("ElevenLabs dubbing job failed");
5510
5745
  }
5511
5746
  } catch (error) {
5512
- throw new Error(`Failed to check dubbing status: ${error instanceof Error ? error.message : "Unknown error"}`);
5747
+ wrapError(error, "Failed to check dubbing status");
5513
5748
  }
5514
5749
  }
5515
5750
  if (dubbingStatus !== "dubbed") {
5516
- throw new Error(`Dubbing job timed out or failed. Final status: ${dubbingStatus}`);
5751
+ throw new MuxAiError("Audio translation timed out or failed. Please try again.", { type: "timeout_error", retryable: true });
5517
5752
  }
5518
5753
  console.warn("\u2705 Dubbing completed successfully!");
5519
- if (!uploadToMux) {
5520
- const targetLanguage2 = getLanguageCodePair(toLanguageCode);
5521
- return {
5522
- assetId,
5523
- targetLanguageCode: targetLanguage2.iso639_1,
5524
- targetLanguage: targetLanguage2,
5525
- dubbingId
5526
- };
5527
- }
5528
- console.warn("\u{1F4E5} Downloading dubbed audio from ElevenLabs...");
5529
- let dubbedAudioBuffer;
5530
- try {
5531
- const requestedLangCode = toISO639_3(toLanguageCode);
5532
- let downloadLangCode = targetLanguages.find(
5533
- (lang) => lang === requestedLangCode
5534
- ) ?? targetLanguages.find(
5535
- (lang) => lang.toLowerCase() === requestedLangCode.toLowerCase()
5536
- );
5537
- if (!downloadLangCode && targetLanguages.length > 0) {
5538
- downloadLangCode = targetLanguages[0];
5539
- console.warn(`\u26A0\uFE0F Requested language "${requestedLangCode}" not found in target_languages. Using "${downloadLangCode}" instead.`);
5540
- }
5541
- if (!downloadLangCode) {
5542
- downloadLangCode = requestedLangCode;
5543
- console.warn(`\u26A0\uFE0F No target_languages available from ElevenLabs status. Using requested language code: ${requestedLangCode}`);
5544
- }
5545
- dubbedAudioBuffer = await downloadDubbedAudioFromElevenLabs({
5546
- dubbingId,
5547
- languageCode: downloadLangCode,
5548
- credentials
5549
- });
5550
- console.warn("\u2705 Dubbed audio downloaded successfully!");
5551
- } catch (error) {
5552
- throw new Error(`Failed to download dubbed audio: ${error instanceof Error ? error.message : "Unknown error"}`);
5553
- }
5554
- console.warn("\u{1F4E4} Uploading dubbed audio to S3-compatible storage...");
5555
5754
  let presignedUrl;
5556
- try {
5557
- presignedUrl = await uploadDubbedAudioToS3({
5558
- dubbedAudioBuffer,
5559
- assetId,
5560
- toLanguageCode,
5561
- s3Endpoint,
5562
- s3Region,
5563
- s3Bucket,
5564
- storageAdapter: effectiveStorageAdapter,
5565
- s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
5566
- });
5567
- } catch (error) {
5568
- throw new Error(`Failed to upload audio to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
5569
- }
5570
- console.warn("\u{1F4F9} Adding dubbed audio track to Mux asset...");
5571
5755
  let uploadedTrackId;
5572
- const muxLangCode = toISO639_1(toLanguageCode);
5573
- try {
5574
- uploadedTrackId = await createAudioTrackOnMux(assetId, muxLangCode, presignedUrl, credentials);
5575
- const languageName = new Intl.DisplayNames(["en"], { type: "language" }).of(muxLangCode) || muxLangCode.toUpperCase();
5576
- const trackName = `${languageName} (auto-dubbed)`;
5577
- console.warn(`\u2705 Track added to Mux asset with ID: ${uploadedTrackId}`);
5578
- console.warn(`\u{1F4CB} Track name: "${trackName}"`);
5579
- } catch (error) {
5580
- console.warn(`\u26A0\uFE0F Failed to add audio track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
5581
- console.warn("\u{1F517} You can manually add the track using this presigned URL:");
5582
- console.warn(presignedUrl);
5756
+ if (uploadToS3) {
5757
+ console.warn("\u{1F4E5} Downloading dubbed audio from ElevenLabs...");
5758
+ let dubbedAudioBuffer;
5759
+ try {
5760
+ const requestedLangCode = toISO639_3(toLanguageCode);
5761
+ let downloadLangCode = targetLanguages.find(
5762
+ (lang) => lang === requestedLangCode
5763
+ ) ?? targetLanguages.find(
5764
+ (lang) => lang.toLowerCase() === requestedLangCode.toLowerCase()
5765
+ );
5766
+ if (!downloadLangCode && targetLanguages.length > 0) {
5767
+ downloadLangCode = targetLanguages[0];
5768
+ console.warn(`\u26A0\uFE0F Requested language "${requestedLangCode}" not found in target_languages. Using "${downloadLangCode}" instead.`);
5769
+ }
5770
+ if (!downloadLangCode) {
5771
+ downloadLangCode = requestedLangCode;
5772
+ console.warn(`\u26A0\uFE0F No target_languages available from ElevenLabs status. Using requested language code: ${requestedLangCode}`);
5773
+ }
5774
+ dubbedAudioBuffer = await downloadDubbedAudioFromElevenLabs({
5775
+ dubbingId,
5776
+ languageCode: downloadLangCode,
5777
+ credentials
5778
+ });
5779
+ console.warn("\u2705 Dubbed audio downloaded successfully!");
5780
+ } catch (error) {
5781
+ wrapError(error, "Failed to download dubbed audio");
5782
+ }
5783
+ console.warn("\u{1F4E4} Uploading dubbed audio to S3-compatible storage...");
5784
+ try {
5785
+ presignedUrl = await uploadDubbedAudioToS3({
5786
+ dubbedAudioBuffer,
5787
+ assetId,
5788
+ toLanguageCode,
5789
+ s3Endpoint,
5790
+ s3Region,
5791
+ s3Bucket,
5792
+ storageAdapter: effectiveStorageAdapter,
5793
+ s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
5794
+ });
5795
+ } catch (error) {
5796
+ wrapError(error, "Failed to upload audio to S3");
5797
+ }
5798
+ if (uploadToMux) {
5799
+ console.warn("\u{1F4F9} Adding dubbed audio track to Mux asset...");
5800
+ const muxLangCode = toISO639_1(toLanguageCode);
5801
+ try {
5802
+ uploadedTrackId = await createAudioTrackOnMux(assetId, muxLangCode, presignedUrl, credentials);
5803
+ const languageName = new Intl.DisplayNames(["en"], { type: "language" }).of(muxLangCode) || muxLangCode.toUpperCase();
5804
+ const trackName = `${languageName} (auto-dubbed)`;
5805
+ console.warn(`\u2705 Track added to Mux asset with ID: ${uploadedTrackId}`);
5806
+ console.warn(`\u{1F4CB} Track name: "${trackName}"`);
5807
+ } catch (error) {
5808
+ console.warn(`\u26A0\uFE0F Failed to add audio track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
5809
+ console.warn("\u{1F517} You can manually add the track using this presigned URL:");
5810
+ console.warn(presignedUrl);
5811
+ }
5812
+ }
5583
5813
  }
5584
5814
  const targetLanguage = getLanguageCodePair(toLanguageCode);
5585
5815
  return {
@@ -5910,9 +6140,7 @@ async function translateChunkWithFallback({
5910
6140
  };
5911
6141
  } catch (error) {
5912
6142
  if (!shouldSplitChunkTranslationError(error) || chunk.cueCount <= 1) {
5913
- throw new Error(
5914
- `Chunk ${chunk.id} failed for ${Math.round(chunk.startTime)}s-${Math.round(chunk.endTime)}s: ${error instanceof Error ? error.message : "Unknown error"}`
5915
- );
6143
+ wrapError(error, `Chunk ${chunk.id} failed for ${Math.round(chunk.startTime)}s-${Math.round(chunk.endTime)}s`);
5916
6144
  }
5917
6145
  const [leftChunk, rightChunk] = splitTranslationChunkAtMidpoint(chunk);
5918
6146
  const [leftResult, rightResult] = await Promise.all([
@@ -6021,7 +6249,7 @@ async function uploadVttToS3({
6021
6249
  expiresInSeconds: s3SignedUrlExpirySeconds ?? 86400
6022
6250
  }, storageAdapter);
6023
6251
  }
6024
- async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, options) {
6252
+ async function translateCaptions(assetId, trackId, toLanguageCode, options) {
6025
6253
  "use workflow";
6026
6254
  const {
6027
6255
  provider = "openai",
@@ -6029,6 +6257,7 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
6029
6257
  s3Endpoint: providedS3Endpoint,
6030
6258
  s3Region: providedS3Region,
6031
6259
  s3Bucket: providedS3Bucket,
6260
+ uploadToS3: uploadToS3Option,
6032
6261
  uploadToMux: uploadToMuxOption,
6033
6262
  storageAdapter,
6034
6263
  credentials: providedCredentials,
@@ -6042,53 +6271,46 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
6042
6271
  const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
6043
6272
  const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
6044
6273
  const uploadToMux = uploadToMuxOption !== false;
6274
+ const uploadToS3 = uploadToS3Option || uploadToMux;
6045
6275
  const modelConfig = resolveLanguageModelConfig({
6046
6276
  ...options,
6047
6277
  model,
6048
6278
  provider
6049
6279
  });
6050
- if (uploadToMux && (!s3Endpoint || !s3Bucket || !effectiveStorageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
6051
- throw new Error("Storage configuration is required for uploading to Mux. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.");
6280
+ if (uploadToS3 && (!s3Endpoint || !s3Bucket || !effectiveStorageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
6281
+ throw new MuxAiError("Storage configuration is required for uploading. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.", { type: "validation_error" });
6052
6282
  }
6053
6283
  const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
6054
6284
  const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
6055
- const isAudioOnly = isAudioOnlyAsset(assetData);
6056
6285
  const signingContext = await resolveMuxSigningContext(credentials);
6057
6286
  if (policy === "signed" && !signingContext) {
6058
- throw new Error(
6059
- "Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
6287
+ throw new MuxAiError(
6288
+ "Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
6289
+ { type: "validation_error" }
6060
6290
  );
6061
6291
  }
6062
6292
  const readyTextTracks = getReadyTextTracks(assetData);
6063
- if (!readyTextTracks.length) {
6064
- throw new Error("No ready text tracks found for this asset");
6065
- }
6066
- let sourceTextTrack = readyTextTracks.find(
6067
- (track) => track.text_type === "subtitles" && track.language_code === fromLanguageCode
6068
- );
6069
- if (!sourceTextTrack && isAudioOnly && readyTextTracks.length === 1) {
6070
- sourceTextTrack = readyTextTracks[0];
6071
- }
6293
+ const sourceTextTrack = readyTextTracks.find((t) => t.id === trackId);
6072
6294
  if (!sourceTextTrack) {
6073
- const availableLanguages = readyTextTracks.map((t) => t.language_code).filter(Boolean).join(", ");
6074
- if (isAudioOnly) {
6075
- throw new Error(
6076
- `No transcript track found with language code '${fromLanguageCode}' for this asset. Audio-only assets require a transcript. Available languages: ${availableLanguages || "none"}`
6077
- );
6078
- }
6079
- throw new Error(
6080
- `No ready text track found with language code '${fromLanguageCode}' for this asset. Available languages: ${availableLanguages || "none"}`
6295
+ const availableTrackIds = readyTextTracks.map((t) => t.id).filter(Boolean).join(", ");
6296
+ throw new MuxAiError(
6297
+ `Track ${trackId} not found or not ready on asset ${assetId}. Available track IDs: ${availableTrackIds || "none"}.`,
6298
+ { type: "validation_error" }
6081
6299
  );
6082
6300
  }
6083
- if (!sourceTextTrack.id) {
6084
- throw new Error("Transcript track is missing an id");
6301
+ const fromLanguageCode = sourceTextTrack.language_code;
6302
+ if (!fromLanguageCode) {
6303
+ throw new MuxAiError(
6304
+ `Track ${trackId} is missing language metadata. Cannot determine source language for translation.`,
6305
+ { type: "validation_error" }
6306
+ );
6085
6307
  }
6086
- const vttUrl = await buildTranscriptUrl(playbackId, sourceTextTrack.id, policy === "signed", credentials);
6308
+ const vttUrl = await buildTranscriptUrl(playbackId, trackId, policy === "signed", credentials);
6087
6309
  let vttContent;
6088
6310
  try {
6089
6311
  vttContent = await fetchVttFromMux(vttUrl);
6090
6312
  } catch (error) {
6091
- throw new Error(`Failed to fetch VTT content: ${error instanceof Error ? error.message : "Unknown error"}`);
6313
+ wrapError(error, "Failed to fetch VTT content");
6092
6314
  }
6093
6315
  let translatedVtt;
6094
6316
  let usage;
@@ -6106,7 +6328,7 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
6106
6328
  translatedVtt = result.translatedVtt;
6107
6329
  usage = result.usage;
6108
6330
  } catch (error) {
6109
- throw new Error(`Failed to translate VTT with ${modelConfig.provider}: ${error instanceof Error ? error.message : "Unknown error"}`);
6331
+ wrapError(error, `Failed to translate VTT with ${modelConfig.provider}`);
6110
6332
  }
6111
6333
  const usageWithMetadata = usage ? {
6112
6334
  ...usage,
@@ -6116,50 +6338,43 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
6116
6338
  } : void 0;
6117
6339
  const sourceLanguage = getLanguageCodePair(fromLanguageCode);
6118
6340
  const targetLanguage = getLanguageCodePair(toLanguageCode);
6119
- if (!uploadToMux) {
6120
- return {
6121
- assetId,
6122
- sourceLanguageCode: fromLanguageCode,
6123
- targetLanguageCode: toLanguageCode,
6124
- sourceLanguage,
6125
- targetLanguage,
6126
- originalVtt: vttContent,
6127
- translatedVtt,
6128
- usage: usageWithMetadata
6129
- };
6130
- }
6131
6341
  let presignedUrl;
6132
- try {
6133
- presignedUrl = await uploadVttToS3({
6134
- translatedVtt,
6135
- assetId,
6136
- fromLanguageCode,
6137
- toLanguageCode,
6138
- s3Endpoint,
6139
- s3Region,
6140
- s3Bucket,
6141
- storageAdapter: effectiveStorageAdapter,
6142
- s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
6143
- });
6144
- } catch (error) {
6145
- throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
6146
- }
6147
6342
  let uploadedTrackId;
6148
- try {
6149
- const languageName = getLanguageName(toLanguageCode);
6150
- const trackName = `${languageName} (auto-translated)`;
6151
- uploadedTrackId = await createTextTrackOnMux(
6152
- assetId,
6153
- toLanguageCode,
6154
- trackName,
6155
- presignedUrl,
6156
- credentials
6157
- );
6158
- } catch (error) {
6159
- console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
6343
+ if (uploadToS3) {
6344
+ try {
6345
+ presignedUrl = await uploadVttToS3({
6346
+ translatedVtt,
6347
+ assetId,
6348
+ fromLanguageCode,
6349
+ toLanguageCode,
6350
+ s3Endpoint,
6351
+ s3Region,
6352
+ s3Bucket,
6353
+ storageAdapter: effectiveStorageAdapter,
6354
+ s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
6355
+ });
6356
+ } catch (error) {
6357
+ wrapError(error, "Failed to upload VTT to S3");
6358
+ }
6359
+ if (uploadToMux) {
6360
+ try {
6361
+ const languageName = getLanguageName(toLanguageCode);
6362
+ const trackName = `${languageName} (auto-translated)`;
6363
+ uploadedTrackId = await createTextTrackOnMux(
6364
+ assetId,
6365
+ toLanguageCode,
6366
+ trackName,
6367
+ presignedUrl,
6368
+ credentials
6369
+ );
6370
+ } catch (error) {
6371
+ console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
6372
+ }
6373
+ }
6160
6374
  }
6161
6375
  return {
6162
6376
  assetId,
6377
+ trackId,
6163
6378
  sourceLanguageCode: fromLanguageCode,
6164
6379
  targetLanguageCode: toLanguageCode,
6165
6380
  sourceLanguage,
@@ -6172,6 +6387,7 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
6172
6387
  };
6173
6388
  }
6174
6389
  export {
6390
+ MuxAiError,
6175
6391
  WorkflowStorageClient,
6176
6392
  createWorkflowStorageClient,
6177
6393
  decryptFromWorkflow,
@@ -6179,6 +6395,7 @@ export {
6179
6395
  primitives_exports as primitives,
6180
6396
  setWorkflowCredentialsProvider,
6181
6397
  version,
6182
- workflows_exports as workflows
6398
+ workflows_exports as workflows,
6399
+ wrapError
6183
6400
  };
6184
6401
  //# sourceMappingURL=index.js.map