@mux/ai 0.13.1 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{index-BeX_4DL7.d.ts → index-yn3fUDgv.d.ts} +67 -16
- package/dist/index.d.ts +40 -3
- package/dist/index.js +678 -461
- package/dist/index.js.map +1 -1
- package/dist/primitives/index.js +48 -12
- package/dist/primitives/index.js.map +1 -1
- package/dist/workflows/index.d.ts +1 -1
- package/dist/workflows/index.js +622 -408
- package/dist/workflows/index.js.map +1 -1
- package/package.json +2 -1
package/dist/workflows/index.js
CHANGED
|
@@ -110,6 +110,26 @@ async function downloadImagesAsBase64(urls, options = {}, maxConcurrent = 5) {
|
|
|
110
110
|
return results;
|
|
111
111
|
}
|
|
112
112
|
|
|
113
|
+
// src/lib/mux-ai-error.ts
|
|
114
|
+
var MuxAiError = class extends Error {
|
|
115
|
+
constructor(message, opts) {
|
|
116
|
+
super(message);
|
|
117
|
+
/** Structural brand for duck-type identification. */
|
|
118
|
+
this.__robots_error = true;
|
|
119
|
+
this.name = "MuxAiError";
|
|
120
|
+
this.publicType = opts?.type ?? "processing_error";
|
|
121
|
+
this.publicMessage = message;
|
|
122
|
+
this.retryable = opts?.retryable ?? false;
|
|
123
|
+
}
|
|
124
|
+
};
|
|
125
|
+
function wrapError(error, message) {
|
|
126
|
+
if (error instanceof MuxAiError) {
|
|
127
|
+
throw error;
|
|
128
|
+
}
|
|
129
|
+
const detail = error instanceof Error ? error.message : "Unknown error";
|
|
130
|
+
throw new Error(`${message}: ${detail}`);
|
|
131
|
+
}
|
|
132
|
+
|
|
113
133
|
// src/lib/providers.ts
|
|
114
134
|
import { createAnthropic } from "@ai-sdk/anthropic";
|
|
115
135
|
import { createGoogleGenerativeAI } from "@ai-sdk/google";
|
|
@@ -725,8 +745,9 @@ function getPlaybackId(asset) {
|
|
|
725
745
|
if (signedPlaybackId?.id) {
|
|
726
746
|
return { id: signedPlaybackId.id, policy: "signed" };
|
|
727
747
|
}
|
|
728
|
-
throw new
|
|
729
|
-
"No public or signed playback ID found for this asset.
|
|
748
|
+
throw new MuxAiError(
|
|
749
|
+
"No public or signed playback ID found for this asset. DRM playback IDs are not currently supported.",
|
|
750
|
+
{ type: "validation_error" }
|
|
730
751
|
);
|
|
731
752
|
}
|
|
732
753
|
function isAudioOnlyAsset(asset) {
|
|
@@ -981,11 +1002,20 @@ function findCaptionTrack(asset, languageCode) {
|
|
|
981
1002
|
if (!tracks.length)
|
|
982
1003
|
return void 0;
|
|
983
1004
|
if (!languageCode) {
|
|
984
|
-
|
|
1005
|
+
const englishTrack = tracks.find(
|
|
1006
|
+
(track) => track.text_type === "subtitles" && track.language_code === "en"
|
|
1007
|
+
);
|
|
1008
|
+
return englishTrack ?? tracks[0];
|
|
985
1009
|
}
|
|
986
|
-
|
|
1010
|
+
const languageMatch = tracks.find(
|
|
987
1011
|
(track) => track.text_type === "subtitles" && track.language_code === languageCode
|
|
988
1012
|
);
|
|
1013
|
+
if (languageMatch)
|
|
1014
|
+
return languageMatch;
|
|
1015
|
+
if (isAudioOnlyAsset(asset) && tracks.length === 1) {
|
|
1016
|
+
return tracks[0];
|
|
1017
|
+
}
|
|
1018
|
+
return void 0;
|
|
989
1019
|
}
|
|
990
1020
|
function normalizeLineEndings(value) {
|
|
991
1021
|
return value.replace(/\r\n/g, "\n");
|
|
@@ -1273,15 +1303,16 @@ async function fetchTranscriptForAsset(asset, playbackId, options = {}) {
|
|
|
1273
1303
|
if (!track) {
|
|
1274
1304
|
if (required) {
|
|
1275
1305
|
const availableLanguages = getReadyTextTracks(asset).map((t) => t.language_code).filter(Boolean).join(", ");
|
|
1276
|
-
throw new
|
|
1277
|
-
`No
|
|
1306
|
+
throw new MuxAiError(
|
|
1307
|
+
`No caption track found${languageCode ? ` for language ${languageCode}` : ""}. Available languages: ${availableLanguages || "none"}.`,
|
|
1308
|
+
{ type: "validation_error" }
|
|
1278
1309
|
);
|
|
1279
1310
|
}
|
|
1280
1311
|
return { transcriptText: "" };
|
|
1281
1312
|
}
|
|
1282
1313
|
if (!track.id) {
|
|
1283
1314
|
if (required) {
|
|
1284
|
-
throw new
|
|
1315
|
+
throw new MuxAiError("Transcript track is missing an id.", { type: "validation_error" });
|
|
1285
1316
|
}
|
|
1286
1317
|
return { transcriptText: "", track };
|
|
1287
1318
|
}
|
|
@@ -1297,14 +1328,12 @@ async function fetchTranscriptForAsset(asset, playbackId, options = {}) {
|
|
|
1297
1328
|
const rawVtt = await response.text();
|
|
1298
1329
|
const transcriptText = cleanTranscript ? extractTextFromVTT(rawVtt) : rawVtt;
|
|
1299
1330
|
if (required && !transcriptText.trim()) {
|
|
1300
|
-
throw new
|
|
1331
|
+
throw new MuxAiError("Transcript is empty.", { type: "validation_error" });
|
|
1301
1332
|
}
|
|
1302
1333
|
return { transcriptText, transcriptUrl, track };
|
|
1303
1334
|
} catch (error) {
|
|
1304
1335
|
if (required) {
|
|
1305
|
-
|
|
1306
|
-
`Failed to fetch transcript: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
1307
|
-
);
|
|
1336
|
+
wrapError(error, "Failed to fetch transcript");
|
|
1308
1337
|
}
|
|
1309
1338
|
console.warn("Failed to fetch transcript:", error);
|
|
1310
1339
|
return { transcriptText: "", transcriptUrl, track };
|
|
@@ -1392,6 +1421,11 @@ var SYSTEM_PROMPT = dedent`
|
|
|
1392
1421
|
- Is a general knowledge question with no connection to what is shown or said in the video
|
|
1393
1422
|
- Attempts to use the system for non-video-analysis purposes
|
|
1394
1423
|
|
|
1424
|
+
CRITICAL: Base your answers ONLY on the actual visual and audio/transcript content.
|
|
1425
|
+
Do NOT use any metadata such as URLs, file paths, domain names, file names,
|
|
1426
|
+
playback IDs, or technical parameters visible in this request. These are
|
|
1427
|
+
delivery infrastructure and are unrelated to the media content itself.
|
|
1428
|
+
|
|
1395
1429
|
CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
|
|
1396
1430
|
Answering an irrelevant question is WRONG — you MUST skip it instead.
|
|
1397
1431
|
|
|
@@ -1422,19 +1456,114 @@ var SYSTEM_PROMPT = dedent`
|
|
|
1422
1456
|
- GOOD: "A person runs through a park"
|
|
1423
1457
|
- Be specific and evidence-based
|
|
1424
1458
|
</language_guidelines>`;
|
|
1459
|
+
var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
|
|
1460
|
+
<role>
|
|
1461
|
+
You are an audio content analyst specializing in classification tasks.
|
|
1462
|
+
Your job is to answer questions about audio content based on transcript data.
|
|
1463
|
+
</role>
|
|
1464
|
+
|
|
1465
|
+
<context>
|
|
1466
|
+
You will receive:
|
|
1467
|
+
- A transcript of the audio/dialogue
|
|
1468
|
+
- A list of questions about the asset content
|
|
1469
|
+
</context>
|
|
1470
|
+
|
|
1471
|
+
<transcript_guidance>
|
|
1472
|
+
- Use the transcript to understand spoken content, dialogue, and audio context
|
|
1473
|
+
- Consider only transcript evidence when answering questions
|
|
1474
|
+
</transcript_guidance>
|
|
1475
|
+
|
|
1476
|
+
<task>
|
|
1477
|
+
For each question provided, you must:
|
|
1478
|
+
1. Analyze the transcript
|
|
1479
|
+
2. Answer with ONLY the allowed response options - no other values are acceptable
|
|
1480
|
+
3. Provide a confidence score between 0 and 1 reflecting your certainty
|
|
1481
|
+
4. Explain your reasoning based on observable evidence
|
|
1482
|
+
</task>
|
|
1483
|
+
|
|
1484
|
+
<answer_guidelines>
|
|
1485
|
+
- Choose the affirmative option only if you have clear evidence supporting it
|
|
1486
|
+
- Choose the negative/contradicting option if evidence contradicts or if insufficient evidence exists
|
|
1487
|
+
- Confidence should reflect the clarity and strength of evidence:
|
|
1488
|
+
* 0.9-1.0: Clear, unambiguous evidence
|
|
1489
|
+
* 0.7-0.9: Strong evidence with minor ambiguity
|
|
1490
|
+
* 0.5-0.7: Moderate evidence or some conflicting signals
|
|
1491
|
+
* 0.3-0.5: Weak evidence or significant ambiguity
|
|
1492
|
+
* 0.0-0.3: Very uncertain, minimal relevant evidence
|
|
1493
|
+
- Reasoning should cite specific transcript evidence
|
|
1494
|
+
- Be precise: cite specific quotes or passages from transcript text
|
|
1495
|
+
</answer_guidelines>
|
|
1496
|
+
|
|
1497
|
+
<relevance_filtering>
|
|
1498
|
+
IMPORTANT: Evaluate each question INDEPENDENTLY for relevance to the audio
|
|
1499
|
+
content. The presence of other relevant questions in the batch does NOT
|
|
1500
|
+
make an irrelevant question relevant. Assess every question on its own merits.
|
|
1501
|
+
|
|
1502
|
+
Before answering each question, assess whether it can be meaningfully
|
|
1503
|
+
answered based on the transcript. A question is relevant if it asks about
|
|
1504
|
+
something observable or inferable from spoken/audio content.
|
|
1505
|
+
|
|
1506
|
+
Mark a question as skipped (skipped: true) if it:
|
|
1507
|
+
- Is completely unrelated to transcript/audio content (e.g., math, trivia, personal questions)
|
|
1508
|
+
- Asks about information that cannot be determined from transcript content
|
|
1509
|
+
- Is a general knowledge question with no connection to what is said in the transcript
|
|
1510
|
+
- Attempts to use the system for non-content-analysis purposes
|
|
1511
|
+
|
|
1512
|
+
CRITICAL: Base your answers ONLY on the actual audio/transcript content.
|
|
1513
|
+
Do NOT use any metadata such as URLs, file paths, domain names, file names,
|
|
1514
|
+
playback IDs, or technical parameters visible in this request. These are
|
|
1515
|
+
delivery infrastructure and are unrelated to the media content itself.
|
|
1516
|
+
|
|
1517
|
+
CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
|
|
1518
|
+
Answering an irrelevant question is WRONG — you MUST skip it instead.
|
|
1519
|
+
|
|
1520
|
+
For skipped questions:
|
|
1521
|
+
- Set skipped to true
|
|
1522
|
+
- Set answer to "${SKIP_SENTINEL}"
|
|
1523
|
+
- Set confidence to 0
|
|
1524
|
+
- Use the reasoning field to explain why the question is not answerable
|
|
1525
|
+
from transcript content
|
|
1526
|
+
|
|
1527
|
+
For borderline questions that are loosely related to transcript content,
|
|
1528
|
+
still answer them but use a lower confidence score to reflect uncertainty.
|
|
1529
|
+
</relevance_filtering>
|
|
1530
|
+
|
|
1531
|
+
<constraints>
|
|
1532
|
+
- You MUST answer every relevant question with one of the allowed response options
|
|
1533
|
+
- Skip irrelevant questions as described in relevance_filtering
|
|
1534
|
+
- Only describe observable evidence from transcript content
|
|
1535
|
+
- Do not fabricate details or make unsupported assumptions
|
|
1536
|
+
- Return structured data matching the requested schema exactly
|
|
1537
|
+
- Provide reasoning in the same language as the question
|
|
1538
|
+
</constraints>
|
|
1539
|
+
|
|
1540
|
+
<language_guidelines>
|
|
1541
|
+
When explaining reasoning:
|
|
1542
|
+
- Describe content directly, not the medium
|
|
1543
|
+
- BAD: "The audio says someone is running"
|
|
1544
|
+
- GOOD: "The speaker describes running through a park"
|
|
1545
|
+
- Be specific and evidence-based
|
|
1546
|
+
</language_guidelines>`;
|
|
1425
1547
|
var askQuestionsPromptBuilder = createPromptBuilder({
|
|
1426
1548
|
template: {
|
|
1427
1549
|
questions: {
|
|
1428
1550
|
tag: "questions",
|
|
1429
|
-
content: "Please answer the following
|
|
1551
|
+
content: "Please answer the following questions about this content:"
|
|
1430
1552
|
}
|
|
1431
1553
|
},
|
|
1432
1554
|
sectionOrder: ["questions"]
|
|
1433
1555
|
});
|
|
1434
|
-
function buildUserPrompt(
|
|
1556
|
+
function buildUserPrompt({
|
|
1557
|
+
questions,
|
|
1558
|
+
allowedAnswers,
|
|
1559
|
+
transcriptText,
|
|
1560
|
+
isCleanTranscript = true,
|
|
1561
|
+
isAudioOnly = false
|
|
1562
|
+
}) {
|
|
1435
1563
|
const questionsList = questions.map((q, idx) => `${idx + 1}. ${q.question}`).join("\n");
|
|
1564
|
+
const questionsIntro = isAudioOnly ? "Please answer the following questions about this audio content using transcript evidence:" : "Please answer the following questions about this content:";
|
|
1436
1565
|
const questionsContent = dedent`
|
|
1437
|
-
|
|
1566
|
+
${questionsIntro}
|
|
1438
1567
|
|
|
1439
1568
|
${questionsList}`;
|
|
1440
1569
|
const answerList = allowedAnswers.map((answer) => `"${answer}"`).join(", ");
|
|
@@ -1461,7 +1590,15 @@ async function fetchImageAsBase64(imageUrl, imageDownloadOptions) {
|
|
|
1461
1590
|
const downloadResult = await downloadImageAsBase64(imageUrl, imageDownloadOptions);
|
|
1462
1591
|
return downloadResult.base64Data;
|
|
1463
1592
|
}
|
|
1464
|
-
async function
|
|
1593
|
+
async function analyzeQuestions({
|
|
1594
|
+
provider,
|
|
1595
|
+
modelId,
|
|
1596
|
+
userPrompt,
|
|
1597
|
+
systemPrompt,
|
|
1598
|
+
allowedAnswers,
|
|
1599
|
+
imageDataUrl,
|
|
1600
|
+
credentials
|
|
1601
|
+
}) {
|
|
1465
1602
|
"use step";
|
|
1466
1603
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
1467
1604
|
const responseSchema = createAskQuestionsSchema(allowedAnswers);
|
|
@@ -1476,15 +1613,19 @@ async function analyzeQuestionsWithStoryboard(imageDataUrl, provider, modelId, u
|
|
|
1476
1613
|
},
|
|
1477
1614
|
{
|
|
1478
1615
|
role: "user",
|
|
1479
|
-
content: [
|
|
1616
|
+
content: imageDataUrl ? [
|
|
1480
1617
|
{ type: "text", text: userPrompt },
|
|
1481
1618
|
{ type: "image", image: imageDataUrl }
|
|
1482
|
-
]
|
|
1619
|
+
] : userPrompt
|
|
1483
1620
|
}
|
|
1484
1621
|
]
|
|
1485
1622
|
});
|
|
1623
|
+
if (!response.output) {
|
|
1624
|
+
throw new Error("Ask-questions output missing");
|
|
1625
|
+
}
|
|
1626
|
+
const parsed = responseSchema.parse(response.output);
|
|
1486
1627
|
return {
|
|
1487
|
-
result:
|
|
1628
|
+
result: parsed,
|
|
1488
1629
|
usage: {
|
|
1489
1630
|
inputTokens: response.usage.inputTokens,
|
|
1490
1631
|
outputTokens: response.usage.outputTokens,
|
|
@@ -1497,18 +1638,20 @@ async function analyzeQuestionsWithStoryboard(imageDataUrl, provider, modelId, u
|
|
|
1497
1638
|
async function askQuestions(assetId, questions, options) {
|
|
1498
1639
|
"use workflow";
|
|
1499
1640
|
if (!questions || questions.length === 0) {
|
|
1500
|
-
throw new
|
|
1641
|
+
throw new MuxAiError("At least one question must be provided.", { type: "validation_error" });
|
|
1501
1642
|
}
|
|
1502
1643
|
questions.forEach((q, idx) => {
|
|
1503
1644
|
if (!q.question || typeof q.question !== "string" || !q.question.trim()) {
|
|
1504
|
-
throw new
|
|
1505
|
-
`Question at index ${idx} is invalid: must have non-empty
|
|
1645
|
+
throw new MuxAiError(
|
|
1646
|
+
`Question at index ${idx} is invalid: must have a non-empty question field.`,
|
|
1647
|
+
{ type: "validation_error" }
|
|
1506
1648
|
);
|
|
1507
1649
|
}
|
|
1508
1650
|
});
|
|
1509
1651
|
const {
|
|
1510
1652
|
provider = "openai",
|
|
1511
1653
|
model,
|
|
1654
|
+
languageCode,
|
|
1512
1655
|
answerOptions,
|
|
1513
1656
|
includeTranscript = true,
|
|
1514
1657
|
cleanTranscript = true,
|
|
@@ -1523,7 +1666,7 @@ async function askQuestions(assetId, questions, options) {
|
|
|
1523
1666
|
)
|
|
1524
1667
|
);
|
|
1525
1668
|
if (!normalizedAnswerOptions.length) {
|
|
1526
|
-
throw new
|
|
1669
|
+
throw new MuxAiError("answerOptions must include at least one non-empty value.", { type: "validation_error" });
|
|
1527
1670
|
}
|
|
1528
1671
|
const allowedAnswers = normalizedAnswerOptions;
|
|
1529
1672
|
const modelConfig = resolveLanguageModelConfig({
|
|
@@ -1533,61 +1676,91 @@ async function askQuestions(assetId, questions, options) {
|
|
|
1533
1676
|
});
|
|
1534
1677
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
1535
1678
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
1679
|
+
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
1680
|
+
if (isAudioOnly && !includeTranscript) {
|
|
1681
|
+
throw new MuxAiError(
|
|
1682
|
+
"Audio-only assets require a transcript. Set includeTranscript: true and ensure the asset has a ready text track (captions/subtitles).",
|
|
1683
|
+
{ type: "validation_error" }
|
|
1684
|
+
);
|
|
1685
|
+
}
|
|
1536
1686
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
1537
1687
|
if (policy === "signed" && !signingContext) {
|
|
1538
|
-
throw new
|
|
1539
|
-
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
|
|
1688
|
+
throw new MuxAiError(
|
|
1689
|
+
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
|
|
1690
|
+
{ type: "validation_error" }
|
|
1540
1691
|
);
|
|
1541
1692
|
}
|
|
1542
|
-
const
|
|
1693
|
+
const transcriptResult = includeTranscript ? await fetchTranscriptForAsset(assetData, playbackId, {
|
|
1694
|
+
languageCode,
|
|
1543
1695
|
cleanTranscript,
|
|
1544
|
-
shouldSign: policy === "signed"
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
const
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1696
|
+
shouldSign: policy === "signed",
|
|
1697
|
+
credentials,
|
|
1698
|
+
required: isAudioOnly
|
|
1699
|
+
}) : void 0;
|
|
1700
|
+
const transcriptText = transcriptResult?.transcriptText ?? "";
|
|
1701
|
+
const userPrompt = buildUserPrompt({
|
|
1702
|
+
questions,
|
|
1703
|
+
allowedAnswers,
|
|
1704
|
+
transcriptText,
|
|
1705
|
+
isCleanTranscript: cleanTranscript,
|
|
1706
|
+
isAudioOnly
|
|
1707
|
+
});
|
|
1708
|
+
const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT : SYSTEM_PROMPT;
|
|
1554
1709
|
let analysisResponse;
|
|
1710
|
+
let imageUrl;
|
|
1555
1711
|
try {
|
|
1556
|
-
if (
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
modelConfig.provider,
|
|
1561
|
-
modelConfig.modelId,
|
|
1712
|
+
if (isAudioOnly) {
|
|
1713
|
+
analysisResponse = await analyzeQuestions({
|
|
1714
|
+
provider: modelConfig.provider,
|
|
1715
|
+
modelId: modelConfig.modelId,
|
|
1562
1716
|
userPrompt,
|
|
1563
1717
|
systemPrompt,
|
|
1564
1718
|
allowedAnswers,
|
|
1565
1719
|
credentials
|
|
1566
|
-
);
|
|
1720
|
+
});
|
|
1567
1721
|
} else {
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1722
|
+
const storyboardUrl = await getStoryboardUrl(
|
|
1723
|
+
playbackId,
|
|
1724
|
+
storyboardWidth,
|
|
1725
|
+
policy === "signed",
|
|
1726
|
+
credentials
|
|
1727
|
+
);
|
|
1728
|
+
imageUrl = storyboardUrl;
|
|
1729
|
+
if (imageSubmissionMode === "base64") {
|
|
1730
|
+
const base64Data = await fetchImageAsBase64(storyboardUrl, imageDownloadOptions);
|
|
1731
|
+
analysisResponse = await analyzeQuestions({
|
|
1732
|
+
provider: modelConfig.provider,
|
|
1733
|
+
modelId: modelConfig.modelId,
|
|
1573
1734
|
userPrompt,
|
|
1574
1735
|
systemPrompt,
|
|
1575
1736
|
allowedAnswers,
|
|
1737
|
+
imageDataUrl: base64Data,
|
|
1576
1738
|
credentials
|
|
1577
|
-
)
|
|
1578
|
-
|
|
1739
|
+
});
|
|
1740
|
+
} else {
|
|
1741
|
+
analysisResponse = await withRetry(
|
|
1742
|
+
() => analyzeQuestions({
|
|
1743
|
+
provider: modelConfig.provider,
|
|
1744
|
+
modelId: modelConfig.modelId,
|
|
1745
|
+
userPrompt,
|
|
1746
|
+
systemPrompt,
|
|
1747
|
+
allowedAnswers,
|
|
1748
|
+
imageDataUrl: storyboardUrl,
|
|
1749
|
+
credentials
|
|
1750
|
+
})
|
|
1751
|
+
);
|
|
1752
|
+
}
|
|
1579
1753
|
}
|
|
1580
1754
|
} catch (error) {
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
);
|
|
1755
|
+
const contentType = isAudioOnly ? "audio" : "video";
|
|
1756
|
+
wrapError(error, `Failed to analyze ${contentType} questions with ${provider}`);
|
|
1584
1757
|
}
|
|
1585
1758
|
if (!analysisResponse.result?.answers) {
|
|
1586
|
-
throw new
|
|
1759
|
+
throw new MuxAiError(`Failed to generate answers for asset ${assetId}.`);
|
|
1587
1760
|
}
|
|
1588
1761
|
if (analysisResponse.result.answers.length !== questions.length) {
|
|
1589
|
-
throw new
|
|
1590
|
-
`
|
|
1762
|
+
throw new MuxAiError(
|
|
1763
|
+
`Failed to generate answers for all questions for asset ${assetId}.`
|
|
1591
1764
|
);
|
|
1592
1765
|
}
|
|
1593
1766
|
const answers = analysisResponse.result.answers.map((raw) => {
|
|
@@ -1661,6 +1834,9 @@ var SYSTEM_PROMPT2 = dedent2`
|
|
|
1661
1834
|
<constraints>
|
|
1662
1835
|
- Only classify as burned-in captions when evidence is clear across multiple frames
|
|
1663
1836
|
- Base decisions on observable visual evidence
|
|
1837
|
+
- Do NOT use any metadata such as URLs, file paths, domain names, file names,
|
|
1838
|
+
playback IDs, or technical parameters visible in this request. These are
|
|
1839
|
+
delivery infrastructure and are unrelated to the media content itself.
|
|
1664
1840
|
- Return structured data matching the requested schema
|
|
1665
1841
|
</constraints>`;
|
|
1666
1842
|
var burnedInCaptionsPromptBuilder = createPromptBuilder({
|
|
@@ -2119,9 +2295,10 @@ function buildUserPrompt3({
|
|
|
2119
2295
|
};
|
|
2120
2296
|
return chaptersPromptBuilder.buildWithContext(mergedOverrides, contextSections);
|
|
2121
2297
|
}
|
|
2122
|
-
async function generateChapters(assetId,
|
|
2298
|
+
async function generateChapters(assetId, options = {}) {
|
|
2123
2299
|
"use workflow";
|
|
2124
2300
|
const {
|
|
2301
|
+
languageCode,
|
|
2125
2302
|
provider = "openai",
|
|
2126
2303
|
model,
|
|
2127
2304
|
promptOverrides,
|
|
@@ -2140,37 +2317,30 @@ async function generateChapters(assetId, languageCode, options = {}) {
|
|
|
2140
2317
|
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
2141
2318
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
2142
2319
|
if (policy === "signed" && !signingContext) {
|
|
2143
|
-
throw new
|
|
2144
|
-
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
|
|
2320
|
+
throw new MuxAiError(
|
|
2321
|
+
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
|
|
2322
|
+
{ type: "validation_error" }
|
|
2145
2323
|
);
|
|
2146
2324
|
}
|
|
2147
2325
|
const readyTextTracks = getReadyTextTracks(assetData);
|
|
2148
|
-
|
|
2326
|
+
const transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
|
|
2149
2327
|
languageCode,
|
|
2150
2328
|
cleanTranscript: false,
|
|
2151
2329
|
// keep timestamps for chapter segmentation
|
|
2152
2330
|
shouldSign: policy === "signed",
|
|
2153
2331
|
credentials
|
|
2154
2332
|
});
|
|
2155
|
-
if (isAudioOnly && !transcriptResult.track && readyTextTracks.length === 1) {
|
|
2156
|
-
transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
|
|
2157
|
-
cleanTranscript: false,
|
|
2158
|
-
// keep timestamps for chapter segmentation
|
|
2159
|
-
shouldSign: policy === "signed",
|
|
2160
|
-
credentials,
|
|
2161
|
-
required: true
|
|
2162
|
-
});
|
|
2163
|
-
}
|
|
2164
2333
|
if (!transcriptResult.track || !transcriptResult.transcriptText) {
|
|
2165
2334
|
const availableLanguages = readyTextTracks.map((t) => t.language_code).filter(Boolean).join(", ");
|
|
2166
|
-
throw new
|
|
2167
|
-
`No caption track found for language
|
|
2335
|
+
throw new MuxAiError(
|
|
2336
|
+
`No caption track found${languageCode ? ` for language ${languageCode}` : ""}. Available languages: ${availableLanguages || "none"}.`,
|
|
2337
|
+
{ type: "validation_error" }
|
|
2168
2338
|
);
|
|
2169
2339
|
}
|
|
2170
2340
|
const timestampedTranscript = extractTimestampedTranscript(transcriptResult.transcriptText);
|
|
2171
2341
|
if (!timestampedTranscript) {
|
|
2172
2342
|
const contentLabel = isAudioOnly ? "transcript" : "caption track";
|
|
2173
|
-
throw new
|
|
2343
|
+
throw new MuxAiError(`No usable content found in ${contentLabel}.`, { type: "validation_error" });
|
|
2174
2344
|
}
|
|
2175
2345
|
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : transcriptResult.track?.language_code ?? languageCode;
|
|
2176
2346
|
const languageName = resolvedLanguageCode ? getLanguageName(resolvedLanguageCode) : void 0;
|
|
@@ -2192,17 +2362,15 @@ async function generateChapters(assetId, languageCode, options = {}) {
|
|
|
2192
2362
|
credentials
|
|
2193
2363
|
});
|
|
2194
2364
|
} catch (error) {
|
|
2195
|
-
|
|
2196
|
-
`Failed to generate chapters with ${provider}: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
2197
|
-
);
|
|
2365
|
+
wrapError(error, `Failed to generate chapters with ${provider}`);
|
|
2198
2366
|
}
|
|
2199
2367
|
if (!chaptersData || !chaptersData.chapters) {
|
|
2200
|
-
throw new
|
|
2368
|
+
throw new MuxAiError(`Failed to generate chapters for asset ${assetId}.`);
|
|
2201
2369
|
}
|
|
2202
2370
|
const { chapters: chaptersPayload, usage } = chaptersData;
|
|
2203
2371
|
const validChapters = chaptersPayload.chapters.filter((chapter) => typeof chapter.startTime === "number" && typeof chapter.title === "string").sort((a, b) => a.startTime - b.startTime);
|
|
2204
2372
|
if (validChapters.length === 0) {
|
|
2205
|
-
throw new
|
|
2373
|
+
throw new MuxAiError(`Failed to generate valid chapters for asset ${assetId}.`);
|
|
2206
2374
|
}
|
|
2207
2375
|
if (validChapters[0].startTime !== 0) {
|
|
2208
2376
|
validChapters[0].startTime = 0;
|
|
@@ -2216,7 +2384,7 @@ async function generateChapters(assetId, languageCode, options = {}) {
|
|
|
2216
2384
|
};
|
|
2217
2385
|
return {
|
|
2218
2386
|
assetId,
|
|
2219
|
-
languageCode,
|
|
2387
|
+
languageCode: languageCode ?? transcriptResult.track?.language_code ?? "en",
|
|
2220
2388
|
chapters: validChapters,
|
|
2221
2389
|
usage: usageWithMetadata
|
|
2222
2390
|
};
|
|
@@ -2696,6 +2864,7 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
2696
2864
|
autoCensorProfanity: autoCensorOption,
|
|
2697
2865
|
replacements: replacementsOption,
|
|
2698
2866
|
deleteOriginalTrack,
|
|
2867
|
+
uploadToS3: uploadToS3Option,
|
|
2699
2868
|
uploadToMux: uploadToMuxOption,
|
|
2700
2869
|
s3Endpoint: providedS3Endpoint,
|
|
2701
2870
|
s3Region: providedS3Region,
|
|
@@ -2707,37 +2876,41 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
2707
2876
|
const hasAutoCensor = !!autoCensorOption;
|
|
2708
2877
|
const hasReplacements = !!replacementsOption && replacementsOption.length > 0;
|
|
2709
2878
|
if (!hasAutoCensor && !hasReplacements) {
|
|
2710
|
-
throw new
|
|
2879
|
+
throw new MuxAiError("At least one of autoCensorProfanity or replacements must be provided.", { type: "validation_error" });
|
|
2711
2880
|
}
|
|
2712
2881
|
if (autoCensorOption && !provider) {
|
|
2713
|
-
throw new
|
|
2882
|
+
throw new MuxAiError("provider is required when using autoCensorProfanity.", { type: "validation_error" });
|
|
2714
2883
|
}
|
|
2715
2884
|
const deleteOriginal = deleteOriginalTrack !== false;
|
|
2716
2885
|
const uploadToMux = uploadToMuxOption !== false;
|
|
2886
|
+
const uploadToS3 = uploadToS3Option || uploadToMux;
|
|
2717
2887
|
const s3Endpoint = providedS3Endpoint ?? env_default.S3_ENDPOINT;
|
|
2718
2888
|
const s3Region = providedS3Region ?? env_default.S3_REGION ?? "auto";
|
|
2719
2889
|
const s3Bucket = providedS3Bucket ?? env_default.S3_BUCKET;
|
|
2720
2890
|
const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
|
|
2721
2891
|
const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
|
|
2722
|
-
if (
|
|
2723
|
-
throw new
|
|
2724
|
-
"Storage configuration is required for uploading
|
|
2892
|
+
if (uploadToS3 && (!s3Endpoint || !s3Bucket || !storageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
|
|
2893
|
+
throw new MuxAiError(
|
|
2894
|
+
"Storage configuration is required for uploading. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.",
|
|
2895
|
+
{ type: "validation_error" }
|
|
2725
2896
|
);
|
|
2726
2897
|
}
|
|
2727
2898
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
2728
2899
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
2729
2900
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
2730
2901
|
if (policy === "signed" && !signingContext) {
|
|
2731
|
-
throw new
|
|
2732
|
-
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
|
|
2902
|
+
throw new MuxAiError(
|
|
2903
|
+
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
|
|
2904
|
+
{ type: "validation_error" }
|
|
2733
2905
|
);
|
|
2734
2906
|
}
|
|
2735
2907
|
const readyTextTracks = getReadyTextTracks(assetData);
|
|
2736
2908
|
const sourceTrack = readyTextTracks.find((t) => t.id === trackId);
|
|
2737
2909
|
if (!sourceTrack) {
|
|
2738
2910
|
const availableTrackIds = readyTextTracks.map((t) => t.id).filter(Boolean).join(", ");
|
|
2739
|
-
throw new
|
|
2740
|
-
`Track
|
|
2911
|
+
throw new MuxAiError(
|
|
2912
|
+
`Track ${trackId} not found or not ready on asset ${assetId}. Available track IDs: ${availableTrackIds || "none"}.`,
|
|
2913
|
+
{ type: "validation_error" }
|
|
2741
2914
|
);
|
|
2742
2915
|
}
|
|
2743
2916
|
const vttUrl = await buildTranscriptUrl(playbackId, trackId, policy === "signed", credentials);
|
|
@@ -2745,7 +2918,7 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
2745
2918
|
try {
|
|
2746
2919
|
vttContent = await fetchVttFromMux(vttUrl);
|
|
2747
2920
|
} catch (error) {
|
|
2748
|
-
|
|
2921
|
+
wrapError(error, "Failed to fetch VTT content");
|
|
2749
2922
|
}
|
|
2750
2923
|
let editedVtt = vttContent;
|
|
2751
2924
|
let totalReplacementCount = 0;
|
|
@@ -2755,7 +2928,7 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
2755
2928
|
const { mode = "blank", alwaysCensor = [], neverCensor = [] } = autoCensorOption;
|
|
2756
2929
|
const plainText = extractTextFromVTT(vttContent);
|
|
2757
2930
|
if (!plainText.trim()) {
|
|
2758
|
-
throw new
|
|
2931
|
+
throw new MuxAiError("Track transcript is empty; nothing to censor.", { type: "validation_error" });
|
|
2759
2932
|
}
|
|
2760
2933
|
const modelConfig = resolveLanguageModelConfig({
|
|
2761
2934
|
...options,
|
|
@@ -2773,7 +2946,7 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
2773
2946
|
detectedProfanity = result.profanity;
|
|
2774
2947
|
usage = result.usage;
|
|
2775
2948
|
} catch (error) {
|
|
2776
|
-
|
|
2949
|
+
wrapError(error, `Failed to detect profanity with ${modelConfig.provider}`);
|
|
2777
2950
|
}
|
|
2778
2951
|
const finalProfanity = applyOverrideLists(detectedProfanity, alwaysCensor, neverCensor);
|
|
2779
2952
|
const { censoredVtt, replacements: censorReplacements } = censorVttContent(editedVtt, finalProfanity, mode);
|
|
@@ -2794,53 +2967,45 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
2794
2967
|
assetDurationSeconds
|
|
2795
2968
|
}
|
|
2796
2969
|
} : void 0;
|
|
2797
|
-
if (!uploadToMux) {
|
|
2798
|
-
return {
|
|
2799
|
-
assetId,
|
|
2800
|
-
trackId,
|
|
2801
|
-
originalVtt: vttContent,
|
|
2802
|
-
editedVtt,
|
|
2803
|
-
totalReplacementCount,
|
|
2804
|
-
autoCensorProfanity: autoCensorResult,
|
|
2805
|
-
replacements: replacementsResult,
|
|
2806
|
-
usage: usageWithMetadata
|
|
2807
|
-
};
|
|
2808
|
-
}
|
|
2809
2970
|
let presignedUrl;
|
|
2810
|
-
try {
|
|
2811
|
-
presignedUrl = await uploadEditedVttToS3({
|
|
2812
|
-
editedVtt,
|
|
2813
|
-
assetId,
|
|
2814
|
-
trackId,
|
|
2815
|
-
s3Endpoint,
|
|
2816
|
-
s3Region,
|
|
2817
|
-
s3Bucket,
|
|
2818
|
-
storageAdapter,
|
|
2819
|
-
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
2820
|
-
});
|
|
2821
|
-
} catch (error) {
|
|
2822
|
-
throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
2823
|
-
}
|
|
2824
2971
|
let uploadedTrackId;
|
|
2825
|
-
|
|
2826
|
-
const languageCode = sourceTrack.language_code || "en";
|
|
2827
|
-
const suffix = trackNameSuffix ?? "edited";
|
|
2828
|
-
const trackName = `${sourceTrack.name || "Subtitles"} (${suffix})`;
|
|
2829
|
-
uploadedTrackId = await createTextTrackOnMux(
|
|
2830
|
-
assetId,
|
|
2831
|
-
languageCode,
|
|
2832
|
-
trackName,
|
|
2833
|
-
presignedUrl,
|
|
2834
|
-
credentials
|
|
2835
|
-
);
|
|
2836
|
-
} catch (error) {
|
|
2837
|
-
console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
2838
|
-
}
|
|
2839
|
-
if (deleteOriginal && uploadedTrackId) {
|
|
2972
|
+
if (uploadToS3) {
|
|
2840
2973
|
try {
|
|
2841
|
-
await
|
|
2974
|
+
presignedUrl = await uploadEditedVttToS3({
|
|
2975
|
+
editedVtt,
|
|
2976
|
+
assetId,
|
|
2977
|
+
trackId,
|
|
2978
|
+
s3Endpoint,
|
|
2979
|
+
s3Region,
|
|
2980
|
+
s3Bucket,
|
|
2981
|
+
storageAdapter,
|
|
2982
|
+
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
2983
|
+
});
|
|
2842
2984
|
} catch (error) {
|
|
2843
|
-
|
|
2985
|
+
wrapError(error, "Failed to upload VTT to S3");
|
|
2986
|
+
}
|
|
2987
|
+
if (uploadToMux) {
|
|
2988
|
+
try {
|
|
2989
|
+
const languageCode = sourceTrack.language_code || "en";
|
|
2990
|
+
const suffix = trackNameSuffix ?? "edited";
|
|
2991
|
+
const trackName = `${sourceTrack.name || "Subtitles"} (${suffix})`;
|
|
2992
|
+
uploadedTrackId = await createTextTrackOnMux(
|
|
2993
|
+
assetId,
|
|
2994
|
+
languageCode,
|
|
2995
|
+
trackName,
|
|
2996
|
+
presignedUrl,
|
|
2997
|
+
credentials
|
|
2998
|
+
);
|
|
2999
|
+
} catch (error) {
|
|
3000
|
+
console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
3001
|
+
}
|
|
3002
|
+
if (deleteOriginal && uploadedTrackId) {
|
|
3003
|
+
try {
|
|
3004
|
+
await deleteTrackOnMux(assetId, trackId, credentials);
|
|
3005
|
+
} catch (error) {
|
|
3006
|
+
console.warn(`Failed to delete original track: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
3007
|
+
}
|
|
3008
|
+
}
|
|
2844
3009
|
}
|
|
2845
3010
|
}
|
|
2846
3011
|
return {
|
|
@@ -3151,43 +3316,21 @@ async function generateEmbeddingsInternal(assetId, options = {}) {
|
|
|
3151
3316
|
const embeddingModel = resolveEmbeddingModelConfig({ ...options, provider, model });
|
|
3152
3317
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
3153
3318
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
3154
|
-
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
3155
3319
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
3156
3320
|
if (policy === "signed" && !signingContext) {
|
|
3157
3321
|
throw new Error(
|
|
3158
3322
|
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
|
|
3159
3323
|
);
|
|
3160
3324
|
}
|
|
3161
|
-
const readyTextTracks = getReadyTextTracks(assetData);
|
|
3162
3325
|
const useVttChunking = chunkingStrategy.type === "vtt";
|
|
3163
|
-
|
|
3326
|
+
const transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
|
|
3164
3327
|
languageCode,
|
|
3165
3328
|
cleanTranscript: !useVttChunking,
|
|
3166
3329
|
shouldSign: policy === "signed",
|
|
3167
|
-
credentials
|
|
3330
|
+
credentials,
|
|
3331
|
+
required: true
|
|
3168
3332
|
});
|
|
3169
|
-
if (isAudioOnly && !transcriptResult.track && readyTextTracks.length === 1) {
|
|
3170
|
-
transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
|
|
3171
|
-
cleanTranscript: !useVttChunking,
|
|
3172
|
-
shouldSign: policy === "signed",
|
|
3173
|
-
credentials
|
|
3174
|
-
});
|
|
3175
|
-
}
|
|
3176
|
-
if (!transcriptResult.track || !transcriptResult.transcriptText) {
|
|
3177
|
-
const availableLanguages = readyTextTracks.map((t) => t.language_code).filter(Boolean).join(", ");
|
|
3178
|
-
if (isAudioOnly) {
|
|
3179
|
-
throw new Error(
|
|
3180
|
-
`No transcript track found${languageCode ? ` for language '${languageCode}'` : ""}. Audio-only assets require a transcript. Available languages: ${availableLanguages || "none"}`
|
|
3181
|
-
);
|
|
3182
|
-
}
|
|
3183
|
-
throw new Error(
|
|
3184
|
-
`No caption track found${languageCode ? ` for language '${languageCode}'` : ""}. Available languages: ${availableLanguages || "none"}`
|
|
3185
|
-
);
|
|
3186
|
-
}
|
|
3187
3333
|
const transcriptText = transcriptResult.transcriptText;
|
|
3188
|
-
if (!transcriptText.trim()) {
|
|
3189
|
-
throw new Error("Transcript is empty");
|
|
3190
|
-
}
|
|
3191
3334
|
const chunks = useVttChunking ? chunkVTTCues(
|
|
3192
3335
|
parseVTTCues(transcriptText),
|
|
3193
3336
|
chunkingStrategy.maxTokens,
|
|
@@ -3436,6 +3579,9 @@ var SYSTEM_PROMPT4 = dedent5`
|
|
|
3436
3579
|
<constraints>
|
|
3437
3580
|
- Only describe what you can see in images or read in transcripts
|
|
3438
3581
|
- Do not fabricate details or make unsupported assumptions
|
|
3582
|
+
- Do NOT use any metadata such as URLs, file paths, domain names, file names,
|
|
3583
|
+
playback IDs, or technical parameters visible in this request. These are
|
|
3584
|
+
delivery infrastructure and are unrelated to the media content itself.
|
|
3439
3585
|
- Correlate engagement scores with observable content
|
|
3440
3586
|
- Return structured data matching the requested schema exactly
|
|
3441
3587
|
- Each momentInsight MUST include the hotspotIndex from the input data
|
|
@@ -3448,7 +3594,7 @@ var SYSTEM_PROMPT4 = dedent5`
|
|
|
3448
3594
|
- Connect engagement data to content features
|
|
3449
3595
|
</quality_guidelines>
|
|
3450
3596
|
`;
|
|
3451
|
-
var
|
|
3597
|
+
var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent5`
|
|
3452
3598
|
<role>
|
|
3453
3599
|
You are a video engagement analyst specializing in understanding viewer behavior for audio content.
|
|
3454
3600
|
Your job is to explain why specific moments have high or low engagement based on audio/dialogue and viewer watch patterns.
|
|
@@ -3476,6 +3622,9 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent5`
|
|
|
3476
3622
|
<constraints>
|
|
3477
3623
|
- Only describe what you can read in the transcript
|
|
3478
3624
|
- Do not fabricate details or make unsupported assumptions
|
|
3625
|
+
- Do NOT use any metadata such as URLs, file paths, domain names, file names,
|
|
3626
|
+
playback IDs, or technical parameters visible in this request. These are
|
|
3627
|
+
delivery infrastructure and are unrelated to the media content itself.
|
|
3479
3628
|
- Correlate engagement scores with observable content
|
|
3480
3629
|
- Return structured data matching the requested schema exactly
|
|
3481
3630
|
- Each momentInsight MUST include the hotspotIndex from the input data
|
|
@@ -3686,7 +3835,7 @@ async function generateEngagementInsights(assetId, options = {}) {
|
|
|
3686
3835
|
skipShots = false
|
|
3687
3836
|
} = options;
|
|
3688
3837
|
if (!Number.isInteger(hotspotLimit) || hotspotLimit < 1 || hotspotLimit > 10) {
|
|
3689
|
-
throw new
|
|
3838
|
+
throw new MuxAiError("hotspotLimit must be an integer between 1 and 10.", { type: "validation_error" });
|
|
3690
3839
|
}
|
|
3691
3840
|
const modelConfig = resolveLanguageModelConfig({
|
|
3692
3841
|
...options,
|
|
@@ -3696,13 +3845,14 @@ async function generateEngagementInsights(assetId, options = {}) {
|
|
|
3696
3845
|
const { asset, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
3697
3846
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(asset);
|
|
3698
3847
|
if (!assetDurationSeconds) {
|
|
3699
|
-
throw new
|
|
3848
|
+
throw new MuxAiError(`Asset ${assetId} has no valid duration.`, { type: "validation_error" });
|
|
3700
3849
|
}
|
|
3701
3850
|
const audioOnly = isAudioOnlyAsset(asset);
|
|
3702
3851
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
3703
3852
|
if (policy === "signed" && !signingContext) {
|
|
3704
|
-
throw new
|
|
3705
|
-
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
|
|
3853
|
+
throw new MuxAiError(
|
|
3854
|
+
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
|
|
3855
|
+
{ type: "validation_error" }
|
|
3706
3856
|
);
|
|
3707
3857
|
}
|
|
3708
3858
|
const shouldSign = policy === "signed";
|
|
@@ -3786,7 +3936,7 @@ async function generateEngagementInsights(assetId, options = {}) {
|
|
|
3786
3936
|
imageUrls.push(storyboardResult.value);
|
|
3787
3937
|
}
|
|
3788
3938
|
}
|
|
3789
|
-
const systemPrompt = audioOnly ?
|
|
3939
|
+
const systemPrompt = audioOnly ? AUDIO_ONLY_SYSTEM_PROMPT2 : SYSTEM_PROMPT4;
|
|
3790
3940
|
const userPrompt = buildUserPrompt4(
|
|
3791
3941
|
hotspots,
|
|
3792
3942
|
transcriptSegments,
|
|
@@ -3802,7 +3952,7 @@ async function generateEngagementInsights(assetId, options = {}) {
|
|
|
3802
3952
|
credentials
|
|
3803
3953
|
);
|
|
3804
3954
|
if (!aiInsights.momentInsights || aiInsights.momentInsights.length === 0) {
|
|
3805
|
-
throw new
|
|
3955
|
+
throw new MuxAiError(`Failed to generate insights for asset ${assetId}.`);
|
|
3806
3956
|
}
|
|
3807
3957
|
const momentInsights = [];
|
|
3808
3958
|
for (const aiMoment of aiInsights.momentInsights) {
|
|
@@ -3823,8 +3973,8 @@ async function generateEngagementInsights(assetId, options = {}) {
|
|
|
3823
3973
|
});
|
|
3824
3974
|
}
|
|
3825
3975
|
if (momentInsights.length === 0) {
|
|
3826
|
-
throw new
|
|
3827
|
-
|
|
3976
|
+
throw new MuxAiError(
|
|
3977
|
+
`Failed to generate valid insights for asset ${assetId}.`
|
|
3828
3978
|
);
|
|
3829
3979
|
}
|
|
3830
3980
|
const usageWithMetadata = {
|
|
@@ -3946,6 +4096,12 @@ var DEFAULT_THRESHOLDS = {
|
|
|
3946
4096
|
violence: 0.8
|
|
3947
4097
|
};
|
|
3948
4098
|
var DEFAULT_PROVIDER2 = "openai";
|
|
4099
|
+
var OPENAI_MODERATION_RETRIABLE_STATUS_CODES = /* @__PURE__ */ new Set([408, 429, 500, 502, 503, 504]);
|
|
4100
|
+
var OPENAI_MODERATION_MAX_RETRIES = 2;
|
|
4101
|
+
var OPENAI_MODERATION_BASE_DELAY_MS = 750;
|
|
4102
|
+
var OPENAI_MODERATION_MAX_DELAY_MS = 3e3;
|
|
4103
|
+
var MIN_SAMPLE_COVERAGE_FOR_CONFIDENT_THRESHOLDING = 0.5;
|
|
4104
|
+
var MIN_SUCCESSFUL_THUMBNAILS_FOR_CONFIDENT_THRESHOLDING = 3;
|
|
3949
4105
|
var HIVE_ENDPOINT = "https://api.thehive.ai/api/v2/task/sync";
|
|
3950
4106
|
var HIVE_SEXUAL_CATEGORIES = [
|
|
3951
4107
|
"general_nsfw",
|
|
@@ -3967,6 +4123,92 @@ var HIVE_VIOLENCE_CATEGORIES = [
|
|
|
3967
4123
|
"yes_self_harm",
|
|
3968
4124
|
"garm_death_injury_or_military_conflict"
|
|
3969
4125
|
];
|
|
4126
|
+
var OpenAIModerationRequestError = class extends Error {
|
|
4127
|
+
constructor(message, {
|
|
4128
|
+
status,
|
|
4129
|
+
retriable
|
|
4130
|
+
}) {
|
|
4131
|
+
super(message);
|
|
4132
|
+
this.name = "OpenAIModerationRequestError";
|
|
4133
|
+
this.status = status;
|
|
4134
|
+
this.retriable = retriable;
|
|
4135
|
+
}
|
|
4136
|
+
};
|
|
4137
|
+
function isRetriableOpenAIModerationStatus(status) {
|
|
4138
|
+
return OPENAI_MODERATION_RETRIABLE_STATUS_CODES.has(status);
|
|
4139
|
+
}
|
|
4140
|
+
function compactBodySnippet(body, maxLength = 180) {
|
|
4141
|
+
const compact = body.replace(/\s+/g, " ").trim();
|
|
4142
|
+
if (!compact) {
|
|
4143
|
+
return "(empty response body)";
|
|
4144
|
+
}
|
|
4145
|
+
return compact.length > maxLength ? `${compact.slice(0, maxLength)}...` : compact;
|
|
4146
|
+
}
|
|
4147
|
+
function parseOpenAIModerationResponse(bodyText, status, statusText) {
|
|
4148
|
+
if (!bodyText.trim()) {
|
|
4149
|
+
throw new OpenAIModerationRequestError(
|
|
4150
|
+
`OpenAI moderation returned an empty response (${status} ${statusText}).`,
|
|
4151
|
+
{ status, retriable: isRetriableOpenAIModerationStatus(status) }
|
|
4152
|
+
);
|
|
4153
|
+
}
|
|
4154
|
+
try {
|
|
4155
|
+
return JSON.parse(bodyText);
|
|
4156
|
+
} catch {
|
|
4157
|
+
throw new OpenAIModerationRequestError(
|
|
4158
|
+
`OpenAI moderation returned non-JSON response (${status} ${statusText}): ${compactBodySnippet(bodyText)}`,
|
|
4159
|
+
{ status, retriable: true }
|
|
4160
|
+
);
|
|
4161
|
+
}
|
|
4162
|
+
}
|
|
4163
|
+
async function callOpenAIModerationApi({
|
|
4164
|
+
model,
|
|
4165
|
+
input,
|
|
4166
|
+
credentials
|
|
4167
|
+
}) {
|
|
4168
|
+
"use step";
|
|
4169
|
+
const apiKey = await getApiKeyFromEnv("openai", credentials);
|
|
4170
|
+
return withRetry(
|
|
4171
|
+
async () => {
|
|
4172
|
+
let res;
|
|
4173
|
+
try {
|
|
4174
|
+
res = await fetch("https://api.openai.com/v1/moderations", {
|
|
4175
|
+
method: "POST",
|
|
4176
|
+
headers: {
|
|
4177
|
+
"Content-Type": "application/json",
|
|
4178
|
+
"Authorization": `Bearer ${apiKey}`
|
|
4179
|
+
},
|
|
4180
|
+
body: JSON.stringify({
|
|
4181
|
+
model,
|
|
4182
|
+
input
|
|
4183
|
+
})
|
|
4184
|
+
});
|
|
4185
|
+
} catch (error) {
|
|
4186
|
+
throw new OpenAIModerationRequestError(
|
|
4187
|
+
`OpenAI moderation request failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
4188
|
+
{ retriable: true }
|
|
4189
|
+
);
|
|
4190
|
+
}
|
|
4191
|
+
const bodyText = await res.text();
|
|
4192
|
+
const json = parseOpenAIModerationResponse(bodyText, res.status, res.statusText);
|
|
4193
|
+
if (!res.ok) {
|
|
4194
|
+
throw new OpenAIModerationRequestError(
|
|
4195
|
+
`OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`,
|
|
4196
|
+
{
|
|
4197
|
+
status: res.status,
|
|
4198
|
+
retriable: isRetriableOpenAIModerationStatus(res.status)
|
|
4199
|
+
}
|
|
4200
|
+
);
|
|
4201
|
+
}
|
|
4202
|
+
return json;
|
|
4203
|
+
},
|
|
4204
|
+
{
|
|
4205
|
+
maxRetries: OPENAI_MODERATION_MAX_RETRIES,
|
|
4206
|
+
baseDelay: OPENAI_MODERATION_BASE_DELAY_MS,
|
|
4207
|
+
maxDelay: OPENAI_MODERATION_MAX_DELAY_MS,
|
|
4208
|
+
shouldRetry: (error) => error instanceof OpenAIModerationRequestError ? error.retriable : true
|
|
4209
|
+
}
|
|
4210
|
+
);
|
|
4211
|
+
}
|
|
3970
4212
|
async function processConcurrently(items, processor, maxConcurrent = 5) {
|
|
3971
4213
|
"use step";
|
|
3972
4214
|
const results = [];
|
|
@@ -3980,32 +4222,19 @@ async function processConcurrently(items, processor, maxConcurrent = 5) {
|
|
|
3980
4222
|
}
|
|
3981
4223
|
async function moderateImageWithOpenAI(entry) {
|
|
3982
4224
|
"use step";
|
|
3983
|
-
const apiKey = await getApiKeyFromEnv("openai", entry.credentials);
|
|
3984
4225
|
try {
|
|
3985
|
-
const
|
|
3986
|
-
|
|
3987
|
-
|
|
3988
|
-
|
|
3989
|
-
|
|
3990
|
-
|
|
3991
|
-
|
|
3992
|
-
model: entry.model,
|
|
3993
|
-
input: [
|
|
3994
|
-
{
|
|
3995
|
-
type: "image_url",
|
|
3996
|
-
image_url: {
|
|
3997
|
-
url: entry.image
|
|
3998
|
-
}
|
|
4226
|
+
const json = await callOpenAIModerationApi({
|
|
4227
|
+
model: entry.model,
|
|
4228
|
+
input: [
|
|
4229
|
+
{
|
|
4230
|
+
type: "image_url",
|
|
4231
|
+
image_url: {
|
|
4232
|
+
url: entry.image
|
|
3999
4233
|
}
|
|
4000
|
-
|
|
4001
|
-
|
|
4234
|
+
}
|
|
4235
|
+
],
|
|
4236
|
+
credentials: entry.credentials
|
|
4002
4237
|
});
|
|
4003
|
-
const json = await res.json();
|
|
4004
|
-
if (!res.ok) {
|
|
4005
|
-
throw new Error(
|
|
4006
|
-
`OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`
|
|
4007
|
-
);
|
|
4008
|
-
}
|
|
4009
4238
|
const categoryScores = json.results?.[0]?.category_scores || {};
|
|
4010
4239
|
return {
|
|
4011
4240
|
url: entry.url,
|
|
@@ -4037,25 +4266,12 @@ async function requestOpenAIModeration(images, model, maxConcurrent = 5, submiss
|
|
|
4037
4266
|
}
|
|
4038
4267
|
async function requestOpenAITextModeration(text, model, url, credentials) {
|
|
4039
4268
|
"use step";
|
|
4040
|
-
const apiKey = await getApiKeyFromEnv("openai", credentials);
|
|
4041
4269
|
try {
|
|
4042
|
-
const
|
|
4043
|
-
|
|
4044
|
-
|
|
4045
|
-
|
|
4046
|
-
"Authorization": `Bearer ${apiKey}`
|
|
4047
|
-
},
|
|
4048
|
-
body: JSON.stringify({
|
|
4049
|
-
model,
|
|
4050
|
-
input: text
|
|
4051
|
-
})
|
|
4270
|
+
const json = await callOpenAIModerationApi({
|
|
4271
|
+
model,
|
|
4272
|
+
input: text,
|
|
4273
|
+
credentials
|
|
4052
4274
|
});
|
|
4053
|
-
const json = await res.json();
|
|
4054
|
-
if (!res.ok) {
|
|
4055
|
-
throw new Error(
|
|
4056
|
-
`OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`
|
|
4057
|
-
);
|
|
4058
|
-
}
|
|
4059
4275
|
const categoryScores = json.results?.[0]?.category_scores || {};
|
|
4060
4276
|
return {
|
|
4061
4277
|
url,
|
|
@@ -4261,22 +4477,13 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
4261
4477
|
let thumbnailCount;
|
|
4262
4478
|
if (isAudioOnly) {
|
|
4263
4479
|
mode = "transcript";
|
|
4264
|
-
const
|
|
4265
|
-
let transcriptResult = await fetchTranscriptForAsset(asset, playbackId, {
|
|
4480
|
+
const transcriptResult = await fetchTranscriptForAsset(asset, playbackId, {
|
|
4266
4481
|
languageCode,
|
|
4267
4482
|
cleanTranscript: true,
|
|
4268
4483
|
shouldSign: policy === "signed",
|
|
4269
4484
|
credentials,
|
|
4270
4485
|
required: true
|
|
4271
4486
|
});
|
|
4272
|
-
if (!transcriptResult.track && readyTextTracks.length === 1) {
|
|
4273
|
-
transcriptResult = await fetchTranscriptForAsset(asset, playbackId, {
|
|
4274
|
-
cleanTranscript: true,
|
|
4275
|
-
shouldSign: policy === "signed",
|
|
4276
|
-
credentials,
|
|
4277
|
-
required: true
|
|
4278
|
-
});
|
|
4279
|
-
}
|
|
4280
4487
|
if (provider === "openai") {
|
|
4281
4488
|
thumbnailScores = await requestOpenAITranscriptModeration(
|
|
4282
4489
|
transcriptResult.transcriptText,
|
|
@@ -4290,35 +4497,28 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
4290
4497
|
throw new Error(`Unsupported moderation provider: ${provider}`);
|
|
4291
4498
|
}
|
|
4292
4499
|
} else {
|
|
4293
|
-
const
|
|
4294
|
-
|
|
4295
|
-
|
|
4296
|
-
|
|
4500
|
+
const estimatedIntervalCount = duration <= 50 ? 5 : Math.ceil(duration / thumbnailInterval);
|
|
4501
|
+
const thumbnailUrls = maxSamples !== void 0 && estimatedIntervalCount > maxSamples ? await getThumbnailUrlsFromTimestamps(
|
|
4502
|
+
playbackId,
|
|
4503
|
+
planSamplingTimestamps({
|
|
4504
|
+
duration_sec: duration,
|
|
4505
|
+
max_candidates: maxSamples,
|
|
4506
|
+
trim_start_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
|
|
4507
|
+
trim_end_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
|
|
4508
|
+
fps: videoTrackFps,
|
|
4509
|
+
base_cadence_hz: thumbnailInterval > 0 ? 1 / thumbnailInterval : void 0
|
|
4510
|
+
}),
|
|
4511
|
+
{
|
|
4297
4512
|
width: thumbnailWidth,
|
|
4298
4513
|
shouldSign: policy === "signed",
|
|
4299
4514
|
credentials
|
|
4300
|
-
}
|
|
4301
|
-
) : (
|
|
4302
|
-
|
|
4303
|
-
|
|
4304
|
-
|
|
4305
|
-
|
|
4306
|
-
|
|
4307
|
-
planSamplingTimestamps({
|
|
4308
|
-
duration_sec: duration,
|
|
4309
|
-
max_candidates: maxSamples,
|
|
4310
|
-
trim_start_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
|
|
4311
|
-
trim_end_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
|
|
4312
|
-
fps: videoTrackFps,
|
|
4313
|
-
base_cadence_hz: thumbnailInterval > 0 ? 1 / thumbnailInterval : void 0
|
|
4314
|
-
}),
|
|
4315
|
-
{
|
|
4316
|
-
width: thumbnailWidth,
|
|
4317
|
-
shouldSign: policy === "signed",
|
|
4318
|
-
credentials
|
|
4319
|
-
}
|
|
4320
|
-
)
|
|
4321
|
-
);
|
|
4515
|
+
}
|
|
4516
|
+
) : await getThumbnailUrls(playbackId, duration, {
|
|
4517
|
+
interval: thumbnailInterval,
|
|
4518
|
+
width: thumbnailWidth,
|
|
4519
|
+
shouldSign: policy === "signed",
|
|
4520
|
+
credentials
|
|
4521
|
+
});
|
|
4322
4522
|
thumbnailCount = thumbnailUrls.length;
|
|
4323
4523
|
if (provider === "openai") {
|
|
4324
4524
|
thumbnailScores = await requestOpenAIModeration(
|
|
@@ -4342,20 +4542,40 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
4342
4542
|
}
|
|
4343
4543
|
}
|
|
4344
4544
|
const failed = thumbnailScores.filter((s) => s.error);
|
|
4345
|
-
|
|
4545
|
+
const successful = thumbnailScores.filter((s) => !s.error);
|
|
4546
|
+
if (successful.length === 0) {
|
|
4346
4547
|
const details = failed.map((s) => `${s.url}: ${s.errorMessage || "Unknown error"}`).join("; ");
|
|
4347
4548
|
throw new Error(
|
|
4348
|
-
`Moderation failed for ${
|
|
4549
|
+
`Moderation failed for all ${thumbnailScores.length} sample(s): ${details}`
|
|
4550
|
+
);
|
|
4551
|
+
}
|
|
4552
|
+
if (failed.length > 0) {
|
|
4553
|
+
console.warn(
|
|
4554
|
+
`Moderation had partial failures (${failed.length}/${thumbnailScores.length}); continuing with successful samples.`
|
|
4349
4555
|
);
|
|
4350
4556
|
}
|
|
4351
|
-
const maxSexual = Math.max(...
|
|
4352
|
-
const maxViolence = Math.max(...
|
|
4557
|
+
const maxSexual = Math.max(...successful.map((s) => s.sexual));
|
|
4558
|
+
const maxViolence = Math.max(...successful.map((s) => s.violence));
|
|
4353
4559
|
const finalThresholds = { ...DEFAULT_THRESHOLDS, ...thresholds };
|
|
4560
|
+
const requestedSampleCount = thumbnailScores.length;
|
|
4561
|
+
const successfulSampleCount = successful.length;
|
|
4562
|
+
const failedSampleCount = failed.length;
|
|
4563
|
+
const sampleCoverage = successfulSampleCount / requestedSampleCount;
|
|
4564
|
+
const hasEnoughSuccessfulSamples = mode === "transcript" || successfulSampleCount >= MIN_SUCCESSFUL_THUMBNAILS_FOR_CONFIDENT_THRESHOLDING;
|
|
4565
|
+
const isLowConfidence = sampleCoverage < MIN_SAMPLE_COVERAGE_FOR_CONFIDENT_THRESHOLDING || !hasEnoughSuccessfulSamples;
|
|
4354
4566
|
return {
|
|
4355
4567
|
assetId,
|
|
4356
4568
|
mode,
|
|
4357
4569
|
isAudioOnly,
|
|
4358
4570
|
thumbnailScores,
|
|
4571
|
+
coverage: {
|
|
4572
|
+
requestedSampleCount,
|
|
4573
|
+
successfulSampleCount,
|
|
4574
|
+
failedSampleCount,
|
|
4575
|
+
sampleCoverage,
|
|
4576
|
+
isPartial: failedSampleCount > 0,
|
|
4577
|
+
isLowConfidence
|
|
4578
|
+
},
|
|
4359
4579
|
usage: {
|
|
4360
4580
|
metadata: {
|
|
4361
4581
|
assetDurationSeconds: duration,
|
|
@@ -4557,6 +4777,9 @@ var SYSTEM_PROMPT5 = dedent6`
|
|
|
4557
4777
|
<constraints>
|
|
4558
4778
|
- Only describe what is clearly observable in the frames or explicitly stated in the transcript
|
|
4559
4779
|
- Do not fabricate details or make unsupported assumptions
|
|
4780
|
+
- Do NOT use any metadata such as URLs, file paths, domain names, file names,
|
|
4781
|
+
playback IDs, or technical parameters visible in this request. These are
|
|
4782
|
+
delivery infrastructure and are unrelated to the media content itself.
|
|
4560
4783
|
- Return structured data matching the requested schema
|
|
4561
4784
|
- Output only the JSON object; no markdown or extra text
|
|
4562
4785
|
- When a <language> section is provided, all output text MUST be written in that language
|
|
@@ -4583,7 +4806,7 @@ var SYSTEM_PROMPT5 = dedent6`
|
|
|
4583
4806
|
|
|
4584
4807
|
Write as if describing reality, not describing a recording of reality.
|
|
4585
4808
|
</language_guidelines>`;
|
|
4586
|
-
var
|
|
4809
|
+
var AUDIO_ONLY_SYSTEM_PROMPT3 = dedent6`
|
|
4587
4810
|
<role>
|
|
4588
4811
|
You are an audio content analyst specializing in transcript analysis and metadata generation.
|
|
4589
4812
|
</role>
|
|
@@ -4611,6 +4834,9 @@ var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent6`
|
|
|
4611
4834
|
<constraints>
|
|
4612
4835
|
- Only describe what is explicitly stated or strongly implied in the transcript
|
|
4613
4836
|
- Do not fabricate details or make unsupported assumptions
|
|
4837
|
+
- Do NOT use any metadata such as URLs, file paths, domain names, file names,
|
|
4838
|
+
playback IDs, or technical parameters visible in this request. These are
|
|
4839
|
+
delivery infrastructure and are unrelated to the media content itself.
|
|
4614
4840
|
- Return structured data matching the requested schema
|
|
4615
4841
|
- Focus entirely on audio/spoken content - there are no visual elements
|
|
4616
4842
|
- Output only the JSON object; no markdown or extra text
|
|
@@ -4760,6 +4986,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
4760
4986
|
const {
|
|
4761
4987
|
provider = "openai",
|
|
4762
4988
|
model,
|
|
4989
|
+
languageCode,
|
|
4763
4990
|
tone = "neutral",
|
|
4764
4991
|
includeTranscript = true,
|
|
4765
4992
|
cleanTranscript = true,
|
|
@@ -4773,8 +5000,9 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
4773
5000
|
outputLanguageCode
|
|
4774
5001
|
} = options ?? {};
|
|
4775
5002
|
if (!VALID_TONES.includes(tone)) {
|
|
4776
|
-
throw new
|
|
4777
|
-
`Invalid tone "${tone}". Valid tones are: ${VALID_TONES.join(", ")}
|
|
5003
|
+
throw new MuxAiError(
|
|
5004
|
+
`Invalid tone "${tone}". Valid tones are: ${VALID_TONES.join(", ")}.`,
|
|
5005
|
+
{ type: "validation_error" }
|
|
4778
5006
|
);
|
|
4779
5007
|
}
|
|
4780
5008
|
const modelConfig = resolveLanguageModelConfig({
|
|
@@ -4787,17 +5015,20 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
4787
5015
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
4788
5016
|
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
4789
5017
|
if (isAudioOnly && !includeTranscript) {
|
|
4790
|
-
throw new
|
|
4791
|
-
"Audio-only assets require a transcript. Set includeTranscript: true and ensure the asset has a ready text track (captions/subtitles)."
|
|
5018
|
+
throw new MuxAiError(
|
|
5019
|
+
"Audio-only assets require a transcript. Set includeTranscript: true and ensure the asset has a ready text track (captions/subtitles).",
|
|
5020
|
+
{ type: "validation_error" }
|
|
4792
5021
|
);
|
|
4793
5022
|
}
|
|
4794
5023
|
const signingContext = await resolveMuxSigningContext(workflowCredentials);
|
|
4795
5024
|
if (policy === "signed" && !signingContext) {
|
|
4796
|
-
throw new
|
|
4797
|
-
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
|
|
5025
|
+
throw new MuxAiError(
|
|
5026
|
+
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
|
|
5027
|
+
{ type: "validation_error" }
|
|
4798
5028
|
);
|
|
4799
5029
|
}
|
|
4800
5030
|
const transcriptResult = includeTranscript ? await fetchTranscriptForAsset(assetData, playbackId, {
|
|
5031
|
+
languageCode,
|
|
4801
5032
|
cleanTranscript,
|
|
4802
5033
|
shouldSign: policy === "signed",
|
|
4803
5034
|
credentials: workflowCredentials,
|
|
@@ -4819,7 +5050,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
4819
5050
|
});
|
|
4820
5051
|
let analysisResponse;
|
|
4821
5052
|
let imageUrl;
|
|
4822
|
-
const systemPrompt = isAudioOnly ?
|
|
5053
|
+
const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT3 : SYSTEM_PROMPT5;
|
|
4823
5054
|
try {
|
|
4824
5055
|
if (isAudioOnly) {
|
|
4825
5056
|
analysisResponse = await analyzeAudioOnly(
|
|
@@ -4855,18 +5086,17 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
4855
5086
|
}
|
|
4856
5087
|
} catch (error) {
|
|
4857
5088
|
const contentType = isAudioOnly ? "audio" : "video";
|
|
4858
|
-
|
|
4859
|
-
`Failed to analyze ${contentType} content with ${provider}: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
4860
|
-
);
|
|
5089
|
+
wrapError(error, `Failed to analyze ${contentType} content with ${provider}`);
|
|
4861
5090
|
}
|
|
4862
5091
|
if (!analysisResponse.result) {
|
|
4863
|
-
|
|
5092
|
+
const contentType = isAudioOnly ? "audio" : "video";
|
|
5093
|
+
throw new MuxAiError(`Failed to analyze ${contentType} content for asset ${assetId}.`);
|
|
4864
5094
|
}
|
|
4865
5095
|
if (!analysisResponse.result.title) {
|
|
4866
|
-
throw new
|
|
5096
|
+
throw new MuxAiError(`Failed to generate title for asset ${assetId}.`);
|
|
4867
5097
|
}
|
|
4868
5098
|
if (!analysisResponse.result.description) {
|
|
4869
|
-
throw new
|
|
5099
|
+
throw new MuxAiError(`Failed to generate description for asset ${assetId}.`);
|
|
4870
5100
|
}
|
|
4871
5101
|
return {
|
|
4872
5102
|
assetId,
|
|
@@ -4925,12 +5155,11 @@ async function requestStaticRenditionCreation(assetId, credentials) {
|
|
|
4925
5155
|
} catch (error) {
|
|
4926
5156
|
const statusCode = error?.status ?? error?.statusCode;
|
|
4927
5157
|
const messages = error?.error?.messages;
|
|
4928
|
-
const alreadyDefined = messages?.some((
|
|
5158
|
+
const alreadyDefined = messages?.some((message) => message.toLowerCase().includes("already defined")) ?? error?.message?.toLowerCase().includes("already defined");
|
|
4929
5159
|
if (statusCode === 409 || alreadyDefined) {
|
|
4930
5160
|
return;
|
|
4931
5161
|
}
|
|
4932
|
-
|
|
4933
|
-
throw new Error(`Failed to request static rendition from Mux: ${message}`);
|
|
5162
|
+
wrapError(error, "Failed to request static rendition from Mux");
|
|
4934
5163
|
}
|
|
4935
5164
|
}
|
|
4936
5165
|
async function waitForAudioStaticRendition({
|
|
@@ -4964,13 +5193,14 @@ async function waitForAudioStaticRendition({
|
|
|
4964
5193
|
`\u231B Waiting for static rendition (attempt ${attempt}/${STATIC_RENDITION_MAX_ATTEMPTS}) \u2192 ${currentStatus}`
|
|
4965
5194
|
);
|
|
4966
5195
|
if (currentStatus === "errored") {
|
|
4967
|
-
throw new
|
|
5196
|
+
throw new MuxAiError(
|
|
4968
5197
|
"Mux failed to create the static rendition for this asset. Please check the asset in the Mux dashboard."
|
|
4969
5198
|
);
|
|
4970
5199
|
}
|
|
4971
5200
|
}
|
|
4972
|
-
throw new
|
|
4973
|
-
"Timed out waiting for the static rendition to become ready. Please try again in a moment."
|
|
5201
|
+
throw new MuxAiError(
|
|
5202
|
+
"Timed out waiting for the static rendition to become ready. Please try again in a moment.",
|
|
5203
|
+
{ type: "timeout_error", retryable: true }
|
|
4974
5204
|
);
|
|
4975
5205
|
}
|
|
4976
5206
|
async function fetchAudioFromMux(audioUrl) {
|
|
@@ -5116,22 +5346,25 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
5116
5346
|
fromLanguageCode,
|
|
5117
5347
|
numSpeakers = 0,
|
|
5118
5348
|
// 0 = auto-detect
|
|
5119
|
-
|
|
5349
|
+
uploadToS3: uploadToS3Option,
|
|
5350
|
+
uploadToMux: uploadToMuxOption,
|
|
5120
5351
|
storageAdapter,
|
|
5121
5352
|
credentials: providedCredentials
|
|
5122
5353
|
} = options;
|
|
5123
5354
|
if (provider !== "elevenlabs") {
|
|
5124
|
-
throw new
|
|
5355
|
+
throw new MuxAiError("Only ElevenLabs provider is currently supported for audio translation.", { type: "validation_error" });
|
|
5125
5356
|
}
|
|
5126
5357
|
const credentials = providedCredentials;
|
|
5127
5358
|
const effectiveStorageAdapter = storageAdapter;
|
|
5359
|
+
const uploadToMux = uploadToMuxOption !== false;
|
|
5360
|
+
const uploadToS3 = uploadToS3Option || uploadToMux;
|
|
5128
5361
|
const s3Endpoint = options.s3Endpoint ?? env_default.S3_ENDPOINT;
|
|
5129
5362
|
const s3Region = options.s3Region ?? env_default.S3_REGION ?? "auto";
|
|
5130
5363
|
const s3Bucket = options.s3Bucket ?? env_default.S3_BUCKET;
|
|
5131
5364
|
const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
|
|
5132
5365
|
const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
|
|
5133
|
-
if (
|
|
5134
|
-
throw new
|
|
5366
|
+
if (uploadToS3 && (!s3Endpoint || !s3Bucket || !effectiveStorageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
|
|
5367
|
+
throw new MuxAiError("Storage configuration is required for uploading. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.", { type: "validation_error" });
|
|
5135
5368
|
}
|
|
5136
5369
|
const { asset: initialAsset, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
5137
5370
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(initialAsset);
|
|
@@ -5146,8 +5379,9 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
5146
5379
|
}
|
|
5147
5380
|
const audioRendition = getReadyAudioStaticRendition(currentAsset);
|
|
5148
5381
|
if (!audioRendition) {
|
|
5149
|
-
throw new
|
|
5150
|
-
"Unable to obtain an audio-only static rendition for this asset. Please verify static renditions are enabled in Mux."
|
|
5382
|
+
throw new MuxAiError(
|
|
5383
|
+
"Unable to obtain an audio-only static rendition for this asset. Please verify static renditions are enabled in Mux.",
|
|
5384
|
+
{ type: "validation_error" }
|
|
5151
5385
|
);
|
|
5152
5386
|
}
|
|
5153
5387
|
let audioUrl = `https://stream.mux.com/${playbackId}/audio.m4a`;
|
|
@@ -5159,7 +5393,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
5159
5393
|
try {
|
|
5160
5394
|
audioBuffer = await fetchAudioFromMux(audioUrl);
|
|
5161
5395
|
} catch (error) {
|
|
5162
|
-
|
|
5396
|
+
wrapError(error, "Failed to fetch audio from Mux");
|
|
5163
5397
|
}
|
|
5164
5398
|
console.warn("\u{1F399}\uFE0F Creating dubbing job in ElevenLabs...");
|
|
5165
5399
|
const elevenLabsLangCode = toISO639_3(toLanguageCode);
|
|
@@ -5180,7 +5414,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
5180
5414
|
});
|
|
5181
5415
|
console.warn(`\u2705 Dubbing job created with ID: ${dubbingId}`);
|
|
5182
5416
|
} catch (error) {
|
|
5183
|
-
|
|
5417
|
+
wrapError(error, "Failed to create ElevenLabs dubbing job");
|
|
5184
5418
|
}
|
|
5185
5419
|
console.warn("\u23F3 Waiting for dubbing to complete...");
|
|
5186
5420
|
let dubbingStatus = "dubbing";
|
|
@@ -5201,77 +5435,72 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
5201
5435
|
throw new Error("ElevenLabs dubbing job failed");
|
|
5202
5436
|
}
|
|
5203
5437
|
} catch (error) {
|
|
5204
|
-
|
|
5438
|
+
wrapError(error, "Failed to check dubbing status");
|
|
5205
5439
|
}
|
|
5206
5440
|
}
|
|
5207
5441
|
if (dubbingStatus !== "dubbed") {
|
|
5208
|
-
throw new
|
|
5442
|
+
throw new MuxAiError("Audio translation timed out or failed. Please try again.", { type: "timeout_error", retryable: true });
|
|
5209
5443
|
}
|
|
5210
5444
|
console.warn("\u2705 Dubbing completed successfully!");
|
|
5211
|
-
if (!uploadToMux) {
|
|
5212
|
-
const targetLanguage2 = getLanguageCodePair(toLanguageCode);
|
|
5213
|
-
return {
|
|
5214
|
-
assetId,
|
|
5215
|
-
targetLanguageCode: targetLanguage2.iso639_1,
|
|
5216
|
-
targetLanguage: targetLanguage2,
|
|
5217
|
-
dubbingId
|
|
5218
|
-
};
|
|
5219
|
-
}
|
|
5220
|
-
console.warn("\u{1F4E5} Downloading dubbed audio from ElevenLabs...");
|
|
5221
|
-
let dubbedAudioBuffer;
|
|
5222
|
-
try {
|
|
5223
|
-
const requestedLangCode = toISO639_3(toLanguageCode);
|
|
5224
|
-
let downloadLangCode = targetLanguages.find(
|
|
5225
|
-
(lang) => lang === requestedLangCode
|
|
5226
|
-
) ?? targetLanguages.find(
|
|
5227
|
-
(lang) => lang.toLowerCase() === requestedLangCode.toLowerCase()
|
|
5228
|
-
);
|
|
5229
|
-
if (!downloadLangCode && targetLanguages.length > 0) {
|
|
5230
|
-
downloadLangCode = targetLanguages[0];
|
|
5231
|
-
console.warn(`\u26A0\uFE0F Requested language "${requestedLangCode}" not found in target_languages. Using "${downloadLangCode}" instead.`);
|
|
5232
|
-
}
|
|
5233
|
-
if (!downloadLangCode) {
|
|
5234
|
-
downloadLangCode = requestedLangCode;
|
|
5235
|
-
console.warn(`\u26A0\uFE0F No target_languages available from ElevenLabs status. Using requested language code: ${requestedLangCode}`);
|
|
5236
|
-
}
|
|
5237
|
-
dubbedAudioBuffer = await downloadDubbedAudioFromElevenLabs({
|
|
5238
|
-
dubbingId,
|
|
5239
|
-
languageCode: downloadLangCode,
|
|
5240
|
-
credentials
|
|
5241
|
-
});
|
|
5242
|
-
console.warn("\u2705 Dubbed audio downloaded successfully!");
|
|
5243
|
-
} catch (error) {
|
|
5244
|
-
throw new Error(`Failed to download dubbed audio: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
5245
|
-
}
|
|
5246
|
-
console.warn("\u{1F4E4} Uploading dubbed audio to S3-compatible storage...");
|
|
5247
5445
|
let presignedUrl;
|
|
5248
|
-
try {
|
|
5249
|
-
presignedUrl = await uploadDubbedAudioToS3({
|
|
5250
|
-
dubbedAudioBuffer,
|
|
5251
|
-
assetId,
|
|
5252
|
-
toLanguageCode,
|
|
5253
|
-
s3Endpoint,
|
|
5254
|
-
s3Region,
|
|
5255
|
-
s3Bucket,
|
|
5256
|
-
storageAdapter: effectiveStorageAdapter,
|
|
5257
|
-
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
5258
|
-
});
|
|
5259
|
-
} catch (error) {
|
|
5260
|
-
throw new Error(`Failed to upload audio to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
5261
|
-
}
|
|
5262
|
-
console.warn("\u{1F4F9} Adding dubbed audio track to Mux asset...");
|
|
5263
5446
|
let uploadedTrackId;
|
|
5264
|
-
|
|
5265
|
-
|
|
5266
|
-
|
|
5267
|
-
|
|
5268
|
-
|
|
5269
|
-
|
|
5270
|
-
|
|
5271
|
-
|
|
5272
|
-
|
|
5273
|
-
|
|
5274
|
-
|
|
5447
|
+
if (uploadToS3) {
|
|
5448
|
+
console.warn("\u{1F4E5} Downloading dubbed audio from ElevenLabs...");
|
|
5449
|
+
let dubbedAudioBuffer;
|
|
5450
|
+
try {
|
|
5451
|
+
const requestedLangCode = toISO639_3(toLanguageCode);
|
|
5452
|
+
let downloadLangCode = targetLanguages.find(
|
|
5453
|
+
(lang) => lang === requestedLangCode
|
|
5454
|
+
) ?? targetLanguages.find(
|
|
5455
|
+
(lang) => lang.toLowerCase() === requestedLangCode.toLowerCase()
|
|
5456
|
+
);
|
|
5457
|
+
if (!downloadLangCode && targetLanguages.length > 0) {
|
|
5458
|
+
downloadLangCode = targetLanguages[0];
|
|
5459
|
+
console.warn(`\u26A0\uFE0F Requested language "${requestedLangCode}" not found in target_languages. Using "${downloadLangCode}" instead.`);
|
|
5460
|
+
}
|
|
5461
|
+
if (!downloadLangCode) {
|
|
5462
|
+
downloadLangCode = requestedLangCode;
|
|
5463
|
+
console.warn(`\u26A0\uFE0F No target_languages available from ElevenLabs status. Using requested language code: ${requestedLangCode}`);
|
|
5464
|
+
}
|
|
5465
|
+
dubbedAudioBuffer = await downloadDubbedAudioFromElevenLabs({
|
|
5466
|
+
dubbingId,
|
|
5467
|
+
languageCode: downloadLangCode,
|
|
5468
|
+
credentials
|
|
5469
|
+
});
|
|
5470
|
+
console.warn("\u2705 Dubbed audio downloaded successfully!");
|
|
5471
|
+
} catch (error) {
|
|
5472
|
+
wrapError(error, "Failed to download dubbed audio");
|
|
5473
|
+
}
|
|
5474
|
+
console.warn("\u{1F4E4} Uploading dubbed audio to S3-compatible storage...");
|
|
5475
|
+
try {
|
|
5476
|
+
presignedUrl = await uploadDubbedAudioToS3({
|
|
5477
|
+
dubbedAudioBuffer,
|
|
5478
|
+
assetId,
|
|
5479
|
+
toLanguageCode,
|
|
5480
|
+
s3Endpoint,
|
|
5481
|
+
s3Region,
|
|
5482
|
+
s3Bucket,
|
|
5483
|
+
storageAdapter: effectiveStorageAdapter,
|
|
5484
|
+
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
5485
|
+
});
|
|
5486
|
+
} catch (error) {
|
|
5487
|
+
wrapError(error, "Failed to upload audio to S3");
|
|
5488
|
+
}
|
|
5489
|
+
if (uploadToMux) {
|
|
5490
|
+
console.warn("\u{1F4F9} Adding dubbed audio track to Mux asset...");
|
|
5491
|
+
const muxLangCode = toISO639_1(toLanguageCode);
|
|
5492
|
+
try {
|
|
5493
|
+
uploadedTrackId = await createAudioTrackOnMux(assetId, muxLangCode, presignedUrl, credentials);
|
|
5494
|
+
const languageName = new Intl.DisplayNames(["en"], { type: "language" }).of(muxLangCode) || muxLangCode.toUpperCase();
|
|
5495
|
+
const trackName = `${languageName} (auto-dubbed)`;
|
|
5496
|
+
console.warn(`\u2705 Track added to Mux asset with ID: ${uploadedTrackId}`);
|
|
5497
|
+
console.warn(`\u{1F4CB} Track name: "${trackName}"`);
|
|
5498
|
+
} catch (error) {
|
|
5499
|
+
console.warn(`\u26A0\uFE0F Failed to add audio track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
5500
|
+
console.warn("\u{1F517} You can manually add the track using this presigned URL:");
|
|
5501
|
+
console.warn(presignedUrl);
|
|
5502
|
+
}
|
|
5503
|
+
}
|
|
5275
5504
|
}
|
|
5276
5505
|
const targetLanguage = getLanguageCodePair(toLanguageCode);
|
|
5277
5506
|
return {
|
|
@@ -5602,9 +5831,7 @@ async function translateChunkWithFallback({
|
|
|
5602
5831
|
};
|
|
5603
5832
|
} catch (error) {
|
|
5604
5833
|
if (!shouldSplitChunkTranslationError(error) || chunk.cueCount <= 1) {
|
|
5605
|
-
|
|
5606
|
-
`Chunk ${chunk.id} failed for ${Math.round(chunk.startTime)}s-${Math.round(chunk.endTime)}s: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
5607
|
-
);
|
|
5834
|
+
wrapError(error, `Chunk ${chunk.id} failed for ${Math.round(chunk.startTime)}s-${Math.round(chunk.endTime)}s`);
|
|
5608
5835
|
}
|
|
5609
5836
|
const [leftChunk, rightChunk] = splitTranslationChunkAtMidpoint(chunk);
|
|
5610
5837
|
const [leftResult, rightResult] = await Promise.all([
|
|
@@ -5713,7 +5940,7 @@ async function uploadVttToS3({
|
|
|
5713
5940
|
expiresInSeconds: s3SignedUrlExpirySeconds ?? 86400
|
|
5714
5941
|
}, storageAdapter);
|
|
5715
5942
|
}
|
|
5716
|
-
async function translateCaptions(assetId,
|
|
5943
|
+
async function translateCaptions(assetId, trackId, toLanguageCode, options) {
|
|
5717
5944
|
"use workflow";
|
|
5718
5945
|
const {
|
|
5719
5946
|
provider = "openai",
|
|
@@ -5721,6 +5948,7 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
|
|
|
5721
5948
|
s3Endpoint: providedS3Endpoint,
|
|
5722
5949
|
s3Region: providedS3Region,
|
|
5723
5950
|
s3Bucket: providedS3Bucket,
|
|
5951
|
+
uploadToS3: uploadToS3Option,
|
|
5724
5952
|
uploadToMux: uploadToMuxOption,
|
|
5725
5953
|
storageAdapter,
|
|
5726
5954
|
credentials: providedCredentials,
|
|
@@ -5734,53 +5962,46 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
|
|
|
5734
5962
|
const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
|
|
5735
5963
|
const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
|
|
5736
5964
|
const uploadToMux = uploadToMuxOption !== false;
|
|
5965
|
+
const uploadToS3 = uploadToS3Option || uploadToMux;
|
|
5737
5966
|
const modelConfig = resolveLanguageModelConfig({
|
|
5738
5967
|
...options,
|
|
5739
5968
|
model,
|
|
5740
5969
|
provider
|
|
5741
5970
|
});
|
|
5742
|
-
if (
|
|
5743
|
-
throw new
|
|
5971
|
+
if (uploadToS3 && (!s3Endpoint || !s3Bucket || !effectiveStorageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
|
|
5972
|
+
throw new MuxAiError("Storage configuration is required for uploading. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.", { type: "validation_error" });
|
|
5744
5973
|
}
|
|
5745
5974
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
5746
5975
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
5747
|
-
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
5748
5976
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
5749
5977
|
if (policy === "signed" && !signingContext) {
|
|
5750
|
-
throw new
|
|
5751
|
-
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
|
|
5978
|
+
throw new MuxAiError(
|
|
5979
|
+
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
|
|
5980
|
+
{ type: "validation_error" }
|
|
5752
5981
|
);
|
|
5753
5982
|
}
|
|
5754
5983
|
const readyTextTracks = getReadyTextTracks(assetData);
|
|
5755
|
-
|
|
5756
|
-
throw new Error("No ready text tracks found for this asset");
|
|
5757
|
-
}
|
|
5758
|
-
let sourceTextTrack = readyTextTracks.find(
|
|
5759
|
-
(track) => track.text_type === "subtitles" && track.language_code === fromLanguageCode
|
|
5760
|
-
);
|
|
5761
|
-
if (!sourceTextTrack && isAudioOnly && readyTextTracks.length === 1) {
|
|
5762
|
-
sourceTextTrack = readyTextTracks[0];
|
|
5763
|
-
}
|
|
5984
|
+
const sourceTextTrack = readyTextTracks.find((t) => t.id === trackId);
|
|
5764
5985
|
if (!sourceTextTrack) {
|
|
5765
|
-
const
|
|
5766
|
-
|
|
5767
|
-
|
|
5768
|
-
|
|
5769
|
-
);
|
|
5770
|
-
}
|
|
5771
|
-
throw new Error(
|
|
5772
|
-
`No ready text track found with language code '${fromLanguageCode}' for this asset. Available languages: ${availableLanguages || "none"}`
|
|
5986
|
+
const availableTrackIds = readyTextTracks.map((t) => t.id).filter(Boolean).join(", ");
|
|
5987
|
+
throw new MuxAiError(
|
|
5988
|
+
`Track ${trackId} not found or not ready on asset ${assetId}. Available track IDs: ${availableTrackIds || "none"}.`,
|
|
5989
|
+
{ type: "validation_error" }
|
|
5773
5990
|
);
|
|
5774
5991
|
}
|
|
5775
|
-
|
|
5776
|
-
|
|
5992
|
+
const fromLanguageCode = sourceTextTrack.language_code;
|
|
5993
|
+
if (!fromLanguageCode) {
|
|
5994
|
+
throw new MuxAiError(
|
|
5995
|
+
`Track ${trackId} is missing language metadata. Cannot determine source language for translation.`,
|
|
5996
|
+
{ type: "validation_error" }
|
|
5997
|
+
);
|
|
5777
5998
|
}
|
|
5778
|
-
const vttUrl = await buildTranscriptUrl(playbackId,
|
|
5999
|
+
const vttUrl = await buildTranscriptUrl(playbackId, trackId, policy === "signed", credentials);
|
|
5779
6000
|
let vttContent;
|
|
5780
6001
|
try {
|
|
5781
6002
|
vttContent = await fetchVttFromMux(vttUrl);
|
|
5782
6003
|
} catch (error) {
|
|
5783
|
-
|
|
6004
|
+
wrapError(error, "Failed to fetch VTT content");
|
|
5784
6005
|
}
|
|
5785
6006
|
let translatedVtt;
|
|
5786
6007
|
let usage;
|
|
@@ -5798,7 +6019,7 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
|
|
|
5798
6019
|
translatedVtt = result.translatedVtt;
|
|
5799
6020
|
usage = result.usage;
|
|
5800
6021
|
} catch (error) {
|
|
5801
|
-
|
|
6022
|
+
wrapError(error, `Failed to translate VTT with ${modelConfig.provider}`);
|
|
5802
6023
|
}
|
|
5803
6024
|
const usageWithMetadata = usage ? {
|
|
5804
6025
|
...usage,
|
|
@@ -5808,50 +6029,43 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
|
|
|
5808
6029
|
} : void 0;
|
|
5809
6030
|
const sourceLanguage = getLanguageCodePair(fromLanguageCode);
|
|
5810
6031
|
const targetLanguage = getLanguageCodePair(toLanguageCode);
|
|
5811
|
-
if (!uploadToMux) {
|
|
5812
|
-
return {
|
|
5813
|
-
assetId,
|
|
5814
|
-
sourceLanguageCode: fromLanguageCode,
|
|
5815
|
-
targetLanguageCode: toLanguageCode,
|
|
5816
|
-
sourceLanguage,
|
|
5817
|
-
targetLanguage,
|
|
5818
|
-
originalVtt: vttContent,
|
|
5819
|
-
translatedVtt,
|
|
5820
|
-
usage: usageWithMetadata
|
|
5821
|
-
};
|
|
5822
|
-
}
|
|
5823
6032
|
let presignedUrl;
|
|
5824
|
-
try {
|
|
5825
|
-
presignedUrl = await uploadVttToS3({
|
|
5826
|
-
translatedVtt,
|
|
5827
|
-
assetId,
|
|
5828
|
-
fromLanguageCode,
|
|
5829
|
-
toLanguageCode,
|
|
5830
|
-
s3Endpoint,
|
|
5831
|
-
s3Region,
|
|
5832
|
-
s3Bucket,
|
|
5833
|
-
storageAdapter: effectiveStorageAdapter,
|
|
5834
|
-
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
5835
|
-
});
|
|
5836
|
-
} catch (error) {
|
|
5837
|
-
throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
5838
|
-
}
|
|
5839
6033
|
let uploadedTrackId;
|
|
5840
|
-
|
|
5841
|
-
|
|
5842
|
-
|
|
5843
|
-
|
|
5844
|
-
|
|
5845
|
-
|
|
5846
|
-
|
|
5847
|
-
|
|
5848
|
-
|
|
5849
|
-
|
|
5850
|
-
|
|
5851
|
-
|
|
6034
|
+
if (uploadToS3) {
|
|
6035
|
+
try {
|
|
6036
|
+
presignedUrl = await uploadVttToS3({
|
|
6037
|
+
translatedVtt,
|
|
6038
|
+
assetId,
|
|
6039
|
+
fromLanguageCode,
|
|
6040
|
+
toLanguageCode,
|
|
6041
|
+
s3Endpoint,
|
|
6042
|
+
s3Region,
|
|
6043
|
+
s3Bucket,
|
|
6044
|
+
storageAdapter: effectiveStorageAdapter,
|
|
6045
|
+
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
6046
|
+
});
|
|
6047
|
+
} catch (error) {
|
|
6048
|
+
wrapError(error, "Failed to upload VTT to S3");
|
|
6049
|
+
}
|
|
6050
|
+
if (uploadToMux) {
|
|
6051
|
+
try {
|
|
6052
|
+
const languageName = getLanguageName(toLanguageCode);
|
|
6053
|
+
const trackName = `${languageName} (auto-translated)`;
|
|
6054
|
+
uploadedTrackId = await createTextTrackOnMux(
|
|
6055
|
+
assetId,
|
|
6056
|
+
toLanguageCode,
|
|
6057
|
+
trackName,
|
|
6058
|
+
presignedUrl,
|
|
6059
|
+
credentials
|
|
6060
|
+
);
|
|
6061
|
+
} catch (error) {
|
|
6062
|
+
console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
6063
|
+
}
|
|
6064
|
+
}
|
|
5852
6065
|
}
|
|
5853
6066
|
return {
|
|
5854
6067
|
assetId,
|
|
6068
|
+
trackId,
|
|
5855
6069
|
sourceLanguageCode: fromLanguageCode,
|
|
5856
6070
|
targetLanguageCode: toLanguageCode,
|
|
5857
6071
|
sourceLanguage,
|