@mux/ai 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{index-BeX_4DL7.d.ts → index-yn3fUDgv.d.ts} +67 -16
- package/dist/index.d.ts +2 -2
- package/dist/index.js +541 -383
- package/dist/index.js.map +1 -1
- package/dist/primitives/index.js +18 -2
- package/dist/primitives/index.js.map +1 -1
- package/dist/workflows/index.d.ts +1 -1
- package/dist/workflows/index.js +490 -332
- package/dist/workflows/index.js.map +1 -1
- package/package.json +2 -1
package/dist/workflows/index.js
CHANGED
|
@@ -981,11 +981,20 @@ function findCaptionTrack(asset, languageCode) {
|
|
|
981
981
|
if (!tracks.length)
|
|
982
982
|
return void 0;
|
|
983
983
|
if (!languageCode) {
|
|
984
|
-
|
|
984
|
+
const englishTrack = tracks.find(
|
|
985
|
+
(track) => track.text_type === "subtitles" && track.language_code === "en"
|
|
986
|
+
);
|
|
987
|
+
return englishTrack ?? tracks[0];
|
|
985
988
|
}
|
|
986
|
-
|
|
989
|
+
const languageMatch = tracks.find(
|
|
987
990
|
(track) => track.text_type === "subtitles" && track.language_code === languageCode
|
|
988
991
|
);
|
|
992
|
+
if (languageMatch)
|
|
993
|
+
return languageMatch;
|
|
994
|
+
if (isAudioOnlyAsset(asset) && tracks.length === 1) {
|
|
995
|
+
return tracks[0];
|
|
996
|
+
}
|
|
997
|
+
return void 0;
|
|
989
998
|
}
|
|
990
999
|
function normalizeLineEndings(value) {
|
|
991
1000
|
return value.replace(/\r\n/g, "\n");
|
|
@@ -1422,19 +1431,102 @@ var SYSTEM_PROMPT = dedent`
|
|
|
1422
1431
|
- GOOD: "A person runs through a park"
|
|
1423
1432
|
- Be specific and evidence-based
|
|
1424
1433
|
</language_guidelines>`;
|
|
1434
|
+
var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
|
|
1435
|
+
<role>
|
|
1436
|
+
You are an audio content analyst specializing in classification tasks.
|
|
1437
|
+
Your job is to answer questions about audio content based on transcript data.
|
|
1438
|
+
</role>
|
|
1439
|
+
|
|
1440
|
+
<context>
|
|
1441
|
+
You will receive:
|
|
1442
|
+
- A transcript of the audio/dialogue
|
|
1443
|
+
- A list of questions about the asset content
|
|
1444
|
+
</context>
|
|
1445
|
+
|
|
1446
|
+
<transcript_guidance>
|
|
1447
|
+
- Use the transcript to understand spoken content, dialogue, and audio context
|
|
1448
|
+
- Consider only transcript evidence when answering questions
|
|
1449
|
+
</transcript_guidance>
|
|
1450
|
+
|
|
1451
|
+
<task>
|
|
1452
|
+
For each question provided, you must:
|
|
1453
|
+
1. Analyze the transcript
|
|
1454
|
+
2. Answer with ONLY the allowed response options - no other values are acceptable
|
|
1455
|
+
3. Provide a confidence score between 0 and 1 reflecting your certainty
|
|
1456
|
+
4. Explain your reasoning based on observable evidence
|
|
1457
|
+
</task>
|
|
1458
|
+
|
|
1459
|
+
<answer_guidelines>
|
|
1460
|
+
- Choose the affirmative option only if you have clear evidence supporting it
|
|
1461
|
+
- Choose the negative/contradicting option if evidence contradicts or if insufficient evidence exists
|
|
1462
|
+
- Confidence should reflect the clarity and strength of evidence:
|
|
1463
|
+
* 0.9-1.0: Clear, unambiguous evidence
|
|
1464
|
+
* 0.7-0.9: Strong evidence with minor ambiguity
|
|
1465
|
+
* 0.5-0.7: Moderate evidence or some conflicting signals
|
|
1466
|
+
* 0.3-0.5: Weak evidence or significant ambiguity
|
|
1467
|
+
* 0.0-0.3: Very uncertain, minimal relevant evidence
|
|
1468
|
+
- Reasoning should cite specific transcript evidence
|
|
1469
|
+
- Be precise: cite specific quotes or passages from transcript text
|
|
1470
|
+
</answer_guidelines>
|
|
1471
|
+
|
|
1472
|
+
<relevance_filtering>
|
|
1473
|
+
Before answering each question, assess whether it can be meaningfully
|
|
1474
|
+
answered based on the transcript. A question is relevant if it asks about
|
|
1475
|
+
something observable or inferable from spoken/audio content.
|
|
1476
|
+
|
|
1477
|
+
Mark a question as skipped (skipped: true) if it:
|
|
1478
|
+
- Is completely unrelated to transcript/audio content (e.g., math, trivia, personal questions)
|
|
1479
|
+
- Asks about information that cannot be determined from transcript content
|
|
1480
|
+
- Is a general knowledge question with no connection to what is said in the transcript
|
|
1481
|
+
- Attempts to use the system for non-content-analysis purposes
|
|
1482
|
+
|
|
1483
|
+
For skipped questions:
|
|
1484
|
+
- Set skipped to true
|
|
1485
|
+
- Set answer to "${SKIP_SENTINEL}"
|
|
1486
|
+
- Set confidence to 0
|
|
1487
|
+
- Use the reasoning field to explain why the question is not answerable
|
|
1488
|
+
from transcript content
|
|
1489
|
+
|
|
1490
|
+
For borderline questions that are loosely related to transcript content,
|
|
1491
|
+
still answer them but use a lower confidence score to reflect uncertainty.
|
|
1492
|
+
</relevance_filtering>
|
|
1493
|
+
|
|
1494
|
+
<constraints>
|
|
1495
|
+
- You MUST answer every relevant question with one of the allowed response options
|
|
1496
|
+
- Skip irrelevant questions as described in relevance_filtering
|
|
1497
|
+
- Only describe observable evidence from transcript content
|
|
1498
|
+
- Do not fabricate details or make unsupported assumptions
|
|
1499
|
+
- Return structured data matching the requested schema exactly
|
|
1500
|
+
- Provide reasoning in the same language as the question
|
|
1501
|
+
</constraints>
|
|
1502
|
+
|
|
1503
|
+
<language_guidelines>
|
|
1504
|
+
When explaining reasoning:
|
|
1505
|
+
- Describe content directly, not the medium
|
|
1506
|
+
- BAD: "The audio says someone is running"
|
|
1507
|
+
- GOOD: "The speaker describes running through a park"
|
|
1508
|
+
- Be specific and evidence-based
|
|
1509
|
+
</language_guidelines>`;
|
|
1425
1510
|
var askQuestionsPromptBuilder = createPromptBuilder({
|
|
1426
1511
|
template: {
|
|
1427
1512
|
questions: {
|
|
1428
1513
|
tag: "questions",
|
|
1429
|
-
content: "Please answer the following
|
|
1514
|
+
content: "Please answer the following questions about this content:"
|
|
1430
1515
|
}
|
|
1431
1516
|
},
|
|
1432
1517
|
sectionOrder: ["questions"]
|
|
1433
1518
|
});
|
|
1434
|
-
function buildUserPrompt(
|
|
1519
|
+
function buildUserPrompt({
|
|
1520
|
+
questions,
|
|
1521
|
+
allowedAnswers,
|
|
1522
|
+
transcriptText,
|
|
1523
|
+
isCleanTranscript = true,
|
|
1524
|
+
isAudioOnly = false
|
|
1525
|
+
}) {
|
|
1435
1526
|
const questionsList = questions.map((q, idx) => `${idx + 1}. ${q.question}`).join("\n");
|
|
1527
|
+
const questionsIntro = isAudioOnly ? "Please answer the following questions about this audio content using transcript evidence:" : "Please answer the following questions about this content:";
|
|
1436
1528
|
const questionsContent = dedent`
|
|
1437
|
-
|
|
1529
|
+
${questionsIntro}
|
|
1438
1530
|
|
|
1439
1531
|
${questionsList}`;
|
|
1440
1532
|
const answerList = allowedAnswers.map((answer) => `"${answer}"`).join(", ");
|
|
@@ -1461,7 +1553,15 @@ async function fetchImageAsBase64(imageUrl, imageDownloadOptions) {
|
|
|
1461
1553
|
const downloadResult = await downloadImageAsBase64(imageUrl, imageDownloadOptions);
|
|
1462
1554
|
return downloadResult.base64Data;
|
|
1463
1555
|
}
|
|
1464
|
-
async function
|
|
1556
|
+
async function analyzeQuestions({
|
|
1557
|
+
provider,
|
|
1558
|
+
modelId,
|
|
1559
|
+
userPrompt,
|
|
1560
|
+
systemPrompt,
|
|
1561
|
+
allowedAnswers,
|
|
1562
|
+
imageDataUrl,
|
|
1563
|
+
credentials
|
|
1564
|
+
}) {
|
|
1465
1565
|
"use step";
|
|
1466
1566
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
1467
1567
|
const responseSchema = createAskQuestionsSchema(allowedAnswers);
|
|
@@ -1476,15 +1576,19 @@ async function analyzeQuestionsWithStoryboard(imageDataUrl, provider, modelId, u
|
|
|
1476
1576
|
},
|
|
1477
1577
|
{
|
|
1478
1578
|
role: "user",
|
|
1479
|
-
content: [
|
|
1579
|
+
content: imageDataUrl ? [
|
|
1480
1580
|
{ type: "text", text: userPrompt },
|
|
1481
1581
|
{ type: "image", image: imageDataUrl }
|
|
1482
|
-
]
|
|
1582
|
+
] : userPrompt
|
|
1483
1583
|
}
|
|
1484
1584
|
]
|
|
1485
1585
|
});
|
|
1586
|
+
if (!response.output) {
|
|
1587
|
+
throw new Error("Ask-questions output missing");
|
|
1588
|
+
}
|
|
1589
|
+
const parsed = responseSchema.parse(response.output);
|
|
1486
1590
|
return {
|
|
1487
|
-
result:
|
|
1591
|
+
result: parsed,
|
|
1488
1592
|
usage: {
|
|
1489
1593
|
inputTokens: response.usage.inputTokens,
|
|
1490
1594
|
outputTokens: response.usage.outputTokens,
|
|
@@ -1509,6 +1613,7 @@ async function askQuestions(assetId, questions, options) {
|
|
|
1509
1613
|
const {
|
|
1510
1614
|
provider = "openai",
|
|
1511
1615
|
model,
|
|
1616
|
+
languageCode,
|
|
1512
1617
|
answerOptions,
|
|
1513
1618
|
includeTranscript = true,
|
|
1514
1619
|
cleanTranscript = true,
|
|
@@ -1533,53 +1638,83 @@ async function askQuestions(assetId, questions, options) {
|
|
|
1533
1638
|
});
|
|
1534
1639
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
1535
1640
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
1641
|
+
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
1642
|
+
if (isAudioOnly && !includeTranscript) {
|
|
1643
|
+
throw new Error(
|
|
1644
|
+
"Audio-only assets require a transcript. Set includeTranscript: true and ensure the asset has a ready text track (captions/subtitles)."
|
|
1645
|
+
);
|
|
1646
|
+
}
|
|
1536
1647
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
1537
1648
|
if (policy === "signed" && !signingContext) {
|
|
1538
1649
|
throw new Error(
|
|
1539
1650
|
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
|
|
1540
1651
|
);
|
|
1541
1652
|
}
|
|
1542
|
-
const
|
|
1653
|
+
const transcriptResult = includeTranscript ? await fetchTranscriptForAsset(assetData, playbackId, {
|
|
1654
|
+
languageCode,
|
|
1543
1655
|
cleanTranscript,
|
|
1544
|
-
shouldSign: policy === "signed"
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
const
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1656
|
+
shouldSign: policy === "signed",
|
|
1657
|
+
credentials,
|
|
1658
|
+
required: isAudioOnly
|
|
1659
|
+
}) : void 0;
|
|
1660
|
+
const transcriptText = transcriptResult?.transcriptText ?? "";
|
|
1661
|
+
const userPrompt = buildUserPrompt({
|
|
1662
|
+
questions,
|
|
1663
|
+
allowedAnswers,
|
|
1664
|
+
transcriptText,
|
|
1665
|
+
isCleanTranscript: cleanTranscript,
|
|
1666
|
+
isAudioOnly
|
|
1667
|
+
});
|
|
1668
|
+
const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT : SYSTEM_PROMPT;
|
|
1554
1669
|
let analysisResponse;
|
|
1670
|
+
let imageUrl;
|
|
1555
1671
|
try {
|
|
1556
|
-
if (
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
modelConfig.provider,
|
|
1561
|
-
modelConfig.modelId,
|
|
1672
|
+
if (isAudioOnly) {
|
|
1673
|
+
analysisResponse = await analyzeQuestions({
|
|
1674
|
+
provider: modelConfig.provider,
|
|
1675
|
+
modelId: modelConfig.modelId,
|
|
1562
1676
|
userPrompt,
|
|
1563
1677
|
systemPrompt,
|
|
1564
1678
|
allowedAnswers,
|
|
1565
1679
|
credentials
|
|
1566
|
-
);
|
|
1680
|
+
});
|
|
1567
1681
|
} else {
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1682
|
+
const storyboardUrl = await getStoryboardUrl(
|
|
1683
|
+
playbackId,
|
|
1684
|
+
storyboardWidth,
|
|
1685
|
+
policy === "signed",
|
|
1686
|
+
credentials
|
|
1687
|
+
);
|
|
1688
|
+
imageUrl = storyboardUrl;
|
|
1689
|
+
if (imageSubmissionMode === "base64") {
|
|
1690
|
+
const base64Data = await fetchImageAsBase64(storyboardUrl, imageDownloadOptions);
|
|
1691
|
+
analysisResponse = await analyzeQuestions({
|
|
1692
|
+
provider: modelConfig.provider,
|
|
1693
|
+
modelId: modelConfig.modelId,
|
|
1573
1694
|
userPrompt,
|
|
1574
1695
|
systemPrompt,
|
|
1575
1696
|
allowedAnswers,
|
|
1697
|
+
imageDataUrl: base64Data,
|
|
1576
1698
|
credentials
|
|
1577
|
-
)
|
|
1578
|
-
|
|
1699
|
+
});
|
|
1700
|
+
} else {
|
|
1701
|
+
analysisResponse = await withRetry(
|
|
1702
|
+
() => analyzeQuestions({
|
|
1703
|
+
provider: modelConfig.provider,
|
|
1704
|
+
modelId: modelConfig.modelId,
|
|
1705
|
+
userPrompt,
|
|
1706
|
+
systemPrompt,
|
|
1707
|
+
allowedAnswers,
|
|
1708
|
+
imageDataUrl: storyboardUrl,
|
|
1709
|
+
credentials
|
|
1710
|
+
})
|
|
1711
|
+
);
|
|
1712
|
+
}
|
|
1579
1713
|
}
|
|
1580
1714
|
} catch (error) {
|
|
1715
|
+
const contentType = isAudioOnly ? "audio" : "video";
|
|
1581
1716
|
throw new Error(
|
|
1582
|
-
`Failed to analyze questions with ${provider}: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
1717
|
+
`Failed to analyze ${contentType} questions with ${provider}: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
1583
1718
|
);
|
|
1584
1719
|
}
|
|
1585
1720
|
if (!analysisResponse.result?.answers) {
|
|
@@ -2119,9 +2254,10 @@ function buildUserPrompt3({
|
|
|
2119
2254
|
};
|
|
2120
2255
|
return chaptersPromptBuilder.buildWithContext(mergedOverrides, contextSections);
|
|
2121
2256
|
}
|
|
2122
|
-
async function generateChapters(assetId,
|
|
2257
|
+
async function generateChapters(assetId, options = {}) {
|
|
2123
2258
|
"use workflow";
|
|
2124
2259
|
const {
|
|
2260
|
+
languageCode,
|
|
2125
2261
|
provider = "openai",
|
|
2126
2262
|
model,
|
|
2127
2263
|
promptOverrides,
|
|
@@ -2145,26 +2281,17 @@ async function generateChapters(assetId, languageCode, options = {}) {
|
|
|
2145
2281
|
);
|
|
2146
2282
|
}
|
|
2147
2283
|
const readyTextTracks = getReadyTextTracks(assetData);
|
|
2148
|
-
|
|
2284
|
+
const transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
|
|
2149
2285
|
languageCode,
|
|
2150
2286
|
cleanTranscript: false,
|
|
2151
2287
|
// keep timestamps for chapter segmentation
|
|
2152
2288
|
shouldSign: policy === "signed",
|
|
2153
2289
|
credentials
|
|
2154
2290
|
});
|
|
2155
|
-
if (isAudioOnly && !transcriptResult.track && readyTextTracks.length === 1) {
|
|
2156
|
-
transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
|
|
2157
|
-
cleanTranscript: false,
|
|
2158
|
-
// keep timestamps for chapter segmentation
|
|
2159
|
-
shouldSign: policy === "signed",
|
|
2160
|
-
credentials,
|
|
2161
|
-
required: true
|
|
2162
|
-
});
|
|
2163
|
-
}
|
|
2164
2291
|
if (!transcriptResult.track || !transcriptResult.transcriptText) {
|
|
2165
2292
|
const availableLanguages = readyTextTracks.map((t) => t.language_code).filter(Boolean).join(", ");
|
|
2166
2293
|
throw new Error(
|
|
2167
|
-
`No caption track found for language '${languageCode}'. Available languages: ${availableLanguages || "none"}`
|
|
2294
|
+
`No caption track found${languageCode ? ` for language '${languageCode}'` : ""}. Available languages: ${availableLanguages || "none"}`
|
|
2168
2295
|
);
|
|
2169
2296
|
}
|
|
2170
2297
|
const timestampedTranscript = extractTimestampedTranscript(transcriptResult.transcriptText);
|
|
@@ -2216,7 +2343,7 @@ async function generateChapters(assetId, languageCode, options = {}) {
|
|
|
2216
2343
|
};
|
|
2217
2344
|
return {
|
|
2218
2345
|
assetId,
|
|
2219
|
-
languageCode,
|
|
2346
|
+
languageCode: languageCode ?? transcriptResult.track?.language_code ?? "en",
|
|
2220
2347
|
chapters: validChapters,
|
|
2221
2348
|
usage: usageWithMetadata
|
|
2222
2349
|
};
|
|
@@ -2696,6 +2823,7 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
2696
2823
|
autoCensorProfanity: autoCensorOption,
|
|
2697
2824
|
replacements: replacementsOption,
|
|
2698
2825
|
deleteOriginalTrack,
|
|
2826
|
+
uploadToS3: uploadToS3Option,
|
|
2699
2827
|
uploadToMux: uploadToMuxOption,
|
|
2700
2828
|
s3Endpoint: providedS3Endpoint,
|
|
2701
2829
|
s3Region: providedS3Region,
|
|
@@ -2714,14 +2842,15 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
2714
2842
|
}
|
|
2715
2843
|
const deleteOriginal = deleteOriginalTrack !== false;
|
|
2716
2844
|
const uploadToMux = uploadToMuxOption !== false;
|
|
2845
|
+
const uploadToS3 = uploadToS3Option || uploadToMux;
|
|
2717
2846
|
const s3Endpoint = providedS3Endpoint ?? env_default.S3_ENDPOINT;
|
|
2718
2847
|
const s3Region = providedS3Region ?? env_default.S3_REGION ?? "auto";
|
|
2719
2848
|
const s3Bucket = providedS3Bucket ?? env_default.S3_BUCKET;
|
|
2720
2849
|
const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
|
|
2721
2850
|
const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
|
|
2722
|
-
if (
|
|
2851
|
+
if (uploadToS3 && (!s3Endpoint || !s3Bucket || !storageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
|
|
2723
2852
|
throw new Error(
|
|
2724
|
-
"Storage configuration is required for uploading
|
|
2853
|
+
"Storage configuration is required for uploading. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables."
|
|
2725
2854
|
);
|
|
2726
2855
|
}
|
|
2727
2856
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
@@ -2794,53 +2923,45 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
2794
2923
|
assetDurationSeconds
|
|
2795
2924
|
}
|
|
2796
2925
|
} : void 0;
|
|
2797
|
-
if (!uploadToMux) {
|
|
2798
|
-
return {
|
|
2799
|
-
assetId,
|
|
2800
|
-
trackId,
|
|
2801
|
-
originalVtt: vttContent,
|
|
2802
|
-
editedVtt,
|
|
2803
|
-
totalReplacementCount,
|
|
2804
|
-
autoCensorProfanity: autoCensorResult,
|
|
2805
|
-
replacements: replacementsResult,
|
|
2806
|
-
usage: usageWithMetadata
|
|
2807
|
-
};
|
|
2808
|
-
}
|
|
2809
2926
|
let presignedUrl;
|
|
2810
|
-
try {
|
|
2811
|
-
presignedUrl = await uploadEditedVttToS3({
|
|
2812
|
-
editedVtt,
|
|
2813
|
-
assetId,
|
|
2814
|
-
trackId,
|
|
2815
|
-
s3Endpoint,
|
|
2816
|
-
s3Region,
|
|
2817
|
-
s3Bucket,
|
|
2818
|
-
storageAdapter,
|
|
2819
|
-
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
2820
|
-
});
|
|
2821
|
-
} catch (error) {
|
|
2822
|
-
throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
2823
|
-
}
|
|
2824
2927
|
let uploadedTrackId;
|
|
2825
|
-
|
|
2826
|
-
const languageCode = sourceTrack.language_code || "en";
|
|
2827
|
-
const suffix = trackNameSuffix ?? "edited";
|
|
2828
|
-
const trackName = `${sourceTrack.name || "Subtitles"} (${suffix})`;
|
|
2829
|
-
uploadedTrackId = await createTextTrackOnMux(
|
|
2830
|
-
assetId,
|
|
2831
|
-
languageCode,
|
|
2832
|
-
trackName,
|
|
2833
|
-
presignedUrl,
|
|
2834
|
-
credentials
|
|
2835
|
-
);
|
|
2836
|
-
} catch (error) {
|
|
2837
|
-
console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
2838
|
-
}
|
|
2839
|
-
if (deleteOriginal && uploadedTrackId) {
|
|
2928
|
+
if (uploadToS3) {
|
|
2840
2929
|
try {
|
|
2841
|
-
await
|
|
2930
|
+
presignedUrl = await uploadEditedVttToS3({
|
|
2931
|
+
editedVtt,
|
|
2932
|
+
assetId,
|
|
2933
|
+
trackId,
|
|
2934
|
+
s3Endpoint,
|
|
2935
|
+
s3Region,
|
|
2936
|
+
s3Bucket,
|
|
2937
|
+
storageAdapter,
|
|
2938
|
+
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
2939
|
+
});
|
|
2842
2940
|
} catch (error) {
|
|
2843
|
-
|
|
2941
|
+
throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
2942
|
+
}
|
|
2943
|
+
if (uploadToMux) {
|
|
2944
|
+
try {
|
|
2945
|
+
const languageCode = sourceTrack.language_code || "en";
|
|
2946
|
+
const suffix = trackNameSuffix ?? "edited";
|
|
2947
|
+
const trackName = `${sourceTrack.name || "Subtitles"} (${suffix})`;
|
|
2948
|
+
uploadedTrackId = await createTextTrackOnMux(
|
|
2949
|
+
assetId,
|
|
2950
|
+
languageCode,
|
|
2951
|
+
trackName,
|
|
2952
|
+
presignedUrl,
|
|
2953
|
+
credentials
|
|
2954
|
+
);
|
|
2955
|
+
} catch (error) {
|
|
2956
|
+
console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
2957
|
+
}
|
|
2958
|
+
if (deleteOriginal && uploadedTrackId) {
|
|
2959
|
+
try {
|
|
2960
|
+
await deleteTrackOnMux(assetId, trackId, credentials);
|
|
2961
|
+
} catch (error) {
|
|
2962
|
+
console.warn(`Failed to delete original track: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
2963
|
+
}
|
|
2964
|
+
}
|
|
2844
2965
|
}
|
|
2845
2966
|
}
|
|
2846
2967
|
return {
|
|
@@ -3151,43 +3272,21 @@ async function generateEmbeddingsInternal(assetId, options = {}) {
|
|
|
3151
3272
|
const embeddingModel = resolveEmbeddingModelConfig({ ...options, provider, model });
|
|
3152
3273
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
3153
3274
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
3154
|
-
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
3155
3275
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
3156
3276
|
if (policy === "signed" && !signingContext) {
|
|
3157
3277
|
throw new Error(
|
|
3158
3278
|
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
|
|
3159
3279
|
);
|
|
3160
3280
|
}
|
|
3161
|
-
const readyTextTracks = getReadyTextTracks(assetData);
|
|
3162
3281
|
const useVttChunking = chunkingStrategy.type === "vtt";
|
|
3163
|
-
|
|
3282
|
+
const transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
|
|
3164
3283
|
languageCode,
|
|
3165
3284
|
cleanTranscript: !useVttChunking,
|
|
3166
3285
|
shouldSign: policy === "signed",
|
|
3167
|
-
credentials
|
|
3286
|
+
credentials,
|
|
3287
|
+
required: true
|
|
3168
3288
|
});
|
|
3169
|
-
if (isAudioOnly && !transcriptResult.track && readyTextTracks.length === 1) {
|
|
3170
|
-
transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
|
|
3171
|
-
cleanTranscript: !useVttChunking,
|
|
3172
|
-
shouldSign: policy === "signed",
|
|
3173
|
-
credentials
|
|
3174
|
-
});
|
|
3175
|
-
}
|
|
3176
|
-
if (!transcriptResult.track || !transcriptResult.transcriptText) {
|
|
3177
|
-
const availableLanguages = readyTextTracks.map((t) => t.language_code).filter(Boolean).join(", ");
|
|
3178
|
-
if (isAudioOnly) {
|
|
3179
|
-
throw new Error(
|
|
3180
|
-
`No transcript track found${languageCode ? ` for language '${languageCode}'` : ""}. Audio-only assets require a transcript. Available languages: ${availableLanguages || "none"}`
|
|
3181
|
-
);
|
|
3182
|
-
}
|
|
3183
|
-
throw new Error(
|
|
3184
|
-
`No caption track found${languageCode ? ` for language '${languageCode}'` : ""}. Available languages: ${availableLanguages || "none"}`
|
|
3185
|
-
);
|
|
3186
|
-
}
|
|
3187
3289
|
const transcriptText = transcriptResult.transcriptText;
|
|
3188
|
-
if (!transcriptText.trim()) {
|
|
3189
|
-
throw new Error("Transcript is empty");
|
|
3190
|
-
}
|
|
3191
3290
|
const chunks = useVttChunking ? chunkVTTCues(
|
|
3192
3291
|
parseVTTCues(transcriptText),
|
|
3193
3292
|
chunkingStrategy.maxTokens,
|
|
@@ -3448,7 +3547,7 @@ var SYSTEM_PROMPT4 = dedent5`
|
|
|
3448
3547
|
- Connect engagement data to content features
|
|
3449
3548
|
</quality_guidelines>
|
|
3450
3549
|
`;
|
|
3451
|
-
var
|
|
3550
|
+
var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent5`
|
|
3452
3551
|
<role>
|
|
3453
3552
|
You are a video engagement analyst specializing in understanding viewer behavior for audio content.
|
|
3454
3553
|
Your job is to explain why specific moments have high or low engagement based on audio/dialogue and viewer watch patterns.
|
|
@@ -3736,9 +3835,14 @@ async function generateEngagementInsights(assetId, options = {}) {
|
|
|
3736
3835
|
}
|
|
3737
3836
|
const { hotspots, heatmap } = engagementResult.value;
|
|
3738
3837
|
if (hotspots.length === 0) {
|
|
3739
|
-
|
|
3740
|
-
|
|
3741
|
-
|
|
3838
|
+
return {
|
|
3839
|
+
assetId,
|
|
3840
|
+
momentInsights: [],
|
|
3841
|
+
overallInsight: {
|
|
3842
|
+
summary: `No engagement data available for asset ${assetId} in timeframe ${timeframe}. Video may not have been viewed yet, not contain notable engagement patterns, or engagement tracking is not enabled.`,
|
|
3843
|
+
trends: []
|
|
3844
|
+
}
|
|
3845
|
+
};
|
|
3742
3846
|
}
|
|
3743
3847
|
let transcriptText = "";
|
|
3744
3848
|
if (transcriptResult.status === "fulfilled") {
|
|
@@ -3781,7 +3885,7 @@ async function generateEngagementInsights(assetId, options = {}) {
|
|
|
3781
3885
|
imageUrls.push(storyboardResult.value);
|
|
3782
3886
|
}
|
|
3783
3887
|
}
|
|
3784
|
-
const systemPrompt = audioOnly ?
|
|
3888
|
+
const systemPrompt = audioOnly ? AUDIO_ONLY_SYSTEM_PROMPT2 : SYSTEM_PROMPT4;
|
|
3785
3889
|
const userPrompt = buildUserPrompt4(
|
|
3786
3890
|
hotspots,
|
|
3787
3891
|
transcriptSegments,
|
|
@@ -3941,6 +4045,12 @@ var DEFAULT_THRESHOLDS = {
|
|
|
3941
4045
|
violence: 0.8
|
|
3942
4046
|
};
|
|
3943
4047
|
var DEFAULT_PROVIDER2 = "openai";
|
|
4048
|
+
var OPENAI_MODERATION_RETRIABLE_STATUS_CODES = /* @__PURE__ */ new Set([408, 429, 500, 502, 503, 504]);
|
|
4049
|
+
var OPENAI_MODERATION_MAX_RETRIES = 2;
|
|
4050
|
+
var OPENAI_MODERATION_BASE_DELAY_MS = 750;
|
|
4051
|
+
var OPENAI_MODERATION_MAX_DELAY_MS = 3e3;
|
|
4052
|
+
var MIN_SAMPLE_COVERAGE_FOR_CONFIDENT_THRESHOLDING = 0.5;
|
|
4053
|
+
var MIN_SUCCESSFUL_THUMBNAILS_FOR_CONFIDENT_THRESHOLDING = 3;
|
|
3944
4054
|
var HIVE_ENDPOINT = "https://api.thehive.ai/api/v2/task/sync";
|
|
3945
4055
|
var HIVE_SEXUAL_CATEGORIES = [
|
|
3946
4056
|
"general_nsfw",
|
|
@@ -3962,6 +4072,92 @@ var HIVE_VIOLENCE_CATEGORIES = [
|
|
|
3962
4072
|
"yes_self_harm",
|
|
3963
4073
|
"garm_death_injury_or_military_conflict"
|
|
3964
4074
|
];
|
|
4075
|
+
var OpenAIModerationRequestError = class extends Error {
|
|
4076
|
+
constructor(message, {
|
|
4077
|
+
status,
|
|
4078
|
+
retriable
|
|
4079
|
+
}) {
|
|
4080
|
+
super(message);
|
|
4081
|
+
this.name = "OpenAIModerationRequestError";
|
|
4082
|
+
this.status = status;
|
|
4083
|
+
this.retriable = retriable;
|
|
4084
|
+
}
|
|
4085
|
+
};
|
|
4086
|
+
function isRetriableOpenAIModerationStatus(status) {
|
|
4087
|
+
return OPENAI_MODERATION_RETRIABLE_STATUS_CODES.has(status);
|
|
4088
|
+
}
|
|
4089
|
+
function compactBodySnippet(body, maxLength = 180) {
|
|
4090
|
+
const compact = body.replace(/\s+/g, " ").trim();
|
|
4091
|
+
if (!compact) {
|
|
4092
|
+
return "(empty response body)";
|
|
4093
|
+
}
|
|
4094
|
+
return compact.length > maxLength ? `${compact.slice(0, maxLength)}...` : compact;
|
|
4095
|
+
}
|
|
4096
|
+
function parseOpenAIModerationResponse(bodyText, status, statusText) {
|
|
4097
|
+
if (!bodyText.trim()) {
|
|
4098
|
+
throw new OpenAIModerationRequestError(
|
|
4099
|
+
`OpenAI moderation returned an empty response (${status} ${statusText}).`,
|
|
4100
|
+
{ status, retriable: isRetriableOpenAIModerationStatus(status) }
|
|
4101
|
+
);
|
|
4102
|
+
}
|
|
4103
|
+
try {
|
|
4104
|
+
return JSON.parse(bodyText);
|
|
4105
|
+
} catch {
|
|
4106
|
+
throw new OpenAIModerationRequestError(
|
|
4107
|
+
`OpenAI moderation returned non-JSON response (${status} ${statusText}): ${compactBodySnippet(bodyText)}`,
|
|
4108
|
+
{ status, retriable: true }
|
|
4109
|
+
);
|
|
4110
|
+
}
|
|
4111
|
+
}
|
|
4112
|
+
async function callOpenAIModerationApi({
|
|
4113
|
+
model,
|
|
4114
|
+
input,
|
|
4115
|
+
credentials
|
|
4116
|
+
}) {
|
|
4117
|
+
"use step";
|
|
4118
|
+
const apiKey = await getApiKeyFromEnv("openai", credentials);
|
|
4119
|
+
return withRetry(
|
|
4120
|
+
async () => {
|
|
4121
|
+
let res;
|
|
4122
|
+
try {
|
|
4123
|
+
res = await fetch("https://api.openai.com/v1/moderations", {
|
|
4124
|
+
method: "POST",
|
|
4125
|
+
headers: {
|
|
4126
|
+
"Content-Type": "application/json",
|
|
4127
|
+
"Authorization": `Bearer ${apiKey}`
|
|
4128
|
+
},
|
|
4129
|
+
body: JSON.stringify({
|
|
4130
|
+
model,
|
|
4131
|
+
input
|
|
4132
|
+
})
|
|
4133
|
+
});
|
|
4134
|
+
} catch (error) {
|
|
4135
|
+
throw new OpenAIModerationRequestError(
|
|
4136
|
+
`OpenAI moderation request failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
4137
|
+
{ retriable: true }
|
|
4138
|
+
);
|
|
4139
|
+
}
|
|
4140
|
+
const bodyText = await res.text();
|
|
4141
|
+
const json = parseOpenAIModerationResponse(bodyText, res.status, res.statusText);
|
|
4142
|
+
if (!res.ok) {
|
|
4143
|
+
throw new OpenAIModerationRequestError(
|
|
4144
|
+
`OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`,
|
|
4145
|
+
{
|
|
4146
|
+
status: res.status,
|
|
4147
|
+
retriable: isRetriableOpenAIModerationStatus(res.status)
|
|
4148
|
+
}
|
|
4149
|
+
);
|
|
4150
|
+
}
|
|
4151
|
+
return json;
|
|
4152
|
+
},
|
|
4153
|
+
{
|
|
4154
|
+
maxRetries: OPENAI_MODERATION_MAX_RETRIES,
|
|
4155
|
+
baseDelay: OPENAI_MODERATION_BASE_DELAY_MS,
|
|
4156
|
+
maxDelay: OPENAI_MODERATION_MAX_DELAY_MS,
|
|
4157
|
+
shouldRetry: (error) => error instanceof OpenAIModerationRequestError ? error.retriable : true
|
|
4158
|
+
}
|
|
4159
|
+
);
|
|
4160
|
+
}
|
|
3965
4161
|
async function processConcurrently(items, processor, maxConcurrent = 5) {
|
|
3966
4162
|
"use step";
|
|
3967
4163
|
const results = [];
|
|
@@ -3975,32 +4171,19 @@ async function processConcurrently(items, processor, maxConcurrent = 5) {
|
|
|
3975
4171
|
}
|
|
3976
4172
|
async function moderateImageWithOpenAI(entry) {
|
|
3977
4173
|
"use step";
|
|
3978
|
-
const apiKey = await getApiKeyFromEnv("openai", entry.credentials);
|
|
3979
4174
|
try {
|
|
3980
|
-
const
|
|
3981
|
-
|
|
3982
|
-
|
|
3983
|
-
|
|
3984
|
-
|
|
3985
|
-
|
|
3986
|
-
|
|
3987
|
-
model: entry.model,
|
|
3988
|
-
input: [
|
|
3989
|
-
{
|
|
3990
|
-
type: "image_url",
|
|
3991
|
-
image_url: {
|
|
3992
|
-
url: entry.image
|
|
3993
|
-
}
|
|
4175
|
+
const json = await callOpenAIModerationApi({
|
|
4176
|
+
model: entry.model,
|
|
4177
|
+
input: [
|
|
4178
|
+
{
|
|
4179
|
+
type: "image_url",
|
|
4180
|
+
image_url: {
|
|
4181
|
+
url: entry.image
|
|
3994
4182
|
}
|
|
3995
|
-
|
|
3996
|
-
|
|
4183
|
+
}
|
|
4184
|
+
],
|
|
4185
|
+
credentials: entry.credentials
|
|
3997
4186
|
});
|
|
3998
|
-
const json = await res.json();
|
|
3999
|
-
if (!res.ok) {
|
|
4000
|
-
throw new Error(
|
|
4001
|
-
`OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`
|
|
4002
|
-
);
|
|
4003
|
-
}
|
|
4004
4187
|
const categoryScores = json.results?.[0]?.category_scores || {};
|
|
4005
4188
|
return {
|
|
4006
4189
|
url: entry.url,
|
|
@@ -4032,25 +4215,12 @@ async function requestOpenAIModeration(images, model, maxConcurrent = 5, submiss
|
|
|
4032
4215
|
}
|
|
4033
4216
|
async function requestOpenAITextModeration(text, model, url, credentials) {
|
|
4034
4217
|
"use step";
|
|
4035
|
-
const apiKey = await getApiKeyFromEnv("openai", credentials);
|
|
4036
4218
|
try {
|
|
4037
|
-
const
|
|
4038
|
-
|
|
4039
|
-
|
|
4040
|
-
|
|
4041
|
-
"Authorization": `Bearer ${apiKey}`
|
|
4042
|
-
},
|
|
4043
|
-
body: JSON.stringify({
|
|
4044
|
-
model,
|
|
4045
|
-
input: text
|
|
4046
|
-
})
|
|
4219
|
+
const json = await callOpenAIModerationApi({
|
|
4220
|
+
model,
|
|
4221
|
+
input: text,
|
|
4222
|
+
credentials
|
|
4047
4223
|
});
|
|
4048
|
-
const json = await res.json();
|
|
4049
|
-
if (!res.ok) {
|
|
4050
|
-
throw new Error(
|
|
4051
|
-
`OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`
|
|
4052
|
-
);
|
|
4053
|
-
}
|
|
4054
4224
|
const categoryScores = json.results?.[0]?.category_scores || {};
|
|
4055
4225
|
return {
|
|
4056
4226
|
url,
|
|
@@ -4256,22 +4426,13 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
4256
4426
|
let thumbnailCount;
|
|
4257
4427
|
if (isAudioOnly) {
|
|
4258
4428
|
mode = "transcript";
|
|
4259
|
-
const
|
|
4260
|
-
let transcriptResult = await fetchTranscriptForAsset(asset, playbackId, {
|
|
4429
|
+
const transcriptResult = await fetchTranscriptForAsset(asset, playbackId, {
|
|
4261
4430
|
languageCode,
|
|
4262
4431
|
cleanTranscript: true,
|
|
4263
4432
|
shouldSign: policy === "signed",
|
|
4264
4433
|
credentials,
|
|
4265
4434
|
required: true
|
|
4266
4435
|
});
|
|
4267
|
-
if (!transcriptResult.track && readyTextTracks.length === 1) {
|
|
4268
|
-
transcriptResult = await fetchTranscriptForAsset(asset, playbackId, {
|
|
4269
|
-
cleanTranscript: true,
|
|
4270
|
-
shouldSign: policy === "signed",
|
|
4271
|
-
credentials,
|
|
4272
|
-
required: true
|
|
4273
|
-
});
|
|
4274
|
-
}
|
|
4275
4436
|
if (provider === "openai") {
|
|
4276
4437
|
thumbnailScores = await requestOpenAITranscriptModeration(
|
|
4277
4438
|
transcriptResult.transcriptText,
|
|
@@ -4285,35 +4446,28 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
4285
4446
|
throw new Error(`Unsupported moderation provider: ${provider}`);
|
|
4286
4447
|
}
|
|
4287
4448
|
} else {
|
|
4288
|
-
const
|
|
4289
|
-
|
|
4290
|
-
|
|
4291
|
-
|
|
4449
|
+
const estimatedIntervalCount = duration <= 50 ? 5 : Math.ceil(duration / thumbnailInterval);
|
|
4450
|
+
const thumbnailUrls = maxSamples !== void 0 && estimatedIntervalCount > maxSamples ? await getThumbnailUrlsFromTimestamps(
|
|
4451
|
+
playbackId,
|
|
4452
|
+
planSamplingTimestamps({
|
|
4453
|
+
duration_sec: duration,
|
|
4454
|
+
max_candidates: maxSamples,
|
|
4455
|
+
trim_start_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
|
|
4456
|
+
trim_end_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
|
|
4457
|
+
fps: videoTrackFps,
|
|
4458
|
+
base_cadence_hz: thumbnailInterval > 0 ? 1 / thumbnailInterval : void 0
|
|
4459
|
+
}),
|
|
4460
|
+
{
|
|
4292
4461
|
width: thumbnailWidth,
|
|
4293
4462
|
shouldSign: policy === "signed",
|
|
4294
4463
|
credentials
|
|
4295
|
-
}
|
|
4296
|
-
) : (
|
|
4297
|
-
|
|
4298
|
-
|
|
4299
|
-
|
|
4300
|
-
|
|
4301
|
-
|
|
4302
|
-
planSamplingTimestamps({
|
|
4303
|
-
duration_sec: duration,
|
|
4304
|
-
max_candidates: maxSamples,
|
|
4305
|
-
trim_start_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
|
|
4306
|
-
trim_end_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
|
|
4307
|
-
fps: videoTrackFps,
|
|
4308
|
-
base_cadence_hz: thumbnailInterval > 0 ? 1 / thumbnailInterval : void 0
|
|
4309
|
-
}),
|
|
4310
|
-
{
|
|
4311
|
-
width: thumbnailWidth,
|
|
4312
|
-
shouldSign: policy === "signed",
|
|
4313
|
-
credentials
|
|
4314
|
-
}
|
|
4315
|
-
)
|
|
4316
|
-
);
|
|
4464
|
+
}
|
|
4465
|
+
) : await getThumbnailUrls(playbackId, duration, {
|
|
4466
|
+
interval: thumbnailInterval,
|
|
4467
|
+
width: thumbnailWidth,
|
|
4468
|
+
shouldSign: policy === "signed",
|
|
4469
|
+
credentials
|
|
4470
|
+
});
|
|
4317
4471
|
thumbnailCount = thumbnailUrls.length;
|
|
4318
4472
|
if (provider === "openai") {
|
|
4319
4473
|
thumbnailScores = await requestOpenAIModeration(
|
|
@@ -4337,20 +4491,40 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
4337
4491
|
}
|
|
4338
4492
|
}
|
|
4339
4493
|
const failed = thumbnailScores.filter((s) => s.error);
|
|
4340
|
-
|
|
4494
|
+
const successful = thumbnailScores.filter((s) => !s.error);
|
|
4495
|
+
if (successful.length === 0) {
|
|
4341
4496
|
const details = failed.map((s) => `${s.url}: ${s.errorMessage || "Unknown error"}`).join("; ");
|
|
4342
4497
|
throw new Error(
|
|
4343
|
-
`Moderation failed for ${
|
|
4498
|
+
`Moderation failed for all ${thumbnailScores.length} sample(s): ${details}`
|
|
4499
|
+
);
|
|
4500
|
+
}
|
|
4501
|
+
if (failed.length > 0) {
|
|
4502
|
+
console.warn(
|
|
4503
|
+
`Moderation had partial failures (${failed.length}/${thumbnailScores.length}); continuing with successful samples.`
|
|
4344
4504
|
);
|
|
4345
4505
|
}
|
|
4346
|
-
const maxSexual = Math.max(...
|
|
4347
|
-
const maxViolence = Math.max(...
|
|
4506
|
+
const maxSexual = Math.max(...successful.map((s) => s.sexual));
|
|
4507
|
+
const maxViolence = Math.max(...successful.map((s) => s.violence));
|
|
4348
4508
|
const finalThresholds = { ...DEFAULT_THRESHOLDS, ...thresholds };
|
|
4509
|
+
const requestedSampleCount = thumbnailScores.length;
|
|
4510
|
+
const successfulSampleCount = successful.length;
|
|
4511
|
+
const failedSampleCount = failed.length;
|
|
4512
|
+
const sampleCoverage = successfulSampleCount / requestedSampleCount;
|
|
4513
|
+
const hasEnoughSuccessfulSamples = mode === "transcript" || successfulSampleCount >= MIN_SUCCESSFUL_THUMBNAILS_FOR_CONFIDENT_THRESHOLDING;
|
|
4514
|
+
const isLowConfidence = sampleCoverage < MIN_SAMPLE_COVERAGE_FOR_CONFIDENT_THRESHOLDING || !hasEnoughSuccessfulSamples;
|
|
4349
4515
|
return {
|
|
4350
4516
|
assetId,
|
|
4351
4517
|
mode,
|
|
4352
4518
|
isAudioOnly,
|
|
4353
4519
|
thumbnailScores,
|
|
4520
|
+
coverage: {
|
|
4521
|
+
requestedSampleCount,
|
|
4522
|
+
successfulSampleCount,
|
|
4523
|
+
failedSampleCount,
|
|
4524
|
+
sampleCoverage,
|
|
4525
|
+
isPartial: failedSampleCount > 0,
|
|
4526
|
+
isLowConfidence
|
|
4527
|
+
},
|
|
4354
4528
|
usage: {
|
|
4355
4529
|
metadata: {
|
|
4356
4530
|
assetDurationSeconds: duration,
|
|
@@ -4578,7 +4752,7 @@ var SYSTEM_PROMPT5 = dedent6`
|
|
|
4578
4752
|
|
|
4579
4753
|
Write as if describing reality, not describing a recording of reality.
|
|
4580
4754
|
</language_guidelines>`;
|
|
4581
|
-
var
|
|
4755
|
+
var AUDIO_ONLY_SYSTEM_PROMPT3 = dedent6`
|
|
4582
4756
|
<role>
|
|
4583
4757
|
You are an audio content analyst specializing in transcript analysis and metadata generation.
|
|
4584
4758
|
</role>
|
|
@@ -4755,6 +4929,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
4755
4929
|
const {
|
|
4756
4930
|
provider = "openai",
|
|
4757
4931
|
model,
|
|
4932
|
+
languageCode,
|
|
4758
4933
|
tone = "neutral",
|
|
4759
4934
|
includeTranscript = true,
|
|
4760
4935
|
cleanTranscript = true,
|
|
@@ -4793,6 +4968,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
4793
4968
|
);
|
|
4794
4969
|
}
|
|
4795
4970
|
const transcriptResult = includeTranscript ? await fetchTranscriptForAsset(assetData, playbackId, {
|
|
4971
|
+
languageCode,
|
|
4796
4972
|
cleanTranscript,
|
|
4797
4973
|
shouldSign: policy === "signed",
|
|
4798
4974
|
credentials: workflowCredentials,
|
|
@@ -4814,7 +4990,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
4814
4990
|
});
|
|
4815
4991
|
let analysisResponse;
|
|
4816
4992
|
let imageUrl;
|
|
4817
|
-
const systemPrompt = isAudioOnly ?
|
|
4993
|
+
const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT3 : SYSTEM_PROMPT5;
|
|
4818
4994
|
try {
|
|
4819
4995
|
if (isAudioOnly) {
|
|
4820
4996
|
analysisResponse = await analyzeAudioOnly(
|
|
@@ -5111,7 +5287,8 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
5111
5287
|
fromLanguageCode,
|
|
5112
5288
|
numSpeakers = 0,
|
|
5113
5289
|
// 0 = auto-detect
|
|
5114
|
-
|
|
5290
|
+
uploadToS3: uploadToS3Option,
|
|
5291
|
+
uploadToMux: uploadToMuxOption,
|
|
5115
5292
|
storageAdapter,
|
|
5116
5293
|
credentials: providedCredentials
|
|
5117
5294
|
} = options;
|
|
@@ -5120,13 +5297,15 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
5120
5297
|
}
|
|
5121
5298
|
const credentials = providedCredentials;
|
|
5122
5299
|
const effectiveStorageAdapter = storageAdapter;
|
|
5300
|
+
const uploadToMux = uploadToMuxOption !== false;
|
|
5301
|
+
const uploadToS3 = uploadToS3Option || uploadToMux;
|
|
5123
5302
|
const s3Endpoint = options.s3Endpoint ?? env_default.S3_ENDPOINT;
|
|
5124
5303
|
const s3Region = options.s3Region ?? env_default.S3_REGION ?? "auto";
|
|
5125
5304
|
const s3Bucket = options.s3Bucket ?? env_default.S3_BUCKET;
|
|
5126
5305
|
const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
|
|
5127
5306
|
const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
|
|
5128
|
-
if (
|
|
5129
|
-
throw new Error("Storage configuration is required for uploading
|
|
5307
|
+
if (uploadToS3 && (!s3Endpoint || !s3Bucket || !effectiveStorageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
|
|
5308
|
+
throw new Error("Storage configuration is required for uploading. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.");
|
|
5130
5309
|
}
|
|
5131
5310
|
const { asset: initialAsset, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
5132
5311
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(initialAsset);
|
|
@@ -5203,70 +5382,65 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
5203
5382
|
throw new Error(`Dubbing job timed out or failed. Final status: ${dubbingStatus}`);
|
|
5204
5383
|
}
|
|
5205
5384
|
console.warn("\u2705 Dubbing completed successfully!");
|
|
5206
|
-
if (!uploadToMux) {
|
|
5207
|
-
const targetLanguage2 = getLanguageCodePair(toLanguageCode);
|
|
5208
|
-
return {
|
|
5209
|
-
assetId,
|
|
5210
|
-
targetLanguageCode: targetLanguage2.iso639_1,
|
|
5211
|
-
targetLanguage: targetLanguage2,
|
|
5212
|
-
dubbingId
|
|
5213
|
-
};
|
|
5214
|
-
}
|
|
5215
|
-
console.warn("\u{1F4E5} Downloading dubbed audio from ElevenLabs...");
|
|
5216
|
-
let dubbedAudioBuffer;
|
|
5217
|
-
try {
|
|
5218
|
-
const requestedLangCode = toISO639_3(toLanguageCode);
|
|
5219
|
-
let downloadLangCode = targetLanguages.find(
|
|
5220
|
-
(lang) => lang === requestedLangCode
|
|
5221
|
-
) ?? targetLanguages.find(
|
|
5222
|
-
(lang) => lang.toLowerCase() === requestedLangCode.toLowerCase()
|
|
5223
|
-
);
|
|
5224
|
-
if (!downloadLangCode && targetLanguages.length > 0) {
|
|
5225
|
-
downloadLangCode = targetLanguages[0];
|
|
5226
|
-
console.warn(`\u26A0\uFE0F Requested language "${requestedLangCode}" not found in target_languages. Using "${downloadLangCode}" instead.`);
|
|
5227
|
-
}
|
|
5228
|
-
if (!downloadLangCode) {
|
|
5229
|
-
downloadLangCode = requestedLangCode;
|
|
5230
|
-
console.warn(`\u26A0\uFE0F No target_languages available from ElevenLabs status. Using requested language code: ${requestedLangCode}`);
|
|
5231
|
-
}
|
|
5232
|
-
dubbedAudioBuffer = await downloadDubbedAudioFromElevenLabs({
|
|
5233
|
-
dubbingId,
|
|
5234
|
-
languageCode: downloadLangCode,
|
|
5235
|
-
credentials
|
|
5236
|
-
});
|
|
5237
|
-
console.warn("\u2705 Dubbed audio downloaded successfully!");
|
|
5238
|
-
} catch (error) {
|
|
5239
|
-
throw new Error(`Failed to download dubbed audio: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
5240
|
-
}
|
|
5241
|
-
console.warn("\u{1F4E4} Uploading dubbed audio to S3-compatible storage...");
|
|
5242
5385
|
let presignedUrl;
|
|
5243
|
-
try {
|
|
5244
|
-
presignedUrl = await uploadDubbedAudioToS3({
|
|
5245
|
-
dubbedAudioBuffer,
|
|
5246
|
-
assetId,
|
|
5247
|
-
toLanguageCode,
|
|
5248
|
-
s3Endpoint,
|
|
5249
|
-
s3Region,
|
|
5250
|
-
s3Bucket,
|
|
5251
|
-
storageAdapter: effectiveStorageAdapter,
|
|
5252
|
-
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
5253
|
-
});
|
|
5254
|
-
} catch (error) {
|
|
5255
|
-
throw new Error(`Failed to upload audio to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
5256
|
-
}
|
|
5257
|
-
console.warn("\u{1F4F9} Adding dubbed audio track to Mux asset...");
|
|
5258
5386
|
let uploadedTrackId;
|
|
5259
|
-
|
|
5260
|
-
|
|
5261
|
-
|
|
5262
|
-
|
|
5263
|
-
|
|
5264
|
-
|
|
5265
|
-
|
|
5266
|
-
|
|
5267
|
-
|
|
5268
|
-
|
|
5269
|
-
|
|
5387
|
+
if (uploadToS3) {
|
|
5388
|
+
console.warn("\u{1F4E5} Downloading dubbed audio from ElevenLabs...");
|
|
5389
|
+
let dubbedAudioBuffer;
|
|
5390
|
+
try {
|
|
5391
|
+
const requestedLangCode = toISO639_3(toLanguageCode);
|
|
5392
|
+
let downloadLangCode = targetLanguages.find(
|
|
5393
|
+
(lang) => lang === requestedLangCode
|
|
5394
|
+
) ?? targetLanguages.find(
|
|
5395
|
+
(lang) => lang.toLowerCase() === requestedLangCode.toLowerCase()
|
|
5396
|
+
);
|
|
5397
|
+
if (!downloadLangCode && targetLanguages.length > 0) {
|
|
5398
|
+
downloadLangCode = targetLanguages[0];
|
|
5399
|
+
console.warn(`\u26A0\uFE0F Requested language "${requestedLangCode}" not found in target_languages. Using "${downloadLangCode}" instead.`);
|
|
5400
|
+
}
|
|
5401
|
+
if (!downloadLangCode) {
|
|
5402
|
+
downloadLangCode = requestedLangCode;
|
|
5403
|
+
console.warn(`\u26A0\uFE0F No target_languages available from ElevenLabs status. Using requested language code: ${requestedLangCode}`);
|
|
5404
|
+
}
|
|
5405
|
+
dubbedAudioBuffer = await downloadDubbedAudioFromElevenLabs({
|
|
5406
|
+
dubbingId,
|
|
5407
|
+
languageCode: downloadLangCode,
|
|
5408
|
+
credentials
|
|
5409
|
+
});
|
|
5410
|
+
console.warn("\u2705 Dubbed audio downloaded successfully!");
|
|
5411
|
+
} catch (error) {
|
|
5412
|
+
throw new Error(`Failed to download dubbed audio: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
5413
|
+
}
|
|
5414
|
+
console.warn("\u{1F4E4} Uploading dubbed audio to S3-compatible storage...");
|
|
5415
|
+
try {
|
|
5416
|
+
presignedUrl = await uploadDubbedAudioToS3({
|
|
5417
|
+
dubbedAudioBuffer,
|
|
5418
|
+
assetId,
|
|
5419
|
+
toLanguageCode,
|
|
5420
|
+
s3Endpoint,
|
|
5421
|
+
s3Region,
|
|
5422
|
+
s3Bucket,
|
|
5423
|
+
storageAdapter: effectiveStorageAdapter,
|
|
5424
|
+
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
5425
|
+
});
|
|
5426
|
+
} catch (error) {
|
|
5427
|
+
throw new Error(`Failed to upload audio to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
5428
|
+
}
|
|
5429
|
+
if (uploadToMux) {
|
|
5430
|
+
console.warn("\u{1F4F9} Adding dubbed audio track to Mux asset...");
|
|
5431
|
+
const muxLangCode = toISO639_1(toLanguageCode);
|
|
5432
|
+
try {
|
|
5433
|
+
uploadedTrackId = await createAudioTrackOnMux(assetId, muxLangCode, presignedUrl, credentials);
|
|
5434
|
+
const languageName = new Intl.DisplayNames(["en"], { type: "language" }).of(muxLangCode) || muxLangCode.toUpperCase();
|
|
5435
|
+
const trackName = `${languageName} (auto-dubbed)`;
|
|
5436
|
+
console.warn(`\u2705 Track added to Mux asset with ID: ${uploadedTrackId}`);
|
|
5437
|
+
console.warn(`\u{1F4CB} Track name: "${trackName}"`);
|
|
5438
|
+
} catch (error) {
|
|
5439
|
+
console.warn(`\u26A0\uFE0F Failed to add audio track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
5440
|
+
console.warn("\u{1F517} You can manually add the track using this presigned URL:");
|
|
5441
|
+
console.warn(presignedUrl);
|
|
5442
|
+
}
|
|
5443
|
+
}
|
|
5270
5444
|
}
|
|
5271
5445
|
const targetLanguage = getLanguageCodePair(toLanguageCode);
|
|
5272
5446
|
return {
|
|
@@ -5708,7 +5882,7 @@ async function uploadVttToS3({
|
|
|
5708
5882
|
expiresInSeconds: s3SignedUrlExpirySeconds ?? 86400
|
|
5709
5883
|
}, storageAdapter);
|
|
5710
5884
|
}
|
|
5711
|
-
async function translateCaptions(assetId,
|
|
5885
|
+
async function translateCaptions(assetId, trackId, toLanguageCode, options) {
|
|
5712
5886
|
"use workflow";
|
|
5713
5887
|
const {
|
|
5714
5888
|
provider = "openai",
|
|
@@ -5716,6 +5890,7 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
|
|
|
5716
5890
|
s3Endpoint: providedS3Endpoint,
|
|
5717
5891
|
s3Region: providedS3Region,
|
|
5718
5892
|
s3Bucket: providedS3Bucket,
|
|
5893
|
+
uploadToS3: uploadToS3Option,
|
|
5719
5894
|
uploadToMux: uploadToMuxOption,
|
|
5720
5895
|
storageAdapter,
|
|
5721
5896
|
credentials: providedCredentials,
|
|
@@ -5729,17 +5904,17 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
|
|
|
5729
5904
|
const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
|
|
5730
5905
|
const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
|
|
5731
5906
|
const uploadToMux = uploadToMuxOption !== false;
|
|
5907
|
+
const uploadToS3 = uploadToS3Option || uploadToMux;
|
|
5732
5908
|
const modelConfig = resolveLanguageModelConfig({
|
|
5733
5909
|
...options,
|
|
5734
5910
|
model,
|
|
5735
5911
|
provider
|
|
5736
5912
|
});
|
|
5737
|
-
if (
|
|
5738
|
-
throw new Error("Storage configuration is required for uploading
|
|
5913
|
+
if (uploadToS3 && (!s3Endpoint || !s3Bucket || !effectiveStorageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
|
|
5914
|
+
throw new Error("Storage configuration is required for uploading. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.");
|
|
5739
5915
|
}
|
|
5740
5916
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
5741
5917
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
5742
|
-
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
5743
5918
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
5744
5919
|
if (policy === "signed" && !signingContext) {
|
|
5745
5920
|
throw new Error(
|
|
@@ -5747,30 +5922,20 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
|
|
|
5747
5922
|
);
|
|
5748
5923
|
}
|
|
5749
5924
|
const readyTextTracks = getReadyTextTracks(assetData);
|
|
5750
|
-
|
|
5751
|
-
throw new Error("No ready text tracks found for this asset");
|
|
5752
|
-
}
|
|
5753
|
-
let sourceTextTrack = readyTextTracks.find(
|
|
5754
|
-
(track) => track.text_type === "subtitles" && track.language_code === fromLanguageCode
|
|
5755
|
-
);
|
|
5756
|
-
if (!sourceTextTrack && isAudioOnly && readyTextTracks.length === 1) {
|
|
5757
|
-
sourceTextTrack = readyTextTracks[0];
|
|
5758
|
-
}
|
|
5925
|
+
const sourceTextTrack = readyTextTracks.find((t) => t.id === trackId);
|
|
5759
5926
|
if (!sourceTextTrack) {
|
|
5760
|
-
const
|
|
5761
|
-
if (isAudioOnly) {
|
|
5762
|
-
throw new Error(
|
|
5763
|
-
`No transcript track found with language code '${fromLanguageCode}' for this asset. Audio-only assets require a transcript. Available languages: ${availableLanguages || "none"}`
|
|
5764
|
-
);
|
|
5765
|
-
}
|
|
5927
|
+
const availableTrackIds = readyTextTracks.map((t) => t.id).filter(Boolean).join(", ");
|
|
5766
5928
|
throw new Error(
|
|
5767
|
-
`
|
|
5929
|
+
`Track '${trackId}' not found or not ready on asset '${assetId}'. Available track IDs: ${availableTrackIds || "none"}`
|
|
5768
5930
|
);
|
|
5769
5931
|
}
|
|
5770
|
-
|
|
5771
|
-
|
|
5932
|
+
const fromLanguageCode = sourceTextTrack.language_code;
|
|
5933
|
+
if (!fromLanguageCode) {
|
|
5934
|
+
throw new Error(
|
|
5935
|
+
`Track '${trackId}' is missing language metadata (language_code). Cannot determine source language for translation.`
|
|
5936
|
+
);
|
|
5772
5937
|
}
|
|
5773
|
-
const vttUrl = await buildTranscriptUrl(playbackId,
|
|
5938
|
+
const vttUrl = await buildTranscriptUrl(playbackId, trackId, policy === "signed", credentials);
|
|
5774
5939
|
let vttContent;
|
|
5775
5940
|
try {
|
|
5776
5941
|
vttContent = await fetchVttFromMux(vttUrl);
|
|
@@ -5803,50 +5968,43 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
|
|
|
5803
5968
|
} : void 0;
|
|
5804
5969
|
const sourceLanguage = getLanguageCodePair(fromLanguageCode);
|
|
5805
5970
|
const targetLanguage = getLanguageCodePair(toLanguageCode);
|
|
5806
|
-
if (!uploadToMux) {
|
|
5807
|
-
return {
|
|
5808
|
-
assetId,
|
|
5809
|
-
sourceLanguageCode: fromLanguageCode,
|
|
5810
|
-
targetLanguageCode: toLanguageCode,
|
|
5811
|
-
sourceLanguage,
|
|
5812
|
-
targetLanguage,
|
|
5813
|
-
originalVtt: vttContent,
|
|
5814
|
-
translatedVtt,
|
|
5815
|
-
usage: usageWithMetadata
|
|
5816
|
-
};
|
|
5817
|
-
}
|
|
5818
5971
|
let presignedUrl;
|
|
5819
|
-
try {
|
|
5820
|
-
presignedUrl = await uploadVttToS3({
|
|
5821
|
-
translatedVtt,
|
|
5822
|
-
assetId,
|
|
5823
|
-
fromLanguageCode,
|
|
5824
|
-
toLanguageCode,
|
|
5825
|
-
s3Endpoint,
|
|
5826
|
-
s3Region,
|
|
5827
|
-
s3Bucket,
|
|
5828
|
-
storageAdapter: effectiveStorageAdapter,
|
|
5829
|
-
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
5830
|
-
});
|
|
5831
|
-
} catch (error) {
|
|
5832
|
-
throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
5833
|
-
}
|
|
5834
5972
|
let uploadedTrackId;
|
|
5835
|
-
|
|
5836
|
-
|
|
5837
|
-
|
|
5838
|
-
|
|
5839
|
-
|
|
5840
|
-
|
|
5841
|
-
|
|
5842
|
-
|
|
5843
|
-
|
|
5844
|
-
|
|
5845
|
-
|
|
5846
|
-
|
|
5973
|
+
if (uploadToS3) {
|
|
5974
|
+
try {
|
|
5975
|
+
presignedUrl = await uploadVttToS3({
|
|
5976
|
+
translatedVtt,
|
|
5977
|
+
assetId,
|
|
5978
|
+
fromLanguageCode,
|
|
5979
|
+
toLanguageCode,
|
|
5980
|
+
s3Endpoint,
|
|
5981
|
+
s3Region,
|
|
5982
|
+
s3Bucket,
|
|
5983
|
+
storageAdapter: effectiveStorageAdapter,
|
|
5984
|
+
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
5985
|
+
});
|
|
5986
|
+
} catch (error) {
|
|
5987
|
+
throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
5988
|
+
}
|
|
5989
|
+
if (uploadToMux) {
|
|
5990
|
+
try {
|
|
5991
|
+
const languageName = getLanguageName(toLanguageCode);
|
|
5992
|
+
const trackName = `${languageName} (auto-translated)`;
|
|
5993
|
+
uploadedTrackId = await createTextTrackOnMux(
|
|
5994
|
+
assetId,
|
|
5995
|
+
toLanguageCode,
|
|
5996
|
+
trackName,
|
|
5997
|
+
presignedUrl,
|
|
5998
|
+
credentials
|
|
5999
|
+
);
|
|
6000
|
+
} catch (error) {
|
|
6001
|
+
console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
6002
|
+
}
|
|
6003
|
+
}
|
|
5847
6004
|
}
|
|
5848
6005
|
return {
|
|
5849
6006
|
assetId,
|
|
6007
|
+
trackId,
|
|
5850
6008
|
sourceLanguageCode: fromLanguageCode,
|
|
5851
6009
|
targetLanguageCode: toLanguageCode,
|
|
5852
6010
|
sourceLanguage,
|