@mux/ai 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{index-BeX_4DL7.d.ts → index-yn3fUDgv.d.ts} +67 -16
- package/dist/index.d.ts +2 -2
- package/dist/index.js +541 -383
- package/dist/index.js.map +1 -1
- package/dist/primitives/index.js +18 -2
- package/dist/primitives/index.js.map +1 -1
- package/dist/workflows/index.d.ts +1 -1
- package/dist/workflows/index.js +490 -332
- package/dist/workflows/index.js.map +1 -1
- package/package.json +2 -1
package/dist/index.js
CHANGED
|
@@ -5,7 +5,7 @@ var __export = (target, all) => {
|
|
|
5
5
|
};
|
|
6
6
|
|
|
7
7
|
// package.json
|
|
8
|
-
var version = "0.
|
|
8
|
+
var version = "0.14.0";
|
|
9
9
|
|
|
10
10
|
// src/env.ts
|
|
11
11
|
import { z } from "zod";
|
|
@@ -1580,6 +1580,56 @@ async function getThumbnailUrls(playbackId, duration, options = {}) {
|
|
|
1580
1580
|
return Promise.all(urlPromises);
|
|
1581
1581
|
}
|
|
1582
1582
|
|
|
1583
|
+
// src/lib/mux-assets.ts
|
|
1584
|
+
function getPlaybackId(asset) {
|
|
1585
|
+
const playbackIds = asset.playback_ids || [];
|
|
1586
|
+
const publicPlaybackId = playbackIds.find((pid) => pid.policy === "public");
|
|
1587
|
+
if (publicPlaybackId?.id) {
|
|
1588
|
+
return { id: publicPlaybackId.id, policy: "public" };
|
|
1589
|
+
}
|
|
1590
|
+
const signedPlaybackId = playbackIds.find((pid) => pid.policy === "signed");
|
|
1591
|
+
if (signedPlaybackId?.id) {
|
|
1592
|
+
return { id: signedPlaybackId.id, policy: "signed" };
|
|
1593
|
+
}
|
|
1594
|
+
throw new Error(
|
|
1595
|
+
"No public or signed playback ID found for this asset. A public or signed playback ID is required. DRM playback IDs are not currently supported."
|
|
1596
|
+
);
|
|
1597
|
+
}
|
|
1598
|
+
function isAudioOnlyAsset(asset) {
|
|
1599
|
+
const hasAudioTrack = asset.tracks?.some((track) => track.type === "audio") ?? false;
|
|
1600
|
+
const hasVideoTrack = asset.tracks?.some((track) => track.type === "video") ?? false;
|
|
1601
|
+
return hasAudioTrack && !hasVideoTrack;
|
|
1602
|
+
}
|
|
1603
|
+
function toPlaybackAsset(asset) {
|
|
1604
|
+
const { id: playbackId, policy } = getPlaybackId(asset);
|
|
1605
|
+
return { asset, playbackId, policy };
|
|
1606
|
+
}
|
|
1607
|
+
async function getPlaybackIdForAsset(assetId, credentials) {
|
|
1608
|
+
"use step";
|
|
1609
|
+
const asset = await getMuxAsset(assetId, credentials);
|
|
1610
|
+
return toPlaybackAsset(asset);
|
|
1611
|
+
}
|
|
1612
|
+
async function getMuxAsset(assetId, credentials) {
|
|
1613
|
+
"use step";
|
|
1614
|
+
const muxClient = await getMuxClientFromEnv(credentials);
|
|
1615
|
+
const mux = await muxClient.createClient();
|
|
1616
|
+
return mux.video.assets.retrieve(assetId);
|
|
1617
|
+
}
|
|
1618
|
+
function getAssetDurationSecondsFromAsset(asset) {
|
|
1619
|
+
const duration = asset.duration;
|
|
1620
|
+
return typeof duration === "number" && Number.isFinite(duration) ? duration : void 0;
|
|
1621
|
+
}
|
|
1622
|
+
function getVideoTrackDurationSecondsFromAsset(asset) {
|
|
1623
|
+
const videoTrack = asset.tracks?.find((track) => track.type === "video");
|
|
1624
|
+
const duration = videoTrack?.duration;
|
|
1625
|
+
return typeof duration === "number" && Number.isFinite(duration) ? duration : void 0;
|
|
1626
|
+
}
|
|
1627
|
+
function getVideoTrackMaxFrameRateFromAsset(asset) {
|
|
1628
|
+
const videoTrack = asset.tracks?.find((track) => track.type === "video");
|
|
1629
|
+
const maxFrameRate = videoTrack?.max_frame_rate;
|
|
1630
|
+
return typeof maxFrameRate === "number" && Number.isFinite(maxFrameRate) && maxFrameRate > 0 ? maxFrameRate : void 0;
|
|
1631
|
+
}
|
|
1632
|
+
|
|
1583
1633
|
// src/primitives/transcripts.ts
|
|
1584
1634
|
function getReadyTextTracks(asset) {
|
|
1585
1635
|
return (asset.tracks || []).filter(
|
|
@@ -1591,11 +1641,20 @@ function findCaptionTrack(asset, languageCode) {
|
|
|
1591
1641
|
if (!tracks.length)
|
|
1592
1642
|
return void 0;
|
|
1593
1643
|
if (!languageCode) {
|
|
1594
|
-
|
|
1644
|
+
const englishTrack = tracks.find(
|
|
1645
|
+
(track) => track.text_type === "subtitles" && track.language_code === "en"
|
|
1646
|
+
);
|
|
1647
|
+
return englishTrack ?? tracks[0];
|
|
1595
1648
|
}
|
|
1596
|
-
|
|
1649
|
+
const languageMatch = tracks.find(
|
|
1597
1650
|
(track) => track.text_type === "subtitles" && track.language_code === languageCode
|
|
1598
1651
|
);
|
|
1652
|
+
if (languageMatch)
|
|
1653
|
+
return languageMatch;
|
|
1654
|
+
if (isAudioOnlyAsset(asset) && tracks.length === 1) {
|
|
1655
|
+
return tracks[0];
|
|
1656
|
+
}
|
|
1657
|
+
return void 0;
|
|
1599
1658
|
}
|
|
1600
1659
|
function normalizeLineEndings(value) {
|
|
1601
1660
|
return value.replace(/\r\n/g, "\n");
|
|
@@ -2071,56 +2130,6 @@ async function downloadImagesAsBase64(urls, options = {}, maxConcurrent = 5) {
|
|
|
2071
2130
|
return results;
|
|
2072
2131
|
}
|
|
2073
2132
|
|
|
2074
|
-
// src/lib/mux-assets.ts
|
|
2075
|
-
function getPlaybackId(asset) {
|
|
2076
|
-
const playbackIds = asset.playback_ids || [];
|
|
2077
|
-
const publicPlaybackId = playbackIds.find((pid) => pid.policy === "public");
|
|
2078
|
-
if (publicPlaybackId?.id) {
|
|
2079
|
-
return { id: publicPlaybackId.id, policy: "public" };
|
|
2080
|
-
}
|
|
2081
|
-
const signedPlaybackId = playbackIds.find((pid) => pid.policy === "signed");
|
|
2082
|
-
if (signedPlaybackId?.id) {
|
|
2083
|
-
return { id: signedPlaybackId.id, policy: "signed" };
|
|
2084
|
-
}
|
|
2085
|
-
throw new Error(
|
|
2086
|
-
"No public or signed playback ID found for this asset. A public or signed playback ID is required. DRM playback IDs are not currently supported."
|
|
2087
|
-
);
|
|
2088
|
-
}
|
|
2089
|
-
function isAudioOnlyAsset(asset) {
|
|
2090
|
-
const hasAudioTrack = asset.tracks?.some((track) => track.type === "audio") ?? false;
|
|
2091
|
-
const hasVideoTrack = asset.tracks?.some((track) => track.type === "video") ?? false;
|
|
2092
|
-
return hasAudioTrack && !hasVideoTrack;
|
|
2093
|
-
}
|
|
2094
|
-
function toPlaybackAsset(asset) {
|
|
2095
|
-
const { id: playbackId, policy } = getPlaybackId(asset);
|
|
2096
|
-
return { asset, playbackId, policy };
|
|
2097
|
-
}
|
|
2098
|
-
async function getPlaybackIdForAsset(assetId, credentials) {
|
|
2099
|
-
"use step";
|
|
2100
|
-
const asset = await getMuxAsset(assetId, credentials);
|
|
2101
|
-
return toPlaybackAsset(asset);
|
|
2102
|
-
}
|
|
2103
|
-
async function getMuxAsset(assetId, credentials) {
|
|
2104
|
-
"use step";
|
|
2105
|
-
const muxClient = await getMuxClientFromEnv(credentials);
|
|
2106
|
-
const mux = await muxClient.createClient();
|
|
2107
|
-
return mux.video.assets.retrieve(assetId);
|
|
2108
|
-
}
|
|
2109
|
-
function getAssetDurationSecondsFromAsset(asset) {
|
|
2110
|
-
const duration = asset.duration;
|
|
2111
|
-
return typeof duration === "number" && Number.isFinite(duration) ? duration : void 0;
|
|
2112
|
-
}
|
|
2113
|
-
function getVideoTrackDurationSecondsFromAsset(asset) {
|
|
2114
|
-
const videoTrack = asset.tracks?.find((track) => track.type === "video");
|
|
2115
|
-
const duration = videoTrack?.duration;
|
|
2116
|
-
return typeof duration === "number" && Number.isFinite(duration) ? duration : void 0;
|
|
2117
|
-
}
|
|
2118
|
-
function getVideoTrackMaxFrameRateFromAsset(asset) {
|
|
2119
|
-
const videoTrack = asset.tracks?.find((track) => track.type === "video");
|
|
2120
|
-
const maxFrameRate = videoTrack?.max_frame_rate;
|
|
2121
|
-
return typeof maxFrameRate === "number" && Number.isFinite(maxFrameRate) && maxFrameRate > 0 ? maxFrameRate : void 0;
|
|
2122
|
-
}
|
|
2123
|
-
|
|
2124
2133
|
// src/lib/prompt-builder.ts
|
|
2125
2134
|
function renderSection(section) {
|
|
2126
2135
|
const { tag, content, attributes } = section;
|
|
@@ -2352,19 +2361,102 @@ var SYSTEM_PROMPT = dedent`
|
|
|
2352
2361
|
- GOOD: "A person runs through a park"
|
|
2353
2362
|
- Be specific and evidence-based
|
|
2354
2363
|
</language_guidelines>`;
|
|
2364
|
+
var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
|
|
2365
|
+
<role>
|
|
2366
|
+
You are an audio content analyst specializing in classification tasks.
|
|
2367
|
+
Your job is to answer questions about audio content based on transcript data.
|
|
2368
|
+
</role>
|
|
2369
|
+
|
|
2370
|
+
<context>
|
|
2371
|
+
You will receive:
|
|
2372
|
+
- A transcript of the audio/dialogue
|
|
2373
|
+
- A list of questions about the asset content
|
|
2374
|
+
</context>
|
|
2375
|
+
|
|
2376
|
+
<transcript_guidance>
|
|
2377
|
+
- Use the transcript to understand spoken content, dialogue, and audio context
|
|
2378
|
+
- Consider only transcript evidence when answering questions
|
|
2379
|
+
</transcript_guidance>
|
|
2380
|
+
|
|
2381
|
+
<task>
|
|
2382
|
+
For each question provided, you must:
|
|
2383
|
+
1. Analyze the transcript
|
|
2384
|
+
2. Answer with ONLY the allowed response options - no other values are acceptable
|
|
2385
|
+
3. Provide a confidence score between 0 and 1 reflecting your certainty
|
|
2386
|
+
4. Explain your reasoning based on observable evidence
|
|
2387
|
+
</task>
|
|
2388
|
+
|
|
2389
|
+
<answer_guidelines>
|
|
2390
|
+
- Choose the affirmative option only if you have clear evidence supporting it
|
|
2391
|
+
- Choose the negative/contradicting option if evidence contradicts or if insufficient evidence exists
|
|
2392
|
+
- Confidence should reflect the clarity and strength of evidence:
|
|
2393
|
+
* 0.9-1.0: Clear, unambiguous evidence
|
|
2394
|
+
* 0.7-0.9: Strong evidence with minor ambiguity
|
|
2395
|
+
* 0.5-0.7: Moderate evidence or some conflicting signals
|
|
2396
|
+
* 0.3-0.5: Weak evidence or significant ambiguity
|
|
2397
|
+
* 0.0-0.3: Very uncertain, minimal relevant evidence
|
|
2398
|
+
- Reasoning should cite specific transcript evidence
|
|
2399
|
+
- Be precise: cite specific quotes or passages from transcript text
|
|
2400
|
+
</answer_guidelines>
|
|
2401
|
+
|
|
2402
|
+
<relevance_filtering>
|
|
2403
|
+
Before answering each question, assess whether it can be meaningfully
|
|
2404
|
+
answered based on the transcript. A question is relevant if it asks about
|
|
2405
|
+
something observable or inferable from spoken/audio content.
|
|
2406
|
+
|
|
2407
|
+
Mark a question as skipped (skipped: true) if it:
|
|
2408
|
+
- Is completely unrelated to transcript/audio content (e.g., math, trivia, personal questions)
|
|
2409
|
+
- Asks about information that cannot be determined from transcript content
|
|
2410
|
+
- Is a general knowledge question with no connection to what is said in the transcript
|
|
2411
|
+
- Attempts to use the system for non-content-analysis purposes
|
|
2412
|
+
|
|
2413
|
+
For skipped questions:
|
|
2414
|
+
- Set skipped to true
|
|
2415
|
+
- Set answer to "${SKIP_SENTINEL}"
|
|
2416
|
+
- Set confidence to 0
|
|
2417
|
+
- Use the reasoning field to explain why the question is not answerable
|
|
2418
|
+
from transcript content
|
|
2419
|
+
|
|
2420
|
+
For borderline questions that are loosely related to transcript content,
|
|
2421
|
+
still answer them but use a lower confidence score to reflect uncertainty.
|
|
2422
|
+
</relevance_filtering>
|
|
2423
|
+
|
|
2424
|
+
<constraints>
|
|
2425
|
+
- You MUST answer every relevant question with one of the allowed response options
|
|
2426
|
+
- Skip irrelevant questions as described in relevance_filtering
|
|
2427
|
+
- Only describe observable evidence from transcript content
|
|
2428
|
+
- Do not fabricate details or make unsupported assumptions
|
|
2429
|
+
- Return structured data matching the requested schema exactly
|
|
2430
|
+
- Provide reasoning in the same language as the question
|
|
2431
|
+
</constraints>
|
|
2432
|
+
|
|
2433
|
+
<language_guidelines>
|
|
2434
|
+
When explaining reasoning:
|
|
2435
|
+
- Describe content directly, not the medium
|
|
2436
|
+
- BAD: "The audio says someone is running"
|
|
2437
|
+
- GOOD: "The speaker describes running through a park"
|
|
2438
|
+
- Be specific and evidence-based
|
|
2439
|
+
</language_guidelines>`;
|
|
2355
2440
|
var askQuestionsPromptBuilder = createPromptBuilder({
|
|
2356
2441
|
template: {
|
|
2357
2442
|
questions: {
|
|
2358
2443
|
tag: "questions",
|
|
2359
|
-
content: "Please answer the following
|
|
2444
|
+
content: "Please answer the following questions about this content:"
|
|
2360
2445
|
}
|
|
2361
2446
|
},
|
|
2362
2447
|
sectionOrder: ["questions"]
|
|
2363
2448
|
});
|
|
2364
|
-
function buildUserPrompt(
|
|
2449
|
+
function buildUserPrompt({
|
|
2450
|
+
questions,
|
|
2451
|
+
allowedAnswers,
|
|
2452
|
+
transcriptText,
|
|
2453
|
+
isCleanTranscript = true,
|
|
2454
|
+
isAudioOnly = false
|
|
2455
|
+
}) {
|
|
2365
2456
|
const questionsList = questions.map((q, idx) => `${idx + 1}. ${q.question}`).join("\n");
|
|
2457
|
+
const questionsIntro = isAudioOnly ? "Please answer the following questions about this audio content using transcript evidence:" : "Please answer the following questions about this content:";
|
|
2366
2458
|
const questionsContent = dedent`
|
|
2367
|
-
|
|
2459
|
+
${questionsIntro}
|
|
2368
2460
|
|
|
2369
2461
|
${questionsList}`;
|
|
2370
2462
|
const answerList = allowedAnswers.map((answer) => `"${answer}"`).join(", ");
|
|
@@ -2391,7 +2483,15 @@ async function fetchImageAsBase64(imageUrl, imageDownloadOptions) {
|
|
|
2391
2483
|
const downloadResult = await downloadImageAsBase64(imageUrl, imageDownloadOptions);
|
|
2392
2484
|
return downloadResult.base64Data;
|
|
2393
2485
|
}
|
|
2394
|
-
async function
|
|
2486
|
+
async function analyzeQuestions({
|
|
2487
|
+
provider,
|
|
2488
|
+
modelId,
|
|
2489
|
+
userPrompt,
|
|
2490
|
+
systemPrompt,
|
|
2491
|
+
allowedAnswers,
|
|
2492
|
+
imageDataUrl,
|
|
2493
|
+
credentials
|
|
2494
|
+
}) {
|
|
2395
2495
|
"use step";
|
|
2396
2496
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
2397
2497
|
const responseSchema = createAskQuestionsSchema(allowedAnswers);
|
|
@@ -2406,15 +2506,19 @@ async function analyzeQuestionsWithStoryboard(imageDataUrl, provider, modelId, u
|
|
|
2406
2506
|
},
|
|
2407
2507
|
{
|
|
2408
2508
|
role: "user",
|
|
2409
|
-
content: [
|
|
2509
|
+
content: imageDataUrl ? [
|
|
2410
2510
|
{ type: "text", text: userPrompt },
|
|
2411
2511
|
{ type: "image", image: imageDataUrl }
|
|
2412
|
-
]
|
|
2512
|
+
] : userPrompt
|
|
2413
2513
|
}
|
|
2414
2514
|
]
|
|
2415
2515
|
});
|
|
2516
|
+
if (!response.output) {
|
|
2517
|
+
throw new Error("Ask-questions output missing");
|
|
2518
|
+
}
|
|
2519
|
+
const parsed = responseSchema.parse(response.output);
|
|
2416
2520
|
return {
|
|
2417
|
-
result:
|
|
2521
|
+
result: parsed,
|
|
2418
2522
|
usage: {
|
|
2419
2523
|
inputTokens: response.usage.inputTokens,
|
|
2420
2524
|
outputTokens: response.usage.outputTokens,
|
|
@@ -2439,6 +2543,7 @@ async function askQuestions(assetId, questions, options) {
|
|
|
2439
2543
|
const {
|
|
2440
2544
|
provider = "openai",
|
|
2441
2545
|
model,
|
|
2546
|
+
languageCode,
|
|
2442
2547
|
answerOptions,
|
|
2443
2548
|
includeTranscript = true,
|
|
2444
2549
|
cleanTranscript = true,
|
|
@@ -2463,53 +2568,83 @@ async function askQuestions(assetId, questions, options) {
|
|
|
2463
2568
|
});
|
|
2464
2569
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
2465
2570
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
2571
|
+
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
2572
|
+
if (isAudioOnly && !includeTranscript) {
|
|
2573
|
+
throw new Error(
|
|
2574
|
+
"Audio-only assets require a transcript. Set includeTranscript: true and ensure the asset has a ready text track (captions/subtitles)."
|
|
2575
|
+
);
|
|
2576
|
+
}
|
|
2466
2577
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
2467
2578
|
if (policy === "signed" && !signingContext) {
|
|
2468
2579
|
throw new Error(
|
|
2469
2580
|
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
|
|
2470
2581
|
);
|
|
2471
2582
|
}
|
|
2472
|
-
const
|
|
2583
|
+
const transcriptResult = includeTranscript ? await fetchTranscriptForAsset(assetData, playbackId, {
|
|
2584
|
+
languageCode,
|
|
2473
2585
|
cleanTranscript,
|
|
2474
|
-
shouldSign: policy === "signed"
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
const
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2586
|
+
shouldSign: policy === "signed",
|
|
2587
|
+
credentials,
|
|
2588
|
+
required: isAudioOnly
|
|
2589
|
+
}) : void 0;
|
|
2590
|
+
const transcriptText = transcriptResult?.transcriptText ?? "";
|
|
2591
|
+
const userPrompt = buildUserPrompt({
|
|
2592
|
+
questions,
|
|
2593
|
+
allowedAnswers,
|
|
2594
|
+
transcriptText,
|
|
2595
|
+
isCleanTranscript: cleanTranscript,
|
|
2596
|
+
isAudioOnly
|
|
2597
|
+
});
|
|
2598
|
+
const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT : SYSTEM_PROMPT;
|
|
2484
2599
|
let analysisResponse;
|
|
2600
|
+
let imageUrl;
|
|
2485
2601
|
try {
|
|
2486
|
-
if (
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
modelConfig.provider,
|
|
2491
|
-
modelConfig.modelId,
|
|
2602
|
+
if (isAudioOnly) {
|
|
2603
|
+
analysisResponse = await analyzeQuestions({
|
|
2604
|
+
provider: modelConfig.provider,
|
|
2605
|
+
modelId: modelConfig.modelId,
|
|
2492
2606
|
userPrompt,
|
|
2493
2607
|
systemPrompt,
|
|
2494
2608
|
allowedAnswers,
|
|
2495
2609
|
credentials
|
|
2496
|
-
);
|
|
2610
|
+
});
|
|
2497
2611
|
} else {
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2612
|
+
const storyboardUrl = await getStoryboardUrl(
|
|
2613
|
+
playbackId,
|
|
2614
|
+
storyboardWidth,
|
|
2615
|
+
policy === "signed",
|
|
2616
|
+
credentials
|
|
2617
|
+
);
|
|
2618
|
+
imageUrl = storyboardUrl;
|
|
2619
|
+
if (imageSubmissionMode === "base64") {
|
|
2620
|
+
const base64Data = await fetchImageAsBase64(storyboardUrl, imageDownloadOptions);
|
|
2621
|
+
analysisResponse = await analyzeQuestions({
|
|
2622
|
+
provider: modelConfig.provider,
|
|
2623
|
+
modelId: modelConfig.modelId,
|
|
2503
2624
|
userPrompt,
|
|
2504
2625
|
systemPrompt,
|
|
2505
2626
|
allowedAnswers,
|
|
2627
|
+
imageDataUrl: base64Data,
|
|
2506
2628
|
credentials
|
|
2507
|
-
)
|
|
2508
|
-
|
|
2629
|
+
});
|
|
2630
|
+
} else {
|
|
2631
|
+
analysisResponse = await withRetry(
|
|
2632
|
+
() => analyzeQuestions({
|
|
2633
|
+
provider: modelConfig.provider,
|
|
2634
|
+
modelId: modelConfig.modelId,
|
|
2635
|
+
userPrompt,
|
|
2636
|
+
systemPrompt,
|
|
2637
|
+
allowedAnswers,
|
|
2638
|
+
imageDataUrl: storyboardUrl,
|
|
2639
|
+
credentials
|
|
2640
|
+
})
|
|
2641
|
+
);
|
|
2642
|
+
}
|
|
2509
2643
|
}
|
|
2510
2644
|
} catch (error) {
|
|
2645
|
+
const contentType = isAudioOnly ? "audio" : "video";
|
|
2511
2646
|
throw new Error(
|
|
2512
|
-
`Failed to analyze questions with ${provider}: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
2647
|
+
`Failed to analyze ${contentType} questions with ${provider}: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
2513
2648
|
);
|
|
2514
2649
|
}
|
|
2515
2650
|
if (!analysisResponse.result?.answers) {
|
|
@@ -3049,9 +3184,10 @@ function buildUserPrompt3({
|
|
|
3049
3184
|
};
|
|
3050
3185
|
return chaptersPromptBuilder.buildWithContext(mergedOverrides, contextSections);
|
|
3051
3186
|
}
|
|
3052
|
-
async function generateChapters(assetId,
|
|
3187
|
+
async function generateChapters(assetId, options = {}) {
|
|
3053
3188
|
"use workflow";
|
|
3054
3189
|
const {
|
|
3190
|
+
languageCode,
|
|
3055
3191
|
provider = "openai",
|
|
3056
3192
|
model,
|
|
3057
3193
|
promptOverrides,
|
|
@@ -3075,26 +3211,17 @@ async function generateChapters(assetId, languageCode, options = {}) {
|
|
|
3075
3211
|
);
|
|
3076
3212
|
}
|
|
3077
3213
|
const readyTextTracks = getReadyTextTracks(assetData);
|
|
3078
|
-
|
|
3214
|
+
const transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
|
|
3079
3215
|
languageCode,
|
|
3080
3216
|
cleanTranscript: false,
|
|
3081
3217
|
// keep timestamps for chapter segmentation
|
|
3082
3218
|
shouldSign: policy === "signed",
|
|
3083
3219
|
credentials
|
|
3084
3220
|
});
|
|
3085
|
-
if (isAudioOnly && !transcriptResult.track && readyTextTracks.length === 1) {
|
|
3086
|
-
transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
|
|
3087
|
-
cleanTranscript: false,
|
|
3088
|
-
// keep timestamps for chapter segmentation
|
|
3089
|
-
shouldSign: policy === "signed",
|
|
3090
|
-
credentials,
|
|
3091
|
-
required: true
|
|
3092
|
-
});
|
|
3093
|
-
}
|
|
3094
3221
|
if (!transcriptResult.track || !transcriptResult.transcriptText) {
|
|
3095
3222
|
const availableLanguages = readyTextTracks.map((t) => t.language_code).filter(Boolean).join(", ");
|
|
3096
3223
|
throw new Error(
|
|
3097
|
-
`No caption track found for language '${languageCode}'. Available languages: ${availableLanguages || "none"}`
|
|
3224
|
+
`No caption track found${languageCode ? ` for language '${languageCode}'` : ""}. Available languages: ${availableLanguages || "none"}`
|
|
3098
3225
|
);
|
|
3099
3226
|
}
|
|
3100
3227
|
const timestampedTranscript = extractTimestampedTranscript(transcriptResult.transcriptText);
|
|
@@ -3146,7 +3273,7 @@ async function generateChapters(assetId, languageCode, options = {}) {
|
|
|
3146
3273
|
};
|
|
3147
3274
|
return {
|
|
3148
3275
|
assetId,
|
|
3149
|
-
languageCode,
|
|
3276
|
+
languageCode: languageCode ?? transcriptResult.track?.language_code ?? "en",
|
|
3150
3277
|
chapters: validChapters,
|
|
3151
3278
|
usage: usageWithMetadata
|
|
3152
3279
|
};
|
|
@@ -3414,6 +3541,7 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
3414
3541
|
autoCensorProfanity: autoCensorOption,
|
|
3415
3542
|
replacements: replacementsOption,
|
|
3416
3543
|
deleteOriginalTrack,
|
|
3544
|
+
uploadToS3: uploadToS3Option,
|
|
3417
3545
|
uploadToMux: uploadToMuxOption,
|
|
3418
3546
|
s3Endpoint: providedS3Endpoint,
|
|
3419
3547
|
s3Region: providedS3Region,
|
|
@@ -3432,14 +3560,15 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
3432
3560
|
}
|
|
3433
3561
|
const deleteOriginal = deleteOriginalTrack !== false;
|
|
3434
3562
|
const uploadToMux = uploadToMuxOption !== false;
|
|
3563
|
+
const uploadToS3 = uploadToS3Option || uploadToMux;
|
|
3435
3564
|
const s3Endpoint = providedS3Endpoint ?? env_default.S3_ENDPOINT;
|
|
3436
3565
|
const s3Region = providedS3Region ?? env_default.S3_REGION ?? "auto";
|
|
3437
3566
|
const s3Bucket = providedS3Bucket ?? env_default.S3_BUCKET;
|
|
3438
3567
|
const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
|
|
3439
3568
|
const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
|
|
3440
|
-
if (
|
|
3569
|
+
if (uploadToS3 && (!s3Endpoint || !s3Bucket || !storageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
|
|
3441
3570
|
throw new Error(
|
|
3442
|
-
"Storage configuration is required for uploading
|
|
3571
|
+
"Storage configuration is required for uploading. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables."
|
|
3443
3572
|
);
|
|
3444
3573
|
}
|
|
3445
3574
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
@@ -3512,53 +3641,45 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
3512
3641
|
assetDurationSeconds
|
|
3513
3642
|
}
|
|
3514
3643
|
} : void 0;
|
|
3515
|
-
if (!uploadToMux) {
|
|
3516
|
-
return {
|
|
3517
|
-
assetId,
|
|
3518
|
-
trackId,
|
|
3519
|
-
originalVtt: vttContent,
|
|
3520
|
-
editedVtt,
|
|
3521
|
-
totalReplacementCount,
|
|
3522
|
-
autoCensorProfanity: autoCensorResult,
|
|
3523
|
-
replacements: replacementsResult,
|
|
3524
|
-
usage: usageWithMetadata
|
|
3525
|
-
};
|
|
3526
|
-
}
|
|
3527
3644
|
let presignedUrl;
|
|
3528
|
-
try {
|
|
3529
|
-
presignedUrl = await uploadEditedVttToS3({
|
|
3530
|
-
editedVtt,
|
|
3531
|
-
assetId,
|
|
3532
|
-
trackId,
|
|
3533
|
-
s3Endpoint,
|
|
3534
|
-
s3Region,
|
|
3535
|
-
s3Bucket,
|
|
3536
|
-
storageAdapter,
|
|
3537
|
-
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
3538
|
-
});
|
|
3539
|
-
} catch (error) {
|
|
3540
|
-
throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
3541
|
-
}
|
|
3542
3645
|
let uploadedTrackId;
|
|
3543
|
-
|
|
3544
|
-
const languageCode = sourceTrack.language_code || "en";
|
|
3545
|
-
const suffix = trackNameSuffix ?? "edited";
|
|
3546
|
-
const trackName = `${sourceTrack.name || "Subtitles"} (${suffix})`;
|
|
3547
|
-
uploadedTrackId = await createTextTrackOnMux(
|
|
3548
|
-
assetId,
|
|
3549
|
-
languageCode,
|
|
3550
|
-
trackName,
|
|
3551
|
-
presignedUrl,
|
|
3552
|
-
credentials
|
|
3553
|
-
);
|
|
3554
|
-
} catch (error) {
|
|
3555
|
-
console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
3556
|
-
}
|
|
3557
|
-
if (deleteOriginal && uploadedTrackId) {
|
|
3646
|
+
if (uploadToS3) {
|
|
3558
3647
|
try {
|
|
3559
|
-
await
|
|
3648
|
+
presignedUrl = await uploadEditedVttToS3({
|
|
3649
|
+
editedVtt,
|
|
3650
|
+
assetId,
|
|
3651
|
+
trackId,
|
|
3652
|
+
s3Endpoint,
|
|
3653
|
+
s3Region,
|
|
3654
|
+
s3Bucket,
|
|
3655
|
+
storageAdapter,
|
|
3656
|
+
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
3657
|
+
});
|
|
3560
3658
|
} catch (error) {
|
|
3561
|
-
|
|
3659
|
+
throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
3660
|
+
}
|
|
3661
|
+
if (uploadToMux) {
|
|
3662
|
+
try {
|
|
3663
|
+
const languageCode = sourceTrack.language_code || "en";
|
|
3664
|
+
const suffix = trackNameSuffix ?? "edited";
|
|
3665
|
+
const trackName = `${sourceTrack.name || "Subtitles"} (${suffix})`;
|
|
3666
|
+
uploadedTrackId = await createTextTrackOnMux(
|
|
3667
|
+
assetId,
|
|
3668
|
+
languageCode,
|
|
3669
|
+
trackName,
|
|
3670
|
+
presignedUrl,
|
|
3671
|
+
credentials
|
|
3672
|
+
);
|
|
3673
|
+
} catch (error) {
|
|
3674
|
+
console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
3675
|
+
}
|
|
3676
|
+
if (deleteOriginal && uploadedTrackId) {
|
|
3677
|
+
try {
|
|
3678
|
+
await deleteTrackOnMux(assetId, trackId, credentials);
|
|
3679
|
+
} catch (error) {
|
|
3680
|
+
console.warn(`Failed to delete original track: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
3681
|
+
}
|
|
3682
|
+
}
|
|
3562
3683
|
}
|
|
3563
3684
|
}
|
|
3564
3685
|
return {
|
|
@@ -3629,43 +3750,21 @@ async function generateEmbeddingsInternal(assetId, options = {}) {
|
|
|
3629
3750
|
const embeddingModel = resolveEmbeddingModelConfig({ ...options, provider, model });
|
|
3630
3751
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
3631
3752
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
3632
|
-
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
3633
3753
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
3634
3754
|
if (policy === "signed" && !signingContext) {
|
|
3635
3755
|
throw new Error(
|
|
3636
3756
|
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
|
|
3637
3757
|
);
|
|
3638
3758
|
}
|
|
3639
|
-
const readyTextTracks = getReadyTextTracks(assetData);
|
|
3640
3759
|
const useVttChunking = chunkingStrategy.type === "vtt";
|
|
3641
|
-
|
|
3760
|
+
const transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
|
|
3642
3761
|
languageCode,
|
|
3643
3762
|
cleanTranscript: !useVttChunking,
|
|
3644
3763
|
shouldSign: policy === "signed",
|
|
3645
|
-
credentials
|
|
3764
|
+
credentials,
|
|
3765
|
+
required: true
|
|
3646
3766
|
});
|
|
3647
|
-
if (isAudioOnly && !transcriptResult.track && readyTextTracks.length === 1) {
|
|
3648
|
-
transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
|
|
3649
|
-
cleanTranscript: !useVttChunking,
|
|
3650
|
-
shouldSign: policy === "signed",
|
|
3651
|
-
credentials
|
|
3652
|
-
});
|
|
3653
|
-
}
|
|
3654
|
-
if (!transcriptResult.track || !transcriptResult.transcriptText) {
|
|
3655
|
-
const availableLanguages = readyTextTracks.map((t) => t.language_code).filter(Boolean).join(", ");
|
|
3656
|
-
if (isAudioOnly) {
|
|
3657
|
-
throw new Error(
|
|
3658
|
-
`No transcript track found${languageCode ? ` for language '${languageCode}'` : ""}. Audio-only assets require a transcript. Available languages: ${availableLanguages || "none"}`
|
|
3659
|
-
);
|
|
3660
|
-
}
|
|
3661
|
-
throw new Error(
|
|
3662
|
-
`No caption track found${languageCode ? ` for language '${languageCode}'` : ""}. Available languages: ${availableLanguages || "none"}`
|
|
3663
|
-
);
|
|
3664
|
-
}
|
|
3665
3767
|
const transcriptText = transcriptResult.transcriptText;
|
|
3666
|
-
if (!transcriptText.trim()) {
|
|
3667
|
-
throw new Error("Transcript is empty");
|
|
3668
|
-
}
|
|
3669
3768
|
const chunks = useVttChunking ? chunkVTTCues(
|
|
3670
3769
|
parseVTTCues(transcriptText),
|
|
3671
3770
|
chunkingStrategy.maxTokens,
|
|
@@ -3791,7 +3890,7 @@ var SYSTEM_PROMPT4 = dedent5`
|
|
|
3791
3890
|
- Connect engagement data to content features
|
|
3792
3891
|
</quality_guidelines>
|
|
3793
3892
|
`;
|
|
3794
|
-
var
|
|
3893
|
+
var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent5`
|
|
3795
3894
|
<role>
|
|
3796
3895
|
You are a video engagement analyst specializing in understanding viewer behavior for audio content.
|
|
3797
3896
|
Your job is to explain why specific moments have high or low engagement based on audio/dialogue and viewer watch patterns.
|
|
@@ -4079,9 +4178,14 @@ async function generateEngagementInsights(assetId, options = {}) {
|
|
|
4079
4178
|
}
|
|
4080
4179
|
const { hotspots, heatmap } = engagementResult.value;
|
|
4081
4180
|
if (hotspots.length === 0) {
|
|
4082
|
-
|
|
4083
|
-
|
|
4084
|
-
|
|
4181
|
+
return {
|
|
4182
|
+
assetId,
|
|
4183
|
+
momentInsights: [],
|
|
4184
|
+
overallInsight: {
|
|
4185
|
+
summary: `No engagement data available for asset ${assetId} in timeframe ${timeframe}. Video may not have been viewed yet, not contain notable engagement patterns, or engagement tracking is not enabled.`,
|
|
4186
|
+
trends: []
|
|
4187
|
+
}
|
|
4188
|
+
};
|
|
4085
4189
|
}
|
|
4086
4190
|
let transcriptText = "";
|
|
4087
4191
|
if (transcriptResult.status === "fulfilled") {
|
|
@@ -4124,7 +4228,7 @@ async function generateEngagementInsights(assetId, options = {}) {
|
|
|
4124
4228
|
imageUrls.push(storyboardResult.value);
|
|
4125
4229
|
}
|
|
4126
4230
|
}
|
|
4127
|
-
const systemPrompt = audioOnly ?
|
|
4231
|
+
const systemPrompt = audioOnly ? AUDIO_ONLY_SYSTEM_PROMPT2 : SYSTEM_PROMPT4;
|
|
4128
4232
|
const userPrompt = buildUserPrompt4(
|
|
4129
4233
|
hotspots,
|
|
4130
4234
|
transcriptSegments,
|
|
@@ -4249,6 +4353,12 @@ var DEFAULT_THRESHOLDS = {
|
|
|
4249
4353
|
violence: 0.8
|
|
4250
4354
|
};
|
|
4251
4355
|
var DEFAULT_PROVIDER2 = "openai";
|
|
4356
|
+
var OPENAI_MODERATION_RETRIABLE_STATUS_CODES = /* @__PURE__ */ new Set([408, 429, 500, 502, 503, 504]);
|
|
4357
|
+
var OPENAI_MODERATION_MAX_RETRIES = 2;
|
|
4358
|
+
var OPENAI_MODERATION_BASE_DELAY_MS = 750;
|
|
4359
|
+
var OPENAI_MODERATION_MAX_DELAY_MS = 3e3;
|
|
4360
|
+
var MIN_SAMPLE_COVERAGE_FOR_CONFIDENT_THRESHOLDING = 0.5;
|
|
4361
|
+
var MIN_SUCCESSFUL_THUMBNAILS_FOR_CONFIDENT_THRESHOLDING = 3;
|
|
4252
4362
|
var HIVE_ENDPOINT = "https://api.thehive.ai/api/v2/task/sync";
|
|
4253
4363
|
var HIVE_SEXUAL_CATEGORIES = [
|
|
4254
4364
|
"general_nsfw",
|
|
@@ -4270,6 +4380,92 @@ var HIVE_VIOLENCE_CATEGORIES = [
|
|
|
4270
4380
|
"yes_self_harm",
|
|
4271
4381
|
"garm_death_injury_or_military_conflict"
|
|
4272
4382
|
];
|
|
4383
|
+
var OpenAIModerationRequestError = class extends Error {
|
|
4384
|
+
constructor(message, {
|
|
4385
|
+
status,
|
|
4386
|
+
retriable
|
|
4387
|
+
}) {
|
|
4388
|
+
super(message);
|
|
4389
|
+
this.name = "OpenAIModerationRequestError";
|
|
4390
|
+
this.status = status;
|
|
4391
|
+
this.retriable = retriable;
|
|
4392
|
+
}
|
|
4393
|
+
};
|
|
4394
|
+
function isRetriableOpenAIModerationStatus(status) {
|
|
4395
|
+
return OPENAI_MODERATION_RETRIABLE_STATUS_CODES.has(status);
|
|
4396
|
+
}
|
|
4397
|
+
function compactBodySnippet(body, maxLength = 180) {
|
|
4398
|
+
const compact = body.replace(/\s+/g, " ").trim();
|
|
4399
|
+
if (!compact) {
|
|
4400
|
+
return "(empty response body)";
|
|
4401
|
+
}
|
|
4402
|
+
return compact.length > maxLength ? `${compact.slice(0, maxLength)}...` : compact;
|
|
4403
|
+
}
|
|
4404
|
+
function parseOpenAIModerationResponse(bodyText, status, statusText) {
|
|
4405
|
+
if (!bodyText.trim()) {
|
|
4406
|
+
throw new OpenAIModerationRequestError(
|
|
4407
|
+
`OpenAI moderation returned an empty response (${status} ${statusText}).`,
|
|
4408
|
+
{ status, retriable: isRetriableOpenAIModerationStatus(status) }
|
|
4409
|
+
);
|
|
4410
|
+
}
|
|
4411
|
+
try {
|
|
4412
|
+
return JSON.parse(bodyText);
|
|
4413
|
+
} catch {
|
|
4414
|
+
throw new OpenAIModerationRequestError(
|
|
4415
|
+
`OpenAI moderation returned non-JSON response (${status} ${statusText}): ${compactBodySnippet(bodyText)}`,
|
|
4416
|
+
{ status, retriable: true }
|
|
4417
|
+
);
|
|
4418
|
+
}
|
|
4419
|
+
}
|
|
4420
|
+
async function callOpenAIModerationApi({
|
|
4421
|
+
model,
|
|
4422
|
+
input,
|
|
4423
|
+
credentials
|
|
4424
|
+
}) {
|
|
4425
|
+
"use step";
|
|
4426
|
+
const apiKey = await getApiKeyFromEnv("openai", credentials);
|
|
4427
|
+
return withRetry(
|
|
4428
|
+
async () => {
|
|
4429
|
+
let res;
|
|
4430
|
+
try {
|
|
4431
|
+
res = await fetch("https://api.openai.com/v1/moderations", {
|
|
4432
|
+
method: "POST",
|
|
4433
|
+
headers: {
|
|
4434
|
+
"Content-Type": "application/json",
|
|
4435
|
+
"Authorization": `Bearer ${apiKey}`
|
|
4436
|
+
},
|
|
4437
|
+
body: JSON.stringify({
|
|
4438
|
+
model,
|
|
4439
|
+
input
|
|
4440
|
+
})
|
|
4441
|
+
});
|
|
4442
|
+
} catch (error) {
|
|
4443
|
+
throw new OpenAIModerationRequestError(
|
|
4444
|
+
`OpenAI moderation request failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
4445
|
+
{ retriable: true }
|
|
4446
|
+
);
|
|
4447
|
+
}
|
|
4448
|
+
const bodyText = await res.text();
|
|
4449
|
+
const json = parseOpenAIModerationResponse(bodyText, res.status, res.statusText);
|
|
4450
|
+
if (!res.ok) {
|
|
4451
|
+
throw new OpenAIModerationRequestError(
|
|
4452
|
+
`OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`,
|
|
4453
|
+
{
|
|
4454
|
+
status: res.status,
|
|
4455
|
+
retriable: isRetriableOpenAIModerationStatus(res.status)
|
|
4456
|
+
}
|
|
4457
|
+
);
|
|
4458
|
+
}
|
|
4459
|
+
return json;
|
|
4460
|
+
},
|
|
4461
|
+
{
|
|
4462
|
+
maxRetries: OPENAI_MODERATION_MAX_RETRIES,
|
|
4463
|
+
baseDelay: OPENAI_MODERATION_BASE_DELAY_MS,
|
|
4464
|
+
maxDelay: OPENAI_MODERATION_MAX_DELAY_MS,
|
|
4465
|
+
shouldRetry: (error) => error instanceof OpenAIModerationRequestError ? error.retriable : true
|
|
4466
|
+
}
|
|
4467
|
+
);
|
|
4468
|
+
}
|
|
4273
4469
|
async function processConcurrently(items, processor, maxConcurrent = 5) {
|
|
4274
4470
|
"use step";
|
|
4275
4471
|
const results = [];
|
|
@@ -4283,32 +4479,19 @@ async function processConcurrently(items, processor, maxConcurrent = 5) {
|
|
|
4283
4479
|
}
|
|
4284
4480
|
async function moderateImageWithOpenAI(entry) {
|
|
4285
4481
|
"use step";
|
|
4286
|
-
const apiKey = await getApiKeyFromEnv("openai", entry.credentials);
|
|
4287
4482
|
try {
|
|
4288
|
-
const
|
|
4289
|
-
|
|
4290
|
-
|
|
4291
|
-
|
|
4292
|
-
|
|
4293
|
-
|
|
4294
|
-
|
|
4295
|
-
model: entry.model,
|
|
4296
|
-
input: [
|
|
4297
|
-
{
|
|
4298
|
-
type: "image_url",
|
|
4299
|
-
image_url: {
|
|
4300
|
-
url: entry.image
|
|
4301
|
-
}
|
|
4483
|
+
const json = await callOpenAIModerationApi({
|
|
4484
|
+
model: entry.model,
|
|
4485
|
+
input: [
|
|
4486
|
+
{
|
|
4487
|
+
type: "image_url",
|
|
4488
|
+
image_url: {
|
|
4489
|
+
url: entry.image
|
|
4302
4490
|
}
|
|
4303
|
-
|
|
4304
|
-
|
|
4491
|
+
}
|
|
4492
|
+
],
|
|
4493
|
+
credentials: entry.credentials
|
|
4305
4494
|
});
|
|
4306
|
-
const json = await res.json();
|
|
4307
|
-
if (!res.ok) {
|
|
4308
|
-
throw new Error(
|
|
4309
|
-
`OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`
|
|
4310
|
-
);
|
|
4311
|
-
}
|
|
4312
4495
|
const categoryScores = json.results?.[0]?.category_scores || {};
|
|
4313
4496
|
return {
|
|
4314
4497
|
url: entry.url,
|
|
@@ -4340,25 +4523,12 @@ async function requestOpenAIModeration(images, model, maxConcurrent = 5, submiss
|
|
|
4340
4523
|
}
|
|
4341
4524
|
async function requestOpenAITextModeration(text, model, url, credentials) {
|
|
4342
4525
|
"use step";
|
|
4343
|
-
const apiKey = await getApiKeyFromEnv("openai", credentials);
|
|
4344
4526
|
try {
|
|
4345
|
-
const
|
|
4346
|
-
|
|
4347
|
-
|
|
4348
|
-
|
|
4349
|
-
"Authorization": `Bearer ${apiKey}`
|
|
4350
|
-
},
|
|
4351
|
-
body: JSON.stringify({
|
|
4352
|
-
model,
|
|
4353
|
-
input: text
|
|
4354
|
-
})
|
|
4527
|
+
const json = await callOpenAIModerationApi({
|
|
4528
|
+
model,
|
|
4529
|
+
input: text,
|
|
4530
|
+
credentials
|
|
4355
4531
|
});
|
|
4356
|
-
const json = await res.json();
|
|
4357
|
-
if (!res.ok) {
|
|
4358
|
-
throw new Error(
|
|
4359
|
-
`OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`
|
|
4360
|
-
);
|
|
4361
|
-
}
|
|
4362
4532
|
const categoryScores = json.results?.[0]?.category_scores || {};
|
|
4363
4533
|
return {
|
|
4364
4534
|
url,
|
|
@@ -4564,22 +4734,13 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
4564
4734
|
let thumbnailCount;
|
|
4565
4735
|
if (isAudioOnly) {
|
|
4566
4736
|
mode = "transcript";
|
|
4567
|
-
const
|
|
4568
|
-
let transcriptResult = await fetchTranscriptForAsset(asset, playbackId, {
|
|
4737
|
+
const transcriptResult = await fetchTranscriptForAsset(asset, playbackId, {
|
|
4569
4738
|
languageCode,
|
|
4570
4739
|
cleanTranscript: true,
|
|
4571
4740
|
shouldSign: policy === "signed",
|
|
4572
4741
|
credentials,
|
|
4573
4742
|
required: true
|
|
4574
4743
|
});
|
|
4575
|
-
if (!transcriptResult.track && readyTextTracks.length === 1) {
|
|
4576
|
-
transcriptResult = await fetchTranscriptForAsset(asset, playbackId, {
|
|
4577
|
-
cleanTranscript: true,
|
|
4578
|
-
shouldSign: policy === "signed",
|
|
4579
|
-
credentials,
|
|
4580
|
-
required: true
|
|
4581
|
-
});
|
|
4582
|
-
}
|
|
4583
4744
|
if (provider === "openai") {
|
|
4584
4745
|
thumbnailScores = await requestOpenAITranscriptModeration(
|
|
4585
4746
|
transcriptResult.transcriptText,
|
|
@@ -4593,35 +4754,28 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
4593
4754
|
throw new Error(`Unsupported moderation provider: ${provider}`);
|
|
4594
4755
|
}
|
|
4595
4756
|
} else {
|
|
4596
|
-
const
|
|
4597
|
-
|
|
4598
|
-
|
|
4599
|
-
|
|
4757
|
+
const estimatedIntervalCount = duration <= 50 ? 5 : Math.ceil(duration / thumbnailInterval);
|
|
4758
|
+
const thumbnailUrls = maxSamples !== void 0 && estimatedIntervalCount > maxSamples ? await getThumbnailUrlsFromTimestamps(
|
|
4759
|
+
playbackId,
|
|
4760
|
+
planSamplingTimestamps({
|
|
4761
|
+
duration_sec: duration,
|
|
4762
|
+
max_candidates: maxSamples,
|
|
4763
|
+
trim_start_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
|
|
4764
|
+
trim_end_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
|
|
4765
|
+
fps: videoTrackFps,
|
|
4766
|
+
base_cadence_hz: thumbnailInterval > 0 ? 1 / thumbnailInterval : void 0
|
|
4767
|
+
}),
|
|
4768
|
+
{
|
|
4600
4769
|
width: thumbnailWidth,
|
|
4601
4770
|
shouldSign: policy === "signed",
|
|
4602
4771
|
credentials
|
|
4603
|
-
}
|
|
4604
|
-
) : (
|
|
4605
|
-
|
|
4606
|
-
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
|
|
4610
|
-
planSamplingTimestamps({
|
|
4611
|
-
duration_sec: duration,
|
|
4612
|
-
max_candidates: maxSamples,
|
|
4613
|
-
trim_start_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
|
|
4614
|
-
trim_end_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
|
|
4615
|
-
fps: videoTrackFps,
|
|
4616
|
-
base_cadence_hz: thumbnailInterval > 0 ? 1 / thumbnailInterval : void 0
|
|
4617
|
-
}),
|
|
4618
|
-
{
|
|
4619
|
-
width: thumbnailWidth,
|
|
4620
|
-
shouldSign: policy === "signed",
|
|
4621
|
-
credentials
|
|
4622
|
-
}
|
|
4623
|
-
)
|
|
4624
|
-
);
|
|
4772
|
+
}
|
|
4773
|
+
) : await getThumbnailUrls(playbackId, duration, {
|
|
4774
|
+
interval: thumbnailInterval,
|
|
4775
|
+
width: thumbnailWidth,
|
|
4776
|
+
shouldSign: policy === "signed",
|
|
4777
|
+
credentials
|
|
4778
|
+
});
|
|
4625
4779
|
thumbnailCount = thumbnailUrls.length;
|
|
4626
4780
|
if (provider === "openai") {
|
|
4627
4781
|
thumbnailScores = await requestOpenAIModeration(
|
|
@@ -4645,20 +4799,40 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
4645
4799
|
}
|
|
4646
4800
|
}
|
|
4647
4801
|
const failed = thumbnailScores.filter((s) => s.error);
|
|
4648
|
-
|
|
4802
|
+
const successful = thumbnailScores.filter((s) => !s.error);
|
|
4803
|
+
if (successful.length === 0) {
|
|
4649
4804
|
const details = failed.map((s) => `${s.url}: ${s.errorMessage || "Unknown error"}`).join("; ");
|
|
4650
4805
|
throw new Error(
|
|
4651
|
-
`Moderation failed for ${
|
|
4806
|
+
`Moderation failed for all ${thumbnailScores.length} sample(s): ${details}`
|
|
4807
|
+
);
|
|
4808
|
+
}
|
|
4809
|
+
if (failed.length > 0) {
|
|
4810
|
+
console.warn(
|
|
4811
|
+
`Moderation had partial failures (${failed.length}/${thumbnailScores.length}); continuing with successful samples.`
|
|
4652
4812
|
);
|
|
4653
4813
|
}
|
|
4654
|
-
const maxSexual = Math.max(...
|
|
4655
|
-
const maxViolence = Math.max(...
|
|
4814
|
+
const maxSexual = Math.max(...successful.map((s) => s.sexual));
|
|
4815
|
+
const maxViolence = Math.max(...successful.map((s) => s.violence));
|
|
4656
4816
|
const finalThresholds = { ...DEFAULT_THRESHOLDS, ...thresholds };
|
|
4817
|
+
const requestedSampleCount = thumbnailScores.length;
|
|
4818
|
+
const successfulSampleCount = successful.length;
|
|
4819
|
+
const failedSampleCount = failed.length;
|
|
4820
|
+
const sampleCoverage = successfulSampleCount / requestedSampleCount;
|
|
4821
|
+
const hasEnoughSuccessfulSamples = mode === "transcript" || successfulSampleCount >= MIN_SUCCESSFUL_THUMBNAILS_FOR_CONFIDENT_THRESHOLDING;
|
|
4822
|
+
const isLowConfidence = sampleCoverage < MIN_SAMPLE_COVERAGE_FOR_CONFIDENT_THRESHOLDING || !hasEnoughSuccessfulSamples;
|
|
4657
4823
|
return {
|
|
4658
4824
|
assetId,
|
|
4659
4825
|
mode,
|
|
4660
4826
|
isAudioOnly,
|
|
4661
4827
|
thumbnailScores,
|
|
4828
|
+
coverage: {
|
|
4829
|
+
requestedSampleCount,
|
|
4830
|
+
successfulSampleCount,
|
|
4831
|
+
failedSampleCount,
|
|
4832
|
+
sampleCoverage,
|
|
4833
|
+
isPartial: failedSampleCount > 0,
|
|
4834
|
+
isLowConfidence
|
|
4835
|
+
},
|
|
4662
4836
|
usage: {
|
|
4663
4837
|
metadata: {
|
|
4664
4838
|
assetDurationSeconds: duration,
|
|
@@ -4886,7 +5060,7 @@ var SYSTEM_PROMPT5 = dedent6`
|
|
|
4886
5060
|
|
|
4887
5061
|
Write as if describing reality, not describing a recording of reality.
|
|
4888
5062
|
</language_guidelines>`;
|
|
4889
|
-
var
|
|
5063
|
+
var AUDIO_ONLY_SYSTEM_PROMPT3 = dedent6`
|
|
4890
5064
|
<role>
|
|
4891
5065
|
You are an audio content analyst specializing in transcript analysis and metadata generation.
|
|
4892
5066
|
</role>
|
|
@@ -5063,6 +5237,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
5063
5237
|
const {
|
|
5064
5238
|
provider = "openai",
|
|
5065
5239
|
model,
|
|
5240
|
+
languageCode,
|
|
5066
5241
|
tone = "neutral",
|
|
5067
5242
|
includeTranscript = true,
|
|
5068
5243
|
cleanTranscript = true,
|
|
@@ -5101,6 +5276,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
5101
5276
|
);
|
|
5102
5277
|
}
|
|
5103
5278
|
const transcriptResult = includeTranscript ? await fetchTranscriptForAsset(assetData, playbackId, {
|
|
5279
|
+
languageCode,
|
|
5104
5280
|
cleanTranscript,
|
|
5105
5281
|
shouldSign: policy === "signed",
|
|
5106
5282
|
credentials: workflowCredentials,
|
|
@@ -5122,7 +5298,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
5122
5298
|
});
|
|
5123
5299
|
let analysisResponse;
|
|
5124
5300
|
let imageUrl;
|
|
5125
|
-
const systemPrompt = isAudioOnly ?
|
|
5301
|
+
const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT3 : SYSTEM_PROMPT5;
|
|
5126
5302
|
try {
|
|
5127
5303
|
if (isAudioOnly) {
|
|
5128
5304
|
analysisResponse = await analyzeAudioOnly(
|
|
@@ -5419,7 +5595,8 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
5419
5595
|
fromLanguageCode,
|
|
5420
5596
|
numSpeakers = 0,
|
|
5421
5597
|
// 0 = auto-detect
|
|
5422
|
-
|
|
5598
|
+
uploadToS3: uploadToS3Option,
|
|
5599
|
+
uploadToMux: uploadToMuxOption,
|
|
5423
5600
|
storageAdapter,
|
|
5424
5601
|
credentials: providedCredentials
|
|
5425
5602
|
} = options;
|
|
@@ -5428,13 +5605,15 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
5428
5605
|
}
|
|
5429
5606
|
const credentials = providedCredentials;
|
|
5430
5607
|
const effectiveStorageAdapter = storageAdapter;
|
|
5608
|
+
const uploadToMux = uploadToMuxOption !== false;
|
|
5609
|
+
const uploadToS3 = uploadToS3Option || uploadToMux;
|
|
5431
5610
|
const s3Endpoint = options.s3Endpoint ?? env_default.S3_ENDPOINT;
|
|
5432
5611
|
const s3Region = options.s3Region ?? env_default.S3_REGION ?? "auto";
|
|
5433
5612
|
const s3Bucket = options.s3Bucket ?? env_default.S3_BUCKET;
|
|
5434
5613
|
const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
|
|
5435
5614
|
const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
|
|
5436
|
-
if (
|
|
5437
|
-
throw new Error("Storage configuration is required for uploading
|
|
5615
|
+
if (uploadToS3 && (!s3Endpoint || !s3Bucket || !effectiveStorageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
|
|
5616
|
+
throw new Error("Storage configuration is required for uploading. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.");
|
|
5438
5617
|
}
|
|
5439
5618
|
const { asset: initialAsset, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
5440
5619
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(initialAsset);
|
|
@@ -5511,70 +5690,65 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
5511
5690
|
throw new Error(`Dubbing job timed out or failed. Final status: ${dubbingStatus}`);
|
|
5512
5691
|
}
|
|
5513
5692
|
console.warn("\u2705 Dubbing completed successfully!");
|
|
5514
|
-
if (!uploadToMux) {
|
|
5515
|
-
const targetLanguage2 = getLanguageCodePair(toLanguageCode);
|
|
5516
|
-
return {
|
|
5517
|
-
assetId,
|
|
5518
|
-
targetLanguageCode: targetLanguage2.iso639_1,
|
|
5519
|
-
targetLanguage: targetLanguage2,
|
|
5520
|
-
dubbingId
|
|
5521
|
-
};
|
|
5522
|
-
}
|
|
5523
|
-
console.warn("\u{1F4E5} Downloading dubbed audio from ElevenLabs...");
|
|
5524
|
-
let dubbedAudioBuffer;
|
|
5525
|
-
try {
|
|
5526
|
-
const requestedLangCode = toISO639_3(toLanguageCode);
|
|
5527
|
-
let downloadLangCode = targetLanguages.find(
|
|
5528
|
-
(lang) => lang === requestedLangCode
|
|
5529
|
-
) ?? targetLanguages.find(
|
|
5530
|
-
(lang) => lang.toLowerCase() === requestedLangCode.toLowerCase()
|
|
5531
|
-
);
|
|
5532
|
-
if (!downloadLangCode && targetLanguages.length > 0) {
|
|
5533
|
-
downloadLangCode = targetLanguages[0];
|
|
5534
|
-
console.warn(`\u26A0\uFE0F Requested language "${requestedLangCode}" not found in target_languages. Using "${downloadLangCode}" instead.`);
|
|
5535
|
-
}
|
|
5536
|
-
if (!downloadLangCode) {
|
|
5537
|
-
downloadLangCode = requestedLangCode;
|
|
5538
|
-
console.warn(`\u26A0\uFE0F No target_languages available from ElevenLabs status. Using requested language code: ${requestedLangCode}`);
|
|
5539
|
-
}
|
|
5540
|
-
dubbedAudioBuffer = await downloadDubbedAudioFromElevenLabs({
|
|
5541
|
-
dubbingId,
|
|
5542
|
-
languageCode: downloadLangCode,
|
|
5543
|
-
credentials
|
|
5544
|
-
});
|
|
5545
|
-
console.warn("\u2705 Dubbed audio downloaded successfully!");
|
|
5546
|
-
} catch (error) {
|
|
5547
|
-
throw new Error(`Failed to download dubbed audio: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
5548
|
-
}
|
|
5549
|
-
console.warn("\u{1F4E4} Uploading dubbed audio to S3-compatible storage...");
|
|
5550
5693
|
let presignedUrl;
|
|
5551
|
-
try {
|
|
5552
|
-
presignedUrl = await uploadDubbedAudioToS3({
|
|
5553
|
-
dubbedAudioBuffer,
|
|
5554
|
-
assetId,
|
|
5555
|
-
toLanguageCode,
|
|
5556
|
-
s3Endpoint,
|
|
5557
|
-
s3Region,
|
|
5558
|
-
s3Bucket,
|
|
5559
|
-
storageAdapter: effectiveStorageAdapter,
|
|
5560
|
-
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
5561
|
-
});
|
|
5562
|
-
} catch (error) {
|
|
5563
|
-
throw new Error(`Failed to upload audio to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
5564
|
-
}
|
|
5565
|
-
console.warn("\u{1F4F9} Adding dubbed audio track to Mux asset...");
|
|
5566
5694
|
let uploadedTrackId;
|
|
5567
|
-
|
|
5568
|
-
|
|
5569
|
-
|
|
5570
|
-
|
|
5571
|
-
|
|
5572
|
-
|
|
5573
|
-
|
|
5574
|
-
|
|
5575
|
-
|
|
5576
|
-
|
|
5577
|
-
|
|
5695
|
+
if (uploadToS3) {
|
|
5696
|
+
console.warn("\u{1F4E5} Downloading dubbed audio from ElevenLabs...");
|
|
5697
|
+
let dubbedAudioBuffer;
|
|
5698
|
+
try {
|
|
5699
|
+
const requestedLangCode = toISO639_3(toLanguageCode);
|
|
5700
|
+
let downloadLangCode = targetLanguages.find(
|
|
5701
|
+
(lang) => lang === requestedLangCode
|
|
5702
|
+
) ?? targetLanguages.find(
|
|
5703
|
+
(lang) => lang.toLowerCase() === requestedLangCode.toLowerCase()
|
|
5704
|
+
);
|
|
5705
|
+
if (!downloadLangCode && targetLanguages.length > 0) {
|
|
5706
|
+
downloadLangCode = targetLanguages[0];
|
|
5707
|
+
console.warn(`\u26A0\uFE0F Requested language "${requestedLangCode}" not found in target_languages. Using "${downloadLangCode}" instead.`);
|
|
5708
|
+
}
|
|
5709
|
+
if (!downloadLangCode) {
|
|
5710
|
+
downloadLangCode = requestedLangCode;
|
|
5711
|
+
console.warn(`\u26A0\uFE0F No target_languages available from ElevenLabs status. Using requested language code: ${requestedLangCode}`);
|
|
5712
|
+
}
|
|
5713
|
+
dubbedAudioBuffer = await downloadDubbedAudioFromElevenLabs({
|
|
5714
|
+
dubbingId,
|
|
5715
|
+
languageCode: downloadLangCode,
|
|
5716
|
+
credentials
|
|
5717
|
+
});
|
|
5718
|
+
console.warn("\u2705 Dubbed audio downloaded successfully!");
|
|
5719
|
+
} catch (error) {
|
|
5720
|
+
throw new Error(`Failed to download dubbed audio: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
5721
|
+
}
|
|
5722
|
+
console.warn("\u{1F4E4} Uploading dubbed audio to S3-compatible storage...");
|
|
5723
|
+
try {
|
|
5724
|
+
presignedUrl = await uploadDubbedAudioToS3({
|
|
5725
|
+
dubbedAudioBuffer,
|
|
5726
|
+
assetId,
|
|
5727
|
+
toLanguageCode,
|
|
5728
|
+
s3Endpoint,
|
|
5729
|
+
s3Region,
|
|
5730
|
+
s3Bucket,
|
|
5731
|
+
storageAdapter: effectiveStorageAdapter,
|
|
5732
|
+
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
5733
|
+
});
|
|
5734
|
+
} catch (error) {
|
|
5735
|
+
throw new Error(`Failed to upload audio to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
5736
|
+
}
|
|
5737
|
+
if (uploadToMux) {
|
|
5738
|
+
console.warn("\u{1F4F9} Adding dubbed audio track to Mux asset...");
|
|
5739
|
+
const muxLangCode = toISO639_1(toLanguageCode);
|
|
5740
|
+
try {
|
|
5741
|
+
uploadedTrackId = await createAudioTrackOnMux(assetId, muxLangCode, presignedUrl, credentials);
|
|
5742
|
+
const languageName = new Intl.DisplayNames(["en"], { type: "language" }).of(muxLangCode) || muxLangCode.toUpperCase();
|
|
5743
|
+
const trackName = `${languageName} (auto-dubbed)`;
|
|
5744
|
+
console.warn(`\u2705 Track added to Mux asset with ID: ${uploadedTrackId}`);
|
|
5745
|
+
console.warn(`\u{1F4CB} Track name: "${trackName}"`);
|
|
5746
|
+
} catch (error) {
|
|
5747
|
+
console.warn(`\u26A0\uFE0F Failed to add audio track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
5748
|
+
console.warn("\u{1F517} You can manually add the track using this presigned URL:");
|
|
5749
|
+
console.warn(presignedUrl);
|
|
5750
|
+
}
|
|
5751
|
+
}
|
|
5578
5752
|
}
|
|
5579
5753
|
const targetLanguage = getLanguageCodePair(toLanguageCode);
|
|
5580
5754
|
return {
|
|
@@ -6016,7 +6190,7 @@ async function uploadVttToS3({
|
|
|
6016
6190
|
expiresInSeconds: s3SignedUrlExpirySeconds ?? 86400
|
|
6017
6191
|
}, storageAdapter);
|
|
6018
6192
|
}
|
|
6019
|
-
async function translateCaptions(assetId,
|
|
6193
|
+
async function translateCaptions(assetId, trackId, toLanguageCode, options) {
|
|
6020
6194
|
"use workflow";
|
|
6021
6195
|
const {
|
|
6022
6196
|
provider = "openai",
|
|
@@ -6024,6 +6198,7 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
|
|
|
6024
6198
|
s3Endpoint: providedS3Endpoint,
|
|
6025
6199
|
s3Region: providedS3Region,
|
|
6026
6200
|
s3Bucket: providedS3Bucket,
|
|
6201
|
+
uploadToS3: uploadToS3Option,
|
|
6027
6202
|
uploadToMux: uploadToMuxOption,
|
|
6028
6203
|
storageAdapter,
|
|
6029
6204
|
credentials: providedCredentials,
|
|
@@ -6037,17 +6212,17 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
|
|
|
6037
6212
|
const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
|
|
6038
6213
|
const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
|
|
6039
6214
|
const uploadToMux = uploadToMuxOption !== false;
|
|
6215
|
+
const uploadToS3 = uploadToS3Option || uploadToMux;
|
|
6040
6216
|
const modelConfig = resolveLanguageModelConfig({
|
|
6041
6217
|
...options,
|
|
6042
6218
|
model,
|
|
6043
6219
|
provider
|
|
6044
6220
|
});
|
|
6045
|
-
if (
|
|
6046
|
-
throw new Error("Storage configuration is required for uploading
|
|
6221
|
+
if (uploadToS3 && (!s3Endpoint || !s3Bucket || !effectiveStorageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
|
|
6222
|
+
throw new Error("Storage configuration is required for uploading. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.");
|
|
6047
6223
|
}
|
|
6048
6224
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
6049
6225
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
6050
|
-
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
6051
6226
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
6052
6227
|
if (policy === "signed" && !signingContext) {
|
|
6053
6228
|
throw new Error(
|
|
@@ -6055,30 +6230,20 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
|
|
|
6055
6230
|
);
|
|
6056
6231
|
}
|
|
6057
6232
|
const readyTextTracks = getReadyTextTracks(assetData);
|
|
6058
|
-
|
|
6059
|
-
throw new Error("No ready text tracks found for this asset");
|
|
6060
|
-
}
|
|
6061
|
-
let sourceTextTrack = readyTextTracks.find(
|
|
6062
|
-
(track) => track.text_type === "subtitles" && track.language_code === fromLanguageCode
|
|
6063
|
-
);
|
|
6064
|
-
if (!sourceTextTrack && isAudioOnly && readyTextTracks.length === 1) {
|
|
6065
|
-
sourceTextTrack = readyTextTracks[0];
|
|
6066
|
-
}
|
|
6233
|
+
const sourceTextTrack = readyTextTracks.find((t) => t.id === trackId);
|
|
6067
6234
|
if (!sourceTextTrack) {
|
|
6068
|
-
const
|
|
6069
|
-
if (isAudioOnly) {
|
|
6070
|
-
throw new Error(
|
|
6071
|
-
`No transcript track found with language code '${fromLanguageCode}' for this asset. Audio-only assets require a transcript. Available languages: ${availableLanguages || "none"}`
|
|
6072
|
-
);
|
|
6073
|
-
}
|
|
6235
|
+
const availableTrackIds = readyTextTracks.map((t) => t.id).filter(Boolean).join(", ");
|
|
6074
6236
|
throw new Error(
|
|
6075
|
-
`
|
|
6237
|
+
`Track '${trackId}' not found or not ready on asset '${assetId}'. Available track IDs: ${availableTrackIds || "none"}`
|
|
6076
6238
|
);
|
|
6077
6239
|
}
|
|
6078
|
-
|
|
6079
|
-
|
|
6240
|
+
const fromLanguageCode = sourceTextTrack.language_code;
|
|
6241
|
+
if (!fromLanguageCode) {
|
|
6242
|
+
throw new Error(
|
|
6243
|
+
`Track '${trackId}' is missing language metadata (language_code). Cannot determine source language for translation.`
|
|
6244
|
+
);
|
|
6080
6245
|
}
|
|
6081
|
-
const vttUrl = await buildTranscriptUrl(playbackId,
|
|
6246
|
+
const vttUrl = await buildTranscriptUrl(playbackId, trackId, policy === "signed", credentials);
|
|
6082
6247
|
let vttContent;
|
|
6083
6248
|
try {
|
|
6084
6249
|
vttContent = await fetchVttFromMux(vttUrl);
|
|
@@ -6111,50 +6276,43 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
|
|
|
6111
6276
|
} : void 0;
|
|
6112
6277
|
const sourceLanguage = getLanguageCodePair(fromLanguageCode);
|
|
6113
6278
|
const targetLanguage = getLanguageCodePair(toLanguageCode);
|
|
6114
|
-
if (!uploadToMux) {
|
|
6115
|
-
return {
|
|
6116
|
-
assetId,
|
|
6117
|
-
sourceLanguageCode: fromLanguageCode,
|
|
6118
|
-
targetLanguageCode: toLanguageCode,
|
|
6119
|
-
sourceLanguage,
|
|
6120
|
-
targetLanguage,
|
|
6121
|
-
originalVtt: vttContent,
|
|
6122
|
-
translatedVtt,
|
|
6123
|
-
usage: usageWithMetadata
|
|
6124
|
-
};
|
|
6125
|
-
}
|
|
6126
6279
|
let presignedUrl;
|
|
6127
|
-
try {
|
|
6128
|
-
presignedUrl = await uploadVttToS3({
|
|
6129
|
-
translatedVtt,
|
|
6130
|
-
assetId,
|
|
6131
|
-
fromLanguageCode,
|
|
6132
|
-
toLanguageCode,
|
|
6133
|
-
s3Endpoint,
|
|
6134
|
-
s3Region,
|
|
6135
|
-
s3Bucket,
|
|
6136
|
-
storageAdapter: effectiveStorageAdapter,
|
|
6137
|
-
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
6138
|
-
});
|
|
6139
|
-
} catch (error) {
|
|
6140
|
-
throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
6141
|
-
}
|
|
6142
6280
|
let uploadedTrackId;
|
|
6143
|
-
|
|
6144
|
-
|
|
6145
|
-
|
|
6146
|
-
|
|
6147
|
-
|
|
6148
|
-
|
|
6149
|
-
|
|
6150
|
-
|
|
6151
|
-
|
|
6152
|
-
|
|
6153
|
-
|
|
6154
|
-
|
|
6281
|
+
if (uploadToS3) {
|
|
6282
|
+
try {
|
|
6283
|
+
presignedUrl = await uploadVttToS3({
|
|
6284
|
+
translatedVtt,
|
|
6285
|
+
assetId,
|
|
6286
|
+
fromLanguageCode,
|
|
6287
|
+
toLanguageCode,
|
|
6288
|
+
s3Endpoint,
|
|
6289
|
+
s3Region,
|
|
6290
|
+
s3Bucket,
|
|
6291
|
+
storageAdapter: effectiveStorageAdapter,
|
|
6292
|
+
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
6293
|
+
});
|
|
6294
|
+
} catch (error) {
|
|
6295
|
+
throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
6296
|
+
}
|
|
6297
|
+
if (uploadToMux) {
|
|
6298
|
+
try {
|
|
6299
|
+
const languageName = getLanguageName(toLanguageCode);
|
|
6300
|
+
const trackName = `${languageName} (auto-translated)`;
|
|
6301
|
+
uploadedTrackId = await createTextTrackOnMux(
|
|
6302
|
+
assetId,
|
|
6303
|
+
toLanguageCode,
|
|
6304
|
+
trackName,
|
|
6305
|
+
presignedUrl,
|
|
6306
|
+
credentials
|
|
6307
|
+
);
|
|
6308
|
+
} catch (error) {
|
|
6309
|
+
console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
6310
|
+
}
|
|
6311
|
+
}
|
|
6155
6312
|
}
|
|
6156
6313
|
return {
|
|
6157
6314
|
assetId,
|
|
6315
|
+
trackId,
|
|
6158
6316
|
sourceLanguageCode: fromLanguageCode,
|
|
6159
6317
|
targetLanguageCode: toLanguageCode,
|
|
6160
6318
|
sourceLanguage,
|