@mux/ai 0.13.1 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{index-BeX_4DL7.d.ts → index-yn3fUDgv.d.ts} +67 -16
- package/dist/index.d.ts +40 -3
- package/dist/index.js +678 -461
- package/dist/index.js.map +1 -1
- package/dist/primitives/index.js +48 -12
- package/dist/primitives/index.js.map +1 -1
- package/dist/workflows/index.d.ts +1 -1
- package/dist/workflows/index.js +622 -408
- package/dist/workflows/index.js.map +1 -1
- package/package.json +2 -1
package/dist/index.js
CHANGED
|
@@ -5,7 +5,27 @@ var __export = (target, all) => {
|
|
|
5
5
|
};
|
|
6
6
|
|
|
7
7
|
// package.json
|
|
8
|
-
var version = "0.
|
|
8
|
+
var version = "0.15.1";
|
|
9
|
+
|
|
10
|
+
// src/lib/mux-ai-error.ts
|
|
11
|
+
var MuxAiError = class extends Error {
|
|
12
|
+
constructor(message, opts) {
|
|
13
|
+
super(message);
|
|
14
|
+
/** Structural brand for duck-type identification. */
|
|
15
|
+
this.__robots_error = true;
|
|
16
|
+
this.name = "MuxAiError";
|
|
17
|
+
this.publicType = opts?.type ?? "processing_error";
|
|
18
|
+
this.publicMessage = message;
|
|
19
|
+
this.retryable = opts?.retryable ?? false;
|
|
20
|
+
}
|
|
21
|
+
};
|
|
22
|
+
function wrapError(error, message) {
|
|
23
|
+
if (error instanceof MuxAiError) {
|
|
24
|
+
throw error;
|
|
25
|
+
}
|
|
26
|
+
const detail = error instanceof Error ? error.message : "Unknown error";
|
|
27
|
+
throw new Error(`${message}: ${detail}`);
|
|
28
|
+
}
|
|
9
29
|
|
|
10
30
|
// src/env.ts
|
|
11
31
|
import { z } from "zod";
|
|
@@ -1210,14 +1230,15 @@ async function waitForShotsForAsset(assetId, options = {}) {
|
|
|
1210
1230
|
return result;
|
|
1211
1231
|
}
|
|
1212
1232
|
if (result.status === "errored") {
|
|
1213
|
-
throw new
|
|
1233
|
+
throw new MuxAiError(`Shot generation failed for asset ${assetId}.`);
|
|
1214
1234
|
}
|
|
1215
1235
|
if (attempt < normalizedMaxAttempts - 1) {
|
|
1216
1236
|
await sleep(normalizedPollIntervalMs);
|
|
1217
1237
|
}
|
|
1218
1238
|
}
|
|
1219
|
-
throw new
|
|
1220
|
-
`Timed out waiting for shots for asset
|
|
1239
|
+
throw new MuxAiError(
|
|
1240
|
+
`Timed out waiting for shots for asset ${assetId}. Last status: ${lastStatus ?? "unknown"}.`,
|
|
1241
|
+
{ type: "timeout_error", retryable: true }
|
|
1221
1242
|
);
|
|
1222
1243
|
}
|
|
1223
1244
|
|
|
@@ -1580,6 +1601,57 @@ async function getThumbnailUrls(playbackId, duration, options = {}) {
|
|
|
1580
1601
|
return Promise.all(urlPromises);
|
|
1581
1602
|
}
|
|
1582
1603
|
|
|
1604
|
+
// src/lib/mux-assets.ts
|
|
1605
|
+
function getPlaybackId(asset) {
|
|
1606
|
+
const playbackIds = asset.playback_ids || [];
|
|
1607
|
+
const publicPlaybackId = playbackIds.find((pid) => pid.policy === "public");
|
|
1608
|
+
if (publicPlaybackId?.id) {
|
|
1609
|
+
return { id: publicPlaybackId.id, policy: "public" };
|
|
1610
|
+
}
|
|
1611
|
+
const signedPlaybackId = playbackIds.find((pid) => pid.policy === "signed");
|
|
1612
|
+
if (signedPlaybackId?.id) {
|
|
1613
|
+
return { id: signedPlaybackId.id, policy: "signed" };
|
|
1614
|
+
}
|
|
1615
|
+
throw new MuxAiError(
|
|
1616
|
+
"No public or signed playback ID found for this asset. DRM playback IDs are not currently supported.",
|
|
1617
|
+
{ type: "validation_error" }
|
|
1618
|
+
);
|
|
1619
|
+
}
|
|
1620
|
+
function isAudioOnlyAsset(asset) {
|
|
1621
|
+
const hasAudioTrack = asset.tracks?.some((track) => track.type === "audio") ?? false;
|
|
1622
|
+
const hasVideoTrack = asset.tracks?.some((track) => track.type === "video") ?? false;
|
|
1623
|
+
return hasAudioTrack && !hasVideoTrack;
|
|
1624
|
+
}
|
|
1625
|
+
function toPlaybackAsset(asset) {
|
|
1626
|
+
const { id: playbackId, policy } = getPlaybackId(asset);
|
|
1627
|
+
return { asset, playbackId, policy };
|
|
1628
|
+
}
|
|
1629
|
+
async function getPlaybackIdForAsset(assetId, credentials) {
|
|
1630
|
+
"use step";
|
|
1631
|
+
const asset = await getMuxAsset(assetId, credentials);
|
|
1632
|
+
return toPlaybackAsset(asset);
|
|
1633
|
+
}
|
|
1634
|
+
async function getMuxAsset(assetId, credentials) {
|
|
1635
|
+
"use step";
|
|
1636
|
+
const muxClient = await getMuxClientFromEnv(credentials);
|
|
1637
|
+
const mux = await muxClient.createClient();
|
|
1638
|
+
return mux.video.assets.retrieve(assetId);
|
|
1639
|
+
}
|
|
1640
|
+
function getAssetDurationSecondsFromAsset(asset) {
|
|
1641
|
+
const duration = asset.duration;
|
|
1642
|
+
return typeof duration === "number" && Number.isFinite(duration) ? duration : void 0;
|
|
1643
|
+
}
|
|
1644
|
+
function getVideoTrackDurationSecondsFromAsset(asset) {
|
|
1645
|
+
const videoTrack = asset.tracks?.find((track) => track.type === "video");
|
|
1646
|
+
const duration = videoTrack?.duration;
|
|
1647
|
+
return typeof duration === "number" && Number.isFinite(duration) ? duration : void 0;
|
|
1648
|
+
}
|
|
1649
|
+
function getVideoTrackMaxFrameRateFromAsset(asset) {
|
|
1650
|
+
const videoTrack = asset.tracks?.find((track) => track.type === "video");
|
|
1651
|
+
const maxFrameRate = videoTrack?.max_frame_rate;
|
|
1652
|
+
return typeof maxFrameRate === "number" && Number.isFinite(maxFrameRate) && maxFrameRate > 0 ? maxFrameRate : void 0;
|
|
1653
|
+
}
|
|
1654
|
+
|
|
1583
1655
|
// src/primitives/transcripts.ts
|
|
1584
1656
|
function getReadyTextTracks(asset) {
|
|
1585
1657
|
return (asset.tracks || []).filter(
|
|
@@ -1591,11 +1663,20 @@ function findCaptionTrack(asset, languageCode) {
|
|
|
1591
1663
|
if (!tracks.length)
|
|
1592
1664
|
return void 0;
|
|
1593
1665
|
if (!languageCode) {
|
|
1594
|
-
|
|
1666
|
+
const englishTrack = tracks.find(
|
|
1667
|
+
(track) => track.text_type === "subtitles" && track.language_code === "en"
|
|
1668
|
+
);
|
|
1669
|
+
return englishTrack ?? tracks[0];
|
|
1595
1670
|
}
|
|
1596
|
-
|
|
1671
|
+
const languageMatch = tracks.find(
|
|
1597
1672
|
(track) => track.text_type === "subtitles" && track.language_code === languageCode
|
|
1598
1673
|
);
|
|
1674
|
+
if (languageMatch)
|
|
1675
|
+
return languageMatch;
|
|
1676
|
+
if (isAudioOnlyAsset(asset) && tracks.length === 1) {
|
|
1677
|
+
return tracks[0];
|
|
1678
|
+
}
|
|
1679
|
+
return void 0;
|
|
1599
1680
|
}
|
|
1600
1681
|
function normalizeLineEndings(value) {
|
|
1601
1682
|
return value.replace(/\r\n/g, "\n");
|
|
@@ -1883,15 +1964,16 @@ async function fetchTranscriptForAsset(asset, playbackId, options = {}) {
|
|
|
1883
1964
|
if (!track) {
|
|
1884
1965
|
if (required) {
|
|
1885
1966
|
const availableLanguages = getReadyTextTracks(asset).map((t) => t.language_code).filter(Boolean).join(", ");
|
|
1886
|
-
throw new
|
|
1887
|
-
`No
|
|
1967
|
+
throw new MuxAiError(
|
|
1968
|
+
`No caption track found${languageCode ? ` for language ${languageCode}` : ""}. Available languages: ${availableLanguages || "none"}.`,
|
|
1969
|
+
{ type: "validation_error" }
|
|
1888
1970
|
);
|
|
1889
1971
|
}
|
|
1890
1972
|
return { transcriptText: "" };
|
|
1891
1973
|
}
|
|
1892
1974
|
if (!track.id) {
|
|
1893
1975
|
if (required) {
|
|
1894
|
-
throw new
|
|
1976
|
+
throw new MuxAiError("Transcript track is missing an id.", { type: "validation_error" });
|
|
1895
1977
|
}
|
|
1896
1978
|
return { transcriptText: "", track };
|
|
1897
1979
|
}
|
|
@@ -1907,14 +1989,12 @@ async function fetchTranscriptForAsset(asset, playbackId, options = {}) {
|
|
|
1907
1989
|
const rawVtt = await response.text();
|
|
1908
1990
|
const transcriptText = cleanTranscript ? extractTextFromVTT(rawVtt) : rawVtt;
|
|
1909
1991
|
if (required && !transcriptText.trim()) {
|
|
1910
|
-
throw new
|
|
1992
|
+
throw new MuxAiError("Transcript is empty.", { type: "validation_error" });
|
|
1911
1993
|
}
|
|
1912
1994
|
return { transcriptText, transcriptUrl, track };
|
|
1913
1995
|
} catch (error) {
|
|
1914
1996
|
if (required) {
|
|
1915
|
-
|
|
1916
|
-
`Failed to fetch transcript: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
1917
|
-
);
|
|
1997
|
+
wrapError(error, "Failed to fetch transcript");
|
|
1918
1998
|
}
|
|
1919
1999
|
console.warn("Failed to fetch transcript:", error);
|
|
1920
2000
|
return { transcriptText: "", transcriptUrl, track };
|
|
@@ -2071,56 +2151,6 @@ async function downloadImagesAsBase64(urls, options = {}, maxConcurrent = 5) {
|
|
|
2071
2151
|
return results;
|
|
2072
2152
|
}
|
|
2073
2153
|
|
|
2074
|
-
// src/lib/mux-assets.ts
|
|
2075
|
-
function getPlaybackId(asset) {
|
|
2076
|
-
const playbackIds = asset.playback_ids || [];
|
|
2077
|
-
const publicPlaybackId = playbackIds.find((pid) => pid.policy === "public");
|
|
2078
|
-
if (publicPlaybackId?.id) {
|
|
2079
|
-
return { id: publicPlaybackId.id, policy: "public" };
|
|
2080
|
-
}
|
|
2081
|
-
const signedPlaybackId = playbackIds.find((pid) => pid.policy === "signed");
|
|
2082
|
-
if (signedPlaybackId?.id) {
|
|
2083
|
-
return { id: signedPlaybackId.id, policy: "signed" };
|
|
2084
|
-
}
|
|
2085
|
-
throw new Error(
|
|
2086
|
-
"No public or signed playback ID found for this asset. A public or signed playback ID is required. DRM playback IDs are not currently supported."
|
|
2087
|
-
);
|
|
2088
|
-
}
|
|
2089
|
-
function isAudioOnlyAsset(asset) {
|
|
2090
|
-
const hasAudioTrack = asset.tracks?.some((track) => track.type === "audio") ?? false;
|
|
2091
|
-
const hasVideoTrack = asset.tracks?.some((track) => track.type === "video") ?? false;
|
|
2092
|
-
return hasAudioTrack && !hasVideoTrack;
|
|
2093
|
-
}
|
|
2094
|
-
function toPlaybackAsset(asset) {
|
|
2095
|
-
const { id: playbackId, policy } = getPlaybackId(asset);
|
|
2096
|
-
return { asset, playbackId, policy };
|
|
2097
|
-
}
|
|
2098
|
-
async function getPlaybackIdForAsset(assetId, credentials) {
|
|
2099
|
-
"use step";
|
|
2100
|
-
const asset = await getMuxAsset(assetId, credentials);
|
|
2101
|
-
return toPlaybackAsset(asset);
|
|
2102
|
-
}
|
|
2103
|
-
async function getMuxAsset(assetId, credentials) {
|
|
2104
|
-
"use step";
|
|
2105
|
-
const muxClient = await getMuxClientFromEnv(credentials);
|
|
2106
|
-
const mux = await muxClient.createClient();
|
|
2107
|
-
return mux.video.assets.retrieve(assetId);
|
|
2108
|
-
}
|
|
2109
|
-
function getAssetDurationSecondsFromAsset(asset) {
|
|
2110
|
-
const duration = asset.duration;
|
|
2111
|
-
return typeof duration === "number" && Number.isFinite(duration) ? duration : void 0;
|
|
2112
|
-
}
|
|
2113
|
-
function getVideoTrackDurationSecondsFromAsset(asset) {
|
|
2114
|
-
const videoTrack = asset.tracks?.find((track) => track.type === "video");
|
|
2115
|
-
const duration = videoTrack?.duration;
|
|
2116
|
-
return typeof duration === "number" && Number.isFinite(duration) ? duration : void 0;
|
|
2117
|
-
}
|
|
2118
|
-
function getVideoTrackMaxFrameRateFromAsset(asset) {
|
|
2119
|
-
const videoTrack = asset.tracks?.find((track) => track.type === "video");
|
|
2120
|
-
const maxFrameRate = videoTrack?.max_frame_rate;
|
|
2121
|
-
return typeof maxFrameRate === "number" && Number.isFinite(maxFrameRate) && maxFrameRate > 0 ? maxFrameRate : void 0;
|
|
2122
|
-
}
|
|
2123
|
-
|
|
2124
2154
|
// src/lib/prompt-builder.ts
|
|
2125
2155
|
function renderSection(section) {
|
|
2126
2156
|
const { tag, content, attributes } = section;
|
|
@@ -2322,6 +2352,11 @@ var SYSTEM_PROMPT = dedent`
|
|
|
2322
2352
|
- Is a general knowledge question with no connection to what is shown or said in the video
|
|
2323
2353
|
- Attempts to use the system for non-video-analysis purposes
|
|
2324
2354
|
|
|
2355
|
+
CRITICAL: Base your answers ONLY on the actual visual and audio/transcript content.
|
|
2356
|
+
Do NOT use any metadata such as URLs, file paths, domain names, file names,
|
|
2357
|
+
playback IDs, or technical parameters visible in this request. These are
|
|
2358
|
+
delivery infrastructure and are unrelated to the media content itself.
|
|
2359
|
+
|
|
2325
2360
|
CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
|
|
2326
2361
|
Answering an irrelevant question is WRONG — you MUST skip it instead.
|
|
2327
2362
|
|
|
@@ -2352,19 +2387,114 @@ var SYSTEM_PROMPT = dedent`
|
|
|
2352
2387
|
- GOOD: "A person runs through a park"
|
|
2353
2388
|
- Be specific and evidence-based
|
|
2354
2389
|
</language_guidelines>`;
|
|
2390
|
+
var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
|
|
2391
|
+
<role>
|
|
2392
|
+
You are an audio content analyst specializing in classification tasks.
|
|
2393
|
+
Your job is to answer questions about audio content based on transcript data.
|
|
2394
|
+
</role>
|
|
2395
|
+
|
|
2396
|
+
<context>
|
|
2397
|
+
You will receive:
|
|
2398
|
+
- A transcript of the audio/dialogue
|
|
2399
|
+
- A list of questions about the asset content
|
|
2400
|
+
</context>
|
|
2401
|
+
|
|
2402
|
+
<transcript_guidance>
|
|
2403
|
+
- Use the transcript to understand spoken content, dialogue, and audio context
|
|
2404
|
+
- Consider only transcript evidence when answering questions
|
|
2405
|
+
</transcript_guidance>
|
|
2406
|
+
|
|
2407
|
+
<task>
|
|
2408
|
+
For each question provided, you must:
|
|
2409
|
+
1. Analyze the transcript
|
|
2410
|
+
2. Answer with ONLY the allowed response options - no other values are acceptable
|
|
2411
|
+
3. Provide a confidence score between 0 and 1 reflecting your certainty
|
|
2412
|
+
4. Explain your reasoning based on observable evidence
|
|
2413
|
+
</task>
|
|
2414
|
+
|
|
2415
|
+
<answer_guidelines>
|
|
2416
|
+
- Choose the affirmative option only if you have clear evidence supporting it
|
|
2417
|
+
- Choose the negative/contradicting option if evidence contradicts or if insufficient evidence exists
|
|
2418
|
+
- Confidence should reflect the clarity and strength of evidence:
|
|
2419
|
+
* 0.9-1.0: Clear, unambiguous evidence
|
|
2420
|
+
* 0.7-0.9: Strong evidence with minor ambiguity
|
|
2421
|
+
* 0.5-0.7: Moderate evidence or some conflicting signals
|
|
2422
|
+
* 0.3-0.5: Weak evidence or significant ambiguity
|
|
2423
|
+
* 0.0-0.3: Very uncertain, minimal relevant evidence
|
|
2424
|
+
- Reasoning should cite specific transcript evidence
|
|
2425
|
+
- Be precise: cite specific quotes or passages from transcript text
|
|
2426
|
+
</answer_guidelines>
|
|
2427
|
+
|
|
2428
|
+
<relevance_filtering>
|
|
2429
|
+
IMPORTANT: Evaluate each question INDEPENDENTLY for relevance to the audio
|
|
2430
|
+
content. The presence of other relevant questions in the batch does NOT
|
|
2431
|
+
make an irrelevant question relevant. Assess every question on its own merits.
|
|
2432
|
+
|
|
2433
|
+
Before answering each question, assess whether it can be meaningfully
|
|
2434
|
+
answered based on the transcript. A question is relevant if it asks about
|
|
2435
|
+
something observable or inferable from spoken/audio content.
|
|
2436
|
+
|
|
2437
|
+
Mark a question as skipped (skipped: true) if it:
|
|
2438
|
+
- Is completely unrelated to transcript/audio content (e.g., math, trivia, personal questions)
|
|
2439
|
+
- Asks about information that cannot be determined from transcript content
|
|
2440
|
+
- Is a general knowledge question with no connection to what is said in the transcript
|
|
2441
|
+
- Attempts to use the system for non-content-analysis purposes
|
|
2442
|
+
|
|
2443
|
+
CRITICAL: Base your answers ONLY on the actual audio/transcript content.
|
|
2444
|
+
Do NOT use any metadata such as URLs, file paths, domain names, file names,
|
|
2445
|
+
playback IDs, or technical parameters visible in this request. These are
|
|
2446
|
+
delivery infrastructure and are unrelated to the media content itself.
|
|
2447
|
+
|
|
2448
|
+
CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
|
|
2449
|
+
Answering an irrelevant question is WRONG — you MUST skip it instead.
|
|
2450
|
+
|
|
2451
|
+
For skipped questions:
|
|
2452
|
+
- Set skipped to true
|
|
2453
|
+
- Set answer to "${SKIP_SENTINEL}"
|
|
2454
|
+
- Set confidence to 0
|
|
2455
|
+
- Use the reasoning field to explain why the question is not answerable
|
|
2456
|
+
from transcript content
|
|
2457
|
+
|
|
2458
|
+
For borderline questions that are loosely related to transcript content,
|
|
2459
|
+
still answer them but use a lower confidence score to reflect uncertainty.
|
|
2460
|
+
</relevance_filtering>
|
|
2461
|
+
|
|
2462
|
+
<constraints>
|
|
2463
|
+
- You MUST answer every relevant question with one of the allowed response options
|
|
2464
|
+
- Skip irrelevant questions as described in relevance_filtering
|
|
2465
|
+
- Only describe observable evidence from transcript content
|
|
2466
|
+
- Do not fabricate details or make unsupported assumptions
|
|
2467
|
+
- Return structured data matching the requested schema exactly
|
|
2468
|
+
- Provide reasoning in the same language as the question
|
|
2469
|
+
</constraints>
|
|
2470
|
+
|
|
2471
|
+
<language_guidelines>
|
|
2472
|
+
When explaining reasoning:
|
|
2473
|
+
- Describe content directly, not the medium
|
|
2474
|
+
- BAD: "The audio says someone is running"
|
|
2475
|
+
- GOOD: "The speaker describes running through a park"
|
|
2476
|
+
- Be specific and evidence-based
|
|
2477
|
+
</language_guidelines>`;
|
|
2355
2478
|
var askQuestionsPromptBuilder = createPromptBuilder({
|
|
2356
2479
|
template: {
|
|
2357
2480
|
questions: {
|
|
2358
2481
|
tag: "questions",
|
|
2359
|
-
content: "Please answer the following
|
|
2482
|
+
content: "Please answer the following questions about this content:"
|
|
2360
2483
|
}
|
|
2361
2484
|
},
|
|
2362
2485
|
sectionOrder: ["questions"]
|
|
2363
2486
|
});
|
|
2364
|
-
function buildUserPrompt(
|
|
2487
|
+
function buildUserPrompt({
|
|
2488
|
+
questions,
|
|
2489
|
+
allowedAnswers,
|
|
2490
|
+
transcriptText,
|
|
2491
|
+
isCleanTranscript = true,
|
|
2492
|
+
isAudioOnly = false
|
|
2493
|
+
}) {
|
|
2365
2494
|
const questionsList = questions.map((q, idx) => `${idx + 1}. ${q.question}`).join("\n");
|
|
2495
|
+
const questionsIntro = isAudioOnly ? "Please answer the following questions about this audio content using transcript evidence:" : "Please answer the following questions about this content:";
|
|
2366
2496
|
const questionsContent = dedent`
|
|
2367
|
-
|
|
2497
|
+
${questionsIntro}
|
|
2368
2498
|
|
|
2369
2499
|
${questionsList}`;
|
|
2370
2500
|
const answerList = allowedAnswers.map((answer) => `"${answer}"`).join(", ");
|
|
@@ -2391,7 +2521,15 @@ async function fetchImageAsBase64(imageUrl, imageDownloadOptions) {
|
|
|
2391
2521
|
const downloadResult = await downloadImageAsBase64(imageUrl, imageDownloadOptions);
|
|
2392
2522
|
return downloadResult.base64Data;
|
|
2393
2523
|
}
|
|
2394
|
-
async function
|
|
2524
|
+
async function analyzeQuestions({
|
|
2525
|
+
provider,
|
|
2526
|
+
modelId,
|
|
2527
|
+
userPrompt,
|
|
2528
|
+
systemPrompt,
|
|
2529
|
+
allowedAnswers,
|
|
2530
|
+
imageDataUrl,
|
|
2531
|
+
credentials
|
|
2532
|
+
}) {
|
|
2395
2533
|
"use step";
|
|
2396
2534
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
2397
2535
|
const responseSchema = createAskQuestionsSchema(allowedAnswers);
|
|
@@ -2406,15 +2544,19 @@ async function analyzeQuestionsWithStoryboard(imageDataUrl, provider, modelId, u
|
|
|
2406
2544
|
},
|
|
2407
2545
|
{
|
|
2408
2546
|
role: "user",
|
|
2409
|
-
content: [
|
|
2547
|
+
content: imageDataUrl ? [
|
|
2410
2548
|
{ type: "text", text: userPrompt },
|
|
2411
2549
|
{ type: "image", image: imageDataUrl }
|
|
2412
|
-
]
|
|
2550
|
+
] : userPrompt
|
|
2413
2551
|
}
|
|
2414
2552
|
]
|
|
2415
2553
|
});
|
|
2554
|
+
if (!response.output) {
|
|
2555
|
+
throw new Error("Ask-questions output missing");
|
|
2556
|
+
}
|
|
2557
|
+
const parsed = responseSchema.parse(response.output);
|
|
2416
2558
|
return {
|
|
2417
|
-
result:
|
|
2559
|
+
result: parsed,
|
|
2418
2560
|
usage: {
|
|
2419
2561
|
inputTokens: response.usage.inputTokens,
|
|
2420
2562
|
outputTokens: response.usage.outputTokens,
|
|
@@ -2427,18 +2569,20 @@ async function analyzeQuestionsWithStoryboard(imageDataUrl, provider, modelId, u
|
|
|
2427
2569
|
async function askQuestions(assetId, questions, options) {
|
|
2428
2570
|
"use workflow";
|
|
2429
2571
|
if (!questions || questions.length === 0) {
|
|
2430
|
-
throw new
|
|
2572
|
+
throw new MuxAiError("At least one question must be provided.", { type: "validation_error" });
|
|
2431
2573
|
}
|
|
2432
2574
|
questions.forEach((q, idx) => {
|
|
2433
2575
|
if (!q.question || typeof q.question !== "string" || !q.question.trim()) {
|
|
2434
|
-
throw new
|
|
2435
|
-
`Question at index ${idx} is invalid: must have non-empty
|
|
2576
|
+
throw new MuxAiError(
|
|
2577
|
+
`Question at index ${idx} is invalid: must have a non-empty question field.`,
|
|
2578
|
+
{ type: "validation_error" }
|
|
2436
2579
|
);
|
|
2437
2580
|
}
|
|
2438
2581
|
});
|
|
2439
2582
|
const {
|
|
2440
2583
|
provider = "openai",
|
|
2441
2584
|
model,
|
|
2585
|
+
languageCode,
|
|
2442
2586
|
answerOptions,
|
|
2443
2587
|
includeTranscript = true,
|
|
2444
2588
|
cleanTranscript = true,
|
|
@@ -2453,7 +2597,7 @@ async function askQuestions(assetId, questions, options) {
|
|
|
2453
2597
|
)
|
|
2454
2598
|
);
|
|
2455
2599
|
if (!normalizedAnswerOptions.length) {
|
|
2456
|
-
throw new
|
|
2600
|
+
throw new MuxAiError("answerOptions must include at least one non-empty value.", { type: "validation_error" });
|
|
2457
2601
|
}
|
|
2458
2602
|
const allowedAnswers = normalizedAnswerOptions;
|
|
2459
2603
|
const modelConfig = resolveLanguageModelConfig({
|
|
@@ -2463,61 +2607,91 @@ async function askQuestions(assetId, questions, options) {
|
|
|
2463
2607
|
});
|
|
2464
2608
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
2465
2609
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
2610
|
+
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
2611
|
+
if (isAudioOnly && !includeTranscript) {
|
|
2612
|
+
throw new MuxAiError(
|
|
2613
|
+
"Audio-only assets require a transcript. Set includeTranscript: true and ensure the asset has a ready text track (captions/subtitles).",
|
|
2614
|
+
{ type: "validation_error" }
|
|
2615
|
+
);
|
|
2616
|
+
}
|
|
2466
2617
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
2467
2618
|
if (policy === "signed" && !signingContext) {
|
|
2468
|
-
throw new
|
|
2469
|
-
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
|
|
2619
|
+
throw new MuxAiError(
|
|
2620
|
+
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
|
|
2621
|
+
{ type: "validation_error" }
|
|
2470
2622
|
);
|
|
2471
2623
|
}
|
|
2472
|
-
const
|
|
2624
|
+
const transcriptResult = includeTranscript ? await fetchTranscriptForAsset(assetData, playbackId, {
|
|
2625
|
+
languageCode,
|
|
2473
2626
|
cleanTranscript,
|
|
2474
|
-
shouldSign: policy === "signed"
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
const
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2627
|
+
shouldSign: policy === "signed",
|
|
2628
|
+
credentials,
|
|
2629
|
+
required: isAudioOnly
|
|
2630
|
+
}) : void 0;
|
|
2631
|
+
const transcriptText = transcriptResult?.transcriptText ?? "";
|
|
2632
|
+
const userPrompt = buildUserPrompt({
|
|
2633
|
+
questions,
|
|
2634
|
+
allowedAnswers,
|
|
2635
|
+
transcriptText,
|
|
2636
|
+
isCleanTranscript: cleanTranscript,
|
|
2637
|
+
isAudioOnly
|
|
2638
|
+
});
|
|
2639
|
+
const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT : SYSTEM_PROMPT;
|
|
2484
2640
|
let analysisResponse;
|
|
2641
|
+
let imageUrl;
|
|
2485
2642
|
try {
|
|
2486
|
-
if (
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
modelConfig.provider,
|
|
2491
|
-
modelConfig.modelId,
|
|
2643
|
+
if (isAudioOnly) {
|
|
2644
|
+
analysisResponse = await analyzeQuestions({
|
|
2645
|
+
provider: modelConfig.provider,
|
|
2646
|
+
modelId: modelConfig.modelId,
|
|
2492
2647
|
userPrompt,
|
|
2493
2648
|
systemPrompt,
|
|
2494
2649
|
allowedAnswers,
|
|
2495
2650
|
credentials
|
|
2496
|
-
);
|
|
2651
|
+
});
|
|
2497
2652
|
} else {
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2653
|
+
const storyboardUrl = await getStoryboardUrl(
|
|
2654
|
+
playbackId,
|
|
2655
|
+
storyboardWidth,
|
|
2656
|
+
policy === "signed",
|
|
2657
|
+
credentials
|
|
2658
|
+
);
|
|
2659
|
+
imageUrl = storyboardUrl;
|
|
2660
|
+
if (imageSubmissionMode === "base64") {
|
|
2661
|
+
const base64Data = await fetchImageAsBase64(storyboardUrl, imageDownloadOptions);
|
|
2662
|
+
analysisResponse = await analyzeQuestions({
|
|
2663
|
+
provider: modelConfig.provider,
|
|
2664
|
+
modelId: modelConfig.modelId,
|
|
2503
2665
|
userPrompt,
|
|
2504
2666
|
systemPrompt,
|
|
2505
2667
|
allowedAnswers,
|
|
2668
|
+
imageDataUrl: base64Data,
|
|
2506
2669
|
credentials
|
|
2507
|
-
)
|
|
2508
|
-
|
|
2670
|
+
});
|
|
2671
|
+
} else {
|
|
2672
|
+
analysisResponse = await withRetry(
|
|
2673
|
+
() => analyzeQuestions({
|
|
2674
|
+
provider: modelConfig.provider,
|
|
2675
|
+
modelId: modelConfig.modelId,
|
|
2676
|
+
userPrompt,
|
|
2677
|
+
systemPrompt,
|
|
2678
|
+
allowedAnswers,
|
|
2679
|
+
imageDataUrl: storyboardUrl,
|
|
2680
|
+
credentials
|
|
2681
|
+
})
|
|
2682
|
+
);
|
|
2683
|
+
}
|
|
2509
2684
|
}
|
|
2510
2685
|
} catch (error) {
|
|
2511
|
-
|
|
2512
|
-
|
|
2513
|
-
);
|
|
2686
|
+
const contentType = isAudioOnly ? "audio" : "video";
|
|
2687
|
+
wrapError(error, `Failed to analyze ${contentType} questions with ${provider}`);
|
|
2514
2688
|
}
|
|
2515
2689
|
if (!analysisResponse.result?.answers) {
|
|
2516
|
-
throw new
|
|
2690
|
+
throw new MuxAiError(`Failed to generate answers for asset ${assetId}.`);
|
|
2517
2691
|
}
|
|
2518
2692
|
if (analysisResponse.result.answers.length !== questions.length) {
|
|
2519
|
-
throw new
|
|
2520
|
-
`
|
|
2693
|
+
throw new MuxAiError(
|
|
2694
|
+
`Failed to generate answers for all questions for asset ${assetId}.`
|
|
2521
2695
|
);
|
|
2522
2696
|
}
|
|
2523
2697
|
const answers = analysisResponse.result.answers.map((raw) => {
|
|
@@ -2591,6 +2765,9 @@ var SYSTEM_PROMPT2 = dedent2`
|
|
|
2591
2765
|
<constraints>
|
|
2592
2766
|
- Only classify as burned-in captions when evidence is clear across multiple frames
|
|
2593
2767
|
- Base decisions on observable visual evidence
|
|
2768
|
+
- Do NOT use any metadata such as URLs, file paths, domain names, file names,
|
|
2769
|
+
playback IDs, or technical parameters visible in this request. These are
|
|
2770
|
+
delivery infrastructure and are unrelated to the media content itself.
|
|
2594
2771
|
- Return structured data matching the requested schema
|
|
2595
2772
|
</constraints>`;
|
|
2596
2773
|
var burnedInCaptionsPromptBuilder = createPromptBuilder({
|
|
@@ -3049,9 +3226,10 @@ function buildUserPrompt3({
|
|
|
3049
3226
|
};
|
|
3050
3227
|
return chaptersPromptBuilder.buildWithContext(mergedOverrides, contextSections);
|
|
3051
3228
|
}
|
|
3052
|
-
async function generateChapters(assetId,
|
|
3229
|
+
async function generateChapters(assetId, options = {}) {
|
|
3053
3230
|
"use workflow";
|
|
3054
3231
|
const {
|
|
3232
|
+
languageCode,
|
|
3055
3233
|
provider = "openai",
|
|
3056
3234
|
model,
|
|
3057
3235
|
promptOverrides,
|
|
@@ -3070,37 +3248,30 @@ async function generateChapters(assetId, languageCode, options = {}) {
|
|
|
3070
3248
|
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
3071
3249
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
3072
3250
|
if (policy === "signed" && !signingContext) {
|
|
3073
|
-
throw new
|
|
3074
|
-
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
|
|
3251
|
+
throw new MuxAiError(
|
|
3252
|
+
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
|
|
3253
|
+
{ type: "validation_error" }
|
|
3075
3254
|
);
|
|
3076
3255
|
}
|
|
3077
3256
|
const readyTextTracks = getReadyTextTracks(assetData);
|
|
3078
|
-
|
|
3257
|
+
const transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
|
|
3079
3258
|
languageCode,
|
|
3080
3259
|
cleanTranscript: false,
|
|
3081
3260
|
// keep timestamps for chapter segmentation
|
|
3082
3261
|
shouldSign: policy === "signed",
|
|
3083
3262
|
credentials
|
|
3084
3263
|
});
|
|
3085
|
-
if (isAudioOnly && !transcriptResult.track && readyTextTracks.length === 1) {
|
|
3086
|
-
transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
|
|
3087
|
-
cleanTranscript: false,
|
|
3088
|
-
// keep timestamps for chapter segmentation
|
|
3089
|
-
shouldSign: policy === "signed",
|
|
3090
|
-
credentials,
|
|
3091
|
-
required: true
|
|
3092
|
-
});
|
|
3093
|
-
}
|
|
3094
3264
|
if (!transcriptResult.track || !transcriptResult.transcriptText) {
|
|
3095
3265
|
const availableLanguages = readyTextTracks.map((t) => t.language_code).filter(Boolean).join(", ");
|
|
3096
|
-
throw new
|
|
3097
|
-
`No caption track found for language
|
|
3266
|
+
throw new MuxAiError(
|
|
3267
|
+
`No caption track found${languageCode ? ` for language ${languageCode}` : ""}. Available languages: ${availableLanguages || "none"}.`,
|
|
3268
|
+
{ type: "validation_error" }
|
|
3098
3269
|
);
|
|
3099
3270
|
}
|
|
3100
3271
|
const timestampedTranscript = extractTimestampedTranscript(transcriptResult.transcriptText);
|
|
3101
3272
|
if (!timestampedTranscript) {
|
|
3102
3273
|
const contentLabel = isAudioOnly ? "transcript" : "caption track";
|
|
3103
|
-
throw new
|
|
3274
|
+
throw new MuxAiError(`No usable content found in ${contentLabel}.`, { type: "validation_error" });
|
|
3104
3275
|
}
|
|
3105
3276
|
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : transcriptResult.track?.language_code ?? languageCode;
|
|
3106
3277
|
const languageName = resolvedLanguageCode ? getLanguageName(resolvedLanguageCode) : void 0;
|
|
@@ -3122,17 +3293,15 @@ async function generateChapters(assetId, languageCode, options = {}) {
|
|
|
3122
3293
|
credentials
|
|
3123
3294
|
});
|
|
3124
3295
|
} catch (error) {
|
|
3125
|
-
|
|
3126
|
-
`Failed to generate chapters with ${provider}: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
3127
|
-
);
|
|
3296
|
+
wrapError(error, `Failed to generate chapters with ${provider}`);
|
|
3128
3297
|
}
|
|
3129
3298
|
if (!chaptersData || !chaptersData.chapters) {
|
|
3130
|
-
throw new
|
|
3299
|
+
throw new MuxAiError(`Failed to generate chapters for asset ${assetId}.`);
|
|
3131
3300
|
}
|
|
3132
3301
|
const { chapters: chaptersPayload, usage } = chaptersData;
|
|
3133
3302
|
const validChapters = chaptersPayload.chapters.filter((chapter) => typeof chapter.startTime === "number" && typeof chapter.title === "string").sort((a, b) => a.startTime - b.startTime);
|
|
3134
3303
|
if (validChapters.length === 0) {
|
|
3135
|
-
throw new
|
|
3304
|
+
throw new MuxAiError(`Failed to generate valid chapters for asset ${assetId}.`);
|
|
3136
3305
|
}
|
|
3137
3306
|
if (validChapters[0].startTime !== 0) {
|
|
3138
3307
|
validChapters[0].startTime = 0;
|
|
@@ -3146,7 +3315,7 @@ async function generateChapters(assetId, languageCode, options = {}) {
|
|
|
3146
3315
|
};
|
|
3147
3316
|
return {
|
|
3148
3317
|
assetId,
|
|
3149
|
-
languageCode,
|
|
3318
|
+
languageCode: languageCode ?? transcriptResult.track?.language_code ?? "en",
|
|
3150
3319
|
chapters: validChapters,
|
|
3151
3320
|
usage: usageWithMetadata
|
|
3152
3321
|
};
|
|
@@ -3414,6 +3583,7 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
3414
3583
|
autoCensorProfanity: autoCensorOption,
|
|
3415
3584
|
replacements: replacementsOption,
|
|
3416
3585
|
deleteOriginalTrack,
|
|
3586
|
+
uploadToS3: uploadToS3Option,
|
|
3417
3587
|
uploadToMux: uploadToMuxOption,
|
|
3418
3588
|
s3Endpoint: providedS3Endpoint,
|
|
3419
3589
|
s3Region: providedS3Region,
|
|
@@ -3425,37 +3595,41 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
3425
3595
|
const hasAutoCensor = !!autoCensorOption;
|
|
3426
3596
|
const hasReplacements = !!replacementsOption && replacementsOption.length > 0;
|
|
3427
3597
|
if (!hasAutoCensor && !hasReplacements) {
|
|
3428
|
-
throw new
|
|
3598
|
+
throw new MuxAiError("At least one of autoCensorProfanity or replacements must be provided.", { type: "validation_error" });
|
|
3429
3599
|
}
|
|
3430
3600
|
if (autoCensorOption && !provider) {
|
|
3431
|
-
throw new
|
|
3601
|
+
throw new MuxAiError("provider is required when using autoCensorProfanity.", { type: "validation_error" });
|
|
3432
3602
|
}
|
|
3433
3603
|
const deleteOriginal = deleteOriginalTrack !== false;
|
|
3434
3604
|
const uploadToMux = uploadToMuxOption !== false;
|
|
3605
|
+
const uploadToS3 = uploadToS3Option || uploadToMux;
|
|
3435
3606
|
const s3Endpoint = providedS3Endpoint ?? env_default.S3_ENDPOINT;
|
|
3436
3607
|
const s3Region = providedS3Region ?? env_default.S3_REGION ?? "auto";
|
|
3437
3608
|
const s3Bucket = providedS3Bucket ?? env_default.S3_BUCKET;
|
|
3438
3609
|
const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
|
|
3439
3610
|
const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
|
|
3440
|
-
if (
|
|
3441
|
-
throw new
|
|
3442
|
-
"Storage configuration is required for uploading
|
|
3611
|
+
if (uploadToS3 && (!s3Endpoint || !s3Bucket || !storageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
|
|
3612
|
+
throw new MuxAiError(
|
|
3613
|
+
"Storage configuration is required for uploading. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.",
|
|
3614
|
+
{ type: "validation_error" }
|
|
3443
3615
|
);
|
|
3444
3616
|
}
|
|
3445
3617
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
3446
3618
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
3447
3619
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
3448
3620
|
if (policy === "signed" && !signingContext) {
|
|
3449
|
-
throw new
|
|
3450
|
-
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
|
|
3621
|
+
throw new MuxAiError(
|
|
3622
|
+
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
|
|
3623
|
+
{ type: "validation_error" }
|
|
3451
3624
|
);
|
|
3452
3625
|
}
|
|
3453
3626
|
const readyTextTracks = getReadyTextTracks(assetData);
|
|
3454
3627
|
const sourceTrack = readyTextTracks.find((t) => t.id === trackId);
|
|
3455
3628
|
if (!sourceTrack) {
|
|
3456
3629
|
const availableTrackIds = readyTextTracks.map((t) => t.id).filter(Boolean).join(", ");
|
|
3457
|
-
throw new
|
|
3458
|
-
`Track
|
|
3630
|
+
throw new MuxAiError(
|
|
3631
|
+
`Track ${trackId} not found or not ready on asset ${assetId}. Available track IDs: ${availableTrackIds || "none"}.`,
|
|
3632
|
+
{ type: "validation_error" }
|
|
3459
3633
|
);
|
|
3460
3634
|
}
|
|
3461
3635
|
const vttUrl = await buildTranscriptUrl(playbackId, trackId, policy === "signed", credentials);
|
|
@@ -3463,7 +3637,7 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
3463
3637
|
try {
|
|
3464
3638
|
vttContent = await fetchVttFromMux(vttUrl);
|
|
3465
3639
|
} catch (error) {
|
|
3466
|
-
|
|
3640
|
+
wrapError(error, "Failed to fetch VTT content");
|
|
3467
3641
|
}
|
|
3468
3642
|
let editedVtt = vttContent;
|
|
3469
3643
|
let totalReplacementCount = 0;
|
|
@@ -3473,7 +3647,7 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
3473
3647
|
const { mode = "blank", alwaysCensor = [], neverCensor = [] } = autoCensorOption;
|
|
3474
3648
|
const plainText = extractTextFromVTT(vttContent);
|
|
3475
3649
|
if (!plainText.trim()) {
|
|
3476
|
-
throw new
|
|
3650
|
+
throw new MuxAiError("Track transcript is empty; nothing to censor.", { type: "validation_error" });
|
|
3477
3651
|
}
|
|
3478
3652
|
const modelConfig = resolveLanguageModelConfig({
|
|
3479
3653
|
...options,
|
|
@@ -3491,7 +3665,7 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
3491
3665
|
detectedProfanity = result.profanity;
|
|
3492
3666
|
usage = result.usage;
|
|
3493
3667
|
} catch (error) {
|
|
3494
|
-
|
|
3668
|
+
wrapError(error, `Failed to detect profanity with ${modelConfig.provider}`);
|
|
3495
3669
|
}
|
|
3496
3670
|
const finalProfanity = applyOverrideLists(detectedProfanity, alwaysCensor, neverCensor);
|
|
3497
3671
|
const { censoredVtt, replacements: censorReplacements } = censorVttContent(editedVtt, finalProfanity, mode);
|
|
@@ -3512,53 +3686,45 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
3512
3686
|
assetDurationSeconds
|
|
3513
3687
|
}
|
|
3514
3688
|
} : void 0;
|
|
3515
|
-
if (!uploadToMux) {
|
|
3516
|
-
return {
|
|
3517
|
-
assetId,
|
|
3518
|
-
trackId,
|
|
3519
|
-
originalVtt: vttContent,
|
|
3520
|
-
editedVtt,
|
|
3521
|
-
totalReplacementCount,
|
|
3522
|
-
autoCensorProfanity: autoCensorResult,
|
|
3523
|
-
replacements: replacementsResult,
|
|
3524
|
-
usage: usageWithMetadata
|
|
3525
|
-
};
|
|
3526
|
-
}
|
|
3527
3689
|
let presignedUrl;
|
|
3528
|
-
try {
|
|
3529
|
-
presignedUrl = await uploadEditedVttToS3({
|
|
3530
|
-
editedVtt,
|
|
3531
|
-
assetId,
|
|
3532
|
-
trackId,
|
|
3533
|
-
s3Endpoint,
|
|
3534
|
-
s3Region,
|
|
3535
|
-
s3Bucket,
|
|
3536
|
-
storageAdapter,
|
|
3537
|
-
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
3538
|
-
});
|
|
3539
|
-
} catch (error) {
|
|
3540
|
-
throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
3541
|
-
}
|
|
3542
3690
|
let uploadedTrackId;
|
|
3543
|
-
|
|
3544
|
-
const languageCode = sourceTrack.language_code || "en";
|
|
3545
|
-
const suffix = trackNameSuffix ?? "edited";
|
|
3546
|
-
const trackName = `${sourceTrack.name || "Subtitles"} (${suffix})`;
|
|
3547
|
-
uploadedTrackId = await createTextTrackOnMux(
|
|
3548
|
-
assetId,
|
|
3549
|
-
languageCode,
|
|
3550
|
-
trackName,
|
|
3551
|
-
presignedUrl,
|
|
3552
|
-
credentials
|
|
3553
|
-
);
|
|
3554
|
-
} catch (error) {
|
|
3555
|
-
console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
3556
|
-
}
|
|
3557
|
-
if (deleteOriginal && uploadedTrackId) {
|
|
3691
|
+
if (uploadToS3) {
|
|
3558
3692
|
try {
|
|
3559
|
-
await
|
|
3693
|
+
presignedUrl = await uploadEditedVttToS3({
|
|
3694
|
+
editedVtt,
|
|
3695
|
+
assetId,
|
|
3696
|
+
trackId,
|
|
3697
|
+
s3Endpoint,
|
|
3698
|
+
s3Region,
|
|
3699
|
+
s3Bucket,
|
|
3700
|
+
storageAdapter,
|
|
3701
|
+
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
3702
|
+
});
|
|
3560
3703
|
} catch (error) {
|
|
3561
|
-
|
|
3704
|
+
wrapError(error, "Failed to upload VTT to S3");
|
|
3705
|
+
}
|
|
3706
|
+
if (uploadToMux) {
|
|
3707
|
+
try {
|
|
3708
|
+
const languageCode = sourceTrack.language_code || "en";
|
|
3709
|
+
const suffix = trackNameSuffix ?? "edited";
|
|
3710
|
+
const trackName = `${sourceTrack.name || "Subtitles"} (${suffix})`;
|
|
3711
|
+
uploadedTrackId = await createTextTrackOnMux(
|
|
3712
|
+
assetId,
|
|
3713
|
+
languageCode,
|
|
3714
|
+
trackName,
|
|
3715
|
+
presignedUrl,
|
|
3716
|
+
credentials
|
|
3717
|
+
);
|
|
3718
|
+
} catch (error) {
|
|
3719
|
+
console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
3720
|
+
}
|
|
3721
|
+
if (deleteOriginal && uploadedTrackId) {
|
|
3722
|
+
try {
|
|
3723
|
+
await deleteTrackOnMux(assetId, trackId, credentials);
|
|
3724
|
+
} catch (error) {
|
|
3725
|
+
console.warn(`Failed to delete original track: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
3726
|
+
}
|
|
3727
|
+
}
|
|
3562
3728
|
}
|
|
3563
3729
|
}
|
|
3564
3730
|
return {
|
|
@@ -3629,43 +3795,21 @@ async function generateEmbeddingsInternal(assetId, options = {}) {
|
|
|
3629
3795
|
const embeddingModel = resolveEmbeddingModelConfig({ ...options, provider, model });
|
|
3630
3796
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
3631
3797
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
3632
|
-
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
3633
3798
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
3634
3799
|
if (policy === "signed" && !signingContext) {
|
|
3635
3800
|
throw new Error(
|
|
3636
3801
|
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
|
|
3637
3802
|
);
|
|
3638
3803
|
}
|
|
3639
|
-
const readyTextTracks = getReadyTextTracks(assetData);
|
|
3640
3804
|
const useVttChunking = chunkingStrategy.type === "vtt";
|
|
3641
|
-
|
|
3805
|
+
const transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
|
|
3642
3806
|
languageCode,
|
|
3643
3807
|
cleanTranscript: !useVttChunking,
|
|
3644
3808
|
shouldSign: policy === "signed",
|
|
3645
|
-
credentials
|
|
3809
|
+
credentials,
|
|
3810
|
+
required: true
|
|
3646
3811
|
});
|
|
3647
|
-
if (isAudioOnly && !transcriptResult.track && readyTextTracks.length === 1) {
|
|
3648
|
-
transcriptResult = await fetchTranscriptForAsset(assetData, playbackId, {
|
|
3649
|
-
cleanTranscript: !useVttChunking,
|
|
3650
|
-
shouldSign: policy === "signed",
|
|
3651
|
-
credentials
|
|
3652
|
-
});
|
|
3653
|
-
}
|
|
3654
|
-
if (!transcriptResult.track || !transcriptResult.transcriptText) {
|
|
3655
|
-
const availableLanguages = readyTextTracks.map((t) => t.language_code).filter(Boolean).join(", ");
|
|
3656
|
-
if (isAudioOnly) {
|
|
3657
|
-
throw new Error(
|
|
3658
|
-
`No transcript track found${languageCode ? ` for language '${languageCode}'` : ""}. Audio-only assets require a transcript. Available languages: ${availableLanguages || "none"}`
|
|
3659
|
-
);
|
|
3660
|
-
}
|
|
3661
|
-
throw new Error(
|
|
3662
|
-
`No caption track found${languageCode ? ` for language '${languageCode}'` : ""}. Available languages: ${availableLanguages || "none"}`
|
|
3663
|
-
);
|
|
3664
|
-
}
|
|
3665
3812
|
const transcriptText = transcriptResult.transcriptText;
|
|
3666
|
-
if (!transcriptText.trim()) {
|
|
3667
|
-
throw new Error("Transcript is empty");
|
|
3668
|
-
}
|
|
3669
3813
|
const chunks = useVttChunking ? chunkVTTCues(
|
|
3670
3814
|
parseVTTCues(transcriptText),
|
|
3671
3815
|
chunkingStrategy.maxTokens,
|
|
@@ -3779,6 +3923,9 @@ var SYSTEM_PROMPT4 = dedent5`
|
|
|
3779
3923
|
<constraints>
|
|
3780
3924
|
- Only describe what you can see in images or read in transcripts
|
|
3781
3925
|
- Do not fabricate details or make unsupported assumptions
|
|
3926
|
+
- Do NOT use any metadata such as URLs, file paths, domain names, file names,
|
|
3927
|
+
playback IDs, or technical parameters visible in this request. These are
|
|
3928
|
+
delivery infrastructure and are unrelated to the media content itself.
|
|
3782
3929
|
- Correlate engagement scores with observable content
|
|
3783
3930
|
- Return structured data matching the requested schema exactly
|
|
3784
3931
|
- Each momentInsight MUST include the hotspotIndex from the input data
|
|
@@ -3791,7 +3938,7 @@ var SYSTEM_PROMPT4 = dedent5`
|
|
|
3791
3938
|
- Connect engagement data to content features
|
|
3792
3939
|
</quality_guidelines>
|
|
3793
3940
|
`;
|
|
3794
|
-
var
|
|
3941
|
+
var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent5`
|
|
3795
3942
|
<role>
|
|
3796
3943
|
You are a video engagement analyst specializing in understanding viewer behavior for audio content.
|
|
3797
3944
|
Your job is to explain why specific moments have high or low engagement based on audio/dialogue and viewer watch patterns.
|
|
@@ -3819,6 +3966,9 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent5`
|
|
|
3819
3966
|
<constraints>
|
|
3820
3967
|
- Only describe what you can read in the transcript
|
|
3821
3968
|
- Do not fabricate details or make unsupported assumptions
|
|
3969
|
+
- Do NOT use any metadata such as URLs, file paths, domain names, file names,
|
|
3970
|
+
playback IDs, or technical parameters visible in this request. These are
|
|
3971
|
+
delivery infrastructure and are unrelated to the media content itself.
|
|
3822
3972
|
- Correlate engagement scores with observable content
|
|
3823
3973
|
- Return structured data matching the requested schema exactly
|
|
3824
3974
|
- Each momentInsight MUST include the hotspotIndex from the input data
|
|
@@ -4029,7 +4179,7 @@ async function generateEngagementInsights(assetId, options = {}) {
|
|
|
4029
4179
|
skipShots = false
|
|
4030
4180
|
} = options;
|
|
4031
4181
|
if (!Number.isInteger(hotspotLimit) || hotspotLimit < 1 || hotspotLimit > 10) {
|
|
4032
|
-
throw new
|
|
4182
|
+
throw new MuxAiError("hotspotLimit must be an integer between 1 and 10.", { type: "validation_error" });
|
|
4033
4183
|
}
|
|
4034
4184
|
const modelConfig = resolveLanguageModelConfig({
|
|
4035
4185
|
...options,
|
|
@@ -4039,13 +4189,14 @@ async function generateEngagementInsights(assetId, options = {}) {
|
|
|
4039
4189
|
const { asset, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
4040
4190
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(asset);
|
|
4041
4191
|
if (!assetDurationSeconds) {
|
|
4042
|
-
throw new
|
|
4192
|
+
throw new MuxAiError(`Asset ${assetId} has no valid duration.`, { type: "validation_error" });
|
|
4043
4193
|
}
|
|
4044
4194
|
const audioOnly = isAudioOnlyAsset(asset);
|
|
4045
4195
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
4046
4196
|
if (policy === "signed" && !signingContext) {
|
|
4047
|
-
throw new
|
|
4048
|
-
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
|
|
4197
|
+
throw new MuxAiError(
|
|
4198
|
+
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
|
|
4199
|
+
{ type: "validation_error" }
|
|
4049
4200
|
);
|
|
4050
4201
|
}
|
|
4051
4202
|
const shouldSign = policy === "signed";
|
|
@@ -4129,7 +4280,7 @@ async function generateEngagementInsights(assetId, options = {}) {
|
|
|
4129
4280
|
imageUrls.push(storyboardResult.value);
|
|
4130
4281
|
}
|
|
4131
4282
|
}
|
|
4132
|
-
const systemPrompt = audioOnly ?
|
|
4283
|
+
const systemPrompt = audioOnly ? AUDIO_ONLY_SYSTEM_PROMPT2 : SYSTEM_PROMPT4;
|
|
4133
4284
|
const userPrompt = buildUserPrompt4(
|
|
4134
4285
|
hotspots,
|
|
4135
4286
|
transcriptSegments,
|
|
@@ -4145,7 +4296,7 @@ async function generateEngagementInsights(assetId, options = {}) {
|
|
|
4145
4296
|
credentials
|
|
4146
4297
|
);
|
|
4147
4298
|
if (!aiInsights.momentInsights || aiInsights.momentInsights.length === 0) {
|
|
4148
|
-
throw new
|
|
4299
|
+
throw new MuxAiError(`Failed to generate insights for asset ${assetId}.`);
|
|
4149
4300
|
}
|
|
4150
4301
|
const momentInsights = [];
|
|
4151
4302
|
for (const aiMoment of aiInsights.momentInsights) {
|
|
@@ -4166,8 +4317,8 @@ async function generateEngagementInsights(assetId, options = {}) {
|
|
|
4166
4317
|
});
|
|
4167
4318
|
}
|
|
4168
4319
|
if (momentInsights.length === 0) {
|
|
4169
|
-
throw new
|
|
4170
|
-
|
|
4320
|
+
throw new MuxAiError(
|
|
4321
|
+
`Failed to generate valid insights for asset ${assetId}.`
|
|
4171
4322
|
);
|
|
4172
4323
|
}
|
|
4173
4324
|
const usageWithMetadata = {
|
|
@@ -4254,6 +4405,12 @@ var DEFAULT_THRESHOLDS = {
|
|
|
4254
4405
|
violence: 0.8
|
|
4255
4406
|
};
|
|
4256
4407
|
var DEFAULT_PROVIDER2 = "openai";
|
|
4408
|
+
var OPENAI_MODERATION_RETRIABLE_STATUS_CODES = /* @__PURE__ */ new Set([408, 429, 500, 502, 503, 504]);
|
|
4409
|
+
var OPENAI_MODERATION_MAX_RETRIES = 2;
|
|
4410
|
+
var OPENAI_MODERATION_BASE_DELAY_MS = 750;
|
|
4411
|
+
var OPENAI_MODERATION_MAX_DELAY_MS = 3e3;
|
|
4412
|
+
var MIN_SAMPLE_COVERAGE_FOR_CONFIDENT_THRESHOLDING = 0.5;
|
|
4413
|
+
var MIN_SUCCESSFUL_THUMBNAILS_FOR_CONFIDENT_THRESHOLDING = 3;
|
|
4257
4414
|
var HIVE_ENDPOINT = "https://api.thehive.ai/api/v2/task/sync";
|
|
4258
4415
|
var HIVE_SEXUAL_CATEGORIES = [
|
|
4259
4416
|
"general_nsfw",
|
|
@@ -4275,6 +4432,92 @@ var HIVE_VIOLENCE_CATEGORIES = [
|
|
|
4275
4432
|
"yes_self_harm",
|
|
4276
4433
|
"garm_death_injury_or_military_conflict"
|
|
4277
4434
|
];
|
|
4435
|
+
var OpenAIModerationRequestError = class extends Error {
|
|
4436
|
+
constructor(message, {
|
|
4437
|
+
status,
|
|
4438
|
+
retriable
|
|
4439
|
+
}) {
|
|
4440
|
+
super(message);
|
|
4441
|
+
this.name = "OpenAIModerationRequestError";
|
|
4442
|
+
this.status = status;
|
|
4443
|
+
this.retriable = retriable;
|
|
4444
|
+
}
|
|
4445
|
+
};
|
|
4446
|
+
function isRetriableOpenAIModerationStatus(status) {
|
|
4447
|
+
return OPENAI_MODERATION_RETRIABLE_STATUS_CODES.has(status);
|
|
4448
|
+
}
|
|
4449
|
+
function compactBodySnippet(body, maxLength = 180) {
|
|
4450
|
+
const compact = body.replace(/\s+/g, " ").trim();
|
|
4451
|
+
if (!compact) {
|
|
4452
|
+
return "(empty response body)";
|
|
4453
|
+
}
|
|
4454
|
+
return compact.length > maxLength ? `${compact.slice(0, maxLength)}...` : compact;
|
|
4455
|
+
}
|
|
4456
|
+
function parseOpenAIModerationResponse(bodyText, status, statusText) {
|
|
4457
|
+
if (!bodyText.trim()) {
|
|
4458
|
+
throw new OpenAIModerationRequestError(
|
|
4459
|
+
`OpenAI moderation returned an empty response (${status} ${statusText}).`,
|
|
4460
|
+
{ status, retriable: isRetriableOpenAIModerationStatus(status) }
|
|
4461
|
+
);
|
|
4462
|
+
}
|
|
4463
|
+
try {
|
|
4464
|
+
return JSON.parse(bodyText);
|
|
4465
|
+
} catch {
|
|
4466
|
+
throw new OpenAIModerationRequestError(
|
|
4467
|
+
`OpenAI moderation returned non-JSON response (${status} ${statusText}): ${compactBodySnippet(bodyText)}`,
|
|
4468
|
+
{ status, retriable: true }
|
|
4469
|
+
);
|
|
4470
|
+
}
|
|
4471
|
+
}
|
|
4472
|
+
async function callOpenAIModerationApi({
|
|
4473
|
+
model,
|
|
4474
|
+
input,
|
|
4475
|
+
credentials
|
|
4476
|
+
}) {
|
|
4477
|
+
"use step";
|
|
4478
|
+
const apiKey = await getApiKeyFromEnv("openai", credentials);
|
|
4479
|
+
return withRetry(
|
|
4480
|
+
async () => {
|
|
4481
|
+
let res;
|
|
4482
|
+
try {
|
|
4483
|
+
res = await fetch("https://api.openai.com/v1/moderations", {
|
|
4484
|
+
method: "POST",
|
|
4485
|
+
headers: {
|
|
4486
|
+
"Content-Type": "application/json",
|
|
4487
|
+
"Authorization": `Bearer ${apiKey}`
|
|
4488
|
+
},
|
|
4489
|
+
body: JSON.stringify({
|
|
4490
|
+
model,
|
|
4491
|
+
input
|
|
4492
|
+
})
|
|
4493
|
+
});
|
|
4494
|
+
} catch (error) {
|
|
4495
|
+
throw new OpenAIModerationRequestError(
|
|
4496
|
+
`OpenAI moderation request failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
4497
|
+
{ retriable: true }
|
|
4498
|
+
);
|
|
4499
|
+
}
|
|
4500
|
+
const bodyText = await res.text();
|
|
4501
|
+
const json = parseOpenAIModerationResponse(bodyText, res.status, res.statusText);
|
|
4502
|
+
if (!res.ok) {
|
|
4503
|
+
throw new OpenAIModerationRequestError(
|
|
4504
|
+
`OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`,
|
|
4505
|
+
{
|
|
4506
|
+
status: res.status,
|
|
4507
|
+
retriable: isRetriableOpenAIModerationStatus(res.status)
|
|
4508
|
+
}
|
|
4509
|
+
);
|
|
4510
|
+
}
|
|
4511
|
+
return json;
|
|
4512
|
+
},
|
|
4513
|
+
{
|
|
4514
|
+
maxRetries: OPENAI_MODERATION_MAX_RETRIES,
|
|
4515
|
+
baseDelay: OPENAI_MODERATION_BASE_DELAY_MS,
|
|
4516
|
+
maxDelay: OPENAI_MODERATION_MAX_DELAY_MS,
|
|
4517
|
+
shouldRetry: (error) => error instanceof OpenAIModerationRequestError ? error.retriable : true
|
|
4518
|
+
}
|
|
4519
|
+
);
|
|
4520
|
+
}
|
|
4278
4521
|
async function processConcurrently(items, processor, maxConcurrent = 5) {
|
|
4279
4522
|
"use step";
|
|
4280
4523
|
const results = [];
|
|
@@ -4288,32 +4531,19 @@ async function processConcurrently(items, processor, maxConcurrent = 5) {
|
|
|
4288
4531
|
}
|
|
4289
4532
|
async function moderateImageWithOpenAI(entry) {
|
|
4290
4533
|
"use step";
|
|
4291
|
-
const apiKey = await getApiKeyFromEnv("openai", entry.credentials);
|
|
4292
4534
|
try {
|
|
4293
|
-
const
|
|
4294
|
-
|
|
4295
|
-
|
|
4296
|
-
|
|
4297
|
-
|
|
4298
|
-
|
|
4299
|
-
|
|
4300
|
-
model: entry.model,
|
|
4301
|
-
input: [
|
|
4302
|
-
{
|
|
4303
|
-
type: "image_url",
|
|
4304
|
-
image_url: {
|
|
4305
|
-
url: entry.image
|
|
4306
|
-
}
|
|
4535
|
+
const json = await callOpenAIModerationApi({
|
|
4536
|
+
model: entry.model,
|
|
4537
|
+
input: [
|
|
4538
|
+
{
|
|
4539
|
+
type: "image_url",
|
|
4540
|
+
image_url: {
|
|
4541
|
+
url: entry.image
|
|
4307
4542
|
}
|
|
4308
|
-
|
|
4309
|
-
|
|
4543
|
+
}
|
|
4544
|
+
],
|
|
4545
|
+
credentials: entry.credentials
|
|
4310
4546
|
});
|
|
4311
|
-
const json = await res.json();
|
|
4312
|
-
if (!res.ok) {
|
|
4313
|
-
throw new Error(
|
|
4314
|
-
`OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`
|
|
4315
|
-
);
|
|
4316
|
-
}
|
|
4317
4547
|
const categoryScores = json.results?.[0]?.category_scores || {};
|
|
4318
4548
|
return {
|
|
4319
4549
|
url: entry.url,
|
|
@@ -4345,25 +4575,12 @@ async function requestOpenAIModeration(images, model, maxConcurrent = 5, submiss
|
|
|
4345
4575
|
}
|
|
4346
4576
|
async function requestOpenAITextModeration(text, model, url, credentials) {
|
|
4347
4577
|
"use step";
|
|
4348
|
-
const apiKey = await getApiKeyFromEnv("openai", credentials);
|
|
4349
4578
|
try {
|
|
4350
|
-
const
|
|
4351
|
-
|
|
4352
|
-
|
|
4353
|
-
|
|
4354
|
-
"Authorization": `Bearer ${apiKey}`
|
|
4355
|
-
},
|
|
4356
|
-
body: JSON.stringify({
|
|
4357
|
-
model,
|
|
4358
|
-
input: text
|
|
4359
|
-
})
|
|
4579
|
+
const json = await callOpenAIModerationApi({
|
|
4580
|
+
model,
|
|
4581
|
+
input: text,
|
|
4582
|
+
credentials
|
|
4360
4583
|
});
|
|
4361
|
-
const json = await res.json();
|
|
4362
|
-
if (!res.ok) {
|
|
4363
|
-
throw new Error(
|
|
4364
|
-
`OpenAI moderation error: ${res.status} ${res.statusText} - ${JSON.stringify(json)}`
|
|
4365
|
-
);
|
|
4366
|
-
}
|
|
4367
4584
|
const categoryScores = json.results?.[0]?.category_scores || {};
|
|
4368
4585
|
return {
|
|
4369
4586
|
url,
|
|
@@ -4569,22 +4786,13 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
4569
4786
|
let thumbnailCount;
|
|
4570
4787
|
if (isAudioOnly) {
|
|
4571
4788
|
mode = "transcript";
|
|
4572
|
-
const
|
|
4573
|
-
let transcriptResult = await fetchTranscriptForAsset(asset, playbackId, {
|
|
4789
|
+
const transcriptResult = await fetchTranscriptForAsset(asset, playbackId, {
|
|
4574
4790
|
languageCode,
|
|
4575
4791
|
cleanTranscript: true,
|
|
4576
4792
|
shouldSign: policy === "signed",
|
|
4577
4793
|
credentials,
|
|
4578
4794
|
required: true
|
|
4579
4795
|
});
|
|
4580
|
-
if (!transcriptResult.track && readyTextTracks.length === 1) {
|
|
4581
|
-
transcriptResult = await fetchTranscriptForAsset(asset, playbackId, {
|
|
4582
|
-
cleanTranscript: true,
|
|
4583
|
-
shouldSign: policy === "signed",
|
|
4584
|
-
credentials,
|
|
4585
|
-
required: true
|
|
4586
|
-
});
|
|
4587
|
-
}
|
|
4588
4796
|
if (provider === "openai") {
|
|
4589
4797
|
thumbnailScores = await requestOpenAITranscriptModeration(
|
|
4590
4798
|
transcriptResult.transcriptText,
|
|
@@ -4598,35 +4806,28 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
4598
4806
|
throw new Error(`Unsupported moderation provider: ${provider}`);
|
|
4599
4807
|
}
|
|
4600
4808
|
} else {
|
|
4601
|
-
const
|
|
4602
|
-
|
|
4603
|
-
|
|
4604
|
-
|
|
4809
|
+
const estimatedIntervalCount = duration <= 50 ? 5 : Math.ceil(duration / thumbnailInterval);
|
|
4810
|
+
const thumbnailUrls = maxSamples !== void 0 && estimatedIntervalCount > maxSamples ? await getThumbnailUrlsFromTimestamps(
|
|
4811
|
+
playbackId,
|
|
4812
|
+
planSamplingTimestamps({
|
|
4813
|
+
duration_sec: duration,
|
|
4814
|
+
max_candidates: maxSamples,
|
|
4815
|
+
trim_start_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
|
|
4816
|
+
trim_end_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
|
|
4817
|
+
fps: videoTrackFps,
|
|
4818
|
+
base_cadence_hz: thumbnailInterval > 0 ? 1 / thumbnailInterval : void 0
|
|
4819
|
+
}),
|
|
4820
|
+
{
|
|
4605
4821
|
width: thumbnailWidth,
|
|
4606
4822
|
shouldSign: policy === "signed",
|
|
4607
4823
|
credentials
|
|
4608
|
-
}
|
|
4609
|
-
) : (
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
|
|
4613
|
-
|
|
4614
|
-
|
|
4615
|
-
planSamplingTimestamps({
|
|
4616
|
-
duration_sec: duration,
|
|
4617
|
-
max_candidates: maxSamples,
|
|
4618
|
-
trim_start_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
|
|
4619
|
-
trim_end_sec: duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
|
|
4620
|
-
fps: videoTrackFps,
|
|
4621
|
-
base_cadence_hz: thumbnailInterval > 0 ? 1 / thumbnailInterval : void 0
|
|
4622
|
-
}),
|
|
4623
|
-
{
|
|
4624
|
-
width: thumbnailWidth,
|
|
4625
|
-
shouldSign: policy === "signed",
|
|
4626
|
-
credentials
|
|
4627
|
-
}
|
|
4628
|
-
)
|
|
4629
|
-
);
|
|
4824
|
+
}
|
|
4825
|
+
) : await getThumbnailUrls(playbackId, duration, {
|
|
4826
|
+
interval: thumbnailInterval,
|
|
4827
|
+
width: thumbnailWidth,
|
|
4828
|
+
shouldSign: policy === "signed",
|
|
4829
|
+
credentials
|
|
4830
|
+
});
|
|
4630
4831
|
thumbnailCount = thumbnailUrls.length;
|
|
4631
4832
|
if (provider === "openai") {
|
|
4632
4833
|
thumbnailScores = await requestOpenAIModeration(
|
|
@@ -4650,20 +4851,40 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
4650
4851
|
}
|
|
4651
4852
|
}
|
|
4652
4853
|
const failed = thumbnailScores.filter((s) => s.error);
|
|
4653
|
-
|
|
4854
|
+
const successful = thumbnailScores.filter((s) => !s.error);
|
|
4855
|
+
if (successful.length === 0) {
|
|
4654
4856
|
const details = failed.map((s) => `${s.url}: ${s.errorMessage || "Unknown error"}`).join("; ");
|
|
4655
4857
|
throw new Error(
|
|
4656
|
-
`Moderation failed for ${
|
|
4858
|
+
`Moderation failed for all ${thumbnailScores.length} sample(s): ${details}`
|
|
4859
|
+
);
|
|
4860
|
+
}
|
|
4861
|
+
if (failed.length > 0) {
|
|
4862
|
+
console.warn(
|
|
4863
|
+
`Moderation had partial failures (${failed.length}/${thumbnailScores.length}); continuing with successful samples.`
|
|
4657
4864
|
);
|
|
4658
4865
|
}
|
|
4659
|
-
const maxSexual = Math.max(...
|
|
4660
|
-
const maxViolence = Math.max(...
|
|
4866
|
+
const maxSexual = Math.max(...successful.map((s) => s.sexual));
|
|
4867
|
+
const maxViolence = Math.max(...successful.map((s) => s.violence));
|
|
4661
4868
|
const finalThresholds = { ...DEFAULT_THRESHOLDS, ...thresholds };
|
|
4869
|
+
const requestedSampleCount = thumbnailScores.length;
|
|
4870
|
+
const successfulSampleCount = successful.length;
|
|
4871
|
+
const failedSampleCount = failed.length;
|
|
4872
|
+
const sampleCoverage = successfulSampleCount / requestedSampleCount;
|
|
4873
|
+
const hasEnoughSuccessfulSamples = mode === "transcript" || successfulSampleCount >= MIN_SUCCESSFUL_THUMBNAILS_FOR_CONFIDENT_THRESHOLDING;
|
|
4874
|
+
const isLowConfidence = sampleCoverage < MIN_SAMPLE_COVERAGE_FOR_CONFIDENT_THRESHOLDING || !hasEnoughSuccessfulSamples;
|
|
4662
4875
|
return {
|
|
4663
4876
|
assetId,
|
|
4664
4877
|
mode,
|
|
4665
4878
|
isAudioOnly,
|
|
4666
4879
|
thumbnailScores,
|
|
4880
|
+
coverage: {
|
|
4881
|
+
requestedSampleCount,
|
|
4882
|
+
successfulSampleCount,
|
|
4883
|
+
failedSampleCount,
|
|
4884
|
+
sampleCoverage,
|
|
4885
|
+
isPartial: failedSampleCount > 0,
|
|
4886
|
+
isLowConfidence
|
|
4887
|
+
},
|
|
4667
4888
|
usage: {
|
|
4668
4889
|
metadata: {
|
|
4669
4890
|
assetDurationSeconds: duration,
|
|
@@ -4865,6 +5086,9 @@ var SYSTEM_PROMPT5 = dedent6`
|
|
|
4865
5086
|
<constraints>
|
|
4866
5087
|
- Only describe what is clearly observable in the frames or explicitly stated in the transcript
|
|
4867
5088
|
- Do not fabricate details or make unsupported assumptions
|
|
5089
|
+
- Do NOT use any metadata such as URLs, file paths, domain names, file names,
|
|
5090
|
+
playback IDs, or technical parameters visible in this request. These are
|
|
5091
|
+
delivery infrastructure and are unrelated to the media content itself.
|
|
4868
5092
|
- Return structured data matching the requested schema
|
|
4869
5093
|
- Output only the JSON object; no markdown or extra text
|
|
4870
5094
|
- When a <language> section is provided, all output text MUST be written in that language
|
|
@@ -4891,7 +5115,7 @@ var SYSTEM_PROMPT5 = dedent6`
|
|
|
4891
5115
|
|
|
4892
5116
|
Write as if describing reality, not describing a recording of reality.
|
|
4893
5117
|
</language_guidelines>`;
|
|
4894
|
-
var
|
|
5118
|
+
var AUDIO_ONLY_SYSTEM_PROMPT3 = dedent6`
|
|
4895
5119
|
<role>
|
|
4896
5120
|
You are an audio content analyst specializing in transcript analysis and metadata generation.
|
|
4897
5121
|
</role>
|
|
@@ -4919,6 +5143,9 @@ var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent6`
|
|
|
4919
5143
|
<constraints>
|
|
4920
5144
|
- Only describe what is explicitly stated or strongly implied in the transcript
|
|
4921
5145
|
- Do not fabricate details or make unsupported assumptions
|
|
5146
|
+
- Do NOT use any metadata such as URLs, file paths, domain names, file names,
|
|
5147
|
+
playback IDs, or technical parameters visible in this request. These are
|
|
5148
|
+
delivery infrastructure and are unrelated to the media content itself.
|
|
4922
5149
|
- Return structured data matching the requested schema
|
|
4923
5150
|
- Focus entirely on audio/spoken content - there are no visual elements
|
|
4924
5151
|
- Output only the JSON object; no markdown or extra text
|
|
@@ -5068,6 +5295,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
5068
5295
|
const {
|
|
5069
5296
|
provider = "openai",
|
|
5070
5297
|
model,
|
|
5298
|
+
languageCode,
|
|
5071
5299
|
tone = "neutral",
|
|
5072
5300
|
includeTranscript = true,
|
|
5073
5301
|
cleanTranscript = true,
|
|
@@ -5081,8 +5309,9 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
5081
5309
|
outputLanguageCode
|
|
5082
5310
|
} = options ?? {};
|
|
5083
5311
|
if (!VALID_TONES.includes(tone)) {
|
|
5084
|
-
throw new
|
|
5085
|
-
`Invalid tone "${tone}". Valid tones are: ${VALID_TONES.join(", ")}
|
|
5312
|
+
throw new MuxAiError(
|
|
5313
|
+
`Invalid tone "${tone}". Valid tones are: ${VALID_TONES.join(", ")}.`,
|
|
5314
|
+
{ type: "validation_error" }
|
|
5086
5315
|
);
|
|
5087
5316
|
}
|
|
5088
5317
|
const modelConfig = resolveLanguageModelConfig({
|
|
@@ -5095,17 +5324,20 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
5095
5324
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
5096
5325
|
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
5097
5326
|
if (isAudioOnly && !includeTranscript) {
|
|
5098
|
-
throw new
|
|
5099
|
-
"Audio-only assets require a transcript. Set includeTranscript: true and ensure the asset has a ready text track (captions/subtitles)."
|
|
5327
|
+
throw new MuxAiError(
|
|
5328
|
+
"Audio-only assets require a transcript. Set includeTranscript: true and ensure the asset has a ready text track (captions/subtitles).",
|
|
5329
|
+
{ type: "validation_error" }
|
|
5100
5330
|
);
|
|
5101
5331
|
}
|
|
5102
5332
|
const signingContext = await resolveMuxSigningContext(workflowCredentials);
|
|
5103
5333
|
if (policy === "signed" && !signingContext) {
|
|
5104
|
-
throw new
|
|
5105
|
-
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
|
|
5334
|
+
throw new MuxAiError(
|
|
5335
|
+
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
|
|
5336
|
+
{ type: "validation_error" }
|
|
5106
5337
|
);
|
|
5107
5338
|
}
|
|
5108
5339
|
const transcriptResult = includeTranscript ? await fetchTranscriptForAsset(assetData, playbackId, {
|
|
5340
|
+
languageCode,
|
|
5109
5341
|
cleanTranscript,
|
|
5110
5342
|
shouldSign: policy === "signed",
|
|
5111
5343
|
credentials: workflowCredentials,
|
|
@@ -5127,7 +5359,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
5127
5359
|
});
|
|
5128
5360
|
let analysisResponse;
|
|
5129
5361
|
let imageUrl;
|
|
5130
|
-
const systemPrompt = isAudioOnly ?
|
|
5362
|
+
const systemPrompt = isAudioOnly ? AUDIO_ONLY_SYSTEM_PROMPT3 : SYSTEM_PROMPT5;
|
|
5131
5363
|
try {
|
|
5132
5364
|
if (isAudioOnly) {
|
|
5133
5365
|
analysisResponse = await analyzeAudioOnly(
|
|
@@ -5163,18 +5395,17 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
5163
5395
|
}
|
|
5164
5396
|
} catch (error) {
|
|
5165
5397
|
const contentType = isAudioOnly ? "audio" : "video";
|
|
5166
|
-
|
|
5167
|
-
`Failed to analyze ${contentType} content with ${provider}: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
5168
|
-
);
|
|
5398
|
+
wrapError(error, `Failed to analyze ${contentType} content with ${provider}`);
|
|
5169
5399
|
}
|
|
5170
5400
|
if (!analysisResponse.result) {
|
|
5171
|
-
|
|
5401
|
+
const contentType = isAudioOnly ? "audio" : "video";
|
|
5402
|
+
throw new MuxAiError(`Failed to analyze ${contentType} content for asset ${assetId}.`);
|
|
5172
5403
|
}
|
|
5173
5404
|
if (!analysisResponse.result.title) {
|
|
5174
|
-
throw new
|
|
5405
|
+
throw new MuxAiError(`Failed to generate title for asset ${assetId}.`);
|
|
5175
5406
|
}
|
|
5176
5407
|
if (!analysisResponse.result.description) {
|
|
5177
|
-
throw new
|
|
5408
|
+
throw new MuxAiError(`Failed to generate description for asset ${assetId}.`);
|
|
5178
5409
|
}
|
|
5179
5410
|
return {
|
|
5180
5411
|
assetId,
|
|
@@ -5233,12 +5464,11 @@ async function requestStaticRenditionCreation(assetId, credentials) {
|
|
|
5233
5464
|
} catch (error) {
|
|
5234
5465
|
const statusCode = error?.status ?? error?.statusCode;
|
|
5235
5466
|
const messages = error?.error?.messages;
|
|
5236
|
-
const alreadyDefined = messages?.some((
|
|
5467
|
+
const alreadyDefined = messages?.some((message) => message.toLowerCase().includes("already defined")) ?? error?.message?.toLowerCase().includes("already defined");
|
|
5237
5468
|
if (statusCode === 409 || alreadyDefined) {
|
|
5238
5469
|
return;
|
|
5239
5470
|
}
|
|
5240
|
-
|
|
5241
|
-
throw new Error(`Failed to request static rendition from Mux: ${message}`);
|
|
5471
|
+
wrapError(error, "Failed to request static rendition from Mux");
|
|
5242
5472
|
}
|
|
5243
5473
|
}
|
|
5244
5474
|
async function waitForAudioStaticRendition({
|
|
@@ -5272,13 +5502,14 @@ async function waitForAudioStaticRendition({
|
|
|
5272
5502
|
`\u231B Waiting for static rendition (attempt ${attempt}/${STATIC_RENDITION_MAX_ATTEMPTS}) \u2192 ${currentStatus}`
|
|
5273
5503
|
);
|
|
5274
5504
|
if (currentStatus === "errored") {
|
|
5275
|
-
throw new
|
|
5505
|
+
throw new MuxAiError(
|
|
5276
5506
|
"Mux failed to create the static rendition for this asset. Please check the asset in the Mux dashboard."
|
|
5277
5507
|
);
|
|
5278
5508
|
}
|
|
5279
5509
|
}
|
|
5280
|
-
throw new
|
|
5281
|
-
"Timed out waiting for the static rendition to become ready. Please try again in a moment."
|
|
5510
|
+
throw new MuxAiError(
|
|
5511
|
+
"Timed out waiting for the static rendition to become ready. Please try again in a moment.",
|
|
5512
|
+
{ type: "timeout_error", retryable: true }
|
|
5282
5513
|
);
|
|
5283
5514
|
}
|
|
5284
5515
|
async function fetchAudioFromMux(audioUrl) {
|
|
@@ -5424,22 +5655,25 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
5424
5655
|
fromLanguageCode,
|
|
5425
5656
|
numSpeakers = 0,
|
|
5426
5657
|
// 0 = auto-detect
|
|
5427
|
-
|
|
5658
|
+
uploadToS3: uploadToS3Option,
|
|
5659
|
+
uploadToMux: uploadToMuxOption,
|
|
5428
5660
|
storageAdapter,
|
|
5429
5661
|
credentials: providedCredentials
|
|
5430
5662
|
} = options;
|
|
5431
5663
|
if (provider !== "elevenlabs") {
|
|
5432
|
-
throw new
|
|
5664
|
+
throw new MuxAiError("Only ElevenLabs provider is currently supported for audio translation.", { type: "validation_error" });
|
|
5433
5665
|
}
|
|
5434
5666
|
const credentials = providedCredentials;
|
|
5435
5667
|
const effectiveStorageAdapter = storageAdapter;
|
|
5668
|
+
const uploadToMux = uploadToMuxOption !== false;
|
|
5669
|
+
const uploadToS3 = uploadToS3Option || uploadToMux;
|
|
5436
5670
|
const s3Endpoint = options.s3Endpoint ?? env_default.S3_ENDPOINT;
|
|
5437
5671
|
const s3Region = options.s3Region ?? env_default.S3_REGION ?? "auto";
|
|
5438
5672
|
const s3Bucket = options.s3Bucket ?? env_default.S3_BUCKET;
|
|
5439
5673
|
const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
|
|
5440
5674
|
const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
|
|
5441
|
-
if (
|
|
5442
|
-
throw new
|
|
5675
|
+
if (uploadToS3 && (!s3Endpoint || !s3Bucket || !effectiveStorageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
|
|
5676
|
+
throw new MuxAiError("Storage configuration is required for uploading. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.", { type: "validation_error" });
|
|
5443
5677
|
}
|
|
5444
5678
|
const { asset: initialAsset, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
5445
5679
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(initialAsset);
|
|
@@ -5454,8 +5688,9 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
5454
5688
|
}
|
|
5455
5689
|
const audioRendition = getReadyAudioStaticRendition(currentAsset);
|
|
5456
5690
|
if (!audioRendition) {
|
|
5457
|
-
throw new
|
|
5458
|
-
"Unable to obtain an audio-only static rendition for this asset. Please verify static renditions are enabled in Mux."
|
|
5691
|
+
throw new MuxAiError(
|
|
5692
|
+
"Unable to obtain an audio-only static rendition for this asset. Please verify static renditions are enabled in Mux.",
|
|
5693
|
+
{ type: "validation_error" }
|
|
5459
5694
|
);
|
|
5460
5695
|
}
|
|
5461
5696
|
let audioUrl = `https://stream.mux.com/${playbackId}/audio.m4a`;
|
|
@@ -5467,7 +5702,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
5467
5702
|
try {
|
|
5468
5703
|
audioBuffer = await fetchAudioFromMux(audioUrl);
|
|
5469
5704
|
} catch (error) {
|
|
5470
|
-
|
|
5705
|
+
wrapError(error, "Failed to fetch audio from Mux");
|
|
5471
5706
|
}
|
|
5472
5707
|
console.warn("\u{1F399}\uFE0F Creating dubbing job in ElevenLabs...");
|
|
5473
5708
|
const elevenLabsLangCode = toISO639_3(toLanguageCode);
|
|
@@ -5488,7 +5723,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
5488
5723
|
});
|
|
5489
5724
|
console.warn(`\u2705 Dubbing job created with ID: ${dubbingId}`);
|
|
5490
5725
|
} catch (error) {
|
|
5491
|
-
|
|
5726
|
+
wrapError(error, "Failed to create ElevenLabs dubbing job");
|
|
5492
5727
|
}
|
|
5493
5728
|
console.warn("\u23F3 Waiting for dubbing to complete...");
|
|
5494
5729
|
let dubbingStatus = "dubbing";
|
|
@@ -5509,77 +5744,72 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
5509
5744
|
throw new Error("ElevenLabs dubbing job failed");
|
|
5510
5745
|
}
|
|
5511
5746
|
} catch (error) {
|
|
5512
|
-
|
|
5747
|
+
wrapError(error, "Failed to check dubbing status");
|
|
5513
5748
|
}
|
|
5514
5749
|
}
|
|
5515
5750
|
if (dubbingStatus !== "dubbed") {
|
|
5516
|
-
throw new
|
|
5751
|
+
throw new MuxAiError("Audio translation timed out or failed. Please try again.", { type: "timeout_error", retryable: true });
|
|
5517
5752
|
}
|
|
5518
5753
|
console.warn("\u2705 Dubbing completed successfully!");
|
|
5519
|
-
if (!uploadToMux) {
|
|
5520
|
-
const targetLanguage2 = getLanguageCodePair(toLanguageCode);
|
|
5521
|
-
return {
|
|
5522
|
-
assetId,
|
|
5523
|
-
targetLanguageCode: targetLanguage2.iso639_1,
|
|
5524
|
-
targetLanguage: targetLanguage2,
|
|
5525
|
-
dubbingId
|
|
5526
|
-
};
|
|
5527
|
-
}
|
|
5528
|
-
console.warn("\u{1F4E5} Downloading dubbed audio from ElevenLabs...");
|
|
5529
|
-
let dubbedAudioBuffer;
|
|
5530
|
-
try {
|
|
5531
|
-
const requestedLangCode = toISO639_3(toLanguageCode);
|
|
5532
|
-
let downloadLangCode = targetLanguages.find(
|
|
5533
|
-
(lang) => lang === requestedLangCode
|
|
5534
|
-
) ?? targetLanguages.find(
|
|
5535
|
-
(lang) => lang.toLowerCase() === requestedLangCode.toLowerCase()
|
|
5536
|
-
);
|
|
5537
|
-
if (!downloadLangCode && targetLanguages.length > 0) {
|
|
5538
|
-
downloadLangCode = targetLanguages[0];
|
|
5539
|
-
console.warn(`\u26A0\uFE0F Requested language "${requestedLangCode}" not found in target_languages. Using "${downloadLangCode}" instead.`);
|
|
5540
|
-
}
|
|
5541
|
-
if (!downloadLangCode) {
|
|
5542
|
-
downloadLangCode = requestedLangCode;
|
|
5543
|
-
console.warn(`\u26A0\uFE0F No target_languages available from ElevenLabs status. Using requested language code: ${requestedLangCode}`);
|
|
5544
|
-
}
|
|
5545
|
-
dubbedAudioBuffer = await downloadDubbedAudioFromElevenLabs({
|
|
5546
|
-
dubbingId,
|
|
5547
|
-
languageCode: downloadLangCode,
|
|
5548
|
-
credentials
|
|
5549
|
-
});
|
|
5550
|
-
console.warn("\u2705 Dubbed audio downloaded successfully!");
|
|
5551
|
-
} catch (error) {
|
|
5552
|
-
throw new Error(`Failed to download dubbed audio: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
5553
|
-
}
|
|
5554
|
-
console.warn("\u{1F4E4} Uploading dubbed audio to S3-compatible storage...");
|
|
5555
5754
|
let presignedUrl;
|
|
5556
|
-
try {
|
|
5557
|
-
presignedUrl = await uploadDubbedAudioToS3({
|
|
5558
|
-
dubbedAudioBuffer,
|
|
5559
|
-
assetId,
|
|
5560
|
-
toLanguageCode,
|
|
5561
|
-
s3Endpoint,
|
|
5562
|
-
s3Region,
|
|
5563
|
-
s3Bucket,
|
|
5564
|
-
storageAdapter: effectiveStorageAdapter,
|
|
5565
|
-
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
5566
|
-
});
|
|
5567
|
-
} catch (error) {
|
|
5568
|
-
throw new Error(`Failed to upload audio to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
5569
|
-
}
|
|
5570
|
-
console.warn("\u{1F4F9} Adding dubbed audio track to Mux asset...");
|
|
5571
5755
|
let uploadedTrackId;
|
|
5572
|
-
|
|
5573
|
-
|
|
5574
|
-
|
|
5575
|
-
|
|
5576
|
-
|
|
5577
|
-
|
|
5578
|
-
|
|
5579
|
-
|
|
5580
|
-
|
|
5581
|
-
|
|
5582
|
-
|
|
5756
|
+
if (uploadToS3) {
|
|
5757
|
+
console.warn("\u{1F4E5} Downloading dubbed audio from ElevenLabs...");
|
|
5758
|
+
let dubbedAudioBuffer;
|
|
5759
|
+
try {
|
|
5760
|
+
const requestedLangCode = toISO639_3(toLanguageCode);
|
|
5761
|
+
let downloadLangCode = targetLanguages.find(
|
|
5762
|
+
(lang) => lang === requestedLangCode
|
|
5763
|
+
) ?? targetLanguages.find(
|
|
5764
|
+
(lang) => lang.toLowerCase() === requestedLangCode.toLowerCase()
|
|
5765
|
+
);
|
|
5766
|
+
if (!downloadLangCode && targetLanguages.length > 0) {
|
|
5767
|
+
downloadLangCode = targetLanguages[0];
|
|
5768
|
+
console.warn(`\u26A0\uFE0F Requested language "${requestedLangCode}" not found in target_languages. Using "${downloadLangCode}" instead.`);
|
|
5769
|
+
}
|
|
5770
|
+
if (!downloadLangCode) {
|
|
5771
|
+
downloadLangCode = requestedLangCode;
|
|
5772
|
+
console.warn(`\u26A0\uFE0F No target_languages available from ElevenLabs status. Using requested language code: ${requestedLangCode}`);
|
|
5773
|
+
}
|
|
5774
|
+
dubbedAudioBuffer = await downloadDubbedAudioFromElevenLabs({
|
|
5775
|
+
dubbingId,
|
|
5776
|
+
languageCode: downloadLangCode,
|
|
5777
|
+
credentials
|
|
5778
|
+
});
|
|
5779
|
+
console.warn("\u2705 Dubbed audio downloaded successfully!");
|
|
5780
|
+
} catch (error) {
|
|
5781
|
+
wrapError(error, "Failed to download dubbed audio");
|
|
5782
|
+
}
|
|
5783
|
+
console.warn("\u{1F4E4} Uploading dubbed audio to S3-compatible storage...");
|
|
5784
|
+
try {
|
|
5785
|
+
presignedUrl = await uploadDubbedAudioToS3({
|
|
5786
|
+
dubbedAudioBuffer,
|
|
5787
|
+
assetId,
|
|
5788
|
+
toLanguageCode,
|
|
5789
|
+
s3Endpoint,
|
|
5790
|
+
s3Region,
|
|
5791
|
+
s3Bucket,
|
|
5792
|
+
storageAdapter: effectiveStorageAdapter,
|
|
5793
|
+
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
5794
|
+
});
|
|
5795
|
+
} catch (error) {
|
|
5796
|
+
wrapError(error, "Failed to upload audio to S3");
|
|
5797
|
+
}
|
|
5798
|
+
if (uploadToMux) {
|
|
5799
|
+
console.warn("\u{1F4F9} Adding dubbed audio track to Mux asset...");
|
|
5800
|
+
const muxLangCode = toISO639_1(toLanguageCode);
|
|
5801
|
+
try {
|
|
5802
|
+
uploadedTrackId = await createAudioTrackOnMux(assetId, muxLangCode, presignedUrl, credentials);
|
|
5803
|
+
const languageName = new Intl.DisplayNames(["en"], { type: "language" }).of(muxLangCode) || muxLangCode.toUpperCase();
|
|
5804
|
+
const trackName = `${languageName} (auto-dubbed)`;
|
|
5805
|
+
console.warn(`\u2705 Track added to Mux asset with ID: ${uploadedTrackId}`);
|
|
5806
|
+
console.warn(`\u{1F4CB} Track name: "${trackName}"`);
|
|
5807
|
+
} catch (error) {
|
|
5808
|
+
console.warn(`\u26A0\uFE0F Failed to add audio track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
5809
|
+
console.warn("\u{1F517} You can manually add the track using this presigned URL:");
|
|
5810
|
+
console.warn(presignedUrl);
|
|
5811
|
+
}
|
|
5812
|
+
}
|
|
5583
5813
|
}
|
|
5584
5814
|
const targetLanguage = getLanguageCodePair(toLanguageCode);
|
|
5585
5815
|
return {
|
|
@@ -5910,9 +6140,7 @@ async function translateChunkWithFallback({
|
|
|
5910
6140
|
};
|
|
5911
6141
|
} catch (error) {
|
|
5912
6142
|
if (!shouldSplitChunkTranslationError(error) || chunk.cueCount <= 1) {
|
|
5913
|
-
|
|
5914
|
-
`Chunk ${chunk.id} failed for ${Math.round(chunk.startTime)}s-${Math.round(chunk.endTime)}s: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
5915
|
-
);
|
|
6143
|
+
wrapError(error, `Chunk ${chunk.id} failed for ${Math.round(chunk.startTime)}s-${Math.round(chunk.endTime)}s`);
|
|
5916
6144
|
}
|
|
5917
6145
|
const [leftChunk, rightChunk] = splitTranslationChunkAtMidpoint(chunk);
|
|
5918
6146
|
const [leftResult, rightResult] = await Promise.all([
|
|
@@ -6021,7 +6249,7 @@ async function uploadVttToS3({
|
|
|
6021
6249
|
expiresInSeconds: s3SignedUrlExpirySeconds ?? 86400
|
|
6022
6250
|
}, storageAdapter);
|
|
6023
6251
|
}
|
|
6024
|
-
async function translateCaptions(assetId,
|
|
6252
|
+
async function translateCaptions(assetId, trackId, toLanguageCode, options) {
|
|
6025
6253
|
"use workflow";
|
|
6026
6254
|
const {
|
|
6027
6255
|
provider = "openai",
|
|
@@ -6029,6 +6257,7 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
|
|
|
6029
6257
|
s3Endpoint: providedS3Endpoint,
|
|
6030
6258
|
s3Region: providedS3Region,
|
|
6031
6259
|
s3Bucket: providedS3Bucket,
|
|
6260
|
+
uploadToS3: uploadToS3Option,
|
|
6032
6261
|
uploadToMux: uploadToMuxOption,
|
|
6033
6262
|
storageAdapter,
|
|
6034
6263
|
credentials: providedCredentials,
|
|
@@ -6042,53 +6271,46 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
|
|
|
6042
6271
|
const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
|
|
6043
6272
|
const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
|
|
6044
6273
|
const uploadToMux = uploadToMuxOption !== false;
|
|
6274
|
+
const uploadToS3 = uploadToS3Option || uploadToMux;
|
|
6045
6275
|
const modelConfig = resolveLanguageModelConfig({
|
|
6046
6276
|
...options,
|
|
6047
6277
|
model,
|
|
6048
6278
|
provider
|
|
6049
6279
|
});
|
|
6050
|
-
if (
|
|
6051
|
-
throw new
|
|
6280
|
+
if (uploadToS3 && (!s3Endpoint || !s3Bucket || !effectiveStorageAdapter && (!s3AccessKeyId || !s3SecretAccessKey))) {
|
|
6281
|
+
throw new MuxAiError("Storage configuration is required for uploading. Provide s3Endpoint and s3Bucket. If no storageAdapter is supplied, also provide s3AccessKeyId and s3SecretAccessKey in options or set S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID, and S3_SECRET_ACCESS_KEY environment variables.", { type: "validation_error" });
|
|
6052
6282
|
}
|
|
6053
6283
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
6054
6284
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
6055
|
-
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
6056
6285
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
6057
6286
|
if (policy === "signed" && !signingContext) {
|
|
6058
|
-
throw new
|
|
6059
|
-
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables."
|
|
6287
|
+
throw new MuxAiError(
|
|
6288
|
+
"Signed playback ID requires signing credentials. Set MUX_SIGNING_KEY and MUX_PRIVATE_KEY environment variables.",
|
|
6289
|
+
{ type: "validation_error" }
|
|
6060
6290
|
);
|
|
6061
6291
|
}
|
|
6062
6292
|
const readyTextTracks = getReadyTextTracks(assetData);
|
|
6063
|
-
|
|
6064
|
-
throw new Error("No ready text tracks found for this asset");
|
|
6065
|
-
}
|
|
6066
|
-
let sourceTextTrack = readyTextTracks.find(
|
|
6067
|
-
(track) => track.text_type === "subtitles" && track.language_code === fromLanguageCode
|
|
6068
|
-
);
|
|
6069
|
-
if (!sourceTextTrack && isAudioOnly && readyTextTracks.length === 1) {
|
|
6070
|
-
sourceTextTrack = readyTextTracks[0];
|
|
6071
|
-
}
|
|
6293
|
+
const sourceTextTrack = readyTextTracks.find((t) => t.id === trackId);
|
|
6072
6294
|
if (!sourceTextTrack) {
|
|
6073
|
-
const
|
|
6074
|
-
|
|
6075
|
-
|
|
6076
|
-
|
|
6077
|
-
);
|
|
6078
|
-
}
|
|
6079
|
-
throw new Error(
|
|
6080
|
-
`No ready text track found with language code '${fromLanguageCode}' for this asset. Available languages: ${availableLanguages || "none"}`
|
|
6295
|
+
const availableTrackIds = readyTextTracks.map((t) => t.id).filter(Boolean).join(", ");
|
|
6296
|
+
throw new MuxAiError(
|
|
6297
|
+
`Track ${trackId} not found or not ready on asset ${assetId}. Available track IDs: ${availableTrackIds || "none"}.`,
|
|
6298
|
+
{ type: "validation_error" }
|
|
6081
6299
|
);
|
|
6082
6300
|
}
|
|
6083
|
-
|
|
6084
|
-
|
|
6301
|
+
const fromLanguageCode = sourceTextTrack.language_code;
|
|
6302
|
+
if (!fromLanguageCode) {
|
|
6303
|
+
throw new MuxAiError(
|
|
6304
|
+
`Track ${trackId} is missing language metadata. Cannot determine source language for translation.`,
|
|
6305
|
+
{ type: "validation_error" }
|
|
6306
|
+
);
|
|
6085
6307
|
}
|
|
6086
|
-
const vttUrl = await buildTranscriptUrl(playbackId,
|
|
6308
|
+
const vttUrl = await buildTranscriptUrl(playbackId, trackId, policy === "signed", credentials);
|
|
6087
6309
|
let vttContent;
|
|
6088
6310
|
try {
|
|
6089
6311
|
vttContent = await fetchVttFromMux(vttUrl);
|
|
6090
6312
|
} catch (error) {
|
|
6091
|
-
|
|
6313
|
+
wrapError(error, "Failed to fetch VTT content");
|
|
6092
6314
|
}
|
|
6093
6315
|
let translatedVtt;
|
|
6094
6316
|
let usage;
|
|
@@ -6106,7 +6328,7 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
|
|
|
6106
6328
|
translatedVtt = result.translatedVtt;
|
|
6107
6329
|
usage = result.usage;
|
|
6108
6330
|
} catch (error) {
|
|
6109
|
-
|
|
6331
|
+
wrapError(error, `Failed to translate VTT with ${modelConfig.provider}`);
|
|
6110
6332
|
}
|
|
6111
6333
|
const usageWithMetadata = usage ? {
|
|
6112
6334
|
...usage,
|
|
@@ -6116,50 +6338,43 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
|
|
|
6116
6338
|
} : void 0;
|
|
6117
6339
|
const sourceLanguage = getLanguageCodePair(fromLanguageCode);
|
|
6118
6340
|
const targetLanguage = getLanguageCodePair(toLanguageCode);
|
|
6119
|
-
if (!uploadToMux) {
|
|
6120
|
-
return {
|
|
6121
|
-
assetId,
|
|
6122
|
-
sourceLanguageCode: fromLanguageCode,
|
|
6123
|
-
targetLanguageCode: toLanguageCode,
|
|
6124
|
-
sourceLanguage,
|
|
6125
|
-
targetLanguage,
|
|
6126
|
-
originalVtt: vttContent,
|
|
6127
|
-
translatedVtt,
|
|
6128
|
-
usage: usageWithMetadata
|
|
6129
|
-
};
|
|
6130
|
-
}
|
|
6131
6341
|
let presignedUrl;
|
|
6132
|
-
try {
|
|
6133
|
-
presignedUrl = await uploadVttToS3({
|
|
6134
|
-
translatedVtt,
|
|
6135
|
-
assetId,
|
|
6136
|
-
fromLanguageCode,
|
|
6137
|
-
toLanguageCode,
|
|
6138
|
-
s3Endpoint,
|
|
6139
|
-
s3Region,
|
|
6140
|
-
s3Bucket,
|
|
6141
|
-
storageAdapter: effectiveStorageAdapter,
|
|
6142
|
-
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
6143
|
-
});
|
|
6144
|
-
} catch (error) {
|
|
6145
|
-
throw new Error(`Failed to upload VTT to S3: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
6146
|
-
}
|
|
6147
6342
|
let uploadedTrackId;
|
|
6148
|
-
|
|
6149
|
-
|
|
6150
|
-
|
|
6151
|
-
|
|
6152
|
-
|
|
6153
|
-
|
|
6154
|
-
|
|
6155
|
-
|
|
6156
|
-
|
|
6157
|
-
|
|
6158
|
-
|
|
6159
|
-
|
|
6343
|
+
if (uploadToS3) {
|
|
6344
|
+
try {
|
|
6345
|
+
presignedUrl = await uploadVttToS3({
|
|
6346
|
+
translatedVtt,
|
|
6347
|
+
assetId,
|
|
6348
|
+
fromLanguageCode,
|
|
6349
|
+
toLanguageCode,
|
|
6350
|
+
s3Endpoint,
|
|
6351
|
+
s3Region,
|
|
6352
|
+
s3Bucket,
|
|
6353
|
+
storageAdapter: effectiveStorageAdapter,
|
|
6354
|
+
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds
|
|
6355
|
+
});
|
|
6356
|
+
} catch (error) {
|
|
6357
|
+
wrapError(error, "Failed to upload VTT to S3");
|
|
6358
|
+
}
|
|
6359
|
+
if (uploadToMux) {
|
|
6360
|
+
try {
|
|
6361
|
+
const languageName = getLanguageName(toLanguageCode);
|
|
6362
|
+
const trackName = `${languageName} (auto-translated)`;
|
|
6363
|
+
uploadedTrackId = await createTextTrackOnMux(
|
|
6364
|
+
assetId,
|
|
6365
|
+
toLanguageCode,
|
|
6366
|
+
trackName,
|
|
6367
|
+
presignedUrl,
|
|
6368
|
+
credentials
|
|
6369
|
+
);
|
|
6370
|
+
} catch (error) {
|
|
6371
|
+
console.warn(`Failed to add track to Mux asset: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
6372
|
+
}
|
|
6373
|
+
}
|
|
6160
6374
|
}
|
|
6161
6375
|
return {
|
|
6162
6376
|
assetId,
|
|
6377
|
+
trackId,
|
|
6163
6378
|
sourceLanguageCode: fromLanguageCode,
|
|
6164
6379
|
targetLanguageCode: toLanguageCode,
|
|
6165
6380
|
sourceLanguage,
|
|
@@ -6172,6 +6387,7 @@ async function translateCaptions(assetId, fromLanguageCode, toLanguageCode, opti
|
|
|
6172
6387
|
};
|
|
6173
6388
|
}
|
|
6174
6389
|
export {
|
|
6390
|
+
MuxAiError,
|
|
6175
6391
|
WorkflowStorageClient,
|
|
6176
6392
|
createWorkflowStorageClient,
|
|
6177
6393
|
decryptFromWorkflow,
|
|
@@ -6179,6 +6395,7 @@ export {
|
|
|
6179
6395
|
primitives_exports as primitives,
|
|
6180
6396
|
setWorkflowCredentialsProvider,
|
|
6181
6397
|
version,
|
|
6182
|
-
workflows_exports as workflows
|
|
6398
|
+
workflows_exports as workflows,
|
|
6399
|
+
wrapError
|
|
6183
6400
|
};
|
|
6184
6401
|
//# sourceMappingURL=index.js.map
|