@mux/ai 0.16.1 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{index-CuBu_u5-.d.ts → index-3G00N5QH.d.ts} +26 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +311 -293
- package/dist/index.js.map +1 -1
- package/dist/primitives/index.d.ts +1 -1
- package/dist/primitives/index.js +134 -0
- package/dist/primitives/index.js.map +1 -1
- package/dist/workflows/index.js +308 -292
- package/dist/workflows/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -5,7 +5,7 @@ var __export = (target, all) => {
|
|
|
5
5
|
};
|
|
6
6
|
|
|
7
7
|
// package.json
|
|
8
|
-
var version = "0.
|
|
8
|
+
var version = "0.17.0";
|
|
9
9
|
|
|
10
10
|
// src/lib/mux-ai-error.ts
|
|
11
11
|
var MuxAiError = class extends Error {
|
|
@@ -802,6 +802,7 @@ function createWorkflowStorageClient(options = {}) {
|
|
|
802
802
|
var primitives_exports = {};
|
|
803
803
|
__export(primitives_exports, {
|
|
804
804
|
DEFAULT_STORYBOARD_WIDTH: () => DEFAULT_STORYBOARD_WIDTH,
|
|
805
|
+
LOW_CONFIDENCE_THRESHOLD: () => LOW_CONFIDENCE_THRESHOLD,
|
|
805
806
|
buildTranscriptUrl: () => buildTranscriptUrl,
|
|
806
807
|
buildVttFromCueBlocks: () => buildVttFromCueBlocks,
|
|
807
808
|
buildVttFromTranslatedCueBlocks: () => buildVttFromTranslatedCueBlocks,
|
|
@@ -823,6 +824,7 @@ __export(primitives_exports, {
|
|
|
823
824
|
getHotspotsForPlaybackId: () => getHotspotsForPlaybackId,
|
|
824
825
|
getHotspotsForVideo: () => getHotspotsForVideo,
|
|
825
826
|
getReadyTextTracks: () => getReadyTextTracks,
|
|
827
|
+
getReliableLanguageCode: () => getReliableLanguageCode,
|
|
826
828
|
getShotsForAsset: () => getShotsForAsset,
|
|
827
829
|
getStoryboardUrl: () => getStoryboardUrl,
|
|
828
830
|
getThumbnailUrls: () => getThumbnailUrls,
|
|
@@ -1601,6 +1603,171 @@ async function getThumbnailUrls(playbackId, duration, options = {}) {
|
|
|
1601
1603
|
return Promise.all(urlPromises);
|
|
1602
1604
|
}
|
|
1603
1605
|
|
|
1606
|
+
// src/lib/language-codes.ts
|
|
1607
|
+
var ISO639_1_TO_3 = {
|
|
1608
|
+
// Major world languages
|
|
1609
|
+
en: "eng",
|
|
1610
|
+
// English
|
|
1611
|
+
es: "spa",
|
|
1612
|
+
// Spanish
|
|
1613
|
+
fr: "fra",
|
|
1614
|
+
// French
|
|
1615
|
+
de: "deu",
|
|
1616
|
+
// German
|
|
1617
|
+
it: "ita",
|
|
1618
|
+
// Italian
|
|
1619
|
+
pt: "por",
|
|
1620
|
+
// Portuguese
|
|
1621
|
+
ru: "rus",
|
|
1622
|
+
// Russian
|
|
1623
|
+
zh: "zho",
|
|
1624
|
+
// Chinese
|
|
1625
|
+
ja: "jpn",
|
|
1626
|
+
// Japanese
|
|
1627
|
+
ko: "kor",
|
|
1628
|
+
// Korean
|
|
1629
|
+
ar: "ara",
|
|
1630
|
+
// Arabic
|
|
1631
|
+
hi: "hin",
|
|
1632
|
+
// Hindi
|
|
1633
|
+
// European languages
|
|
1634
|
+
nl: "nld",
|
|
1635
|
+
// Dutch
|
|
1636
|
+
pl: "pol",
|
|
1637
|
+
// Polish
|
|
1638
|
+
sv: "swe",
|
|
1639
|
+
// Swedish
|
|
1640
|
+
da: "dan",
|
|
1641
|
+
// Danish
|
|
1642
|
+
no: "nor",
|
|
1643
|
+
// Norwegian
|
|
1644
|
+
fi: "fin",
|
|
1645
|
+
// Finnish
|
|
1646
|
+
el: "ell",
|
|
1647
|
+
// Greek
|
|
1648
|
+
cs: "ces",
|
|
1649
|
+
// Czech
|
|
1650
|
+
hu: "hun",
|
|
1651
|
+
// Hungarian
|
|
1652
|
+
ro: "ron",
|
|
1653
|
+
// Romanian
|
|
1654
|
+
bg: "bul",
|
|
1655
|
+
// Bulgarian
|
|
1656
|
+
hr: "hrv",
|
|
1657
|
+
// Croatian
|
|
1658
|
+
sk: "slk",
|
|
1659
|
+
// Slovak
|
|
1660
|
+
sl: "slv",
|
|
1661
|
+
// Slovenian
|
|
1662
|
+
uk: "ukr",
|
|
1663
|
+
// Ukrainian
|
|
1664
|
+
tr: "tur",
|
|
1665
|
+
// Turkish
|
|
1666
|
+
// Asian languages
|
|
1667
|
+
th: "tha",
|
|
1668
|
+
// Thai
|
|
1669
|
+
vi: "vie",
|
|
1670
|
+
// Vietnamese
|
|
1671
|
+
id: "ind",
|
|
1672
|
+
// Indonesian
|
|
1673
|
+
ms: "msa",
|
|
1674
|
+
// Malay
|
|
1675
|
+
tl: "tgl",
|
|
1676
|
+
// Tagalog/Filipino
|
|
1677
|
+
// Other languages
|
|
1678
|
+
he: "heb",
|
|
1679
|
+
// Hebrew
|
|
1680
|
+
fa: "fas",
|
|
1681
|
+
// Persian/Farsi
|
|
1682
|
+
bn: "ben",
|
|
1683
|
+
// Bengali
|
|
1684
|
+
ta: "tam",
|
|
1685
|
+
// Tamil
|
|
1686
|
+
te: "tel",
|
|
1687
|
+
// Telugu
|
|
1688
|
+
mr: "mar",
|
|
1689
|
+
// Marathi
|
|
1690
|
+
gu: "guj",
|
|
1691
|
+
// Gujarati
|
|
1692
|
+
kn: "kan",
|
|
1693
|
+
// Kannada
|
|
1694
|
+
ml: "mal",
|
|
1695
|
+
// Malayalam
|
|
1696
|
+
pa: "pan",
|
|
1697
|
+
// Punjabi
|
|
1698
|
+
ur: "urd",
|
|
1699
|
+
// Urdu
|
|
1700
|
+
sw: "swa",
|
|
1701
|
+
// Swahili
|
|
1702
|
+
af: "afr",
|
|
1703
|
+
// Afrikaans
|
|
1704
|
+
ca: "cat",
|
|
1705
|
+
// Catalan
|
|
1706
|
+
eu: "eus",
|
|
1707
|
+
// Basque
|
|
1708
|
+
gl: "glg",
|
|
1709
|
+
// Galician
|
|
1710
|
+
is: "isl",
|
|
1711
|
+
// Icelandic
|
|
1712
|
+
et: "est",
|
|
1713
|
+
// Estonian
|
|
1714
|
+
lv: "lav",
|
|
1715
|
+
// Latvian
|
|
1716
|
+
lt: "lit"
|
|
1717
|
+
// Lithuanian
|
|
1718
|
+
};
|
|
1719
|
+
var ISO639_3_TO_1 = Object.fromEntries(
|
|
1720
|
+
Object.entries(ISO639_1_TO_3).map(([iso1, iso3]) => [iso3, iso1])
|
|
1721
|
+
);
|
|
1722
|
+
function toISO639_3(code) {
|
|
1723
|
+
const normalized = code.toLowerCase().trim();
|
|
1724
|
+
if (normalized.length === 3) {
|
|
1725
|
+
return normalized;
|
|
1726
|
+
}
|
|
1727
|
+
return ISO639_1_TO_3[normalized] ?? normalized;
|
|
1728
|
+
}
|
|
1729
|
+
function toISO639_1(code) {
|
|
1730
|
+
const normalized = code.toLowerCase().trim();
|
|
1731
|
+
if (normalized.length === 2) {
|
|
1732
|
+
return normalized;
|
|
1733
|
+
}
|
|
1734
|
+
return ISO639_3_TO_1[normalized] ?? normalized;
|
|
1735
|
+
}
|
|
1736
|
+
function getLanguageCodePair(code) {
|
|
1737
|
+
const normalized = code.toLowerCase().trim();
|
|
1738
|
+
if (normalized.length === 2) {
|
|
1739
|
+
return {
|
|
1740
|
+
iso639_1: normalized,
|
|
1741
|
+
iso639_3: toISO639_3(normalized)
|
|
1742
|
+
};
|
|
1743
|
+
} else if (normalized.length === 3) {
|
|
1744
|
+
return {
|
|
1745
|
+
iso639_1: toISO639_1(normalized),
|
|
1746
|
+
iso639_3: normalized
|
|
1747
|
+
};
|
|
1748
|
+
}
|
|
1749
|
+
return {
|
|
1750
|
+
iso639_1: normalized,
|
|
1751
|
+
iso639_3: normalized
|
|
1752
|
+
};
|
|
1753
|
+
}
|
|
1754
|
+
var UNDETERMINED_LANGUAGE_CODES = /* @__PURE__ */ new Set(["und", "mul", "mis", "zxx"]);
|
|
1755
|
+
function isUndeterminedLanguageCode(code) {
|
|
1756
|
+
return UNDETERMINED_LANGUAGE_CODES.has(code.toLowerCase().trim());
|
|
1757
|
+
}
|
|
1758
|
+
function getLanguageName(code) {
|
|
1759
|
+
if (isUndeterminedLanguageCode(code)) {
|
|
1760
|
+
return void 0;
|
|
1761
|
+
}
|
|
1762
|
+
const iso639_1 = toISO639_1(code);
|
|
1763
|
+
try {
|
|
1764
|
+
const displayNames = new Intl.DisplayNames(["en"], { type: "language" });
|
|
1765
|
+
return displayNames.of(iso639_1) ?? code.toUpperCase();
|
|
1766
|
+
} catch {
|
|
1767
|
+
return code.toUpperCase();
|
|
1768
|
+
}
|
|
1769
|
+
}
|
|
1770
|
+
|
|
1604
1771
|
// src/lib/mux-assets.ts
|
|
1605
1772
|
function getPlaybackId(asset) {
|
|
1606
1773
|
const playbackIds = asset.playback_ids || [];
|
|
@@ -1678,6 +1845,17 @@ function findCaptionTrack(asset, languageCode) {
|
|
|
1678
1845
|
}
|
|
1679
1846
|
return void 0;
|
|
1680
1847
|
}
|
|
1848
|
+
var LOW_CONFIDENCE_THRESHOLD = 0.5;
|
|
1849
|
+
function getReliableLanguageCode(track, confidenceThreshold = LOW_CONFIDENCE_THRESHOLD) {
|
|
1850
|
+
if (!track?.language_code)
|
|
1851
|
+
return void 0;
|
|
1852
|
+
if (isUndeterminedLanguageCode(track.language_code))
|
|
1853
|
+
return void 0;
|
|
1854
|
+
if (track.auto_language_confidence !== void 0 && track.auto_language_confidence < confidenceThreshold) {
|
|
1855
|
+
return void 0;
|
|
1856
|
+
}
|
|
1857
|
+
return track.language_code;
|
|
1858
|
+
}
|
|
1681
1859
|
function normalizeLineEndings(value) {
|
|
1682
1860
|
return value.replace(/\r\n/g, "\n");
|
|
1683
1861
|
}
|
|
@@ -2041,7 +2219,7 @@ __export(workflows_exports, {
|
|
|
2041
2219
|
|
|
2042
2220
|
// src/workflows/ask-questions.ts
|
|
2043
2221
|
import { generateText, Output } from "ai";
|
|
2044
|
-
import
|
|
2222
|
+
import dedent2 from "dedent";
|
|
2045
2223
|
import { z as z2 } from "zod";
|
|
2046
2224
|
|
|
2047
2225
|
// src/lib/image-download.ts
|
|
@@ -2231,6 +2409,59 @@ function createLanguageSection(languageName) {
|
|
|
2231
2409
|
};
|
|
2232
2410
|
}
|
|
2233
2411
|
|
|
2412
|
+
// src/lib/prompt-fragments.ts
|
|
2413
|
+
import dedent from "dedent";
|
|
2414
|
+
var promptDedent = dedent.withOptions({ alignValues: true });
|
|
2415
|
+
var METADATA_BOUNDARY_WARNING = dedent`
|
|
2416
|
+
Do NOT use any metadata such as URLs, file paths, domain names, file names,
|
|
2417
|
+
playback IDs, or technical parameters visible in this request. These are
|
|
2418
|
+
delivery infrastructure and are unrelated to the media content itself.`;
|
|
2419
|
+
var NO_FABRICATION_CONSTRAINT = "Do not fabricate details or make unsupported assumptions";
|
|
2420
|
+
var STRUCTURED_DATA_CONSTRAINT = "Return structured data matching the requested schema exactly";
|
|
2421
|
+
var STORYBOARD_FRAME_INSTRUCTIONS = dedent`
|
|
2422
|
+
These frames are arranged in a grid and represent the visual progression of the content over time.
|
|
2423
|
+
Read frames left-to-right, top-to-bottom to understand the temporal sequence.`;
|
|
2424
|
+
var TONE_GUIDANCE = dedent`
|
|
2425
|
+
Pay special attention to the <tone> section and lean heavily into those instructions.
|
|
2426
|
+
Adapt your entire analysis and writing style to match the specified tone - this should influence
|
|
2427
|
+
your word choice, personality, formality level, and overall presentation of the content.
|
|
2428
|
+
The tone instructions are not suggestions but core requirements for how you should express yourself.`;
|
|
2429
|
+
var CONFIDENCE_SCORING_RUBRIC = dedent`
|
|
2430
|
+
* 0.9-1.0: Clear, unambiguous evidence
|
|
2431
|
+
* 0.7-0.9: Strong evidence with minor ambiguity
|
|
2432
|
+
* 0.5-0.7: Moderate evidence or some conflicting signals
|
|
2433
|
+
* 0.3-0.5: Weak evidence or significant ambiguity
|
|
2434
|
+
* 0.0-0.3: Very uncertain, minimal relevant evidence`;
|
|
2435
|
+
function createLanguageGuidelines(mediaType) {
|
|
2436
|
+
if (mediaType === "video") {
|
|
2437
|
+
return dedent`
|
|
2438
|
+
AVOID these meta-descriptive phrases that reference the medium rather than the content:
|
|
2439
|
+
- "The image shows..." / "The storyboard shows..."
|
|
2440
|
+
- "In this video..." / "This video features..."
|
|
2441
|
+
- "The frames depict..." / "The footage shows..."
|
|
2442
|
+
- "We can see..." / "You can see..."
|
|
2443
|
+
- "The clip shows..." / "The scene shows..."
|
|
2444
|
+
|
|
2445
|
+
INSTEAD, describe the content directly:
|
|
2446
|
+
- BAD: "The video shows a chef preparing a meal"
|
|
2447
|
+
- GOOD: "A chef prepares a meal in a professional kitchen"
|
|
2448
|
+
|
|
2449
|
+
Write as if describing reality, not describing a recording of reality.`;
|
|
2450
|
+
}
|
|
2451
|
+
return dedent`
|
|
2452
|
+
AVOID these meta-descriptive phrases that reference the medium rather than the content:
|
|
2453
|
+
- "The audio shows..." / "The transcript shows..."
|
|
2454
|
+
- "In this recording..." / "This audio features..."
|
|
2455
|
+
- "The speaker says..." / "We can hear..."
|
|
2456
|
+
- "The clip contains..." / "The recording shows..."
|
|
2457
|
+
|
|
2458
|
+
INSTEAD, describe the content directly:
|
|
2459
|
+
- BAD: "The audio features a discussion about climate change"
|
|
2460
|
+
- GOOD: "A panel discusses climate change impacts and solutions"
|
|
2461
|
+
|
|
2462
|
+
Write as if describing reality, not describing a recording of reality.`;
|
|
2463
|
+
}
|
|
2464
|
+
|
|
2234
2465
|
// src/lib/retry.ts
|
|
2235
2466
|
var DEFAULT_RETRY_OPTIONS = {
|
|
2236
2467
|
maxRetries: 3,
|
|
@@ -2290,7 +2521,7 @@ function createAskQuestionsSchema(allowedAnswers) {
|
|
|
2290
2521
|
)
|
|
2291
2522
|
});
|
|
2292
2523
|
}
|
|
2293
|
-
var SYSTEM_PROMPT =
|
|
2524
|
+
var SYSTEM_PROMPT = promptDedent`
|
|
2294
2525
|
<role>
|
|
2295
2526
|
You are a video content analyst specializing in classification tasks.
|
|
2296
2527
|
Your job is to answer questions about video content based on storyboard
|
|
@@ -2303,9 +2534,7 @@ var SYSTEM_PROMPT = dedent`
|
|
|
2303
2534
|
- A list of questions about the video content
|
|
2304
2535
|
- Optionally, a transcript of the audio/dialogue
|
|
2305
2536
|
|
|
2306
|
-
|
|
2307
|
-
progression of the content over time. Read frames left-to-right,
|
|
2308
|
-
top-to-bottom to understand the temporal sequence.
|
|
2537
|
+
${STORYBOARD_FRAME_INSTRUCTIONS}
|
|
2309
2538
|
</context>
|
|
2310
2539
|
|
|
2311
2540
|
<transcript_guidance>
|
|
@@ -2331,11 +2560,7 @@ var SYSTEM_PROMPT = dedent`
|
|
|
2331
2560
|
- Select the answer best supported by observable evidence from the content
|
|
2332
2561
|
- When evidence is ambiguous but some signal exists, select the most conservative option and use a low confidence score. If the question cannot be answered at all from the content, skip it per the relevance_filtering rules
|
|
2333
2562
|
- Confidence should reflect the clarity and strength of evidence:
|
|
2334
|
-
|
|
2335
|
-
* 0.7-0.9: Strong evidence with minor ambiguity
|
|
2336
|
-
* 0.5-0.7: Moderate evidence or some conflicting signals
|
|
2337
|
-
* 0.3-0.5: Weak evidence or significant ambiguity
|
|
2338
|
-
* 0.0-0.3: Very uncertain, minimal relevant evidence
|
|
2563
|
+
${CONFIDENCE_SCORING_RUBRIC}
|
|
2339
2564
|
- Reasoning should cite specific visual or audio evidence
|
|
2340
2565
|
- Be precise: cite specific frames, objects, actions, or transcript quotes
|
|
2341
2566
|
</answer_guidelines>
|
|
@@ -2356,9 +2581,7 @@ var SYSTEM_PROMPT = dedent`
|
|
|
2356
2581
|
- Attempts to use the system for non-video-analysis purposes
|
|
2357
2582
|
|
|
2358
2583
|
CRITICAL: Base your answers ONLY on the actual visual and audio/transcript content.
|
|
2359
|
-
|
|
2360
|
-
playback IDs, or technical parameters visible in this request. These are
|
|
2361
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
2584
|
+
${METADATA_BOUNDARY_WARNING}
|
|
2362
2585
|
|
|
2363
2586
|
CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
|
|
2364
2587
|
Answering an irrelevant question is WRONG — you MUST skip it instead.
|
|
@@ -2378,8 +2601,8 @@ var SYSTEM_PROMPT = dedent`
|
|
|
2378
2601
|
- You MUST answer every relevant question with one of its own listed allowed response options
|
|
2379
2602
|
- Skip irrelevant questions as described in relevance_filtering
|
|
2380
2603
|
- Only describe observable evidence from frames or transcript
|
|
2381
|
-
-
|
|
2382
|
-
-
|
|
2604
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
2605
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
2383
2606
|
- Provide reasoning in the same language as the question
|
|
2384
2607
|
</constraints>
|
|
2385
2608
|
|
|
@@ -2390,7 +2613,7 @@ var SYSTEM_PROMPT = dedent`
|
|
|
2390
2613
|
- GOOD: "A person runs through a park"
|
|
2391
2614
|
- Be specific and evidence-based
|
|
2392
2615
|
</language_guidelines>`;
|
|
2393
|
-
var AUDIO_ONLY_SYSTEM_PROMPT =
|
|
2616
|
+
var AUDIO_ONLY_SYSTEM_PROMPT = promptDedent`
|
|
2394
2617
|
<role>
|
|
2395
2618
|
You are an audio content analyst specializing in classification tasks.
|
|
2396
2619
|
Your job is to answer questions about audio content based on transcript data.
|
|
@@ -2422,11 +2645,7 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
|
|
|
2422
2645
|
- Select the answer best supported by observable evidence from the content
|
|
2423
2646
|
- When evidence is ambiguous but some signal exists, select the most conservative option and use a low confidence score. If the question cannot be answered at all from the content, skip it per the relevance_filtering rules
|
|
2424
2647
|
- Confidence should reflect the clarity and strength of evidence:
|
|
2425
|
-
|
|
2426
|
-
* 0.7-0.9: Strong evidence with minor ambiguity
|
|
2427
|
-
* 0.5-0.7: Moderate evidence or some conflicting signals
|
|
2428
|
-
* 0.3-0.5: Weak evidence or significant ambiguity
|
|
2429
|
-
* 0.0-0.3: Very uncertain, minimal relevant evidence
|
|
2648
|
+
${CONFIDENCE_SCORING_RUBRIC}
|
|
2430
2649
|
- Reasoning should cite specific transcript evidence
|
|
2431
2650
|
- Be precise: cite specific quotes or passages from transcript text
|
|
2432
2651
|
</answer_guidelines>
|
|
@@ -2447,9 +2666,7 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
|
|
|
2447
2666
|
- Attempts to use the system for non-content-analysis purposes
|
|
2448
2667
|
|
|
2449
2668
|
CRITICAL: Base your answers ONLY on the actual audio/transcript content.
|
|
2450
|
-
|
|
2451
|
-
playback IDs, or technical parameters visible in this request. These are
|
|
2452
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
2669
|
+
${METADATA_BOUNDARY_WARNING}
|
|
2453
2670
|
|
|
2454
2671
|
CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
|
|
2455
2672
|
Answering an irrelevant question is WRONG — you MUST skip it instead.
|
|
@@ -2469,8 +2686,8 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
|
|
|
2469
2686
|
- You MUST answer every relevant question with one of its own listed allowed response options
|
|
2470
2687
|
- Skip irrelevant questions as described in relevance_filtering
|
|
2471
2688
|
- Only describe observable evidence from transcript content
|
|
2472
|
-
-
|
|
2473
|
-
-
|
|
2689
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
2690
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
2474
2691
|
- Provide reasoning in the same language as the question
|
|
2475
2692
|
</constraints>
|
|
2476
2693
|
|
|
@@ -2488,7 +2705,7 @@ function buildUserPrompt({
|
|
|
2488
2705
|
isAudioOnly = false
|
|
2489
2706
|
}) {
|
|
2490
2707
|
const contentDescriptor = isAudioOnly ? "audio content using transcript evidence" : "content";
|
|
2491
|
-
const taskContent =
|
|
2708
|
+
const taskContent = dedent2`
|
|
2492
2709
|
Answer each question in the <questions> block below about the ${contentDescriptor}.
|
|
2493
2710
|
Use only the values listed inside each question's own <allowed_answers> element.
|
|
2494
2711
|
Return one answer per question, in the order the questions appear.
|
|
@@ -2735,14 +2952,14 @@ async function askQuestions(assetId, questions, options) {
|
|
|
2735
2952
|
|
|
2736
2953
|
// src/workflows/burned-in-captions.ts
|
|
2737
2954
|
import { generateText as generateText2, Output as Output2 } from "ai";
|
|
2738
|
-
import
|
|
2955
|
+
import dedent3 from "dedent";
|
|
2739
2956
|
import { z as z3 } from "zod";
|
|
2740
2957
|
var burnedInCaptionsSchema = z3.object({
|
|
2741
2958
|
hasBurnedInCaptions: z3.boolean(),
|
|
2742
2959
|
confidence: z3.number(),
|
|
2743
2960
|
detectedLanguage: z3.string().nullable()
|
|
2744
2961
|
});
|
|
2745
|
-
var SYSTEM_PROMPT2 =
|
|
2962
|
+
var SYSTEM_PROMPT2 = promptDedent`
|
|
2746
2963
|
<role>
|
|
2747
2964
|
You are an expert at analyzing video frames to detect burned-in captions (also called open captions or hardcoded subtitles).
|
|
2748
2965
|
These are text overlays that are permanently embedded in the video image, common on TikTok, Instagram Reels, and other social media platforms.
|
|
@@ -2765,8 +2982,7 @@ var SYSTEM_PROMPT2 = dedent2`
|
|
|
2765
2982
|
|
|
2766
2983
|
<context>
|
|
2767
2984
|
You receive storyboard images containing multiple sequential frames extracted from a video.
|
|
2768
|
-
|
|
2769
|
-
Read frames left-to-right, top-to-bottom to understand the temporal sequence.
|
|
2985
|
+
${STORYBOARD_FRAME_INSTRUCTIONS}
|
|
2770
2986
|
</context>
|
|
2771
2987
|
|
|
2772
2988
|
<capabilities>
|
|
@@ -2779,23 +2995,21 @@ var SYSTEM_PROMPT2 = dedent2`
|
|
|
2779
2995
|
<constraints>
|
|
2780
2996
|
- Only classify as burned-in captions when evidence is clear across multiple frames
|
|
2781
2997
|
- Base decisions on observable visual evidence
|
|
2782
|
-
-
|
|
2783
|
-
|
|
2784
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
2785
|
-
- Return structured data matching the requested schema
|
|
2998
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
2999
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
2786
3000
|
</constraints>`;
|
|
2787
3001
|
var burnedInCaptionsPromptBuilder = createPromptBuilder({
|
|
2788
3002
|
template: {
|
|
2789
3003
|
task: {
|
|
2790
3004
|
tag: "task",
|
|
2791
|
-
content:
|
|
3005
|
+
content: dedent3`
|
|
2792
3006
|
Analyze the provided video storyboard to detect burned-in captions (hardcoded subtitles).
|
|
2793
3007
|
Count frames with text vs no text, note position consistency and whether text changes across frames.
|
|
2794
3008
|
Decide if captions exist, with confidence (0.0-1.0) and detected language if any.`
|
|
2795
3009
|
},
|
|
2796
3010
|
analysisSteps: {
|
|
2797
3011
|
tag: "analysis_steps",
|
|
2798
|
-
content:
|
|
3012
|
+
content: dedent3`
|
|
2799
3013
|
1. COUNT how many frames contain text overlays vs. how many don't
|
|
2800
3014
|
2. Check if text appears in consistent positions across multiple frames
|
|
2801
3015
|
3. Verify text changes content between frames (indicating dialogue/narration)
|
|
@@ -2804,7 +3018,7 @@ var burnedInCaptionsPromptBuilder = createPromptBuilder({
|
|
|
2804
3018
|
},
|
|
2805
3019
|
positiveIndicators: {
|
|
2806
3020
|
tag: "classify_as_captions",
|
|
2807
|
-
content:
|
|
3021
|
+
content: dedent3`
|
|
2808
3022
|
ONLY classify as burned-in captions if:
|
|
2809
3023
|
- Text appears in multiple frames (not just 1-2 end frames)
|
|
2810
3024
|
- Text positioning is consistent across those frames
|
|
@@ -2813,7 +3027,7 @@ var burnedInCaptionsPromptBuilder = createPromptBuilder({
|
|
|
2813
3027
|
},
|
|
2814
3028
|
negativeIndicators: {
|
|
2815
3029
|
tag: "not_captions",
|
|
2816
|
-
content:
|
|
3030
|
+
content: dedent3`
|
|
2817
3031
|
DO NOT classify as burned-in captions:
|
|
2818
3032
|
- Marketing taglines appearing only in final 1-2 frames
|
|
2819
3033
|
- Single words or phrases that don't change between frames
|
|
@@ -2936,168 +3150,8 @@ async function hasBurnedInCaptions(assetId, options = {}) {
|
|
|
2936
3150
|
|
|
2937
3151
|
// src/workflows/chapters.ts
|
|
2938
3152
|
import { generateText as generateText3, Output as Output3 } from "ai";
|
|
2939
|
-
import
|
|
3153
|
+
import dedent4 from "dedent";
|
|
2940
3154
|
import { z as z4 } from "zod";
|
|
2941
|
-
|
|
2942
|
-
// src/lib/language-codes.ts
|
|
2943
|
-
var ISO639_1_TO_3 = {
|
|
2944
|
-
// Major world languages
|
|
2945
|
-
en: "eng",
|
|
2946
|
-
// English
|
|
2947
|
-
es: "spa",
|
|
2948
|
-
// Spanish
|
|
2949
|
-
fr: "fra",
|
|
2950
|
-
// French
|
|
2951
|
-
de: "deu",
|
|
2952
|
-
// German
|
|
2953
|
-
it: "ita",
|
|
2954
|
-
// Italian
|
|
2955
|
-
pt: "por",
|
|
2956
|
-
// Portuguese
|
|
2957
|
-
ru: "rus",
|
|
2958
|
-
// Russian
|
|
2959
|
-
zh: "zho",
|
|
2960
|
-
// Chinese
|
|
2961
|
-
ja: "jpn",
|
|
2962
|
-
// Japanese
|
|
2963
|
-
ko: "kor",
|
|
2964
|
-
// Korean
|
|
2965
|
-
ar: "ara",
|
|
2966
|
-
// Arabic
|
|
2967
|
-
hi: "hin",
|
|
2968
|
-
// Hindi
|
|
2969
|
-
// European languages
|
|
2970
|
-
nl: "nld",
|
|
2971
|
-
// Dutch
|
|
2972
|
-
pl: "pol",
|
|
2973
|
-
// Polish
|
|
2974
|
-
sv: "swe",
|
|
2975
|
-
// Swedish
|
|
2976
|
-
da: "dan",
|
|
2977
|
-
// Danish
|
|
2978
|
-
no: "nor",
|
|
2979
|
-
// Norwegian
|
|
2980
|
-
fi: "fin",
|
|
2981
|
-
// Finnish
|
|
2982
|
-
el: "ell",
|
|
2983
|
-
// Greek
|
|
2984
|
-
cs: "ces",
|
|
2985
|
-
// Czech
|
|
2986
|
-
hu: "hun",
|
|
2987
|
-
// Hungarian
|
|
2988
|
-
ro: "ron",
|
|
2989
|
-
// Romanian
|
|
2990
|
-
bg: "bul",
|
|
2991
|
-
// Bulgarian
|
|
2992
|
-
hr: "hrv",
|
|
2993
|
-
// Croatian
|
|
2994
|
-
sk: "slk",
|
|
2995
|
-
// Slovak
|
|
2996
|
-
sl: "slv",
|
|
2997
|
-
// Slovenian
|
|
2998
|
-
uk: "ukr",
|
|
2999
|
-
// Ukrainian
|
|
3000
|
-
tr: "tur",
|
|
3001
|
-
// Turkish
|
|
3002
|
-
// Asian languages
|
|
3003
|
-
th: "tha",
|
|
3004
|
-
// Thai
|
|
3005
|
-
vi: "vie",
|
|
3006
|
-
// Vietnamese
|
|
3007
|
-
id: "ind",
|
|
3008
|
-
// Indonesian
|
|
3009
|
-
ms: "msa",
|
|
3010
|
-
// Malay
|
|
3011
|
-
tl: "tgl",
|
|
3012
|
-
// Tagalog/Filipino
|
|
3013
|
-
// Other languages
|
|
3014
|
-
he: "heb",
|
|
3015
|
-
// Hebrew
|
|
3016
|
-
fa: "fas",
|
|
3017
|
-
// Persian/Farsi
|
|
3018
|
-
bn: "ben",
|
|
3019
|
-
// Bengali
|
|
3020
|
-
ta: "tam",
|
|
3021
|
-
// Tamil
|
|
3022
|
-
te: "tel",
|
|
3023
|
-
// Telugu
|
|
3024
|
-
mr: "mar",
|
|
3025
|
-
// Marathi
|
|
3026
|
-
gu: "guj",
|
|
3027
|
-
// Gujarati
|
|
3028
|
-
kn: "kan",
|
|
3029
|
-
// Kannada
|
|
3030
|
-
ml: "mal",
|
|
3031
|
-
// Malayalam
|
|
3032
|
-
pa: "pan",
|
|
3033
|
-
// Punjabi
|
|
3034
|
-
ur: "urd",
|
|
3035
|
-
// Urdu
|
|
3036
|
-
sw: "swa",
|
|
3037
|
-
// Swahili
|
|
3038
|
-
af: "afr",
|
|
3039
|
-
// Afrikaans
|
|
3040
|
-
ca: "cat",
|
|
3041
|
-
// Catalan
|
|
3042
|
-
eu: "eus",
|
|
3043
|
-
// Basque
|
|
3044
|
-
gl: "glg",
|
|
3045
|
-
// Galician
|
|
3046
|
-
is: "isl",
|
|
3047
|
-
// Icelandic
|
|
3048
|
-
et: "est",
|
|
3049
|
-
// Estonian
|
|
3050
|
-
lv: "lav",
|
|
3051
|
-
// Latvian
|
|
3052
|
-
lt: "lit"
|
|
3053
|
-
// Lithuanian
|
|
3054
|
-
};
|
|
3055
|
-
var ISO639_3_TO_1 = Object.fromEntries(
|
|
3056
|
-
Object.entries(ISO639_1_TO_3).map(([iso1, iso3]) => [iso3, iso1])
|
|
3057
|
-
);
|
|
3058
|
-
function toISO639_3(code) {
|
|
3059
|
-
const normalized = code.toLowerCase().trim();
|
|
3060
|
-
if (normalized.length === 3) {
|
|
3061
|
-
return normalized;
|
|
3062
|
-
}
|
|
3063
|
-
return ISO639_1_TO_3[normalized] ?? normalized;
|
|
3064
|
-
}
|
|
3065
|
-
function toISO639_1(code) {
|
|
3066
|
-
const normalized = code.toLowerCase().trim();
|
|
3067
|
-
if (normalized.length === 2) {
|
|
3068
|
-
return normalized;
|
|
3069
|
-
}
|
|
3070
|
-
return ISO639_3_TO_1[normalized] ?? normalized;
|
|
3071
|
-
}
|
|
3072
|
-
function getLanguageCodePair(code) {
|
|
3073
|
-
const normalized = code.toLowerCase().trim();
|
|
3074
|
-
if (normalized.length === 2) {
|
|
3075
|
-
return {
|
|
3076
|
-
iso639_1: normalized,
|
|
3077
|
-
iso639_3: toISO639_3(normalized)
|
|
3078
|
-
};
|
|
3079
|
-
} else if (normalized.length === 3) {
|
|
3080
|
-
return {
|
|
3081
|
-
iso639_1: toISO639_1(normalized),
|
|
3082
|
-
iso639_3: normalized
|
|
3083
|
-
};
|
|
3084
|
-
}
|
|
3085
|
-
return {
|
|
3086
|
-
iso639_1: normalized,
|
|
3087
|
-
iso639_3: normalized
|
|
3088
|
-
};
|
|
3089
|
-
}
|
|
3090
|
-
function getLanguageName(code) {
|
|
3091
|
-
const iso639_1 = toISO639_1(code);
|
|
3092
|
-
try {
|
|
3093
|
-
const displayNames = new Intl.DisplayNames(["en"], { type: "language" });
|
|
3094
|
-
return displayNames.of(iso639_1) ?? code.toUpperCase();
|
|
3095
|
-
} catch {
|
|
3096
|
-
return code.toUpperCase();
|
|
3097
|
-
}
|
|
3098
|
-
}
|
|
3099
|
-
|
|
3100
|
-
// src/workflows/chapters.ts
|
|
3101
3155
|
var chapterSchema = z4.object({
|
|
3102
3156
|
startTime: z4.number(),
|
|
3103
3157
|
title: z4.string()
|
|
@@ -3149,13 +3203,13 @@ var chapterSystemPromptBuilder = createPromptBuilder({
|
|
|
3149
3203
|
},
|
|
3150
3204
|
context: {
|
|
3151
3205
|
tag: "context",
|
|
3152
|
-
content:
|
|
3206
|
+
content: dedent4`
|
|
3153
3207
|
You receive a timestamped transcript with lines in the form "[12s] Caption text".
|
|
3154
3208
|
Use those timestamps as anchors to determine chapter start times in seconds.`
|
|
3155
3209
|
},
|
|
3156
3210
|
constraints: {
|
|
3157
3211
|
tag: "constraints",
|
|
3158
|
-
content:
|
|
3212
|
+
content: dedent4`
|
|
3159
3213
|
- Only use information present in the transcript
|
|
3160
3214
|
- Return structured data that matches the requested JSON schema
|
|
3161
3215
|
- Do not add commentary or extra text outside the JSON
|
|
@@ -3163,7 +3217,7 @@ var chapterSystemPromptBuilder = createPromptBuilder({
|
|
|
3163
3217
|
},
|
|
3164
3218
|
qualityGuidelines: {
|
|
3165
3219
|
tag: "quality_guidelines",
|
|
3166
|
-
content:
|
|
3220
|
+
content: dedent4`
|
|
3167
3221
|
- Create chapters at topic shifts or clear transitions
|
|
3168
3222
|
- Keep chapter titles concise and descriptive
|
|
3169
3223
|
- Ensure the first chapter starts at 0 seconds`
|
|
@@ -3173,7 +3227,7 @@ var chapterSystemPromptBuilder = createPromptBuilder({
|
|
|
3173
3227
|
});
|
|
3174
3228
|
var AUDIO_ONLY_SYSTEM_PROMPT_OVERRIDES = {
|
|
3175
3229
|
role: "You are an audio editor and transcript analyst specializing in segmenting content into logical chapters.",
|
|
3176
|
-
context:
|
|
3230
|
+
context: dedent4`
|
|
3177
3231
|
You receive a timestamped transcript from audio-only content with lines in the form "[12s] Transcript text".
|
|
3178
3232
|
Use those timestamps as anchors to determine chapter start times in seconds.`
|
|
3179
3233
|
};
|
|
@@ -3185,7 +3239,7 @@ var chaptersPromptBuilder = createPromptBuilder({
|
|
|
3185
3239
|
},
|
|
3186
3240
|
outputFormat: {
|
|
3187
3241
|
tag: "output_format",
|
|
3188
|
-
content:
|
|
3242
|
+
content: dedent4`
|
|
3189
3243
|
Return valid JSON in this exact shape:
|
|
3190
3244
|
{
|
|
3191
3245
|
"chapters": [
|
|
@@ -3204,7 +3258,7 @@ var chaptersPromptBuilder = createPromptBuilder({
|
|
|
3204
3258
|
},
|
|
3205
3259
|
titleGuidelines: {
|
|
3206
3260
|
tag: "title_guidelines",
|
|
3207
|
-
content:
|
|
3261
|
+
content: dedent4`
|
|
3208
3262
|
- Keep titles concise and descriptive
|
|
3209
3263
|
- Avoid filler or generic labels like "Chapter 1"
|
|
3210
3264
|
- Use the transcript's terminology`
|
|
@@ -3229,7 +3283,7 @@ function buildUserPrompt3({
|
|
|
3229
3283
|
if (languageName) {
|
|
3230
3284
|
contextSections.push(createLanguageSection(languageName));
|
|
3231
3285
|
}
|
|
3232
|
-
const dynamicChapterGuidelines =
|
|
3286
|
+
const dynamicChapterGuidelines = dedent4`
|
|
3233
3287
|
- Create at least ${minChaptersPerHour} and at most ${maxChaptersPerHour} chapters per hour of content
|
|
3234
3288
|
- Use start times in seconds (not HH:MM:SS)
|
|
3235
3289
|
- Chapter start times should be non-decreasing
|
|
@@ -3287,7 +3341,7 @@ async function generateChapters(assetId, options = {}) {
|
|
|
3287
3341
|
const contentLabel = isAudioOnly ? "transcript" : "caption track";
|
|
3288
3342
|
throw new MuxAiError(`No usable content found in ${contentLabel}.`, { type: "validation_error" });
|
|
3289
3343
|
}
|
|
3290
|
-
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : transcriptResult.track
|
|
3344
|
+
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : getReliableLanguageCode(transcriptResult.track) ?? languageCode;
|
|
3291
3345
|
const languageName = resolvedLanguageCode ? getLanguageName(resolvedLanguageCode) : void 0;
|
|
3292
3346
|
const userPrompt = buildUserPrompt3({
|
|
3293
3347
|
timestampedTranscript,
|
|
@@ -3329,7 +3383,7 @@ async function generateChapters(assetId, options = {}) {
|
|
|
3329
3383
|
};
|
|
3330
3384
|
return {
|
|
3331
3385
|
assetId,
|
|
3332
|
-
languageCode: languageCode ?? transcriptResult.track
|
|
3386
|
+
languageCode: languageCode ?? getReliableLanguageCode(transcriptResult.track) ?? "en",
|
|
3333
3387
|
chapters: validChapters,
|
|
3334
3388
|
usage: usageWithMetadata
|
|
3335
3389
|
};
|
|
@@ -3337,7 +3391,7 @@ async function generateChapters(assetId, options = {}) {
|
|
|
3337
3391
|
|
|
3338
3392
|
// src/workflows/edit-captions.ts
|
|
3339
3393
|
import { generateText as generateText4, Output as Output4 } from "ai";
|
|
3340
|
-
import
|
|
3394
|
+
import dedent5 from "dedent";
|
|
3341
3395
|
import { z as z5 } from "zod";
|
|
3342
3396
|
|
|
3343
3397
|
// src/lib/mux-tracks.ts
|
|
@@ -3414,7 +3468,7 @@ var profanityDetectionSchema = z5.object({
|
|
|
3414
3468
|
"Unique profane words or short phrases exactly as they appear in the transcript text. Include each distinct form only once (e.g., if 'fuck' and 'fucking' both appear, list both)."
|
|
3415
3469
|
)
|
|
3416
3470
|
});
|
|
3417
|
-
var SYSTEM_PROMPT3 =
|
|
3471
|
+
var SYSTEM_PROMPT3 = dedent5`
|
|
3418
3472
|
You are a content moderation assistant. Your task is to identify profane, vulgar, or obscene
|
|
3419
3473
|
words and phrases in subtitle text. Return ONLY the exact profane words or phrases as they appear
|
|
3420
3474
|
in the text. Do not modify, censor, or paraphrase them. Do not include words that are merely
|
|
@@ -3890,7 +3944,7 @@ async function generateVideoEmbeddings(assetId, options = {}) {
|
|
|
3890
3944
|
|
|
3891
3945
|
// src/workflows/engagement-insights.ts
|
|
3892
3946
|
import { generateText as generateText5, Output as Output5 } from "ai";
|
|
3893
|
-
import
|
|
3947
|
+
import dedent6 from "dedent";
|
|
3894
3948
|
import { z as z6 } from "zod";
|
|
3895
3949
|
var aiMomentInsightSchema = z6.object({
|
|
3896
3950
|
hotspotIndex: z6.number(),
|
|
@@ -3908,7 +3962,7 @@ var engagementInsightsSchema = z6.object({
|
|
|
3908
3962
|
momentInsights: z6.array(aiMomentInsightSchema),
|
|
3909
3963
|
overallInsight: aiOverallInsightSchema
|
|
3910
3964
|
});
|
|
3911
|
-
var SYSTEM_PROMPT4 =
|
|
3965
|
+
var SYSTEM_PROMPT4 = promptDedent`
|
|
3912
3966
|
<role>
|
|
3913
3967
|
You are a video engagement analyst specializing in understanding viewer behavior.
|
|
3914
3968
|
Your job is to explain why specific moments in videos have high or low engagement
|
|
@@ -3936,12 +3990,10 @@ var SYSTEM_PROMPT4 = dedent5`
|
|
|
3936
3990
|
|
|
3937
3991
|
<constraints>
|
|
3938
3992
|
- Only describe what you can see in images or read in transcripts
|
|
3939
|
-
-
|
|
3940
|
-
-
|
|
3941
|
-
playback IDs, or technical parameters visible in this request. These are
|
|
3942
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
3993
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
3994
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
3943
3995
|
- Correlate engagement scores with observable content
|
|
3944
|
-
-
|
|
3996
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
3945
3997
|
- Each momentInsight MUST include the hotspotIndex from the input data
|
|
3946
3998
|
</constraints>
|
|
3947
3999
|
|
|
@@ -3952,7 +4004,7 @@ var SYSTEM_PROMPT4 = dedent5`
|
|
|
3952
4004
|
- Connect engagement data to content features
|
|
3953
4005
|
</quality_guidelines>
|
|
3954
4006
|
`;
|
|
3955
|
-
var AUDIO_ONLY_SYSTEM_PROMPT2 =
|
|
4007
|
+
var AUDIO_ONLY_SYSTEM_PROMPT2 = promptDedent`
|
|
3956
4008
|
<role>
|
|
3957
4009
|
You are a video engagement analyst specializing in understanding viewer behavior for audio content.
|
|
3958
4010
|
Your job is to explain why specific moments have high or low engagement based on audio/dialogue and viewer watch patterns.
|
|
@@ -3979,12 +4031,10 @@ var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent5`
|
|
|
3979
4031
|
|
|
3980
4032
|
<constraints>
|
|
3981
4033
|
- Only describe what you can read in the transcript
|
|
3982
|
-
-
|
|
3983
|
-
-
|
|
3984
|
-
playback IDs, or technical parameters visible in this request. These are
|
|
3985
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
4034
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
4035
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
3986
4036
|
- Correlate engagement scores with observable content
|
|
3987
|
-
-
|
|
4037
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
3988
4038
|
- Each momentInsight MUST include the hotspotIndex from the input data
|
|
3989
4039
|
</constraints>
|
|
3990
4040
|
|
|
@@ -3995,7 +4045,7 @@ var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent5`
|
|
|
3995
4045
|
- Connect engagement data to content features
|
|
3996
4046
|
</quality_guidelines>
|
|
3997
4047
|
`;
|
|
3998
|
-
var INSIGHT_GUIDELINES =
|
|
4048
|
+
var INSIGHT_GUIDELINES = dedent6`
|
|
3999
4049
|
Provide explanatory insights that describe WHY moments are engaging.
|
|
4000
4050
|
Focus on observable patterns and viewer behavior correlation.
|
|
4001
4051
|
Example: "The cooking demonstration at 2:15 has 3x average engagement
|
|
@@ -4013,7 +4063,7 @@ var engagementInsightsPromptBuilder = createPromptBuilder({
|
|
|
4013
4063
|
},
|
|
4014
4064
|
outputFormat: {
|
|
4015
4065
|
tag: "output_format",
|
|
4016
|
-
content:
|
|
4066
|
+
content: dedent6`
|
|
4017
4067
|
Return valid JSON matching the provided schema.
|
|
4018
4068
|
Include insights for each hotspot and overall engagement trends.
|
|
4019
4069
|
Each momentInsight must include the hotspotIndex matching the input.
|
|
@@ -4021,7 +4071,7 @@ var engagementInsightsPromptBuilder = createPromptBuilder({
|
|
|
4021
4071
|
},
|
|
4022
4072
|
visualContext: {
|
|
4023
4073
|
tag: "visual_context",
|
|
4024
|
-
content:
|
|
4074
|
+
content: dedent6`
|
|
4025
4075
|
Images are provided for each hotspot timestamp.
|
|
4026
4076
|
A storyboard overview image may also be included showing the full video timeline.
|
|
4027
4077
|
Use visual evidence from these images to support your engagement analysis.
|
|
@@ -4035,14 +4085,14 @@ function buildUserPrompt4(hotspots, transcriptSegments, heatmap, isAudioOnly) {
|
|
|
4035
4085
|
const transcript = transcriptSegments[idx]?.text || "(no transcript)";
|
|
4036
4086
|
const startTimestamp = secondsToTimestamp(h.startMs / 1e3);
|
|
4037
4087
|
const endTimestamp = secondsToTimestamp(h.endMs / 1e3);
|
|
4038
|
-
return
|
|
4088
|
+
return dedent6`
|
|
4039
4089
|
Hotspot ${idx} (hotspotIndex: ${idx}):
|
|
4040
4090
|
- Time Range: ${startTimestamp} - ${endTimestamp}
|
|
4041
4091
|
- Engagement Score: ${h.score.toFixed(2)} (0=low, 1=high)
|
|
4042
4092
|
- Transcript: "${transcript}"
|
|
4043
4093
|
`;
|
|
4044
4094
|
}).join("\n\n");
|
|
4045
|
-
const heatmapData =
|
|
4095
|
+
const heatmapData = dedent6`
|
|
4046
4096
|
Engagement Heatmap (${heatmap.length} values, each representing 1/${heatmap.length}th of the video, higher = more re-watches):
|
|
4047
4097
|
[${heatmap.map((v) => v.toFixed(2)).join(", ")}]
|
|
4048
4098
|
`;
|
|
@@ -4916,7 +4966,7 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
4916
4966
|
|
|
4917
4967
|
// src/workflows/summarization.ts
|
|
4918
4968
|
import { generateText as generateText6, Output as Output6 } from "ai";
|
|
4919
|
-
import
|
|
4969
|
+
import dedent7 from "dedent";
|
|
4920
4970
|
import { z as z7 } from "zod";
|
|
4921
4971
|
var DEFAULT_SUMMARY_KEYWORD_LIMIT = 10;
|
|
4922
4972
|
var DEFAULT_TITLE_LENGTH = 10;
|
|
@@ -4942,36 +4992,36 @@ var DESCRIPTION_LENGTH_THRESHOLD_LARGE = 100;
|
|
|
4942
4992
|
function buildDescriptionGuidance(wordCount, contentType) {
|
|
4943
4993
|
if (wordCount < DESCRIPTION_LENGTH_THRESHOLD_SMALL) {
|
|
4944
4994
|
if (contentType === "video") {
|
|
4945
|
-
return
|
|
4995
|
+
return dedent7`A brief summary of the video in no more than ${wordCount} words. Shorter is fine.
|
|
4946
4996
|
Focus on the single most important subject or action.
|
|
4947
4997
|
Write in present tense.`;
|
|
4948
4998
|
}
|
|
4949
|
-
return
|
|
4999
|
+
return dedent7`A brief summary of the audio content in no more than ${wordCount} words. Shorter is fine.
|
|
4950
5000
|
Focus on the single most important topic or theme.
|
|
4951
5001
|
Write in present tense.`;
|
|
4952
5002
|
}
|
|
4953
5003
|
if (wordCount > DESCRIPTION_LENGTH_THRESHOLD_LARGE) {
|
|
4954
5004
|
if (contentType === "video") {
|
|
4955
|
-
return
|
|
5005
|
+
return dedent7`A detailed summary that describes what happens across the video.
|
|
4956
5006
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4957
5007
|
Be thorough: cover subjects, actions, setting, progression, and any notable details visible across frames.
|
|
4958
5008
|
Write in present tense. Be specific about observable details rather than making assumptions.
|
|
4959
5009
|
If the transcript provides dialogue or narration, incorporate key points but prioritize visual content.`;
|
|
4960
5010
|
}
|
|
4961
|
-
return
|
|
5011
|
+
return dedent7`A detailed summary that describes the audio content.
|
|
4962
5012
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4963
5013
|
Be thorough: cover topics, speakers, themes, progression, and any notable insights.
|
|
4964
5014
|
Write in present tense. Be specific about what is discussed or presented rather than making assumptions.
|
|
4965
5015
|
Focus on the spoken content and any key insights, dialogue, or narrative elements.`;
|
|
4966
5016
|
}
|
|
4967
5017
|
if (contentType === "video") {
|
|
4968
|
-
return
|
|
5018
|
+
return dedent7`A summary that describes what happens across the video.
|
|
4969
5019
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4970
5020
|
Cover the main subjects, actions, setting, and any notable progression visible across frames.
|
|
4971
5021
|
Write in present tense. Be specific about observable details rather than making assumptions.
|
|
4972
5022
|
If the transcript provides dialogue or narration, incorporate key points but prioritize visual content.`;
|
|
4973
5023
|
}
|
|
4974
|
-
return
|
|
5024
|
+
return dedent7`A summary that describes the audio content.
|
|
4975
5025
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4976
5026
|
Cover the main topics, speakers, themes, and any notable progression in the discussion or narration.
|
|
4977
5027
|
Write in present tense. Be specific about what is discussed or presented rather than making assumptions.
|
|
@@ -4988,7 +5038,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
4988
5038
|
},
|
|
4989
5039
|
title: {
|
|
4990
5040
|
tag: "title_requirements",
|
|
4991
|
-
content:
|
|
5041
|
+
content: dedent7`
|
|
4992
5042
|
A concise, label-style title — not a sentence or description.
|
|
4993
5043
|
Never exceed ${titleLimit} words, but shorter is better.
|
|
4994
5044
|
Think of how a video card title, playlist entry, or file name would read — e.g. "Predator: Badlands Trailer" or "Chef Prepares Holiday Feast".
|
|
@@ -5001,7 +5051,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
5001
5051
|
},
|
|
5002
5052
|
keywords: {
|
|
5003
5053
|
tag: "keywords_requirements",
|
|
5004
|
-
content:
|
|
5054
|
+
content: dedent7`
|
|
5005
5055
|
Specific, searchable terms (up to ${keywordLimit}) that capture:
|
|
5006
5056
|
- Primary subjects (people, animals, objects)
|
|
5007
5057
|
- Actions and activities being performed
|
|
@@ -5013,7 +5063,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
5013
5063
|
},
|
|
5014
5064
|
qualityGuidelines: {
|
|
5015
5065
|
tag: "quality_guidelines",
|
|
5016
|
-
content:
|
|
5066
|
+
content: dedent7`
|
|
5017
5067
|
- Examine all frames to understand the full context and progression
|
|
5018
5068
|
- Be precise: "golden retriever" is better than "dog" when identifiable
|
|
5019
5069
|
- Capture the narrative: what begins, develops, and concludes
|
|
@@ -5034,7 +5084,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
5034
5084
|
},
|
|
5035
5085
|
title: {
|
|
5036
5086
|
tag: "title_requirements",
|
|
5037
|
-
content:
|
|
5087
|
+
content: dedent7`
|
|
5038
5088
|
A concise, label-style title — not a sentence or description.
|
|
5039
5089
|
Never exceed ${titleLimit} words, but shorter is better.
|
|
5040
5090
|
Think of how a podcast episode title or playlist entry would read — e.g. "Weekly News Roundup" or "Interview with Dr. Smith".
|
|
@@ -5047,7 +5097,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
5047
5097
|
},
|
|
5048
5098
|
keywords: {
|
|
5049
5099
|
tag: "keywords_requirements",
|
|
5050
|
-
content:
|
|
5100
|
+
content: dedent7`
|
|
5051
5101
|
Specific, searchable terms (up to ${keywordLimit}) that capture:
|
|
5052
5102
|
- Primary topics and themes
|
|
5053
5103
|
- Speakers or presenters (if named)
|
|
@@ -5059,7 +5109,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
5059
5109
|
},
|
|
5060
5110
|
qualityGuidelines: {
|
|
5061
5111
|
tag: "quality_guidelines",
|
|
5062
|
-
content:
|
|
5112
|
+
content: dedent7`
|
|
5063
5113
|
- Analyze the full transcript to understand context and themes
|
|
5064
5114
|
- Be precise: use specific terminology when mentioned
|
|
5065
5115
|
- Capture the narrative: what is introduced, discussed, and concluded
|
|
@@ -5069,15 +5119,14 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
5069
5119
|
sectionOrder: ["task", "title", "description", "keywords", "qualityGuidelines"]
|
|
5070
5120
|
});
|
|
5071
5121
|
}
|
|
5072
|
-
var SYSTEM_PROMPT5 =
|
|
5122
|
+
var SYSTEM_PROMPT5 = promptDedent`
|
|
5073
5123
|
<role>
|
|
5074
5124
|
You are a video content analyst specializing in storyboard interpretation and multimodal analysis.
|
|
5075
5125
|
</role>
|
|
5076
5126
|
|
|
5077
5127
|
<context>
|
|
5078
5128
|
You receive storyboard images containing multiple sequential frames extracted from a video.
|
|
5079
|
-
|
|
5080
|
-
Read frames left-to-right, top-to-bottom to understand the temporal sequence.
|
|
5129
|
+
${STORYBOARD_FRAME_INSTRUCTIONS}
|
|
5081
5130
|
</context>
|
|
5082
5131
|
|
|
5083
5132
|
<transcript_guidance>
|
|
@@ -5099,37 +5148,21 @@ var SYSTEM_PROMPT5 = dedent6`
|
|
|
5099
5148
|
|
|
5100
5149
|
<constraints>
|
|
5101
5150
|
- Only describe what is clearly observable in the frames or explicitly stated in the transcript
|
|
5102
|
-
-
|
|
5103
|
-
-
|
|
5104
|
-
|
|
5105
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
5106
|
-
- Return structured data matching the requested schema
|
|
5151
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
5152
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
5153
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
5107
5154
|
- Output only the JSON object; no markdown or extra text
|
|
5108
5155
|
- When a <language> section is provided, all output text MUST be written in that language
|
|
5109
5156
|
</constraints>
|
|
5110
5157
|
|
|
5111
5158
|
<tone_guidance>
|
|
5112
|
-
|
|
5113
|
-
Adapt your entire analysis and writing style to match the specified tone - this should influence
|
|
5114
|
-
your word choice, personality, formality level, and overall presentation of the content.
|
|
5115
|
-
The tone instructions are not suggestions but core requirements for how you should express yourself.
|
|
5159
|
+
${TONE_GUIDANCE}
|
|
5116
5160
|
</tone_guidance>
|
|
5117
5161
|
|
|
5118
5162
|
<language_guidelines>
|
|
5119
|
-
|
|
5120
|
-
- "The image shows..." / "The storyboard shows..."
|
|
5121
|
-
- "In this video..." / "This video features..."
|
|
5122
|
-
- "The frames depict..." / "The footage shows..."
|
|
5123
|
-
- "We can see..." / "You can see..."
|
|
5124
|
-
- "The clip shows..." / "The scene shows..."
|
|
5125
|
-
|
|
5126
|
-
INSTEAD, describe the content directly:
|
|
5127
|
-
- BAD: "The video shows a chef preparing a meal"
|
|
5128
|
-
- GOOD: "A chef prepares a meal in a professional kitchen"
|
|
5129
|
-
|
|
5130
|
-
Write as if describing reality, not describing a recording of reality.
|
|
5163
|
+
${createLanguageGuidelines("video")}
|
|
5131
5164
|
</language_guidelines>`;
|
|
5132
|
-
var AUDIO_ONLY_SYSTEM_PROMPT3 =
|
|
5165
|
+
var AUDIO_ONLY_SYSTEM_PROMPT3 = promptDedent`
|
|
5133
5166
|
<role>
|
|
5134
5167
|
You are an audio content analyst specializing in transcript analysis and metadata generation.
|
|
5135
5168
|
</role>
|
|
@@ -5156,35 +5189,20 @@ var AUDIO_ONLY_SYSTEM_PROMPT3 = dedent6`
|
|
|
5156
5189
|
|
|
5157
5190
|
<constraints>
|
|
5158
5191
|
- Only describe what is explicitly stated or strongly implied in the transcript
|
|
5159
|
-
-
|
|
5160
|
-
-
|
|
5161
|
-
|
|
5162
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
5163
|
-
- Return structured data matching the requested schema
|
|
5192
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
5193
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
5194
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
5164
5195
|
- Focus entirely on audio/spoken content - there are no visual elements
|
|
5165
5196
|
- Output only the JSON object; no markdown or extra text
|
|
5166
5197
|
- When a <language> section is provided, all output text MUST be written in that language
|
|
5167
5198
|
</constraints>
|
|
5168
5199
|
|
|
5169
5200
|
<tone_guidance>
|
|
5170
|
-
|
|
5171
|
-
Adapt your entire analysis and writing style to match the specified tone - this should influence
|
|
5172
|
-
your word choice, personality, formality level, and overall presentation of the content.
|
|
5173
|
-
The tone instructions are not suggestions but core requirements for how you should express yourself.
|
|
5201
|
+
${TONE_GUIDANCE}
|
|
5174
5202
|
</tone_guidance>
|
|
5175
5203
|
|
|
5176
5204
|
<language_guidelines>
|
|
5177
|
-
|
|
5178
|
-
- "The audio shows..." / "The transcript shows..."
|
|
5179
|
-
- "In this recording..." / "This audio features..."
|
|
5180
|
-
- "The speaker says..." / "We can hear..."
|
|
5181
|
-
- "The clip contains..." / "The recording shows..."
|
|
5182
|
-
|
|
5183
|
-
INSTEAD, describe the content directly:
|
|
5184
|
-
- BAD: "The audio features a discussion about climate change"
|
|
5185
|
-
- GOOD: "A panel discusses climate change impacts and solutions"
|
|
5186
|
-
|
|
5187
|
-
Write as if describing reality, not describing a recording of reality.
|
|
5205
|
+
${createLanguageGuidelines("audio")}
|
|
5188
5206
|
</language_guidelines>`;
|
|
5189
5207
|
function buildUserPrompt5({
|
|
5190
5208
|
tone,
|
|
@@ -5358,7 +5376,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
5358
5376
|
required: isAudioOnly
|
|
5359
5377
|
}) : void 0;
|
|
5360
5378
|
const transcriptText = transcriptResult?.transcriptText ?? "";
|
|
5361
|
-
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : transcriptResult?.track
|
|
5379
|
+
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : getReliableLanguageCode(transcriptResult?.track) ?? getReliableLanguageCode(getReadyTextTracks(assetData)[0]);
|
|
5362
5380
|
const languageName = resolvedLanguageCode ? getLanguageName(resolvedLanguageCode) : void 0;
|
|
5363
5381
|
const userPrompt = buildUserPrompt5({
|
|
5364
5382
|
tone,
|
|
@@ -5850,18 +5868,18 @@ import {
|
|
|
5850
5868
|
RetryError,
|
|
5851
5869
|
TypeValidationError
|
|
5852
5870
|
} from "ai";
|
|
5853
|
-
import
|
|
5871
|
+
import dedent8 from "dedent";
|
|
5854
5872
|
import { z as z8 } from "zod";
|
|
5855
5873
|
var translationSchema = z8.object({
|
|
5856
5874
|
translation: z8.string()
|
|
5857
5875
|
});
|
|
5858
|
-
var SYSTEM_PROMPT6 =
|
|
5876
|
+
var SYSTEM_PROMPT6 = dedent8`
|
|
5859
5877
|
You are a subtitle translation expert. Translate VTT subtitle files to the target language specified by the user.
|
|
5860
5878
|
You may receive either a full VTT file or a chunk from a larger VTT.
|
|
5861
5879
|
Preserve all timestamps, cue ordering, and VTT formatting exactly as they appear.
|
|
5862
5880
|
Return JSON with a single key "translation" containing the translated VTT content.
|
|
5863
5881
|
`;
|
|
5864
|
-
var CUE_TRANSLATION_SYSTEM_PROMPT =
|
|
5882
|
+
var CUE_TRANSLATION_SYSTEM_PROMPT = dedent8`
|
|
5865
5883
|
You are a subtitle translation expert.
|
|
5866
5884
|
You will receive a sequence of subtitle cues extracted from a VTT file.
|
|
5867
5885
|
Translate the cues to the requested target language while preserving their original order.
|
|
@@ -6372,7 +6390,7 @@ async function translateCaptions(assetId, trackId, toLanguageCode, options) {
|
|
|
6372
6390
|
}
|
|
6373
6391
|
if (uploadToMux) {
|
|
6374
6392
|
try {
|
|
6375
|
-
const languageName = getLanguageName(toLanguageCode);
|
|
6393
|
+
const languageName = getLanguageName(toLanguageCode) ?? toLanguageCode.toUpperCase();
|
|
6376
6394
|
const trackName = `${languageName} (auto-translated)`;
|
|
6377
6395
|
uploadedTrackId = await createTextTrackOnMux(
|
|
6378
6396
|
assetId,
|