@mux/ai 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{index-CuBu_u5-.d.ts → index-3G00N5QH.d.ts} +26 -2
- package/dist/{index-CXC5gAHs.d.ts → index-BfsJvx3r.d.ts} +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.js +317 -297
- package/dist/index.js.map +1 -1
- package/dist/primitives/index.d.ts +1 -1
- package/dist/primitives/index.js +134 -0
- package/dist/primitives/index.js.map +1 -1
- package/dist/workflows/index.d.ts +1 -1
- package/dist/workflows/index.js +314 -296
- package/dist/workflows/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -5,7 +5,7 @@ var __export = (target, all) => {
|
|
|
5
5
|
};
|
|
6
6
|
|
|
7
7
|
// package.json
|
|
8
|
-
var version = "0.
|
|
8
|
+
var version = "0.17.0";
|
|
9
9
|
|
|
10
10
|
// src/lib/mux-ai-error.ts
|
|
11
11
|
var MuxAiError = class extends Error {
|
|
@@ -802,6 +802,7 @@ function createWorkflowStorageClient(options = {}) {
|
|
|
802
802
|
var primitives_exports = {};
|
|
803
803
|
__export(primitives_exports, {
|
|
804
804
|
DEFAULT_STORYBOARD_WIDTH: () => DEFAULT_STORYBOARD_WIDTH,
|
|
805
|
+
LOW_CONFIDENCE_THRESHOLD: () => LOW_CONFIDENCE_THRESHOLD,
|
|
805
806
|
buildTranscriptUrl: () => buildTranscriptUrl,
|
|
806
807
|
buildVttFromCueBlocks: () => buildVttFromCueBlocks,
|
|
807
808
|
buildVttFromTranslatedCueBlocks: () => buildVttFromTranslatedCueBlocks,
|
|
@@ -823,6 +824,7 @@ __export(primitives_exports, {
|
|
|
823
824
|
getHotspotsForPlaybackId: () => getHotspotsForPlaybackId,
|
|
824
825
|
getHotspotsForVideo: () => getHotspotsForVideo,
|
|
825
826
|
getReadyTextTracks: () => getReadyTextTracks,
|
|
827
|
+
getReliableLanguageCode: () => getReliableLanguageCode,
|
|
826
828
|
getShotsForAsset: () => getShotsForAsset,
|
|
827
829
|
getStoryboardUrl: () => getStoryboardUrl,
|
|
828
830
|
getThumbnailUrls: () => getThumbnailUrls,
|
|
@@ -1601,6 +1603,171 @@ async function getThumbnailUrls(playbackId, duration, options = {}) {
|
|
|
1601
1603
|
return Promise.all(urlPromises);
|
|
1602
1604
|
}
|
|
1603
1605
|
|
|
1606
|
+
// src/lib/language-codes.ts
|
|
1607
|
+
var ISO639_1_TO_3 = {
|
|
1608
|
+
// Major world languages
|
|
1609
|
+
en: "eng",
|
|
1610
|
+
// English
|
|
1611
|
+
es: "spa",
|
|
1612
|
+
// Spanish
|
|
1613
|
+
fr: "fra",
|
|
1614
|
+
// French
|
|
1615
|
+
de: "deu",
|
|
1616
|
+
// German
|
|
1617
|
+
it: "ita",
|
|
1618
|
+
// Italian
|
|
1619
|
+
pt: "por",
|
|
1620
|
+
// Portuguese
|
|
1621
|
+
ru: "rus",
|
|
1622
|
+
// Russian
|
|
1623
|
+
zh: "zho",
|
|
1624
|
+
// Chinese
|
|
1625
|
+
ja: "jpn",
|
|
1626
|
+
// Japanese
|
|
1627
|
+
ko: "kor",
|
|
1628
|
+
// Korean
|
|
1629
|
+
ar: "ara",
|
|
1630
|
+
// Arabic
|
|
1631
|
+
hi: "hin",
|
|
1632
|
+
// Hindi
|
|
1633
|
+
// European languages
|
|
1634
|
+
nl: "nld",
|
|
1635
|
+
// Dutch
|
|
1636
|
+
pl: "pol",
|
|
1637
|
+
// Polish
|
|
1638
|
+
sv: "swe",
|
|
1639
|
+
// Swedish
|
|
1640
|
+
da: "dan",
|
|
1641
|
+
// Danish
|
|
1642
|
+
no: "nor",
|
|
1643
|
+
// Norwegian
|
|
1644
|
+
fi: "fin",
|
|
1645
|
+
// Finnish
|
|
1646
|
+
el: "ell",
|
|
1647
|
+
// Greek
|
|
1648
|
+
cs: "ces",
|
|
1649
|
+
// Czech
|
|
1650
|
+
hu: "hun",
|
|
1651
|
+
// Hungarian
|
|
1652
|
+
ro: "ron",
|
|
1653
|
+
// Romanian
|
|
1654
|
+
bg: "bul",
|
|
1655
|
+
// Bulgarian
|
|
1656
|
+
hr: "hrv",
|
|
1657
|
+
// Croatian
|
|
1658
|
+
sk: "slk",
|
|
1659
|
+
// Slovak
|
|
1660
|
+
sl: "slv",
|
|
1661
|
+
// Slovenian
|
|
1662
|
+
uk: "ukr",
|
|
1663
|
+
// Ukrainian
|
|
1664
|
+
tr: "tur",
|
|
1665
|
+
// Turkish
|
|
1666
|
+
// Asian languages
|
|
1667
|
+
th: "tha",
|
|
1668
|
+
// Thai
|
|
1669
|
+
vi: "vie",
|
|
1670
|
+
// Vietnamese
|
|
1671
|
+
id: "ind",
|
|
1672
|
+
// Indonesian
|
|
1673
|
+
ms: "msa",
|
|
1674
|
+
// Malay
|
|
1675
|
+
tl: "tgl",
|
|
1676
|
+
// Tagalog/Filipino
|
|
1677
|
+
// Other languages
|
|
1678
|
+
he: "heb",
|
|
1679
|
+
// Hebrew
|
|
1680
|
+
fa: "fas",
|
|
1681
|
+
// Persian/Farsi
|
|
1682
|
+
bn: "ben",
|
|
1683
|
+
// Bengali
|
|
1684
|
+
ta: "tam",
|
|
1685
|
+
// Tamil
|
|
1686
|
+
te: "tel",
|
|
1687
|
+
// Telugu
|
|
1688
|
+
mr: "mar",
|
|
1689
|
+
// Marathi
|
|
1690
|
+
gu: "guj",
|
|
1691
|
+
// Gujarati
|
|
1692
|
+
kn: "kan",
|
|
1693
|
+
// Kannada
|
|
1694
|
+
ml: "mal",
|
|
1695
|
+
// Malayalam
|
|
1696
|
+
pa: "pan",
|
|
1697
|
+
// Punjabi
|
|
1698
|
+
ur: "urd",
|
|
1699
|
+
// Urdu
|
|
1700
|
+
sw: "swa",
|
|
1701
|
+
// Swahili
|
|
1702
|
+
af: "afr",
|
|
1703
|
+
// Afrikaans
|
|
1704
|
+
ca: "cat",
|
|
1705
|
+
// Catalan
|
|
1706
|
+
eu: "eus",
|
|
1707
|
+
// Basque
|
|
1708
|
+
gl: "glg",
|
|
1709
|
+
// Galician
|
|
1710
|
+
is: "isl",
|
|
1711
|
+
// Icelandic
|
|
1712
|
+
et: "est",
|
|
1713
|
+
// Estonian
|
|
1714
|
+
lv: "lav",
|
|
1715
|
+
// Latvian
|
|
1716
|
+
lt: "lit"
|
|
1717
|
+
// Lithuanian
|
|
1718
|
+
};
|
|
1719
|
+
var ISO639_3_TO_1 = Object.fromEntries(
|
|
1720
|
+
Object.entries(ISO639_1_TO_3).map(([iso1, iso3]) => [iso3, iso1])
|
|
1721
|
+
);
|
|
1722
|
+
function toISO639_3(code) {
|
|
1723
|
+
const normalized = code.toLowerCase().trim();
|
|
1724
|
+
if (normalized.length === 3) {
|
|
1725
|
+
return normalized;
|
|
1726
|
+
}
|
|
1727
|
+
return ISO639_1_TO_3[normalized] ?? normalized;
|
|
1728
|
+
}
|
|
1729
|
+
function toISO639_1(code) {
|
|
1730
|
+
const normalized = code.toLowerCase().trim();
|
|
1731
|
+
if (normalized.length === 2) {
|
|
1732
|
+
return normalized;
|
|
1733
|
+
}
|
|
1734
|
+
return ISO639_3_TO_1[normalized] ?? normalized;
|
|
1735
|
+
}
|
|
1736
|
+
function getLanguageCodePair(code) {
|
|
1737
|
+
const normalized = code.toLowerCase().trim();
|
|
1738
|
+
if (normalized.length === 2) {
|
|
1739
|
+
return {
|
|
1740
|
+
iso639_1: normalized,
|
|
1741
|
+
iso639_3: toISO639_3(normalized)
|
|
1742
|
+
};
|
|
1743
|
+
} else if (normalized.length === 3) {
|
|
1744
|
+
return {
|
|
1745
|
+
iso639_1: toISO639_1(normalized),
|
|
1746
|
+
iso639_3: normalized
|
|
1747
|
+
};
|
|
1748
|
+
}
|
|
1749
|
+
return {
|
|
1750
|
+
iso639_1: normalized,
|
|
1751
|
+
iso639_3: normalized
|
|
1752
|
+
};
|
|
1753
|
+
}
|
|
1754
|
+
var UNDETERMINED_LANGUAGE_CODES = /* @__PURE__ */ new Set(["und", "mul", "mis", "zxx"]);
|
|
1755
|
+
function isUndeterminedLanguageCode(code) {
|
|
1756
|
+
return UNDETERMINED_LANGUAGE_CODES.has(code.toLowerCase().trim());
|
|
1757
|
+
}
|
|
1758
|
+
function getLanguageName(code) {
|
|
1759
|
+
if (isUndeterminedLanguageCode(code)) {
|
|
1760
|
+
return void 0;
|
|
1761
|
+
}
|
|
1762
|
+
const iso639_1 = toISO639_1(code);
|
|
1763
|
+
try {
|
|
1764
|
+
const displayNames = new Intl.DisplayNames(["en"], { type: "language" });
|
|
1765
|
+
return displayNames.of(iso639_1) ?? code.toUpperCase();
|
|
1766
|
+
} catch {
|
|
1767
|
+
return code.toUpperCase();
|
|
1768
|
+
}
|
|
1769
|
+
}
|
|
1770
|
+
|
|
1604
1771
|
// src/lib/mux-assets.ts
|
|
1605
1772
|
function getPlaybackId(asset) {
|
|
1606
1773
|
const playbackIds = asset.playback_ids || [];
|
|
@@ -1678,6 +1845,17 @@ function findCaptionTrack(asset, languageCode) {
|
|
|
1678
1845
|
}
|
|
1679
1846
|
return void 0;
|
|
1680
1847
|
}
|
|
1848
|
+
var LOW_CONFIDENCE_THRESHOLD = 0.5;
|
|
1849
|
+
function getReliableLanguageCode(track, confidenceThreshold = LOW_CONFIDENCE_THRESHOLD) {
|
|
1850
|
+
if (!track?.language_code)
|
|
1851
|
+
return void 0;
|
|
1852
|
+
if (isUndeterminedLanguageCode(track.language_code))
|
|
1853
|
+
return void 0;
|
|
1854
|
+
if (track.auto_language_confidence !== void 0 && track.auto_language_confidence < confidenceThreshold) {
|
|
1855
|
+
return void 0;
|
|
1856
|
+
}
|
|
1857
|
+
return track.language_code;
|
|
1858
|
+
}
|
|
1681
1859
|
function normalizeLineEndings(value) {
|
|
1682
1860
|
return value.replace(/\r\n/g, "\n");
|
|
1683
1861
|
}
|
|
@@ -2041,7 +2219,7 @@ __export(workflows_exports, {
|
|
|
2041
2219
|
|
|
2042
2220
|
// src/workflows/ask-questions.ts
|
|
2043
2221
|
import { generateText, Output } from "ai";
|
|
2044
|
-
import
|
|
2222
|
+
import dedent2 from "dedent";
|
|
2045
2223
|
import { z as z2 } from "zod";
|
|
2046
2224
|
|
|
2047
2225
|
// src/lib/image-download.ts
|
|
@@ -2231,6 +2409,59 @@ function createLanguageSection(languageName) {
|
|
|
2231
2409
|
};
|
|
2232
2410
|
}
|
|
2233
2411
|
|
|
2412
|
+
// src/lib/prompt-fragments.ts
|
|
2413
|
+
import dedent from "dedent";
|
|
2414
|
+
var promptDedent = dedent.withOptions({ alignValues: true });
|
|
2415
|
+
var METADATA_BOUNDARY_WARNING = dedent`
|
|
2416
|
+
Do NOT use any metadata such as URLs, file paths, domain names, file names,
|
|
2417
|
+
playback IDs, or technical parameters visible in this request. These are
|
|
2418
|
+
delivery infrastructure and are unrelated to the media content itself.`;
|
|
2419
|
+
var NO_FABRICATION_CONSTRAINT = "Do not fabricate details or make unsupported assumptions";
|
|
2420
|
+
var STRUCTURED_DATA_CONSTRAINT = "Return structured data matching the requested schema exactly";
|
|
2421
|
+
var STORYBOARD_FRAME_INSTRUCTIONS = dedent`
|
|
2422
|
+
These frames are arranged in a grid and represent the visual progression of the content over time.
|
|
2423
|
+
Read frames left-to-right, top-to-bottom to understand the temporal sequence.`;
|
|
2424
|
+
var TONE_GUIDANCE = dedent`
|
|
2425
|
+
Pay special attention to the <tone> section and lean heavily into those instructions.
|
|
2426
|
+
Adapt your entire analysis and writing style to match the specified tone - this should influence
|
|
2427
|
+
your word choice, personality, formality level, and overall presentation of the content.
|
|
2428
|
+
The tone instructions are not suggestions but core requirements for how you should express yourself.`;
|
|
2429
|
+
var CONFIDENCE_SCORING_RUBRIC = dedent`
|
|
2430
|
+
* 0.9-1.0: Clear, unambiguous evidence
|
|
2431
|
+
* 0.7-0.9: Strong evidence with minor ambiguity
|
|
2432
|
+
* 0.5-0.7: Moderate evidence or some conflicting signals
|
|
2433
|
+
* 0.3-0.5: Weak evidence or significant ambiguity
|
|
2434
|
+
* 0.0-0.3: Very uncertain, minimal relevant evidence`;
|
|
2435
|
+
function createLanguageGuidelines(mediaType) {
|
|
2436
|
+
if (mediaType === "video") {
|
|
2437
|
+
return dedent`
|
|
2438
|
+
AVOID these meta-descriptive phrases that reference the medium rather than the content:
|
|
2439
|
+
- "The image shows..." / "The storyboard shows..."
|
|
2440
|
+
- "In this video..." / "This video features..."
|
|
2441
|
+
- "The frames depict..." / "The footage shows..."
|
|
2442
|
+
- "We can see..." / "You can see..."
|
|
2443
|
+
- "The clip shows..." / "The scene shows..."
|
|
2444
|
+
|
|
2445
|
+
INSTEAD, describe the content directly:
|
|
2446
|
+
- BAD: "The video shows a chef preparing a meal"
|
|
2447
|
+
- GOOD: "A chef prepares a meal in a professional kitchen"
|
|
2448
|
+
|
|
2449
|
+
Write as if describing reality, not describing a recording of reality.`;
|
|
2450
|
+
}
|
|
2451
|
+
return dedent`
|
|
2452
|
+
AVOID these meta-descriptive phrases that reference the medium rather than the content:
|
|
2453
|
+
- "The audio shows..." / "The transcript shows..."
|
|
2454
|
+
- "In this recording..." / "This audio features..."
|
|
2455
|
+
- "The speaker says..." / "We can hear..."
|
|
2456
|
+
- "The clip contains..." / "The recording shows..."
|
|
2457
|
+
|
|
2458
|
+
INSTEAD, describe the content directly:
|
|
2459
|
+
- BAD: "The audio features a discussion about climate change"
|
|
2460
|
+
- GOOD: "A panel discusses climate change impacts and solutions"
|
|
2461
|
+
|
|
2462
|
+
Write as if describing reality, not describing a recording of reality.`;
|
|
2463
|
+
}
|
|
2464
|
+
|
|
2234
2465
|
// src/lib/retry.ts
|
|
2235
2466
|
var DEFAULT_RETRY_OPTIONS = {
|
|
2236
2467
|
maxRetries: 3,
|
|
@@ -2290,7 +2521,7 @@ function createAskQuestionsSchema(allowedAnswers) {
|
|
|
2290
2521
|
)
|
|
2291
2522
|
});
|
|
2292
2523
|
}
|
|
2293
|
-
var SYSTEM_PROMPT =
|
|
2524
|
+
var SYSTEM_PROMPT = promptDedent`
|
|
2294
2525
|
<role>
|
|
2295
2526
|
You are a video content analyst specializing in classification tasks.
|
|
2296
2527
|
Your job is to answer questions about video content based on storyboard
|
|
@@ -2303,9 +2534,7 @@ var SYSTEM_PROMPT = dedent`
|
|
|
2303
2534
|
- A list of questions about the video content
|
|
2304
2535
|
- Optionally, a transcript of the audio/dialogue
|
|
2305
2536
|
|
|
2306
|
-
|
|
2307
|
-
progression of the content over time. Read frames left-to-right,
|
|
2308
|
-
top-to-bottom to understand the temporal sequence.
|
|
2537
|
+
${STORYBOARD_FRAME_INSTRUCTIONS}
|
|
2309
2538
|
</context>
|
|
2310
2539
|
|
|
2311
2540
|
<transcript_guidance>
|
|
@@ -2327,14 +2556,11 @@ var SYSTEM_PROMPT = dedent`
|
|
|
2327
2556
|
<answer_guidelines>
|
|
2328
2557
|
- Each question is presented as its own <question> block with a <text> element (the question) and an <allowed_answers> element (the permitted response values)
|
|
2329
2558
|
- Choose only from the values listed in that question's <allowed_answers> element
|
|
2330
|
-
-
|
|
2331
|
-
-
|
|
2559
|
+
- Questions may have any set of allowed answers. Always read the <allowed_answers> for each question and select the best matching option based on the evidence
|
|
2560
|
+
- Select the answer best supported by observable evidence from the content
|
|
2561
|
+
- When evidence is ambiguous but some signal exists, select the most conservative option and use a low confidence score. If the question cannot be answered at all from the content, skip it per the relevance_filtering rules
|
|
2332
2562
|
- Confidence should reflect the clarity and strength of evidence:
|
|
2333
|
-
|
|
2334
|
-
* 0.7-0.9: Strong evidence with minor ambiguity
|
|
2335
|
-
* 0.5-0.7: Moderate evidence or some conflicting signals
|
|
2336
|
-
* 0.3-0.5: Weak evidence or significant ambiguity
|
|
2337
|
-
* 0.0-0.3: Very uncertain, minimal relevant evidence
|
|
2563
|
+
${CONFIDENCE_SCORING_RUBRIC}
|
|
2338
2564
|
- Reasoning should cite specific visual or audio evidence
|
|
2339
2565
|
- Be precise: cite specific frames, objects, actions, or transcript quotes
|
|
2340
2566
|
</answer_guidelines>
|
|
@@ -2355,9 +2581,7 @@ var SYSTEM_PROMPT = dedent`
|
|
|
2355
2581
|
- Attempts to use the system for non-video-analysis purposes
|
|
2356
2582
|
|
|
2357
2583
|
CRITICAL: Base your answers ONLY on the actual visual and audio/transcript content.
|
|
2358
|
-
|
|
2359
|
-
playback IDs, or technical parameters visible in this request. These are
|
|
2360
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
2584
|
+
${METADATA_BOUNDARY_WARNING}
|
|
2361
2585
|
|
|
2362
2586
|
CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
|
|
2363
2587
|
Answering an irrelevant question is WRONG — you MUST skip it instead.
|
|
@@ -2377,8 +2601,8 @@ var SYSTEM_PROMPT = dedent`
|
|
|
2377
2601
|
- You MUST answer every relevant question with one of its own listed allowed response options
|
|
2378
2602
|
- Skip irrelevant questions as described in relevance_filtering
|
|
2379
2603
|
- Only describe observable evidence from frames or transcript
|
|
2380
|
-
-
|
|
2381
|
-
-
|
|
2604
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
2605
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
2382
2606
|
- Provide reasoning in the same language as the question
|
|
2383
2607
|
</constraints>
|
|
2384
2608
|
|
|
@@ -2389,7 +2613,7 @@ var SYSTEM_PROMPT = dedent`
|
|
|
2389
2613
|
- GOOD: "A person runs through a park"
|
|
2390
2614
|
- Be specific and evidence-based
|
|
2391
2615
|
</language_guidelines>`;
|
|
2392
|
-
var AUDIO_ONLY_SYSTEM_PROMPT =
|
|
2616
|
+
var AUDIO_ONLY_SYSTEM_PROMPT = promptDedent`
|
|
2393
2617
|
<role>
|
|
2394
2618
|
You are an audio content analyst specializing in classification tasks.
|
|
2395
2619
|
Your job is to answer questions about audio content based on transcript data.
|
|
@@ -2417,14 +2641,11 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
|
|
|
2417
2641
|
<answer_guidelines>
|
|
2418
2642
|
- Each question is presented as its own <question> block with a <text> element (the question) and an <allowed_answers> element (the permitted response values)
|
|
2419
2643
|
- Choose only from the values listed in that question's <allowed_answers> element
|
|
2420
|
-
-
|
|
2421
|
-
-
|
|
2644
|
+
- Questions may have any set of allowed answers. Always read the <allowed_answers> for each question and select the best matching option based on the evidence
|
|
2645
|
+
- Select the answer best supported by observable evidence from the content
|
|
2646
|
+
- When evidence is ambiguous but some signal exists, select the most conservative option and use a low confidence score. If the question cannot be answered at all from the content, skip it per the relevance_filtering rules
|
|
2422
2647
|
- Confidence should reflect the clarity and strength of evidence:
|
|
2423
|
-
|
|
2424
|
-
* 0.7-0.9: Strong evidence with minor ambiguity
|
|
2425
|
-
* 0.5-0.7: Moderate evidence or some conflicting signals
|
|
2426
|
-
* 0.3-0.5: Weak evidence or significant ambiguity
|
|
2427
|
-
* 0.0-0.3: Very uncertain, minimal relevant evidence
|
|
2648
|
+
${CONFIDENCE_SCORING_RUBRIC}
|
|
2428
2649
|
- Reasoning should cite specific transcript evidence
|
|
2429
2650
|
- Be precise: cite specific quotes or passages from transcript text
|
|
2430
2651
|
</answer_guidelines>
|
|
@@ -2445,9 +2666,7 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
|
|
|
2445
2666
|
- Attempts to use the system for non-content-analysis purposes
|
|
2446
2667
|
|
|
2447
2668
|
CRITICAL: Base your answers ONLY on the actual audio/transcript content.
|
|
2448
|
-
|
|
2449
|
-
playback IDs, or technical parameters visible in this request. These are
|
|
2450
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
2669
|
+
${METADATA_BOUNDARY_WARNING}
|
|
2451
2670
|
|
|
2452
2671
|
CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
|
|
2453
2672
|
Answering an irrelevant question is WRONG — you MUST skip it instead.
|
|
@@ -2467,8 +2686,8 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
|
|
|
2467
2686
|
- You MUST answer every relevant question with one of its own listed allowed response options
|
|
2468
2687
|
- Skip irrelevant questions as described in relevance_filtering
|
|
2469
2688
|
- Only describe observable evidence from transcript content
|
|
2470
|
-
-
|
|
2471
|
-
-
|
|
2689
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
2690
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
2472
2691
|
- Provide reasoning in the same language as the question
|
|
2473
2692
|
</constraints>
|
|
2474
2693
|
|
|
@@ -2486,7 +2705,7 @@ function buildUserPrompt({
|
|
|
2486
2705
|
isAudioOnly = false
|
|
2487
2706
|
}) {
|
|
2488
2707
|
const contentDescriptor = isAudioOnly ? "audio content using transcript evidence" : "content";
|
|
2489
|
-
const taskContent =
|
|
2708
|
+
const taskContent = dedent2`
|
|
2490
2709
|
Answer each question in the <questions> block below about the ${contentDescriptor}.
|
|
2491
2710
|
Use only the values listed inside each question's own <allowed_answers> element.
|
|
2492
2711
|
Return one answer per question, in the order the questions appear.
|
|
@@ -2733,14 +2952,14 @@ async function askQuestions(assetId, questions, options) {
|
|
|
2733
2952
|
|
|
2734
2953
|
// src/workflows/burned-in-captions.ts
|
|
2735
2954
|
import { generateText as generateText2, Output as Output2 } from "ai";
|
|
2736
|
-
import
|
|
2955
|
+
import dedent3 from "dedent";
|
|
2737
2956
|
import { z as z3 } from "zod";
|
|
2738
2957
|
var burnedInCaptionsSchema = z3.object({
|
|
2739
2958
|
hasBurnedInCaptions: z3.boolean(),
|
|
2740
2959
|
confidence: z3.number(),
|
|
2741
2960
|
detectedLanguage: z3.string().nullable()
|
|
2742
2961
|
});
|
|
2743
|
-
var SYSTEM_PROMPT2 =
|
|
2962
|
+
var SYSTEM_PROMPT2 = promptDedent`
|
|
2744
2963
|
<role>
|
|
2745
2964
|
You are an expert at analyzing video frames to detect burned-in captions (also called open captions or hardcoded subtitles).
|
|
2746
2965
|
These are text overlays that are permanently embedded in the video image, common on TikTok, Instagram Reels, and other social media platforms.
|
|
@@ -2763,8 +2982,7 @@ var SYSTEM_PROMPT2 = dedent2`
|
|
|
2763
2982
|
|
|
2764
2983
|
<context>
|
|
2765
2984
|
You receive storyboard images containing multiple sequential frames extracted from a video.
|
|
2766
|
-
|
|
2767
|
-
Read frames left-to-right, top-to-bottom to understand the temporal sequence.
|
|
2985
|
+
${STORYBOARD_FRAME_INSTRUCTIONS}
|
|
2768
2986
|
</context>
|
|
2769
2987
|
|
|
2770
2988
|
<capabilities>
|
|
@@ -2777,23 +2995,21 @@ var SYSTEM_PROMPT2 = dedent2`
|
|
|
2777
2995
|
<constraints>
|
|
2778
2996
|
- Only classify as burned-in captions when evidence is clear across multiple frames
|
|
2779
2997
|
- Base decisions on observable visual evidence
|
|
2780
|
-
-
|
|
2781
|
-
|
|
2782
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
2783
|
-
- Return structured data matching the requested schema
|
|
2998
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
2999
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
2784
3000
|
</constraints>`;
|
|
2785
3001
|
var burnedInCaptionsPromptBuilder = createPromptBuilder({
|
|
2786
3002
|
template: {
|
|
2787
3003
|
task: {
|
|
2788
3004
|
tag: "task",
|
|
2789
|
-
content:
|
|
3005
|
+
content: dedent3`
|
|
2790
3006
|
Analyze the provided video storyboard to detect burned-in captions (hardcoded subtitles).
|
|
2791
3007
|
Count frames with text vs no text, note position consistency and whether text changes across frames.
|
|
2792
3008
|
Decide if captions exist, with confidence (0.0-1.0) and detected language if any.`
|
|
2793
3009
|
},
|
|
2794
3010
|
analysisSteps: {
|
|
2795
3011
|
tag: "analysis_steps",
|
|
2796
|
-
content:
|
|
3012
|
+
content: dedent3`
|
|
2797
3013
|
1. COUNT how many frames contain text overlays vs. how many don't
|
|
2798
3014
|
2. Check if text appears in consistent positions across multiple frames
|
|
2799
3015
|
3. Verify text changes content between frames (indicating dialogue/narration)
|
|
@@ -2802,7 +3018,7 @@ var burnedInCaptionsPromptBuilder = createPromptBuilder({
|
|
|
2802
3018
|
},
|
|
2803
3019
|
positiveIndicators: {
|
|
2804
3020
|
tag: "classify_as_captions",
|
|
2805
|
-
content:
|
|
3021
|
+
content: dedent3`
|
|
2806
3022
|
ONLY classify as burned-in captions if:
|
|
2807
3023
|
- Text appears in multiple frames (not just 1-2 end frames)
|
|
2808
3024
|
- Text positioning is consistent across those frames
|
|
@@ -2811,7 +3027,7 @@ var burnedInCaptionsPromptBuilder = createPromptBuilder({
|
|
|
2811
3027
|
},
|
|
2812
3028
|
negativeIndicators: {
|
|
2813
3029
|
tag: "not_captions",
|
|
2814
|
-
content:
|
|
3030
|
+
content: dedent3`
|
|
2815
3031
|
DO NOT classify as burned-in captions:
|
|
2816
3032
|
- Marketing taglines appearing only in final 1-2 frames
|
|
2817
3033
|
- Single words or phrases that don't change between frames
|
|
@@ -2934,168 +3150,8 @@ async function hasBurnedInCaptions(assetId, options = {}) {
|
|
|
2934
3150
|
|
|
2935
3151
|
// src/workflows/chapters.ts
|
|
2936
3152
|
import { generateText as generateText3, Output as Output3 } from "ai";
|
|
2937
|
-
import
|
|
3153
|
+
import dedent4 from "dedent";
|
|
2938
3154
|
import { z as z4 } from "zod";
|
|
2939
|
-
|
|
2940
|
-
// src/lib/language-codes.ts
|
|
2941
|
-
var ISO639_1_TO_3 = {
|
|
2942
|
-
// Major world languages
|
|
2943
|
-
en: "eng",
|
|
2944
|
-
// English
|
|
2945
|
-
es: "spa",
|
|
2946
|
-
// Spanish
|
|
2947
|
-
fr: "fra",
|
|
2948
|
-
// French
|
|
2949
|
-
de: "deu",
|
|
2950
|
-
// German
|
|
2951
|
-
it: "ita",
|
|
2952
|
-
// Italian
|
|
2953
|
-
pt: "por",
|
|
2954
|
-
// Portuguese
|
|
2955
|
-
ru: "rus",
|
|
2956
|
-
// Russian
|
|
2957
|
-
zh: "zho",
|
|
2958
|
-
// Chinese
|
|
2959
|
-
ja: "jpn",
|
|
2960
|
-
// Japanese
|
|
2961
|
-
ko: "kor",
|
|
2962
|
-
// Korean
|
|
2963
|
-
ar: "ara",
|
|
2964
|
-
// Arabic
|
|
2965
|
-
hi: "hin",
|
|
2966
|
-
// Hindi
|
|
2967
|
-
// European languages
|
|
2968
|
-
nl: "nld",
|
|
2969
|
-
// Dutch
|
|
2970
|
-
pl: "pol",
|
|
2971
|
-
// Polish
|
|
2972
|
-
sv: "swe",
|
|
2973
|
-
// Swedish
|
|
2974
|
-
da: "dan",
|
|
2975
|
-
// Danish
|
|
2976
|
-
no: "nor",
|
|
2977
|
-
// Norwegian
|
|
2978
|
-
fi: "fin",
|
|
2979
|
-
// Finnish
|
|
2980
|
-
el: "ell",
|
|
2981
|
-
// Greek
|
|
2982
|
-
cs: "ces",
|
|
2983
|
-
// Czech
|
|
2984
|
-
hu: "hun",
|
|
2985
|
-
// Hungarian
|
|
2986
|
-
ro: "ron",
|
|
2987
|
-
// Romanian
|
|
2988
|
-
bg: "bul",
|
|
2989
|
-
// Bulgarian
|
|
2990
|
-
hr: "hrv",
|
|
2991
|
-
// Croatian
|
|
2992
|
-
sk: "slk",
|
|
2993
|
-
// Slovak
|
|
2994
|
-
sl: "slv",
|
|
2995
|
-
// Slovenian
|
|
2996
|
-
uk: "ukr",
|
|
2997
|
-
// Ukrainian
|
|
2998
|
-
tr: "tur",
|
|
2999
|
-
// Turkish
|
|
3000
|
-
// Asian languages
|
|
3001
|
-
th: "tha",
|
|
3002
|
-
// Thai
|
|
3003
|
-
vi: "vie",
|
|
3004
|
-
// Vietnamese
|
|
3005
|
-
id: "ind",
|
|
3006
|
-
// Indonesian
|
|
3007
|
-
ms: "msa",
|
|
3008
|
-
// Malay
|
|
3009
|
-
tl: "tgl",
|
|
3010
|
-
// Tagalog/Filipino
|
|
3011
|
-
// Other languages
|
|
3012
|
-
he: "heb",
|
|
3013
|
-
// Hebrew
|
|
3014
|
-
fa: "fas",
|
|
3015
|
-
// Persian/Farsi
|
|
3016
|
-
bn: "ben",
|
|
3017
|
-
// Bengali
|
|
3018
|
-
ta: "tam",
|
|
3019
|
-
// Tamil
|
|
3020
|
-
te: "tel",
|
|
3021
|
-
// Telugu
|
|
3022
|
-
mr: "mar",
|
|
3023
|
-
// Marathi
|
|
3024
|
-
gu: "guj",
|
|
3025
|
-
// Gujarati
|
|
3026
|
-
kn: "kan",
|
|
3027
|
-
// Kannada
|
|
3028
|
-
ml: "mal",
|
|
3029
|
-
// Malayalam
|
|
3030
|
-
pa: "pan",
|
|
3031
|
-
// Punjabi
|
|
3032
|
-
ur: "urd",
|
|
3033
|
-
// Urdu
|
|
3034
|
-
sw: "swa",
|
|
3035
|
-
// Swahili
|
|
3036
|
-
af: "afr",
|
|
3037
|
-
// Afrikaans
|
|
3038
|
-
ca: "cat",
|
|
3039
|
-
// Catalan
|
|
3040
|
-
eu: "eus",
|
|
3041
|
-
// Basque
|
|
3042
|
-
gl: "glg",
|
|
3043
|
-
// Galician
|
|
3044
|
-
is: "isl",
|
|
3045
|
-
// Icelandic
|
|
3046
|
-
et: "est",
|
|
3047
|
-
// Estonian
|
|
3048
|
-
lv: "lav",
|
|
3049
|
-
// Latvian
|
|
3050
|
-
lt: "lit"
|
|
3051
|
-
// Lithuanian
|
|
3052
|
-
};
|
|
3053
|
-
var ISO639_3_TO_1 = Object.fromEntries(
|
|
3054
|
-
Object.entries(ISO639_1_TO_3).map(([iso1, iso3]) => [iso3, iso1])
|
|
3055
|
-
);
|
|
3056
|
-
function toISO639_3(code) {
|
|
3057
|
-
const normalized = code.toLowerCase().trim();
|
|
3058
|
-
if (normalized.length === 3) {
|
|
3059
|
-
return normalized;
|
|
3060
|
-
}
|
|
3061
|
-
return ISO639_1_TO_3[normalized] ?? normalized;
|
|
3062
|
-
}
|
|
3063
|
-
function toISO639_1(code) {
|
|
3064
|
-
const normalized = code.toLowerCase().trim();
|
|
3065
|
-
if (normalized.length === 2) {
|
|
3066
|
-
return normalized;
|
|
3067
|
-
}
|
|
3068
|
-
return ISO639_3_TO_1[normalized] ?? normalized;
|
|
3069
|
-
}
|
|
3070
|
-
function getLanguageCodePair(code) {
|
|
3071
|
-
const normalized = code.toLowerCase().trim();
|
|
3072
|
-
if (normalized.length === 2) {
|
|
3073
|
-
return {
|
|
3074
|
-
iso639_1: normalized,
|
|
3075
|
-
iso639_3: toISO639_3(normalized)
|
|
3076
|
-
};
|
|
3077
|
-
} else if (normalized.length === 3) {
|
|
3078
|
-
return {
|
|
3079
|
-
iso639_1: toISO639_1(normalized),
|
|
3080
|
-
iso639_3: normalized
|
|
3081
|
-
};
|
|
3082
|
-
}
|
|
3083
|
-
return {
|
|
3084
|
-
iso639_1: normalized,
|
|
3085
|
-
iso639_3: normalized
|
|
3086
|
-
};
|
|
3087
|
-
}
|
|
3088
|
-
function getLanguageName(code) {
|
|
3089
|
-
const iso639_1 = toISO639_1(code);
|
|
3090
|
-
try {
|
|
3091
|
-
const displayNames = new Intl.DisplayNames(["en"], { type: "language" });
|
|
3092
|
-
return displayNames.of(iso639_1) ?? code.toUpperCase();
|
|
3093
|
-
} catch {
|
|
3094
|
-
return code.toUpperCase();
|
|
3095
|
-
}
|
|
3096
|
-
}
|
|
3097
|
-
|
|
3098
|
-
// src/workflows/chapters.ts
|
|
3099
3155
|
var chapterSchema = z4.object({
|
|
3100
3156
|
startTime: z4.number(),
|
|
3101
3157
|
title: z4.string()
|
|
@@ -3147,13 +3203,13 @@ var chapterSystemPromptBuilder = createPromptBuilder({
|
|
|
3147
3203
|
},
|
|
3148
3204
|
context: {
|
|
3149
3205
|
tag: "context",
|
|
3150
|
-
content:
|
|
3206
|
+
content: dedent4`
|
|
3151
3207
|
You receive a timestamped transcript with lines in the form "[12s] Caption text".
|
|
3152
3208
|
Use those timestamps as anchors to determine chapter start times in seconds.`
|
|
3153
3209
|
},
|
|
3154
3210
|
constraints: {
|
|
3155
3211
|
tag: "constraints",
|
|
3156
|
-
content:
|
|
3212
|
+
content: dedent4`
|
|
3157
3213
|
- Only use information present in the transcript
|
|
3158
3214
|
- Return structured data that matches the requested JSON schema
|
|
3159
3215
|
- Do not add commentary or extra text outside the JSON
|
|
@@ -3161,7 +3217,7 @@ var chapterSystemPromptBuilder = createPromptBuilder({
|
|
|
3161
3217
|
},
|
|
3162
3218
|
qualityGuidelines: {
|
|
3163
3219
|
tag: "quality_guidelines",
|
|
3164
|
-
content:
|
|
3220
|
+
content: dedent4`
|
|
3165
3221
|
- Create chapters at topic shifts or clear transitions
|
|
3166
3222
|
- Keep chapter titles concise and descriptive
|
|
3167
3223
|
- Ensure the first chapter starts at 0 seconds`
|
|
@@ -3171,7 +3227,7 @@ var chapterSystemPromptBuilder = createPromptBuilder({
|
|
|
3171
3227
|
});
|
|
3172
3228
|
var AUDIO_ONLY_SYSTEM_PROMPT_OVERRIDES = {
|
|
3173
3229
|
role: "You are an audio editor and transcript analyst specializing in segmenting content into logical chapters.",
|
|
3174
|
-
context:
|
|
3230
|
+
context: dedent4`
|
|
3175
3231
|
You receive a timestamped transcript from audio-only content with lines in the form "[12s] Transcript text".
|
|
3176
3232
|
Use those timestamps as anchors to determine chapter start times in seconds.`
|
|
3177
3233
|
};
|
|
@@ -3183,7 +3239,7 @@ var chaptersPromptBuilder = createPromptBuilder({
|
|
|
3183
3239
|
},
|
|
3184
3240
|
outputFormat: {
|
|
3185
3241
|
tag: "output_format",
|
|
3186
|
-
content:
|
|
3242
|
+
content: dedent4`
|
|
3187
3243
|
Return valid JSON in this exact shape:
|
|
3188
3244
|
{
|
|
3189
3245
|
"chapters": [
|
|
@@ -3202,7 +3258,7 @@ var chaptersPromptBuilder = createPromptBuilder({
|
|
|
3202
3258
|
},
|
|
3203
3259
|
titleGuidelines: {
|
|
3204
3260
|
tag: "title_guidelines",
|
|
3205
|
-
content:
|
|
3261
|
+
content: dedent4`
|
|
3206
3262
|
- Keep titles concise and descriptive
|
|
3207
3263
|
- Avoid filler or generic labels like "Chapter 1"
|
|
3208
3264
|
- Use the transcript's terminology`
|
|
@@ -3227,7 +3283,7 @@ function buildUserPrompt3({
|
|
|
3227
3283
|
if (languageName) {
|
|
3228
3284
|
contextSections.push(createLanguageSection(languageName));
|
|
3229
3285
|
}
|
|
3230
|
-
const dynamicChapterGuidelines =
|
|
3286
|
+
const dynamicChapterGuidelines = dedent4`
|
|
3231
3287
|
- Create at least ${minChaptersPerHour} and at most ${maxChaptersPerHour} chapters per hour of content
|
|
3232
3288
|
- Use start times in seconds (not HH:MM:SS)
|
|
3233
3289
|
- Chapter start times should be non-decreasing
|
|
@@ -3285,7 +3341,7 @@ async function generateChapters(assetId, options = {}) {
|
|
|
3285
3341
|
const contentLabel = isAudioOnly ? "transcript" : "caption track";
|
|
3286
3342
|
throw new MuxAiError(`No usable content found in ${contentLabel}.`, { type: "validation_error" });
|
|
3287
3343
|
}
|
|
3288
|
-
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : transcriptResult.track
|
|
3344
|
+
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : getReliableLanguageCode(transcriptResult.track) ?? languageCode;
|
|
3289
3345
|
const languageName = resolvedLanguageCode ? getLanguageName(resolvedLanguageCode) : void 0;
|
|
3290
3346
|
const userPrompt = buildUserPrompt3({
|
|
3291
3347
|
timestampedTranscript,
|
|
@@ -3327,7 +3383,7 @@ async function generateChapters(assetId, options = {}) {
|
|
|
3327
3383
|
};
|
|
3328
3384
|
return {
|
|
3329
3385
|
assetId,
|
|
3330
|
-
languageCode: languageCode ?? transcriptResult.track
|
|
3386
|
+
languageCode: languageCode ?? getReliableLanguageCode(transcriptResult.track) ?? "en",
|
|
3331
3387
|
chapters: validChapters,
|
|
3332
3388
|
usage: usageWithMetadata
|
|
3333
3389
|
};
|
|
@@ -3335,7 +3391,7 @@ async function generateChapters(assetId, options = {}) {
|
|
|
3335
3391
|
|
|
3336
3392
|
// src/workflows/edit-captions.ts
|
|
3337
3393
|
import { generateText as generateText4, Output as Output4 } from "ai";
|
|
3338
|
-
import
|
|
3394
|
+
import dedent5 from "dedent";
|
|
3339
3395
|
import { z as z5 } from "zod";
|
|
3340
3396
|
|
|
3341
3397
|
// src/lib/mux-tracks.ts
|
|
@@ -3412,7 +3468,7 @@ var profanityDetectionSchema = z5.object({
|
|
|
3412
3468
|
"Unique profane words or short phrases exactly as they appear in the transcript text. Include each distinct form only once (e.g., if 'fuck' and 'fucking' both appear, list both)."
|
|
3413
3469
|
)
|
|
3414
3470
|
});
|
|
3415
|
-
var SYSTEM_PROMPT3 =
|
|
3471
|
+
var SYSTEM_PROMPT3 = dedent5`
|
|
3416
3472
|
You are a content moderation assistant. Your task is to identify profane, vulgar, or obscene
|
|
3417
3473
|
words and phrases in subtitle text. Return ONLY the exact profane words or phrases as they appear
|
|
3418
3474
|
in the text. Do not modify, censor, or paraphrase them. Do not include words that are merely
|
|
@@ -3888,7 +3944,7 @@ async function generateVideoEmbeddings(assetId, options = {}) {
|
|
|
3888
3944
|
|
|
3889
3945
|
// src/workflows/engagement-insights.ts
|
|
3890
3946
|
import { generateText as generateText5, Output as Output5 } from "ai";
|
|
3891
|
-
import
|
|
3947
|
+
import dedent6 from "dedent";
|
|
3892
3948
|
import { z as z6 } from "zod";
|
|
3893
3949
|
var aiMomentInsightSchema = z6.object({
|
|
3894
3950
|
hotspotIndex: z6.number(),
|
|
@@ -3906,7 +3962,7 @@ var engagementInsightsSchema = z6.object({
|
|
|
3906
3962
|
momentInsights: z6.array(aiMomentInsightSchema),
|
|
3907
3963
|
overallInsight: aiOverallInsightSchema
|
|
3908
3964
|
});
|
|
3909
|
-
var SYSTEM_PROMPT4 =
|
|
3965
|
+
var SYSTEM_PROMPT4 = promptDedent`
|
|
3910
3966
|
<role>
|
|
3911
3967
|
You are a video engagement analyst specializing in understanding viewer behavior.
|
|
3912
3968
|
Your job is to explain why specific moments in videos have high or low engagement
|
|
@@ -3934,12 +3990,10 @@ var SYSTEM_PROMPT4 = dedent5`
|
|
|
3934
3990
|
|
|
3935
3991
|
<constraints>
|
|
3936
3992
|
- Only describe what you can see in images or read in transcripts
|
|
3937
|
-
-
|
|
3938
|
-
-
|
|
3939
|
-
playback IDs, or technical parameters visible in this request. These are
|
|
3940
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
3993
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
3994
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
3941
3995
|
- Correlate engagement scores with observable content
|
|
3942
|
-
-
|
|
3996
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
3943
3997
|
- Each momentInsight MUST include the hotspotIndex from the input data
|
|
3944
3998
|
</constraints>
|
|
3945
3999
|
|
|
@@ -3950,7 +4004,7 @@ var SYSTEM_PROMPT4 = dedent5`
|
|
|
3950
4004
|
- Connect engagement data to content features
|
|
3951
4005
|
</quality_guidelines>
|
|
3952
4006
|
`;
|
|
3953
|
-
var AUDIO_ONLY_SYSTEM_PROMPT2 =
|
|
4007
|
+
var AUDIO_ONLY_SYSTEM_PROMPT2 = promptDedent`
|
|
3954
4008
|
<role>
|
|
3955
4009
|
You are a video engagement analyst specializing in understanding viewer behavior for audio content.
|
|
3956
4010
|
Your job is to explain why specific moments have high or low engagement based on audio/dialogue and viewer watch patterns.
|
|
@@ -3977,12 +4031,10 @@ var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent5`
|
|
|
3977
4031
|
|
|
3978
4032
|
<constraints>
|
|
3979
4033
|
- Only describe what you can read in the transcript
|
|
3980
|
-
-
|
|
3981
|
-
-
|
|
3982
|
-
playback IDs, or technical parameters visible in this request. These are
|
|
3983
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
4034
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
4035
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
3984
4036
|
- Correlate engagement scores with observable content
|
|
3985
|
-
-
|
|
4037
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
3986
4038
|
- Each momentInsight MUST include the hotspotIndex from the input data
|
|
3987
4039
|
</constraints>
|
|
3988
4040
|
|
|
@@ -3993,7 +4045,7 @@ var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent5`
|
|
|
3993
4045
|
- Connect engagement data to content features
|
|
3994
4046
|
</quality_guidelines>
|
|
3995
4047
|
`;
|
|
3996
|
-
var INSIGHT_GUIDELINES =
|
|
4048
|
+
var INSIGHT_GUIDELINES = dedent6`
|
|
3997
4049
|
Provide explanatory insights that describe WHY moments are engaging.
|
|
3998
4050
|
Focus on observable patterns and viewer behavior correlation.
|
|
3999
4051
|
Example: "The cooking demonstration at 2:15 has 3x average engagement
|
|
@@ -4011,7 +4063,7 @@ var engagementInsightsPromptBuilder = createPromptBuilder({
|
|
|
4011
4063
|
},
|
|
4012
4064
|
outputFormat: {
|
|
4013
4065
|
tag: "output_format",
|
|
4014
|
-
content:
|
|
4066
|
+
content: dedent6`
|
|
4015
4067
|
Return valid JSON matching the provided schema.
|
|
4016
4068
|
Include insights for each hotspot and overall engagement trends.
|
|
4017
4069
|
Each momentInsight must include the hotspotIndex matching the input.
|
|
@@ -4019,7 +4071,7 @@ var engagementInsightsPromptBuilder = createPromptBuilder({
|
|
|
4019
4071
|
},
|
|
4020
4072
|
visualContext: {
|
|
4021
4073
|
tag: "visual_context",
|
|
4022
|
-
content:
|
|
4074
|
+
content: dedent6`
|
|
4023
4075
|
Images are provided for each hotspot timestamp.
|
|
4024
4076
|
A storyboard overview image may also be included showing the full video timeline.
|
|
4025
4077
|
Use visual evidence from these images to support your engagement analysis.
|
|
@@ -4033,14 +4085,14 @@ function buildUserPrompt4(hotspots, transcriptSegments, heatmap, isAudioOnly) {
|
|
|
4033
4085
|
const transcript = transcriptSegments[idx]?.text || "(no transcript)";
|
|
4034
4086
|
const startTimestamp = secondsToTimestamp(h.startMs / 1e3);
|
|
4035
4087
|
const endTimestamp = secondsToTimestamp(h.endMs / 1e3);
|
|
4036
|
-
return
|
|
4088
|
+
return dedent6`
|
|
4037
4089
|
Hotspot ${idx} (hotspotIndex: ${idx}):
|
|
4038
4090
|
- Time Range: ${startTimestamp} - ${endTimestamp}
|
|
4039
4091
|
- Engagement Score: ${h.score.toFixed(2)} (0=low, 1=high)
|
|
4040
4092
|
- Transcript: "${transcript}"
|
|
4041
4093
|
`;
|
|
4042
4094
|
}).join("\n\n");
|
|
4043
|
-
const heatmapData =
|
|
4095
|
+
const heatmapData = dedent6`
|
|
4044
4096
|
Engagement Heatmap (${heatmap.length} values, each representing 1/${heatmap.length}th of the video, higher = more re-watches):
|
|
4045
4097
|
[${heatmap.map((v) => v.toFixed(2)).join(", ")}]
|
|
4046
4098
|
`;
|
|
@@ -4914,7 +4966,7 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
4914
4966
|
|
|
4915
4967
|
// src/workflows/summarization.ts
|
|
4916
4968
|
import { generateText as generateText6, Output as Output6 } from "ai";
|
|
4917
|
-
import
|
|
4969
|
+
import dedent7 from "dedent";
|
|
4918
4970
|
import { z as z7 } from "zod";
|
|
4919
4971
|
var DEFAULT_SUMMARY_KEYWORD_LIMIT = 10;
|
|
4920
4972
|
var DEFAULT_TITLE_LENGTH = 10;
|
|
@@ -4940,36 +4992,36 @@ var DESCRIPTION_LENGTH_THRESHOLD_LARGE = 100;
|
|
|
4940
4992
|
function buildDescriptionGuidance(wordCount, contentType) {
|
|
4941
4993
|
if (wordCount < DESCRIPTION_LENGTH_THRESHOLD_SMALL) {
|
|
4942
4994
|
if (contentType === "video") {
|
|
4943
|
-
return
|
|
4995
|
+
return dedent7`A brief summary of the video in no more than ${wordCount} words. Shorter is fine.
|
|
4944
4996
|
Focus on the single most important subject or action.
|
|
4945
4997
|
Write in present tense.`;
|
|
4946
4998
|
}
|
|
4947
|
-
return
|
|
4999
|
+
return dedent7`A brief summary of the audio content in no more than ${wordCount} words. Shorter is fine.
|
|
4948
5000
|
Focus on the single most important topic or theme.
|
|
4949
5001
|
Write in present tense.`;
|
|
4950
5002
|
}
|
|
4951
5003
|
if (wordCount > DESCRIPTION_LENGTH_THRESHOLD_LARGE) {
|
|
4952
5004
|
if (contentType === "video") {
|
|
4953
|
-
return
|
|
5005
|
+
return dedent7`A detailed summary that describes what happens across the video.
|
|
4954
5006
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4955
5007
|
Be thorough: cover subjects, actions, setting, progression, and any notable details visible across frames.
|
|
4956
5008
|
Write in present tense. Be specific about observable details rather than making assumptions.
|
|
4957
5009
|
If the transcript provides dialogue or narration, incorporate key points but prioritize visual content.`;
|
|
4958
5010
|
}
|
|
4959
|
-
return
|
|
5011
|
+
return dedent7`A detailed summary that describes the audio content.
|
|
4960
5012
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4961
5013
|
Be thorough: cover topics, speakers, themes, progression, and any notable insights.
|
|
4962
5014
|
Write in present tense. Be specific about what is discussed or presented rather than making assumptions.
|
|
4963
5015
|
Focus on the spoken content and any key insights, dialogue, or narrative elements.`;
|
|
4964
5016
|
}
|
|
4965
5017
|
if (contentType === "video") {
|
|
4966
|
-
return
|
|
5018
|
+
return dedent7`A summary that describes what happens across the video.
|
|
4967
5019
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4968
5020
|
Cover the main subjects, actions, setting, and any notable progression visible across frames.
|
|
4969
5021
|
Write in present tense. Be specific about observable details rather than making assumptions.
|
|
4970
5022
|
If the transcript provides dialogue or narration, incorporate key points but prioritize visual content.`;
|
|
4971
5023
|
}
|
|
4972
|
-
return
|
|
5024
|
+
return dedent7`A summary that describes the audio content.
|
|
4973
5025
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4974
5026
|
Cover the main topics, speakers, themes, and any notable progression in the discussion or narration.
|
|
4975
5027
|
Write in present tense. Be specific about what is discussed or presented rather than making assumptions.
|
|
@@ -4986,7 +5038,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
4986
5038
|
},
|
|
4987
5039
|
title: {
|
|
4988
5040
|
tag: "title_requirements",
|
|
4989
|
-
content:
|
|
5041
|
+
content: dedent7`
|
|
4990
5042
|
A concise, label-style title — not a sentence or description.
|
|
4991
5043
|
Never exceed ${titleLimit} words, but shorter is better.
|
|
4992
5044
|
Think of how a video card title, playlist entry, or file name would read — e.g. "Predator: Badlands Trailer" or "Chef Prepares Holiday Feast".
|
|
@@ -4999,7 +5051,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
4999
5051
|
},
|
|
5000
5052
|
keywords: {
|
|
5001
5053
|
tag: "keywords_requirements",
|
|
5002
|
-
content:
|
|
5054
|
+
content: dedent7`
|
|
5003
5055
|
Specific, searchable terms (up to ${keywordLimit}) that capture:
|
|
5004
5056
|
- Primary subjects (people, animals, objects)
|
|
5005
5057
|
- Actions and activities being performed
|
|
@@ -5011,7 +5063,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
5011
5063
|
},
|
|
5012
5064
|
qualityGuidelines: {
|
|
5013
5065
|
tag: "quality_guidelines",
|
|
5014
|
-
content:
|
|
5066
|
+
content: dedent7`
|
|
5015
5067
|
- Examine all frames to understand the full context and progression
|
|
5016
5068
|
- Be precise: "golden retriever" is better than "dog" when identifiable
|
|
5017
5069
|
- Capture the narrative: what begins, develops, and concludes
|
|
@@ -5032,7 +5084,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
5032
5084
|
},
|
|
5033
5085
|
title: {
|
|
5034
5086
|
tag: "title_requirements",
|
|
5035
|
-
content:
|
|
5087
|
+
content: dedent7`
|
|
5036
5088
|
A concise, label-style title — not a sentence or description.
|
|
5037
5089
|
Never exceed ${titleLimit} words, but shorter is better.
|
|
5038
5090
|
Think of how a podcast episode title or playlist entry would read — e.g. "Weekly News Roundup" or "Interview with Dr. Smith".
|
|
@@ -5045,7 +5097,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
5045
5097
|
},
|
|
5046
5098
|
keywords: {
|
|
5047
5099
|
tag: "keywords_requirements",
|
|
5048
|
-
content:
|
|
5100
|
+
content: dedent7`
|
|
5049
5101
|
Specific, searchable terms (up to ${keywordLimit}) that capture:
|
|
5050
5102
|
- Primary topics and themes
|
|
5051
5103
|
- Speakers or presenters (if named)
|
|
@@ -5057,7 +5109,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
5057
5109
|
},
|
|
5058
5110
|
qualityGuidelines: {
|
|
5059
5111
|
tag: "quality_guidelines",
|
|
5060
|
-
content:
|
|
5112
|
+
content: dedent7`
|
|
5061
5113
|
- Analyze the full transcript to understand context and themes
|
|
5062
5114
|
- Be precise: use specific terminology when mentioned
|
|
5063
5115
|
- Capture the narrative: what is introduced, discussed, and concluded
|
|
@@ -5067,15 +5119,14 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
5067
5119
|
sectionOrder: ["task", "title", "description", "keywords", "qualityGuidelines"]
|
|
5068
5120
|
});
|
|
5069
5121
|
}
|
|
5070
|
-
var SYSTEM_PROMPT5 =
|
|
5122
|
+
var SYSTEM_PROMPT5 = promptDedent`
|
|
5071
5123
|
<role>
|
|
5072
5124
|
You are a video content analyst specializing in storyboard interpretation and multimodal analysis.
|
|
5073
5125
|
</role>
|
|
5074
5126
|
|
|
5075
5127
|
<context>
|
|
5076
5128
|
You receive storyboard images containing multiple sequential frames extracted from a video.
|
|
5077
|
-
|
|
5078
|
-
Read frames left-to-right, top-to-bottom to understand the temporal sequence.
|
|
5129
|
+
${STORYBOARD_FRAME_INSTRUCTIONS}
|
|
5079
5130
|
</context>
|
|
5080
5131
|
|
|
5081
5132
|
<transcript_guidance>
|
|
@@ -5097,37 +5148,21 @@ var SYSTEM_PROMPT5 = dedent6`
|
|
|
5097
5148
|
|
|
5098
5149
|
<constraints>
|
|
5099
5150
|
- Only describe what is clearly observable in the frames or explicitly stated in the transcript
|
|
5100
|
-
-
|
|
5101
|
-
-
|
|
5102
|
-
|
|
5103
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
5104
|
-
- Return structured data matching the requested schema
|
|
5151
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
5152
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
5153
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
5105
5154
|
- Output only the JSON object; no markdown or extra text
|
|
5106
5155
|
- When a <language> section is provided, all output text MUST be written in that language
|
|
5107
5156
|
</constraints>
|
|
5108
5157
|
|
|
5109
5158
|
<tone_guidance>
|
|
5110
|
-
|
|
5111
|
-
Adapt your entire analysis and writing style to match the specified tone - this should influence
|
|
5112
|
-
your word choice, personality, formality level, and overall presentation of the content.
|
|
5113
|
-
The tone instructions are not suggestions but core requirements for how you should express yourself.
|
|
5159
|
+
${TONE_GUIDANCE}
|
|
5114
5160
|
</tone_guidance>
|
|
5115
5161
|
|
|
5116
5162
|
<language_guidelines>
|
|
5117
|
-
|
|
5118
|
-
- "The image shows..." / "The storyboard shows..."
|
|
5119
|
-
- "In this video..." / "This video features..."
|
|
5120
|
-
- "The frames depict..." / "The footage shows..."
|
|
5121
|
-
- "We can see..." / "You can see..."
|
|
5122
|
-
- "The clip shows..." / "The scene shows..."
|
|
5123
|
-
|
|
5124
|
-
INSTEAD, describe the content directly:
|
|
5125
|
-
- BAD: "The video shows a chef preparing a meal"
|
|
5126
|
-
- GOOD: "A chef prepares a meal in a professional kitchen"
|
|
5127
|
-
|
|
5128
|
-
Write as if describing reality, not describing a recording of reality.
|
|
5163
|
+
${createLanguageGuidelines("video")}
|
|
5129
5164
|
</language_guidelines>`;
|
|
5130
|
-
var AUDIO_ONLY_SYSTEM_PROMPT3 =
|
|
5165
|
+
var AUDIO_ONLY_SYSTEM_PROMPT3 = promptDedent`
|
|
5131
5166
|
<role>
|
|
5132
5167
|
You are an audio content analyst specializing in transcript analysis and metadata generation.
|
|
5133
5168
|
</role>
|
|
@@ -5154,35 +5189,20 @@ var AUDIO_ONLY_SYSTEM_PROMPT3 = dedent6`
|
|
|
5154
5189
|
|
|
5155
5190
|
<constraints>
|
|
5156
5191
|
- Only describe what is explicitly stated or strongly implied in the transcript
|
|
5157
|
-
-
|
|
5158
|
-
-
|
|
5159
|
-
|
|
5160
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
5161
|
-
- Return structured data matching the requested schema
|
|
5192
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
5193
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
5194
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
5162
5195
|
- Focus entirely on audio/spoken content - there are no visual elements
|
|
5163
5196
|
- Output only the JSON object; no markdown or extra text
|
|
5164
5197
|
- When a <language> section is provided, all output text MUST be written in that language
|
|
5165
5198
|
</constraints>
|
|
5166
5199
|
|
|
5167
5200
|
<tone_guidance>
|
|
5168
|
-
|
|
5169
|
-
Adapt your entire analysis and writing style to match the specified tone - this should influence
|
|
5170
|
-
your word choice, personality, formality level, and overall presentation of the content.
|
|
5171
|
-
The tone instructions are not suggestions but core requirements for how you should express yourself.
|
|
5201
|
+
${TONE_GUIDANCE}
|
|
5172
5202
|
</tone_guidance>
|
|
5173
5203
|
|
|
5174
5204
|
<language_guidelines>
|
|
5175
|
-
|
|
5176
|
-
- "The audio shows..." / "The transcript shows..."
|
|
5177
|
-
- "In this recording..." / "This audio features..."
|
|
5178
|
-
- "The speaker says..." / "We can hear..."
|
|
5179
|
-
- "The clip contains..." / "The recording shows..."
|
|
5180
|
-
|
|
5181
|
-
INSTEAD, describe the content directly:
|
|
5182
|
-
- BAD: "The audio features a discussion about climate change"
|
|
5183
|
-
- GOOD: "A panel discusses climate change impacts and solutions"
|
|
5184
|
-
|
|
5185
|
-
Write as if describing reality, not describing a recording of reality.
|
|
5205
|
+
${createLanguageGuidelines("audio")}
|
|
5186
5206
|
</language_guidelines>`;
|
|
5187
5207
|
function buildUserPrompt5({
|
|
5188
5208
|
tone,
|
|
@@ -5356,7 +5376,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
5356
5376
|
required: isAudioOnly
|
|
5357
5377
|
}) : void 0;
|
|
5358
5378
|
const transcriptText = transcriptResult?.transcriptText ?? "";
|
|
5359
|
-
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : transcriptResult?.track
|
|
5379
|
+
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : getReliableLanguageCode(transcriptResult?.track) ?? getReliableLanguageCode(getReadyTextTracks(assetData)[0]);
|
|
5360
5380
|
const languageName = resolvedLanguageCode ? getLanguageName(resolvedLanguageCode) : void 0;
|
|
5361
5381
|
const userPrompt = buildUserPrompt5({
|
|
5362
5382
|
tone,
|
|
@@ -5848,18 +5868,18 @@ import {
|
|
|
5848
5868
|
RetryError,
|
|
5849
5869
|
TypeValidationError
|
|
5850
5870
|
} from "ai";
|
|
5851
|
-
import
|
|
5871
|
+
import dedent8 from "dedent";
|
|
5852
5872
|
import { z as z8 } from "zod";
|
|
5853
5873
|
var translationSchema = z8.object({
|
|
5854
5874
|
translation: z8.string()
|
|
5855
5875
|
});
|
|
5856
|
-
var SYSTEM_PROMPT6 =
|
|
5876
|
+
var SYSTEM_PROMPT6 = dedent8`
|
|
5857
5877
|
You are a subtitle translation expert. Translate VTT subtitle files to the target language specified by the user.
|
|
5858
5878
|
You may receive either a full VTT file or a chunk from a larger VTT.
|
|
5859
5879
|
Preserve all timestamps, cue ordering, and VTT formatting exactly as they appear.
|
|
5860
5880
|
Return JSON with a single key "translation" containing the translated VTT content.
|
|
5861
5881
|
`;
|
|
5862
|
-
var CUE_TRANSLATION_SYSTEM_PROMPT =
|
|
5882
|
+
var CUE_TRANSLATION_SYSTEM_PROMPT = dedent8`
|
|
5863
5883
|
You are a subtitle translation expert.
|
|
5864
5884
|
You will receive a sequence of subtitle cues extracted from a VTT file.
|
|
5865
5885
|
Translate the cues to the requested target language while preserving their original order.
|
|
@@ -6370,7 +6390,7 @@ async function translateCaptions(assetId, trackId, toLanguageCode, options) {
|
|
|
6370
6390
|
}
|
|
6371
6391
|
if (uploadToMux) {
|
|
6372
6392
|
try {
|
|
6373
|
-
const languageName = getLanguageName(toLanguageCode);
|
|
6393
|
+
const languageName = getLanguageName(toLanguageCode) ?? toLanguageCode.toUpperCase();
|
|
6374
6394
|
const trackName = `${languageName} (auto-translated)`;
|
|
6375
6395
|
uploadedTrackId = await createTextTrackOnMux(
|
|
6376
6396
|
assetId,
|