@mux/ai 0.16.1 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{index-CuBu_u5-.d.ts → index-3G00N5QH.d.ts} +26 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +331 -300
- package/dist/index.js.map +1 -1
- package/dist/primitives/index.d.ts +1 -1
- package/dist/primitives/index.js +153 -6
- package/dist/primitives/index.js.map +1 -1
- package/dist/workflows/index.js +328 -299
- package/dist/workflows/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -5,7 +5,7 @@ var __export = (target, all) => {
|
|
|
5
5
|
};
|
|
6
6
|
|
|
7
7
|
// package.json
|
|
8
|
-
var version = "0.
|
|
8
|
+
var version = "0.17.1";
|
|
9
9
|
|
|
10
10
|
// src/lib/mux-ai-error.ts
|
|
11
11
|
var MuxAiError = class extends Error {
|
|
@@ -54,6 +54,10 @@ var EnvSchema = z.object({
|
|
|
54
54
|
"Override for Mux image base URL (defaults to https://image.mux.com).",
|
|
55
55
|
"Mux image URL override"
|
|
56
56
|
),
|
|
57
|
+
MUX_STREAM_URL_OVERRIDE: optionalString(
|
|
58
|
+
"Override for Mux stream base URL (defaults to https://stream.mux.com).",
|
|
59
|
+
"Mux stream URL override"
|
|
60
|
+
),
|
|
57
61
|
// Test-only helpers (used by this repo's integration tests)
|
|
58
62
|
MUX_TEST_ASSET_ID: optionalString("Mux asset ID used by integration tests.", "Mux test asset id"),
|
|
59
63
|
MUX_TEST_ASSET_ID_CHAPTERS: optionalString("Mux asset ID used by integration tests for chapters.", "Mux test asset id for chapters"),
|
|
@@ -802,6 +806,7 @@ function createWorkflowStorageClient(options = {}) {
|
|
|
802
806
|
var primitives_exports = {};
|
|
803
807
|
__export(primitives_exports, {
|
|
804
808
|
DEFAULT_STORYBOARD_WIDTH: () => DEFAULT_STORYBOARD_WIDTH,
|
|
809
|
+
LOW_CONFIDENCE_THRESHOLD: () => LOW_CONFIDENCE_THRESHOLD,
|
|
805
810
|
buildTranscriptUrl: () => buildTranscriptUrl,
|
|
806
811
|
buildVttFromCueBlocks: () => buildVttFromCueBlocks,
|
|
807
812
|
buildVttFromTranslatedCueBlocks: () => buildVttFromTranslatedCueBlocks,
|
|
@@ -823,6 +828,7 @@ __export(primitives_exports, {
|
|
|
823
828
|
getHotspotsForPlaybackId: () => getHotspotsForPlaybackId,
|
|
824
829
|
getHotspotsForVideo: () => getHotspotsForVideo,
|
|
825
830
|
getReadyTextTracks: () => getReadyTextTracks,
|
|
831
|
+
getReliableLanguageCode: () => getReliableLanguageCode,
|
|
826
832
|
getShotsForAsset: () => getShotsForAsset,
|
|
827
833
|
getStoryboardUrl: () => getStoryboardUrl,
|
|
828
834
|
getThumbnailUrls: () => getThumbnailUrls,
|
|
@@ -1242,22 +1248,24 @@ async function waitForShotsForAsset(assetId, options = {}) {
|
|
|
1242
1248
|
);
|
|
1243
1249
|
}
|
|
1244
1250
|
|
|
1245
|
-
// src/lib/mux-
|
|
1251
|
+
// src/lib/mux-url.ts
|
|
1246
1252
|
var DEFAULT_MUX_IMAGE_ORIGIN = "https://image.mux.com";
|
|
1247
|
-
|
|
1253
|
+
var DEFAULT_MUX_STREAM_ORIGIN = "https://stream.mux.com";
|
|
1254
|
+
function normalizeOrigin(value, envVarName) {
|
|
1248
1255
|
const trimmed = value.trim();
|
|
1249
1256
|
const candidate = trimmed.includes("://") ? trimmed : `https://${trimmed}`;
|
|
1257
|
+
const exampleHostname = envVarName === "MUX_IMAGE_URL_OVERRIDE" ? "image.example.mux.com" : "stream.example.mux.com";
|
|
1250
1258
|
let parsed;
|
|
1251
1259
|
try {
|
|
1252
1260
|
parsed = new URL(candidate);
|
|
1253
1261
|
} catch {
|
|
1254
1262
|
throw new Error(
|
|
1255
|
-
`Invalid
|
|
1263
|
+
`Invalid ${envVarName}. Provide a hostname (e.g. "${exampleHostname}") or a URL origin (e.g. "https://${exampleHostname}").`
|
|
1256
1264
|
);
|
|
1257
1265
|
}
|
|
1258
1266
|
if (parsed.username || parsed.password || parsed.search || parsed.hash || parsed.pathname && parsed.pathname !== "/") {
|
|
1259
1267
|
throw new Error(
|
|
1260
|
-
|
|
1268
|
+
`Invalid ${envVarName}. Only a hostname/origin is allowed (no credentials, query params, hash fragments, or path).`
|
|
1261
1269
|
);
|
|
1262
1270
|
}
|
|
1263
1271
|
return parsed.origin;
|
|
@@ -1267,7 +1275,14 @@ function getMuxImageOrigin() {
|
|
|
1267
1275
|
if (!override) {
|
|
1268
1276
|
return DEFAULT_MUX_IMAGE_ORIGIN;
|
|
1269
1277
|
}
|
|
1270
|
-
return
|
|
1278
|
+
return normalizeOrigin(override, "MUX_IMAGE_URL_OVERRIDE");
|
|
1279
|
+
}
|
|
1280
|
+
function getMuxStreamOrigin() {
|
|
1281
|
+
const override = env_default.MUX_STREAM_URL_OVERRIDE;
|
|
1282
|
+
if (!override) {
|
|
1283
|
+
return DEFAULT_MUX_STREAM_ORIGIN;
|
|
1284
|
+
}
|
|
1285
|
+
return normalizeOrigin(override, "MUX_STREAM_URL_OVERRIDE");
|
|
1271
1286
|
}
|
|
1272
1287
|
function getMuxImageBaseUrl(playbackId, assetType) {
|
|
1273
1288
|
const origin = getMuxImageOrigin();
|
|
@@ -1601,6 +1616,171 @@ async function getThumbnailUrls(playbackId, duration, options = {}) {
|
|
|
1601
1616
|
return Promise.all(urlPromises);
|
|
1602
1617
|
}
|
|
1603
1618
|
|
|
1619
|
+
// src/lib/language-codes.ts
|
|
1620
|
+
var ISO639_1_TO_3 = {
|
|
1621
|
+
// Major world languages
|
|
1622
|
+
en: "eng",
|
|
1623
|
+
// English
|
|
1624
|
+
es: "spa",
|
|
1625
|
+
// Spanish
|
|
1626
|
+
fr: "fra",
|
|
1627
|
+
// French
|
|
1628
|
+
de: "deu",
|
|
1629
|
+
// German
|
|
1630
|
+
it: "ita",
|
|
1631
|
+
// Italian
|
|
1632
|
+
pt: "por",
|
|
1633
|
+
// Portuguese
|
|
1634
|
+
ru: "rus",
|
|
1635
|
+
// Russian
|
|
1636
|
+
zh: "zho",
|
|
1637
|
+
// Chinese
|
|
1638
|
+
ja: "jpn",
|
|
1639
|
+
// Japanese
|
|
1640
|
+
ko: "kor",
|
|
1641
|
+
// Korean
|
|
1642
|
+
ar: "ara",
|
|
1643
|
+
// Arabic
|
|
1644
|
+
hi: "hin",
|
|
1645
|
+
// Hindi
|
|
1646
|
+
// European languages
|
|
1647
|
+
nl: "nld",
|
|
1648
|
+
// Dutch
|
|
1649
|
+
pl: "pol",
|
|
1650
|
+
// Polish
|
|
1651
|
+
sv: "swe",
|
|
1652
|
+
// Swedish
|
|
1653
|
+
da: "dan",
|
|
1654
|
+
// Danish
|
|
1655
|
+
no: "nor",
|
|
1656
|
+
// Norwegian
|
|
1657
|
+
fi: "fin",
|
|
1658
|
+
// Finnish
|
|
1659
|
+
el: "ell",
|
|
1660
|
+
// Greek
|
|
1661
|
+
cs: "ces",
|
|
1662
|
+
// Czech
|
|
1663
|
+
hu: "hun",
|
|
1664
|
+
// Hungarian
|
|
1665
|
+
ro: "ron",
|
|
1666
|
+
// Romanian
|
|
1667
|
+
bg: "bul",
|
|
1668
|
+
// Bulgarian
|
|
1669
|
+
hr: "hrv",
|
|
1670
|
+
// Croatian
|
|
1671
|
+
sk: "slk",
|
|
1672
|
+
// Slovak
|
|
1673
|
+
sl: "slv",
|
|
1674
|
+
// Slovenian
|
|
1675
|
+
uk: "ukr",
|
|
1676
|
+
// Ukrainian
|
|
1677
|
+
tr: "tur",
|
|
1678
|
+
// Turkish
|
|
1679
|
+
// Asian languages
|
|
1680
|
+
th: "tha",
|
|
1681
|
+
// Thai
|
|
1682
|
+
vi: "vie",
|
|
1683
|
+
// Vietnamese
|
|
1684
|
+
id: "ind",
|
|
1685
|
+
// Indonesian
|
|
1686
|
+
ms: "msa",
|
|
1687
|
+
// Malay
|
|
1688
|
+
tl: "tgl",
|
|
1689
|
+
// Tagalog/Filipino
|
|
1690
|
+
// Other languages
|
|
1691
|
+
he: "heb",
|
|
1692
|
+
// Hebrew
|
|
1693
|
+
fa: "fas",
|
|
1694
|
+
// Persian/Farsi
|
|
1695
|
+
bn: "ben",
|
|
1696
|
+
// Bengali
|
|
1697
|
+
ta: "tam",
|
|
1698
|
+
// Tamil
|
|
1699
|
+
te: "tel",
|
|
1700
|
+
// Telugu
|
|
1701
|
+
mr: "mar",
|
|
1702
|
+
// Marathi
|
|
1703
|
+
gu: "guj",
|
|
1704
|
+
// Gujarati
|
|
1705
|
+
kn: "kan",
|
|
1706
|
+
// Kannada
|
|
1707
|
+
ml: "mal",
|
|
1708
|
+
// Malayalam
|
|
1709
|
+
pa: "pan",
|
|
1710
|
+
// Punjabi
|
|
1711
|
+
ur: "urd",
|
|
1712
|
+
// Urdu
|
|
1713
|
+
sw: "swa",
|
|
1714
|
+
// Swahili
|
|
1715
|
+
af: "afr",
|
|
1716
|
+
// Afrikaans
|
|
1717
|
+
ca: "cat",
|
|
1718
|
+
// Catalan
|
|
1719
|
+
eu: "eus",
|
|
1720
|
+
// Basque
|
|
1721
|
+
gl: "glg",
|
|
1722
|
+
// Galician
|
|
1723
|
+
is: "isl",
|
|
1724
|
+
// Icelandic
|
|
1725
|
+
et: "est",
|
|
1726
|
+
// Estonian
|
|
1727
|
+
lv: "lav",
|
|
1728
|
+
// Latvian
|
|
1729
|
+
lt: "lit"
|
|
1730
|
+
// Lithuanian
|
|
1731
|
+
};
|
|
1732
|
+
var ISO639_3_TO_1 = Object.fromEntries(
|
|
1733
|
+
Object.entries(ISO639_1_TO_3).map(([iso1, iso3]) => [iso3, iso1])
|
|
1734
|
+
);
|
|
1735
|
+
function toISO639_3(code) {
|
|
1736
|
+
const normalized = code.toLowerCase().trim();
|
|
1737
|
+
if (normalized.length === 3) {
|
|
1738
|
+
return normalized;
|
|
1739
|
+
}
|
|
1740
|
+
return ISO639_1_TO_3[normalized] ?? normalized;
|
|
1741
|
+
}
|
|
1742
|
+
function toISO639_1(code) {
|
|
1743
|
+
const normalized = code.toLowerCase().trim();
|
|
1744
|
+
if (normalized.length === 2) {
|
|
1745
|
+
return normalized;
|
|
1746
|
+
}
|
|
1747
|
+
return ISO639_3_TO_1[normalized] ?? normalized;
|
|
1748
|
+
}
|
|
1749
|
+
function getLanguageCodePair(code) {
|
|
1750
|
+
const normalized = code.toLowerCase().trim();
|
|
1751
|
+
if (normalized.length === 2) {
|
|
1752
|
+
return {
|
|
1753
|
+
iso639_1: normalized,
|
|
1754
|
+
iso639_3: toISO639_3(normalized)
|
|
1755
|
+
};
|
|
1756
|
+
} else if (normalized.length === 3) {
|
|
1757
|
+
return {
|
|
1758
|
+
iso639_1: toISO639_1(normalized),
|
|
1759
|
+
iso639_3: normalized
|
|
1760
|
+
};
|
|
1761
|
+
}
|
|
1762
|
+
return {
|
|
1763
|
+
iso639_1: normalized,
|
|
1764
|
+
iso639_3: normalized
|
|
1765
|
+
};
|
|
1766
|
+
}
|
|
1767
|
+
var UNDETERMINED_LANGUAGE_CODES = /* @__PURE__ */ new Set(["und", "mul", "mis", "zxx"]);
|
|
1768
|
+
function isUndeterminedLanguageCode(code) {
|
|
1769
|
+
return UNDETERMINED_LANGUAGE_CODES.has(code.toLowerCase().trim());
|
|
1770
|
+
}
|
|
1771
|
+
function getLanguageName(code) {
|
|
1772
|
+
if (isUndeterminedLanguageCode(code)) {
|
|
1773
|
+
return void 0;
|
|
1774
|
+
}
|
|
1775
|
+
const iso639_1 = toISO639_1(code);
|
|
1776
|
+
try {
|
|
1777
|
+
const displayNames = new Intl.DisplayNames(["en"], { type: "language" });
|
|
1778
|
+
return displayNames.of(iso639_1) ?? code.toUpperCase();
|
|
1779
|
+
} catch {
|
|
1780
|
+
return code.toUpperCase();
|
|
1781
|
+
}
|
|
1782
|
+
}
|
|
1783
|
+
|
|
1604
1784
|
// src/lib/mux-assets.ts
|
|
1605
1785
|
function getPlaybackId(asset) {
|
|
1606
1786
|
const playbackIds = asset.playback_ids || [];
|
|
@@ -1678,6 +1858,17 @@ function findCaptionTrack(asset, languageCode) {
|
|
|
1678
1858
|
}
|
|
1679
1859
|
return void 0;
|
|
1680
1860
|
}
|
|
1861
|
+
var LOW_CONFIDENCE_THRESHOLD = 0.5;
|
|
1862
|
+
function getReliableLanguageCode(track, confidenceThreshold = LOW_CONFIDENCE_THRESHOLD) {
|
|
1863
|
+
if (!track?.language_code)
|
|
1864
|
+
return void 0;
|
|
1865
|
+
if (isUndeterminedLanguageCode(track.language_code))
|
|
1866
|
+
return void 0;
|
|
1867
|
+
if (track.auto_language_confidence !== void 0 && track.auto_language_confidence < confidenceThreshold) {
|
|
1868
|
+
return void 0;
|
|
1869
|
+
}
|
|
1870
|
+
return track.language_code;
|
|
1871
|
+
}
|
|
1681
1872
|
function normalizeLineEndings(value) {
|
|
1682
1873
|
return value.replace(/\r\n/g, "\n");
|
|
1683
1874
|
}
|
|
@@ -1945,7 +2136,7 @@ function concatenateVttSegments(segments, preamble = "WEBVTT") {
|
|
|
1945
2136
|
}
|
|
1946
2137
|
async function buildTranscriptUrl(playbackId, trackId, shouldSign = false, credentials) {
|
|
1947
2138
|
"use step";
|
|
1948
|
-
const baseUrl =
|
|
2139
|
+
const baseUrl = `${getMuxStreamOrigin()}/${playbackId}/text/${trackId}.vtt`;
|
|
1949
2140
|
if (shouldSign) {
|
|
1950
2141
|
return signUrl(baseUrl, playbackId, "video", void 0, credentials);
|
|
1951
2142
|
}
|
|
@@ -2041,7 +2232,7 @@ __export(workflows_exports, {
|
|
|
2041
2232
|
|
|
2042
2233
|
// src/workflows/ask-questions.ts
|
|
2043
2234
|
import { generateText, Output } from "ai";
|
|
2044
|
-
import
|
|
2235
|
+
import dedent2 from "dedent";
|
|
2045
2236
|
import { z as z2 } from "zod";
|
|
2046
2237
|
|
|
2047
2238
|
// src/lib/image-download.ts
|
|
@@ -2231,6 +2422,59 @@ function createLanguageSection(languageName) {
|
|
|
2231
2422
|
};
|
|
2232
2423
|
}
|
|
2233
2424
|
|
|
2425
|
+
// src/lib/prompt-fragments.ts
|
|
2426
|
+
import dedent from "dedent";
|
|
2427
|
+
var promptDedent = dedent.withOptions({ alignValues: true });
|
|
2428
|
+
var METADATA_BOUNDARY_WARNING = dedent`
|
|
2429
|
+
Do NOT use any metadata such as URLs, file paths, domain names, file names,
|
|
2430
|
+
playback IDs, or technical parameters visible in this request. These are
|
|
2431
|
+
delivery infrastructure and are unrelated to the media content itself.`;
|
|
2432
|
+
var NO_FABRICATION_CONSTRAINT = "Do not fabricate details or make unsupported assumptions";
|
|
2433
|
+
var STRUCTURED_DATA_CONSTRAINT = "Return structured data matching the requested schema exactly";
|
|
2434
|
+
var STORYBOARD_FRAME_INSTRUCTIONS = dedent`
|
|
2435
|
+
These frames are arranged in a grid and represent the visual progression of the content over time.
|
|
2436
|
+
Read frames left-to-right, top-to-bottom to understand the temporal sequence.`;
|
|
2437
|
+
var TONE_GUIDANCE = dedent`
|
|
2438
|
+
Pay special attention to the <tone> section and lean heavily into those instructions.
|
|
2439
|
+
Adapt your entire analysis and writing style to match the specified tone - this should influence
|
|
2440
|
+
your word choice, personality, formality level, and overall presentation of the content.
|
|
2441
|
+
The tone instructions are not suggestions but core requirements for how you should express yourself.`;
|
|
2442
|
+
var CONFIDENCE_SCORING_RUBRIC = dedent`
|
|
2443
|
+
* 0.9-1.0: Clear, unambiguous evidence
|
|
2444
|
+
* 0.7-0.9: Strong evidence with minor ambiguity
|
|
2445
|
+
* 0.5-0.7: Moderate evidence or some conflicting signals
|
|
2446
|
+
* 0.3-0.5: Weak evidence or significant ambiguity
|
|
2447
|
+
* 0.0-0.3: Very uncertain, minimal relevant evidence`;
|
|
2448
|
+
function createLanguageGuidelines(mediaType) {
|
|
2449
|
+
if (mediaType === "video") {
|
|
2450
|
+
return dedent`
|
|
2451
|
+
AVOID these meta-descriptive phrases that reference the medium rather than the content:
|
|
2452
|
+
- "The image shows..." / "The storyboard shows..."
|
|
2453
|
+
- "In this video..." / "This video features..."
|
|
2454
|
+
- "The frames depict..." / "The footage shows..."
|
|
2455
|
+
- "We can see..." / "You can see..."
|
|
2456
|
+
- "The clip shows..." / "The scene shows..."
|
|
2457
|
+
|
|
2458
|
+
INSTEAD, describe the content directly:
|
|
2459
|
+
- BAD: "The video shows a chef preparing a meal"
|
|
2460
|
+
- GOOD: "A chef prepares a meal in a professional kitchen"
|
|
2461
|
+
|
|
2462
|
+
Write as if describing reality, not describing a recording of reality.`;
|
|
2463
|
+
}
|
|
2464
|
+
return dedent`
|
|
2465
|
+
AVOID these meta-descriptive phrases that reference the medium rather than the content:
|
|
2466
|
+
- "The audio shows..." / "The transcript shows..."
|
|
2467
|
+
- "In this recording..." / "This audio features..."
|
|
2468
|
+
- "The speaker says..." / "We can hear..."
|
|
2469
|
+
- "The clip contains..." / "The recording shows..."
|
|
2470
|
+
|
|
2471
|
+
INSTEAD, describe the content directly:
|
|
2472
|
+
- BAD: "The audio features a discussion about climate change"
|
|
2473
|
+
- GOOD: "A panel discusses climate change impacts and solutions"
|
|
2474
|
+
|
|
2475
|
+
Write as if describing reality, not describing a recording of reality.`;
|
|
2476
|
+
}
|
|
2477
|
+
|
|
2234
2478
|
// src/lib/retry.ts
|
|
2235
2479
|
var DEFAULT_RETRY_OPTIONS = {
|
|
2236
2480
|
maxRetries: 3,
|
|
@@ -2290,7 +2534,7 @@ function createAskQuestionsSchema(allowedAnswers) {
|
|
|
2290
2534
|
)
|
|
2291
2535
|
});
|
|
2292
2536
|
}
|
|
2293
|
-
var SYSTEM_PROMPT =
|
|
2537
|
+
var SYSTEM_PROMPT = promptDedent`
|
|
2294
2538
|
<role>
|
|
2295
2539
|
You are a video content analyst specializing in classification tasks.
|
|
2296
2540
|
Your job is to answer questions about video content based on storyboard
|
|
@@ -2303,9 +2547,7 @@ var SYSTEM_PROMPT = dedent`
|
|
|
2303
2547
|
- A list of questions about the video content
|
|
2304
2548
|
- Optionally, a transcript of the audio/dialogue
|
|
2305
2549
|
|
|
2306
|
-
|
|
2307
|
-
progression of the content over time. Read frames left-to-right,
|
|
2308
|
-
top-to-bottom to understand the temporal sequence.
|
|
2550
|
+
${STORYBOARD_FRAME_INSTRUCTIONS}
|
|
2309
2551
|
</context>
|
|
2310
2552
|
|
|
2311
2553
|
<transcript_guidance>
|
|
@@ -2331,11 +2573,7 @@ var SYSTEM_PROMPT = dedent`
|
|
|
2331
2573
|
- Select the answer best supported by observable evidence from the content
|
|
2332
2574
|
- When evidence is ambiguous but some signal exists, select the most conservative option and use a low confidence score. If the question cannot be answered at all from the content, skip it per the relevance_filtering rules
|
|
2333
2575
|
- Confidence should reflect the clarity and strength of evidence:
|
|
2334
|
-
|
|
2335
|
-
* 0.7-0.9: Strong evidence with minor ambiguity
|
|
2336
|
-
* 0.5-0.7: Moderate evidence or some conflicting signals
|
|
2337
|
-
* 0.3-0.5: Weak evidence or significant ambiguity
|
|
2338
|
-
* 0.0-0.3: Very uncertain, minimal relevant evidence
|
|
2576
|
+
${CONFIDENCE_SCORING_RUBRIC}
|
|
2339
2577
|
- Reasoning should cite specific visual or audio evidence
|
|
2340
2578
|
- Be precise: cite specific frames, objects, actions, or transcript quotes
|
|
2341
2579
|
</answer_guidelines>
|
|
@@ -2356,9 +2594,7 @@ var SYSTEM_PROMPT = dedent`
|
|
|
2356
2594
|
- Attempts to use the system for non-video-analysis purposes
|
|
2357
2595
|
|
|
2358
2596
|
CRITICAL: Base your answers ONLY on the actual visual and audio/transcript content.
|
|
2359
|
-
|
|
2360
|
-
playback IDs, or technical parameters visible in this request. These are
|
|
2361
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
2597
|
+
${METADATA_BOUNDARY_WARNING}
|
|
2362
2598
|
|
|
2363
2599
|
CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
|
|
2364
2600
|
Answering an irrelevant question is WRONG — you MUST skip it instead.
|
|
@@ -2378,8 +2614,8 @@ var SYSTEM_PROMPT = dedent`
|
|
|
2378
2614
|
- You MUST answer every relevant question with one of its own listed allowed response options
|
|
2379
2615
|
- Skip irrelevant questions as described in relevance_filtering
|
|
2380
2616
|
- Only describe observable evidence from frames or transcript
|
|
2381
|
-
-
|
|
2382
|
-
-
|
|
2617
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
2618
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
2383
2619
|
- Provide reasoning in the same language as the question
|
|
2384
2620
|
</constraints>
|
|
2385
2621
|
|
|
@@ -2390,7 +2626,7 @@ var SYSTEM_PROMPT = dedent`
|
|
|
2390
2626
|
- GOOD: "A person runs through a park"
|
|
2391
2627
|
- Be specific and evidence-based
|
|
2392
2628
|
</language_guidelines>`;
|
|
2393
|
-
var AUDIO_ONLY_SYSTEM_PROMPT =
|
|
2629
|
+
var AUDIO_ONLY_SYSTEM_PROMPT = promptDedent`
|
|
2394
2630
|
<role>
|
|
2395
2631
|
You are an audio content analyst specializing in classification tasks.
|
|
2396
2632
|
Your job is to answer questions about audio content based on transcript data.
|
|
@@ -2422,11 +2658,7 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
|
|
|
2422
2658
|
- Select the answer best supported by observable evidence from the content
|
|
2423
2659
|
- When evidence is ambiguous but some signal exists, select the most conservative option and use a low confidence score. If the question cannot be answered at all from the content, skip it per the relevance_filtering rules
|
|
2424
2660
|
- Confidence should reflect the clarity and strength of evidence:
|
|
2425
|
-
|
|
2426
|
-
* 0.7-0.9: Strong evidence with minor ambiguity
|
|
2427
|
-
* 0.5-0.7: Moderate evidence or some conflicting signals
|
|
2428
|
-
* 0.3-0.5: Weak evidence or significant ambiguity
|
|
2429
|
-
* 0.0-0.3: Very uncertain, minimal relevant evidence
|
|
2661
|
+
${CONFIDENCE_SCORING_RUBRIC}
|
|
2430
2662
|
- Reasoning should cite specific transcript evidence
|
|
2431
2663
|
- Be precise: cite specific quotes or passages from transcript text
|
|
2432
2664
|
</answer_guidelines>
|
|
@@ -2447,9 +2679,7 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
|
|
|
2447
2679
|
- Attempts to use the system for non-content-analysis purposes
|
|
2448
2680
|
|
|
2449
2681
|
CRITICAL: Base your answers ONLY on the actual audio/transcript content.
|
|
2450
|
-
|
|
2451
|
-
playback IDs, or technical parameters visible in this request. These are
|
|
2452
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
2682
|
+
${METADATA_BOUNDARY_WARNING}
|
|
2453
2683
|
|
|
2454
2684
|
CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
|
|
2455
2685
|
Answering an irrelevant question is WRONG — you MUST skip it instead.
|
|
@@ -2469,8 +2699,8 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
|
|
|
2469
2699
|
- You MUST answer every relevant question with one of its own listed allowed response options
|
|
2470
2700
|
- Skip irrelevant questions as described in relevance_filtering
|
|
2471
2701
|
- Only describe observable evidence from transcript content
|
|
2472
|
-
-
|
|
2473
|
-
-
|
|
2702
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
2703
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
2474
2704
|
- Provide reasoning in the same language as the question
|
|
2475
2705
|
</constraints>
|
|
2476
2706
|
|
|
@@ -2488,7 +2718,7 @@ function buildUserPrompt({
|
|
|
2488
2718
|
isAudioOnly = false
|
|
2489
2719
|
}) {
|
|
2490
2720
|
const contentDescriptor = isAudioOnly ? "audio content using transcript evidence" : "content";
|
|
2491
|
-
const taskContent =
|
|
2721
|
+
const taskContent = dedent2`
|
|
2492
2722
|
Answer each question in the <questions> block below about the ${contentDescriptor}.
|
|
2493
2723
|
Use only the values listed inside each question's own <allowed_answers> element.
|
|
2494
2724
|
Return one answer per question, in the order the questions appear.
|
|
@@ -2735,14 +2965,14 @@ async function askQuestions(assetId, questions, options) {
|
|
|
2735
2965
|
|
|
2736
2966
|
// src/workflows/burned-in-captions.ts
|
|
2737
2967
|
import { generateText as generateText2, Output as Output2 } from "ai";
|
|
2738
|
-
import
|
|
2968
|
+
import dedent3 from "dedent";
|
|
2739
2969
|
import { z as z3 } from "zod";
|
|
2740
2970
|
var burnedInCaptionsSchema = z3.object({
|
|
2741
2971
|
hasBurnedInCaptions: z3.boolean(),
|
|
2742
2972
|
confidence: z3.number(),
|
|
2743
2973
|
detectedLanguage: z3.string().nullable()
|
|
2744
2974
|
});
|
|
2745
|
-
var SYSTEM_PROMPT2 =
|
|
2975
|
+
var SYSTEM_PROMPT2 = promptDedent`
|
|
2746
2976
|
<role>
|
|
2747
2977
|
You are an expert at analyzing video frames to detect burned-in captions (also called open captions or hardcoded subtitles).
|
|
2748
2978
|
These are text overlays that are permanently embedded in the video image, common on TikTok, Instagram Reels, and other social media platforms.
|
|
@@ -2765,8 +2995,7 @@ var SYSTEM_PROMPT2 = dedent2`
|
|
|
2765
2995
|
|
|
2766
2996
|
<context>
|
|
2767
2997
|
You receive storyboard images containing multiple sequential frames extracted from a video.
|
|
2768
|
-
|
|
2769
|
-
Read frames left-to-right, top-to-bottom to understand the temporal sequence.
|
|
2998
|
+
${STORYBOARD_FRAME_INSTRUCTIONS}
|
|
2770
2999
|
</context>
|
|
2771
3000
|
|
|
2772
3001
|
<capabilities>
|
|
@@ -2779,23 +3008,21 @@ var SYSTEM_PROMPT2 = dedent2`
|
|
|
2779
3008
|
<constraints>
|
|
2780
3009
|
- Only classify as burned-in captions when evidence is clear across multiple frames
|
|
2781
3010
|
- Base decisions on observable visual evidence
|
|
2782
|
-
-
|
|
2783
|
-
|
|
2784
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
2785
|
-
- Return structured data matching the requested schema
|
|
3011
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
3012
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
2786
3013
|
</constraints>`;
|
|
2787
3014
|
var burnedInCaptionsPromptBuilder = createPromptBuilder({
|
|
2788
3015
|
template: {
|
|
2789
3016
|
task: {
|
|
2790
3017
|
tag: "task",
|
|
2791
|
-
content:
|
|
3018
|
+
content: dedent3`
|
|
2792
3019
|
Analyze the provided video storyboard to detect burned-in captions (hardcoded subtitles).
|
|
2793
3020
|
Count frames with text vs no text, note position consistency and whether text changes across frames.
|
|
2794
3021
|
Decide if captions exist, with confidence (0.0-1.0) and detected language if any.`
|
|
2795
3022
|
},
|
|
2796
3023
|
analysisSteps: {
|
|
2797
3024
|
tag: "analysis_steps",
|
|
2798
|
-
content:
|
|
3025
|
+
content: dedent3`
|
|
2799
3026
|
1. COUNT how many frames contain text overlays vs. how many don't
|
|
2800
3027
|
2. Check if text appears in consistent positions across multiple frames
|
|
2801
3028
|
3. Verify text changes content between frames (indicating dialogue/narration)
|
|
@@ -2804,7 +3031,7 @@ var burnedInCaptionsPromptBuilder = createPromptBuilder({
|
|
|
2804
3031
|
},
|
|
2805
3032
|
positiveIndicators: {
|
|
2806
3033
|
tag: "classify_as_captions",
|
|
2807
|
-
content:
|
|
3034
|
+
content: dedent3`
|
|
2808
3035
|
ONLY classify as burned-in captions if:
|
|
2809
3036
|
- Text appears in multiple frames (not just 1-2 end frames)
|
|
2810
3037
|
- Text positioning is consistent across those frames
|
|
@@ -2813,7 +3040,7 @@ var burnedInCaptionsPromptBuilder = createPromptBuilder({
|
|
|
2813
3040
|
},
|
|
2814
3041
|
negativeIndicators: {
|
|
2815
3042
|
tag: "not_captions",
|
|
2816
|
-
content:
|
|
3043
|
+
content: dedent3`
|
|
2817
3044
|
DO NOT classify as burned-in captions:
|
|
2818
3045
|
- Marketing taglines appearing only in final 1-2 frames
|
|
2819
3046
|
- Single words or phrases that don't change between frames
|
|
@@ -2936,168 +3163,8 @@ async function hasBurnedInCaptions(assetId, options = {}) {
|
|
|
2936
3163
|
|
|
2937
3164
|
// src/workflows/chapters.ts
|
|
2938
3165
|
import { generateText as generateText3, Output as Output3 } from "ai";
|
|
2939
|
-
import
|
|
3166
|
+
import dedent4 from "dedent";
|
|
2940
3167
|
import { z as z4 } from "zod";
|
|
2941
|
-
|
|
2942
|
-
// src/lib/language-codes.ts
|
|
2943
|
-
var ISO639_1_TO_3 = {
|
|
2944
|
-
// Major world languages
|
|
2945
|
-
en: "eng",
|
|
2946
|
-
// English
|
|
2947
|
-
es: "spa",
|
|
2948
|
-
// Spanish
|
|
2949
|
-
fr: "fra",
|
|
2950
|
-
// French
|
|
2951
|
-
de: "deu",
|
|
2952
|
-
// German
|
|
2953
|
-
it: "ita",
|
|
2954
|
-
// Italian
|
|
2955
|
-
pt: "por",
|
|
2956
|
-
// Portuguese
|
|
2957
|
-
ru: "rus",
|
|
2958
|
-
// Russian
|
|
2959
|
-
zh: "zho",
|
|
2960
|
-
// Chinese
|
|
2961
|
-
ja: "jpn",
|
|
2962
|
-
// Japanese
|
|
2963
|
-
ko: "kor",
|
|
2964
|
-
// Korean
|
|
2965
|
-
ar: "ara",
|
|
2966
|
-
// Arabic
|
|
2967
|
-
hi: "hin",
|
|
2968
|
-
// Hindi
|
|
2969
|
-
// European languages
|
|
2970
|
-
nl: "nld",
|
|
2971
|
-
// Dutch
|
|
2972
|
-
pl: "pol",
|
|
2973
|
-
// Polish
|
|
2974
|
-
sv: "swe",
|
|
2975
|
-
// Swedish
|
|
2976
|
-
da: "dan",
|
|
2977
|
-
// Danish
|
|
2978
|
-
no: "nor",
|
|
2979
|
-
// Norwegian
|
|
2980
|
-
fi: "fin",
|
|
2981
|
-
// Finnish
|
|
2982
|
-
el: "ell",
|
|
2983
|
-
// Greek
|
|
2984
|
-
cs: "ces",
|
|
2985
|
-
// Czech
|
|
2986
|
-
hu: "hun",
|
|
2987
|
-
// Hungarian
|
|
2988
|
-
ro: "ron",
|
|
2989
|
-
// Romanian
|
|
2990
|
-
bg: "bul",
|
|
2991
|
-
// Bulgarian
|
|
2992
|
-
hr: "hrv",
|
|
2993
|
-
// Croatian
|
|
2994
|
-
sk: "slk",
|
|
2995
|
-
// Slovak
|
|
2996
|
-
sl: "slv",
|
|
2997
|
-
// Slovenian
|
|
2998
|
-
uk: "ukr",
|
|
2999
|
-
// Ukrainian
|
|
3000
|
-
tr: "tur",
|
|
3001
|
-
// Turkish
|
|
3002
|
-
// Asian languages
|
|
3003
|
-
th: "tha",
|
|
3004
|
-
// Thai
|
|
3005
|
-
vi: "vie",
|
|
3006
|
-
// Vietnamese
|
|
3007
|
-
id: "ind",
|
|
3008
|
-
// Indonesian
|
|
3009
|
-
ms: "msa",
|
|
3010
|
-
// Malay
|
|
3011
|
-
tl: "tgl",
|
|
3012
|
-
// Tagalog/Filipino
|
|
3013
|
-
// Other languages
|
|
3014
|
-
he: "heb",
|
|
3015
|
-
// Hebrew
|
|
3016
|
-
fa: "fas",
|
|
3017
|
-
// Persian/Farsi
|
|
3018
|
-
bn: "ben",
|
|
3019
|
-
// Bengali
|
|
3020
|
-
ta: "tam",
|
|
3021
|
-
// Tamil
|
|
3022
|
-
te: "tel",
|
|
3023
|
-
// Telugu
|
|
3024
|
-
mr: "mar",
|
|
3025
|
-
// Marathi
|
|
3026
|
-
gu: "guj",
|
|
3027
|
-
// Gujarati
|
|
3028
|
-
kn: "kan",
|
|
3029
|
-
// Kannada
|
|
3030
|
-
ml: "mal",
|
|
3031
|
-
// Malayalam
|
|
3032
|
-
pa: "pan",
|
|
3033
|
-
// Punjabi
|
|
3034
|
-
ur: "urd",
|
|
3035
|
-
// Urdu
|
|
3036
|
-
sw: "swa",
|
|
3037
|
-
// Swahili
|
|
3038
|
-
af: "afr",
|
|
3039
|
-
// Afrikaans
|
|
3040
|
-
ca: "cat",
|
|
3041
|
-
// Catalan
|
|
3042
|
-
eu: "eus",
|
|
3043
|
-
// Basque
|
|
3044
|
-
gl: "glg",
|
|
3045
|
-
// Galician
|
|
3046
|
-
is: "isl",
|
|
3047
|
-
// Icelandic
|
|
3048
|
-
et: "est",
|
|
3049
|
-
// Estonian
|
|
3050
|
-
lv: "lav",
|
|
3051
|
-
// Latvian
|
|
3052
|
-
lt: "lit"
|
|
3053
|
-
// Lithuanian
|
|
3054
|
-
};
|
|
3055
|
-
var ISO639_3_TO_1 = Object.fromEntries(
|
|
3056
|
-
Object.entries(ISO639_1_TO_3).map(([iso1, iso3]) => [iso3, iso1])
|
|
3057
|
-
);
|
|
3058
|
-
function toISO639_3(code) {
|
|
3059
|
-
const normalized = code.toLowerCase().trim();
|
|
3060
|
-
if (normalized.length === 3) {
|
|
3061
|
-
return normalized;
|
|
3062
|
-
}
|
|
3063
|
-
return ISO639_1_TO_3[normalized] ?? normalized;
|
|
3064
|
-
}
|
|
3065
|
-
function toISO639_1(code) {
|
|
3066
|
-
const normalized = code.toLowerCase().trim();
|
|
3067
|
-
if (normalized.length === 2) {
|
|
3068
|
-
return normalized;
|
|
3069
|
-
}
|
|
3070
|
-
return ISO639_3_TO_1[normalized] ?? normalized;
|
|
3071
|
-
}
|
|
3072
|
-
function getLanguageCodePair(code) {
|
|
3073
|
-
const normalized = code.toLowerCase().trim();
|
|
3074
|
-
if (normalized.length === 2) {
|
|
3075
|
-
return {
|
|
3076
|
-
iso639_1: normalized,
|
|
3077
|
-
iso639_3: toISO639_3(normalized)
|
|
3078
|
-
};
|
|
3079
|
-
} else if (normalized.length === 3) {
|
|
3080
|
-
return {
|
|
3081
|
-
iso639_1: toISO639_1(normalized),
|
|
3082
|
-
iso639_3: normalized
|
|
3083
|
-
};
|
|
3084
|
-
}
|
|
3085
|
-
return {
|
|
3086
|
-
iso639_1: normalized,
|
|
3087
|
-
iso639_3: normalized
|
|
3088
|
-
};
|
|
3089
|
-
}
|
|
3090
|
-
function getLanguageName(code) {
|
|
3091
|
-
const iso639_1 = toISO639_1(code);
|
|
3092
|
-
try {
|
|
3093
|
-
const displayNames = new Intl.DisplayNames(["en"], { type: "language" });
|
|
3094
|
-
return displayNames.of(iso639_1) ?? code.toUpperCase();
|
|
3095
|
-
} catch {
|
|
3096
|
-
return code.toUpperCase();
|
|
3097
|
-
}
|
|
3098
|
-
}
|
|
3099
|
-
|
|
3100
|
-
// src/workflows/chapters.ts
|
|
3101
3168
|
var chapterSchema = z4.object({
|
|
3102
3169
|
startTime: z4.number(),
|
|
3103
3170
|
title: z4.string()
|
|
@@ -3149,13 +3216,13 @@ var chapterSystemPromptBuilder = createPromptBuilder({
|
|
|
3149
3216
|
},
|
|
3150
3217
|
context: {
|
|
3151
3218
|
tag: "context",
|
|
3152
|
-
content:
|
|
3219
|
+
content: dedent4`
|
|
3153
3220
|
You receive a timestamped transcript with lines in the form "[12s] Caption text".
|
|
3154
3221
|
Use those timestamps as anchors to determine chapter start times in seconds.`
|
|
3155
3222
|
},
|
|
3156
3223
|
constraints: {
|
|
3157
3224
|
tag: "constraints",
|
|
3158
|
-
content:
|
|
3225
|
+
content: dedent4`
|
|
3159
3226
|
- Only use information present in the transcript
|
|
3160
3227
|
- Return structured data that matches the requested JSON schema
|
|
3161
3228
|
- Do not add commentary or extra text outside the JSON
|
|
@@ -3163,7 +3230,7 @@ var chapterSystemPromptBuilder = createPromptBuilder({
|
|
|
3163
3230
|
},
|
|
3164
3231
|
qualityGuidelines: {
|
|
3165
3232
|
tag: "quality_guidelines",
|
|
3166
|
-
content:
|
|
3233
|
+
content: dedent4`
|
|
3167
3234
|
- Create chapters at topic shifts or clear transitions
|
|
3168
3235
|
- Keep chapter titles concise and descriptive
|
|
3169
3236
|
- Ensure the first chapter starts at 0 seconds`
|
|
@@ -3173,7 +3240,7 @@ var chapterSystemPromptBuilder = createPromptBuilder({
|
|
|
3173
3240
|
});
|
|
3174
3241
|
var AUDIO_ONLY_SYSTEM_PROMPT_OVERRIDES = {
|
|
3175
3242
|
role: "You are an audio editor and transcript analyst specializing in segmenting content into logical chapters.",
|
|
3176
|
-
context:
|
|
3243
|
+
context: dedent4`
|
|
3177
3244
|
You receive a timestamped transcript from audio-only content with lines in the form "[12s] Transcript text".
|
|
3178
3245
|
Use those timestamps as anchors to determine chapter start times in seconds.`
|
|
3179
3246
|
};
|
|
@@ -3185,7 +3252,7 @@ var chaptersPromptBuilder = createPromptBuilder({
|
|
|
3185
3252
|
},
|
|
3186
3253
|
outputFormat: {
|
|
3187
3254
|
tag: "output_format",
|
|
3188
|
-
content:
|
|
3255
|
+
content: dedent4`
|
|
3189
3256
|
Return valid JSON in this exact shape:
|
|
3190
3257
|
{
|
|
3191
3258
|
"chapters": [
|
|
@@ -3204,7 +3271,7 @@ var chaptersPromptBuilder = createPromptBuilder({
|
|
|
3204
3271
|
},
|
|
3205
3272
|
titleGuidelines: {
|
|
3206
3273
|
tag: "title_guidelines",
|
|
3207
|
-
content:
|
|
3274
|
+
content: dedent4`
|
|
3208
3275
|
- Keep titles concise and descriptive
|
|
3209
3276
|
- Avoid filler or generic labels like "Chapter 1"
|
|
3210
3277
|
- Use the transcript's terminology`
|
|
@@ -3229,7 +3296,7 @@ function buildUserPrompt3({
|
|
|
3229
3296
|
if (languageName) {
|
|
3230
3297
|
contextSections.push(createLanguageSection(languageName));
|
|
3231
3298
|
}
|
|
3232
|
-
const dynamicChapterGuidelines =
|
|
3299
|
+
const dynamicChapterGuidelines = dedent4`
|
|
3233
3300
|
- Create at least ${minChaptersPerHour} and at most ${maxChaptersPerHour} chapters per hour of content
|
|
3234
3301
|
- Use start times in seconds (not HH:MM:SS)
|
|
3235
3302
|
- Chapter start times should be non-decreasing
|
|
@@ -3287,7 +3354,7 @@ async function generateChapters(assetId, options = {}) {
|
|
|
3287
3354
|
const contentLabel = isAudioOnly ? "transcript" : "caption track";
|
|
3288
3355
|
throw new MuxAiError(`No usable content found in ${contentLabel}.`, { type: "validation_error" });
|
|
3289
3356
|
}
|
|
3290
|
-
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : transcriptResult.track
|
|
3357
|
+
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : getReliableLanguageCode(transcriptResult.track) ?? languageCode;
|
|
3291
3358
|
const languageName = resolvedLanguageCode ? getLanguageName(resolvedLanguageCode) : void 0;
|
|
3292
3359
|
const userPrompt = buildUserPrompt3({
|
|
3293
3360
|
timestampedTranscript,
|
|
@@ -3329,7 +3396,7 @@ async function generateChapters(assetId, options = {}) {
|
|
|
3329
3396
|
};
|
|
3330
3397
|
return {
|
|
3331
3398
|
assetId,
|
|
3332
|
-
languageCode: languageCode ?? transcriptResult.track
|
|
3399
|
+
languageCode: languageCode ?? getReliableLanguageCode(transcriptResult.track) ?? "en",
|
|
3333
3400
|
chapters: validChapters,
|
|
3334
3401
|
usage: usageWithMetadata
|
|
3335
3402
|
};
|
|
@@ -3337,7 +3404,7 @@ async function generateChapters(assetId, options = {}) {
|
|
|
3337
3404
|
|
|
3338
3405
|
// src/workflows/edit-captions.ts
|
|
3339
3406
|
import { generateText as generateText4, Output as Output4 } from "ai";
|
|
3340
|
-
import
|
|
3407
|
+
import dedent5 from "dedent";
|
|
3341
3408
|
import { z as z5 } from "zod";
|
|
3342
3409
|
|
|
3343
3410
|
// src/lib/mux-tracks.ts
|
|
@@ -3414,7 +3481,7 @@ var profanityDetectionSchema = z5.object({
|
|
|
3414
3481
|
"Unique profane words or short phrases exactly as they appear in the transcript text. Include each distinct form only once (e.g., if 'fuck' and 'fucking' both appear, list both)."
|
|
3415
3482
|
)
|
|
3416
3483
|
});
|
|
3417
|
-
var SYSTEM_PROMPT3 =
|
|
3484
|
+
var SYSTEM_PROMPT3 = dedent5`
|
|
3418
3485
|
You are a content moderation assistant. Your task is to identify profane, vulgar, or obscene
|
|
3419
3486
|
words and phrases in subtitle text. Return ONLY the exact profane words or phrases as they appear
|
|
3420
3487
|
in the text. Do not modify, censor, or paraphrase them. Do not include words that are merely
|
|
@@ -3890,7 +3957,7 @@ async function generateVideoEmbeddings(assetId, options = {}) {
|
|
|
3890
3957
|
|
|
3891
3958
|
// src/workflows/engagement-insights.ts
|
|
3892
3959
|
import { generateText as generateText5, Output as Output5 } from "ai";
|
|
3893
|
-
import
|
|
3960
|
+
import dedent6 from "dedent";
|
|
3894
3961
|
import { z as z6 } from "zod";
|
|
3895
3962
|
var aiMomentInsightSchema = z6.object({
|
|
3896
3963
|
hotspotIndex: z6.number(),
|
|
@@ -3908,7 +3975,7 @@ var engagementInsightsSchema = z6.object({
|
|
|
3908
3975
|
momentInsights: z6.array(aiMomentInsightSchema),
|
|
3909
3976
|
overallInsight: aiOverallInsightSchema
|
|
3910
3977
|
});
|
|
3911
|
-
var SYSTEM_PROMPT4 =
|
|
3978
|
+
var SYSTEM_PROMPT4 = promptDedent`
|
|
3912
3979
|
<role>
|
|
3913
3980
|
You are a video engagement analyst specializing in understanding viewer behavior.
|
|
3914
3981
|
Your job is to explain why specific moments in videos have high or low engagement
|
|
@@ -3936,12 +4003,10 @@ var SYSTEM_PROMPT4 = dedent5`
|
|
|
3936
4003
|
|
|
3937
4004
|
<constraints>
|
|
3938
4005
|
- Only describe what you can see in images or read in transcripts
|
|
3939
|
-
-
|
|
3940
|
-
-
|
|
3941
|
-
playback IDs, or technical parameters visible in this request. These are
|
|
3942
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
4006
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
4007
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
3943
4008
|
- Correlate engagement scores with observable content
|
|
3944
|
-
-
|
|
4009
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
3945
4010
|
- Each momentInsight MUST include the hotspotIndex from the input data
|
|
3946
4011
|
</constraints>
|
|
3947
4012
|
|
|
@@ -3952,7 +4017,7 @@ var SYSTEM_PROMPT4 = dedent5`
|
|
|
3952
4017
|
- Connect engagement data to content features
|
|
3953
4018
|
</quality_guidelines>
|
|
3954
4019
|
`;
|
|
3955
|
-
var AUDIO_ONLY_SYSTEM_PROMPT2 =
|
|
4020
|
+
var AUDIO_ONLY_SYSTEM_PROMPT2 = promptDedent`
|
|
3956
4021
|
<role>
|
|
3957
4022
|
You are a video engagement analyst specializing in understanding viewer behavior for audio content.
|
|
3958
4023
|
Your job is to explain why specific moments have high or low engagement based on audio/dialogue and viewer watch patterns.
|
|
@@ -3979,12 +4044,10 @@ var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent5`
|
|
|
3979
4044
|
|
|
3980
4045
|
<constraints>
|
|
3981
4046
|
- Only describe what you can read in the transcript
|
|
3982
|
-
-
|
|
3983
|
-
-
|
|
3984
|
-
playback IDs, or technical parameters visible in this request. These are
|
|
3985
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
4047
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
4048
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
3986
4049
|
- Correlate engagement scores with observable content
|
|
3987
|
-
-
|
|
4050
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
3988
4051
|
- Each momentInsight MUST include the hotspotIndex from the input data
|
|
3989
4052
|
</constraints>
|
|
3990
4053
|
|
|
@@ -3995,7 +4058,7 @@ var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent5`
|
|
|
3995
4058
|
- Connect engagement data to content features
|
|
3996
4059
|
</quality_guidelines>
|
|
3997
4060
|
`;
|
|
3998
|
-
var INSIGHT_GUIDELINES =
|
|
4061
|
+
var INSIGHT_GUIDELINES = dedent6`
|
|
3999
4062
|
Provide explanatory insights that describe WHY moments are engaging.
|
|
4000
4063
|
Focus on observable patterns and viewer behavior correlation.
|
|
4001
4064
|
Example: "The cooking demonstration at 2:15 has 3x average engagement
|
|
@@ -4013,7 +4076,7 @@ var engagementInsightsPromptBuilder = createPromptBuilder({
|
|
|
4013
4076
|
},
|
|
4014
4077
|
outputFormat: {
|
|
4015
4078
|
tag: "output_format",
|
|
4016
|
-
content:
|
|
4079
|
+
content: dedent6`
|
|
4017
4080
|
Return valid JSON matching the provided schema.
|
|
4018
4081
|
Include insights for each hotspot and overall engagement trends.
|
|
4019
4082
|
Each momentInsight must include the hotspotIndex matching the input.
|
|
@@ -4021,7 +4084,7 @@ var engagementInsightsPromptBuilder = createPromptBuilder({
|
|
|
4021
4084
|
},
|
|
4022
4085
|
visualContext: {
|
|
4023
4086
|
tag: "visual_context",
|
|
4024
|
-
content:
|
|
4087
|
+
content: dedent6`
|
|
4025
4088
|
Images are provided for each hotspot timestamp.
|
|
4026
4089
|
A storyboard overview image may also be included showing the full video timeline.
|
|
4027
4090
|
Use visual evidence from these images to support your engagement analysis.
|
|
@@ -4035,14 +4098,14 @@ function buildUserPrompt4(hotspots, transcriptSegments, heatmap, isAudioOnly) {
|
|
|
4035
4098
|
const transcript = transcriptSegments[idx]?.text || "(no transcript)";
|
|
4036
4099
|
const startTimestamp = secondsToTimestamp(h.startMs / 1e3);
|
|
4037
4100
|
const endTimestamp = secondsToTimestamp(h.endMs / 1e3);
|
|
4038
|
-
return
|
|
4101
|
+
return dedent6`
|
|
4039
4102
|
Hotspot ${idx} (hotspotIndex: ${idx}):
|
|
4040
4103
|
- Time Range: ${startTimestamp} - ${endTimestamp}
|
|
4041
4104
|
- Engagement Score: ${h.score.toFixed(2)} (0=low, 1=high)
|
|
4042
4105
|
- Transcript: "${transcript}"
|
|
4043
4106
|
`;
|
|
4044
4107
|
}).join("\n\n");
|
|
4045
|
-
const heatmapData =
|
|
4108
|
+
const heatmapData = dedent6`
|
|
4046
4109
|
Engagement Heatmap (${heatmap.length} values, each representing 1/${heatmap.length}th of the video, higher = more re-watches):
|
|
4047
4110
|
[${heatmap.map((v) => v.toFixed(2)).join(", ")}]
|
|
4048
4111
|
`;
|
|
@@ -4916,7 +4979,7 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
4916
4979
|
|
|
4917
4980
|
// src/workflows/summarization.ts
|
|
4918
4981
|
import { generateText as generateText6, Output as Output6 } from "ai";
|
|
4919
|
-
import
|
|
4982
|
+
import dedent7 from "dedent";
|
|
4920
4983
|
import { z as z7 } from "zod";
|
|
4921
4984
|
var DEFAULT_SUMMARY_KEYWORD_LIMIT = 10;
|
|
4922
4985
|
var DEFAULT_TITLE_LENGTH = 10;
|
|
@@ -4942,36 +5005,36 @@ var DESCRIPTION_LENGTH_THRESHOLD_LARGE = 100;
|
|
|
4942
5005
|
function buildDescriptionGuidance(wordCount, contentType) {
|
|
4943
5006
|
if (wordCount < DESCRIPTION_LENGTH_THRESHOLD_SMALL) {
|
|
4944
5007
|
if (contentType === "video") {
|
|
4945
|
-
return
|
|
5008
|
+
return dedent7`A brief summary of the video in no more than ${wordCount} words. Shorter is fine.
|
|
4946
5009
|
Focus on the single most important subject or action.
|
|
4947
5010
|
Write in present tense.`;
|
|
4948
5011
|
}
|
|
4949
|
-
return
|
|
5012
|
+
return dedent7`A brief summary of the audio content in no more than ${wordCount} words. Shorter is fine.
|
|
4950
5013
|
Focus on the single most important topic or theme.
|
|
4951
5014
|
Write in present tense.`;
|
|
4952
5015
|
}
|
|
4953
5016
|
if (wordCount > DESCRIPTION_LENGTH_THRESHOLD_LARGE) {
|
|
4954
5017
|
if (contentType === "video") {
|
|
4955
|
-
return
|
|
5018
|
+
return dedent7`A detailed summary that describes what happens across the video.
|
|
4956
5019
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4957
5020
|
Be thorough: cover subjects, actions, setting, progression, and any notable details visible across frames.
|
|
4958
5021
|
Write in present tense. Be specific about observable details rather than making assumptions.
|
|
4959
5022
|
If the transcript provides dialogue or narration, incorporate key points but prioritize visual content.`;
|
|
4960
5023
|
}
|
|
4961
|
-
return
|
|
5024
|
+
return dedent7`A detailed summary that describes the audio content.
|
|
4962
5025
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4963
5026
|
Be thorough: cover topics, speakers, themes, progression, and any notable insights.
|
|
4964
5027
|
Write in present tense. Be specific about what is discussed or presented rather than making assumptions.
|
|
4965
5028
|
Focus on the spoken content and any key insights, dialogue, or narrative elements.`;
|
|
4966
5029
|
}
|
|
4967
5030
|
if (contentType === "video") {
|
|
4968
|
-
return
|
|
5031
|
+
return dedent7`A summary that describes what happens across the video.
|
|
4969
5032
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4970
5033
|
Cover the main subjects, actions, setting, and any notable progression visible across frames.
|
|
4971
5034
|
Write in present tense. Be specific about observable details rather than making assumptions.
|
|
4972
5035
|
If the transcript provides dialogue or narration, incorporate key points but prioritize visual content.`;
|
|
4973
5036
|
}
|
|
4974
|
-
return
|
|
5037
|
+
return dedent7`A summary that describes the audio content.
|
|
4975
5038
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4976
5039
|
Cover the main topics, speakers, themes, and any notable progression in the discussion or narration.
|
|
4977
5040
|
Write in present tense. Be specific about what is discussed or presented rather than making assumptions.
|
|
@@ -4988,7 +5051,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
4988
5051
|
},
|
|
4989
5052
|
title: {
|
|
4990
5053
|
tag: "title_requirements",
|
|
4991
|
-
content:
|
|
5054
|
+
content: dedent7`
|
|
4992
5055
|
A concise, label-style title — not a sentence or description.
|
|
4993
5056
|
Never exceed ${titleLimit} words, but shorter is better.
|
|
4994
5057
|
Think of how a video card title, playlist entry, or file name would read — e.g. "Predator: Badlands Trailer" or "Chef Prepares Holiday Feast".
|
|
@@ -5001,7 +5064,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
5001
5064
|
},
|
|
5002
5065
|
keywords: {
|
|
5003
5066
|
tag: "keywords_requirements",
|
|
5004
|
-
content:
|
|
5067
|
+
content: dedent7`
|
|
5005
5068
|
Specific, searchable terms (up to ${keywordLimit}) that capture:
|
|
5006
5069
|
- Primary subjects (people, animals, objects)
|
|
5007
5070
|
- Actions and activities being performed
|
|
@@ -5013,7 +5076,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
5013
5076
|
},
|
|
5014
5077
|
qualityGuidelines: {
|
|
5015
5078
|
tag: "quality_guidelines",
|
|
5016
|
-
content:
|
|
5079
|
+
content: dedent7`
|
|
5017
5080
|
- Examine all frames to understand the full context and progression
|
|
5018
5081
|
- Be precise: "golden retriever" is better than "dog" when identifiable
|
|
5019
5082
|
- Capture the narrative: what begins, develops, and concludes
|
|
@@ -5034,7 +5097,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
5034
5097
|
},
|
|
5035
5098
|
title: {
|
|
5036
5099
|
tag: "title_requirements",
|
|
5037
|
-
content:
|
|
5100
|
+
content: dedent7`
|
|
5038
5101
|
A concise, label-style title — not a sentence or description.
|
|
5039
5102
|
Never exceed ${titleLimit} words, but shorter is better.
|
|
5040
5103
|
Think of how a podcast episode title or playlist entry would read — e.g. "Weekly News Roundup" or "Interview with Dr. Smith".
|
|
@@ -5047,7 +5110,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
5047
5110
|
},
|
|
5048
5111
|
keywords: {
|
|
5049
5112
|
tag: "keywords_requirements",
|
|
5050
|
-
content:
|
|
5113
|
+
content: dedent7`
|
|
5051
5114
|
Specific, searchable terms (up to ${keywordLimit}) that capture:
|
|
5052
5115
|
- Primary topics and themes
|
|
5053
5116
|
- Speakers or presenters (if named)
|
|
@@ -5059,7 +5122,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
5059
5122
|
},
|
|
5060
5123
|
qualityGuidelines: {
|
|
5061
5124
|
tag: "quality_guidelines",
|
|
5062
|
-
content:
|
|
5125
|
+
content: dedent7`
|
|
5063
5126
|
- Analyze the full transcript to understand context and themes
|
|
5064
5127
|
- Be precise: use specific terminology when mentioned
|
|
5065
5128
|
- Capture the narrative: what is introduced, discussed, and concluded
|
|
@@ -5069,15 +5132,14 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
5069
5132
|
sectionOrder: ["task", "title", "description", "keywords", "qualityGuidelines"]
|
|
5070
5133
|
});
|
|
5071
5134
|
}
|
|
5072
|
-
var SYSTEM_PROMPT5 =
|
|
5135
|
+
var SYSTEM_PROMPT5 = promptDedent`
|
|
5073
5136
|
<role>
|
|
5074
5137
|
You are a video content analyst specializing in storyboard interpretation and multimodal analysis.
|
|
5075
5138
|
</role>
|
|
5076
5139
|
|
|
5077
5140
|
<context>
|
|
5078
5141
|
You receive storyboard images containing multiple sequential frames extracted from a video.
|
|
5079
|
-
|
|
5080
|
-
Read frames left-to-right, top-to-bottom to understand the temporal sequence.
|
|
5142
|
+
${STORYBOARD_FRAME_INSTRUCTIONS}
|
|
5081
5143
|
</context>
|
|
5082
5144
|
|
|
5083
5145
|
<transcript_guidance>
|
|
@@ -5099,37 +5161,21 @@ var SYSTEM_PROMPT5 = dedent6`
|
|
|
5099
5161
|
|
|
5100
5162
|
<constraints>
|
|
5101
5163
|
- Only describe what is clearly observable in the frames or explicitly stated in the transcript
|
|
5102
|
-
-
|
|
5103
|
-
-
|
|
5104
|
-
|
|
5105
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
5106
|
-
- Return structured data matching the requested schema
|
|
5164
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
5165
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
5166
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
5107
5167
|
- Output only the JSON object; no markdown or extra text
|
|
5108
5168
|
- When a <language> section is provided, all output text MUST be written in that language
|
|
5109
5169
|
</constraints>
|
|
5110
5170
|
|
|
5111
5171
|
<tone_guidance>
|
|
5112
|
-
|
|
5113
|
-
Adapt your entire analysis and writing style to match the specified tone - this should influence
|
|
5114
|
-
your word choice, personality, formality level, and overall presentation of the content.
|
|
5115
|
-
The tone instructions are not suggestions but core requirements for how you should express yourself.
|
|
5172
|
+
${TONE_GUIDANCE}
|
|
5116
5173
|
</tone_guidance>
|
|
5117
5174
|
|
|
5118
5175
|
<language_guidelines>
|
|
5119
|
-
|
|
5120
|
-
- "The image shows..." / "The storyboard shows..."
|
|
5121
|
-
- "In this video..." / "This video features..."
|
|
5122
|
-
- "The frames depict..." / "The footage shows..."
|
|
5123
|
-
- "We can see..." / "You can see..."
|
|
5124
|
-
- "The clip shows..." / "The scene shows..."
|
|
5125
|
-
|
|
5126
|
-
INSTEAD, describe the content directly:
|
|
5127
|
-
- BAD: "The video shows a chef preparing a meal"
|
|
5128
|
-
- GOOD: "A chef prepares a meal in a professional kitchen"
|
|
5129
|
-
|
|
5130
|
-
Write as if describing reality, not describing a recording of reality.
|
|
5176
|
+
${createLanguageGuidelines("video")}
|
|
5131
5177
|
</language_guidelines>`;
|
|
5132
|
-
var AUDIO_ONLY_SYSTEM_PROMPT3 =
|
|
5178
|
+
var AUDIO_ONLY_SYSTEM_PROMPT3 = promptDedent`
|
|
5133
5179
|
<role>
|
|
5134
5180
|
You are an audio content analyst specializing in transcript analysis and metadata generation.
|
|
5135
5181
|
</role>
|
|
@@ -5156,35 +5202,20 @@ var AUDIO_ONLY_SYSTEM_PROMPT3 = dedent6`
|
|
|
5156
5202
|
|
|
5157
5203
|
<constraints>
|
|
5158
5204
|
- Only describe what is explicitly stated or strongly implied in the transcript
|
|
5159
|
-
-
|
|
5160
|
-
-
|
|
5161
|
-
|
|
5162
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
5163
|
-
- Return structured data matching the requested schema
|
|
5205
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
5206
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
5207
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
5164
5208
|
- Focus entirely on audio/spoken content - there are no visual elements
|
|
5165
5209
|
- Output only the JSON object; no markdown or extra text
|
|
5166
5210
|
- When a <language> section is provided, all output text MUST be written in that language
|
|
5167
5211
|
</constraints>
|
|
5168
5212
|
|
|
5169
5213
|
<tone_guidance>
|
|
5170
|
-
|
|
5171
|
-
Adapt your entire analysis and writing style to match the specified tone - this should influence
|
|
5172
|
-
your word choice, personality, formality level, and overall presentation of the content.
|
|
5173
|
-
The tone instructions are not suggestions but core requirements for how you should express yourself.
|
|
5214
|
+
${TONE_GUIDANCE}
|
|
5174
5215
|
</tone_guidance>
|
|
5175
5216
|
|
|
5176
5217
|
<language_guidelines>
|
|
5177
|
-
|
|
5178
|
-
- "The audio shows..." / "The transcript shows..."
|
|
5179
|
-
- "In this recording..." / "This audio features..."
|
|
5180
|
-
- "The speaker says..." / "We can hear..."
|
|
5181
|
-
- "The clip contains..." / "The recording shows..."
|
|
5182
|
-
|
|
5183
|
-
INSTEAD, describe the content directly:
|
|
5184
|
-
- BAD: "The audio features a discussion about climate change"
|
|
5185
|
-
- GOOD: "A panel discusses climate change impacts and solutions"
|
|
5186
|
-
|
|
5187
|
-
Write as if describing reality, not describing a recording of reality.
|
|
5218
|
+
${createLanguageGuidelines("audio")}
|
|
5188
5219
|
</language_guidelines>`;
|
|
5189
5220
|
function buildUserPrompt5({
|
|
5190
5221
|
tone,
|
|
@@ -5358,7 +5389,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
5358
5389
|
required: isAudioOnly
|
|
5359
5390
|
}) : void 0;
|
|
5360
5391
|
const transcriptText = transcriptResult?.transcriptText ?? "";
|
|
5361
|
-
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : transcriptResult?.track
|
|
5392
|
+
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : getReliableLanguageCode(transcriptResult?.track) ?? getReliableLanguageCode(getReadyTextTracks(assetData)[0]);
|
|
5362
5393
|
const languageName = resolvedLanguageCode ? getLanguageName(resolvedLanguageCode) : void 0;
|
|
5363
5394
|
const userPrompt = buildUserPrompt5({
|
|
5364
5395
|
tone,
|
|
@@ -5707,7 +5738,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
5707
5738
|
{ type: "validation_error" }
|
|
5708
5739
|
);
|
|
5709
5740
|
}
|
|
5710
|
-
let audioUrl =
|
|
5741
|
+
let audioUrl = `${getMuxStreamOrigin()}/${playbackId}/audio.m4a`;
|
|
5711
5742
|
if (policy === "signed") {
|
|
5712
5743
|
audioUrl = await signUrl(audioUrl, playbackId, "video", void 0, credentials);
|
|
5713
5744
|
}
|
|
@@ -5850,18 +5881,18 @@ import {
|
|
|
5850
5881
|
RetryError,
|
|
5851
5882
|
TypeValidationError
|
|
5852
5883
|
} from "ai";
|
|
5853
|
-
import
|
|
5884
|
+
import dedent8 from "dedent";
|
|
5854
5885
|
import { z as z8 } from "zod";
|
|
5855
5886
|
var translationSchema = z8.object({
|
|
5856
5887
|
translation: z8.string()
|
|
5857
5888
|
});
|
|
5858
|
-
var SYSTEM_PROMPT6 =
|
|
5889
|
+
var SYSTEM_PROMPT6 = dedent8`
|
|
5859
5890
|
You are a subtitle translation expert. Translate VTT subtitle files to the target language specified by the user.
|
|
5860
5891
|
You may receive either a full VTT file or a chunk from a larger VTT.
|
|
5861
5892
|
Preserve all timestamps, cue ordering, and VTT formatting exactly as they appear.
|
|
5862
5893
|
Return JSON with a single key "translation" containing the translated VTT content.
|
|
5863
5894
|
`;
|
|
5864
|
-
var CUE_TRANSLATION_SYSTEM_PROMPT =
|
|
5895
|
+
var CUE_TRANSLATION_SYSTEM_PROMPT = dedent8`
|
|
5865
5896
|
You are a subtitle translation expert.
|
|
5866
5897
|
You will receive a sequence of subtitle cues extracted from a VTT file.
|
|
5867
5898
|
Translate the cues to the requested target language while preserving their original order.
|
|
@@ -6372,7 +6403,7 @@ async function translateCaptions(assetId, trackId, toLanguageCode, options) {
|
|
|
6372
6403
|
}
|
|
6373
6404
|
if (uploadToMux) {
|
|
6374
6405
|
try {
|
|
6375
|
-
const languageName = getLanguageName(toLanguageCode);
|
|
6406
|
+
const languageName = getLanguageName(toLanguageCode) ?? toLanguageCode.toUpperCase();
|
|
6376
6407
|
const trackName = `${languageName} (auto-translated)`;
|
|
6377
6408
|
uploadedTrackId = await createTextTrackOnMux(
|
|
6378
6409
|
assetId,
|