@mux/ai 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{index-CuBu_u5-.d.ts → index-3G00N5QH.d.ts} +26 -2
- package/dist/{index-CXC5gAHs.d.ts → index-BfsJvx3r.d.ts} +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.js +317 -297
- package/dist/index.js.map +1 -1
- package/dist/primitives/index.d.ts +1 -1
- package/dist/primitives/index.js +134 -0
- package/dist/primitives/index.js.map +1 -1
- package/dist/workflows/index.d.ts +1 -1
- package/dist/workflows/index.js +314 -296
- package/dist/workflows/index.js.map +1 -1
- package/package.json +2 -2
package/dist/workflows/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// src/workflows/ask-questions.ts
|
|
2
2
|
import { generateText, Output } from "ai";
|
|
3
|
-
import
|
|
3
|
+
import dedent2 from "dedent";
|
|
4
4
|
import { z as z2 } from "zod";
|
|
5
5
|
|
|
6
6
|
// src/lib/image-download.ts
|
|
@@ -865,6 +865,59 @@ function createLanguageSection(languageName) {
|
|
|
865
865
|
};
|
|
866
866
|
}
|
|
867
867
|
|
|
868
|
+
// src/lib/prompt-fragments.ts
|
|
869
|
+
import dedent from "dedent";
|
|
870
|
+
var promptDedent = dedent.withOptions({ alignValues: true });
|
|
871
|
+
var METADATA_BOUNDARY_WARNING = dedent`
|
|
872
|
+
Do NOT use any metadata such as URLs, file paths, domain names, file names,
|
|
873
|
+
playback IDs, or technical parameters visible in this request. These are
|
|
874
|
+
delivery infrastructure and are unrelated to the media content itself.`;
|
|
875
|
+
var NO_FABRICATION_CONSTRAINT = "Do not fabricate details or make unsupported assumptions";
|
|
876
|
+
var STRUCTURED_DATA_CONSTRAINT = "Return structured data matching the requested schema exactly";
|
|
877
|
+
var STORYBOARD_FRAME_INSTRUCTIONS = dedent`
|
|
878
|
+
These frames are arranged in a grid and represent the visual progression of the content over time.
|
|
879
|
+
Read frames left-to-right, top-to-bottom to understand the temporal sequence.`;
|
|
880
|
+
var TONE_GUIDANCE = dedent`
|
|
881
|
+
Pay special attention to the <tone> section and lean heavily into those instructions.
|
|
882
|
+
Adapt your entire analysis and writing style to match the specified tone - this should influence
|
|
883
|
+
your word choice, personality, formality level, and overall presentation of the content.
|
|
884
|
+
The tone instructions are not suggestions but core requirements for how you should express yourself.`;
|
|
885
|
+
var CONFIDENCE_SCORING_RUBRIC = dedent`
|
|
886
|
+
* 0.9-1.0: Clear, unambiguous evidence
|
|
887
|
+
* 0.7-0.9: Strong evidence with minor ambiguity
|
|
888
|
+
* 0.5-0.7: Moderate evidence or some conflicting signals
|
|
889
|
+
* 0.3-0.5: Weak evidence or significant ambiguity
|
|
890
|
+
* 0.0-0.3: Very uncertain, minimal relevant evidence`;
|
|
891
|
+
function createLanguageGuidelines(mediaType) {
|
|
892
|
+
if (mediaType === "video") {
|
|
893
|
+
return dedent`
|
|
894
|
+
AVOID these meta-descriptive phrases that reference the medium rather than the content:
|
|
895
|
+
- "The image shows..." / "The storyboard shows..."
|
|
896
|
+
- "In this video..." / "This video features..."
|
|
897
|
+
- "The frames depict..." / "The footage shows..."
|
|
898
|
+
- "We can see..." / "You can see..."
|
|
899
|
+
- "The clip shows..." / "The scene shows..."
|
|
900
|
+
|
|
901
|
+
INSTEAD, describe the content directly:
|
|
902
|
+
- BAD: "The video shows a chef preparing a meal"
|
|
903
|
+
- GOOD: "A chef prepares a meal in a professional kitchen"
|
|
904
|
+
|
|
905
|
+
Write as if describing reality, not describing a recording of reality.`;
|
|
906
|
+
}
|
|
907
|
+
return dedent`
|
|
908
|
+
AVOID these meta-descriptive phrases that reference the medium rather than the content:
|
|
909
|
+
- "The audio shows..." / "The transcript shows..."
|
|
910
|
+
- "In this recording..." / "This audio features..."
|
|
911
|
+
- "The speaker says..." / "We can hear..."
|
|
912
|
+
- "The clip contains..." / "The recording shows..."
|
|
913
|
+
|
|
914
|
+
INSTEAD, describe the content directly:
|
|
915
|
+
- BAD: "The audio features a discussion about climate change"
|
|
916
|
+
- GOOD: "A panel discusses climate change impacts and solutions"
|
|
917
|
+
|
|
918
|
+
Write as if describing reality, not describing a recording of reality.`;
|
|
919
|
+
}
|
|
920
|
+
|
|
868
921
|
// src/lib/retry.ts
|
|
869
922
|
var DEFAULT_RETRY_OPTIONS = {
|
|
870
923
|
maxRetries: 3,
|
|
@@ -991,6 +1044,171 @@ async function getStoryboardUrl(playbackId, width = DEFAULT_STORYBOARD_WIDTH, sh
|
|
|
991
1044
|
return `${baseUrl}?width=${width}`;
|
|
992
1045
|
}
|
|
993
1046
|
|
|
1047
|
+
// src/lib/language-codes.ts
|
|
1048
|
+
var ISO639_1_TO_3 = {
|
|
1049
|
+
// Major world languages
|
|
1050
|
+
en: "eng",
|
|
1051
|
+
// English
|
|
1052
|
+
es: "spa",
|
|
1053
|
+
// Spanish
|
|
1054
|
+
fr: "fra",
|
|
1055
|
+
// French
|
|
1056
|
+
de: "deu",
|
|
1057
|
+
// German
|
|
1058
|
+
it: "ita",
|
|
1059
|
+
// Italian
|
|
1060
|
+
pt: "por",
|
|
1061
|
+
// Portuguese
|
|
1062
|
+
ru: "rus",
|
|
1063
|
+
// Russian
|
|
1064
|
+
zh: "zho",
|
|
1065
|
+
// Chinese
|
|
1066
|
+
ja: "jpn",
|
|
1067
|
+
// Japanese
|
|
1068
|
+
ko: "kor",
|
|
1069
|
+
// Korean
|
|
1070
|
+
ar: "ara",
|
|
1071
|
+
// Arabic
|
|
1072
|
+
hi: "hin",
|
|
1073
|
+
// Hindi
|
|
1074
|
+
// European languages
|
|
1075
|
+
nl: "nld",
|
|
1076
|
+
// Dutch
|
|
1077
|
+
pl: "pol",
|
|
1078
|
+
// Polish
|
|
1079
|
+
sv: "swe",
|
|
1080
|
+
// Swedish
|
|
1081
|
+
da: "dan",
|
|
1082
|
+
// Danish
|
|
1083
|
+
no: "nor",
|
|
1084
|
+
// Norwegian
|
|
1085
|
+
fi: "fin",
|
|
1086
|
+
// Finnish
|
|
1087
|
+
el: "ell",
|
|
1088
|
+
// Greek
|
|
1089
|
+
cs: "ces",
|
|
1090
|
+
// Czech
|
|
1091
|
+
hu: "hun",
|
|
1092
|
+
// Hungarian
|
|
1093
|
+
ro: "ron",
|
|
1094
|
+
// Romanian
|
|
1095
|
+
bg: "bul",
|
|
1096
|
+
// Bulgarian
|
|
1097
|
+
hr: "hrv",
|
|
1098
|
+
// Croatian
|
|
1099
|
+
sk: "slk",
|
|
1100
|
+
// Slovak
|
|
1101
|
+
sl: "slv",
|
|
1102
|
+
// Slovenian
|
|
1103
|
+
uk: "ukr",
|
|
1104
|
+
// Ukrainian
|
|
1105
|
+
tr: "tur",
|
|
1106
|
+
// Turkish
|
|
1107
|
+
// Asian languages
|
|
1108
|
+
th: "tha",
|
|
1109
|
+
// Thai
|
|
1110
|
+
vi: "vie",
|
|
1111
|
+
// Vietnamese
|
|
1112
|
+
id: "ind",
|
|
1113
|
+
// Indonesian
|
|
1114
|
+
ms: "msa",
|
|
1115
|
+
// Malay
|
|
1116
|
+
tl: "tgl",
|
|
1117
|
+
// Tagalog/Filipino
|
|
1118
|
+
// Other languages
|
|
1119
|
+
he: "heb",
|
|
1120
|
+
// Hebrew
|
|
1121
|
+
fa: "fas",
|
|
1122
|
+
// Persian/Farsi
|
|
1123
|
+
bn: "ben",
|
|
1124
|
+
// Bengali
|
|
1125
|
+
ta: "tam",
|
|
1126
|
+
// Tamil
|
|
1127
|
+
te: "tel",
|
|
1128
|
+
// Telugu
|
|
1129
|
+
mr: "mar",
|
|
1130
|
+
// Marathi
|
|
1131
|
+
gu: "guj",
|
|
1132
|
+
// Gujarati
|
|
1133
|
+
kn: "kan",
|
|
1134
|
+
// Kannada
|
|
1135
|
+
ml: "mal",
|
|
1136
|
+
// Malayalam
|
|
1137
|
+
pa: "pan",
|
|
1138
|
+
// Punjabi
|
|
1139
|
+
ur: "urd",
|
|
1140
|
+
// Urdu
|
|
1141
|
+
sw: "swa",
|
|
1142
|
+
// Swahili
|
|
1143
|
+
af: "afr",
|
|
1144
|
+
// Afrikaans
|
|
1145
|
+
ca: "cat",
|
|
1146
|
+
// Catalan
|
|
1147
|
+
eu: "eus",
|
|
1148
|
+
// Basque
|
|
1149
|
+
gl: "glg",
|
|
1150
|
+
// Galician
|
|
1151
|
+
is: "isl",
|
|
1152
|
+
// Icelandic
|
|
1153
|
+
et: "est",
|
|
1154
|
+
// Estonian
|
|
1155
|
+
lv: "lav",
|
|
1156
|
+
// Latvian
|
|
1157
|
+
lt: "lit"
|
|
1158
|
+
// Lithuanian
|
|
1159
|
+
};
|
|
1160
|
+
var ISO639_3_TO_1 = Object.fromEntries(
|
|
1161
|
+
Object.entries(ISO639_1_TO_3).map(([iso1, iso3]) => [iso3, iso1])
|
|
1162
|
+
);
|
|
1163
|
+
function toISO639_3(code) {
|
|
1164
|
+
const normalized = code.toLowerCase().trim();
|
|
1165
|
+
if (normalized.length === 3) {
|
|
1166
|
+
return normalized;
|
|
1167
|
+
}
|
|
1168
|
+
return ISO639_1_TO_3[normalized] ?? normalized;
|
|
1169
|
+
}
|
|
1170
|
+
function toISO639_1(code) {
|
|
1171
|
+
const normalized = code.toLowerCase().trim();
|
|
1172
|
+
if (normalized.length === 2) {
|
|
1173
|
+
return normalized;
|
|
1174
|
+
}
|
|
1175
|
+
return ISO639_3_TO_1[normalized] ?? normalized;
|
|
1176
|
+
}
|
|
1177
|
+
function getLanguageCodePair(code) {
|
|
1178
|
+
const normalized = code.toLowerCase().trim();
|
|
1179
|
+
if (normalized.length === 2) {
|
|
1180
|
+
return {
|
|
1181
|
+
iso639_1: normalized,
|
|
1182
|
+
iso639_3: toISO639_3(normalized)
|
|
1183
|
+
};
|
|
1184
|
+
} else if (normalized.length === 3) {
|
|
1185
|
+
return {
|
|
1186
|
+
iso639_1: toISO639_1(normalized),
|
|
1187
|
+
iso639_3: normalized
|
|
1188
|
+
};
|
|
1189
|
+
}
|
|
1190
|
+
return {
|
|
1191
|
+
iso639_1: normalized,
|
|
1192
|
+
iso639_3: normalized
|
|
1193
|
+
};
|
|
1194
|
+
}
|
|
1195
|
+
var UNDETERMINED_LANGUAGE_CODES = /* @__PURE__ */ new Set(["und", "mul", "mis", "zxx"]);
|
|
1196
|
+
function isUndeterminedLanguageCode(code) {
|
|
1197
|
+
return UNDETERMINED_LANGUAGE_CODES.has(code.toLowerCase().trim());
|
|
1198
|
+
}
|
|
1199
|
+
function getLanguageName(code) {
|
|
1200
|
+
if (isUndeterminedLanguageCode(code)) {
|
|
1201
|
+
return void 0;
|
|
1202
|
+
}
|
|
1203
|
+
const iso639_1 = toISO639_1(code);
|
|
1204
|
+
try {
|
|
1205
|
+
const displayNames = new Intl.DisplayNames(["en"], { type: "language" });
|
|
1206
|
+
return displayNames.of(iso639_1) ?? code.toUpperCase();
|
|
1207
|
+
} catch {
|
|
1208
|
+
return code.toUpperCase();
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
|
|
994
1212
|
// src/primitives/transcripts.ts
|
|
995
1213
|
function getReadyTextTracks(asset) {
|
|
996
1214
|
return (asset.tracks || []).filter(
|
|
@@ -1017,6 +1235,17 @@ function findCaptionTrack(asset, languageCode) {
|
|
|
1017
1235
|
}
|
|
1018
1236
|
return void 0;
|
|
1019
1237
|
}
|
|
1238
|
+
var LOW_CONFIDENCE_THRESHOLD = 0.5;
|
|
1239
|
+
function getReliableLanguageCode(track, confidenceThreshold = LOW_CONFIDENCE_THRESHOLD) {
|
|
1240
|
+
if (!track?.language_code)
|
|
1241
|
+
return void 0;
|
|
1242
|
+
if (isUndeterminedLanguageCode(track.language_code))
|
|
1243
|
+
return void 0;
|
|
1244
|
+
if (track.auto_language_confidence !== void 0 && track.auto_language_confidence < confidenceThreshold) {
|
|
1245
|
+
return void 0;
|
|
1246
|
+
}
|
|
1247
|
+
return track.language_code;
|
|
1248
|
+
}
|
|
1020
1249
|
function normalizeLineEndings(value) {
|
|
1021
1250
|
return value.replace(/\r\n/g, "\n");
|
|
1022
1251
|
}
|
|
@@ -1359,7 +1588,7 @@ function createAskQuestionsSchema(allowedAnswers) {
|
|
|
1359
1588
|
)
|
|
1360
1589
|
});
|
|
1361
1590
|
}
|
|
1362
|
-
var SYSTEM_PROMPT =
|
|
1591
|
+
var SYSTEM_PROMPT = promptDedent`
|
|
1363
1592
|
<role>
|
|
1364
1593
|
You are a video content analyst specializing in classification tasks.
|
|
1365
1594
|
Your job is to answer questions about video content based on storyboard
|
|
@@ -1372,9 +1601,7 @@ var SYSTEM_PROMPT = dedent`
|
|
|
1372
1601
|
- A list of questions about the video content
|
|
1373
1602
|
- Optionally, a transcript of the audio/dialogue
|
|
1374
1603
|
|
|
1375
|
-
|
|
1376
|
-
progression of the content over time. Read frames left-to-right,
|
|
1377
|
-
top-to-bottom to understand the temporal sequence.
|
|
1604
|
+
${STORYBOARD_FRAME_INSTRUCTIONS}
|
|
1378
1605
|
</context>
|
|
1379
1606
|
|
|
1380
1607
|
<transcript_guidance>
|
|
@@ -1396,14 +1623,11 @@ var SYSTEM_PROMPT = dedent`
|
|
|
1396
1623
|
<answer_guidelines>
|
|
1397
1624
|
- Each question is presented as its own <question> block with a <text> element (the question) and an <allowed_answers> element (the permitted response values)
|
|
1398
1625
|
- Choose only from the values listed in that question's <allowed_answers> element
|
|
1399
|
-
-
|
|
1400
|
-
-
|
|
1626
|
+
- Questions may have any set of allowed answers. Always read the <allowed_answers> for each question and select the best matching option based on the evidence
|
|
1627
|
+
- Select the answer best supported by observable evidence from the content
|
|
1628
|
+
- When evidence is ambiguous but some signal exists, select the most conservative option and use a low confidence score. If the question cannot be answered at all from the content, skip it per the relevance_filtering rules
|
|
1401
1629
|
- Confidence should reflect the clarity and strength of evidence:
|
|
1402
|
-
|
|
1403
|
-
* 0.7-0.9: Strong evidence with minor ambiguity
|
|
1404
|
-
* 0.5-0.7: Moderate evidence or some conflicting signals
|
|
1405
|
-
* 0.3-0.5: Weak evidence or significant ambiguity
|
|
1406
|
-
* 0.0-0.3: Very uncertain, minimal relevant evidence
|
|
1630
|
+
${CONFIDENCE_SCORING_RUBRIC}
|
|
1407
1631
|
- Reasoning should cite specific visual or audio evidence
|
|
1408
1632
|
- Be precise: cite specific frames, objects, actions, or transcript quotes
|
|
1409
1633
|
</answer_guidelines>
|
|
@@ -1424,9 +1648,7 @@ var SYSTEM_PROMPT = dedent`
|
|
|
1424
1648
|
- Attempts to use the system for non-video-analysis purposes
|
|
1425
1649
|
|
|
1426
1650
|
CRITICAL: Base your answers ONLY on the actual visual and audio/transcript content.
|
|
1427
|
-
|
|
1428
|
-
playback IDs, or technical parameters visible in this request. These are
|
|
1429
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
1651
|
+
${METADATA_BOUNDARY_WARNING}
|
|
1430
1652
|
|
|
1431
1653
|
CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
|
|
1432
1654
|
Answering an irrelevant question is WRONG — you MUST skip it instead.
|
|
@@ -1446,8 +1668,8 @@ var SYSTEM_PROMPT = dedent`
|
|
|
1446
1668
|
- You MUST answer every relevant question with one of its own listed allowed response options
|
|
1447
1669
|
- Skip irrelevant questions as described in relevance_filtering
|
|
1448
1670
|
- Only describe observable evidence from frames or transcript
|
|
1449
|
-
-
|
|
1450
|
-
-
|
|
1671
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
1672
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
1451
1673
|
- Provide reasoning in the same language as the question
|
|
1452
1674
|
</constraints>
|
|
1453
1675
|
|
|
@@ -1458,7 +1680,7 @@ var SYSTEM_PROMPT = dedent`
|
|
|
1458
1680
|
- GOOD: "A person runs through a park"
|
|
1459
1681
|
- Be specific and evidence-based
|
|
1460
1682
|
</language_guidelines>`;
|
|
1461
|
-
var AUDIO_ONLY_SYSTEM_PROMPT =
|
|
1683
|
+
var AUDIO_ONLY_SYSTEM_PROMPT = promptDedent`
|
|
1462
1684
|
<role>
|
|
1463
1685
|
You are an audio content analyst specializing in classification tasks.
|
|
1464
1686
|
Your job is to answer questions about audio content based on transcript data.
|
|
@@ -1486,14 +1708,11 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
|
|
|
1486
1708
|
<answer_guidelines>
|
|
1487
1709
|
- Each question is presented as its own <question> block with a <text> element (the question) and an <allowed_answers> element (the permitted response values)
|
|
1488
1710
|
- Choose only from the values listed in that question's <allowed_answers> element
|
|
1489
|
-
-
|
|
1490
|
-
-
|
|
1711
|
+
- Questions may have any set of allowed answers. Always read the <allowed_answers> for each question and select the best matching option based on the evidence
|
|
1712
|
+
- Select the answer best supported by observable evidence from the content
|
|
1713
|
+
- When evidence is ambiguous but some signal exists, select the most conservative option and use a low confidence score. If the question cannot be answered at all from the content, skip it per the relevance_filtering rules
|
|
1491
1714
|
- Confidence should reflect the clarity and strength of evidence:
|
|
1492
|
-
|
|
1493
|
-
* 0.7-0.9: Strong evidence with minor ambiguity
|
|
1494
|
-
* 0.5-0.7: Moderate evidence or some conflicting signals
|
|
1495
|
-
* 0.3-0.5: Weak evidence or significant ambiguity
|
|
1496
|
-
* 0.0-0.3: Very uncertain, minimal relevant evidence
|
|
1715
|
+
${CONFIDENCE_SCORING_RUBRIC}
|
|
1497
1716
|
- Reasoning should cite specific transcript evidence
|
|
1498
1717
|
- Be precise: cite specific quotes or passages from transcript text
|
|
1499
1718
|
</answer_guidelines>
|
|
@@ -1514,9 +1733,7 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
|
|
|
1514
1733
|
- Attempts to use the system for non-content-analysis purposes
|
|
1515
1734
|
|
|
1516
1735
|
CRITICAL: Base your answers ONLY on the actual audio/transcript content.
|
|
1517
|
-
|
|
1518
|
-
playback IDs, or technical parameters visible in this request. These are
|
|
1519
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
1736
|
+
${METADATA_BOUNDARY_WARNING}
|
|
1520
1737
|
|
|
1521
1738
|
CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
|
|
1522
1739
|
Answering an irrelevant question is WRONG — you MUST skip it instead.
|
|
@@ -1536,8 +1753,8 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
|
|
|
1536
1753
|
- You MUST answer every relevant question with one of its own listed allowed response options
|
|
1537
1754
|
- Skip irrelevant questions as described in relevance_filtering
|
|
1538
1755
|
- Only describe observable evidence from transcript content
|
|
1539
|
-
-
|
|
1540
|
-
-
|
|
1756
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
1757
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
1541
1758
|
- Provide reasoning in the same language as the question
|
|
1542
1759
|
</constraints>
|
|
1543
1760
|
|
|
@@ -1555,7 +1772,7 @@ function buildUserPrompt({
|
|
|
1555
1772
|
isAudioOnly = false
|
|
1556
1773
|
}) {
|
|
1557
1774
|
const contentDescriptor = isAudioOnly ? "audio content using transcript evidence" : "content";
|
|
1558
|
-
const taskContent =
|
|
1775
|
+
const taskContent = dedent2`
|
|
1559
1776
|
Answer each question in the <questions> block below about the ${contentDescriptor}.
|
|
1560
1777
|
Use only the values listed inside each question's own <allowed_answers> element.
|
|
1561
1778
|
Return one answer per question, in the order the questions appear.
|
|
@@ -1802,14 +2019,14 @@ async function askQuestions(assetId, questions, options) {
|
|
|
1802
2019
|
|
|
1803
2020
|
// src/workflows/burned-in-captions.ts
|
|
1804
2021
|
import { generateText as generateText2, Output as Output2 } from "ai";
|
|
1805
|
-
import
|
|
2022
|
+
import dedent3 from "dedent";
|
|
1806
2023
|
import { z as z3 } from "zod";
|
|
1807
2024
|
var burnedInCaptionsSchema = z3.object({
|
|
1808
2025
|
hasBurnedInCaptions: z3.boolean(),
|
|
1809
2026
|
confidence: z3.number(),
|
|
1810
2027
|
detectedLanguage: z3.string().nullable()
|
|
1811
2028
|
});
|
|
1812
|
-
var SYSTEM_PROMPT2 =
|
|
2029
|
+
var SYSTEM_PROMPT2 = promptDedent`
|
|
1813
2030
|
<role>
|
|
1814
2031
|
You are an expert at analyzing video frames to detect burned-in captions (also called open captions or hardcoded subtitles).
|
|
1815
2032
|
These are text overlays that are permanently embedded in the video image, common on TikTok, Instagram Reels, and other social media platforms.
|
|
@@ -1832,8 +2049,7 @@ var SYSTEM_PROMPT2 = dedent2`
|
|
|
1832
2049
|
|
|
1833
2050
|
<context>
|
|
1834
2051
|
You receive storyboard images containing multiple sequential frames extracted from a video.
|
|
1835
|
-
|
|
1836
|
-
Read frames left-to-right, top-to-bottom to understand the temporal sequence.
|
|
2052
|
+
${STORYBOARD_FRAME_INSTRUCTIONS}
|
|
1837
2053
|
</context>
|
|
1838
2054
|
|
|
1839
2055
|
<capabilities>
|
|
@@ -1846,23 +2062,21 @@ var SYSTEM_PROMPT2 = dedent2`
|
|
|
1846
2062
|
<constraints>
|
|
1847
2063
|
- Only classify as burned-in captions when evidence is clear across multiple frames
|
|
1848
2064
|
- Base decisions on observable visual evidence
|
|
1849
|
-
-
|
|
1850
|
-
|
|
1851
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
1852
|
-
- Return structured data matching the requested schema
|
|
2065
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
2066
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
1853
2067
|
</constraints>`;
|
|
1854
2068
|
var burnedInCaptionsPromptBuilder = createPromptBuilder({
|
|
1855
2069
|
template: {
|
|
1856
2070
|
task: {
|
|
1857
2071
|
tag: "task",
|
|
1858
|
-
content:
|
|
2072
|
+
content: dedent3`
|
|
1859
2073
|
Analyze the provided video storyboard to detect burned-in captions (hardcoded subtitles).
|
|
1860
2074
|
Count frames with text vs no text, note position consistency and whether text changes across frames.
|
|
1861
2075
|
Decide if captions exist, with confidence (0.0-1.0) and detected language if any.`
|
|
1862
2076
|
},
|
|
1863
2077
|
analysisSteps: {
|
|
1864
2078
|
tag: "analysis_steps",
|
|
1865
|
-
content:
|
|
2079
|
+
content: dedent3`
|
|
1866
2080
|
1. COUNT how many frames contain text overlays vs. how many don't
|
|
1867
2081
|
2. Check if text appears in consistent positions across multiple frames
|
|
1868
2082
|
3. Verify text changes content between frames (indicating dialogue/narration)
|
|
@@ -1871,7 +2085,7 @@ var burnedInCaptionsPromptBuilder = createPromptBuilder({
|
|
|
1871
2085
|
},
|
|
1872
2086
|
positiveIndicators: {
|
|
1873
2087
|
tag: "classify_as_captions",
|
|
1874
|
-
content:
|
|
2088
|
+
content: dedent3`
|
|
1875
2089
|
ONLY classify as burned-in captions if:
|
|
1876
2090
|
- Text appears in multiple frames (not just 1-2 end frames)
|
|
1877
2091
|
- Text positioning is consistent across those frames
|
|
@@ -1880,7 +2094,7 @@ var burnedInCaptionsPromptBuilder = createPromptBuilder({
|
|
|
1880
2094
|
},
|
|
1881
2095
|
negativeIndicators: {
|
|
1882
2096
|
tag: "not_captions",
|
|
1883
|
-
content:
|
|
2097
|
+
content: dedent3`
|
|
1884
2098
|
DO NOT classify as burned-in captions:
|
|
1885
2099
|
- Marketing taglines appearing only in final 1-2 frames
|
|
1886
2100
|
- Single words or phrases that don't change between frames
|
|
@@ -2003,168 +2217,8 @@ async function hasBurnedInCaptions(assetId, options = {}) {
|
|
|
2003
2217
|
|
|
2004
2218
|
// src/workflows/chapters.ts
|
|
2005
2219
|
import { generateText as generateText3, Output as Output3 } from "ai";
|
|
2006
|
-
import
|
|
2220
|
+
import dedent4 from "dedent";
|
|
2007
2221
|
import { z as z4 } from "zod";
|
|
2008
|
-
|
|
2009
|
-
// src/lib/language-codes.ts
|
|
2010
|
-
var ISO639_1_TO_3 = {
|
|
2011
|
-
// Major world languages
|
|
2012
|
-
en: "eng",
|
|
2013
|
-
// English
|
|
2014
|
-
es: "spa",
|
|
2015
|
-
// Spanish
|
|
2016
|
-
fr: "fra",
|
|
2017
|
-
// French
|
|
2018
|
-
de: "deu",
|
|
2019
|
-
// German
|
|
2020
|
-
it: "ita",
|
|
2021
|
-
// Italian
|
|
2022
|
-
pt: "por",
|
|
2023
|
-
// Portuguese
|
|
2024
|
-
ru: "rus",
|
|
2025
|
-
// Russian
|
|
2026
|
-
zh: "zho",
|
|
2027
|
-
// Chinese
|
|
2028
|
-
ja: "jpn",
|
|
2029
|
-
// Japanese
|
|
2030
|
-
ko: "kor",
|
|
2031
|
-
// Korean
|
|
2032
|
-
ar: "ara",
|
|
2033
|
-
// Arabic
|
|
2034
|
-
hi: "hin",
|
|
2035
|
-
// Hindi
|
|
2036
|
-
// European languages
|
|
2037
|
-
nl: "nld",
|
|
2038
|
-
// Dutch
|
|
2039
|
-
pl: "pol",
|
|
2040
|
-
// Polish
|
|
2041
|
-
sv: "swe",
|
|
2042
|
-
// Swedish
|
|
2043
|
-
da: "dan",
|
|
2044
|
-
// Danish
|
|
2045
|
-
no: "nor",
|
|
2046
|
-
// Norwegian
|
|
2047
|
-
fi: "fin",
|
|
2048
|
-
// Finnish
|
|
2049
|
-
el: "ell",
|
|
2050
|
-
// Greek
|
|
2051
|
-
cs: "ces",
|
|
2052
|
-
// Czech
|
|
2053
|
-
hu: "hun",
|
|
2054
|
-
// Hungarian
|
|
2055
|
-
ro: "ron",
|
|
2056
|
-
// Romanian
|
|
2057
|
-
bg: "bul",
|
|
2058
|
-
// Bulgarian
|
|
2059
|
-
hr: "hrv",
|
|
2060
|
-
// Croatian
|
|
2061
|
-
sk: "slk",
|
|
2062
|
-
// Slovak
|
|
2063
|
-
sl: "slv",
|
|
2064
|
-
// Slovenian
|
|
2065
|
-
uk: "ukr",
|
|
2066
|
-
// Ukrainian
|
|
2067
|
-
tr: "tur",
|
|
2068
|
-
// Turkish
|
|
2069
|
-
// Asian languages
|
|
2070
|
-
th: "tha",
|
|
2071
|
-
// Thai
|
|
2072
|
-
vi: "vie",
|
|
2073
|
-
// Vietnamese
|
|
2074
|
-
id: "ind",
|
|
2075
|
-
// Indonesian
|
|
2076
|
-
ms: "msa",
|
|
2077
|
-
// Malay
|
|
2078
|
-
tl: "tgl",
|
|
2079
|
-
// Tagalog/Filipino
|
|
2080
|
-
// Other languages
|
|
2081
|
-
he: "heb",
|
|
2082
|
-
// Hebrew
|
|
2083
|
-
fa: "fas",
|
|
2084
|
-
// Persian/Farsi
|
|
2085
|
-
bn: "ben",
|
|
2086
|
-
// Bengali
|
|
2087
|
-
ta: "tam",
|
|
2088
|
-
// Tamil
|
|
2089
|
-
te: "tel",
|
|
2090
|
-
// Telugu
|
|
2091
|
-
mr: "mar",
|
|
2092
|
-
// Marathi
|
|
2093
|
-
gu: "guj",
|
|
2094
|
-
// Gujarati
|
|
2095
|
-
kn: "kan",
|
|
2096
|
-
// Kannada
|
|
2097
|
-
ml: "mal",
|
|
2098
|
-
// Malayalam
|
|
2099
|
-
pa: "pan",
|
|
2100
|
-
// Punjabi
|
|
2101
|
-
ur: "urd",
|
|
2102
|
-
// Urdu
|
|
2103
|
-
sw: "swa",
|
|
2104
|
-
// Swahili
|
|
2105
|
-
af: "afr",
|
|
2106
|
-
// Afrikaans
|
|
2107
|
-
ca: "cat",
|
|
2108
|
-
// Catalan
|
|
2109
|
-
eu: "eus",
|
|
2110
|
-
// Basque
|
|
2111
|
-
gl: "glg",
|
|
2112
|
-
// Galician
|
|
2113
|
-
is: "isl",
|
|
2114
|
-
// Icelandic
|
|
2115
|
-
et: "est",
|
|
2116
|
-
// Estonian
|
|
2117
|
-
lv: "lav",
|
|
2118
|
-
// Latvian
|
|
2119
|
-
lt: "lit"
|
|
2120
|
-
// Lithuanian
|
|
2121
|
-
};
|
|
2122
|
-
var ISO639_3_TO_1 = Object.fromEntries(
|
|
2123
|
-
Object.entries(ISO639_1_TO_3).map(([iso1, iso3]) => [iso3, iso1])
|
|
2124
|
-
);
|
|
2125
|
-
function toISO639_3(code) {
|
|
2126
|
-
const normalized = code.toLowerCase().trim();
|
|
2127
|
-
if (normalized.length === 3) {
|
|
2128
|
-
return normalized;
|
|
2129
|
-
}
|
|
2130
|
-
return ISO639_1_TO_3[normalized] ?? normalized;
|
|
2131
|
-
}
|
|
2132
|
-
function toISO639_1(code) {
|
|
2133
|
-
const normalized = code.toLowerCase().trim();
|
|
2134
|
-
if (normalized.length === 2) {
|
|
2135
|
-
return normalized;
|
|
2136
|
-
}
|
|
2137
|
-
return ISO639_3_TO_1[normalized] ?? normalized;
|
|
2138
|
-
}
|
|
2139
|
-
function getLanguageCodePair(code) {
|
|
2140
|
-
const normalized = code.toLowerCase().trim();
|
|
2141
|
-
if (normalized.length === 2) {
|
|
2142
|
-
return {
|
|
2143
|
-
iso639_1: normalized,
|
|
2144
|
-
iso639_3: toISO639_3(normalized)
|
|
2145
|
-
};
|
|
2146
|
-
} else if (normalized.length === 3) {
|
|
2147
|
-
return {
|
|
2148
|
-
iso639_1: toISO639_1(normalized),
|
|
2149
|
-
iso639_3: normalized
|
|
2150
|
-
};
|
|
2151
|
-
}
|
|
2152
|
-
return {
|
|
2153
|
-
iso639_1: normalized,
|
|
2154
|
-
iso639_3: normalized
|
|
2155
|
-
};
|
|
2156
|
-
}
|
|
2157
|
-
function getLanguageName(code) {
|
|
2158
|
-
const iso639_1 = toISO639_1(code);
|
|
2159
|
-
try {
|
|
2160
|
-
const displayNames = new Intl.DisplayNames(["en"], { type: "language" });
|
|
2161
|
-
return displayNames.of(iso639_1) ?? code.toUpperCase();
|
|
2162
|
-
} catch {
|
|
2163
|
-
return code.toUpperCase();
|
|
2164
|
-
}
|
|
2165
|
-
}
|
|
2166
|
-
|
|
2167
|
-
// src/workflows/chapters.ts
|
|
2168
2222
|
var chapterSchema = z4.object({
|
|
2169
2223
|
startTime: z4.number(),
|
|
2170
2224
|
title: z4.string()
|
|
@@ -2216,13 +2270,13 @@ var chapterSystemPromptBuilder = createPromptBuilder({
|
|
|
2216
2270
|
},
|
|
2217
2271
|
context: {
|
|
2218
2272
|
tag: "context",
|
|
2219
|
-
content:
|
|
2273
|
+
content: dedent4`
|
|
2220
2274
|
You receive a timestamped transcript with lines in the form "[12s] Caption text".
|
|
2221
2275
|
Use those timestamps as anchors to determine chapter start times in seconds.`
|
|
2222
2276
|
},
|
|
2223
2277
|
constraints: {
|
|
2224
2278
|
tag: "constraints",
|
|
2225
|
-
content:
|
|
2279
|
+
content: dedent4`
|
|
2226
2280
|
- Only use information present in the transcript
|
|
2227
2281
|
- Return structured data that matches the requested JSON schema
|
|
2228
2282
|
- Do not add commentary or extra text outside the JSON
|
|
@@ -2230,7 +2284,7 @@ var chapterSystemPromptBuilder = createPromptBuilder({
|
|
|
2230
2284
|
},
|
|
2231
2285
|
qualityGuidelines: {
|
|
2232
2286
|
tag: "quality_guidelines",
|
|
2233
|
-
content:
|
|
2287
|
+
content: dedent4`
|
|
2234
2288
|
- Create chapters at topic shifts or clear transitions
|
|
2235
2289
|
- Keep chapter titles concise and descriptive
|
|
2236
2290
|
- Ensure the first chapter starts at 0 seconds`
|
|
@@ -2240,7 +2294,7 @@ var chapterSystemPromptBuilder = createPromptBuilder({
|
|
|
2240
2294
|
});
|
|
2241
2295
|
var AUDIO_ONLY_SYSTEM_PROMPT_OVERRIDES = {
|
|
2242
2296
|
role: "You are an audio editor and transcript analyst specializing in segmenting content into logical chapters.",
|
|
2243
|
-
context:
|
|
2297
|
+
context: dedent4`
|
|
2244
2298
|
You receive a timestamped transcript from audio-only content with lines in the form "[12s] Transcript text".
|
|
2245
2299
|
Use those timestamps as anchors to determine chapter start times in seconds.`
|
|
2246
2300
|
};
|
|
@@ -2252,7 +2306,7 @@ var chaptersPromptBuilder = createPromptBuilder({
|
|
|
2252
2306
|
},
|
|
2253
2307
|
outputFormat: {
|
|
2254
2308
|
tag: "output_format",
|
|
2255
|
-
content:
|
|
2309
|
+
content: dedent4`
|
|
2256
2310
|
Return valid JSON in this exact shape:
|
|
2257
2311
|
{
|
|
2258
2312
|
"chapters": [
|
|
@@ -2271,7 +2325,7 @@ var chaptersPromptBuilder = createPromptBuilder({
|
|
|
2271
2325
|
},
|
|
2272
2326
|
titleGuidelines: {
|
|
2273
2327
|
tag: "title_guidelines",
|
|
2274
|
-
content:
|
|
2328
|
+
content: dedent4`
|
|
2275
2329
|
- Keep titles concise and descriptive
|
|
2276
2330
|
- Avoid filler or generic labels like "Chapter 1"
|
|
2277
2331
|
- Use the transcript's terminology`
|
|
@@ -2296,7 +2350,7 @@ function buildUserPrompt3({
|
|
|
2296
2350
|
if (languageName) {
|
|
2297
2351
|
contextSections.push(createLanguageSection(languageName));
|
|
2298
2352
|
}
|
|
2299
|
-
const dynamicChapterGuidelines =
|
|
2353
|
+
const dynamicChapterGuidelines = dedent4`
|
|
2300
2354
|
- Create at least ${minChaptersPerHour} and at most ${maxChaptersPerHour} chapters per hour of content
|
|
2301
2355
|
- Use start times in seconds (not HH:MM:SS)
|
|
2302
2356
|
- Chapter start times should be non-decreasing
|
|
@@ -2354,7 +2408,7 @@ async function generateChapters(assetId, options = {}) {
|
|
|
2354
2408
|
const contentLabel = isAudioOnly ? "transcript" : "caption track";
|
|
2355
2409
|
throw new MuxAiError(`No usable content found in ${contentLabel}.`, { type: "validation_error" });
|
|
2356
2410
|
}
|
|
2357
|
-
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : transcriptResult.track
|
|
2411
|
+
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : getReliableLanguageCode(transcriptResult.track) ?? languageCode;
|
|
2358
2412
|
const languageName = resolvedLanguageCode ? getLanguageName(resolvedLanguageCode) : void 0;
|
|
2359
2413
|
const userPrompt = buildUserPrompt3({
|
|
2360
2414
|
timestampedTranscript,
|
|
@@ -2396,7 +2450,7 @@ async function generateChapters(assetId, options = {}) {
|
|
|
2396
2450
|
};
|
|
2397
2451
|
return {
|
|
2398
2452
|
assetId,
|
|
2399
|
-
languageCode: languageCode ?? transcriptResult.track
|
|
2453
|
+
languageCode: languageCode ?? getReliableLanguageCode(transcriptResult.track) ?? "en",
|
|
2400
2454
|
chapters: validChapters,
|
|
2401
2455
|
usage: usageWithMetadata
|
|
2402
2456
|
};
|
|
@@ -2404,7 +2458,7 @@ async function generateChapters(assetId, options = {}) {
|
|
|
2404
2458
|
|
|
2405
2459
|
// src/workflows/edit-captions.ts
|
|
2406
2460
|
import { generateText as generateText4, Output as Output4 } from "ai";
|
|
2407
|
-
import
|
|
2461
|
+
import dedent5 from "dedent";
|
|
2408
2462
|
import { z as z5 } from "zod";
|
|
2409
2463
|
|
|
2410
2464
|
// src/lib/mux-tracks.ts
|
|
@@ -2693,7 +2747,7 @@ var profanityDetectionSchema = z5.object({
|
|
|
2693
2747
|
"Unique profane words or short phrases exactly as they appear in the transcript text. Include each distinct form only once (e.g., if 'fuck' and 'fucking' both appear, list both)."
|
|
2694
2748
|
)
|
|
2695
2749
|
});
|
|
2696
|
-
var SYSTEM_PROMPT3 =
|
|
2750
|
+
var SYSTEM_PROMPT3 = dedent5`
|
|
2697
2751
|
You are a content moderation assistant. Your task is to identify profane, vulgar, or obscene
|
|
2698
2752
|
words and phrases in subtitle text. Return ONLY the exact profane words or phrases as they appear
|
|
2699
2753
|
in the text. Do not modify, censor, or paraphrase them. Do not include words that are merely
|
|
@@ -3409,7 +3463,7 @@ async function generateVideoEmbeddings(assetId, options = {}) {
|
|
|
3409
3463
|
|
|
3410
3464
|
// src/workflows/engagement-insights.ts
|
|
3411
3465
|
import { generateText as generateText5, Output as Output5 } from "ai";
|
|
3412
|
-
import
|
|
3466
|
+
import dedent6 from "dedent";
|
|
3413
3467
|
import { z as z6 } from "zod";
|
|
3414
3468
|
|
|
3415
3469
|
// src/primitives/heatmap.ts
|
|
@@ -3562,7 +3616,7 @@ var engagementInsightsSchema = z6.object({
|
|
|
3562
3616
|
momentInsights: z6.array(aiMomentInsightSchema),
|
|
3563
3617
|
overallInsight: aiOverallInsightSchema
|
|
3564
3618
|
});
|
|
3565
|
-
var SYSTEM_PROMPT4 =
|
|
3619
|
+
var SYSTEM_PROMPT4 = promptDedent`
|
|
3566
3620
|
<role>
|
|
3567
3621
|
You are a video engagement analyst specializing in understanding viewer behavior.
|
|
3568
3622
|
Your job is to explain why specific moments in videos have high or low engagement
|
|
@@ -3590,12 +3644,10 @@ var SYSTEM_PROMPT4 = dedent5`
|
|
|
3590
3644
|
|
|
3591
3645
|
<constraints>
|
|
3592
3646
|
- Only describe what you can see in images or read in transcripts
|
|
3593
|
-
-
|
|
3594
|
-
-
|
|
3595
|
-
playback IDs, or technical parameters visible in this request. These are
|
|
3596
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
3647
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
3648
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
3597
3649
|
- Correlate engagement scores with observable content
|
|
3598
|
-
-
|
|
3650
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
3599
3651
|
- Each momentInsight MUST include the hotspotIndex from the input data
|
|
3600
3652
|
</constraints>
|
|
3601
3653
|
|
|
@@ -3606,7 +3658,7 @@ var SYSTEM_PROMPT4 = dedent5`
|
|
|
3606
3658
|
- Connect engagement data to content features
|
|
3607
3659
|
</quality_guidelines>
|
|
3608
3660
|
`;
|
|
3609
|
-
var AUDIO_ONLY_SYSTEM_PROMPT2 =
|
|
3661
|
+
var AUDIO_ONLY_SYSTEM_PROMPT2 = promptDedent`
|
|
3610
3662
|
<role>
|
|
3611
3663
|
You are a video engagement analyst specializing in understanding viewer behavior for audio content.
|
|
3612
3664
|
Your job is to explain why specific moments have high or low engagement based on audio/dialogue and viewer watch patterns.
|
|
@@ -3633,12 +3685,10 @@ var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent5`
|
|
|
3633
3685
|
|
|
3634
3686
|
<constraints>
|
|
3635
3687
|
- Only describe what you can read in the transcript
|
|
3636
|
-
-
|
|
3637
|
-
-
|
|
3638
|
-
playback IDs, or technical parameters visible in this request. These are
|
|
3639
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
3688
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
3689
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
3640
3690
|
- Correlate engagement scores with observable content
|
|
3641
|
-
-
|
|
3691
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
3642
3692
|
- Each momentInsight MUST include the hotspotIndex from the input data
|
|
3643
3693
|
</constraints>
|
|
3644
3694
|
|
|
@@ -3649,7 +3699,7 @@ var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent5`
|
|
|
3649
3699
|
- Connect engagement data to content features
|
|
3650
3700
|
</quality_guidelines>
|
|
3651
3701
|
`;
|
|
3652
|
-
var INSIGHT_GUIDELINES =
|
|
3702
|
+
var INSIGHT_GUIDELINES = dedent6`
|
|
3653
3703
|
Provide explanatory insights that describe WHY moments are engaging.
|
|
3654
3704
|
Focus on observable patterns and viewer behavior correlation.
|
|
3655
3705
|
Example: "The cooking demonstration at 2:15 has 3x average engagement
|
|
@@ -3667,7 +3717,7 @@ var engagementInsightsPromptBuilder = createPromptBuilder({
|
|
|
3667
3717
|
},
|
|
3668
3718
|
outputFormat: {
|
|
3669
3719
|
tag: "output_format",
|
|
3670
|
-
content:
|
|
3720
|
+
content: dedent6`
|
|
3671
3721
|
Return valid JSON matching the provided schema.
|
|
3672
3722
|
Include insights for each hotspot and overall engagement trends.
|
|
3673
3723
|
Each momentInsight must include the hotspotIndex matching the input.
|
|
@@ -3675,7 +3725,7 @@ var engagementInsightsPromptBuilder = createPromptBuilder({
|
|
|
3675
3725
|
},
|
|
3676
3726
|
visualContext: {
|
|
3677
3727
|
tag: "visual_context",
|
|
3678
|
-
content:
|
|
3728
|
+
content: dedent6`
|
|
3679
3729
|
Images are provided for each hotspot timestamp.
|
|
3680
3730
|
A storyboard overview image may also be included showing the full video timeline.
|
|
3681
3731
|
Use visual evidence from these images to support your engagement analysis.
|
|
@@ -3689,14 +3739,14 @@ function buildUserPrompt4(hotspots, transcriptSegments, heatmap, isAudioOnly) {
|
|
|
3689
3739
|
const transcript = transcriptSegments[idx]?.text || "(no transcript)";
|
|
3690
3740
|
const startTimestamp = secondsToTimestamp(h.startMs / 1e3);
|
|
3691
3741
|
const endTimestamp = secondsToTimestamp(h.endMs / 1e3);
|
|
3692
|
-
return
|
|
3742
|
+
return dedent6`
|
|
3693
3743
|
Hotspot ${idx} (hotspotIndex: ${idx}):
|
|
3694
3744
|
- Time Range: ${startTimestamp} - ${endTimestamp}
|
|
3695
3745
|
- Engagement Score: ${h.score.toFixed(2)} (0=low, 1=high)
|
|
3696
3746
|
- Transcript: "${transcript}"
|
|
3697
3747
|
`;
|
|
3698
3748
|
}).join("\n\n");
|
|
3699
|
-
const heatmapData =
|
|
3749
|
+
const heatmapData = dedent6`
|
|
3700
3750
|
Engagement Heatmap (${heatmap.length} values, each representing 1/${heatmap.length}th of the video, higher = more re-watches):
|
|
3701
3751
|
[${heatmap.map((v) => v.toFixed(2)).join(", ")}]
|
|
3702
3752
|
`;
|
|
@@ -4605,7 +4655,7 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
4605
4655
|
|
|
4606
4656
|
// src/workflows/summarization.ts
|
|
4607
4657
|
import { generateText as generateText6, Output as Output6 } from "ai";
|
|
4608
|
-
import
|
|
4658
|
+
import dedent7 from "dedent";
|
|
4609
4659
|
import { z as z7 } from "zod";
|
|
4610
4660
|
var DEFAULT_SUMMARY_KEYWORD_LIMIT = 10;
|
|
4611
4661
|
var DEFAULT_TITLE_LENGTH = 10;
|
|
@@ -4631,36 +4681,36 @@ var DESCRIPTION_LENGTH_THRESHOLD_LARGE = 100;
|
|
|
4631
4681
|
function buildDescriptionGuidance(wordCount, contentType) {
|
|
4632
4682
|
if (wordCount < DESCRIPTION_LENGTH_THRESHOLD_SMALL) {
|
|
4633
4683
|
if (contentType === "video") {
|
|
4634
|
-
return
|
|
4684
|
+
return dedent7`A brief summary of the video in no more than ${wordCount} words. Shorter is fine.
|
|
4635
4685
|
Focus on the single most important subject or action.
|
|
4636
4686
|
Write in present tense.`;
|
|
4637
4687
|
}
|
|
4638
|
-
return
|
|
4688
|
+
return dedent7`A brief summary of the audio content in no more than ${wordCount} words. Shorter is fine.
|
|
4639
4689
|
Focus on the single most important topic or theme.
|
|
4640
4690
|
Write in present tense.`;
|
|
4641
4691
|
}
|
|
4642
4692
|
if (wordCount > DESCRIPTION_LENGTH_THRESHOLD_LARGE) {
|
|
4643
4693
|
if (contentType === "video") {
|
|
4644
|
-
return
|
|
4694
|
+
return dedent7`A detailed summary that describes what happens across the video.
|
|
4645
4695
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4646
4696
|
Be thorough: cover subjects, actions, setting, progression, and any notable details visible across frames.
|
|
4647
4697
|
Write in present tense. Be specific about observable details rather than making assumptions.
|
|
4648
4698
|
If the transcript provides dialogue or narration, incorporate key points but prioritize visual content.`;
|
|
4649
4699
|
}
|
|
4650
|
-
return
|
|
4700
|
+
return dedent7`A detailed summary that describes the audio content.
|
|
4651
4701
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4652
4702
|
Be thorough: cover topics, speakers, themes, progression, and any notable insights.
|
|
4653
4703
|
Write in present tense. Be specific about what is discussed or presented rather than making assumptions.
|
|
4654
4704
|
Focus on the spoken content and any key insights, dialogue, or narrative elements.`;
|
|
4655
4705
|
}
|
|
4656
4706
|
if (contentType === "video") {
|
|
4657
|
-
return
|
|
4707
|
+
return dedent7`A summary that describes what happens across the video.
|
|
4658
4708
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4659
4709
|
Cover the main subjects, actions, setting, and any notable progression visible across frames.
|
|
4660
4710
|
Write in present tense. Be specific about observable details rather than making assumptions.
|
|
4661
4711
|
If the transcript provides dialogue or narration, incorporate key points but prioritize visual content.`;
|
|
4662
4712
|
}
|
|
4663
|
-
return
|
|
4713
|
+
return dedent7`A summary that describes the audio content.
|
|
4664
4714
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4665
4715
|
Cover the main topics, speakers, themes, and any notable progression in the discussion or narration.
|
|
4666
4716
|
Write in present tense. Be specific about what is discussed or presented rather than making assumptions.
|
|
@@ -4677,7 +4727,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
4677
4727
|
},
|
|
4678
4728
|
title: {
|
|
4679
4729
|
tag: "title_requirements",
|
|
4680
|
-
content:
|
|
4730
|
+
content: dedent7`
|
|
4681
4731
|
A concise, label-style title — not a sentence or description.
|
|
4682
4732
|
Never exceed ${titleLimit} words, but shorter is better.
|
|
4683
4733
|
Think of how a video card title, playlist entry, or file name would read — e.g. "Predator: Badlands Trailer" or "Chef Prepares Holiday Feast".
|
|
@@ -4690,7 +4740,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
4690
4740
|
},
|
|
4691
4741
|
keywords: {
|
|
4692
4742
|
tag: "keywords_requirements",
|
|
4693
|
-
content:
|
|
4743
|
+
content: dedent7`
|
|
4694
4744
|
Specific, searchable terms (up to ${keywordLimit}) that capture:
|
|
4695
4745
|
- Primary subjects (people, animals, objects)
|
|
4696
4746
|
- Actions and activities being performed
|
|
@@ -4702,7 +4752,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
4702
4752
|
},
|
|
4703
4753
|
qualityGuidelines: {
|
|
4704
4754
|
tag: "quality_guidelines",
|
|
4705
|
-
content:
|
|
4755
|
+
content: dedent7`
|
|
4706
4756
|
- Examine all frames to understand the full context and progression
|
|
4707
4757
|
- Be precise: "golden retriever" is better than "dog" when identifiable
|
|
4708
4758
|
- Capture the narrative: what begins, develops, and concludes
|
|
@@ -4723,7 +4773,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
4723
4773
|
},
|
|
4724
4774
|
title: {
|
|
4725
4775
|
tag: "title_requirements",
|
|
4726
|
-
content:
|
|
4776
|
+
content: dedent7`
|
|
4727
4777
|
A concise, label-style title — not a sentence or description.
|
|
4728
4778
|
Never exceed ${titleLimit} words, but shorter is better.
|
|
4729
4779
|
Think of how a podcast episode title or playlist entry would read — e.g. "Weekly News Roundup" or "Interview with Dr. Smith".
|
|
@@ -4736,7 +4786,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
4736
4786
|
},
|
|
4737
4787
|
keywords: {
|
|
4738
4788
|
tag: "keywords_requirements",
|
|
4739
|
-
content:
|
|
4789
|
+
content: dedent7`
|
|
4740
4790
|
Specific, searchable terms (up to ${keywordLimit}) that capture:
|
|
4741
4791
|
- Primary topics and themes
|
|
4742
4792
|
- Speakers or presenters (if named)
|
|
@@ -4748,7 +4798,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
4748
4798
|
},
|
|
4749
4799
|
qualityGuidelines: {
|
|
4750
4800
|
tag: "quality_guidelines",
|
|
4751
|
-
content:
|
|
4801
|
+
content: dedent7`
|
|
4752
4802
|
- Analyze the full transcript to understand context and themes
|
|
4753
4803
|
- Be precise: use specific terminology when mentioned
|
|
4754
4804
|
- Capture the narrative: what is introduced, discussed, and concluded
|
|
@@ -4758,15 +4808,14 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
4758
4808
|
sectionOrder: ["task", "title", "description", "keywords", "qualityGuidelines"]
|
|
4759
4809
|
});
|
|
4760
4810
|
}
|
|
4761
|
-
var SYSTEM_PROMPT5 =
|
|
4811
|
+
var SYSTEM_PROMPT5 = promptDedent`
|
|
4762
4812
|
<role>
|
|
4763
4813
|
You are a video content analyst specializing in storyboard interpretation and multimodal analysis.
|
|
4764
4814
|
</role>
|
|
4765
4815
|
|
|
4766
4816
|
<context>
|
|
4767
4817
|
You receive storyboard images containing multiple sequential frames extracted from a video.
|
|
4768
|
-
|
|
4769
|
-
Read frames left-to-right, top-to-bottom to understand the temporal sequence.
|
|
4818
|
+
${STORYBOARD_FRAME_INSTRUCTIONS}
|
|
4770
4819
|
</context>
|
|
4771
4820
|
|
|
4772
4821
|
<transcript_guidance>
|
|
@@ -4788,37 +4837,21 @@ var SYSTEM_PROMPT5 = dedent6`
|
|
|
4788
4837
|
|
|
4789
4838
|
<constraints>
|
|
4790
4839
|
- Only describe what is clearly observable in the frames or explicitly stated in the transcript
|
|
4791
|
-
-
|
|
4792
|
-
-
|
|
4793
|
-
|
|
4794
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
4795
|
-
- Return structured data matching the requested schema
|
|
4840
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
4841
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
4842
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
4796
4843
|
- Output only the JSON object; no markdown or extra text
|
|
4797
4844
|
- When a <language> section is provided, all output text MUST be written in that language
|
|
4798
4845
|
</constraints>
|
|
4799
4846
|
|
|
4800
4847
|
<tone_guidance>
|
|
4801
|
-
|
|
4802
|
-
Adapt your entire analysis and writing style to match the specified tone - this should influence
|
|
4803
|
-
your word choice, personality, formality level, and overall presentation of the content.
|
|
4804
|
-
The tone instructions are not suggestions but core requirements for how you should express yourself.
|
|
4848
|
+
${TONE_GUIDANCE}
|
|
4805
4849
|
</tone_guidance>
|
|
4806
4850
|
|
|
4807
4851
|
<language_guidelines>
|
|
4808
|
-
|
|
4809
|
-
- "The image shows..." / "The storyboard shows..."
|
|
4810
|
-
- "In this video..." / "This video features..."
|
|
4811
|
-
- "The frames depict..." / "The footage shows..."
|
|
4812
|
-
- "We can see..." / "You can see..."
|
|
4813
|
-
- "The clip shows..." / "The scene shows..."
|
|
4814
|
-
|
|
4815
|
-
INSTEAD, describe the content directly:
|
|
4816
|
-
- BAD: "The video shows a chef preparing a meal"
|
|
4817
|
-
- GOOD: "A chef prepares a meal in a professional kitchen"
|
|
4818
|
-
|
|
4819
|
-
Write as if describing reality, not describing a recording of reality.
|
|
4852
|
+
${createLanguageGuidelines("video")}
|
|
4820
4853
|
</language_guidelines>`;
|
|
4821
|
-
var AUDIO_ONLY_SYSTEM_PROMPT3 =
|
|
4854
|
+
var AUDIO_ONLY_SYSTEM_PROMPT3 = promptDedent`
|
|
4822
4855
|
<role>
|
|
4823
4856
|
You are an audio content analyst specializing in transcript analysis and metadata generation.
|
|
4824
4857
|
</role>
|
|
@@ -4845,35 +4878,20 @@ var AUDIO_ONLY_SYSTEM_PROMPT3 = dedent6`
|
|
|
4845
4878
|
|
|
4846
4879
|
<constraints>
|
|
4847
4880
|
- Only describe what is explicitly stated or strongly implied in the transcript
|
|
4848
|
-
-
|
|
4849
|
-
-
|
|
4850
|
-
|
|
4851
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
4852
|
-
- Return structured data matching the requested schema
|
|
4881
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
4882
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
4883
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
4853
4884
|
- Focus entirely on audio/spoken content - there are no visual elements
|
|
4854
4885
|
- Output only the JSON object; no markdown or extra text
|
|
4855
4886
|
- When a <language> section is provided, all output text MUST be written in that language
|
|
4856
4887
|
</constraints>
|
|
4857
4888
|
|
|
4858
4889
|
<tone_guidance>
|
|
4859
|
-
|
|
4860
|
-
Adapt your entire analysis and writing style to match the specified tone - this should influence
|
|
4861
|
-
your word choice, personality, formality level, and overall presentation of the content.
|
|
4862
|
-
The tone instructions are not suggestions but core requirements for how you should express yourself.
|
|
4890
|
+
${TONE_GUIDANCE}
|
|
4863
4891
|
</tone_guidance>
|
|
4864
4892
|
|
|
4865
4893
|
<language_guidelines>
|
|
4866
|
-
|
|
4867
|
-
- "The audio shows..." / "The transcript shows..."
|
|
4868
|
-
- "In this recording..." / "This audio features..."
|
|
4869
|
-
- "The speaker says..." / "We can hear..."
|
|
4870
|
-
- "The clip contains..." / "The recording shows..."
|
|
4871
|
-
|
|
4872
|
-
INSTEAD, describe the content directly:
|
|
4873
|
-
- BAD: "The audio features a discussion about climate change"
|
|
4874
|
-
- GOOD: "A panel discusses climate change impacts and solutions"
|
|
4875
|
-
|
|
4876
|
-
Write as if describing reality, not describing a recording of reality.
|
|
4894
|
+
${createLanguageGuidelines("audio")}
|
|
4877
4895
|
</language_guidelines>`;
|
|
4878
4896
|
function buildUserPrompt5({
|
|
4879
4897
|
tone,
|
|
@@ -5047,7 +5065,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
5047
5065
|
required: isAudioOnly
|
|
5048
5066
|
}) : void 0;
|
|
5049
5067
|
const transcriptText = transcriptResult?.transcriptText ?? "";
|
|
5050
|
-
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : transcriptResult?.track
|
|
5068
|
+
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : getReliableLanguageCode(transcriptResult?.track) ?? getReliableLanguageCode(getReadyTextTracks(assetData)[0]);
|
|
5051
5069
|
const languageName = resolvedLanguageCode ? getLanguageName(resolvedLanguageCode) : void 0;
|
|
5052
5070
|
const userPrompt = buildUserPrompt5({
|
|
5053
5071
|
tone,
|
|
@@ -5539,18 +5557,18 @@ import {
|
|
|
5539
5557
|
RetryError,
|
|
5540
5558
|
TypeValidationError
|
|
5541
5559
|
} from "ai";
|
|
5542
|
-
import
|
|
5560
|
+
import dedent8 from "dedent";
|
|
5543
5561
|
import { z as z8 } from "zod";
|
|
5544
5562
|
var translationSchema = z8.object({
|
|
5545
5563
|
translation: z8.string()
|
|
5546
5564
|
});
|
|
5547
|
-
var SYSTEM_PROMPT6 =
|
|
5565
|
+
var SYSTEM_PROMPT6 = dedent8`
|
|
5548
5566
|
You are a subtitle translation expert. Translate VTT subtitle files to the target language specified by the user.
|
|
5549
5567
|
You may receive either a full VTT file or a chunk from a larger VTT.
|
|
5550
5568
|
Preserve all timestamps, cue ordering, and VTT formatting exactly as they appear.
|
|
5551
5569
|
Return JSON with a single key "translation" containing the translated VTT content.
|
|
5552
5570
|
`;
|
|
5553
|
-
var CUE_TRANSLATION_SYSTEM_PROMPT =
|
|
5571
|
+
var CUE_TRANSLATION_SYSTEM_PROMPT = dedent8`
|
|
5554
5572
|
You are a subtitle translation expert.
|
|
5555
5573
|
You will receive a sequence of subtitle cues extracted from a VTT file.
|
|
5556
5574
|
Translate the cues to the requested target language while preserving their original order.
|
|
@@ -6061,7 +6079,7 @@ async function translateCaptions(assetId, trackId, toLanguageCode, options) {
|
|
|
6061
6079
|
}
|
|
6062
6080
|
if (uploadToMux) {
|
|
6063
6081
|
try {
|
|
6064
|
-
const languageName = getLanguageName(toLanguageCode);
|
|
6082
|
+
const languageName = getLanguageName(toLanguageCode) ?? toLanguageCode.toUpperCase();
|
|
6065
6083
|
const trackName = `${languageName} (auto-translated)`;
|
|
6066
6084
|
uploadedTrackId = await createTextTrackOnMux(
|
|
6067
6085
|
assetId,
|