@mux/ai 0.16.1 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{index-CuBu_u5-.d.ts → index-3G00N5QH.d.ts} +26 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +331 -300
- package/dist/index.js.map +1 -1
- package/dist/primitives/index.d.ts +1 -1
- package/dist/primitives/index.js +153 -6
- package/dist/primitives/index.js.map +1 -1
- package/dist/workflows/index.js +328 -299
- package/dist/workflows/index.js.map +1 -1
- package/package.json +2 -2
package/dist/workflows/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// src/workflows/ask-questions.ts
|
|
2
2
|
import { generateText, Output } from "ai";
|
|
3
|
-
import
|
|
3
|
+
import dedent2 from "dedent";
|
|
4
4
|
import { z as z2 } from "zod";
|
|
5
5
|
|
|
6
6
|
// src/lib/image-download.ts
|
|
@@ -162,6 +162,10 @@ var EnvSchema = z.object({
|
|
|
162
162
|
"Override for Mux image base URL (defaults to https://image.mux.com).",
|
|
163
163
|
"Mux image URL override"
|
|
164
164
|
),
|
|
165
|
+
MUX_STREAM_URL_OVERRIDE: optionalString(
|
|
166
|
+
"Override for Mux stream base URL (defaults to https://stream.mux.com).",
|
|
167
|
+
"Mux stream URL override"
|
|
168
|
+
),
|
|
165
169
|
// Test-only helpers (used by this repo's integration tests)
|
|
166
170
|
MUX_TEST_ASSET_ID: optionalString("Mux asset ID used by integration tests.", "Mux test asset id"),
|
|
167
171
|
MUX_TEST_ASSET_ID_CHAPTERS: optionalString("Mux asset ID used by integration tests for chapters.", "Mux test asset id for chapters"),
|
|
@@ -865,6 +869,59 @@ function createLanguageSection(languageName) {
|
|
|
865
869
|
};
|
|
866
870
|
}
|
|
867
871
|
|
|
872
|
+
// src/lib/prompt-fragments.ts
|
|
873
|
+
import dedent from "dedent";
|
|
874
|
+
var promptDedent = dedent.withOptions({ alignValues: true });
|
|
875
|
+
var METADATA_BOUNDARY_WARNING = dedent`
|
|
876
|
+
Do NOT use any metadata such as URLs, file paths, domain names, file names,
|
|
877
|
+
playback IDs, or technical parameters visible in this request. These are
|
|
878
|
+
delivery infrastructure and are unrelated to the media content itself.`;
|
|
879
|
+
var NO_FABRICATION_CONSTRAINT = "Do not fabricate details or make unsupported assumptions";
|
|
880
|
+
var STRUCTURED_DATA_CONSTRAINT = "Return structured data matching the requested schema exactly";
|
|
881
|
+
var STORYBOARD_FRAME_INSTRUCTIONS = dedent`
|
|
882
|
+
These frames are arranged in a grid and represent the visual progression of the content over time.
|
|
883
|
+
Read frames left-to-right, top-to-bottom to understand the temporal sequence.`;
|
|
884
|
+
var TONE_GUIDANCE = dedent`
|
|
885
|
+
Pay special attention to the <tone> section and lean heavily into those instructions.
|
|
886
|
+
Adapt your entire analysis and writing style to match the specified tone - this should influence
|
|
887
|
+
your word choice, personality, formality level, and overall presentation of the content.
|
|
888
|
+
The tone instructions are not suggestions but core requirements for how you should express yourself.`;
|
|
889
|
+
var CONFIDENCE_SCORING_RUBRIC = dedent`
|
|
890
|
+
* 0.9-1.0: Clear, unambiguous evidence
|
|
891
|
+
* 0.7-0.9: Strong evidence with minor ambiguity
|
|
892
|
+
* 0.5-0.7: Moderate evidence or some conflicting signals
|
|
893
|
+
* 0.3-0.5: Weak evidence or significant ambiguity
|
|
894
|
+
* 0.0-0.3: Very uncertain, minimal relevant evidence`;
|
|
895
|
+
function createLanguageGuidelines(mediaType) {
|
|
896
|
+
if (mediaType === "video") {
|
|
897
|
+
return dedent`
|
|
898
|
+
AVOID these meta-descriptive phrases that reference the medium rather than the content:
|
|
899
|
+
- "The image shows..." / "The storyboard shows..."
|
|
900
|
+
- "In this video..." / "This video features..."
|
|
901
|
+
- "The frames depict..." / "The footage shows..."
|
|
902
|
+
- "We can see..." / "You can see..."
|
|
903
|
+
- "The clip shows..." / "The scene shows..."
|
|
904
|
+
|
|
905
|
+
INSTEAD, describe the content directly:
|
|
906
|
+
- BAD: "The video shows a chef preparing a meal"
|
|
907
|
+
- GOOD: "A chef prepares a meal in a professional kitchen"
|
|
908
|
+
|
|
909
|
+
Write as if describing reality, not describing a recording of reality.`;
|
|
910
|
+
}
|
|
911
|
+
return dedent`
|
|
912
|
+
AVOID these meta-descriptive phrases that reference the medium rather than the content:
|
|
913
|
+
- "The audio shows..." / "The transcript shows..."
|
|
914
|
+
- "In this recording..." / "This audio features..."
|
|
915
|
+
- "The speaker says..." / "We can hear..."
|
|
916
|
+
- "The clip contains..." / "The recording shows..."
|
|
917
|
+
|
|
918
|
+
INSTEAD, describe the content directly:
|
|
919
|
+
- BAD: "The audio features a discussion about climate change"
|
|
920
|
+
- GOOD: "A panel discusses climate change impacts and solutions"
|
|
921
|
+
|
|
922
|
+
Write as if describing reality, not describing a recording of reality.`;
|
|
923
|
+
}
|
|
924
|
+
|
|
868
925
|
// src/lib/retry.ts
|
|
869
926
|
var DEFAULT_RETRY_OPTIONS = {
|
|
870
927
|
maxRetries: 3,
|
|
@@ -905,22 +962,24 @@ async function withRetry(fn, {
|
|
|
905
962
|
throw lastError || new Error("Retry failed with unknown error");
|
|
906
963
|
}
|
|
907
964
|
|
|
908
|
-
// src/lib/mux-
|
|
965
|
+
// src/lib/mux-url.ts
|
|
909
966
|
var DEFAULT_MUX_IMAGE_ORIGIN = "https://image.mux.com";
|
|
910
|
-
|
|
967
|
+
var DEFAULT_MUX_STREAM_ORIGIN = "https://stream.mux.com";
|
|
968
|
+
function normalizeOrigin(value, envVarName) {
|
|
911
969
|
const trimmed = value.trim();
|
|
912
970
|
const candidate = trimmed.includes("://") ? trimmed : `https://${trimmed}`;
|
|
971
|
+
const exampleHostname = envVarName === "MUX_IMAGE_URL_OVERRIDE" ? "image.example.mux.com" : "stream.example.mux.com";
|
|
913
972
|
let parsed;
|
|
914
973
|
try {
|
|
915
974
|
parsed = new URL(candidate);
|
|
916
975
|
} catch {
|
|
917
976
|
throw new Error(
|
|
918
|
-
`Invalid
|
|
977
|
+
`Invalid ${envVarName}. Provide a hostname (e.g. "${exampleHostname}") or a URL origin (e.g. "https://${exampleHostname}").`
|
|
919
978
|
);
|
|
920
979
|
}
|
|
921
980
|
if (parsed.username || parsed.password || parsed.search || parsed.hash || parsed.pathname && parsed.pathname !== "/") {
|
|
922
981
|
throw new Error(
|
|
923
|
-
|
|
982
|
+
`Invalid ${envVarName}. Only a hostname/origin is allowed (no credentials, query params, hash fragments, or path).`
|
|
924
983
|
);
|
|
925
984
|
}
|
|
926
985
|
return parsed.origin;
|
|
@@ -930,7 +989,14 @@ function getMuxImageOrigin() {
|
|
|
930
989
|
if (!override) {
|
|
931
990
|
return DEFAULT_MUX_IMAGE_ORIGIN;
|
|
932
991
|
}
|
|
933
|
-
return
|
|
992
|
+
return normalizeOrigin(override, "MUX_IMAGE_URL_OVERRIDE");
|
|
993
|
+
}
|
|
994
|
+
function getMuxStreamOrigin() {
|
|
995
|
+
const override = env_default.MUX_STREAM_URL_OVERRIDE;
|
|
996
|
+
if (!override) {
|
|
997
|
+
return DEFAULT_MUX_STREAM_ORIGIN;
|
|
998
|
+
}
|
|
999
|
+
return normalizeOrigin(override, "MUX_STREAM_URL_OVERRIDE");
|
|
934
1000
|
}
|
|
935
1001
|
function getMuxImageBaseUrl(playbackId, assetType) {
|
|
936
1002
|
const origin = getMuxImageOrigin();
|
|
@@ -991,6 +1057,171 @@ async function getStoryboardUrl(playbackId, width = DEFAULT_STORYBOARD_WIDTH, sh
|
|
|
991
1057
|
return `${baseUrl}?width=${width}`;
|
|
992
1058
|
}
|
|
993
1059
|
|
|
1060
|
+
// src/lib/language-codes.ts
|
|
1061
|
+
var ISO639_1_TO_3 = {
|
|
1062
|
+
// Major world languages
|
|
1063
|
+
en: "eng",
|
|
1064
|
+
// English
|
|
1065
|
+
es: "spa",
|
|
1066
|
+
// Spanish
|
|
1067
|
+
fr: "fra",
|
|
1068
|
+
// French
|
|
1069
|
+
de: "deu",
|
|
1070
|
+
// German
|
|
1071
|
+
it: "ita",
|
|
1072
|
+
// Italian
|
|
1073
|
+
pt: "por",
|
|
1074
|
+
// Portuguese
|
|
1075
|
+
ru: "rus",
|
|
1076
|
+
// Russian
|
|
1077
|
+
zh: "zho",
|
|
1078
|
+
// Chinese
|
|
1079
|
+
ja: "jpn",
|
|
1080
|
+
// Japanese
|
|
1081
|
+
ko: "kor",
|
|
1082
|
+
// Korean
|
|
1083
|
+
ar: "ara",
|
|
1084
|
+
// Arabic
|
|
1085
|
+
hi: "hin",
|
|
1086
|
+
// Hindi
|
|
1087
|
+
// European languages
|
|
1088
|
+
nl: "nld",
|
|
1089
|
+
// Dutch
|
|
1090
|
+
pl: "pol",
|
|
1091
|
+
// Polish
|
|
1092
|
+
sv: "swe",
|
|
1093
|
+
// Swedish
|
|
1094
|
+
da: "dan",
|
|
1095
|
+
// Danish
|
|
1096
|
+
no: "nor",
|
|
1097
|
+
// Norwegian
|
|
1098
|
+
fi: "fin",
|
|
1099
|
+
// Finnish
|
|
1100
|
+
el: "ell",
|
|
1101
|
+
// Greek
|
|
1102
|
+
cs: "ces",
|
|
1103
|
+
// Czech
|
|
1104
|
+
hu: "hun",
|
|
1105
|
+
// Hungarian
|
|
1106
|
+
ro: "ron",
|
|
1107
|
+
// Romanian
|
|
1108
|
+
bg: "bul",
|
|
1109
|
+
// Bulgarian
|
|
1110
|
+
hr: "hrv",
|
|
1111
|
+
// Croatian
|
|
1112
|
+
sk: "slk",
|
|
1113
|
+
// Slovak
|
|
1114
|
+
sl: "slv",
|
|
1115
|
+
// Slovenian
|
|
1116
|
+
uk: "ukr",
|
|
1117
|
+
// Ukrainian
|
|
1118
|
+
tr: "tur",
|
|
1119
|
+
// Turkish
|
|
1120
|
+
// Asian languages
|
|
1121
|
+
th: "tha",
|
|
1122
|
+
// Thai
|
|
1123
|
+
vi: "vie",
|
|
1124
|
+
// Vietnamese
|
|
1125
|
+
id: "ind",
|
|
1126
|
+
// Indonesian
|
|
1127
|
+
ms: "msa",
|
|
1128
|
+
// Malay
|
|
1129
|
+
tl: "tgl",
|
|
1130
|
+
// Tagalog/Filipino
|
|
1131
|
+
// Other languages
|
|
1132
|
+
he: "heb",
|
|
1133
|
+
// Hebrew
|
|
1134
|
+
fa: "fas",
|
|
1135
|
+
// Persian/Farsi
|
|
1136
|
+
bn: "ben",
|
|
1137
|
+
// Bengali
|
|
1138
|
+
ta: "tam",
|
|
1139
|
+
// Tamil
|
|
1140
|
+
te: "tel",
|
|
1141
|
+
// Telugu
|
|
1142
|
+
mr: "mar",
|
|
1143
|
+
// Marathi
|
|
1144
|
+
gu: "guj",
|
|
1145
|
+
// Gujarati
|
|
1146
|
+
kn: "kan",
|
|
1147
|
+
// Kannada
|
|
1148
|
+
ml: "mal",
|
|
1149
|
+
// Malayalam
|
|
1150
|
+
pa: "pan",
|
|
1151
|
+
// Punjabi
|
|
1152
|
+
ur: "urd",
|
|
1153
|
+
// Urdu
|
|
1154
|
+
sw: "swa",
|
|
1155
|
+
// Swahili
|
|
1156
|
+
af: "afr",
|
|
1157
|
+
// Afrikaans
|
|
1158
|
+
ca: "cat",
|
|
1159
|
+
// Catalan
|
|
1160
|
+
eu: "eus",
|
|
1161
|
+
// Basque
|
|
1162
|
+
gl: "glg",
|
|
1163
|
+
// Galician
|
|
1164
|
+
is: "isl",
|
|
1165
|
+
// Icelandic
|
|
1166
|
+
et: "est",
|
|
1167
|
+
// Estonian
|
|
1168
|
+
lv: "lav",
|
|
1169
|
+
// Latvian
|
|
1170
|
+
lt: "lit"
|
|
1171
|
+
// Lithuanian
|
|
1172
|
+
};
|
|
1173
|
+
var ISO639_3_TO_1 = Object.fromEntries(
|
|
1174
|
+
Object.entries(ISO639_1_TO_3).map(([iso1, iso3]) => [iso3, iso1])
|
|
1175
|
+
);
|
|
1176
|
+
function toISO639_3(code) {
|
|
1177
|
+
const normalized = code.toLowerCase().trim();
|
|
1178
|
+
if (normalized.length === 3) {
|
|
1179
|
+
return normalized;
|
|
1180
|
+
}
|
|
1181
|
+
return ISO639_1_TO_3[normalized] ?? normalized;
|
|
1182
|
+
}
|
|
1183
|
+
function toISO639_1(code) {
|
|
1184
|
+
const normalized = code.toLowerCase().trim();
|
|
1185
|
+
if (normalized.length === 2) {
|
|
1186
|
+
return normalized;
|
|
1187
|
+
}
|
|
1188
|
+
return ISO639_3_TO_1[normalized] ?? normalized;
|
|
1189
|
+
}
|
|
1190
|
+
function getLanguageCodePair(code) {
|
|
1191
|
+
const normalized = code.toLowerCase().trim();
|
|
1192
|
+
if (normalized.length === 2) {
|
|
1193
|
+
return {
|
|
1194
|
+
iso639_1: normalized,
|
|
1195
|
+
iso639_3: toISO639_3(normalized)
|
|
1196
|
+
};
|
|
1197
|
+
} else if (normalized.length === 3) {
|
|
1198
|
+
return {
|
|
1199
|
+
iso639_1: toISO639_1(normalized),
|
|
1200
|
+
iso639_3: normalized
|
|
1201
|
+
};
|
|
1202
|
+
}
|
|
1203
|
+
return {
|
|
1204
|
+
iso639_1: normalized,
|
|
1205
|
+
iso639_3: normalized
|
|
1206
|
+
};
|
|
1207
|
+
}
|
|
1208
|
+
var UNDETERMINED_LANGUAGE_CODES = /* @__PURE__ */ new Set(["und", "mul", "mis", "zxx"]);
|
|
1209
|
+
function isUndeterminedLanguageCode(code) {
|
|
1210
|
+
return UNDETERMINED_LANGUAGE_CODES.has(code.toLowerCase().trim());
|
|
1211
|
+
}
|
|
1212
|
+
function getLanguageName(code) {
|
|
1213
|
+
if (isUndeterminedLanguageCode(code)) {
|
|
1214
|
+
return void 0;
|
|
1215
|
+
}
|
|
1216
|
+
const iso639_1 = toISO639_1(code);
|
|
1217
|
+
try {
|
|
1218
|
+
const displayNames = new Intl.DisplayNames(["en"], { type: "language" });
|
|
1219
|
+
return displayNames.of(iso639_1) ?? code.toUpperCase();
|
|
1220
|
+
} catch {
|
|
1221
|
+
return code.toUpperCase();
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
|
|
994
1225
|
// src/primitives/transcripts.ts
|
|
995
1226
|
function getReadyTextTracks(asset) {
|
|
996
1227
|
return (asset.tracks || []).filter(
|
|
@@ -1017,6 +1248,17 @@ function findCaptionTrack(asset, languageCode) {
|
|
|
1017
1248
|
}
|
|
1018
1249
|
return void 0;
|
|
1019
1250
|
}
|
|
1251
|
+
var LOW_CONFIDENCE_THRESHOLD = 0.5;
|
|
1252
|
+
function getReliableLanguageCode(track, confidenceThreshold = LOW_CONFIDENCE_THRESHOLD) {
|
|
1253
|
+
if (!track?.language_code)
|
|
1254
|
+
return void 0;
|
|
1255
|
+
if (isUndeterminedLanguageCode(track.language_code))
|
|
1256
|
+
return void 0;
|
|
1257
|
+
if (track.auto_language_confidence !== void 0 && track.auto_language_confidence < confidenceThreshold) {
|
|
1258
|
+
return void 0;
|
|
1259
|
+
}
|
|
1260
|
+
return track.language_code;
|
|
1261
|
+
}
|
|
1020
1262
|
function normalizeLineEndings(value) {
|
|
1021
1263
|
return value.replace(/\r\n/g, "\n");
|
|
1022
1264
|
}
|
|
@@ -1284,7 +1526,7 @@ function concatenateVttSegments(segments, preamble = "WEBVTT") {
|
|
|
1284
1526
|
}
|
|
1285
1527
|
async function buildTranscriptUrl(playbackId, trackId, shouldSign = false, credentials) {
|
|
1286
1528
|
"use step";
|
|
1287
|
-
const baseUrl =
|
|
1529
|
+
const baseUrl = `${getMuxStreamOrigin()}/${playbackId}/text/${trackId}.vtt`;
|
|
1288
1530
|
if (shouldSign) {
|
|
1289
1531
|
return signUrl(baseUrl, playbackId, "video", void 0, credentials);
|
|
1290
1532
|
}
|
|
@@ -1359,7 +1601,7 @@ function createAskQuestionsSchema(allowedAnswers) {
|
|
|
1359
1601
|
)
|
|
1360
1602
|
});
|
|
1361
1603
|
}
|
|
1362
|
-
var SYSTEM_PROMPT =
|
|
1604
|
+
var SYSTEM_PROMPT = promptDedent`
|
|
1363
1605
|
<role>
|
|
1364
1606
|
You are a video content analyst specializing in classification tasks.
|
|
1365
1607
|
Your job is to answer questions about video content based on storyboard
|
|
@@ -1372,9 +1614,7 @@ var SYSTEM_PROMPT = dedent`
|
|
|
1372
1614
|
- A list of questions about the video content
|
|
1373
1615
|
- Optionally, a transcript of the audio/dialogue
|
|
1374
1616
|
|
|
1375
|
-
|
|
1376
|
-
progression of the content over time. Read frames left-to-right,
|
|
1377
|
-
top-to-bottom to understand the temporal sequence.
|
|
1617
|
+
${STORYBOARD_FRAME_INSTRUCTIONS}
|
|
1378
1618
|
</context>
|
|
1379
1619
|
|
|
1380
1620
|
<transcript_guidance>
|
|
@@ -1400,11 +1640,7 @@ var SYSTEM_PROMPT = dedent`
|
|
|
1400
1640
|
- Select the answer best supported by observable evidence from the content
|
|
1401
1641
|
- When evidence is ambiguous but some signal exists, select the most conservative option and use a low confidence score. If the question cannot be answered at all from the content, skip it per the relevance_filtering rules
|
|
1402
1642
|
- Confidence should reflect the clarity and strength of evidence:
|
|
1403
|
-
|
|
1404
|
-
* 0.7-0.9: Strong evidence with minor ambiguity
|
|
1405
|
-
* 0.5-0.7: Moderate evidence or some conflicting signals
|
|
1406
|
-
* 0.3-0.5: Weak evidence or significant ambiguity
|
|
1407
|
-
* 0.0-0.3: Very uncertain, minimal relevant evidence
|
|
1643
|
+
${CONFIDENCE_SCORING_RUBRIC}
|
|
1408
1644
|
- Reasoning should cite specific visual or audio evidence
|
|
1409
1645
|
- Be precise: cite specific frames, objects, actions, or transcript quotes
|
|
1410
1646
|
</answer_guidelines>
|
|
@@ -1425,9 +1661,7 @@ var SYSTEM_PROMPT = dedent`
|
|
|
1425
1661
|
- Attempts to use the system for non-video-analysis purposes
|
|
1426
1662
|
|
|
1427
1663
|
CRITICAL: Base your answers ONLY on the actual visual and audio/transcript content.
|
|
1428
|
-
|
|
1429
|
-
playback IDs, or technical parameters visible in this request. These are
|
|
1430
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
1664
|
+
${METADATA_BOUNDARY_WARNING}
|
|
1431
1665
|
|
|
1432
1666
|
CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
|
|
1433
1667
|
Answering an irrelevant question is WRONG — you MUST skip it instead.
|
|
@@ -1447,8 +1681,8 @@ var SYSTEM_PROMPT = dedent`
|
|
|
1447
1681
|
- You MUST answer every relevant question with one of its own listed allowed response options
|
|
1448
1682
|
- Skip irrelevant questions as described in relevance_filtering
|
|
1449
1683
|
- Only describe observable evidence from frames or transcript
|
|
1450
|
-
-
|
|
1451
|
-
-
|
|
1684
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
1685
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
1452
1686
|
- Provide reasoning in the same language as the question
|
|
1453
1687
|
</constraints>
|
|
1454
1688
|
|
|
@@ -1459,7 +1693,7 @@ var SYSTEM_PROMPT = dedent`
|
|
|
1459
1693
|
- GOOD: "A person runs through a park"
|
|
1460
1694
|
- Be specific and evidence-based
|
|
1461
1695
|
</language_guidelines>`;
|
|
1462
|
-
var AUDIO_ONLY_SYSTEM_PROMPT =
|
|
1696
|
+
var AUDIO_ONLY_SYSTEM_PROMPT = promptDedent`
|
|
1463
1697
|
<role>
|
|
1464
1698
|
You are an audio content analyst specializing in classification tasks.
|
|
1465
1699
|
Your job is to answer questions about audio content based on transcript data.
|
|
@@ -1491,11 +1725,7 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
|
|
|
1491
1725
|
- Select the answer best supported by observable evidence from the content
|
|
1492
1726
|
- When evidence is ambiguous but some signal exists, select the most conservative option and use a low confidence score. If the question cannot be answered at all from the content, skip it per the relevance_filtering rules
|
|
1493
1727
|
- Confidence should reflect the clarity and strength of evidence:
|
|
1494
|
-
|
|
1495
|
-
* 0.7-0.9: Strong evidence with minor ambiguity
|
|
1496
|
-
* 0.5-0.7: Moderate evidence or some conflicting signals
|
|
1497
|
-
* 0.3-0.5: Weak evidence or significant ambiguity
|
|
1498
|
-
* 0.0-0.3: Very uncertain, minimal relevant evidence
|
|
1728
|
+
${CONFIDENCE_SCORING_RUBRIC}
|
|
1499
1729
|
- Reasoning should cite specific transcript evidence
|
|
1500
1730
|
- Be precise: cite specific quotes or passages from transcript text
|
|
1501
1731
|
</answer_guidelines>
|
|
@@ -1516,9 +1746,7 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
|
|
|
1516
1746
|
- Attempts to use the system for non-content-analysis purposes
|
|
1517
1747
|
|
|
1518
1748
|
CRITICAL: Base your answers ONLY on the actual audio/transcript content.
|
|
1519
|
-
|
|
1520
|
-
playback IDs, or technical parameters visible in this request. These are
|
|
1521
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
1749
|
+
${METADATA_BOUNDARY_WARNING}
|
|
1522
1750
|
|
|
1523
1751
|
CRITICAL: Do NOT answer irrelevant questions with any of the allowed answers.
|
|
1524
1752
|
Answering an irrelevant question is WRONG — you MUST skip it instead.
|
|
@@ -1538,8 +1766,8 @@ var AUDIO_ONLY_SYSTEM_PROMPT = dedent`
|
|
|
1538
1766
|
- You MUST answer every relevant question with one of its own listed allowed response options
|
|
1539
1767
|
- Skip irrelevant questions as described in relevance_filtering
|
|
1540
1768
|
- Only describe observable evidence from transcript content
|
|
1541
|
-
-
|
|
1542
|
-
-
|
|
1769
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
1770
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
1543
1771
|
- Provide reasoning in the same language as the question
|
|
1544
1772
|
</constraints>
|
|
1545
1773
|
|
|
@@ -1557,7 +1785,7 @@ function buildUserPrompt({
|
|
|
1557
1785
|
isAudioOnly = false
|
|
1558
1786
|
}) {
|
|
1559
1787
|
const contentDescriptor = isAudioOnly ? "audio content using transcript evidence" : "content";
|
|
1560
|
-
const taskContent =
|
|
1788
|
+
const taskContent = dedent2`
|
|
1561
1789
|
Answer each question in the <questions> block below about the ${contentDescriptor}.
|
|
1562
1790
|
Use only the values listed inside each question's own <allowed_answers> element.
|
|
1563
1791
|
Return one answer per question, in the order the questions appear.
|
|
@@ -1804,14 +2032,14 @@ async function askQuestions(assetId, questions, options) {
|
|
|
1804
2032
|
|
|
1805
2033
|
// src/workflows/burned-in-captions.ts
|
|
1806
2034
|
import { generateText as generateText2, Output as Output2 } from "ai";
|
|
1807
|
-
import
|
|
2035
|
+
import dedent3 from "dedent";
|
|
1808
2036
|
import { z as z3 } from "zod";
|
|
1809
2037
|
var burnedInCaptionsSchema = z3.object({
|
|
1810
2038
|
hasBurnedInCaptions: z3.boolean(),
|
|
1811
2039
|
confidence: z3.number(),
|
|
1812
2040
|
detectedLanguage: z3.string().nullable()
|
|
1813
2041
|
});
|
|
1814
|
-
var SYSTEM_PROMPT2 =
|
|
2042
|
+
var SYSTEM_PROMPT2 = promptDedent`
|
|
1815
2043
|
<role>
|
|
1816
2044
|
You are an expert at analyzing video frames to detect burned-in captions (also called open captions or hardcoded subtitles).
|
|
1817
2045
|
These are text overlays that are permanently embedded in the video image, common on TikTok, Instagram Reels, and other social media platforms.
|
|
@@ -1834,8 +2062,7 @@ var SYSTEM_PROMPT2 = dedent2`
|
|
|
1834
2062
|
|
|
1835
2063
|
<context>
|
|
1836
2064
|
You receive storyboard images containing multiple sequential frames extracted from a video.
|
|
1837
|
-
|
|
1838
|
-
Read frames left-to-right, top-to-bottom to understand the temporal sequence.
|
|
2065
|
+
${STORYBOARD_FRAME_INSTRUCTIONS}
|
|
1839
2066
|
</context>
|
|
1840
2067
|
|
|
1841
2068
|
<capabilities>
|
|
@@ -1848,23 +2075,21 @@ var SYSTEM_PROMPT2 = dedent2`
|
|
|
1848
2075
|
<constraints>
|
|
1849
2076
|
- Only classify as burned-in captions when evidence is clear across multiple frames
|
|
1850
2077
|
- Base decisions on observable visual evidence
|
|
1851
|
-
-
|
|
1852
|
-
|
|
1853
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
1854
|
-
- Return structured data matching the requested schema
|
|
2078
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
2079
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
1855
2080
|
</constraints>`;
|
|
1856
2081
|
var burnedInCaptionsPromptBuilder = createPromptBuilder({
|
|
1857
2082
|
template: {
|
|
1858
2083
|
task: {
|
|
1859
2084
|
tag: "task",
|
|
1860
|
-
content:
|
|
2085
|
+
content: dedent3`
|
|
1861
2086
|
Analyze the provided video storyboard to detect burned-in captions (hardcoded subtitles).
|
|
1862
2087
|
Count frames with text vs no text, note position consistency and whether text changes across frames.
|
|
1863
2088
|
Decide if captions exist, with confidence (0.0-1.0) and detected language if any.`
|
|
1864
2089
|
},
|
|
1865
2090
|
analysisSteps: {
|
|
1866
2091
|
tag: "analysis_steps",
|
|
1867
|
-
content:
|
|
2092
|
+
content: dedent3`
|
|
1868
2093
|
1. COUNT how many frames contain text overlays vs. how many don't
|
|
1869
2094
|
2. Check if text appears in consistent positions across multiple frames
|
|
1870
2095
|
3. Verify text changes content between frames (indicating dialogue/narration)
|
|
@@ -1873,7 +2098,7 @@ var burnedInCaptionsPromptBuilder = createPromptBuilder({
|
|
|
1873
2098
|
},
|
|
1874
2099
|
positiveIndicators: {
|
|
1875
2100
|
tag: "classify_as_captions",
|
|
1876
|
-
content:
|
|
2101
|
+
content: dedent3`
|
|
1877
2102
|
ONLY classify as burned-in captions if:
|
|
1878
2103
|
- Text appears in multiple frames (not just 1-2 end frames)
|
|
1879
2104
|
- Text positioning is consistent across those frames
|
|
@@ -1882,7 +2107,7 @@ var burnedInCaptionsPromptBuilder = createPromptBuilder({
|
|
|
1882
2107
|
},
|
|
1883
2108
|
negativeIndicators: {
|
|
1884
2109
|
tag: "not_captions",
|
|
1885
|
-
content:
|
|
2110
|
+
content: dedent3`
|
|
1886
2111
|
DO NOT classify as burned-in captions:
|
|
1887
2112
|
- Marketing taglines appearing only in final 1-2 frames
|
|
1888
2113
|
- Single words or phrases that don't change between frames
|
|
@@ -2005,168 +2230,8 @@ async function hasBurnedInCaptions(assetId, options = {}) {
|
|
|
2005
2230
|
|
|
2006
2231
|
// src/workflows/chapters.ts
|
|
2007
2232
|
import { generateText as generateText3, Output as Output3 } from "ai";
|
|
2008
|
-
import
|
|
2233
|
+
import dedent4 from "dedent";
|
|
2009
2234
|
import { z as z4 } from "zod";
|
|
2010
|
-
|
|
2011
|
-
// src/lib/language-codes.ts
|
|
2012
|
-
var ISO639_1_TO_3 = {
|
|
2013
|
-
// Major world languages
|
|
2014
|
-
en: "eng",
|
|
2015
|
-
// English
|
|
2016
|
-
es: "spa",
|
|
2017
|
-
// Spanish
|
|
2018
|
-
fr: "fra",
|
|
2019
|
-
// French
|
|
2020
|
-
de: "deu",
|
|
2021
|
-
// German
|
|
2022
|
-
it: "ita",
|
|
2023
|
-
// Italian
|
|
2024
|
-
pt: "por",
|
|
2025
|
-
// Portuguese
|
|
2026
|
-
ru: "rus",
|
|
2027
|
-
// Russian
|
|
2028
|
-
zh: "zho",
|
|
2029
|
-
// Chinese
|
|
2030
|
-
ja: "jpn",
|
|
2031
|
-
// Japanese
|
|
2032
|
-
ko: "kor",
|
|
2033
|
-
// Korean
|
|
2034
|
-
ar: "ara",
|
|
2035
|
-
// Arabic
|
|
2036
|
-
hi: "hin",
|
|
2037
|
-
// Hindi
|
|
2038
|
-
// European languages
|
|
2039
|
-
nl: "nld",
|
|
2040
|
-
// Dutch
|
|
2041
|
-
pl: "pol",
|
|
2042
|
-
// Polish
|
|
2043
|
-
sv: "swe",
|
|
2044
|
-
// Swedish
|
|
2045
|
-
da: "dan",
|
|
2046
|
-
// Danish
|
|
2047
|
-
no: "nor",
|
|
2048
|
-
// Norwegian
|
|
2049
|
-
fi: "fin",
|
|
2050
|
-
// Finnish
|
|
2051
|
-
el: "ell",
|
|
2052
|
-
// Greek
|
|
2053
|
-
cs: "ces",
|
|
2054
|
-
// Czech
|
|
2055
|
-
hu: "hun",
|
|
2056
|
-
// Hungarian
|
|
2057
|
-
ro: "ron",
|
|
2058
|
-
// Romanian
|
|
2059
|
-
bg: "bul",
|
|
2060
|
-
// Bulgarian
|
|
2061
|
-
hr: "hrv",
|
|
2062
|
-
// Croatian
|
|
2063
|
-
sk: "slk",
|
|
2064
|
-
// Slovak
|
|
2065
|
-
sl: "slv",
|
|
2066
|
-
// Slovenian
|
|
2067
|
-
uk: "ukr",
|
|
2068
|
-
// Ukrainian
|
|
2069
|
-
tr: "tur",
|
|
2070
|
-
// Turkish
|
|
2071
|
-
// Asian languages
|
|
2072
|
-
th: "tha",
|
|
2073
|
-
// Thai
|
|
2074
|
-
vi: "vie",
|
|
2075
|
-
// Vietnamese
|
|
2076
|
-
id: "ind",
|
|
2077
|
-
// Indonesian
|
|
2078
|
-
ms: "msa",
|
|
2079
|
-
// Malay
|
|
2080
|
-
tl: "tgl",
|
|
2081
|
-
// Tagalog/Filipino
|
|
2082
|
-
// Other languages
|
|
2083
|
-
he: "heb",
|
|
2084
|
-
// Hebrew
|
|
2085
|
-
fa: "fas",
|
|
2086
|
-
// Persian/Farsi
|
|
2087
|
-
bn: "ben",
|
|
2088
|
-
// Bengali
|
|
2089
|
-
ta: "tam",
|
|
2090
|
-
// Tamil
|
|
2091
|
-
te: "tel",
|
|
2092
|
-
// Telugu
|
|
2093
|
-
mr: "mar",
|
|
2094
|
-
// Marathi
|
|
2095
|
-
gu: "guj",
|
|
2096
|
-
// Gujarati
|
|
2097
|
-
kn: "kan",
|
|
2098
|
-
// Kannada
|
|
2099
|
-
ml: "mal",
|
|
2100
|
-
// Malayalam
|
|
2101
|
-
pa: "pan",
|
|
2102
|
-
// Punjabi
|
|
2103
|
-
ur: "urd",
|
|
2104
|
-
// Urdu
|
|
2105
|
-
sw: "swa",
|
|
2106
|
-
// Swahili
|
|
2107
|
-
af: "afr",
|
|
2108
|
-
// Afrikaans
|
|
2109
|
-
ca: "cat",
|
|
2110
|
-
// Catalan
|
|
2111
|
-
eu: "eus",
|
|
2112
|
-
// Basque
|
|
2113
|
-
gl: "glg",
|
|
2114
|
-
// Galician
|
|
2115
|
-
is: "isl",
|
|
2116
|
-
// Icelandic
|
|
2117
|
-
et: "est",
|
|
2118
|
-
// Estonian
|
|
2119
|
-
lv: "lav",
|
|
2120
|
-
// Latvian
|
|
2121
|
-
lt: "lit"
|
|
2122
|
-
// Lithuanian
|
|
2123
|
-
};
|
|
2124
|
-
var ISO639_3_TO_1 = Object.fromEntries(
|
|
2125
|
-
Object.entries(ISO639_1_TO_3).map(([iso1, iso3]) => [iso3, iso1])
|
|
2126
|
-
);
|
|
2127
|
-
function toISO639_3(code) {
|
|
2128
|
-
const normalized = code.toLowerCase().trim();
|
|
2129
|
-
if (normalized.length === 3) {
|
|
2130
|
-
return normalized;
|
|
2131
|
-
}
|
|
2132
|
-
return ISO639_1_TO_3[normalized] ?? normalized;
|
|
2133
|
-
}
|
|
2134
|
-
function toISO639_1(code) {
|
|
2135
|
-
const normalized = code.toLowerCase().trim();
|
|
2136
|
-
if (normalized.length === 2) {
|
|
2137
|
-
return normalized;
|
|
2138
|
-
}
|
|
2139
|
-
return ISO639_3_TO_1[normalized] ?? normalized;
|
|
2140
|
-
}
|
|
2141
|
-
function getLanguageCodePair(code) {
|
|
2142
|
-
const normalized = code.toLowerCase().trim();
|
|
2143
|
-
if (normalized.length === 2) {
|
|
2144
|
-
return {
|
|
2145
|
-
iso639_1: normalized,
|
|
2146
|
-
iso639_3: toISO639_3(normalized)
|
|
2147
|
-
};
|
|
2148
|
-
} else if (normalized.length === 3) {
|
|
2149
|
-
return {
|
|
2150
|
-
iso639_1: toISO639_1(normalized),
|
|
2151
|
-
iso639_3: normalized
|
|
2152
|
-
};
|
|
2153
|
-
}
|
|
2154
|
-
return {
|
|
2155
|
-
iso639_1: normalized,
|
|
2156
|
-
iso639_3: normalized
|
|
2157
|
-
};
|
|
2158
|
-
}
|
|
2159
|
-
function getLanguageName(code) {
|
|
2160
|
-
const iso639_1 = toISO639_1(code);
|
|
2161
|
-
try {
|
|
2162
|
-
const displayNames = new Intl.DisplayNames(["en"], { type: "language" });
|
|
2163
|
-
return displayNames.of(iso639_1) ?? code.toUpperCase();
|
|
2164
|
-
} catch {
|
|
2165
|
-
return code.toUpperCase();
|
|
2166
|
-
}
|
|
2167
|
-
}
|
|
2168
|
-
|
|
2169
|
-
// src/workflows/chapters.ts
|
|
2170
2235
|
var chapterSchema = z4.object({
|
|
2171
2236
|
startTime: z4.number(),
|
|
2172
2237
|
title: z4.string()
|
|
@@ -2218,13 +2283,13 @@ var chapterSystemPromptBuilder = createPromptBuilder({
|
|
|
2218
2283
|
},
|
|
2219
2284
|
context: {
|
|
2220
2285
|
tag: "context",
|
|
2221
|
-
content:
|
|
2286
|
+
content: dedent4`
|
|
2222
2287
|
You receive a timestamped transcript with lines in the form "[12s] Caption text".
|
|
2223
2288
|
Use those timestamps as anchors to determine chapter start times in seconds.`
|
|
2224
2289
|
},
|
|
2225
2290
|
constraints: {
|
|
2226
2291
|
tag: "constraints",
|
|
2227
|
-
content:
|
|
2292
|
+
content: dedent4`
|
|
2228
2293
|
- Only use information present in the transcript
|
|
2229
2294
|
- Return structured data that matches the requested JSON schema
|
|
2230
2295
|
- Do not add commentary or extra text outside the JSON
|
|
@@ -2232,7 +2297,7 @@ var chapterSystemPromptBuilder = createPromptBuilder({
|
|
|
2232
2297
|
},
|
|
2233
2298
|
qualityGuidelines: {
|
|
2234
2299
|
tag: "quality_guidelines",
|
|
2235
|
-
content:
|
|
2300
|
+
content: dedent4`
|
|
2236
2301
|
- Create chapters at topic shifts or clear transitions
|
|
2237
2302
|
- Keep chapter titles concise and descriptive
|
|
2238
2303
|
- Ensure the first chapter starts at 0 seconds`
|
|
@@ -2242,7 +2307,7 @@ var chapterSystemPromptBuilder = createPromptBuilder({
|
|
|
2242
2307
|
});
|
|
2243
2308
|
var AUDIO_ONLY_SYSTEM_PROMPT_OVERRIDES = {
|
|
2244
2309
|
role: "You are an audio editor and transcript analyst specializing in segmenting content into logical chapters.",
|
|
2245
|
-
context:
|
|
2310
|
+
context: dedent4`
|
|
2246
2311
|
You receive a timestamped transcript from audio-only content with lines in the form "[12s] Transcript text".
|
|
2247
2312
|
Use those timestamps as anchors to determine chapter start times in seconds.`
|
|
2248
2313
|
};
|
|
@@ -2254,7 +2319,7 @@ var chaptersPromptBuilder = createPromptBuilder({
|
|
|
2254
2319
|
},
|
|
2255
2320
|
outputFormat: {
|
|
2256
2321
|
tag: "output_format",
|
|
2257
|
-
content:
|
|
2322
|
+
content: dedent4`
|
|
2258
2323
|
Return valid JSON in this exact shape:
|
|
2259
2324
|
{
|
|
2260
2325
|
"chapters": [
|
|
@@ -2273,7 +2338,7 @@ var chaptersPromptBuilder = createPromptBuilder({
|
|
|
2273
2338
|
},
|
|
2274
2339
|
titleGuidelines: {
|
|
2275
2340
|
tag: "title_guidelines",
|
|
2276
|
-
content:
|
|
2341
|
+
content: dedent4`
|
|
2277
2342
|
- Keep titles concise and descriptive
|
|
2278
2343
|
- Avoid filler or generic labels like "Chapter 1"
|
|
2279
2344
|
- Use the transcript's terminology`
|
|
@@ -2298,7 +2363,7 @@ function buildUserPrompt3({
|
|
|
2298
2363
|
if (languageName) {
|
|
2299
2364
|
contextSections.push(createLanguageSection(languageName));
|
|
2300
2365
|
}
|
|
2301
|
-
const dynamicChapterGuidelines =
|
|
2366
|
+
const dynamicChapterGuidelines = dedent4`
|
|
2302
2367
|
- Create at least ${minChaptersPerHour} and at most ${maxChaptersPerHour} chapters per hour of content
|
|
2303
2368
|
- Use start times in seconds (not HH:MM:SS)
|
|
2304
2369
|
- Chapter start times should be non-decreasing
|
|
@@ -2356,7 +2421,7 @@ async function generateChapters(assetId, options = {}) {
|
|
|
2356
2421
|
const contentLabel = isAudioOnly ? "transcript" : "caption track";
|
|
2357
2422
|
throw new MuxAiError(`No usable content found in ${contentLabel}.`, { type: "validation_error" });
|
|
2358
2423
|
}
|
|
2359
|
-
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : transcriptResult.track
|
|
2424
|
+
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : getReliableLanguageCode(transcriptResult.track) ?? languageCode;
|
|
2360
2425
|
const languageName = resolvedLanguageCode ? getLanguageName(resolvedLanguageCode) : void 0;
|
|
2361
2426
|
const userPrompt = buildUserPrompt3({
|
|
2362
2427
|
timestampedTranscript,
|
|
@@ -2398,7 +2463,7 @@ async function generateChapters(assetId, options = {}) {
|
|
|
2398
2463
|
};
|
|
2399
2464
|
return {
|
|
2400
2465
|
assetId,
|
|
2401
|
-
languageCode: languageCode ?? transcriptResult.track
|
|
2466
|
+
languageCode: languageCode ?? getReliableLanguageCode(transcriptResult.track) ?? "en",
|
|
2402
2467
|
chapters: validChapters,
|
|
2403
2468
|
usage: usageWithMetadata
|
|
2404
2469
|
};
|
|
@@ -2406,7 +2471,7 @@ async function generateChapters(assetId, options = {}) {
|
|
|
2406
2471
|
|
|
2407
2472
|
// src/workflows/edit-captions.ts
|
|
2408
2473
|
import { generateText as generateText4, Output as Output4 } from "ai";
|
|
2409
|
-
import
|
|
2474
|
+
import dedent5 from "dedent";
|
|
2410
2475
|
import { z as z5 } from "zod";
|
|
2411
2476
|
|
|
2412
2477
|
// src/lib/mux-tracks.ts
|
|
@@ -2695,7 +2760,7 @@ var profanityDetectionSchema = z5.object({
|
|
|
2695
2760
|
"Unique profane words or short phrases exactly as they appear in the transcript text. Include each distinct form only once (e.g., if 'fuck' and 'fucking' both appear, list both)."
|
|
2696
2761
|
)
|
|
2697
2762
|
});
|
|
2698
|
-
var SYSTEM_PROMPT3 =
|
|
2763
|
+
var SYSTEM_PROMPT3 = dedent5`
|
|
2699
2764
|
You are a content moderation assistant. Your task is to identify profane, vulgar, or obscene
|
|
2700
2765
|
words and phrases in subtitle text. Return ONLY the exact profane words or phrases as they appear
|
|
2701
2766
|
in the text. Do not modify, censor, or paraphrase them. Do not include words that are merely
|
|
@@ -3411,7 +3476,7 @@ async function generateVideoEmbeddings(assetId, options = {}) {
|
|
|
3411
3476
|
|
|
3412
3477
|
// src/workflows/engagement-insights.ts
|
|
3413
3478
|
import { generateText as generateText5, Output as Output5 } from "ai";
|
|
3414
|
-
import
|
|
3479
|
+
import dedent6 from "dedent";
|
|
3415
3480
|
import { z as z6 } from "zod";
|
|
3416
3481
|
|
|
3417
3482
|
// src/primitives/heatmap.ts
|
|
@@ -3564,7 +3629,7 @@ var engagementInsightsSchema = z6.object({
|
|
|
3564
3629
|
momentInsights: z6.array(aiMomentInsightSchema),
|
|
3565
3630
|
overallInsight: aiOverallInsightSchema
|
|
3566
3631
|
});
|
|
3567
|
-
var SYSTEM_PROMPT4 =
|
|
3632
|
+
var SYSTEM_PROMPT4 = promptDedent`
|
|
3568
3633
|
<role>
|
|
3569
3634
|
You are a video engagement analyst specializing in understanding viewer behavior.
|
|
3570
3635
|
Your job is to explain why specific moments in videos have high or low engagement
|
|
@@ -3592,12 +3657,10 @@ var SYSTEM_PROMPT4 = dedent5`
|
|
|
3592
3657
|
|
|
3593
3658
|
<constraints>
|
|
3594
3659
|
- Only describe what you can see in images or read in transcripts
|
|
3595
|
-
-
|
|
3596
|
-
-
|
|
3597
|
-
playback IDs, or technical parameters visible in this request. These are
|
|
3598
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
3660
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
3661
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
3599
3662
|
- Correlate engagement scores with observable content
|
|
3600
|
-
-
|
|
3663
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
3601
3664
|
- Each momentInsight MUST include the hotspotIndex from the input data
|
|
3602
3665
|
</constraints>
|
|
3603
3666
|
|
|
@@ -3608,7 +3671,7 @@ var SYSTEM_PROMPT4 = dedent5`
|
|
|
3608
3671
|
- Connect engagement data to content features
|
|
3609
3672
|
</quality_guidelines>
|
|
3610
3673
|
`;
|
|
3611
|
-
var AUDIO_ONLY_SYSTEM_PROMPT2 =
|
|
3674
|
+
var AUDIO_ONLY_SYSTEM_PROMPT2 = promptDedent`
|
|
3612
3675
|
<role>
|
|
3613
3676
|
You are a video engagement analyst specializing in understanding viewer behavior for audio content.
|
|
3614
3677
|
Your job is to explain why specific moments have high or low engagement based on audio/dialogue and viewer watch patterns.
|
|
@@ -3635,12 +3698,10 @@ var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent5`
|
|
|
3635
3698
|
|
|
3636
3699
|
<constraints>
|
|
3637
3700
|
- Only describe what you can read in the transcript
|
|
3638
|
-
-
|
|
3639
|
-
-
|
|
3640
|
-
playback IDs, or technical parameters visible in this request. These are
|
|
3641
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
3701
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
3702
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
3642
3703
|
- Correlate engagement scores with observable content
|
|
3643
|
-
-
|
|
3704
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
3644
3705
|
- Each momentInsight MUST include the hotspotIndex from the input data
|
|
3645
3706
|
</constraints>
|
|
3646
3707
|
|
|
@@ -3651,7 +3712,7 @@ var AUDIO_ONLY_SYSTEM_PROMPT2 = dedent5`
|
|
|
3651
3712
|
- Connect engagement data to content features
|
|
3652
3713
|
</quality_guidelines>
|
|
3653
3714
|
`;
|
|
3654
|
-
var INSIGHT_GUIDELINES =
|
|
3715
|
+
var INSIGHT_GUIDELINES = dedent6`
|
|
3655
3716
|
Provide explanatory insights that describe WHY moments are engaging.
|
|
3656
3717
|
Focus on observable patterns and viewer behavior correlation.
|
|
3657
3718
|
Example: "The cooking demonstration at 2:15 has 3x average engagement
|
|
@@ -3669,7 +3730,7 @@ var engagementInsightsPromptBuilder = createPromptBuilder({
|
|
|
3669
3730
|
},
|
|
3670
3731
|
outputFormat: {
|
|
3671
3732
|
tag: "output_format",
|
|
3672
|
-
content:
|
|
3733
|
+
content: dedent6`
|
|
3673
3734
|
Return valid JSON matching the provided schema.
|
|
3674
3735
|
Include insights for each hotspot and overall engagement trends.
|
|
3675
3736
|
Each momentInsight must include the hotspotIndex matching the input.
|
|
@@ -3677,7 +3738,7 @@ var engagementInsightsPromptBuilder = createPromptBuilder({
|
|
|
3677
3738
|
},
|
|
3678
3739
|
visualContext: {
|
|
3679
3740
|
tag: "visual_context",
|
|
3680
|
-
content:
|
|
3741
|
+
content: dedent6`
|
|
3681
3742
|
Images are provided for each hotspot timestamp.
|
|
3682
3743
|
A storyboard overview image may also be included showing the full video timeline.
|
|
3683
3744
|
Use visual evidence from these images to support your engagement analysis.
|
|
@@ -3691,14 +3752,14 @@ function buildUserPrompt4(hotspots, transcriptSegments, heatmap, isAudioOnly) {
|
|
|
3691
3752
|
const transcript = transcriptSegments[idx]?.text || "(no transcript)";
|
|
3692
3753
|
const startTimestamp = secondsToTimestamp(h.startMs / 1e3);
|
|
3693
3754
|
const endTimestamp = secondsToTimestamp(h.endMs / 1e3);
|
|
3694
|
-
return
|
|
3755
|
+
return dedent6`
|
|
3695
3756
|
Hotspot ${idx} (hotspotIndex: ${idx}):
|
|
3696
3757
|
- Time Range: ${startTimestamp} - ${endTimestamp}
|
|
3697
3758
|
- Engagement Score: ${h.score.toFixed(2)} (0=low, 1=high)
|
|
3698
3759
|
- Transcript: "${transcript}"
|
|
3699
3760
|
`;
|
|
3700
3761
|
}).join("\n\n");
|
|
3701
|
-
const heatmapData =
|
|
3762
|
+
const heatmapData = dedent6`
|
|
3702
3763
|
Engagement Heatmap (${heatmap.length} values, each representing 1/${heatmap.length}th of the video, higher = more re-watches):
|
|
3703
3764
|
[${heatmap.map((v) => v.toFixed(2)).join(", ")}]
|
|
3704
3765
|
`;
|
|
@@ -4607,7 +4668,7 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
4607
4668
|
|
|
4608
4669
|
// src/workflows/summarization.ts
|
|
4609
4670
|
import { generateText as generateText6, Output as Output6 } from "ai";
|
|
4610
|
-
import
|
|
4671
|
+
import dedent7 from "dedent";
|
|
4611
4672
|
import { z as z7 } from "zod";
|
|
4612
4673
|
var DEFAULT_SUMMARY_KEYWORD_LIMIT = 10;
|
|
4613
4674
|
var DEFAULT_TITLE_LENGTH = 10;
|
|
@@ -4633,36 +4694,36 @@ var DESCRIPTION_LENGTH_THRESHOLD_LARGE = 100;
|
|
|
4633
4694
|
function buildDescriptionGuidance(wordCount, contentType) {
|
|
4634
4695
|
if (wordCount < DESCRIPTION_LENGTH_THRESHOLD_SMALL) {
|
|
4635
4696
|
if (contentType === "video") {
|
|
4636
|
-
return
|
|
4697
|
+
return dedent7`A brief summary of the video in no more than ${wordCount} words. Shorter is fine.
|
|
4637
4698
|
Focus on the single most important subject or action.
|
|
4638
4699
|
Write in present tense.`;
|
|
4639
4700
|
}
|
|
4640
|
-
return
|
|
4701
|
+
return dedent7`A brief summary of the audio content in no more than ${wordCount} words. Shorter is fine.
|
|
4641
4702
|
Focus on the single most important topic or theme.
|
|
4642
4703
|
Write in present tense.`;
|
|
4643
4704
|
}
|
|
4644
4705
|
if (wordCount > DESCRIPTION_LENGTH_THRESHOLD_LARGE) {
|
|
4645
4706
|
if (contentType === "video") {
|
|
4646
|
-
return
|
|
4707
|
+
return dedent7`A detailed summary that describes what happens across the video.
|
|
4647
4708
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4648
4709
|
Be thorough: cover subjects, actions, setting, progression, and any notable details visible across frames.
|
|
4649
4710
|
Write in present tense. Be specific about observable details rather than making assumptions.
|
|
4650
4711
|
If the transcript provides dialogue or narration, incorporate key points but prioritize visual content.`;
|
|
4651
4712
|
}
|
|
4652
|
-
return
|
|
4713
|
+
return dedent7`A detailed summary that describes the audio content.
|
|
4653
4714
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4654
4715
|
Be thorough: cover topics, speakers, themes, progression, and any notable insights.
|
|
4655
4716
|
Write in present tense. Be specific about what is discussed or presented rather than making assumptions.
|
|
4656
4717
|
Focus on the spoken content and any key insights, dialogue, or narrative elements.`;
|
|
4657
4718
|
}
|
|
4658
4719
|
if (contentType === "video") {
|
|
4659
|
-
return
|
|
4720
|
+
return dedent7`A summary that describes what happens across the video.
|
|
4660
4721
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4661
4722
|
Cover the main subjects, actions, setting, and any notable progression visible across frames.
|
|
4662
4723
|
Write in present tense. Be specific about observable details rather than making assumptions.
|
|
4663
4724
|
If the transcript provides dialogue or narration, incorporate key points but prioritize visual content.`;
|
|
4664
4725
|
}
|
|
4665
|
-
return
|
|
4726
|
+
return dedent7`A summary that describes the audio content.
|
|
4666
4727
|
Never exceed ${wordCount} words, but shorter is perfectly fine. You may use multiple sentences.
|
|
4667
4728
|
Cover the main topics, speakers, themes, and any notable progression in the discussion or narration.
|
|
4668
4729
|
Write in present tense. Be specific about what is discussed or presented rather than making assumptions.
|
|
@@ -4679,7 +4740,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
4679
4740
|
},
|
|
4680
4741
|
title: {
|
|
4681
4742
|
tag: "title_requirements",
|
|
4682
|
-
content:
|
|
4743
|
+
content: dedent7`
|
|
4683
4744
|
A concise, label-style title — not a sentence or description.
|
|
4684
4745
|
Never exceed ${titleLimit} words, but shorter is better.
|
|
4685
4746
|
Think of how a video card title, playlist entry, or file name would read — e.g. "Predator: Badlands Trailer" or "Chef Prepares Holiday Feast".
|
|
@@ -4692,7 +4753,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
4692
4753
|
},
|
|
4693
4754
|
keywords: {
|
|
4694
4755
|
tag: "keywords_requirements",
|
|
4695
|
-
content:
|
|
4756
|
+
content: dedent7`
|
|
4696
4757
|
Specific, searchable terms (up to ${keywordLimit}) that capture:
|
|
4697
4758
|
- Primary subjects (people, animals, objects)
|
|
4698
4759
|
- Actions and activities being performed
|
|
@@ -4704,7 +4765,7 @@ function createSummarizationBuilder({ titleLength, descriptionLength, tagCount }
|
|
|
4704
4765
|
},
|
|
4705
4766
|
qualityGuidelines: {
|
|
4706
4767
|
tag: "quality_guidelines",
|
|
4707
|
-
content:
|
|
4768
|
+
content: dedent7`
|
|
4708
4769
|
- Examine all frames to understand the full context and progression
|
|
4709
4770
|
- Be precise: "golden retriever" is better than "dog" when identifiable
|
|
4710
4771
|
- Capture the narrative: what begins, develops, and concludes
|
|
@@ -4725,7 +4786,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
4725
4786
|
},
|
|
4726
4787
|
title: {
|
|
4727
4788
|
tag: "title_requirements",
|
|
4728
|
-
content:
|
|
4789
|
+
content: dedent7`
|
|
4729
4790
|
A concise, label-style title — not a sentence or description.
|
|
4730
4791
|
Never exceed ${titleLimit} words, but shorter is better.
|
|
4731
4792
|
Think of how a podcast episode title or playlist entry would read — e.g. "Weekly News Roundup" or "Interview with Dr. Smith".
|
|
@@ -4738,7 +4799,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
4738
4799
|
},
|
|
4739
4800
|
keywords: {
|
|
4740
4801
|
tag: "keywords_requirements",
|
|
4741
|
-
content:
|
|
4802
|
+
content: dedent7`
|
|
4742
4803
|
Specific, searchable terms (up to ${keywordLimit}) that capture:
|
|
4743
4804
|
- Primary topics and themes
|
|
4744
4805
|
- Speakers or presenters (if named)
|
|
@@ -4750,7 +4811,7 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
4750
4811
|
},
|
|
4751
4812
|
qualityGuidelines: {
|
|
4752
4813
|
tag: "quality_guidelines",
|
|
4753
|
-
content:
|
|
4814
|
+
content: dedent7`
|
|
4754
4815
|
- Analyze the full transcript to understand context and themes
|
|
4755
4816
|
- Be precise: use specific terminology when mentioned
|
|
4756
4817
|
- Capture the narrative: what is introduced, discussed, and concluded
|
|
@@ -4760,15 +4821,14 @@ function createAudioOnlyBuilder({ titleLength, descriptionLength, tagCount } = {
|
|
|
4760
4821
|
sectionOrder: ["task", "title", "description", "keywords", "qualityGuidelines"]
|
|
4761
4822
|
});
|
|
4762
4823
|
}
|
|
4763
|
-
var SYSTEM_PROMPT5 =
|
|
4824
|
+
var SYSTEM_PROMPT5 = promptDedent`
|
|
4764
4825
|
<role>
|
|
4765
4826
|
You are a video content analyst specializing in storyboard interpretation and multimodal analysis.
|
|
4766
4827
|
</role>
|
|
4767
4828
|
|
|
4768
4829
|
<context>
|
|
4769
4830
|
You receive storyboard images containing multiple sequential frames extracted from a video.
|
|
4770
|
-
|
|
4771
|
-
Read frames left-to-right, top-to-bottom to understand the temporal sequence.
|
|
4831
|
+
${STORYBOARD_FRAME_INSTRUCTIONS}
|
|
4772
4832
|
</context>
|
|
4773
4833
|
|
|
4774
4834
|
<transcript_guidance>
|
|
@@ -4790,37 +4850,21 @@ var SYSTEM_PROMPT5 = dedent6`
|
|
|
4790
4850
|
|
|
4791
4851
|
<constraints>
|
|
4792
4852
|
- Only describe what is clearly observable in the frames or explicitly stated in the transcript
|
|
4793
|
-
-
|
|
4794
|
-
-
|
|
4795
|
-
|
|
4796
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
4797
|
-
- Return structured data matching the requested schema
|
|
4853
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
4854
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
4855
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
4798
4856
|
- Output only the JSON object; no markdown or extra text
|
|
4799
4857
|
- When a <language> section is provided, all output text MUST be written in that language
|
|
4800
4858
|
</constraints>
|
|
4801
4859
|
|
|
4802
4860
|
<tone_guidance>
|
|
4803
|
-
|
|
4804
|
-
Adapt your entire analysis and writing style to match the specified tone - this should influence
|
|
4805
|
-
your word choice, personality, formality level, and overall presentation of the content.
|
|
4806
|
-
The tone instructions are not suggestions but core requirements for how you should express yourself.
|
|
4861
|
+
${TONE_GUIDANCE}
|
|
4807
4862
|
</tone_guidance>
|
|
4808
4863
|
|
|
4809
4864
|
<language_guidelines>
|
|
4810
|
-
|
|
4811
|
-
- "The image shows..." / "The storyboard shows..."
|
|
4812
|
-
- "In this video..." / "This video features..."
|
|
4813
|
-
- "The frames depict..." / "The footage shows..."
|
|
4814
|
-
- "We can see..." / "You can see..."
|
|
4815
|
-
- "The clip shows..." / "The scene shows..."
|
|
4816
|
-
|
|
4817
|
-
INSTEAD, describe the content directly:
|
|
4818
|
-
- BAD: "The video shows a chef preparing a meal"
|
|
4819
|
-
- GOOD: "A chef prepares a meal in a professional kitchen"
|
|
4820
|
-
|
|
4821
|
-
Write as if describing reality, not describing a recording of reality.
|
|
4865
|
+
${createLanguageGuidelines("video")}
|
|
4822
4866
|
</language_guidelines>`;
|
|
4823
|
-
var AUDIO_ONLY_SYSTEM_PROMPT3 =
|
|
4867
|
+
var AUDIO_ONLY_SYSTEM_PROMPT3 = promptDedent`
|
|
4824
4868
|
<role>
|
|
4825
4869
|
You are an audio content analyst specializing in transcript analysis and metadata generation.
|
|
4826
4870
|
</role>
|
|
@@ -4847,35 +4891,20 @@ var AUDIO_ONLY_SYSTEM_PROMPT3 = dedent6`
|
|
|
4847
4891
|
|
|
4848
4892
|
<constraints>
|
|
4849
4893
|
- Only describe what is explicitly stated or strongly implied in the transcript
|
|
4850
|
-
-
|
|
4851
|
-
-
|
|
4852
|
-
|
|
4853
|
-
delivery infrastructure and are unrelated to the media content itself.
|
|
4854
|
-
- Return structured data matching the requested schema
|
|
4894
|
+
- ${NO_FABRICATION_CONSTRAINT}
|
|
4895
|
+
- ${METADATA_BOUNDARY_WARNING}
|
|
4896
|
+
- ${STRUCTURED_DATA_CONSTRAINT}
|
|
4855
4897
|
- Focus entirely on audio/spoken content - there are no visual elements
|
|
4856
4898
|
- Output only the JSON object; no markdown or extra text
|
|
4857
4899
|
- When a <language> section is provided, all output text MUST be written in that language
|
|
4858
4900
|
</constraints>
|
|
4859
4901
|
|
|
4860
4902
|
<tone_guidance>
|
|
4861
|
-
|
|
4862
|
-
Adapt your entire analysis and writing style to match the specified tone - this should influence
|
|
4863
|
-
your word choice, personality, formality level, and overall presentation of the content.
|
|
4864
|
-
The tone instructions are not suggestions but core requirements for how you should express yourself.
|
|
4903
|
+
${TONE_GUIDANCE}
|
|
4865
4904
|
</tone_guidance>
|
|
4866
4905
|
|
|
4867
4906
|
<language_guidelines>
|
|
4868
|
-
|
|
4869
|
-
- "The audio shows..." / "The transcript shows..."
|
|
4870
|
-
- "In this recording..." / "This audio features..."
|
|
4871
|
-
- "The speaker says..." / "We can hear..."
|
|
4872
|
-
- "The clip contains..." / "The recording shows..."
|
|
4873
|
-
|
|
4874
|
-
INSTEAD, describe the content directly:
|
|
4875
|
-
- BAD: "The audio features a discussion about climate change"
|
|
4876
|
-
- GOOD: "A panel discusses climate change impacts and solutions"
|
|
4877
|
-
|
|
4878
|
-
Write as if describing reality, not describing a recording of reality.
|
|
4907
|
+
${createLanguageGuidelines("audio")}
|
|
4879
4908
|
</language_guidelines>`;
|
|
4880
4909
|
function buildUserPrompt5({
|
|
4881
4910
|
tone,
|
|
@@ -5049,7 +5078,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
5049
5078
|
required: isAudioOnly
|
|
5050
5079
|
}) : void 0;
|
|
5051
5080
|
const transcriptText = transcriptResult?.transcriptText ?? "";
|
|
5052
|
-
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : transcriptResult?.track
|
|
5081
|
+
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : getReliableLanguageCode(transcriptResult?.track) ?? getReliableLanguageCode(getReadyTextTracks(assetData)[0]);
|
|
5053
5082
|
const languageName = resolvedLanguageCode ? getLanguageName(resolvedLanguageCode) : void 0;
|
|
5054
5083
|
const userPrompt = buildUserPrompt5({
|
|
5055
5084
|
tone,
|
|
@@ -5398,7 +5427,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
5398
5427
|
{ type: "validation_error" }
|
|
5399
5428
|
);
|
|
5400
5429
|
}
|
|
5401
|
-
let audioUrl =
|
|
5430
|
+
let audioUrl = `${getMuxStreamOrigin()}/${playbackId}/audio.m4a`;
|
|
5402
5431
|
if (policy === "signed") {
|
|
5403
5432
|
audioUrl = await signUrl(audioUrl, playbackId, "video", void 0, credentials);
|
|
5404
5433
|
}
|
|
@@ -5541,18 +5570,18 @@ import {
|
|
|
5541
5570
|
RetryError,
|
|
5542
5571
|
TypeValidationError
|
|
5543
5572
|
} from "ai";
|
|
5544
|
-
import
|
|
5573
|
+
import dedent8 from "dedent";
|
|
5545
5574
|
import { z as z8 } from "zod";
|
|
5546
5575
|
var translationSchema = z8.object({
|
|
5547
5576
|
translation: z8.string()
|
|
5548
5577
|
});
|
|
5549
|
-
var SYSTEM_PROMPT6 =
|
|
5578
|
+
var SYSTEM_PROMPT6 = dedent8`
|
|
5550
5579
|
You are a subtitle translation expert. Translate VTT subtitle files to the target language specified by the user.
|
|
5551
5580
|
You may receive either a full VTT file or a chunk from a larger VTT.
|
|
5552
5581
|
Preserve all timestamps, cue ordering, and VTT formatting exactly as they appear.
|
|
5553
5582
|
Return JSON with a single key "translation" containing the translated VTT content.
|
|
5554
5583
|
`;
|
|
5555
|
-
var CUE_TRANSLATION_SYSTEM_PROMPT =
|
|
5584
|
+
var CUE_TRANSLATION_SYSTEM_PROMPT = dedent8`
|
|
5556
5585
|
You are a subtitle translation expert.
|
|
5557
5586
|
You will receive a sequence of subtitle cues extracted from a VTT file.
|
|
5558
5587
|
Translate the cues to the requested target language while preserving their original order.
|
|
@@ -6063,7 +6092,7 @@ async function translateCaptions(assetId, trackId, toLanguageCode, options) {
|
|
|
6063
6092
|
}
|
|
6064
6093
|
if (uploadToMux) {
|
|
6065
6094
|
try {
|
|
6066
|
-
const languageName = getLanguageName(toLanguageCode);
|
|
6095
|
+
const languageName = getLanguageName(toLanguageCode) ?? toLanguageCode.toUpperCase();
|
|
6067
6096
|
const trackName = `${languageName} (auto-translated)`;
|
|
6068
6097
|
uploadedTrackId = await createTextTrackOnMux(
|
|
6069
6098
|
assetId,
|