@mux/ai 0.28.1 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/{index-bjQi1ZCj.d.ts → index-CJtQiGj7.d.ts} +18 -4
- package/dist/{index-DUVBpnj5.d.ts → index-D3cB2ltG.d.ts} +47 -15
- package/dist/index.d.ts +7 -6
- package/dist/index.js +571 -189
- package/dist/index.js.map +1 -1
- package/dist/primitives/index.d.ts +2 -2
- package/dist/primitives/index.js +122 -21
- package/dist/primitives/index.js.map +1 -1
- package/dist/{shots-mGJfYGiO.d.ts → shots-Bh0GrvLx.d.ts} +21 -1
- package/dist/workflows/index.d.ts +2 -2
- package/dist/workflows/index.js +678 -295
- package/dist/workflows/index.js.map +1 -1
- package/package.json +7 -1
package/dist/index.js
CHANGED
|
@@ -5,7 +5,7 @@ var __export = (target, all) => {
|
|
|
5
5
|
};
|
|
6
6
|
|
|
7
7
|
// package.json
|
|
8
|
-
var version = "0.
|
|
8
|
+
var version = "0.30.0";
|
|
9
9
|
|
|
10
10
|
// src/lib/language-codes.ts
|
|
11
11
|
var ISO639_1_TO_3 = {
|
|
@@ -247,7 +247,7 @@ var EnvSchema = z.object({
|
|
|
247
247
|
MUX_TEST_ASSET_ID_VIOLENT_AUDIO_ONLY: optionalString("Mux test asset ID for audio-only assets with violent content.", "Mux test asset id for audio-only assets with violent content for testing"),
|
|
248
248
|
// Eval config
|
|
249
249
|
MUX_AI_EVAL_MODEL_SET: optionalString("Eval model selection mode.", "Choose between 'default' (provider defaults only) or 'all' (all configured models)"),
|
|
250
|
-
MUX_AI_EVAL_MODELS: optionalString("Comma-separated eval model pairs.", "Comma-separated provider:model pairs (e.g. 'openai:gpt-5.
|
|
250
|
+
MUX_AI_EVAL_MODELS: optionalString("Comma-separated eval model pairs.", "Comma-separated provider:model pairs (e.g. 'openai:gpt-5.6-luna,anthropic:claude-sonnet-4-5,google:gemini-3-flash-preview')"),
|
|
251
251
|
// AI Providers
|
|
252
252
|
OPENAI_API_KEY: optionalString("OpenAI API key for OpenAI-backed workflows.", "OpenAI API key"),
|
|
253
253
|
ANTHROPIC_API_KEY: optionalString("Anthropic API key for Claude-backed workflows.", "Anthropic API key"),
|
|
@@ -566,7 +566,8 @@ var TOKEN_USAGE_FIELDS = [
|
|
|
566
566
|
"outputTokens",
|
|
567
567
|
"totalTokens",
|
|
568
568
|
"reasoningTokens",
|
|
569
|
-
"cachedInputTokens"
|
|
569
|
+
"cachedInputTokens",
|
|
570
|
+
"cacheWriteTokens"
|
|
570
571
|
];
|
|
571
572
|
function isDefinedTokenUsageValue(value) {
|
|
572
573
|
return typeof value === "number";
|
|
@@ -608,6 +609,12 @@ function getErrorTokenUsage(error) {
|
|
|
608
609
|
extracted.cachedInputTokens = cacheReadTokens;
|
|
609
610
|
}
|
|
610
611
|
}
|
|
612
|
+
if (extracted.cacheWriteTokens === void 0) {
|
|
613
|
+
const cacheWriteTokens = source.inputTokenDetails?.cacheWriteTokens;
|
|
614
|
+
if (typeof cacheWriteTokens === "number") {
|
|
615
|
+
extracted.cacheWriteTokens = cacheWriteTokens;
|
|
616
|
+
}
|
|
617
|
+
}
|
|
611
618
|
return Object.keys(extracted).length > 0 ? extracted : void 0;
|
|
612
619
|
}
|
|
613
620
|
function rethrowWithTokenUsage(error, collectedUsage) {
|
|
@@ -633,6 +640,9 @@ var MuxAiError = class extends Error {
|
|
|
633
640
|
this.publicMessage = message;
|
|
634
641
|
this.retryable = opts?.retryable ?? false;
|
|
635
642
|
}
|
|
643
|
+
static is(value) {
|
|
644
|
+
return typeof value === "object" && value !== null && "__robots_error" in value && value.__robots_error === true;
|
|
645
|
+
}
|
|
636
646
|
};
|
|
637
647
|
function extractErrorDetail(error) {
|
|
638
648
|
if (error instanceof Error && error.message) {
|
|
@@ -650,7 +660,7 @@ function extractErrorDetail(error) {
|
|
|
650
660
|
return "Unknown error";
|
|
651
661
|
}
|
|
652
662
|
function wrapError(error, message) {
|
|
653
|
-
if (error
|
|
663
|
+
if (MuxAiError.is(error)) {
|
|
654
664
|
throw error;
|
|
655
665
|
}
|
|
656
666
|
const rawDetail = extractErrorDetail(error);
|
|
@@ -1372,6 +1382,7 @@ __export(primitives_exports, {
|
|
|
1372
1382
|
extractTextFromVTT: () => extractTextFromVTT,
|
|
1373
1383
|
extractTimestampedTranscript: () => extractTimestampedTranscript,
|
|
1374
1384
|
fetchTranscriptForAsset: () => fetchTranscriptForAsset,
|
|
1385
|
+
filterVttByScope: () => filterVttByScope,
|
|
1375
1386
|
findCaptionTrack: () => findCaptionTrack,
|
|
1376
1387
|
getHeatmapForAsset: () => getHeatmapForAsset,
|
|
1377
1388
|
getHeatmapForPlaybackId: () => getHeatmapForPlaybackId,
|
|
@@ -1400,7 +1411,8 @@ import { createAnthropic } from "@ai-sdk/anthropic";
|
|
|
1400
1411
|
import { createGoogleGenerativeAI } from "@ai-sdk/google";
|
|
1401
1412
|
import { createOpenAI } from "@ai-sdk/openai";
|
|
1402
1413
|
var DEFAULT_LANGUAGE_MODELS = {
|
|
1403
|
-
|
|
1414
|
+
// GPT-5.6 models default to medium reasoning when no effort is specified.
|
|
1415
|
+
openai: "gpt-5.6-luna",
|
|
1404
1416
|
anthropic: "claude-sonnet-4-5",
|
|
1405
1417
|
google: "gemini-3-flash-preview"
|
|
1406
1418
|
};
|
|
@@ -1409,11 +1421,20 @@ var DEFAULT_EMBEDDING_MODELS = {
|
|
|
1409
1421
|
google: "gemini-embedding-001"
|
|
1410
1422
|
};
|
|
1411
1423
|
var LANGUAGE_MODELS = {
|
|
1412
|
-
openai: ["gpt-5.1", "gpt-5-mini"],
|
|
1424
|
+
openai: ["gpt-5.6-luna", "gpt-5.1", "gpt-5-mini"],
|
|
1413
1425
|
anthropic: ["claude-sonnet-4-5"],
|
|
1414
1426
|
google: ["gemini-3-flash-preview", "gemini-3.1-flash-lite", "gemini-2.5-flash"]
|
|
1415
1427
|
};
|
|
1416
1428
|
var LANGUAGE_MODEL_DEPRECATIONS = [
|
|
1429
|
+
{
|
|
1430
|
+
provider: "openai",
|
|
1431
|
+
modelId: "gpt-5.1",
|
|
1432
|
+
replacementModelId: "gpt-5.6-luna",
|
|
1433
|
+
phase: "warn",
|
|
1434
|
+
deprecatedOn: "2026-08-01",
|
|
1435
|
+
sunsetOn: "2026-10-01",
|
|
1436
|
+
reason: "GPT-5.6 Luna with its default medium reasoning effort is the new OpenAI default."
|
|
1437
|
+
},
|
|
1417
1438
|
{
|
|
1418
1439
|
provider: "google",
|
|
1419
1440
|
modelId: "gemini-2.5-flash",
|
|
@@ -1474,7 +1495,7 @@ function parseEvalModelPair(value) {
|
|
|
1474
1495
|
const modelId = modelIdRaw?.trim();
|
|
1475
1496
|
if (!provider || !modelId) {
|
|
1476
1497
|
throw new Error(
|
|
1477
|
-
`Invalid eval model pair "${value}". Use "provider:model" (example: "openai:gpt-5.
|
|
1498
|
+
`Invalid eval model pair "${value}". Use "provider:model" (example: "openai:gpt-5.6-luna").`
|
|
1478
1499
|
);
|
|
1479
1500
|
}
|
|
1480
1501
|
if (!isSupportedProvider(provider)) {
|
|
@@ -1908,13 +1929,22 @@ async function signUrl(url, playbackId, type = "video", params, credentials) {
|
|
|
1908
1929
|
|
|
1909
1930
|
// src/primitives/storyboards.ts
|
|
1910
1931
|
var DEFAULT_STORYBOARD_WIDTH = 640;
|
|
1911
|
-
async function getStoryboardUrl(playbackId, width = DEFAULT_STORYBOARD_WIDTH, shouldSign = false, credentials) {
|
|
1932
|
+
async function getStoryboardUrl(playbackId, width = DEFAULT_STORYBOARD_WIDTH, shouldSign = false, credentials, scope) {
|
|
1912
1933
|
"use step";
|
|
1913
1934
|
const baseUrl = getMuxStoryboardBaseUrl(playbackId);
|
|
1935
|
+
const params = { width };
|
|
1936
|
+
if (scope?.startTime !== void 0) {
|
|
1937
|
+
params.asset_start_time = scope.startTime;
|
|
1938
|
+
}
|
|
1939
|
+
if (scope?.endTime !== void 0) {
|
|
1940
|
+
params.asset_end_time = scope.endTime;
|
|
1941
|
+
}
|
|
1914
1942
|
if (shouldSign) {
|
|
1915
|
-
return signUrl(baseUrl, playbackId, "storyboard",
|
|
1943
|
+
return signUrl(baseUrl, playbackId, "storyboard", params, credentials);
|
|
1916
1944
|
}
|
|
1917
|
-
return `${baseUrl}
|
|
1945
|
+
return `${baseUrl}?${new URLSearchParams(
|
|
1946
|
+
Object.entries(params).map(([key, value]) => [key, String(value)])
|
|
1947
|
+
).toString()}`;
|
|
1918
1948
|
}
|
|
1919
1949
|
|
|
1920
1950
|
// src/primitives/text-chunking.ts
|
|
@@ -2155,30 +2185,97 @@ function chunkText(text, strategy) {
|
|
|
2155
2185
|
}
|
|
2156
2186
|
}
|
|
2157
2187
|
|
|
2188
|
+
// src/lib/workflow-scope.ts
|
|
2189
|
+
function hasWorkflowScopeBoundaries(scope) {
|
|
2190
|
+
return scope?.startTime !== void 0 || scope?.endTime !== void 0;
|
|
2191
|
+
}
|
|
2192
|
+
function resolveWorkflowScope(scope, assetDurationSeconds) {
|
|
2193
|
+
if (assetDurationSeconds === void 0 || !Number.isFinite(assetDurationSeconds) || assetDurationSeconds <= 0) {
|
|
2194
|
+
throw new MuxAiError("Asset has no valid duration.", { type: "validation_error" });
|
|
2195
|
+
}
|
|
2196
|
+
const startTime = scope?.startTime ?? 0;
|
|
2197
|
+
const endTime = scope?.endTime ?? assetDurationSeconds;
|
|
2198
|
+
if (!Number.isFinite(startTime) || startTime < 0) {
|
|
2199
|
+
throw new MuxAiError(
|
|
2200
|
+
`scope.startTime must be a finite, non-negative number (received ${startTime}).`,
|
|
2201
|
+
{ type: "validation_error" }
|
|
2202
|
+
);
|
|
2203
|
+
}
|
|
2204
|
+
if (!Number.isFinite(endTime) || endTime < 0) {
|
|
2205
|
+
throw new MuxAiError(
|
|
2206
|
+
`scope.endTime must be a finite, non-negative number (received ${endTime}).`,
|
|
2207
|
+
{ type: "validation_error" }
|
|
2208
|
+
);
|
|
2209
|
+
}
|
|
2210
|
+
if (endTime > assetDurationSeconds) {
|
|
2211
|
+
throw new MuxAiError(
|
|
2212
|
+
`scope.endTime (${endTime}) cannot exceed the asset duration (${assetDurationSeconds}).`,
|
|
2213
|
+
{ type: "validation_error" }
|
|
2214
|
+
);
|
|
2215
|
+
}
|
|
2216
|
+
if (startTime >= endTime) {
|
|
2217
|
+
throw new MuxAiError(
|
|
2218
|
+
`scope.startTime (${startTime}) must be less than scope.endTime (${endTime}).`,
|
|
2219
|
+
{ type: "validation_error" }
|
|
2220
|
+
);
|
|
2221
|
+
}
|
|
2222
|
+
return { startTime, endTime };
|
|
2223
|
+
}
|
|
2224
|
+
function resolveRenderableVideoScope(scope, assetDurationSeconds, videoTrackDurationSeconds) {
|
|
2225
|
+
if (!hasWorkflowScopeBoundaries(scope)) {
|
|
2226
|
+
return void 0;
|
|
2227
|
+
}
|
|
2228
|
+
const resolvedScope = resolveWorkflowScope(scope, assetDurationSeconds);
|
|
2229
|
+
let renderableEndTime = resolvedScope.endTime;
|
|
2230
|
+
if (videoTrackDurationSeconds !== void 0 && Number.isFinite(videoTrackDurationSeconds) && videoTrackDurationSeconds >= 0) {
|
|
2231
|
+
renderableEndTime = Math.min(videoTrackDurationSeconds, resolvedScope.endTime);
|
|
2232
|
+
}
|
|
2233
|
+
const renderableScope = {
|
|
2234
|
+
startTime: Math.min(resolvedScope.startTime, renderableEndTime),
|
|
2235
|
+
endTime: renderableEndTime
|
|
2236
|
+
};
|
|
2237
|
+
if (renderableScope.startTime >= renderableScope.endTime) {
|
|
2238
|
+
throw new MuxAiError("The requested scope does not include any renderable video.", {
|
|
2239
|
+
type: "validation_error"
|
|
2240
|
+
});
|
|
2241
|
+
}
|
|
2242
|
+
return renderableScope;
|
|
2243
|
+
}
|
|
2244
|
+
function timeRangesOverlap(firstStart, firstEnd, second) {
|
|
2245
|
+
const secondStart = second.startTime ?? 0;
|
|
2246
|
+
const secondEnd = second.endTime ?? Number.POSITIVE_INFINITY;
|
|
2247
|
+
return firstStart < secondEnd && firstEnd > secondStart;
|
|
2248
|
+
}
|
|
2249
|
+
|
|
2158
2250
|
// src/primitives/thumbnails.ts
|
|
2159
2251
|
async function getThumbnailUrls(playbackId, duration, options = {}) {
|
|
2160
2252
|
"use step";
|
|
2161
|
-
const { interval = 10, width = 640, shouldSign = false, maxSamples, credentials } = options;
|
|
2253
|
+
const { interval = 10, width = 640, shouldSign = false, maxSamples, credentials, scope } = options;
|
|
2254
|
+
const effectiveScope = hasWorkflowScopeBoundaries(scope) ? scope : void 0;
|
|
2255
|
+
const resolvedScope = effectiveScope ? resolveWorkflowScope(effectiveScope, duration) : { startTime: 0, endTime: duration };
|
|
2256
|
+
const rangeDuration = resolvedScope.endTime - resolvedScope.startTime;
|
|
2162
2257
|
let timestamps = [];
|
|
2163
|
-
if (
|
|
2164
|
-
const spacing =
|
|
2258
|
+
if (rangeDuration <= 50) {
|
|
2259
|
+
const spacing = rangeDuration / 6;
|
|
2165
2260
|
for (let i = 1; i <= 5; i++) {
|
|
2166
|
-
|
|
2261
|
+
const time = resolvedScope.startTime + i * spacing;
|
|
2262
|
+
timestamps.push(effectiveScope ? Number(time.toFixed(3)) : Math.round(time));
|
|
2167
2263
|
}
|
|
2168
2264
|
} else {
|
|
2169
|
-
for (let time =
|
|
2265
|
+
for (let time = resolvedScope.startTime; time < resolvedScope.endTime; time += interval) {
|
|
2170
2266
|
timestamps.push(time);
|
|
2171
2267
|
}
|
|
2172
2268
|
}
|
|
2173
2269
|
if (maxSamples !== void 0 && timestamps.length > maxSamples) {
|
|
2174
2270
|
const newTimestamps = [];
|
|
2175
|
-
newTimestamps.push(
|
|
2271
|
+
newTimestamps.push(resolvedScope.startTime);
|
|
2176
2272
|
if (maxSamples >= 2) {
|
|
2177
|
-
const
|
|
2273
|
+
const lastTime = effectiveScope ? Math.max(resolvedScope.startTime, resolvedScope.endTime - 1e-3) : resolvedScope.endTime;
|
|
2274
|
+
const spacing = (lastTime - resolvedScope.startTime) / (maxSamples - 1);
|
|
2178
2275
|
for (let i = 1; i < maxSamples - 1; i++) {
|
|
2179
|
-
newTimestamps.push(spacing * i);
|
|
2276
|
+
newTimestamps.push(resolvedScope.startTime + spacing * i);
|
|
2180
2277
|
}
|
|
2181
|
-
newTimestamps.push(
|
|
2278
|
+
newTimestamps.push(lastTime);
|
|
2182
2279
|
}
|
|
2183
2280
|
timestamps = newTimestamps;
|
|
2184
2281
|
}
|
|
@@ -2512,6 +2609,23 @@ function parseVTTCues(vttContent) {
|
|
|
2512
2609
|
}
|
|
2513
2610
|
return cues;
|
|
2514
2611
|
}
|
|
2612
|
+
function filterVttByScope(vttContent, scope) {
|
|
2613
|
+
if (!scope || scope.startTime === void 0 && scope.endTime === void 0) {
|
|
2614
|
+
return vttContent;
|
|
2615
|
+
}
|
|
2616
|
+
const { preamble, cueBlocks } = splitVttPreambleAndCueBlocks(vttContent);
|
|
2617
|
+
const scopedCueBlocks = cueBlocks.filter((cueBlock) => {
|
|
2618
|
+
const cue = parseVTTCues(cueBlock)[0];
|
|
2619
|
+
return cue ? timeRangesOverlap(cue.startTime, cue.endTime, scope) : false;
|
|
2620
|
+
}).map((cueBlock, index) => {
|
|
2621
|
+
const lines = cueBlock.split("\n");
|
|
2622
|
+
if (/^\d+$/.test(lines[0]?.trim()) && lines[1]?.includes("-->")) {
|
|
2623
|
+
lines[0] = String(index + 1);
|
|
2624
|
+
}
|
|
2625
|
+
return lines.join("\n");
|
|
2626
|
+
});
|
|
2627
|
+
return buildVttFromCueBlocks(scopedCueBlocks, preamble);
|
|
2628
|
+
}
|
|
2515
2629
|
function splitVttPreambleAndCueBlocks(vttContent) {
|
|
2516
2630
|
const normalizedContent = normalizeLineEndings(vttContent).trim();
|
|
2517
2631
|
if (!normalizedContent) {
|
|
@@ -2616,6 +2730,7 @@ async function fetchTranscriptForAsset(asset, playbackId, options = {}) {
|
|
|
2616
2730
|
const {
|
|
2617
2731
|
languageCode,
|
|
2618
2732
|
cleanTranscript = true,
|
|
2733
|
+
scope,
|
|
2619
2734
|
shouldSign,
|
|
2620
2735
|
credentials,
|
|
2621
2736
|
required = false
|
|
@@ -2647,9 +2762,15 @@ async function fetchTranscriptForAsset(asset, playbackId, options = {}) {
|
|
|
2647
2762
|
return { transcriptText: "", transcriptUrl, track };
|
|
2648
2763
|
}
|
|
2649
2764
|
const rawVtt = await response.text();
|
|
2650
|
-
const
|
|
2651
|
-
|
|
2652
|
-
|
|
2765
|
+
const effectiveScope = hasWorkflowScopeBoundaries(scope) ? scope : void 0;
|
|
2766
|
+
const scopedVtt = filterVttByScope(rawVtt, effectiveScope);
|
|
2767
|
+
const transcriptText = cleanTranscript ? extractTextFromVTT(scopedVtt) : scopedVtt;
|
|
2768
|
+
const hasScopedCues = !effectiveScope || parseVTTCues(scopedVtt).length > 0;
|
|
2769
|
+
if (required && (!transcriptText.trim() || !hasScopedCues)) {
|
|
2770
|
+
throw new MuxAiError(
|
|
2771
|
+
effectiveScope ? "Transcript has no cues in the requested scope." : "Transcript is empty.",
|
|
2772
|
+
{ type: "validation_error" }
|
|
2773
|
+
);
|
|
2653
2774
|
}
|
|
2654
2775
|
return { transcriptText, transcriptUrl, track };
|
|
2655
2776
|
} catch (error) {
|
|
@@ -2664,6 +2785,9 @@ async function fetchTranscriptForAsset(asset, playbackId, options = {}) {
|
|
|
2664
2785
|
// src/workflows/index.ts
|
|
2665
2786
|
var workflows_exports = {};
|
|
2666
2787
|
__export(workflows_exports, {
|
|
2788
|
+
ASK_QUESTIONS_DEFAULT_MAX_ANSWER_OPTION_LENGTH: () => ASK_QUESTIONS_DEFAULT_MAX_ANSWER_OPTION_LENGTH,
|
|
2789
|
+
ASK_QUESTIONS_DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH: () => ASK_QUESTIONS_DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH,
|
|
2790
|
+
ASK_QUESTIONS_MAX_QUESTION_LENGTH: () => ASK_QUESTIONS_MAX_QUESTION_LENGTH,
|
|
2667
2791
|
DEFAULT_DESCRIPTION_LENGTH: () => DEFAULT_DESCRIPTION_LENGTH,
|
|
2668
2792
|
DEFAULT_SUMMARY_KEYWORD_LIMIT: () => DEFAULT_SUMMARY_KEYWORD_LIMIT,
|
|
2669
2793
|
DEFAULT_TITLE_LENGTH: () => DEFAULT_TITLE_LENGTH,
|
|
@@ -2704,7 +2828,156 @@ __export(workflows_exports, {
|
|
|
2704
2828
|
// src/workflows/ask-questions.ts
|
|
2705
2829
|
import { generateText, Output } from "ai";
|
|
2706
2830
|
import dedent2 from "dedent";
|
|
2831
|
+
import { z as z3 } from "zod";
|
|
2832
|
+
|
|
2833
|
+
// src/lib/content-policy-error.ts
|
|
2834
|
+
import { APICallError, NoObjectGeneratedError, RetryError } from "ai";
|
|
2707
2835
|
import { z as z2 } from "zod";
|
|
2836
|
+
var PolicyTokenSchema = z2.string().trim().min(1).max(64).regex(/^[A-Z0-9_]+$/);
|
|
2837
|
+
var SafetyRatingSchema = z2.object({
|
|
2838
|
+
category: PolicyTokenSchema.nullish(),
|
|
2839
|
+
blocked: z2.boolean().nullish()
|
|
2840
|
+
});
|
|
2841
|
+
var ContentPolicyResponseSchema = z2.object({
|
|
2842
|
+
promptFeedback: z2.object({
|
|
2843
|
+
blockReason: PolicyTokenSchema.nullish(),
|
|
2844
|
+
safetyRatings: z2.array(SafetyRatingSchema).max(32).nullish()
|
|
2845
|
+
}).nullish(),
|
|
2846
|
+
candidates: z2.array(z2.object({
|
|
2847
|
+
finishReason: PolicyTokenSchema.nullish(),
|
|
2848
|
+
safetyRatings: z2.array(SafetyRatingSchema).max(32).nullish()
|
|
2849
|
+
})).max(8).nullish()
|
|
2850
|
+
});
|
|
2851
|
+
var ContentPolicyUsageResponseSchema = z2.object({
|
|
2852
|
+
usageMetadata: z2.object({
|
|
2853
|
+
promptTokenCount: z2.number().finite().nonnegative().nullish(),
|
|
2854
|
+
candidatesTokenCount: z2.number().finite().nonnegative().nullish(),
|
|
2855
|
+
totalTokenCount: z2.number().finite().nonnegative().nullish(),
|
|
2856
|
+
cachedContentTokenCount: z2.number().finite().nonnegative().nullish(),
|
|
2857
|
+
thoughtsTokenCount: z2.number().finite().nonnegative().nullish()
|
|
2858
|
+
}).nullish()
|
|
2859
|
+
});
|
|
2860
|
+
var CONTENT_POLICY_REASONS = /* @__PURE__ */ new Set([
|
|
2861
|
+
"BLOCKLIST",
|
|
2862
|
+
"ESCALATION",
|
|
2863
|
+
"IMAGE_PROHIBITED_CONTENT",
|
|
2864
|
+
"IMAGE_RECITATION",
|
|
2865
|
+
"IMAGE_SAFETY",
|
|
2866
|
+
"JAILBREAK",
|
|
2867
|
+
"MODEL_ARMOR",
|
|
2868
|
+
"PROHIBITED_CONTENT",
|
|
2869
|
+
"RECITATION",
|
|
2870
|
+
"SAFETY",
|
|
2871
|
+
"SPII"
|
|
2872
|
+
]);
|
|
2873
|
+
async function withContentPolicyErrorHandling(operation) {
|
|
2874
|
+
try {
|
|
2875
|
+
return await operation();
|
|
2876
|
+
} catch (error) {
|
|
2877
|
+
rethrowContentPolicyError(error);
|
|
2878
|
+
}
|
|
2879
|
+
}
|
|
2880
|
+
function rethrowContentPolicyError(error) {
|
|
2881
|
+
const block = extractContentPolicyBlock(error);
|
|
2882
|
+
if (!block) {
|
|
2883
|
+
throw error;
|
|
2884
|
+
}
|
|
2885
|
+
throwContentPolicyError(block, getContentPolicyTokenUsage(error));
|
|
2886
|
+
}
|
|
2887
|
+
function getGeneratedOutputWithContentPolicyHandling(response) {
|
|
2888
|
+
if (response.finishReason === "content-filter") {
|
|
2889
|
+
const rawReason = PolicyTokenSchema.safeParse(response.rawFinishReason);
|
|
2890
|
+
const reason = rawReason.success && isContentPolicyReason(rawReason.data) ? rawReason.data : "CONTENT_FILTER";
|
|
2891
|
+
throwContentPolicyError({ reason }, getErrorTokenUsage(response));
|
|
2892
|
+
}
|
|
2893
|
+
return response.output;
|
|
2894
|
+
}
|
|
2895
|
+
function throwContentPolicyError(block, usage) {
|
|
2896
|
+
const category = block.category ? `; category: ${block.category}` : "";
|
|
2897
|
+
const contentPolicyError = new MuxAiError(
|
|
2898
|
+
`The supplied content was blocked by a content policy (reason: ${block.reason}${category}).`,
|
|
2899
|
+
{
|
|
2900
|
+
type: "content_policy_error",
|
|
2901
|
+
retryable: false
|
|
2902
|
+
}
|
|
2903
|
+
);
|
|
2904
|
+
if (usage) {
|
|
2905
|
+
contentPolicyError.usage = usage;
|
|
2906
|
+
}
|
|
2907
|
+
throw contentPolicyError;
|
|
2908
|
+
}
|
|
2909
|
+
function extractContentPolicyBlock(error) {
|
|
2910
|
+
if (RetryError.isInstance(error)) {
|
|
2911
|
+
return extractContentPolicyBlock(error.lastError);
|
|
2912
|
+
}
|
|
2913
|
+
if (NoObjectGeneratedError.isInstance(error) && error.finishReason === "content-filter") {
|
|
2914
|
+
return { reason: "CONTENT_FILTER" };
|
|
2915
|
+
}
|
|
2916
|
+
if (!APICallError.isInstance(error) || !error.responseBody) {
|
|
2917
|
+
return void 0;
|
|
2918
|
+
}
|
|
2919
|
+
try {
|
|
2920
|
+
const parsed = ContentPolicyResponseSchema.safeParse(JSON.parse(error.responseBody));
|
|
2921
|
+
if (!parsed.success) {
|
|
2922
|
+
return void 0;
|
|
2923
|
+
}
|
|
2924
|
+
const promptReason = parsed.data.promptFeedback?.blockReason;
|
|
2925
|
+
if (isContentPolicyReason(promptReason)) {
|
|
2926
|
+
return {
|
|
2927
|
+
reason: promptReason,
|
|
2928
|
+
category: findBlockedCategory(parsed.data.promptFeedback?.safetyRatings)
|
|
2929
|
+
};
|
|
2930
|
+
}
|
|
2931
|
+
for (const candidate of parsed.data.candidates ?? []) {
|
|
2932
|
+
if (isContentPolicyReason(candidate.finishReason)) {
|
|
2933
|
+
return {
|
|
2934
|
+
reason: candidate.finishReason,
|
|
2935
|
+
category: findBlockedCategory(candidate.safetyRatings)
|
|
2936
|
+
};
|
|
2937
|
+
}
|
|
2938
|
+
}
|
|
2939
|
+
} catch {
|
|
2940
|
+
return void 0;
|
|
2941
|
+
}
|
|
2942
|
+
return void 0;
|
|
2943
|
+
}
|
|
2944
|
+
function isContentPolicyReason(reason) {
|
|
2945
|
+
return Boolean(reason && CONTENT_POLICY_REASONS.has(reason));
|
|
2946
|
+
}
|
|
2947
|
+
function getContentPolicyTokenUsage(error) {
|
|
2948
|
+
const errorUsage = getErrorTokenUsage(error);
|
|
2949
|
+
if (errorUsage) {
|
|
2950
|
+
return errorUsage;
|
|
2951
|
+
}
|
|
2952
|
+
if (RetryError.isInstance(error)) {
|
|
2953
|
+
return getContentPolicyTokenUsage(error.lastError);
|
|
2954
|
+
}
|
|
2955
|
+
if (!APICallError.isInstance(error) || !error.responseBody) {
|
|
2956
|
+
return void 0;
|
|
2957
|
+
}
|
|
2958
|
+
try {
|
|
2959
|
+
const parsed = ContentPolicyUsageResponseSchema.safeParse(JSON.parse(error.responseBody));
|
|
2960
|
+
const usage = parsed.success ? parsed.data.usageMetadata : void 0;
|
|
2961
|
+
if (!usage) {
|
|
2962
|
+
return void 0;
|
|
2963
|
+
}
|
|
2964
|
+
const inputTokens = usage.promptTokenCount ?? 0;
|
|
2965
|
+
const reasoningTokens = usage.thoughtsTokenCount ?? 0;
|
|
2966
|
+
const outputTokens = (usage.candidatesTokenCount ?? 0) + reasoningTokens;
|
|
2967
|
+
return {
|
|
2968
|
+
inputTokens,
|
|
2969
|
+
outputTokens,
|
|
2970
|
+
totalTokens: usage.totalTokenCount ?? inputTokens + outputTokens,
|
|
2971
|
+
reasoningTokens,
|
|
2972
|
+
cachedInputTokens: usage.cachedContentTokenCount ?? 0
|
|
2973
|
+
};
|
|
2974
|
+
} catch {
|
|
2975
|
+
return void 0;
|
|
2976
|
+
}
|
|
2977
|
+
}
|
|
2978
|
+
function findBlockedCategory(ratings) {
|
|
2979
|
+
return ratings?.find((rating) => rating.blocked === true)?.category ?? void 0;
|
|
2980
|
+
}
|
|
2708
2981
|
|
|
2709
2982
|
// src/lib/image-download.ts
|
|
2710
2983
|
import pRetry, { AbortError } from "p-retry";
|
|
@@ -2918,33 +3191,37 @@ async function withRetry(fn, {
|
|
|
2918
3191
|
try {
|
|
2919
3192
|
return await fn();
|
|
2920
3193
|
} catch (error) {
|
|
2921
|
-
lastError = error
|
|
3194
|
+
lastError = error;
|
|
3195
|
+
const retryError = error instanceof Error ? error : new Error(String(error));
|
|
2922
3196
|
const isLastAttempt = attempt === maxRetries;
|
|
2923
|
-
if (isLastAttempt || !shouldRetry(
|
|
2924
|
-
throw
|
|
3197
|
+
if (isLastAttempt || !shouldRetry(retryError, attempt + 1)) {
|
|
3198
|
+
throw error;
|
|
2925
3199
|
}
|
|
2926
3200
|
const delay = calculateDelay(attempt + 1, baseDelay, maxDelay);
|
|
2927
3201
|
console.warn(
|
|
2928
|
-
`Attempt ${attempt + 1} failed: ${
|
|
3202
|
+
`Attempt ${attempt + 1} failed: ${retryError.message}. Retrying in ${Math.round(delay)}ms...`
|
|
2929
3203
|
);
|
|
2930
3204
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
2931
3205
|
}
|
|
2932
3206
|
}
|
|
2933
|
-
throw lastError
|
|
3207
|
+
throw lastError ?? new Error("Retry failed with unknown error");
|
|
2934
3208
|
}
|
|
2935
3209
|
|
|
2936
3210
|
// src/workflows/ask-questions.ts
|
|
2937
|
-
var
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
|
|
2941
|
-
|
|
2942
|
-
|
|
3211
|
+
var ASK_QUESTIONS_MAX_QUESTION_LENGTH = 600;
|
|
3212
|
+
var ASK_QUESTIONS_DEFAULT_MAX_ANSWER_OPTION_LENGTH = 150;
|
|
3213
|
+
var ASK_QUESTIONS_DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH = 500;
|
|
3214
|
+
var questionAnswerSchema = z3.object({
|
|
3215
|
+
question: z3.string().describe("The full text of the original question"),
|
|
3216
|
+
answer: z3.string().nullable(),
|
|
3217
|
+
confidence: z3.number(),
|
|
3218
|
+
reasoning: z3.string().max(1e3),
|
|
3219
|
+
skipped: z3.boolean()
|
|
2943
3220
|
});
|
|
2944
3221
|
var SKIP_SENTINEL = "__SKIPPED__";
|
|
2945
3222
|
function createAskQuestionsSchema(answerSchema) {
|
|
2946
|
-
return
|
|
2947
|
-
answers:
|
|
3223
|
+
return z3.object({
|
|
3224
|
+
answers: z3.array(
|
|
2948
3225
|
questionAnswerSchema.extend({
|
|
2949
3226
|
answer: answerSchema
|
|
2950
3227
|
})
|
|
@@ -3294,12 +3571,12 @@ function buildResponseSchemaForQuestions(normalizedQuestions, maxFreeFormAnswerL
|
|
|
3294
3571
|
SKIP_SENTINEL.length,
|
|
3295
3572
|
longestConstrained
|
|
3296
3573
|
);
|
|
3297
|
-
return createAskQuestionsSchema(
|
|
3574
|
+
return createAskQuestionsSchema(z3.string().min(1).max(schemaMax));
|
|
3298
3575
|
}
|
|
3299
3576
|
const allowed = Array.from(
|
|
3300
3577
|
new Set(normalizedQuestions.flatMap(allowedAnswersForQuestion))
|
|
3301
3578
|
);
|
|
3302
|
-
return createAskQuestionsSchema(
|
|
3579
|
+
return createAskQuestionsSchema(z3.enum([...allowed, SKIP_SENTINEL]));
|
|
3303
3580
|
}
|
|
3304
3581
|
async function analyzeQuestions({
|
|
3305
3582
|
provider,
|
|
@@ -3317,7 +3594,7 @@ async function analyzeQuestions({
|
|
|
3317
3594
|
normalizedQuestions,
|
|
3318
3595
|
maxFreeFormAnswerLength
|
|
3319
3596
|
);
|
|
3320
|
-
const response = await generateText({
|
|
3597
|
+
const response = await withContentPolicyErrorHandling(() => generateText({
|
|
3321
3598
|
model,
|
|
3322
3599
|
output: Output.object({ schema: responseSchema }),
|
|
3323
3600
|
experimental_telemetry: { isEnabled: true },
|
|
@@ -3334,11 +3611,12 @@ async function analyzeQuestions({
|
|
|
3334
3611
|
] : userPrompt
|
|
3335
3612
|
}
|
|
3336
3613
|
]
|
|
3337
|
-
});
|
|
3338
|
-
|
|
3614
|
+
}));
|
|
3615
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
3616
|
+
if (!output) {
|
|
3339
3617
|
throw new Error("Ask-questions output missing");
|
|
3340
3618
|
}
|
|
3341
|
-
const parsed = responseSchema.parse(
|
|
3619
|
+
const parsed = responseSchema.parse(output);
|
|
3342
3620
|
const unexpectedRootKeys = detectUnexpectedKeysFromRawText(
|
|
3343
3621
|
response.text,
|
|
3344
3622
|
responseSchema.keyof().options
|
|
@@ -3360,7 +3638,8 @@ async function analyzeQuestions({
|
|
|
3360
3638
|
outputTokens: response.usage.outputTokens,
|
|
3361
3639
|
totalTokens: response.usage.totalTokens,
|
|
3362
3640
|
reasoningTokens: response.usage.reasoningTokens,
|
|
3363
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
3641
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
3642
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
3364
3643
|
},
|
|
3365
3644
|
unexpectedRootKeys,
|
|
3366
3645
|
unexpectedAnswerKeys
|
|
@@ -3379,7 +3658,6 @@ async function askQuestionsInternal(assetId, questions, options, collectedUsage)
|
|
|
3379
3658
|
if (!questions || questions.length === 0) {
|
|
3380
3659
|
throw new MuxAiError("At least one question must be provided.", { type: "validation_error" });
|
|
3381
3660
|
}
|
|
3382
|
-
const MAX_QUESTION_LENGTH = 500;
|
|
3383
3661
|
questions.forEach((q, idx) => {
|
|
3384
3662
|
if (!q.question || typeof q.question !== "string" || !q.question.trim()) {
|
|
3385
3663
|
throw new MuxAiError(
|
|
@@ -3387,15 +3665,14 @@ async function askQuestionsInternal(assetId, questions, options, collectedUsage)
|
|
|
3387
3665
|
{ type: "validation_error" }
|
|
3388
3666
|
);
|
|
3389
3667
|
}
|
|
3390
|
-
if (q.question.length >
|
|
3668
|
+
if (q.question.length > ASK_QUESTIONS_MAX_QUESTION_LENGTH) {
|
|
3391
3669
|
throw new MuxAiError(
|
|
3392
|
-
`Question at index ${idx} exceeds the ${
|
|
3670
|
+
`Question at index ${idx} exceeds the ${ASK_QUESTIONS_MAX_QUESTION_LENGTH}-character limit (received ${q.question.length}).`,
|
|
3393
3671
|
{ type: "validation_error" }
|
|
3394
3672
|
);
|
|
3395
3673
|
}
|
|
3396
3674
|
});
|
|
3397
|
-
const
|
|
3398
|
-
const maxAnswerOptionLength = options?.maxAnswerOptionLength ?? DEFAULT_MAX_ANSWER_OPTION_LENGTH;
|
|
3675
|
+
const maxAnswerOptionLength = options?.maxAnswerOptionLength ?? ASK_QUESTIONS_DEFAULT_MAX_ANSWER_OPTION_LENGTH;
|
|
3399
3676
|
if (!Number.isFinite(maxAnswerOptionLength) || maxAnswerOptionLength <= 0) {
|
|
3400
3677
|
throw new MuxAiError(
|
|
3401
3678
|
`maxAnswerOptionLength must be a positive number (received ${maxAnswerOptionLength}).`,
|
|
@@ -3414,8 +3691,7 @@ async function askQuestionsInternal(assetId, questions, options, collectedUsage)
|
|
|
3414
3691
|
}
|
|
3415
3692
|
}
|
|
3416
3693
|
});
|
|
3417
|
-
const
|
|
3418
|
-
const maxFreeFormAnswerLength = options?.maxFreeFormAnswerLength ?? DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH;
|
|
3694
|
+
const maxFreeFormAnswerLength = options?.maxFreeFormAnswerLength ?? ASK_QUESTIONS_DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH;
|
|
3419
3695
|
if (!Number.isFinite(maxFreeFormAnswerLength) || maxFreeFormAnswerLength <= 0) {
|
|
3420
3696
|
throw new MuxAiError(
|
|
3421
3697
|
`maxFreeFormAnswerLength must be a positive number (received ${maxFreeFormAnswerLength}).`,
|
|
@@ -3431,7 +3707,8 @@ async function askQuestionsInternal(assetId, questions, options, collectedUsage)
|
|
|
3431
3707
|
imageSubmissionMode = "url",
|
|
3432
3708
|
imageDownloadOptions,
|
|
3433
3709
|
storyboardWidth = 640,
|
|
3434
|
-
credentials
|
|
3710
|
+
credentials,
|
|
3711
|
+
scope
|
|
3435
3712
|
} = options ?? {};
|
|
3436
3713
|
const normalizedQuestions = questions.map((q, idx) => normalizeQuestion(q, idx));
|
|
3437
3714
|
const hasFreeForm = normalizedQuestions.some((q) => q.mode.kind === "freeForm");
|
|
@@ -3443,6 +3720,15 @@ async function askQuestionsInternal(assetId, questions, options, collectedUsage)
|
|
|
3443
3720
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
3444
3721
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
3445
3722
|
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
3723
|
+
const effectiveScope = hasWorkflowScopeBoundaries(scope) ? scope : void 0;
|
|
3724
|
+
const storyboardScope = isAudioOnly ? void 0 : resolveRenderableVideoScope(
|
|
3725
|
+
effectiveScope,
|
|
3726
|
+
assetDurationSeconds,
|
|
3727
|
+
getVideoTrackDurationSecondsFromAsset(assetData)
|
|
3728
|
+
);
|
|
3729
|
+
if (isAudioOnly && effectiveScope) {
|
|
3730
|
+
resolveWorkflowScope(effectiveScope, assetDurationSeconds);
|
|
3731
|
+
}
|
|
3446
3732
|
if (isAudioOnly && !includeTranscript) {
|
|
3447
3733
|
throw new MuxAiError(
|
|
3448
3734
|
"Audio-only assets require a transcript. Set includeTranscript: true and ensure the asset has a ready text track (captions/subtitles).",
|
|
@@ -3461,7 +3747,8 @@ async function askQuestionsInternal(assetId, questions, options, collectedUsage)
|
|
|
3461
3747
|
cleanTranscript,
|
|
3462
3748
|
shouldSign: policy === "signed",
|
|
3463
3749
|
credentials,
|
|
3464
|
-
required: isAudioOnly
|
|
3750
|
+
required: isAudioOnly,
|
|
3751
|
+
scope: effectiveScope
|
|
3465
3752
|
}) : void 0;
|
|
3466
3753
|
const transcriptText = transcriptResult?.transcriptText ?? "";
|
|
3467
3754
|
const userPrompt = buildUserPrompt({
|
|
@@ -3490,7 +3777,8 @@ async function askQuestionsInternal(assetId, questions, options, collectedUsage)
|
|
|
3490
3777
|
playbackId,
|
|
3491
3778
|
storyboardWidth,
|
|
3492
3779
|
policy === "signed",
|
|
3493
|
-
credentials
|
|
3780
|
+
credentials,
|
|
3781
|
+
storyboardScope
|
|
3494
3782
|
);
|
|
3495
3783
|
imageUrl = storyboardUrl;
|
|
3496
3784
|
if (imageSubmissionMode === "base64") {
|
|
@@ -3605,11 +3893,11 @@ async function askQuestionsInternal(assetId, questions, options, collectedUsage)
|
|
|
3605
3893
|
// src/workflows/burned-in-captions.ts
|
|
3606
3894
|
import { generateText as generateText2, Output as Output2 } from "ai";
|
|
3607
3895
|
import dedent3 from "dedent";
|
|
3608
|
-
import { z as
|
|
3609
|
-
var burnedInCaptionsSchema =
|
|
3610
|
-
hasBurnedInCaptions:
|
|
3611
|
-
confidence:
|
|
3612
|
-
detectedLanguage:
|
|
3896
|
+
import { z as z4 } from "zod";
|
|
3897
|
+
var burnedInCaptionsSchema = z4.object({
|
|
3898
|
+
hasBurnedInCaptions: z4.boolean(),
|
|
3899
|
+
confidence: z4.number(),
|
|
3900
|
+
detectedLanguage: z4.string().max(100).nullable()
|
|
3613
3901
|
});
|
|
3614
3902
|
var SYSTEM_PROMPT2 = promptDedent`
|
|
3615
3903
|
<role>
|
|
@@ -3719,7 +4007,7 @@ async function analyzeStoryboard({
|
|
|
3719
4007
|
}) {
|
|
3720
4008
|
"use step";
|
|
3721
4009
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
3722
|
-
const response = await generateText2({
|
|
4010
|
+
const response = await withContentPolicyErrorHandling(() => generateText2({
|
|
3723
4011
|
model,
|
|
3724
4012
|
output: Output2.object({ schema: burnedInCaptionsSchema }),
|
|
3725
4013
|
experimental_telemetry: { isEnabled: true },
|
|
@@ -3736,22 +4024,24 @@ async function analyzeStoryboard({
|
|
|
3736
4024
|
]
|
|
3737
4025
|
}
|
|
3738
4026
|
]
|
|
3739
|
-
});
|
|
4027
|
+
}));
|
|
4028
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
3740
4029
|
const unexpectedKeys = detectUnexpectedKeysFromRawText(
|
|
3741
4030
|
response.text,
|
|
3742
4031
|
burnedInCaptionsSchema.keyof().options
|
|
3743
4032
|
);
|
|
3744
4033
|
return {
|
|
3745
4034
|
result: {
|
|
3746
|
-
...
|
|
3747
|
-
confidence: Math.min(1, Math.max(0,
|
|
4035
|
+
...output,
|
|
4036
|
+
confidence: Math.min(1, Math.max(0, output.confidence))
|
|
3748
4037
|
},
|
|
3749
4038
|
usage: {
|
|
3750
4039
|
inputTokens: response.usage.inputTokens,
|
|
3751
4040
|
outputTokens: response.usage.outputTokens,
|
|
3752
4041
|
totalTokens: response.usage.totalTokens,
|
|
3753
4042
|
reasoningTokens: response.usage.reasoningTokens,
|
|
3754
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
4043
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
4044
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
3755
4045
|
},
|
|
3756
4046
|
unexpectedKeys
|
|
3757
4047
|
};
|
|
@@ -3773,6 +4063,7 @@ async function hasBurnedInCaptionsInternal(assetId, options, collectedUsage) {
|
|
|
3773
4063
|
imageDownloadOptions,
|
|
3774
4064
|
promptOverrides,
|
|
3775
4065
|
credentials,
|
|
4066
|
+
scope,
|
|
3776
4067
|
...config
|
|
3777
4068
|
} = options;
|
|
3778
4069
|
const userPrompt = buildUserPrompt2(promptOverrides);
|
|
@@ -3783,7 +4074,18 @@ async function hasBurnedInCaptionsInternal(assetId, options, collectedUsage) {
|
|
|
3783
4074
|
});
|
|
3784
4075
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
3785
4076
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
3786
|
-
const
|
|
4077
|
+
const storyboardScope = resolveRenderableVideoScope(
|
|
4078
|
+
scope,
|
|
4079
|
+
assetDurationSeconds,
|
|
4080
|
+
getVideoTrackDurationSecondsFromAsset(assetData)
|
|
4081
|
+
);
|
|
4082
|
+
const imageUrl = await getStoryboardUrl(
|
|
4083
|
+
playbackId,
|
|
4084
|
+
640,
|
|
4085
|
+
policy === "signed",
|
|
4086
|
+
credentials,
|
|
4087
|
+
storyboardScope
|
|
4088
|
+
);
|
|
3787
4089
|
let analysisResponse;
|
|
3788
4090
|
if (imageSubmissionMode === "base64") {
|
|
3789
4091
|
const base64Data = await fetchImageAsBase642(imageUrl, imageDownloadOptions);
|
|
@@ -3837,13 +4139,13 @@ async function hasBurnedInCaptionsInternal(assetId, options, collectedUsage) {
|
|
|
3837
4139
|
// src/workflows/chapters.ts
|
|
3838
4140
|
import { generateText as generateText3, Output as Output3 } from "ai";
|
|
3839
4141
|
import dedent4 from "dedent";
|
|
3840
|
-
import { z as
|
|
3841
|
-
var chapterSchema =
|
|
3842
|
-
startTime:
|
|
3843
|
-
title:
|
|
4142
|
+
import { z as z5 } from "zod";
|
|
4143
|
+
var chapterSchema = z5.object({
|
|
4144
|
+
startTime: z5.number(),
|
|
4145
|
+
title: z5.string().max(300)
|
|
3844
4146
|
});
|
|
3845
|
-
var chaptersSchema =
|
|
3846
|
-
chapters:
|
|
4147
|
+
var chaptersSchema = z5.object({
|
|
4148
|
+
chapters: z5.array(chapterSchema)
|
|
3847
4149
|
});
|
|
3848
4150
|
async function generateChaptersWithAI({
|
|
3849
4151
|
provider,
|
|
@@ -3855,21 +4157,24 @@ async function generateChaptersWithAI({
|
|
|
3855
4157
|
"use step";
|
|
3856
4158
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
3857
4159
|
const response = await withRetry(
|
|
3858
|
-
() =>
|
|
3859
|
-
|
|
3860
|
-
|
|
3861
|
-
|
|
3862
|
-
|
|
3863
|
-
|
|
3864
|
-
|
|
3865
|
-
|
|
3866
|
-
|
|
3867
|
-
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
|
|
3871
|
-
|
|
4160
|
+
() => withContentPolicyErrorHandling(
|
|
4161
|
+
() => generateText3({
|
|
4162
|
+
model,
|
|
4163
|
+
output: Output3.object({ schema: chaptersSchema }),
|
|
4164
|
+
messages: [
|
|
4165
|
+
{
|
|
4166
|
+
role: "system",
|
|
4167
|
+
content: systemPrompt
|
|
4168
|
+
},
|
|
4169
|
+
{
|
|
4170
|
+
role: "user",
|
|
4171
|
+
content: userPrompt
|
|
4172
|
+
}
|
|
4173
|
+
]
|
|
4174
|
+
})
|
|
4175
|
+
)
|
|
3872
4176
|
);
|
|
4177
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
3873
4178
|
const unexpectedRootKeys = detectUnexpectedKeysFromRawText(
|
|
3874
4179
|
response.text,
|
|
3875
4180
|
chaptersSchema.keyof().options
|
|
@@ -3885,13 +4190,14 @@ async function generateChaptersWithAI({
|
|
|
3885
4190
|
} catch {
|
|
3886
4191
|
}
|
|
3887
4192
|
return {
|
|
3888
|
-
chapters:
|
|
4193
|
+
chapters: output,
|
|
3889
4194
|
usage: {
|
|
3890
4195
|
inputTokens: response.usage.inputTokens,
|
|
3891
4196
|
outputTokens: response.usage.outputTokens,
|
|
3892
4197
|
totalTokens: response.usage.totalTokens,
|
|
3893
4198
|
reasoningTokens: response.usage.reasoningTokens,
|
|
3894
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
4199
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
4200
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
3895
4201
|
},
|
|
3896
4202
|
unexpectedRootKeys,
|
|
3897
4203
|
unexpectedChapterKeys
|
|
@@ -3982,7 +4288,8 @@ function buildUserPrompt3({
|
|
|
3982
4288
|
promptOverrides,
|
|
3983
4289
|
minChaptersPerHour = 3,
|
|
3984
4290
|
maxChaptersPerHour = 8,
|
|
3985
|
-
languageName
|
|
4291
|
+
languageName,
|
|
4292
|
+
scope
|
|
3986
4293
|
}) {
|
|
3987
4294
|
const contextSections = [
|
|
3988
4295
|
{
|
|
@@ -3997,6 +4304,8 @@ function buildUserPrompt3({
|
|
|
3997
4304
|
const dynamicChapterGuidelines = dedent4`
|
|
3998
4305
|
- Create at least ${minChaptersPerHour} and at most ${maxChaptersPerHour} chapters per hour of content
|
|
3999
4306
|
- Use start times in seconds (not HH:MM:SS)
|
|
4307
|
+
${scope ? `- Analyze only the range from ${scope.startTime}s (inclusive) to ${scope.endTime}s (exclusive)
|
|
4308
|
+
- The first chapter must start at ${scope.startTime}s` : ""}
|
|
4000
4309
|
- Chapter start times should be non-decreasing
|
|
4001
4310
|
- Do not include text before or after the JSON`;
|
|
4002
4311
|
const mergedOverrides = {
|
|
@@ -4023,7 +4332,8 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
|
4023
4332
|
minChaptersPerHour,
|
|
4024
4333
|
maxChaptersPerHour,
|
|
4025
4334
|
credentials,
|
|
4026
|
-
outputLanguageCode
|
|
4335
|
+
outputLanguageCode,
|
|
4336
|
+
scope
|
|
4027
4337
|
} = options;
|
|
4028
4338
|
const modelConfig = resolveLanguageModelConfig({
|
|
4029
4339
|
...options,
|
|
@@ -4032,6 +4342,8 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
|
4032
4342
|
});
|
|
4033
4343
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
4034
4344
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
4345
|
+
const effectiveScope = hasWorkflowScopeBoundaries(scope) ? scope : void 0;
|
|
4346
|
+
const resolvedScope = effectiveScope ? resolveWorkflowScope(effectiveScope, assetDurationSeconds) : void 0;
|
|
4035
4347
|
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
4036
4348
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
4037
4349
|
if (policy === "signed" && !signingContext) {
|
|
@@ -4046,7 +4358,8 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
|
4046
4358
|
cleanTranscript: false,
|
|
4047
4359
|
// keep timestamps for chapter segmentation
|
|
4048
4360
|
shouldSign: policy === "signed",
|
|
4049
|
-
credentials
|
|
4361
|
+
credentials,
|
|
4362
|
+
scope: effectiveScope
|
|
4050
4363
|
});
|
|
4051
4364
|
if (!transcriptResult.track || !transcriptResult.transcriptText) {
|
|
4052
4365
|
const availableLanguages = readyTextTracks.map((t) => t.language_code).filter(Boolean).join(", ");
|
|
@@ -4067,7 +4380,8 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
|
4067
4380
|
promptOverrides,
|
|
4068
4381
|
minChaptersPerHour,
|
|
4069
4382
|
maxChaptersPerHour,
|
|
4070
|
-
languageName
|
|
4383
|
+
languageName,
|
|
4384
|
+
scope: resolvedScope
|
|
4071
4385
|
});
|
|
4072
4386
|
let chaptersData = null;
|
|
4073
4387
|
try {
|
|
@@ -4089,7 +4403,9 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
|
4089
4403
|
throw new MuxAiError(`Failed to generate chapters for asset ${assetId}.`);
|
|
4090
4404
|
}
|
|
4091
4405
|
const { chapters: chaptersPayload, usage } = chaptersData;
|
|
4092
|
-
const validChapters = chaptersPayload.chapters.filter((chapter) => typeof chapter.startTime === "number" && typeof chapter.title === "string").
|
|
4406
|
+
const validChapters = chaptersPayload.chapters.filter((chapter) => typeof chapter.startTime === "number" && typeof chapter.title === "string").filter(
|
|
4407
|
+
(chapter) => resolvedScope ? chapter.startTime >= resolvedScope.startTime && chapter.startTime < resolvedScope.endTime : true
|
|
4408
|
+
).sort((a, b) => a.startTime - b.startTime);
|
|
4093
4409
|
if (validChapters.length === 0) {
|
|
4094
4410
|
throw new MuxAiError(`Failed to generate valid chapters for asset ${assetId}.`);
|
|
4095
4411
|
}
|
|
@@ -4115,8 +4431,9 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
|
4115
4431
|
if (scrubbedChapters.length === 0) {
|
|
4116
4432
|
throw new MuxAiError(`Failed to generate valid chapters for asset ${assetId}.`);
|
|
4117
4433
|
}
|
|
4118
|
-
|
|
4119
|
-
|
|
4434
|
+
const firstChapterStartTime = resolvedScope?.startTime ?? 0;
|
|
4435
|
+
if (scrubbedChapters[0].startTime !== firstChapterStartTime) {
|
|
4436
|
+
scrubbedChapters[0].startTime = firstChapterStartTime;
|
|
4120
4437
|
}
|
|
4121
4438
|
const usageWithMetadata = {
|
|
4122
4439
|
...usage,
|
|
@@ -4136,7 +4453,7 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
|
4136
4453
|
|
|
4137
4454
|
// src/workflows/edit-captions.ts
|
|
4138
4455
|
import { generateText as generateText4, Output as Output4 } from "ai";
|
|
4139
|
-
import { z as
|
|
4456
|
+
import { z as z6 } from "zod";
|
|
4140
4457
|
|
|
4141
4458
|
// src/lib/mux-tracks.ts
|
|
4142
4459
|
async function fetchVttFromMux(vttUrl) {
|
|
@@ -4207,8 +4524,8 @@ async function createPresignedGetUrlWithStorageAdapter(input, adapter) {
|
|
|
4207
4524
|
}
|
|
4208
4525
|
|
|
4209
4526
|
// src/workflows/edit-captions.ts
|
|
4210
|
-
var profanityDetectionSchema =
|
|
4211
|
-
profanity:
|
|
4527
|
+
var profanityDetectionSchema = z6.object({
|
|
4528
|
+
profanity: z6.array(z6.string().max(100)).describe(
|
|
4212
4529
|
"Unique profane words or short phrases exactly as they appear in the transcript text. Include each distinct form only once (e.g., if 'fuck' and 'fucking' both appear, list both)."
|
|
4213
4530
|
)
|
|
4214
4531
|
});
|
|
@@ -4385,7 +4702,7 @@ async function identifyProfanityWithAI({
|
|
|
4385
4702
|
tag: "transcript",
|
|
4386
4703
|
content: plainText
|
|
4387
4704
|
});
|
|
4388
|
-
const response = await generateText4({
|
|
4705
|
+
const response = await withContentPolicyErrorHandling(() => generateText4({
|
|
4389
4706
|
model,
|
|
4390
4707
|
output: Output4.object({ schema: profanityDetectionSchema }),
|
|
4391
4708
|
messages: [
|
|
@@ -4400,19 +4717,21 @@ async function identifyProfanityWithAI({
|
|
|
4400
4717
|
${transcriptSection}`
|
|
4401
4718
|
}
|
|
4402
4719
|
]
|
|
4403
|
-
});
|
|
4720
|
+
}));
|
|
4721
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
4404
4722
|
const unexpectedKeys = detectUnexpectedKeysFromRawText(
|
|
4405
4723
|
response.text,
|
|
4406
4724
|
profanityDetectionSchema.keyof().options
|
|
4407
4725
|
);
|
|
4408
4726
|
return {
|
|
4409
|
-
profanity:
|
|
4727
|
+
profanity: output.profanity,
|
|
4410
4728
|
usage: {
|
|
4411
4729
|
inputTokens: response.usage.inputTokens,
|
|
4412
4730
|
outputTokens: response.usage.outputTokens,
|
|
4413
4731
|
totalTokens: response.usage.totalTokens,
|
|
4414
4732
|
reasoningTokens: response.usage.reasoningTokens,
|
|
4415
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
4733
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
4734
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
4416
4735
|
},
|
|
4417
4736
|
unexpectedKeys
|
|
4418
4737
|
};
|
|
@@ -4695,11 +5014,16 @@ async function generateEmbeddingsInternal(assetId, options, collectedUsage) {
|
|
|
4695
5014
|
languageCode,
|
|
4696
5015
|
chunkingStrategy = { type: "token", maxTokens: 500, overlap: 100 },
|
|
4697
5016
|
batchSize = 5,
|
|
4698
|
-
credentials
|
|
5017
|
+
credentials,
|
|
5018
|
+
scope
|
|
4699
5019
|
} = options;
|
|
4700
5020
|
const embeddingModel = resolveEmbeddingModelConfig({ ...options, provider, model });
|
|
4701
5021
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
4702
5022
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
5023
|
+
const effectiveScope = hasWorkflowScopeBoundaries(scope) ? scope : void 0;
|
|
5024
|
+
if (effectiveScope) {
|
|
5025
|
+
resolveWorkflowScope(effectiveScope, assetDurationSeconds);
|
|
5026
|
+
}
|
|
4703
5027
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
4704
5028
|
if (policy === "signed" && !signingContext) {
|
|
4705
5029
|
throw new Error(
|
|
@@ -4712,7 +5036,8 @@ async function generateEmbeddingsInternal(assetId, options, collectedUsage) {
|
|
|
4712
5036
|
cleanTranscript: !useVttChunking,
|
|
4713
5037
|
shouldSign: policy === "signed",
|
|
4714
5038
|
credentials,
|
|
4715
|
-
required: true
|
|
5039
|
+
required: true,
|
|
5040
|
+
scope: effectiveScope
|
|
4716
5041
|
});
|
|
4717
5042
|
const transcriptText = transcriptResult.transcriptText;
|
|
4718
5043
|
const chunks = useVttChunking ? chunkVTTCues(
|
|
@@ -4809,21 +5134,21 @@ async function generateVideoEmbeddings(assetId, options = {}) {
|
|
|
4809
5134
|
// src/workflows/engagement-insights.ts
|
|
4810
5135
|
import { generateText as generateText5, Output as Output5 } from "ai";
|
|
4811
5136
|
import dedent5 from "dedent";
|
|
4812
|
-
import { z as
|
|
4813
|
-
var aiMomentInsightSchema =
|
|
4814
|
-
hotspotIndex:
|
|
4815
|
-
startMs:
|
|
4816
|
-
endMs:
|
|
4817
|
-
timestamp:
|
|
4818
|
-
engagementScore:
|
|
4819
|
-
insight:
|
|
5137
|
+
import { z as z7 } from "zod";
|
|
5138
|
+
var aiMomentInsightSchema = z7.object({
|
|
5139
|
+
hotspotIndex: z7.number(),
|
|
5140
|
+
startMs: z7.number(),
|
|
5141
|
+
endMs: z7.number(),
|
|
5142
|
+
timestamp: z7.string(),
|
|
5143
|
+
engagementScore: z7.number(),
|
|
5144
|
+
insight: z7.string().max(1e3)
|
|
4820
5145
|
});
|
|
4821
|
-
var aiOverallInsightSchema =
|
|
4822
|
-
summary:
|
|
4823
|
-
trends:
|
|
5146
|
+
var aiOverallInsightSchema = z7.object({
|
|
5147
|
+
summary: z7.string().max(2e3),
|
|
5148
|
+
trends: z7.array(z7.string().max(500))
|
|
4824
5149
|
});
|
|
4825
|
-
var engagementInsightsSchema =
|
|
4826
|
-
momentInsights:
|
|
5150
|
+
var engagementInsightsSchema = z7.object({
|
|
5151
|
+
momentInsights: z7.array(aiMomentInsightSchema),
|
|
4827
5152
|
overallInsight: aiOverallInsightSchema
|
|
4828
5153
|
});
|
|
4829
5154
|
var SYSTEM_PROMPT4 = promptDedent`
|
|
@@ -5085,26 +5410,29 @@ async function generateInsightsWithAI(provider, modelId, systemPrompt, userPromp
|
|
|
5085
5410
|
"use step";
|
|
5086
5411
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
5087
5412
|
const response = await withRetry(
|
|
5088
|
-
() =>
|
|
5089
|
-
|
|
5090
|
-
|
|
5091
|
-
|
|
5092
|
-
|
|
5093
|
-
|
|
5094
|
-
|
|
5095
|
-
|
|
5096
|
-
|
|
5097
|
-
|
|
5098
|
-
|
|
5099
|
-
|
|
5100
|
-
|
|
5101
|
-
|
|
5102
|
-
|
|
5103
|
-
|
|
5104
|
-
|
|
5105
|
-
|
|
5413
|
+
() => withContentPolicyErrorHandling(
|
|
5414
|
+
() => generateText5({
|
|
5415
|
+
model,
|
|
5416
|
+
output: Output5.object({ schema: engagementInsightsSchema }),
|
|
5417
|
+
experimental_telemetry: { isEnabled: true },
|
|
5418
|
+
messages: [
|
|
5419
|
+
{
|
|
5420
|
+
role: "system",
|
|
5421
|
+
content: systemPrompt
|
|
5422
|
+
},
|
|
5423
|
+
{
|
|
5424
|
+
role: "user",
|
|
5425
|
+
content: [
|
|
5426
|
+
{ type: "text", text: userPrompt },
|
|
5427
|
+
...imageUrls.map((url) => ({ type: "image", image: url }))
|
|
5428
|
+
]
|
|
5429
|
+
}
|
|
5430
|
+
]
|
|
5431
|
+
})
|
|
5432
|
+
)
|
|
5106
5433
|
);
|
|
5107
|
-
|
|
5434
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
5435
|
+
if (!output) {
|
|
5108
5436
|
throw new Error("AI returned empty or unparseable response");
|
|
5109
5437
|
}
|
|
5110
5438
|
const unexpectedRootKeys = detectUnexpectedKeysFromRawText(
|
|
@@ -5127,13 +5455,14 @@ async function generateInsightsWithAI(provider, modelId, systemPrompt, userPromp
|
|
|
5127
5455
|
} catch {
|
|
5128
5456
|
}
|
|
5129
5457
|
return {
|
|
5130
|
-
result:
|
|
5458
|
+
result: output,
|
|
5131
5459
|
usage: {
|
|
5132
5460
|
inputTokens: response.usage.inputTokens,
|
|
5133
5461
|
outputTokens: response.usage.outputTokens,
|
|
5134
5462
|
totalTokens: response.usage.totalTokens,
|
|
5135
5463
|
reasoningTokens: response.usage.reasoningTokens,
|
|
5136
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
5464
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
5465
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
5137
5466
|
},
|
|
5138
5467
|
unexpectedRootKeys,
|
|
5139
5468
|
unexpectedMomentKeys,
|
|
@@ -5843,12 +6172,24 @@ async function requestGoogleVisionModeration(images, maxConcurrent = 5, submissi
|
|
|
5843
6172
|
}));
|
|
5844
6173
|
return processConcurrently(targets, moderateImageWithGoogleVision, maxConcurrent);
|
|
5845
6174
|
}
|
|
6175
|
+
function toThumbnailTimeSeconds(timestampMs) {
|
|
6176
|
+
return Number((timestampMs / 1e3).toFixed(2));
|
|
6177
|
+
}
|
|
6178
|
+
function filterTimestampsWithinExclusiveScope(timestampsMs, scope) {
|
|
6179
|
+
if (!scope) {
|
|
6180
|
+
return timestampsMs;
|
|
6181
|
+
}
|
|
6182
|
+
return timestampsMs.filter((timestampMs) => {
|
|
6183
|
+
const time = toThumbnailTimeSeconds(timestampMs);
|
|
6184
|
+
return time >= scope.startTime && time < scope.endTime;
|
|
6185
|
+
});
|
|
6186
|
+
}
|
|
5846
6187
|
async function getThumbnailUrlsFromTimestamps(playbackId, timestampsMs, options) {
|
|
5847
6188
|
"use step";
|
|
5848
6189
|
const { width, shouldSign, credentials } = options;
|
|
5849
6190
|
const baseUrl = getMuxThumbnailBaseUrl(playbackId);
|
|
5850
6191
|
const urlPromises = timestampsMs.map(async (tsMs) => {
|
|
5851
|
-
const time =
|
|
6192
|
+
const time = toThumbnailTimeSeconds(tsMs);
|
|
5852
6193
|
const url = shouldSign ? await signUrl(baseUrl, playbackId, "thumbnail", { time, width }, credentials) : `${baseUrl}?time=${time}&width=${width}`;
|
|
5853
6194
|
return { url, time };
|
|
5854
6195
|
});
|
|
@@ -5867,7 +6208,8 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
5867
6208
|
maxConcurrent = 5,
|
|
5868
6209
|
imageSubmissionMode = "url",
|
|
5869
6210
|
imageDownloadOptions,
|
|
5870
|
-
credentials: providedCredentials
|
|
6211
|
+
credentials: providedCredentials,
|
|
6212
|
+
scope
|
|
5871
6213
|
} = options;
|
|
5872
6214
|
const credentials = providedCredentials;
|
|
5873
6215
|
const { asset, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials);
|
|
@@ -5878,7 +6220,16 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
5878
6220
|
(d) => d != null
|
|
5879
6221
|
);
|
|
5880
6222
|
const duration = candidateDurations.length > 0 ? Math.min(...candidateDurations) : 0;
|
|
6223
|
+
const effectiveScope = hasWorkflowScopeBoundaries(scope) ? scope : void 0;
|
|
6224
|
+
const resolvedScope = effectiveScope ? resolveWorkflowScope(effectiveScope, assetDurationSeconds) : void 0;
|
|
5881
6225
|
const isAudioOnly = isAudioOnlyAsset(asset);
|
|
6226
|
+
const renderableScope = resolvedScope ? {
|
|
6227
|
+
startTime: Math.min(resolvedScope.startTime, duration),
|
|
6228
|
+
endTime: Math.min(resolvedScope.endTime, duration)
|
|
6229
|
+
} : void 0;
|
|
6230
|
+
if (!isAudioOnly && renderableScope && renderableScope.startTime >= renderableScope.endTime) {
|
|
6231
|
+
throw new Error("The requested scope does not include any renderable video.");
|
|
6232
|
+
}
|
|
5882
6233
|
const signingContext = await resolveMuxSigningContext(credentials);
|
|
5883
6234
|
if (policy === "signed" && !signingContext) {
|
|
5884
6235
|
throw new Error(
|
|
@@ -5895,7 +6246,8 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
5895
6246
|
cleanTranscript: true,
|
|
5896
6247
|
shouldSign: policy === "signed",
|
|
5897
6248
|
credentials,
|
|
5898
|
-
required: true
|
|
6249
|
+
required: true,
|
|
6250
|
+
scope: effectiveScope
|
|
5899
6251
|
});
|
|
5900
6252
|
if (provider === "openai") {
|
|
5901
6253
|
thumbnailScores = await requestOpenAITranscriptModeration(
|
|
@@ -5913,17 +6265,21 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
5913
6265
|
throw new Error(`Unsupported moderation provider: ${exhaustiveCheck}`);
|
|
5914
6266
|
}
|
|
5915
6267
|
} else {
|
|
5916
|
-
const
|
|
6268
|
+
const scopedDuration = renderableScope ? renderableScope.endTime - renderableScope.startTime : duration;
|
|
6269
|
+
const estimatedIntervalCount = scopedDuration <= 50 ? 5 : Math.ceil(scopedDuration / thumbnailInterval);
|
|
5917
6270
|
const thumbnailUrls = maxSamples !== void 0 && estimatedIntervalCount > maxSamples ? await getThumbnailUrlsFromTimestamps(
|
|
5918
6271
|
playbackId,
|
|
5919
|
-
|
|
5920
|
-
|
|
5921
|
-
|
|
5922
|
-
|
|
5923
|
-
|
|
5924
|
-
|
|
5925
|
-
|
|
5926
|
-
|
|
6272
|
+
filterTimestampsWithinExclusiveScope(
|
|
6273
|
+
planSamplingTimestamps({
|
|
6274
|
+
duration_sec: duration,
|
|
6275
|
+
max_candidates: maxSamples,
|
|
6276
|
+
trim_start_sec: renderableScope?.startTime ?? (duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0),
|
|
6277
|
+
trim_end_sec: renderableScope ? duration - renderableScope.endTime : duration > 2 ? Math.min(5, Math.max(1, duration / 6)) : 0,
|
|
6278
|
+
fps: videoTrackFps,
|
|
6279
|
+
base_cadence_hz: thumbnailInterval > 0 ? 1 / thumbnailInterval : void 0
|
|
6280
|
+
}),
|
|
6281
|
+
renderableScope
|
|
6282
|
+
),
|
|
5927
6283
|
{
|
|
5928
6284
|
width: thumbnailWidth,
|
|
5929
6285
|
shouldSign: policy === "signed",
|
|
@@ -5933,7 +6289,8 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
5933
6289
|
interval: thumbnailInterval,
|
|
5934
6290
|
width: thumbnailWidth,
|
|
5935
6291
|
shouldSign: policy === "signed",
|
|
5936
|
-
credentials
|
|
6292
|
+
credentials,
|
|
6293
|
+
scope: renderableScope
|
|
5937
6294
|
});
|
|
5938
6295
|
thumbnailCount = thumbnailUrls.length;
|
|
5939
6296
|
if (provider === "openai") {
|
|
@@ -6019,7 +6376,7 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
6019
6376
|
// src/workflows/summarization.ts
|
|
6020
6377
|
import { generateText as generateText6, Output as Output6 } from "ai";
|
|
6021
6378
|
import dedent6 from "dedent";
|
|
6022
|
-
import { z as
|
|
6379
|
+
import { z as z8 } from "zod";
|
|
6023
6380
|
|
|
6024
6381
|
// src/lib/markdown-normalization.ts
|
|
6025
6382
|
var ESCAPED_LINE_BREAK_REGEX = /\\r\\n|\\n|\\r/g;
|
|
@@ -6032,17 +6389,17 @@ function normalizeMarkdownDescription(description) {
|
|
|
6032
6389
|
var DEFAULT_SUMMARY_KEYWORD_LIMIT = 10;
|
|
6033
6390
|
var DEFAULT_TITLE_LENGTH = 10;
|
|
6034
6391
|
var DEFAULT_DESCRIPTION_LENGTH = 50;
|
|
6035
|
-
var summarySchema =
|
|
6036
|
-
keywords:
|
|
6037
|
-
title:
|
|
6038
|
-
description:
|
|
6392
|
+
var summarySchema = z8.object({
|
|
6393
|
+
keywords: z8.array(z8.string().max(200)),
|
|
6394
|
+
title: z8.string().max(500),
|
|
6395
|
+
description: z8.string().max(1e4)
|
|
6039
6396
|
});
|
|
6040
6397
|
function buildSummarySchema(descriptionLength) {
|
|
6041
6398
|
const descriptionMaxChars = Math.max(2e3, descriptionLength * 15);
|
|
6042
|
-
return
|
|
6043
|
-
keywords:
|
|
6044
|
-
title:
|
|
6045
|
-
description:
|
|
6399
|
+
return z8.object({
|
|
6400
|
+
keywords: z8.array(z8.string().max(200)),
|
|
6401
|
+
title: z8.string().max(500),
|
|
6402
|
+
description: z8.string().max(descriptionMaxChars)
|
|
6046
6403
|
});
|
|
6047
6404
|
}
|
|
6048
6405
|
var VALID_TONES = ["neutral", "playful", "professional"];
|
|
@@ -6318,7 +6675,7 @@ async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, s
|
|
|
6318
6675
|
"use step";
|
|
6319
6676
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
6320
6677
|
const schema = buildSummarySchema(descriptionLength);
|
|
6321
|
-
const response = await generateText6({
|
|
6678
|
+
const response = await withContentPolicyErrorHandling(() => generateText6({
|
|
6322
6679
|
model,
|
|
6323
6680
|
output: Output6.object({
|
|
6324
6681
|
name: "summary_metadata",
|
|
@@ -6338,11 +6695,12 @@ async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, s
|
|
|
6338
6695
|
]
|
|
6339
6696
|
}
|
|
6340
6697
|
]
|
|
6341
|
-
});
|
|
6342
|
-
|
|
6698
|
+
}));
|
|
6699
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
6700
|
+
if (!output) {
|
|
6343
6701
|
throw new Error("Summarization output missing");
|
|
6344
6702
|
}
|
|
6345
|
-
const parsed = schema.parse(
|
|
6703
|
+
const parsed = schema.parse(output);
|
|
6346
6704
|
const unexpectedKeys = detectUnexpectedKeysFromRawText(
|
|
6347
6705
|
response.text,
|
|
6348
6706
|
schema.keyof().options
|
|
@@ -6354,7 +6712,8 @@ async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, s
|
|
|
6354
6712
|
outputTokens: response.usage.outputTokens,
|
|
6355
6713
|
totalTokens: response.usage.totalTokens,
|
|
6356
6714
|
reasoningTokens: response.usage.reasoningTokens,
|
|
6357
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
6715
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
6716
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
6358
6717
|
},
|
|
6359
6718
|
unexpectedKeys
|
|
6360
6719
|
};
|
|
@@ -6363,7 +6722,7 @@ async function analyzeAudioOnly(provider, modelId, userPrompt, systemPrompt, des
|
|
|
6363
6722
|
"use step";
|
|
6364
6723
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
6365
6724
|
const schema = buildSummarySchema(descriptionLength);
|
|
6366
|
-
const response = await generateText6({
|
|
6725
|
+
const response = await withContentPolicyErrorHandling(() => generateText6({
|
|
6367
6726
|
model,
|
|
6368
6727
|
output: Output6.object({
|
|
6369
6728
|
name: "summary_metadata",
|
|
@@ -6380,11 +6739,12 @@ async function analyzeAudioOnly(provider, modelId, userPrompt, systemPrompt, des
|
|
|
6380
6739
|
content: userPrompt
|
|
6381
6740
|
}
|
|
6382
6741
|
]
|
|
6383
|
-
});
|
|
6384
|
-
|
|
6742
|
+
}));
|
|
6743
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
6744
|
+
if (!output) {
|
|
6385
6745
|
throw new Error("Summarization output missing");
|
|
6386
6746
|
}
|
|
6387
|
-
const parsed = schema.parse(
|
|
6747
|
+
const parsed = schema.parse(output);
|
|
6388
6748
|
const unexpectedKeys = detectUnexpectedKeysFromRawText(
|
|
6389
6749
|
response.text,
|
|
6390
6750
|
schema.keyof().options
|
|
@@ -6396,7 +6756,8 @@ async function analyzeAudioOnly(provider, modelId, userPrompt, systemPrompt, des
|
|
|
6396
6756
|
outputTokens: response.usage.outputTokens,
|
|
6397
6757
|
totalTokens: response.usage.totalTokens,
|
|
6398
6758
|
reasoningTokens: response.usage.reasoningTokens,
|
|
6399
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
6759
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
6760
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
6400
6761
|
},
|
|
6401
6762
|
unexpectedKeys
|
|
6402
6763
|
};
|
|
@@ -6448,7 +6809,8 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
|
|
|
6448
6809
|
titleLength,
|
|
6449
6810
|
descriptionLength,
|
|
6450
6811
|
tagCount,
|
|
6451
|
-
outputLanguageCode
|
|
6812
|
+
outputLanguageCode,
|
|
6813
|
+
scope
|
|
6452
6814
|
} = options ?? {};
|
|
6453
6815
|
if (!VALID_TONES.includes(tone)) {
|
|
6454
6816
|
throw new MuxAiError(
|
|
@@ -6465,6 +6827,15 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
|
|
|
6465
6827
|
const { asset: assetData, playbackId, policy } = await getPlaybackIdForAsset(assetId, workflowCredentials);
|
|
6466
6828
|
const assetDurationSeconds = getAssetDurationSecondsFromAsset(assetData);
|
|
6467
6829
|
const isAudioOnly = isAudioOnlyAsset(assetData);
|
|
6830
|
+
const effectiveScope = hasWorkflowScopeBoundaries(scope) ? scope : void 0;
|
|
6831
|
+
const storyboardScope = isAudioOnly ? void 0 : resolveRenderableVideoScope(
|
|
6832
|
+
effectiveScope,
|
|
6833
|
+
assetDurationSeconds,
|
|
6834
|
+
getVideoTrackDurationSecondsFromAsset(assetData)
|
|
6835
|
+
);
|
|
6836
|
+
if (isAudioOnly && effectiveScope) {
|
|
6837
|
+
resolveWorkflowScope(effectiveScope, assetDurationSeconds);
|
|
6838
|
+
}
|
|
6468
6839
|
if (isAudioOnly && !includeTranscript) {
|
|
6469
6840
|
throw new MuxAiError(
|
|
6470
6841
|
"Audio-only assets require a transcript. Set includeTranscript: true and ensure the asset has a ready text track (captions/subtitles).",
|
|
@@ -6483,7 +6854,8 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
|
|
|
6483
6854
|
cleanTranscript,
|
|
6484
6855
|
shouldSign: policy === "signed",
|
|
6485
6856
|
credentials: workflowCredentials,
|
|
6486
|
-
required: isAudioOnly
|
|
6857
|
+
required: isAudioOnly,
|
|
6858
|
+
scope: effectiveScope
|
|
6487
6859
|
}) : void 0;
|
|
6488
6860
|
const transcriptText = transcriptResult?.transcriptText ?? "";
|
|
6489
6861
|
const resolvedLanguageCode = outputLanguageCode && outputLanguageCode !== "auto" ? outputLanguageCode : getReliableLanguageCode(transcriptResult?.track) ?? getReliableLanguageCode(getReadyTextTracks(assetData)[0]);
|
|
@@ -6514,7 +6886,13 @@ async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
|
|
|
6514
6886
|
workflowCredentials
|
|
6515
6887
|
);
|
|
6516
6888
|
} else {
|
|
6517
|
-
const storyboardUrl = await getStoryboardUrl(
|
|
6889
|
+
const storyboardUrl = await getStoryboardUrl(
|
|
6890
|
+
playbackId,
|
|
6891
|
+
640,
|
|
6892
|
+
policy === "signed",
|
|
6893
|
+
workflowCredentials,
|
|
6894
|
+
storyboardScope
|
|
6895
|
+
);
|
|
6518
6896
|
imageUrl = storyboardUrl;
|
|
6519
6897
|
if (imageSubmissionMode === "base64") {
|
|
6520
6898
|
const downloadResult = await downloadImageAsBase64(storyboardUrl, imageDownloadOptions);
|
|
@@ -6950,16 +7328,16 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
6950
7328
|
|
|
6951
7329
|
// src/workflows/translate-captions.ts
|
|
6952
7330
|
import {
|
|
6953
|
-
APICallError,
|
|
7331
|
+
APICallError as APICallError2,
|
|
6954
7332
|
generateText as generateText7,
|
|
6955
|
-
NoObjectGeneratedError,
|
|
7333
|
+
NoObjectGeneratedError as NoObjectGeneratedError2,
|
|
6956
7334
|
Output as Output7,
|
|
6957
|
-
RetryError,
|
|
7335
|
+
RetryError as RetryError2,
|
|
6958
7336
|
TypeValidationError
|
|
6959
7337
|
} from "ai";
|
|
6960
|
-
import { z as
|
|
6961
|
-
var translationSchema =
|
|
6962
|
-
translation:
|
|
7338
|
+
import { z as z9 } from "zod";
|
|
7339
|
+
var translationSchema = z9.object({
|
|
7340
|
+
translation: z9.string()
|
|
6963
7341
|
});
|
|
6964
7342
|
var SYSTEM_PROMPT6 = promptDedent`
|
|
6965
7343
|
You are a subtitle translation expert. Translate VTT subtitle files to the target language specified by the user.
|
|
@@ -7026,10 +7404,10 @@ function isProviderServiceError(error) {
|
|
|
7026
7404
|
if (!error) {
|
|
7027
7405
|
return false;
|
|
7028
7406
|
}
|
|
7029
|
-
if (
|
|
7407
|
+
if (RetryError2.isInstance(error)) {
|
|
7030
7408
|
return isProviderServiceError(error.lastError);
|
|
7031
7409
|
}
|
|
7032
|
-
if (
|
|
7410
|
+
if (APICallError2.isInstance(error)) {
|
|
7033
7411
|
return true;
|
|
7034
7412
|
}
|
|
7035
7413
|
if (error instanceof Error && "cause" in error) {
|
|
@@ -7041,7 +7419,7 @@ function shouldSplitChunkTranslationError(error) {
|
|
|
7041
7419
|
if (isProviderServiceError(error)) {
|
|
7042
7420
|
return false;
|
|
7043
7421
|
}
|
|
7044
|
-
return
|
|
7422
|
+
return NoObjectGeneratedError2.isInstance(error) || TypeValidationError.isInstance(error) || isTranslationChunkValidationError(error);
|
|
7045
7423
|
}
|
|
7046
7424
|
function resolveTranslationChunkingOptions(options) {
|
|
7047
7425
|
const targetChunkDurationSeconds = Math.max(
|
|
@@ -7202,7 +7580,7 @@ async function translateVttWithAI({
|
|
|
7202
7580
|
"use step";
|
|
7203
7581
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
7204
7582
|
const sanitisedVttContent = stripVttMetadataBlocks(vttContent);
|
|
7205
|
-
const response = await generateText7({
|
|
7583
|
+
const response = await withContentPolicyErrorHandling(() => generateText7({
|
|
7206
7584
|
model,
|
|
7207
7585
|
output: Output7.object({ schema: translationSchema }),
|
|
7208
7586
|
messages: [
|
|
@@ -7217,8 +7595,9 @@ async function translateVttWithAI({
|
|
|
7217
7595
|
${sanitisedVttContent}`
|
|
7218
7596
|
}
|
|
7219
7597
|
]
|
|
7220
|
-
});
|
|
7221
|
-
const
|
|
7598
|
+
}));
|
|
7599
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
7600
|
+
const translated = normalizeTranslatedVtt(output.translation);
|
|
7222
7601
|
const leakReason = detectLeakReason(translated);
|
|
7223
7602
|
const safeTranslated = leakReason !== null ? vttContent : translated;
|
|
7224
7603
|
if (leakReason !== null) {
|
|
@@ -7243,7 +7622,8 @@ ${sanitisedVttContent}`
|
|
|
7243
7622
|
outputTokens: response.usage.outputTokens,
|
|
7244
7623
|
totalTokens: response.usage.totalTokens,
|
|
7245
7624
|
reasoningTokens: response.usage.reasoningTokens,
|
|
7246
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
7625
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
7626
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
7247
7627
|
}
|
|
7248
7628
|
};
|
|
7249
7629
|
}
|
|
@@ -7257,8 +7637,8 @@ async function translateCueChunkWithAI({
|
|
|
7257
7637
|
}) {
|
|
7258
7638
|
"use step";
|
|
7259
7639
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
7260
|
-
const schema =
|
|
7261
|
-
translations:
|
|
7640
|
+
const schema = z9.object({
|
|
7641
|
+
translations: z9.array(z9.string().min(1).max(2e3)).length(cues.length)
|
|
7262
7642
|
});
|
|
7263
7643
|
const cuePayload = cues.map((cue, index) => ({
|
|
7264
7644
|
index,
|
|
@@ -7266,7 +7646,7 @@ async function translateCueChunkWithAI({
|
|
|
7266
7646
|
endTime: cue.endTime,
|
|
7267
7647
|
text: cue.text
|
|
7268
7648
|
}));
|
|
7269
|
-
const response = await generateText7({
|
|
7649
|
+
const response = await withContentPolicyErrorHandling(() => generateText7({
|
|
7270
7650
|
model,
|
|
7271
7651
|
output: Output7.object({ schema }),
|
|
7272
7652
|
messages: [
|
|
@@ -7282,7 +7662,8 @@ Return exactly ${cues.length} translated cues in the same order as the input.
|
|
|
7282
7662
|
${JSON.stringify(cuePayload, null, 2)}`
|
|
7283
7663
|
}
|
|
7284
7664
|
]
|
|
7285
|
-
});
|
|
7665
|
+
}));
|
|
7666
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
7286
7667
|
const unexpectedKeys = detectUnexpectedKeysFromRawText(
|
|
7287
7668
|
response.text,
|
|
7288
7669
|
schema.keyof().options
|
|
@@ -7293,13 +7674,14 @@ ${JSON.stringify(cuePayload, null, 2)}`
|
|
|
7293
7674
|
);
|
|
7294
7675
|
}
|
|
7295
7676
|
return {
|
|
7296
|
-
translations:
|
|
7677
|
+
translations: output.translations,
|
|
7297
7678
|
usage: {
|
|
7298
7679
|
inputTokens: response.usage.inputTokens,
|
|
7299
7680
|
outputTokens: response.usage.outputTokens,
|
|
7300
7681
|
totalTokens: response.usage.totalTokens,
|
|
7301
7682
|
reasoningTokens: response.usage.reasoningTokens,
|
|
7302
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
7683
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
7684
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
7303
7685
|
},
|
|
7304
7686
|
unexpectedKeyCount: unexpectedKeys.length
|
|
7305
7687
|
};
|