@mux/ai 0.29.0 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/{index-CXwsu679.d.ts → index-CJtQiGj7.d.ts} +1 -1
- package/dist/{index-DJirK4fh.d.ts → index-ooikuED7.d.ts} +46 -3
- package/dist/index.d.ts +7 -6
- package/dist/index.js +447 -150
- package/dist/index.js.map +1 -1
- package/dist/primitives/index.d.ts +2 -2
- package/dist/primitives/index.js +26 -6
- package/dist/primitives/index.js.map +1 -1
- package/dist/{shots-BKd7TMFN.d.ts → shots-Bh0GrvLx.d.ts} +2 -0
- package/dist/workflows/index.d.ts +2 -2
- package/dist/workflows/index.js +555 -256
- package/dist/workflows/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -5,7 +5,7 @@ var __export = (target, all) => {
|
|
|
5
5
|
};
|
|
6
6
|
|
|
7
7
|
// package.json
|
|
8
|
-
var version = "0.
|
|
8
|
+
var version = "0.31.0";
|
|
9
9
|
|
|
10
10
|
// src/lib/language-codes.ts
|
|
11
11
|
var ISO639_1_TO_3 = {
|
|
@@ -247,7 +247,7 @@ var EnvSchema = z.object({
|
|
|
247
247
|
MUX_TEST_ASSET_ID_VIOLENT_AUDIO_ONLY: optionalString("Mux test asset ID for audio-only assets with violent content.", "Mux test asset id for audio-only assets with violent content for testing"),
|
|
248
248
|
// Eval config
|
|
249
249
|
MUX_AI_EVAL_MODEL_SET: optionalString("Eval model selection mode.", "Choose between 'default' (provider defaults only) or 'all' (all configured models)"),
|
|
250
|
-
MUX_AI_EVAL_MODELS: optionalString("Comma-separated eval model pairs.", "Comma-separated provider:model pairs (e.g. 'openai:gpt-5.
|
|
250
|
+
MUX_AI_EVAL_MODELS: optionalString("Comma-separated eval model pairs.", "Comma-separated provider:model pairs (e.g. 'openai:gpt-5.6-luna,anthropic:claude-sonnet-4-5,google:gemini-3-flash-preview')"),
|
|
251
251
|
// AI Providers
|
|
252
252
|
OPENAI_API_KEY: optionalString("OpenAI API key for OpenAI-backed workflows.", "OpenAI API key"),
|
|
253
253
|
ANTHROPIC_API_KEY: optionalString("Anthropic API key for Claude-backed workflows.", "Anthropic API key"),
|
|
@@ -566,7 +566,8 @@ var TOKEN_USAGE_FIELDS = [
|
|
|
566
566
|
"outputTokens",
|
|
567
567
|
"totalTokens",
|
|
568
568
|
"reasoningTokens",
|
|
569
|
-
"cachedInputTokens"
|
|
569
|
+
"cachedInputTokens",
|
|
570
|
+
"cacheWriteTokens"
|
|
570
571
|
];
|
|
571
572
|
function isDefinedTokenUsageValue(value) {
|
|
572
573
|
return typeof value === "number";
|
|
@@ -608,6 +609,12 @@ function getErrorTokenUsage(error) {
|
|
|
608
609
|
extracted.cachedInputTokens = cacheReadTokens;
|
|
609
610
|
}
|
|
610
611
|
}
|
|
612
|
+
if (extracted.cacheWriteTokens === void 0) {
|
|
613
|
+
const cacheWriteTokens = source.inputTokenDetails?.cacheWriteTokens;
|
|
614
|
+
if (typeof cacheWriteTokens === "number") {
|
|
615
|
+
extracted.cacheWriteTokens = cacheWriteTokens;
|
|
616
|
+
}
|
|
617
|
+
}
|
|
611
618
|
return Object.keys(extracted).length > 0 ? extracted : void 0;
|
|
612
619
|
}
|
|
613
620
|
function rethrowWithTokenUsage(error, collectedUsage) {
|
|
@@ -633,6 +640,9 @@ var MuxAiError = class extends Error {
|
|
|
633
640
|
this.publicMessage = message;
|
|
634
641
|
this.retryable = opts?.retryable ?? false;
|
|
635
642
|
}
|
|
643
|
+
static is(value) {
|
|
644
|
+
return typeof value === "object" && value !== null && "__robots_error" in value && value.__robots_error === true;
|
|
645
|
+
}
|
|
636
646
|
};
|
|
637
647
|
function extractErrorDetail(error) {
|
|
638
648
|
if (error instanceof Error && error.message) {
|
|
@@ -650,7 +660,7 @@ function extractErrorDetail(error) {
|
|
|
650
660
|
return "Unknown error";
|
|
651
661
|
}
|
|
652
662
|
function wrapError(error, message) {
|
|
653
|
-
if (error
|
|
663
|
+
if (MuxAiError.is(error)) {
|
|
654
664
|
throw error;
|
|
655
665
|
}
|
|
656
666
|
const rawDetail = extractErrorDetail(error);
|
|
@@ -1401,7 +1411,8 @@ import { createAnthropic } from "@ai-sdk/anthropic";
|
|
|
1401
1411
|
import { createGoogleGenerativeAI } from "@ai-sdk/google";
|
|
1402
1412
|
import { createOpenAI } from "@ai-sdk/openai";
|
|
1403
1413
|
var DEFAULT_LANGUAGE_MODELS = {
|
|
1404
|
-
|
|
1414
|
+
// GPT-5.6 models default to medium reasoning when no effort is specified.
|
|
1415
|
+
openai: "gpt-5.6-luna",
|
|
1405
1416
|
anthropic: "claude-sonnet-4-5",
|
|
1406
1417
|
google: "gemini-3-flash-preview"
|
|
1407
1418
|
};
|
|
@@ -1410,11 +1421,20 @@ var DEFAULT_EMBEDDING_MODELS = {
|
|
|
1410
1421
|
google: "gemini-embedding-001"
|
|
1411
1422
|
};
|
|
1412
1423
|
var LANGUAGE_MODELS = {
|
|
1413
|
-
openai: ["gpt-5.1", "gpt-5-mini"],
|
|
1424
|
+
openai: ["gpt-5.6-luna", "gpt-5.1", "gpt-5-mini"],
|
|
1414
1425
|
anthropic: ["claude-sonnet-4-5"],
|
|
1415
1426
|
google: ["gemini-3-flash-preview", "gemini-3.1-flash-lite", "gemini-2.5-flash"]
|
|
1416
1427
|
};
|
|
1417
1428
|
var LANGUAGE_MODEL_DEPRECATIONS = [
|
|
1429
|
+
{
|
|
1430
|
+
provider: "openai",
|
|
1431
|
+
modelId: "gpt-5.1",
|
|
1432
|
+
replacementModelId: "gpt-5.6-luna",
|
|
1433
|
+
phase: "warn",
|
|
1434
|
+
deprecatedOn: "2026-08-01",
|
|
1435
|
+
sunsetOn: "2026-10-01",
|
|
1436
|
+
reason: "GPT-5.6 Luna with its default medium reasoning effort is the new OpenAI default."
|
|
1437
|
+
},
|
|
1418
1438
|
{
|
|
1419
1439
|
provider: "google",
|
|
1420
1440
|
modelId: "gemini-2.5-flash",
|
|
@@ -1475,7 +1495,7 @@ function parseEvalModelPair(value) {
|
|
|
1475
1495
|
const modelId = modelIdRaw?.trim();
|
|
1476
1496
|
if (!provider || !modelId) {
|
|
1477
1497
|
throw new Error(
|
|
1478
|
-
`Invalid eval model pair "${value}". Use "provider:model" (example: "openai:gpt-5.
|
|
1498
|
+
`Invalid eval model pair "${value}". Use "provider:model" (example: "openai:gpt-5.6-luna").`
|
|
1479
1499
|
);
|
|
1480
1500
|
}
|
|
1481
1501
|
if (!isSupportedProvider(provider)) {
|
|
@@ -2765,6 +2785,9 @@ async function fetchTranscriptForAsset(asset, playbackId, options = {}) {
|
|
|
2765
2785
|
// src/workflows/index.ts
|
|
2766
2786
|
var workflows_exports = {};
|
|
2767
2787
|
__export(workflows_exports, {
|
|
2788
|
+
ASK_QUESTIONS_DEFAULT_MAX_ANSWER_OPTION_LENGTH: () => ASK_QUESTIONS_DEFAULT_MAX_ANSWER_OPTION_LENGTH,
|
|
2789
|
+
ASK_QUESTIONS_DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH: () => ASK_QUESTIONS_DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH,
|
|
2790
|
+
ASK_QUESTIONS_MAX_QUESTION_LENGTH: () => ASK_QUESTIONS_MAX_QUESTION_LENGTH,
|
|
2768
2791
|
DEFAULT_DESCRIPTION_LENGTH: () => DEFAULT_DESCRIPTION_LENGTH,
|
|
2769
2792
|
DEFAULT_SUMMARY_KEYWORD_LIMIT: () => DEFAULT_SUMMARY_KEYWORD_LIMIT,
|
|
2770
2793
|
DEFAULT_TITLE_LENGTH: () => DEFAULT_TITLE_LENGTH,
|
|
@@ -2805,7 +2828,156 @@ __export(workflows_exports, {
|
|
|
2805
2828
|
// src/workflows/ask-questions.ts
|
|
2806
2829
|
import { generateText, Output } from "ai";
|
|
2807
2830
|
import dedent2 from "dedent";
|
|
2831
|
+
import { z as z3 } from "zod";
|
|
2832
|
+
|
|
2833
|
+
// src/lib/content-policy-error.ts
|
|
2834
|
+
import { APICallError, NoObjectGeneratedError, RetryError } from "ai";
|
|
2808
2835
|
import { z as z2 } from "zod";
|
|
2836
|
+
var PolicyTokenSchema = z2.string().trim().min(1).max(64).regex(/^[A-Z0-9_]+$/);
|
|
2837
|
+
var SafetyRatingSchema = z2.object({
|
|
2838
|
+
category: PolicyTokenSchema.nullish(),
|
|
2839
|
+
blocked: z2.boolean().nullish()
|
|
2840
|
+
});
|
|
2841
|
+
var ContentPolicyResponseSchema = z2.object({
|
|
2842
|
+
promptFeedback: z2.object({
|
|
2843
|
+
blockReason: PolicyTokenSchema.nullish(),
|
|
2844
|
+
safetyRatings: z2.array(SafetyRatingSchema).max(32).nullish()
|
|
2845
|
+
}).nullish(),
|
|
2846
|
+
candidates: z2.array(z2.object({
|
|
2847
|
+
finishReason: PolicyTokenSchema.nullish(),
|
|
2848
|
+
safetyRatings: z2.array(SafetyRatingSchema).max(32).nullish()
|
|
2849
|
+
})).max(8).nullish()
|
|
2850
|
+
});
|
|
2851
|
+
var ContentPolicyUsageResponseSchema = z2.object({
|
|
2852
|
+
usageMetadata: z2.object({
|
|
2853
|
+
promptTokenCount: z2.number().finite().nonnegative().nullish(),
|
|
2854
|
+
candidatesTokenCount: z2.number().finite().nonnegative().nullish(),
|
|
2855
|
+
totalTokenCount: z2.number().finite().nonnegative().nullish(),
|
|
2856
|
+
cachedContentTokenCount: z2.number().finite().nonnegative().nullish(),
|
|
2857
|
+
thoughtsTokenCount: z2.number().finite().nonnegative().nullish()
|
|
2858
|
+
}).nullish()
|
|
2859
|
+
});
|
|
2860
|
+
var CONTENT_POLICY_REASONS = /* @__PURE__ */ new Set([
|
|
2861
|
+
"BLOCKLIST",
|
|
2862
|
+
"ESCALATION",
|
|
2863
|
+
"IMAGE_PROHIBITED_CONTENT",
|
|
2864
|
+
"IMAGE_RECITATION",
|
|
2865
|
+
"IMAGE_SAFETY",
|
|
2866
|
+
"JAILBREAK",
|
|
2867
|
+
"MODEL_ARMOR",
|
|
2868
|
+
"PROHIBITED_CONTENT",
|
|
2869
|
+
"RECITATION",
|
|
2870
|
+
"SAFETY",
|
|
2871
|
+
"SPII"
|
|
2872
|
+
]);
|
|
2873
|
+
async function withContentPolicyErrorHandling(operation) {
|
|
2874
|
+
try {
|
|
2875
|
+
return await operation();
|
|
2876
|
+
} catch (error) {
|
|
2877
|
+
rethrowContentPolicyError(error);
|
|
2878
|
+
}
|
|
2879
|
+
}
|
|
2880
|
+
function rethrowContentPolicyError(error) {
|
|
2881
|
+
const block = extractContentPolicyBlock(error);
|
|
2882
|
+
if (!block) {
|
|
2883
|
+
throw error;
|
|
2884
|
+
}
|
|
2885
|
+
throwContentPolicyError(block, getContentPolicyTokenUsage(error));
|
|
2886
|
+
}
|
|
2887
|
+
function getGeneratedOutputWithContentPolicyHandling(response) {
|
|
2888
|
+
if (response.finishReason === "content-filter") {
|
|
2889
|
+
const rawReason = PolicyTokenSchema.safeParse(response.rawFinishReason);
|
|
2890
|
+
const reason = rawReason.success && isContentPolicyReason(rawReason.data) ? rawReason.data : "CONTENT_FILTER";
|
|
2891
|
+
throwContentPolicyError({ reason }, getErrorTokenUsage(response));
|
|
2892
|
+
}
|
|
2893
|
+
return response.output;
|
|
2894
|
+
}
|
|
2895
|
+
function throwContentPolicyError(block, usage) {
|
|
2896
|
+
const category = block.category ? `; category: ${block.category}` : "";
|
|
2897
|
+
const contentPolicyError = new MuxAiError(
|
|
2898
|
+
`The supplied content was blocked by a content policy (reason: ${block.reason}${category}).`,
|
|
2899
|
+
{
|
|
2900
|
+
type: "content_policy_error",
|
|
2901
|
+
retryable: false
|
|
2902
|
+
}
|
|
2903
|
+
);
|
|
2904
|
+
if (usage) {
|
|
2905
|
+
contentPolicyError.usage = usage;
|
|
2906
|
+
}
|
|
2907
|
+
throw contentPolicyError;
|
|
2908
|
+
}
|
|
2909
|
+
function extractContentPolicyBlock(error) {
|
|
2910
|
+
if (RetryError.isInstance(error)) {
|
|
2911
|
+
return extractContentPolicyBlock(error.lastError);
|
|
2912
|
+
}
|
|
2913
|
+
if (NoObjectGeneratedError.isInstance(error) && error.finishReason === "content-filter") {
|
|
2914
|
+
return { reason: "CONTENT_FILTER" };
|
|
2915
|
+
}
|
|
2916
|
+
if (!APICallError.isInstance(error) || !error.responseBody) {
|
|
2917
|
+
return void 0;
|
|
2918
|
+
}
|
|
2919
|
+
try {
|
|
2920
|
+
const parsed = ContentPolicyResponseSchema.safeParse(JSON.parse(error.responseBody));
|
|
2921
|
+
if (!parsed.success) {
|
|
2922
|
+
return void 0;
|
|
2923
|
+
}
|
|
2924
|
+
const promptReason = parsed.data.promptFeedback?.blockReason;
|
|
2925
|
+
if (isContentPolicyReason(promptReason)) {
|
|
2926
|
+
return {
|
|
2927
|
+
reason: promptReason,
|
|
2928
|
+
category: findBlockedCategory(parsed.data.promptFeedback?.safetyRatings)
|
|
2929
|
+
};
|
|
2930
|
+
}
|
|
2931
|
+
for (const candidate of parsed.data.candidates ?? []) {
|
|
2932
|
+
if (isContentPolicyReason(candidate.finishReason)) {
|
|
2933
|
+
return {
|
|
2934
|
+
reason: candidate.finishReason,
|
|
2935
|
+
category: findBlockedCategory(candidate.safetyRatings)
|
|
2936
|
+
};
|
|
2937
|
+
}
|
|
2938
|
+
}
|
|
2939
|
+
} catch {
|
|
2940
|
+
return void 0;
|
|
2941
|
+
}
|
|
2942
|
+
return void 0;
|
|
2943
|
+
}
|
|
2944
|
+
function isContentPolicyReason(reason) {
|
|
2945
|
+
return Boolean(reason && CONTENT_POLICY_REASONS.has(reason));
|
|
2946
|
+
}
|
|
2947
|
+
function getContentPolicyTokenUsage(error) {
|
|
2948
|
+
const errorUsage = getErrorTokenUsage(error);
|
|
2949
|
+
if (errorUsage) {
|
|
2950
|
+
return errorUsage;
|
|
2951
|
+
}
|
|
2952
|
+
if (RetryError.isInstance(error)) {
|
|
2953
|
+
return getContentPolicyTokenUsage(error.lastError);
|
|
2954
|
+
}
|
|
2955
|
+
if (!APICallError.isInstance(error) || !error.responseBody) {
|
|
2956
|
+
return void 0;
|
|
2957
|
+
}
|
|
2958
|
+
try {
|
|
2959
|
+
const parsed = ContentPolicyUsageResponseSchema.safeParse(JSON.parse(error.responseBody));
|
|
2960
|
+
const usage = parsed.success ? parsed.data.usageMetadata : void 0;
|
|
2961
|
+
if (!usage) {
|
|
2962
|
+
return void 0;
|
|
2963
|
+
}
|
|
2964
|
+
const inputTokens = usage.promptTokenCount ?? 0;
|
|
2965
|
+
const reasoningTokens = usage.thoughtsTokenCount ?? 0;
|
|
2966
|
+
const outputTokens = (usage.candidatesTokenCount ?? 0) + reasoningTokens;
|
|
2967
|
+
return {
|
|
2968
|
+
inputTokens,
|
|
2969
|
+
outputTokens,
|
|
2970
|
+
totalTokens: usage.totalTokenCount ?? inputTokens + outputTokens,
|
|
2971
|
+
reasoningTokens,
|
|
2972
|
+
cachedInputTokens: usage.cachedContentTokenCount ?? 0
|
|
2973
|
+
};
|
|
2974
|
+
} catch {
|
|
2975
|
+
return void 0;
|
|
2976
|
+
}
|
|
2977
|
+
}
|
|
2978
|
+
function findBlockedCategory(ratings) {
|
|
2979
|
+
return ratings?.find((rating) => rating.blocked === true)?.category ?? void 0;
|
|
2980
|
+
}
|
|
2809
2981
|
|
|
2810
2982
|
// src/lib/image-download.ts
|
|
2811
2983
|
import pRetry, { AbortError } from "p-retry";
|
|
@@ -2995,13 +3167,23 @@ function createLanguageSection(languageName) {
|
|
|
2995
3167
|
}
|
|
2996
3168
|
|
|
2997
3169
|
// src/lib/retry.ts
|
|
3170
|
+
import { DownloadError } from "ai";
|
|
2998
3171
|
var DEFAULT_RETRY_OPTIONS = {
|
|
2999
3172
|
maxRetries: 3,
|
|
3000
3173
|
baseDelay: 2e3,
|
|
3001
3174
|
maxDelay: 1e4
|
|
3002
3175
|
};
|
|
3003
3176
|
function defaultShouldRetry(error, _attempt) {
|
|
3004
|
-
|
|
3177
|
+
if (error.message.includes("Timeout while downloading")) {
|
|
3178
|
+
return true;
|
|
3179
|
+
}
|
|
3180
|
+
const isDownloadError = DownloadError.isInstance(error);
|
|
3181
|
+
const isSerializedDownloadError = error.name === "AI_DownloadError";
|
|
3182
|
+
if (!isDownloadError && !isSerializedDownloadError) {
|
|
3183
|
+
return false;
|
|
3184
|
+
}
|
|
3185
|
+
const statusCode = error.statusCode;
|
|
3186
|
+
return statusCode === void 0 || statusCode === 408 || statusCode === 425 || statusCode === 429 || statusCode >= 500;
|
|
3005
3187
|
}
|
|
3006
3188
|
function calculateDelay(attempt, baseDelay, maxDelay) {
|
|
3007
3189
|
const exponentialDelay = baseDelay * 2 ** (attempt - 1);
|
|
@@ -3019,33 +3201,37 @@ async function withRetry(fn, {
|
|
|
3019
3201
|
try {
|
|
3020
3202
|
return await fn();
|
|
3021
3203
|
} catch (error) {
|
|
3022
|
-
lastError = error
|
|
3204
|
+
lastError = error;
|
|
3205
|
+
const retryError = error instanceof Error ? error : new Error(String(error));
|
|
3023
3206
|
const isLastAttempt = attempt === maxRetries;
|
|
3024
|
-
if (isLastAttempt || !shouldRetry(
|
|
3025
|
-
throw
|
|
3207
|
+
if (isLastAttempt || !shouldRetry(retryError, attempt + 1)) {
|
|
3208
|
+
throw error;
|
|
3026
3209
|
}
|
|
3027
3210
|
const delay = calculateDelay(attempt + 1, baseDelay, maxDelay);
|
|
3028
3211
|
console.warn(
|
|
3029
|
-
`Attempt ${attempt + 1} failed: ${
|
|
3212
|
+
`Attempt ${attempt + 1} failed: ${retryError.message}. Retrying in ${Math.round(delay)}ms...`
|
|
3030
3213
|
);
|
|
3031
3214
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
3032
3215
|
}
|
|
3033
3216
|
}
|
|
3034
|
-
throw lastError
|
|
3217
|
+
throw lastError ?? new Error("Retry failed with unknown error");
|
|
3035
3218
|
}
|
|
3036
3219
|
|
|
3037
3220
|
// src/workflows/ask-questions.ts
|
|
3038
|
-
var
|
|
3039
|
-
|
|
3040
|
-
|
|
3041
|
-
|
|
3042
|
-
|
|
3043
|
-
|
|
3221
|
+
var ASK_QUESTIONS_MAX_QUESTION_LENGTH = 600;
|
|
3222
|
+
var ASK_QUESTIONS_DEFAULT_MAX_ANSWER_OPTION_LENGTH = 150;
|
|
3223
|
+
var ASK_QUESTIONS_DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH = 500;
|
|
3224
|
+
var questionAnswerSchema = z3.object({
|
|
3225
|
+
question: z3.string().describe("The full text of the original question"),
|
|
3226
|
+
answer: z3.string().nullable(),
|
|
3227
|
+
confidence: z3.number(),
|
|
3228
|
+
reasoning: z3.string().max(1e3),
|
|
3229
|
+
skipped: z3.boolean()
|
|
3044
3230
|
});
|
|
3045
3231
|
var SKIP_SENTINEL = "__SKIPPED__";
|
|
3046
3232
|
function createAskQuestionsSchema(answerSchema) {
|
|
3047
|
-
return
|
|
3048
|
-
answers:
|
|
3233
|
+
return z3.object({
|
|
3234
|
+
answers: z3.array(
|
|
3049
3235
|
questionAnswerSchema.extend({
|
|
3050
3236
|
answer: answerSchema
|
|
3051
3237
|
})
|
|
@@ -3395,12 +3581,12 @@ function buildResponseSchemaForQuestions(normalizedQuestions, maxFreeFormAnswerL
|
|
|
3395
3581
|
SKIP_SENTINEL.length,
|
|
3396
3582
|
longestConstrained
|
|
3397
3583
|
);
|
|
3398
|
-
return createAskQuestionsSchema(
|
|
3584
|
+
return createAskQuestionsSchema(z3.string().min(1).max(schemaMax));
|
|
3399
3585
|
}
|
|
3400
3586
|
const allowed = Array.from(
|
|
3401
3587
|
new Set(normalizedQuestions.flatMap(allowedAnswersForQuestion))
|
|
3402
3588
|
);
|
|
3403
|
-
return createAskQuestionsSchema(
|
|
3589
|
+
return createAskQuestionsSchema(z3.enum([...allowed, SKIP_SENTINEL]));
|
|
3404
3590
|
}
|
|
3405
3591
|
async function analyzeQuestions({
|
|
3406
3592
|
provider,
|
|
@@ -3418,7 +3604,7 @@ async function analyzeQuestions({
|
|
|
3418
3604
|
normalizedQuestions,
|
|
3419
3605
|
maxFreeFormAnswerLength
|
|
3420
3606
|
);
|
|
3421
|
-
const response = await generateText({
|
|
3607
|
+
const response = await withContentPolicyErrorHandling(() => generateText({
|
|
3422
3608
|
model,
|
|
3423
3609
|
output: Output.object({ schema: responseSchema }),
|
|
3424
3610
|
experimental_telemetry: { isEnabled: true },
|
|
@@ -3435,11 +3621,12 @@ async function analyzeQuestions({
|
|
|
3435
3621
|
] : userPrompt
|
|
3436
3622
|
}
|
|
3437
3623
|
]
|
|
3438
|
-
});
|
|
3439
|
-
|
|
3624
|
+
}));
|
|
3625
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
3626
|
+
if (!output) {
|
|
3440
3627
|
throw new Error("Ask-questions output missing");
|
|
3441
3628
|
}
|
|
3442
|
-
const parsed = responseSchema.parse(
|
|
3629
|
+
const parsed = responseSchema.parse(output);
|
|
3443
3630
|
const unexpectedRootKeys = detectUnexpectedKeysFromRawText(
|
|
3444
3631
|
response.text,
|
|
3445
3632
|
responseSchema.keyof().options
|
|
@@ -3461,7 +3648,8 @@ async function analyzeQuestions({
|
|
|
3461
3648
|
outputTokens: response.usage.outputTokens,
|
|
3462
3649
|
totalTokens: response.usage.totalTokens,
|
|
3463
3650
|
reasoningTokens: response.usage.reasoningTokens,
|
|
3464
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
3651
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
3652
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
3465
3653
|
},
|
|
3466
3654
|
unexpectedRootKeys,
|
|
3467
3655
|
unexpectedAnswerKeys
|
|
@@ -3480,7 +3668,6 @@ async function askQuestionsInternal(assetId, questions, options, collectedUsage)
|
|
|
3480
3668
|
if (!questions || questions.length === 0) {
|
|
3481
3669
|
throw new MuxAiError("At least one question must be provided.", { type: "validation_error" });
|
|
3482
3670
|
}
|
|
3483
|
-
const MAX_QUESTION_LENGTH = 500;
|
|
3484
3671
|
questions.forEach((q, idx) => {
|
|
3485
3672
|
if (!q.question || typeof q.question !== "string" || !q.question.trim()) {
|
|
3486
3673
|
throw new MuxAiError(
|
|
@@ -3488,15 +3675,14 @@ async function askQuestionsInternal(assetId, questions, options, collectedUsage)
|
|
|
3488
3675
|
{ type: "validation_error" }
|
|
3489
3676
|
);
|
|
3490
3677
|
}
|
|
3491
|
-
if (q.question.length >
|
|
3678
|
+
if (q.question.length > ASK_QUESTIONS_MAX_QUESTION_LENGTH) {
|
|
3492
3679
|
throw new MuxAiError(
|
|
3493
|
-
`Question at index ${idx} exceeds the ${
|
|
3680
|
+
`Question at index ${idx} exceeds the ${ASK_QUESTIONS_MAX_QUESTION_LENGTH}-character limit (received ${q.question.length}).`,
|
|
3494
3681
|
{ type: "validation_error" }
|
|
3495
3682
|
);
|
|
3496
3683
|
}
|
|
3497
3684
|
});
|
|
3498
|
-
const
|
|
3499
|
-
const maxAnswerOptionLength = options?.maxAnswerOptionLength ?? DEFAULT_MAX_ANSWER_OPTION_LENGTH;
|
|
3685
|
+
const maxAnswerOptionLength = options?.maxAnswerOptionLength ?? ASK_QUESTIONS_DEFAULT_MAX_ANSWER_OPTION_LENGTH;
|
|
3500
3686
|
if (!Number.isFinite(maxAnswerOptionLength) || maxAnswerOptionLength <= 0) {
|
|
3501
3687
|
throw new MuxAiError(
|
|
3502
3688
|
`maxAnswerOptionLength must be a positive number (received ${maxAnswerOptionLength}).`,
|
|
@@ -3515,8 +3701,7 @@ async function askQuestionsInternal(assetId, questions, options, collectedUsage)
|
|
|
3515
3701
|
}
|
|
3516
3702
|
}
|
|
3517
3703
|
});
|
|
3518
|
-
const
|
|
3519
|
-
const maxFreeFormAnswerLength = options?.maxFreeFormAnswerLength ?? DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH;
|
|
3704
|
+
const maxFreeFormAnswerLength = options?.maxFreeFormAnswerLength ?? ASK_QUESTIONS_DEFAULT_MAX_FREE_FORM_ANSWER_LENGTH;
|
|
3520
3705
|
if (!Number.isFinite(maxFreeFormAnswerLength) || maxFreeFormAnswerLength <= 0) {
|
|
3521
3706
|
throw new MuxAiError(
|
|
3522
3707
|
`maxFreeFormAnswerLength must be a positive number (received ${maxFreeFormAnswerLength}).`,
|
|
@@ -3718,11 +3903,11 @@ async function askQuestionsInternal(assetId, questions, options, collectedUsage)
|
|
|
3718
3903
|
// src/workflows/burned-in-captions.ts
|
|
3719
3904
|
import { generateText as generateText2, Output as Output2 } from "ai";
|
|
3720
3905
|
import dedent3 from "dedent";
|
|
3721
|
-
import { z as
|
|
3722
|
-
var burnedInCaptionsSchema =
|
|
3723
|
-
hasBurnedInCaptions:
|
|
3724
|
-
confidence:
|
|
3725
|
-
detectedLanguage:
|
|
3906
|
+
import { z as z4 } from "zod";
|
|
3907
|
+
var burnedInCaptionsSchema = z4.object({
|
|
3908
|
+
hasBurnedInCaptions: z4.boolean(),
|
|
3909
|
+
confidence: z4.number(),
|
|
3910
|
+
detectedLanguage: z4.string().max(100).nullable()
|
|
3726
3911
|
});
|
|
3727
3912
|
var SYSTEM_PROMPT2 = promptDedent`
|
|
3728
3913
|
<role>
|
|
@@ -3832,7 +4017,7 @@ async function analyzeStoryboard({
|
|
|
3832
4017
|
}) {
|
|
3833
4018
|
"use step";
|
|
3834
4019
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
3835
|
-
const response = await generateText2({
|
|
4020
|
+
const response = await withContentPolicyErrorHandling(() => generateText2({
|
|
3836
4021
|
model,
|
|
3837
4022
|
output: Output2.object({ schema: burnedInCaptionsSchema }),
|
|
3838
4023
|
experimental_telemetry: { isEnabled: true },
|
|
@@ -3849,22 +4034,24 @@ async function analyzeStoryboard({
|
|
|
3849
4034
|
]
|
|
3850
4035
|
}
|
|
3851
4036
|
]
|
|
3852
|
-
});
|
|
4037
|
+
}));
|
|
4038
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
3853
4039
|
const unexpectedKeys = detectUnexpectedKeysFromRawText(
|
|
3854
4040
|
response.text,
|
|
3855
4041
|
burnedInCaptionsSchema.keyof().options
|
|
3856
4042
|
);
|
|
3857
4043
|
return {
|
|
3858
4044
|
result: {
|
|
3859
|
-
...
|
|
3860
|
-
confidence: Math.min(1, Math.max(0,
|
|
4045
|
+
...output,
|
|
4046
|
+
confidence: Math.min(1, Math.max(0, output.confidence))
|
|
3861
4047
|
},
|
|
3862
4048
|
usage: {
|
|
3863
4049
|
inputTokens: response.usage.inputTokens,
|
|
3864
4050
|
outputTokens: response.usage.outputTokens,
|
|
3865
4051
|
totalTokens: response.usage.totalTokens,
|
|
3866
4052
|
reasoningTokens: response.usage.reasoningTokens,
|
|
3867
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
4053
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
4054
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
3868
4055
|
},
|
|
3869
4056
|
unexpectedKeys
|
|
3870
4057
|
};
|
|
@@ -3921,14 +4108,14 @@ async function hasBurnedInCaptionsInternal(assetId, options, collectedUsage) {
|
|
|
3921
4108
|
credentials
|
|
3922
4109
|
});
|
|
3923
4110
|
} else {
|
|
3924
|
-
analysisResponse = await analyzeStoryboard({
|
|
4111
|
+
analysisResponse = await withRetry(() => analyzeStoryboard({
|
|
3925
4112
|
imageDataUrl: imageUrl,
|
|
3926
4113
|
provider: modelConfig.provider,
|
|
3927
4114
|
modelId: modelConfig.modelId,
|
|
3928
4115
|
userPrompt,
|
|
3929
4116
|
systemPrompt: SYSTEM_PROMPT2,
|
|
3930
4117
|
credentials
|
|
3931
|
-
});
|
|
4118
|
+
}));
|
|
3932
4119
|
}
|
|
3933
4120
|
collectedUsage.push(analysisResponse.usage);
|
|
3934
4121
|
if (!analysisResponse.result) {
|
|
@@ -3962,13 +4149,13 @@ async function hasBurnedInCaptionsInternal(assetId, options, collectedUsage) {
|
|
|
3962
4149
|
// src/workflows/chapters.ts
|
|
3963
4150
|
import { generateText as generateText3, Output as Output3 } from "ai";
|
|
3964
4151
|
import dedent4 from "dedent";
|
|
3965
|
-
import { z as
|
|
3966
|
-
var chapterSchema =
|
|
3967
|
-
startTime:
|
|
3968
|
-
title:
|
|
4152
|
+
import { z as z5 } from "zod";
|
|
4153
|
+
var chapterSchema = z5.object({
|
|
4154
|
+
startTime: z5.number(),
|
|
4155
|
+
title: z5.string().max(300)
|
|
3969
4156
|
});
|
|
3970
|
-
var chaptersSchema =
|
|
3971
|
-
chapters:
|
|
4157
|
+
var chaptersSchema = z5.object({
|
|
4158
|
+
chapters: z5.array(chapterSchema)
|
|
3972
4159
|
});
|
|
3973
4160
|
async function generateChaptersWithAI({
|
|
3974
4161
|
provider,
|
|
@@ -3980,21 +4167,24 @@ async function generateChaptersWithAI({
|
|
|
3980
4167
|
"use step";
|
|
3981
4168
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
3982
4169
|
const response = await withRetry(
|
|
3983
|
-
() =>
|
|
3984
|
-
|
|
3985
|
-
|
|
3986
|
-
|
|
3987
|
-
|
|
3988
|
-
|
|
3989
|
-
|
|
3990
|
-
|
|
3991
|
-
|
|
3992
|
-
|
|
3993
|
-
|
|
3994
|
-
|
|
3995
|
-
|
|
3996
|
-
|
|
4170
|
+
() => withContentPolicyErrorHandling(
|
|
4171
|
+
() => generateText3({
|
|
4172
|
+
model,
|
|
4173
|
+
output: Output3.object({ schema: chaptersSchema }),
|
|
4174
|
+
messages: [
|
|
4175
|
+
{
|
|
4176
|
+
role: "system",
|
|
4177
|
+
content: systemPrompt
|
|
4178
|
+
},
|
|
4179
|
+
{
|
|
4180
|
+
role: "user",
|
|
4181
|
+
content: userPrompt
|
|
4182
|
+
}
|
|
4183
|
+
]
|
|
4184
|
+
})
|
|
4185
|
+
)
|
|
3997
4186
|
);
|
|
4187
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
3998
4188
|
const unexpectedRootKeys = detectUnexpectedKeysFromRawText(
|
|
3999
4189
|
response.text,
|
|
4000
4190
|
chaptersSchema.keyof().options
|
|
@@ -4010,13 +4200,14 @@ async function generateChaptersWithAI({
|
|
|
4010
4200
|
} catch {
|
|
4011
4201
|
}
|
|
4012
4202
|
return {
|
|
4013
|
-
chapters:
|
|
4203
|
+
chapters: output,
|
|
4014
4204
|
usage: {
|
|
4015
4205
|
inputTokens: response.usage.inputTokens,
|
|
4016
4206
|
outputTokens: response.usage.outputTokens,
|
|
4017
4207
|
totalTokens: response.usage.totalTokens,
|
|
4018
4208
|
reasoningTokens: response.usage.reasoningTokens,
|
|
4019
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
4209
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
4210
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
4020
4211
|
},
|
|
4021
4212
|
unexpectedRootKeys,
|
|
4022
4213
|
unexpectedChapterKeys
|
|
@@ -4272,7 +4463,7 @@ async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
|
4272
4463
|
|
|
4273
4464
|
// src/workflows/edit-captions.ts
|
|
4274
4465
|
import { generateText as generateText4, Output as Output4 } from "ai";
|
|
4275
|
-
import { z as
|
|
4466
|
+
import { z as z6 } from "zod";
|
|
4276
4467
|
|
|
4277
4468
|
// src/lib/mux-tracks.ts
|
|
4278
4469
|
async function fetchVttFromMux(vttUrl) {
|
|
@@ -4343,8 +4534,8 @@ async function createPresignedGetUrlWithStorageAdapter(input, adapter) {
|
|
|
4343
4534
|
}
|
|
4344
4535
|
|
|
4345
4536
|
// src/workflows/edit-captions.ts
|
|
4346
|
-
var profanityDetectionSchema =
|
|
4347
|
-
profanity:
|
|
4537
|
+
var profanityDetectionSchema = z6.object({
|
|
4538
|
+
profanity: z6.array(z6.string().max(100)).describe(
|
|
4348
4539
|
"Unique profane words or short phrases exactly as they appear in the transcript text. Include each distinct form only once (e.g., if 'fuck' and 'fucking' both appear, list both)."
|
|
4349
4540
|
)
|
|
4350
4541
|
});
|
|
@@ -4521,7 +4712,7 @@ async function identifyProfanityWithAI({
|
|
|
4521
4712
|
tag: "transcript",
|
|
4522
4713
|
content: plainText
|
|
4523
4714
|
});
|
|
4524
|
-
const response = await generateText4({
|
|
4715
|
+
const response = await withContentPolicyErrorHandling(() => generateText4({
|
|
4525
4716
|
model,
|
|
4526
4717
|
output: Output4.object({ schema: profanityDetectionSchema }),
|
|
4527
4718
|
messages: [
|
|
@@ -4536,19 +4727,21 @@ async function identifyProfanityWithAI({
|
|
|
4536
4727
|
${transcriptSection}`
|
|
4537
4728
|
}
|
|
4538
4729
|
]
|
|
4539
|
-
});
|
|
4730
|
+
}));
|
|
4731
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
4540
4732
|
const unexpectedKeys = detectUnexpectedKeysFromRawText(
|
|
4541
4733
|
response.text,
|
|
4542
4734
|
profanityDetectionSchema.keyof().options
|
|
4543
4735
|
);
|
|
4544
4736
|
return {
|
|
4545
|
-
profanity:
|
|
4737
|
+
profanity: output.profanity,
|
|
4546
4738
|
usage: {
|
|
4547
4739
|
inputTokens: response.usage.inputTokens,
|
|
4548
4740
|
outputTokens: response.usage.outputTokens,
|
|
4549
4741
|
totalTokens: response.usage.totalTokens,
|
|
4550
4742
|
reasoningTokens: response.usage.reasoningTokens,
|
|
4551
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
4743
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
4744
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
4552
4745
|
},
|
|
4553
4746
|
unexpectedKeys
|
|
4554
4747
|
};
|
|
@@ -4951,21 +5144,21 @@ async function generateVideoEmbeddings(assetId, options = {}) {
|
|
|
4951
5144
|
// src/workflows/engagement-insights.ts
|
|
4952
5145
|
import { generateText as generateText5, Output as Output5 } from "ai";
|
|
4953
5146
|
import dedent5 from "dedent";
|
|
4954
|
-
import { z as
|
|
4955
|
-
var aiMomentInsightSchema =
|
|
4956
|
-
hotspotIndex:
|
|
4957
|
-
startMs:
|
|
4958
|
-
endMs:
|
|
4959
|
-
timestamp:
|
|
4960
|
-
engagementScore:
|
|
4961
|
-
insight:
|
|
5147
|
+
import { z as z7 } from "zod";
|
|
5148
|
+
var aiMomentInsightSchema = z7.object({
|
|
5149
|
+
hotspotIndex: z7.number(),
|
|
5150
|
+
startMs: z7.number(),
|
|
5151
|
+
endMs: z7.number(),
|
|
5152
|
+
timestamp: z7.string(),
|
|
5153
|
+
engagementScore: z7.number(),
|
|
5154
|
+
insight: z7.string().max(1e3)
|
|
4962
5155
|
});
|
|
4963
|
-
var aiOverallInsightSchema =
|
|
4964
|
-
summary:
|
|
4965
|
-
trends:
|
|
5156
|
+
var aiOverallInsightSchema = z7.object({
|
|
5157
|
+
summary: z7.string().max(2e3),
|
|
5158
|
+
trends: z7.array(z7.string().max(500))
|
|
4966
5159
|
});
|
|
4967
|
-
var engagementInsightsSchema =
|
|
4968
|
-
momentInsights:
|
|
5160
|
+
var engagementInsightsSchema = z7.object({
|
|
5161
|
+
momentInsights: z7.array(aiMomentInsightSchema),
|
|
4969
5162
|
overallInsight: aiOverallInsightSchema
|
|
4970
5163
|
});
|
|
4971
5164
|
var SYSTEM_PROMPT4 = promptDedent`
|
|
@@ -5227,26 +5420,29 @@ async function generateInsightsWithAI(provider, modelId, systemPrompt, userPromp
|
|
|
5227
5420
|
"use step";
|
|
5228
5421
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
5229
5422
|
const response = await withRetry(
|
|
5230
|
-
() =>
|
|
5231
|
-
|
|
5232
|
-
|
|
5233
|
-
|
|
5234
|
-
|
|
5235
|
-
|
|
5236
|
-
|
|
5237
|
-
|
|
5238
|
-
|
|
5239
|
-
|
|
5240
|
-
|
|
5241
|
-
|
|
5242
|
-
|
|
5243
|
-
|
|
5244
|
-
|
|
5245
|
-
|
|
5246
|
-
|
|
5247
|
-
|
|
5423
|
+
() => withContentPolicyErrorHandling(
|
|
5424
|
+
() => generateText5({
|
|
5425
|
+
model,
|
|
5426
|
+
output: Output5.object({ schema: engagementInsightsSchema }),
|
|
5427
|
+
experimental_telemetry: { isEnabled: true },
|
|
5428
|
+
messages: [
|
|
5429
|
+
{
|
|
5430
|
+
role: "system",
|
|
5431
|
+
content: systemPrompt
|
|
5432
|
+
},
|
|
5433
|
+
{
|
|
5434
|
+
role: "user",
|
|
5435
|
+
content: [
|
|
5436
|
+
{ type: "text", text: userPrompt },
|
|
5437
|
+
...imageUrls.map((url) => ({ type: "image", image: url }))
|
|
5438
|
+
]
|
|
5439
|
+
}
|
|
5440
|
+
]
|
|
5441
|
+
})
|
|
5442
|
+
)
|
|
5248
5443
|
);
|
|
5249
|
-
|
|
5444
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
5445
|
+
if (!output) {
|
|
5250
5446
|
throw new Error("AI returned empty or unparseable response");
|
|
5251
5447
|
}
|
|
5252
5448
|
const unexpectedRootKeys = detectUnexpectedKeysFromRawText(
|
|
@@ -5269,13 +5465,14 @@ async function generateInsightsWithAI(provider, modelId, systemPrompt, userPromp
|
|
|
5269
5465
|
} catch {
|
|
5270
5466
|
}
|
|
5271
5467
|
return {
|
|
5272
|
-
result:
|
|
5468
|
+
result: output,
|
|
5273
5469
|
usage: {
|
|
5274
5470
|
inputTokens: response.usage.inputTokens,
|
|
5275
5471
|
outputTokens: response.usage.outputTokens,
|
|
5276
5472
|
totalTokens: response.usage.totalTokens,
|
|
5277
5473
|
reasoningTokens: response.usage.reasoningTokens,
|
|
5278
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
5474
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
5475
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
5279
5476
|
},
|
|
5280
5477
|
unexpectedRootKeys,
|
|
5281
5478
|
unexpectedMomentKeys,
|
|
@@ -6189,7 +6386,7 @@ async function getModerationScores(assetId, options = {}) {
|
|
|
6189
6386
|
// src/workflows/summarization.ts
|
|
6190
6387
|
import { generateText as generateText6, Output as Output6 } from "ai";
|
|
6191
6388
|
import dedent6 from "dedent";
|
|
6192
|
-
import { z as
|
|
6389
|
+
import { z as z8 } from "zod";
|
|
6193
6390
|
|
|
6194
6391
|
// src/lib/markdown-normalization.ts
|
|
6195
6392
|
var ESCAPED_LINE_BREAK_REGEX = /\\r\\n|\\n|\\r/g;
|
|
@@ -6202,17 +6399,17 @@ function normalizeMarkdownDescription(description) {
|
|
|
6202
6399
|
var DEFAULT_SUMMARY_KEYWORD_LIMIT = 10;
|
|
6203
6400
|
var DEFAULT_TITLE_LENGTH = 10;
|
|
6204
6401
|
var DEFAULT_DESCRIPTION_LENGTH = 50;
|
|
6205
|
-
var summarySchema =
|
|
6206
|
-
keywords:
|
|
6207
|
-
title:
|
|
6208
|
-
description:
|
|
6402
|
+
var summarySchema = z8.object({
|
|
6403
|
+
keywords: z8.array(z8.string().max(200)),
|
|
6404
|
+
title: z8.string().max(500),
|
|
6405
|
+
description: z8.string().max(1e4)
|
|
6209
6406
|
});
|
|
6210
6407
|
function buildSummarySchema(descriptionLength) {
|
|
6211
6408
|
const descriptionMaxChars = Math.max(2e3, descriptionLength * 15);
|
|
6212
|
-
return
|
|
6213
|
-
keywords:
|
|
6214
|
-
title:
|
|
6215
|
-
description:
|
|
6409
|
+
return z8.object({
|
|
6410
|
+
keywords: z8.array(z8.string().max(200)),
|
|
6411
|
+
title: z8.string().max(500),
|
|
6412
|
+
description: z8.string().max(descriptionMaxChars)
|
|
6216
6413
|
});
|
|
6217
6414
|
}
|
|
6218
6415
|
var VALID_TONES = ["neutral", "playful", "professional"];
|
|
@@ -6488,7 +6685,7 @@ async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, s
|
|
|
6488
6685
|
"use step";
|
|
6489
6686
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
6490
6687
|
const schema = buildSummarySchema(descriptionLength);
|
|
6491
|
-
const response = await generateText6({
|
|
6688
|
+
const response = await withContentPolicyErrorHandling(() => generateText6({
|
|
6492
6689
|
model,
|
|
6493
6690
|
output: Output6.object({
|
|
6494
6691
|
name: "summary_metadata",
|
|
@@ -6508,11 +6705,12 @@ async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, s
|
|
|
6508
6705
|
]
|
|
6509
6706
|
}
|
|
6510
6707
|
]
|
|
6511
|
-
});
|
|
6512
|
-
|
|
6708
|
+
}));
|
|
6709
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
6710
|
+
if (!output) {
|
|
6513
6711
|
throw new Error("Summarization output missing");
|
|
6514
6712
|
}
|
|
6515
|
-
const parsed = schema.parse(
|
|
6713
|
+
const parsed = schema.parse(output);
|
|
6516
6714
|
const unexpectedKeys = detectUnexpectedKeysFromRawText(
|
|
6517
6715
|
response.text,
|
|
6518
6716
|
schema.keyof().options
|
|
@@ -6524,7 +6722,8 @@ async function analyzeStoryboard2(imageDataUrl, provider, modelId, userPrompt, s
|
|
|
6524
6722
|
outputTokens: response.usage.outputTokens,
|
|
6525
6723
|
totalTokens: response.usage.totalTokens,
|
|
6526
6724
|
reasoningTokens: response.usage.reasoningTokens,
|
|
6527
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
6725
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
6726
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
6528
6727
|
},
|
|
6529
6728
|
unexpectedKeys
|
|
6530
6729
|
};
|
|
@@ -6533,7 +6732,7 @@ async function analyzeAudioOnly(provider, modelId, userPrompt, systemPrompt, des
|
|
|
6533
6732
|
"use step";
|
|
6534
6733
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
6535
6734
|
const schema = buildSummarySchema(descriptionLength);
|
|
6536
|
-
const response = await generateText6({
|
|
6735
|
+
const response = await withContentPolicyErrorHandling(() => generateText6({
|
|
6537
6736
|
model,
|
|
6538
6737
|
output: Output6.object({
|
|
6539
6738
|
name: "summary_metadata",
|
|
@@ -6550,11 +6749,12 @@ async function analyzeAudioOnly(provider, modelId, userPrompt, systemPrompt, des
|
|
|
6550
6749
|
content: userPrompt
|
|
6551
6750
|
}
|
|
6552
6751
|
]
|
|
6553
|
-
});
|
|
6554
|
-
|
|
6752
|
+
}));
|
|
6753
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
6754
|
+
if (!output) {
|
|
6555
6755
|
throw new Error("Summarization output missing");
|
|
6556
6756
|
}
|
|
6557
|
-
const parsed = schema.parse(
|
|
6757
|
+
const parsed = schema.parse(output);
|
|
6558
6758
|
const unexpectedKeys = detectUnexpectedKeysFromRawText(
|
|
6559
6759
|
response.text,
|
|
6560
6760
|
schema.keyof().options
|
|
@@ -6566,7 +6766,8 @@ async function analyzeAudioOnly(provider, modelId, userPrompt, systemPrompt, des
|
|
|
6566
6766
|
outputTokens: response.usage.outputTokens,
|
|
6567
6767
|
totalTokens: response.usage.totalTokens,
|
|
6568
6768
|
reasoningTokens: response.usage.reasoningTokens,
|
|
6569
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
6769
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
6770
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
6570
6771
|
},
|
|
6571
6772
|
unexpectedKeys
|
|
6572
6773
|
};
|
|
@@ -6967,6 +7168,59 @@ async function downloadAndUploadDubbedAudio({
|
|
|
6967
7168
|
console.warn(`\u{1F517} Generated presigned URL (expires in ${expiryHours} hour${expiryHours === 1 ? "" : "s"})`);
|
|
6968
7169
|
return presignedUrl;
|
|
6969
7170
|
}
|
|
7171
|
+
async function downloadAndUploadDubTranscript({
|
|
7172
|
+
dubbingId,
|
|
7173
|
+
languageCode,
|
|
7174
|
+
assetId,
|
|
7175
|
+
toLanguageCode,
|
|
7176
|
+
s3Endpoint,
|
|
7177
|
+
s3Region,
|
|
7178
|
+
s3Bucket,
|
|
7179
|
+
storageAdapter,
|
|
7180
|
+
s3SignedUrlExpirySeconds,
|
|
7181
|
+
credentials
|
|
7182
|
+
}) {
|
|
7183
|
+
"use step";
|
|
7184
|
+
const elevenLabsApiKey = await getApiKeyFromEnv("elevenlabs", credentials);
|
|
7185
|
+
const transcriptUrl = `https://api.elevenlabs.io/v1/dubbing/${dubbingId}/transcripts/${languageCode}/format/webvtt`;
|
|
7186
|
+
const transcriptResponse = await fetch(transcriptUrl, {
|
|
7187
|
+
headers: {
|
|
7188
|
+
"xi-api-key": elevenLabsApiKey
|
|
7189
|
+
}
|
|
7190
|
+
});
|
|
7191
|
+
if (!transcriptResponse.ok) {
|
|
7192
|
+
throw new Error(`Failed to fetch dub transcript: ${transcriptResponse.statusText}`);
|
|
7193
|
+
}
|
|
7194
|
+
const transcriptBody = await transcriptResponse.json();
|
|
7195
|
+
const transcriptVtt = transcriptBody.webvtt;
|
|
7196
|
+
if (!transcriptVtt) {
|
|
7197
|
+
throw new Error("Dub transcript response did not include webvtt content");
|
|
7198
|
+
}
|
|
7199
|
+
const s3AccessKeyId = env_default.S3_ACCESS_KEY_ID;
|
|
7200
|
+
const s3SecretAccessKey = env_default.S3_SECRET_ACCESS_KEY;
|
|
7201
|
+
const vttKey = `audio-translations/${assetId}/auto-to-${toLanguageCode}-${Date.now()}.vtt`;
|
|
7202
|
+
await putObjectWithStorageAdapter({
|
|
7203
|
+
accessKeyId: s3AccessKeyId,
|
|
7204
|
+
secretAccessKey: s3SecretAccessKey,
|
|
7205
|
+
endpoint: s3Endpoint,
|
|
7206
|
+
region: s3Region,
|
|
7207
|
+
bucket: s3Bucket,
|
|
7208
|
+
key: vttKey,
|
|
7209
|
+
body: transcriptVtt,
|
|
7210
|
+
contentType: "text/vtt"
|
|
7211
|
+
}, storageAdapter);
|
|
7212
|
+
const presignedUrl = await createPresignedGetUrlWithStorageAdapter({
|
|
7213
|
+
accessKeyId: s3AccessKeyId,
|
|
7214
|
+
secretAccessKey: s3SecretAccessKey,
|
|
7215
|
+
endpoint: s3Endpoint,
|
|
7216
|
+
region: s3Region,
|
|
7217
|
+
bucket: s3Bucket,
|
|
7218
|
+
key: vttKey,
|
|
7219
|
+
expiresInSeconds: s3SignedUrlExpirySeconds ?? 86400
|
|
7220
|
+
}, storageAdapter);
|
|
7221
|
+
console.warn(`\u2705 Dub transcript uploaded successfully to: ${vttKey}`);
|
|
7222
|
+
return presignedUrl;
|
|
7223
|
+
}
|
|
6970
7224
|
async function createAudioTrackOnMux(assetId, languageCode, presignedUrl, credentials) {
|
|
6971
7225
|
"use step";
|
|
6972
7226
|
const muxClient = await resolveMuxClient(credentials);
|
|
@@ -6993,6 +7247,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
6993
7247
|
// 0 = auto-detect
|
|
6994
7248
|
uploadToS3: uploadToS3Option,
|
|
6995
7249
|
uploadToMux: uploadToMuxOption,
|
|
7250
|
+
uploadCaptionsToMux = false,
|
|
6996
7251
|
storageAdapter,
|
|
6997
7252
|
credentials: providedCredentials
|
|
6998
7253
|
} = options;
|
|
@@ -7002,7 +7257,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
7002
7257
|
const credentials = providedCredentials;
|
|
7003
7258
|
const effectiveStorageAdapter = storageAdapter;
|
|
7004
7259
|
const uploadToMux = uploadToMuxOption !== false;
|
|
7005
|
-
const uploadToS3 = uploadToS3Option || uploadToMux;
|
|
7260
|
+
const uploadToS3 = uploadToS3Option || uploadToMux || uploadCaptionsToMux;
|
|
7006
7261
|
const s3Endpoint = options.s3Endpoint ?? env_default.S3_ENDPOINT;
|
|
7007
7262
|
const s3Region = options.s3Region ?? env_default.S3_REGION ?? "auto";
|
|
7008
7263
|
const s3Bucket = options.s3Bucket ?? env_default.S3_BUCKET;
|
|
@@ -7083,10 +7338,12 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
7083
7338
|
console.warn("\u2705 Dubbing completed successfully!");
|
|
7084
7339
|
let presignedUrl;
|
|
7085
7340
|
let uploadedTrackId;
|
|
7341
|
+
let captionsPresignedUrl;
|
|
7342
|
+
let captionsTrackId;
|
|
7086
7343
|
if (uploadToS3) {
|
|
7087
7344
|
console.warn("\u{1F4E5} Downloading dubbed audio from ElevenLabs and staging to S3...");
|
|
7345
|
+
const downloadLangCode = targetLanguages[0] ?? toISO639_1(toLanguageCode);
|
|
7088
7346
|
try {
|
|
7089
|
-
const downloadLangCode = targetLanguages[0] ?? toISO639_1(toLanguageCode);
|
|
7090
7347
|
presignedUrl = await downloadAndUploadDubbedAudio({
|
|
7091
7348
|
dubbingId,
|
|
7092
7349
|
languageCode: downloadLangCode,
|
|
@@ -7118,6 +7375,40 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
7118
7375
|
console.warn(presignedUrl);
|
|
7119
7376
|
}
|
|
7120
7377
|
}
|
|
7378
|
+
try {
|
|
7379
|
+
console.warn("\u{1F4E5} Downloading dub transcript from ElevenLabs and staging to S3...");
|
|
7380
|
+
captionsPresignedUrl = await downloadAndUploadDubTranscript({
|
|
7381
|
+
dubbingId,
|
|
7382
|
+
languageCode: downloadLangCode,
|
|
7383
|
+
assetId,
|
|
7384
|
+
toLanguageCode,
|
|
7385
|
+
s3Endpoint,
|
|
7386
|
+
s3Region,
|
|
7387
|
+
s3Bucket,
|
|
7388
|
+
storageAdapter: effectiveStorageAdapter,
|
|
7389
|
+
s3SignedUrlExpirySeconds: options.s3SignedUrlExpirySeconds,
|
|
7390
|
+
credentials
|
|
7391
|
+
});
|
|
7392
|
+
if (uploadCaptionsToMux) {
|
|
7393
|
+
console.warn("\u{1F4F9} Adding dubbed captions track to Mux asset...");
|
|
7394
|
+
const muxLangCode = toISO639_1(toLanguageCode);
|
|
7395
|
+
const languageName = new Intl.DisplayNames(["en"], { type: "language" }).of(muxLangCode) || muxLangCode.toUpperCase();
|
|
7396
|
+
captionsTrackId = await createTextTrackOnMux(
|
|
7397
|
+
assetId,
|
|
7398
|
+
muxLangCode,
|
|
7399
|
+
`${languageName} (auto-dubbed)`,
|
|
7400
|
+
captionsPresignedUrl,
|
|
7401
|
+
credentials
|
|
7402
|
+
);
|
|
7403
|
+
console.warn(`\u2705 Captions track added to Mux asset with ID: ${captionsTrackId}`);
|
|
7404
|
+
}
|
|
7405
|
+
} catch (error) {
|
|
7406
|
+
console.warn(`\u26A0\uFE0F Failed to attach dubbed captions: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
7407
|
+
if (captionsPresignedUrl) {
|
|
7408
|
+
console.warn("\u{1F517} You can manually add the captions track using this presigned URL:");
|
|
7409
|
+
console.warn(captionsPresignedUrl);
|
|
7410
|
+
}
|
|
7411
|
+
}
|
|
7121
7412
|
}
|
|
7122
7413
|
const targetLanguage = getLanguageCodePair(toLanguageCode);
|
|
7123
7414
|
return {
|
|
@@ -7127,6 +7418,8 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
7127
7418
|
dubbingId,
|
|
7128
7419
|
uploadedTrackId,
|
|
7129
7420
|
presignedUrl,
|
|
7421
|
+
captionsTrackId,
|
|
7422
|
+
captionsPresignedUrl,
|
|
7130
7423
|
usage: {
|
|
7131
7424
|
metadata: {
|
|
7132
7425
|
assetDurationSeconds
|
|
@@ -7137,16 +7430,16 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
7137
7430
|
|
|
7138
7431
|
// src/workflows/translate-captions.ts
|
|
7139
7432
|
import {
|
|
7140
|
-
APICallError,
|
|
7433
|
+
APICallError as APICallError2,
|
|
7141
7434
|
generateText as generateText7,
|
|
7142
|
-
NoObjectGeneratedError,
|
|
7435
|
+
NoObjectGeneratedError as NoObjectGeneratedError2,
|
|
7143
7436
|
Output as Output7,
|
|
7144
|
-
RetryError,
|
|
7437
|
+
RetryError as RetryError2,
|
|
7145
7438
|
TypeValidationError
|
|
7146
7439
|
} from "ai";
|
|
7147
|
-
import { z as
|
|
7148
|
-
var translationSchema =
|
|
7149
|
-
translation:
|
|
7440
|
+
import { z as z9 } from "zod";
|
|
7441
|
+
var translationSchema = z9.object({
|
|
7442
|
+
translation: z9.string()
|
|
7150
7443
|
});
|
|
7151
7444
|
var SYSTEM_PROMPT6 = promptDedent`
|
|
7152
7445
|
You are a subtitle translation expert. Translate VTT subtitle files to the target language specified by the user.
|
|
@@ -7213,10 +7506,10 @@ function isProviderServiceError(error) {
|
|
|
7213
7506
|
if (!error) {
|
|
7214
7507
|
return false;
|
|
7215
7508
|
}
|
|
7216
|
-
if (
|
|
7509
|
+
if (RetryError2.isInstance(error)) {
|
|
7217
7510
|
return isProviderServiceError(error.lastError);
|
|
7218
7511
|
}
|
|
7219
|
-
if (
|
|
7512
|
+
if (APICallError2.isInstance(error)) {
|
|
7220
7513
|
return true;
|
|
7221
7514
|
}
|
|
7222
7515
|
if (error instanceof Error && "cause" in error) {
|
|
@@ -7228,7 +7521,7 @@ function shouldSplitChunkTranslationError(error) {
|
|
|
7228
7521
|
if (isProviderServiceError(error)) {
|
|
7229
7522
|
return false;
|
|
7230
7523
|
}
|
|
7231
|
-
return
|
|
7524
|
+
return NoObjectGeneratedError2.isInstance(error) || TypeValidationError.isInstance(error) || isTranslationChunkValidationError(error);
|
|
7232
7525
|
}
|
|
7233
7526
|
function resolveTranslationChunkingOptions(options) {
|
|
7234
7527
|
const targetChunkDurationSeconds = Math.max(
|
|
@@ -7389,7 +7682,7 @@ async function translateVttWithAI({
|
|
|
7389
7682
|
"use step";
|
|
7390
7683
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
7391
7684
|
const sanitisedVttContent = stripVttMetadataBlocks(vttContent);
|
|
7392
|
-
const response = await generateText7({
|
|
7685
|
+
const response = await withContentPolicyErrorHandling(() => generateText7({
|
|
7393
7686
|
model,
|
|
7394
7687
|
output: Output7.object({ schema: translationSchema }),
|
|
7395
7688
|
messages: [
|
|
@@ -7404,8 +7697,9 @@ async function translateVttWithAI({
|
|
|
7404
7697
|
${sanitisedVttContent}`
|
|
7405
7698
|
}
|
|
7406
7699
|
]
|
|
7407
|
-
});
|
|
7408
|
-
const
|
|
7700
|
+
}));
|
|
7701
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
7702
|
+
const translated = normalizeTranslatedVtt(output.translation);
|
|
7409
7703
|
const leakReason = detectLeakReason(translated);
|
|
7410
7704
|
const safeTranslated = leakReason !== null ? vttContent : translated;
|
|
7411
7705
|
if (leakReason !== null) {
|
|
@@ -7430,7 +7724,8 @@ ${sanitisedVttContent}`
|
|
|
7430
7724
|
outputTokens: response.usage.outputTokens,
|
|
7431
7725
|
totalTokens: response.usage.totalTokens,
|
|
7432
7726
|
reasoningTokens: response.usage.reasoningTokens,
|
|
7433
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
7727
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
7728
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
7434
7729
|
}
|
|
7435
7730
|
};
|
|
7436
7731
|
}
|
|
@@ -7444,8 +7739,8 @@ async function translateCueChunkWithAI({
|
|
|
7444
7739
|
}) {
|
|
7445
7740
|
"use step";
|
|
7446
7741
|
const model = await createLanguageModelFromConfig(provider, modelId, credentials);
|
|
7447
|
-
const schema =
|
|
7448
|
-
translations:
|
|
7742
|
+
const schema = z9.object({
|
|
7743
|
+
translations: z9.array(z9.string().min(1).max(2e3)).length(cues.length)
|
|
7449
7744
|
});
|
|
7450
7745
|
const cuePayload = cues.map((cue, index) => ({
|
|
7451
7746
|
index,
|
|
@@ -7453,7 +7748,7 @@ async function translateCueChunkWithAI({
|
|
|
7453
7748
|
endTime: cue.endTime,
|
|
7454
7749
|
text: cue.text
|
|
7455
7750
|
}));
|
|
7456
|
-
const response = await generateText7({
|
|
7751
|
+
const response = await withContentPolicyErrorHandling(() => generateText7({
|
|
7457
7752
|
model,
|
|
7458
7753
|
output: Output7.object({ schema }),
|
|
7459
7754
|
messages: [
|
|
@@ -7469,7 +7764,8 @@ Return exactly ${cues.length} translated cues in the same order as the input.
|
|
|
7469
7764
|
${JSON.stringify(cuePayload, null, 2)}`
|
|
7470
7765
|
}
|
|
7471
7766
|
]
|
|
7472
|
-
});
|
|
7767
|
+
}));
|
|
7768
|
+
const output = getGeneratedOutputWithContentPolicyHandling(response);
|
|
7473
7769
|
const unexpectedKeys = detectUnexpectedKeysFromRawText(
|
|
7474
7770
|
response.text,
|
|
7475
7771
|
schema.keyof().options
|
|
@@ -7480,13 +7776,14 @@ ${JSON.stringify(cuePayload, null, 2)}`
|
|
|
7480
7776
|
);
|
|
7481
7777
|
}
|
|
7482
7778
|
return {
|
|
7483
|
-
translations:
|
|
7779
|
+
translations: output.translations,
|
|
7484
7780
|
usage: {
|
|
7485
7781
|
inputTokens: response.usage.inputTokens,
|
|
7486
7782
|
outputTokens: response.usage.outputTokens,
|
|
7487
7783
|
totalTokens: response.usage.totalTokens,
|
|
7488
7784
|
reasoningTokens: response.usage.reasoningTokens,
|
|
7489
|
-
cachedInputTokens: response.usage.cachedInputTokens
|
|
7785
|
+
cachedInputTokens: response.usage.cachedInputTokens,
|
|
7786
|
+
cacheWriteTokens: response.usage.inputTokenDetails?.cacheWriteTokens
|
|
7490
7787
|
},
|
|
7491
7788
|
unexpectedKeyCount: unexpectedKeys.length
|
|
7492
7789
|
};
|