@mux/ai 0.26.0 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{index-C5zhbQHK.d.ts → index-DUVBpnj5.d.ts} +5 -1
- package/dist/index.d.ts +2 -28
- package/dist/index.js +271 -97
- package/dist/index.js.map +1 -1
- package/dist/primitives/index.js +64 -5
- package/dist/primitives/index.js.map +1 -1
- package/dist/workflows/index.d.ts +1 -1
- package/dist/workflows/index.js +270 -96
- package/dist/workflows/index.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { A as AskQuestionsOptions, j as AskQuestionsResult, k as AskQuestionsType, l as AudioTranslationOptions, m as AudioTranslationResult, n as AutoCensorProfanityOptions, B as BurnedInCaptionsAnalysis, o as BurnedInCaptionsOptions, p as BurnedInCaptionsPromptOverrides, q as BurnedInCaptionsPromptSections, r as BurnedInCaptionsResult, C as CaptionReplacement, s as CensorMode, u as Chapter, v as ChapterSystemPromptSections, w as ChaptersOptions, x as ChaptersPromptOverrides, y as ChaptersPromptSections, z as ChaptersResult, D as ChaptersType, E as DEFAULT_DESCRIPTION_LENGTH, F as DEFAULT_SUMMARY_KEYWORD_LIMIT, G as DEFAULT_TITLE_LENGTH, H as EditCaptionsOptions, J as EditCaptionsResult, K as EmbeddingsOptions, M as EmbeddingsResult, N as EngagementInsightsOptions, O as EngagementInsightsResult, P as GOOGLE_VISION_LIKELIHOOD_TO_SCORE, Q as HIVE_SEXUAL_CATEGORIES, R as HIVE_VIOLENCE_CATEGORIES, T as HiveModerationOutput, U as HiveModerationSource, V as ModerationOptions, W as ModerationProvider, X as ModerationResult, Y as MomentInsight, Z as OverallInsight, _ as ProfanityDetectionPayload, $ as Question, a0 as QuestionAnswer, a1 as QuestionAnswerType, a2 as ReplacementRecord, a3 as SummarizationOptions, a4 as SummarizationPromptOverrides, a5 as SummarizationPromptSections, a6 as SummaryAndTagsResult, a7 as SummaryType, a8 as ThumbnailModerationScore, a9 as TranslationChunkingOptions, aa as TranslationOptions, ab as TranslationPayload, ac as TranslationResult, ad as aggregateTokenUsage, ae as applyOverrideLists, af as applyReplacements, ag as askQuestions, ah as buildReplacementRegex, ai as burnedInCaptionsSchema, aj as censorVttContent, ak as chapterSchema, al as chaptersSchema, am as createReplacer, an as editCaptions, ao as extractTranscriptSegmentsForHotspots, ap as generateChapters, aq as generateEmbeddings, ar as generateEngagementInsights, as as generateVideoEmbeddings, at as getModerationScores, au as getSummaryAndTags, av as hasBurnedInCaptions, aw as mapShotsToHotspots, ax as normalizeTranslatedVtt, ay as profanityDetectionSchema, az as questionAnswerSchema, aA as shouldSplitChunkTranslationError, aB as summarySchema, aC as transformCueText, aD as translateAudio, aE as translateCaptions, aF as translationSchema } from '../index-
|
|
1
|
+
export { A as AskQuestionsOptions, j as AskQuestionsResult, k as AskQuestionsType, l as AudioTranslationOptions, m as AudioTranslationResult, n as AutoCensorProfanityOptions, B as BurnedInCaptionsAnalysis, o as BurnedInCaptionsOptions, p as BurnedInCaptionsPromptOverrides, q as BurnedInCaptionsPromptSections, r as BurnedInCaptionsResult, C as CaptionReplacement, s as CensorMode, u as Chapter, v as ChapterSystemPromptSections, w as ChaptersOptions, x as ChaptersPromptOverrides, y as ChaptersPromptSections, z as ChaptersResult, D as ChaptersType, E as DEFAULT_DESCRIPTION_LENGTH, F as DEFAULT_SUMMARY_KEYWORD_LIMIT, G as DEFAULT_TITLE_LENGTH, H as EditCaptionsOptions, J as EditCaptionsResult, K as EmbeddingsOptions, M as EmbeddingsResult, N as EngagementInsightsOptions, O as EngagementInsightsResult, P as GOOGLE_VISION_LIKELIHOOD_TO_SCORE, Q as HIVE_SEXUAL_CATEGORIES, R as HIVE_VIOLENCE_CATEGORIES, T as HiveModerationOutput, U as HiveModerationSource, V as ModerationOptions, W as ModerationProvider, X as ModerationResult, Y as MomentInsight, Z as OverallInsight, _ as ProfanityDetectionPayload, $ as Question, a0 as QuestionAnswer, a1 as QuestionAnswerType, a2 as ReplacementRecord, a3 as SummarizationOptions, a4 as SummarizationPromptOverrides, a5 as SummarizationPromptSections, a6 as SummaryAndTagsResult, a7 as SummaryType, a8 as ThumbnailModerationScore, a9 as TranslationChunkingOptions, aa as TranslationOptions, ab as TranslationPayload, ac as TranslationResult, ad as aggregateTokenUsage, ae as applyOverrideLists, af as applyReplacements, ag as askQuestions, ah as buildReplacementRegex, ai as burnedInCaptionsSchema, aj as censorVttContent, ak as chapterSchema, al as chaptersSchema, am as createReplacer, an as editCaptions, ao as extractTranscriptSegmentsForHotspots, ap as generateChapters, aq as generateEmbeddings, ar as generateEngagementInsights, as as generateVideoEmbeddings, at as getModerationScores, au as getSummaryAndTags, av as hasBurnedInCaptions, aw as mapShotsToHotspots, ax as normalizeTranslatedVtt, ay as profanityDetectionSchema, az as questionAnswerSchema, aA as shouldSplitChunkTranslationError, aB as summarySchema, aC as transformCueText, aD as translateAudio, aE as translateCaptions, aF as translationSchema } from '../index-DUVBpnj5.js';
|
|
2
2
|
import 'zod';
|
|
3
3
|
import '@ai-sdk/anthropic';
|
|
4
4
|
import '@ai-sdk/google';
|
package/dist/workflows/index.js
CHANGED
|
@@ -492,6 +492,68 @@ function createSafetyReporter() {
|
|
|
492
492
|
};
|
|
493
493
|
}
|
|
494
494
|
|
|
495
|
+
// src/lib/token-usage.ts
|
|
496
|
+
var TOKEN_USAGE_FIELDS = [
|
|
497
|
+
"inputTokens",
|
|
498
|
+
"outputTokens",
|
|
499
|
+
"totalTokens",
|
|
500
|
+
"reasoningTokens",
|
|
501
|
+
"cachedInputTokens"
|
|
502
|
+
];
|
|
503
|
+
function isDefinedTokenUsageValue(value) {
|
|
504
|
+
return typeof value === "number";
|
|
505
|
+
}
|
|
506
|
+
function aggregateTokenUsage(usages) {
|
|
507
|
+
return TOKEN_USAGE_FIELDS.reduce((aggregate, field) => {
|
|
508
|
+
const values = usages.map((usage) => usage[field]).filter(isDefinedTokenUsageValue);
|
|
509
|
+
if (values.length > 0) {
|
|
510
|
+
aggregate[field] = values.reduce((total, value) => total + value, 0);
|
|
511
|
+
}
|
|
512
|
+
return aggregate;
|
|
513
|
+
}, {});
|
|
514
|
+
}
|
|
515
|
+
function getErrorTokenUsage(error) {
|
|
516
|
+
if (typeof error !== "object" || error === null) {
|
|
517
|
+
return void 0;
|
|
518
|
+
}
|
|
519
|
+
const usage = error.usage;
|
|
520
|
+
if (typeof usage !== "object" || usage === null) {
|
|
521
|
+
return void 0;
|
|
522
|
+
}
|
|
523
|
+
const source = usage;
|
|
524
|
+
const extracted = {};
|
|
525
|
+
for (const field of TOKEN_USAGE_FIELDS) {
|
|
526
|
+
const value = source[field];
|
|
527
|
+
if (typeof value === "number") {
|
|
528
|
+
extracted[field] = value;
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
if (extracted.reasoningTokens === void 0) {
|
|
532
|
+
const reasoningTokens = source.outputTokenDetails?.reasoningTokens;
|
|
533
|
+
if (typeof reasoningTokens === "number") {
|
|
534
|
+
extracted.reasoningTokens = reasoningTokens;
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
if (extracted.cachedInputTokens === void 0) {
|
|
538
|
+
const cacheReadTokens = source.inputTokenDetails?.cacheReadTokens;
|
|
539
|
+
if (typeof cacheReadTokens === "number") {
|
|
540
|
+
extracted.cachedInputTokens = cacheReadTokens;
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
return Object.keys(extracted).length > 0 ? extracted : void 0;
|
|
544
|
+
}
|
|
545
|
+
function rethrowWithTokenUsage(error, collectedUsage) {
|
|
546
|
+
const usages = [...collectedUsage];
|
|
547
|
+
const errorUsage = getErrorTokenUsage(error);
|
|
548
|
+
if (errorUsage) {
|
|
549
|
+
usages.push(errorUsage);
|
|
550
|
+
}
|
|
551
|
+
if (usages.length > 0 && typeof error === "object" && error !== null) {
|
|
552
|
+
error.usage = aggregateTokenUsage(usages);
|
|
553
|
+
}
|
|
554
|
+
throw error;
|
|
555
|
+
}
|
|
556
|
+
|
|
495
557
|
// src/lib/mux-ai-error.ts
|
|
496
558
|
var MuxAiError = class extends Error {
|
|
497
559
|
constructor(message, opts) {
|
|
@@ -504,18 +566,38 @@ var MuxAiError = class extends Error {
|
|
|
504
566
|
this.retryable = opts?.retryable ?? false;
|
|
505
567
|
}
|
|
506
568
|
};
|
|
569
|
+
function extractErrorDetail(error) {
|
|
570
|
+
if (error instanceof Error && error.message) {
|
|
571
|
+
return error.message;
|
|
572
|
+
}
|
|
573
|
+
if (typeof error === "string" && error.length > 0) {
|
|
574
|
+
return error;
|
|
575
|
+
}
|
|
576
|
+
if (typeof error === "object" && error !== null) {
|
|
577
|
+
const message = error.message;
|
|
578
|
+
if (typeof message === "string" && message.length > 0) {
|
|
579
|
+
return message;
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
return "Unknown error";
|
|
583
|
+
}
|
|
507
584
|
function wrapError(error, message) {
|
|
508
585
|
if (error instanceof MuxAiError) {
|
|
509
586
|
throw error;
|
|
510
587
|
}
|
|
511
|
-
const rawDetail = error
|
|
588
|
+
const rawDetail = extractErrorDetail(error);
|
|
512
589
|
const leakReason = detectLeakReason(rawDetail);
|
|
513
590
|
const shouldSuppress = leakReason === "canary" || leakReason === "prompt_tag";
|
|
514
591
|
const detail = shouldSuppress ? "Upstream error details suppressed by safety filter" : rawDetail;
|
|
515
592
|
if (shouldSuppress) {
|
|
516
593
|
console.warn(`[@mux/ai] Suppressed suspected prompt leak in wrapped error (context: ${message}, reason: ${leakReason}).`);
|
|
517
594
|
}
|
|
518
|
-
|
|
595
|
+
const wrapped = new Error(`${message}: ${detail}`);
|
|
596
|
+
const usage = getErrorTokenUsage(error);
|
|
597
|
+
if (usage) {
|
|
598
|
+
wrapped.usage = usage;
|
|
599
|
+
}
|
|
600
|
+
throw wrapped;
|
|
519
601
|
}
|
|
520
602
|
|
|
521
603
|
// src/lib/providers.ts
|
|
@@ -865,7 +947,7 @@ var DEFAULT_EMBEDDING_MODELS = {
|
|
|
865
947
|
var LANGUAGE_MODELS = {
|
|
866
948
|
openai: ["gpt-5.1", "gpt-5-mini"],
|
|
867
949
|
anthropic: ["claude-sonnet-4-5"],
|
|
868
|
-
google: ["gemini-3-flash-preview", "gemini-3.1-flash-lite", "gemini-
|
|
950
|
+
google: ["gemini-3-flash-preview", "gemini-3.1-flash-lite", "gemini-2.5-flash"]
|
|
869
951
|
};
|
|
870
952
|
var LANGUAGE_MODEL_DEPRECATIONS = [
|
|
871
953
|
{
|
|
@@ -881,10 +963,10 @@ var LANGUAGE_MODEL_DEPRECATIONS = [
|
|
|
881
963
|
provider: "google",
|
|
882
964
|
modelId: "gemini-3.1-flash-lite-preview",
|
|
883
965
|
replacementModelId: "gemini-3.1-flash-lite",
|
|
884
|
-
phase: "
|
|
966
|
+
phase: "blocked",
|
|
885
967
|
deprecatedOn: "2026-05-12",
|
|
886
968
|
sunsetOn: "2026-05-25",
|
|
887
|
-
reason: "
|
|
969
|
+
reason: "Google shut down Gemini 3.1 Flash-Lite Preview on 2026-05-25."
|
|
888
970
|
}
|
|
889
971
|
];
|
|
890
972
|
var warnedDeprecatedLanguageModels = /* @__PURE__ */ new Set();
|
|
@@ -2331,6 +2413,14 @@ async function analyzeQuestions({
|
|
|
2331
2413
|
}
|
|
2332
2414
|
async function askQuestions(assetId, questions, options) {
|
|
2333
2415
|
"use workflow";
|
|
2416
|
+
const collectedUsage = [];
|
|
2417
|
+
try {
|
|
2418
|
+
return await askQuestionsInternal(assetId, questions, options, collectedUsage);
|
|
2419
|
+
} catch (error) {
|
|
2420
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
2421
|
+
}
|
|
2422
|
+
}
|
|
2423
|
+
async function askQuestionsInternal(assetId, questions, options, collectedUsage) {
|
|
2334
2424
|
if (!questions || questions.length === 0) {
|
|
2335
2425
|
throw new MuxAiError("At least one question must be provided.", { type: "validation_error" });
|
|
2336
2426
|
}
|
|
@@ -2479,6 +2569,7 @@ async function askQuestions(assetId, questions, options) {
|
|
|
2479
2569
|
const contentType = isAudioOnly ? "audio" : "video";
|
|
2480
2570
|
wrapError(error, `Failed to analyze ${contentType} questions with ${provider}`);
|
|
2481
2571
|
}
|
|
2572
|
+
collectedUsage.push(analysisResponse.usage);
|
|
2482
2573
|
if (!analysisResponse.result?.answers) {
|
|
2483
2574
|
throw new MuxAiError(`Failed to generate answers for asset ${assetId}.`);
|
|
2484
2575
|
}
|
|
@@ -2712,6 +2803,14 @@ async function analyzeStoryboard({
|
|
|
2712
2803
|
}
|
|
2713
2804
|
async function hasBurnedInCaptions(assetId, options = {}) {
|
|
2714
2805
|
"use workflow";
|
|
2806
|
+
const collectedUsage = [];
|
|
2807
|
+
try {
|
|
2808
|
+
return await hasBurnedInCaptionsInternal(assetId, options, collectedUsage);
|
|
2809
|
+
} catch (error) {
|
|
2810
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
2811
|
+
}
|
|
2812
|
+
}
|
|
2813
|
+
async function hasBurnedInCaptionsInternal(assetId, options, collectedUsage) {
|
|
2715
2814
|
const {
|
|
2716
2815
|
provider = DEFAULT_PROVIDER,
|
|
2717
2816
|
model,
|
|
@@ -2751,6 +2850,7 @@ async function hasBurnedInCaptions(assetId, options = {}) {
|
|
|
2751
2850
|
credentials
|
|
2752
2851
|
});
|
|
2753
2852
|
}
|
|
2853
|
+
collectedUsage.push(analysisResponse.usage);
|
|
2754
2854
|
if (!analysisResponse.result) {
|
|
2755
2855
|
throw new Error("No analysis result received from AI provider");
|
|
2756
2856
|
}
|
|
@@ -2952,6 +3052,14 @@ function buildUserPrompt3({
|
|
|
2952
3052
|
}
|
|
2953
3053
|
async function generateChapters(assetId, options = {}) {
|
|
2954
3054
|
"use workflow";
|
|
3055
|
+
const collectedUsage = [];
|
|
3056
|
+
try {
|
|
3057
|
+
return await generateChaptersInternal(assetId, options, collectedUsage);
|
|
3058
|
+
} catch (error) {
|
|
3059
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
3060
|
+
}
|
|
3061
|
+
}
|
|
3062
|
+
async function generateChaptersInternal(assetId, options, collectedUsage) {
|
|
2955
3063
|
const {
|
|
2956
3064
|
languageCode,
|
|
2957
3065
|
provider = "openai",
|
|
@@ -3019,6 +3127,9 @@ async function generateChapters(assetId, options = {}) {
|
|
|
3019
3127
|
} catch (error) {
|
|
3020
3128
|
wrapError(error, `Failed to generate chapters with ${provider}`);
|
|
3021
3129
|
}
|
|
3130
|
+
if (chaptersData) {
|
|
3131
|
+
collectedUsage.push(chaptersData.usage);
|
|
3132
|
+
}
|
|
3022
3133
|
if (!chaptersData || !chaptersData.chapters) {
|
|
3023
3134
|
throw new MuxAiError(`Failed to generate chapters for asset ${assetId}.`);
|
|
3024
3135
|
}
|
|
@@ -3605,6 +3716,14 @@ async function deleteTrackOnMux(assetId, trackId, credentials) {
|
|
|
3605
3716
|
}
|
|
3606
3717
|
async function editCaptions(assetId, trackId, options) {
|
|
3607
3718
|
"use workflow";
|
|
3719
|
+
const collectedUsage = [];
|
|
3720
|
+
try {
|
|
3721
|
+
return await editCaptionsInternal(assetId, trackId, options, collectedUsage);
|
|
3722
|
+
} catch (error) {
|
|
3723
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
3724
|
+
}
|
|
3725
|
+
}
|
|
3726
|
+
async function editCaptionsInternal(assetId, trackId, options, collectedUsage) {
|
|
3608
3727
|
const {
|
|
3609
3728
|
provider,
|
|
3610
3729
|
model,
|
|
@@ -3693,6 +3812,7 @@ async function editCaptions(assetId, trackId, options) {
|
|
|
3693
3812
|
});
|
|
3694
3813
|
detectedProfanity = result.profanity;
|
|
3695
3814
|
usage = result.usage;
|
|
3815
|
+
collectedUsage.push(result.usage);
|
|
3696
3816
|
if (result.unexpectedKeys.length > 0) {
|
|
3697
3817
|
console.warn(
|
|
3698
3818
|
`[@mux/ai] Model emitted unexpected keys in profanity detection (stripped): ${result.unexpectedKeys.join(", ")}.`
|
|
@@ -4053,16 +4173,19 @@ async function generateSingleChunkEmbedding({
|
|
|
4053
4173
|
})
|
|
4054
4174
|
);
|
|
4055
4175
|
return {
|
|
4056
|
-
|
|
4057
|
-
|
|
4058
|
-
|
|
4059
|
-
|
|
4060
|
-
|
|
4061
|
-
|
|
4062
|
-
|
|
4176
|
+
chunk: {
|
|
4177
|
+
chunkId: chunk.id,
|
|
4178
|
+
embedding: response.embedding,
|
|
4179
|
+
metadata: {
|
|
4180
|
+
startTime: chunk.startTime,
|
|
4181
|
+
endTime: chunk.endTime,
|
|
4182
|
+
tokenCount: chunk.tokenCount
|
|
4183
|
+
}
|
|
4184
|
+
},
|
|
4185
|
+
usage: typeof response.usage?.tokens === "number" ? { inputTokens: response.usage.tokens, totalTokens: response.usage.tokens } : void 0
|
|
4063
4186
|
};
|
|
4064
4187
|
}
|
|
4065
|
-
async function generateEmbeddingsInternal(assetId, options
|
|
4188
|
+
async function generateEmbeddingsInternal(assetId, options, collectedUsage) {
|
|
4066
4189
|
const {
|
|
4067
4190
|
provider = "openai",
|
|
4068
4191
|
model,
|
|
@@ -4101,7 +4224,7 @@ async function generateEmbeddingsInternal(assetId, options = {}) {
|
|
|
4101
4224
|
try {
|
|
4102
4225
|
for (let i = 0; i < chunks.length; i += batchSize) {
|
|
4103
4226
|
const batch = chunks.slice(i, i + batchSize);
|
|
4104
|
-
const
|
|
4227
|
+
const batchOutcomes = await Promise.allSettled(
|
|
4105
4228
|
batch.map(
|
|
4106
4229
|
(chunk) => generateSingleChunkEmbedding({
|
|
4107
4230
|
chunk,
|
|
@@ -4111,7 +4234,24 @@ async function generateEmbeddingsInternal(assetId, options = {}) {
|
|
|
4111
4234
|
})
|
|
4112
4235
|
)
|
|
4113
4236
|
);
|
|
4114
|
-
|
|
4237
|
+
const failures = [];
|
|
4238
|
+
for (const outcome of batchOutcomes) {
|
|
4239
|
+
if (outcome.status === "fulfilled") {
|
|
4240
|
+
if (outcome.value.usage) {
|
|
4241
|
+
collectedUsage.push(outcome.value.usage);
|
|
4242
|
+
}
|
|
4243
|
+
chunkEmbeddings.push(outcome.value.chunk);
|
|
4244
|
+
} else {
|
|
4245
|
+
const failureUsage = getErrorTokenUsage(outcome.reason);
|
|
4246
|
+
if (failureUsage) {
|
|
4247
|
+
collectedUsage.push(failureUsage);
|
|
4248
|
+
}
|
|
4249
|
+
failures.push(outcome.reason);
|
|
4250
|
+
}
|
|
4251
|
+
}
|
|
4252
|
+
if (failures.length > 0) {
|
|
4253
|
+
throw failures[0];
|
|
4254
|
+
}
|
|
4115
4255
|
}
|
|
4116
4256
|
} catch (error) {
|
|
4117
4257
|
throw new Error(
|
|
@@ -4145,12 +4285,22 @@ async function generateEmbeddingsInternal(assetId, options = {}) {
|
|
|
4145
4285
|
}
|
|
4146
4286
|
async function generateEmbeddings(assetId, options = {}) {
|
|
4147
4287
|
"use workflow";
|
|
4148
|
-
|
|
4288
|
+
const collectedUsage = [];
|
|
4289
|
+
try {
|
|
4290
|
+
return await generateEmbeddingsInternal(assetId, options, collectedUsage);
|
|
4291
|
+
} catch (error) {
|
|
4292
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
4293
|
+
}
|
|
4149
4294
|
}
|
|
4150
4295
|
async function generateVideoEmbeddings(assetId, options = {}) {
|
|
4151
4296
|
"use workflow";
|
|
4152
4297
|
console.warn("generateVideoEmbeddings is deprecated. Use generateEmbeddings instead.");
|
|
4153
|
-
|
|
4298
|
+
const collectedUsage = [];
|
|
4299
|
+
try {
|
|
4300
|
+
return await generateEmbeddingsInternal(assetId, options, collectedUsage);
|
|
4301
|
+
} catch (error) {
|
|
4302
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
4303
|
+
}
|
|
4154
4304
|
}
|
|
4155
4305
|
|
|
4156
4306
|
// src/workflows/engagement-insights.ts
|
|
@@ -4635,6 +4785,14 @@ async function generateInsightsWithAI(provider, modelId, systemPrompt, userPromp
|
|
|
4635
4785
|
}
|
|
4636
4786
|
async function generateEngagementInsights(assetId, options = {}) {
|
|
4637
4787
|
"use workflow";
|
|
4788
|
+
const collectedUsage = [];
|
|
4789
|
+
try {
|
|
4790
|
+
return await generateEngagementInsightsInternal(assetId, options, collectedUsage);
|
|
4791
|
+
} catch (error) {
|
|
4792
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
4793
|
+
}
|
|
4794
|
+
}
|
|
4795
|
+
async function generateEngagementInsightsInternal(assetId, options, collectedUsage) {
|
|
4638
4796
|
const {
|
|
4639
4797
|
provider = "openai",
|
|
4640
4798
|
model,
|
|
@@ -4761,6 +4919,7 @@ async function generateEngagementInsights(assetId, options = {}) {
|
|
|
4761
4919
|
credentials
|
|
4762
4920
|
);
|
|
4763
4921
|
const { result: aiInsights, usage } = aiStepResult;
|
|
4922
|
+
collectedUsage.push(usage);
|
|
4764
4923
|
if (!aiInsights.momentInsights || aiInsights.momentInsights.length === 0) {
|
|
4765
4924
|
throw new MuxAiError(`Failed to generate insights for asset ${assetId}.`);
|
|
4766
4925
|
}
|
|
@@ -5945,6 +6104,14 @@ function normalizeKeywords(keywords, limit = DEFAULT_SUMMARY_KEYWORD_LIMIT) {
|
|
|
5945
6104
|
}
|
|
5946
6105
|
async function getSummaryAndTags(assetId, options) {
|
|
5947
6106
|
"use workflow";
|
|
6107
|
+
const collectedUsage = [];
|
|
6108
|
+
try {
|
|
6109
|
+
return await getSummaryAndTagsInternal(assetId, options, collectedUsage);
|
|
6110
|
+
} catch (error) {
|
|
6111
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
6112
|
+
}
|
|
6113
|
+
}
|
|
6114
|
+
async function getSummaryAndTagsInternal(assetId, options, collectedUsage) {
|
|
5948
6115
|
const {
|
|
5949
6116
|
provider = "openai",
|
|
5950
6117
|
model,
|
|
@@ -6054,6 +6221,7 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
6054
6221
|
const contentType = isAudioOnly ? "audio" : "video";
|
|
6055
6222
|
wrapError(error, `Failed to analyze ${contentType} content with ${provider}`);
|
|
6056
6223
|
}
|
|
6224
|
+
collectedUsage.push(analysisResponse.usage);
|
|
6057
6225
|
if (!analysisResponse.result) {
|
|
6058
6226
|
const contentType = isAudioOnly ? "audio" : "video";
|
|
6059
6227
|
throw new MuxAiError(`Failed to analyze ${contentType} content for asset ${assetId}.`);
|
|
@@ -6096,12 +6264,11 @@ async function getSummaryAndTags(assetId, options) {
|
|
|
6096
6264
|
}
|
|
6097
6265
|
|
|
6098
6266
|
// src/workflows/translate-audio.ts
|
|
6267
|
+
import { sleep } from "workflow";
|
|
6099
6268
|
var STATIC_RENDITION_POLL_INTERVAL_MS = 5e3;
|
|
6100
6269
|
var STATIC_RENDITION_MAX_ATTEMPTS = 36;
|
|
6101
|
-
|
|
6102
|
-
|
|
6103
|
-
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
6104
|
-
}
|
|
6270
|
+
var DUBBING_POLL_INTERVAL_MS = 1e4;
|
|
6271
|
+
var DEFAULT_DUBBING_POLL_TIMEOUT_SECONDS = 7200;
|
|
6105
6272
|
function getReadyAudioStaticRendition(asset) {
|
|
6106
6273
|
const files = asset.static_renditions?.files;
|
|
6107
6274
|
if (!files || files.length === 0) {
|
|
@@ -6142,14 +6309,17 @@ async function requestStaticRenditionCreation(assetId, credentials) {
|
|
|
6142
6309
|
wrapError(error, "Failed to request static rendition from Mux");
|
|
6143
6310
|
}
|
|
6144
6311
|
}
|
|
6312
|
+
async function retrieveAsset(assetId, credentials) {
|
|
6313
|
+
"use step";
|
|
6314
|
+
const muxClient = await resolveMuxClient(credentials);
|
|
6315
|
+
const mux = await muxClient.createClient();
|
|
6316
|
+
return mux.video.assets.retrieve(assetId);
|
|
6317
|
+
}
|
|
6145
6318
|
async function waitForAudioStaticRendition({
|
|
6146
6319
|
assetId,
|
|
6147
6320
|
initialAsset,
|
|
6148
6321
|
credentials
|
|
6149
6322
|
}) {
|
|
6150
|
-
"use step";
|
|
6151
|
-
const muxClient = await resolveMuxClient(credentials);
|
|
6152
|
-
const mux = await muxClient.createClient();
|
|
6153
6323
|
let currentAsset = initialAsset;
|
|
6154
6324
|
if (hasReadyAudioStaticRendition(currentAsset)) {
|
|
6155
6325
|
return currentAsset;
|
|
@@ -6164,7 +6334,7 @@ async function waitForAudioStaticRendition({
|
|
|
6164
6334
|
}
|
|
6165
6335
|
for (let attempt = 1; attempt <= STATIC_RENDITION_MAX_ATTEMPTS; attempt++) {
|
|
6166
6336
|
await sleep(STATIC_RENDITION_POLL_INTERVAL_MS);
|
|
6167
|
-
currentAsset = await
|
|
6337
|
+
currentAsset = await retrieveAsset(assetId, credentials);
|
|
6168
6338
|
if (hasReadyAudioStaticRendition(currentAsset)) {
|
|
6169
6339
|
return currentAsset;
|
|
6170
6340
|
}
|
|
@@ -6183,16 +6353,8 @@ async function waitForAudioStaticRendition({
|
|
|
6183
6353
|
{ type: "timeout_error", retryable: true }
|
|
6184
6354
|
);
|
|
6185
6355
|
}
|
|
6186
|
-
async function fetchAudioFromMux(audioUrl) {
|
|
6187
|
-
"use step";
|
|
6188
|
-
const audioResponse = await fetch(audioUrl);
|
|
6189
|
-
if (!audioResponse.ok) {
|
|
6190
|
-
throw new Error(`Failed to fetch audio file: ${audioResponse.statusText}`);
|
|
6191
|
-
}
|
|
6192
|
-
return audioResponse.arrayBuffer();
|
|
6193
|
-
}
|
|
6194
6356
|
async function createElevenLabsDubbingJob({
|
|
6195
|
-
|
|
6357
|
+
sourceUrl,
|
|
6196
6358
|
assetId,
|
|
6197
6359
|
elevenLabsLangCode,
|
|
6198
6360
|
elevenLabsSourceLangCode,
|
|
@@ -6201,9 +6363,8 @@ async function createElevenLabsDubbingJob({
|
|
|
6201
6363
|
}) {
|
|
6202
6364
|
"use step";
|
|
6203
6365
|
const elevenLabsApiKey = await getApiKeyFromEnv("elevenlabs", credentials);
|
|
6204
|
-
const audioBlob = new Blob([audioBuffer], { type: "audio/mp4" });
|
|
6205
6366
|
const formData = new FormData();
|
|
6206
|
-
formData.append("
|
|
6367
|
+
formData.append("source_url", sourceUrl);
|
|
6207
6368
|
formData.append("target_lang", elevenLabsLangCode);
|
|
6208
6369
|
if (elevenLabsSourceLangCode) {
|
|
6209
6370
|
formData.append("source_lang", elevenLabsSourceLangCode);
|
|
@@ -6368,13 +6529,6 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
6368
6529
|
if (policy === "signed") {
|
|
6369
6530
|
audioUrl = await signUrl(audioUrl, playbackId, "video", void 0, credentials);
|
|
6370
6531
|
}
|
|
6371
|
-
console.warn("\u{1F399}\uFE0F Fetching audio from Mux...");
|
|
6372
|
-
let audioBuffer;
|
|
6373
|
-
try {
|
|
6374
|
-
audioBuffer = await fetchAudioFromMux(audioUrl);
|
|
6375
|
-
} catch (error) {
|
|
6376
|
-
wrapError(error, "Failed to fetch audio from Mux");
|
|
6377
|
-
}
|
|
6378
6532
|
console.warn("\u{1F399}\uFE0F Creating dubbing job in ElevenLabs...");
|
|
6379
6533
|
const elevenLabsLangCode = toISO639_3(toLanguageCode);
|
|
6380
6534
|
const normalizedFromLanguageCode = fromLanguageCode?.trim();
|
|
@@ -6385,7 +6539,7 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
6385
6539
|
let dubbingId;
|
|
6386
6540
|
try {
|
|
6387
6541
|
dubbingId = await createElevenLabsDubbingJob({
|
|
6388
|
-
|
|
6542
|
+
sourceUrl: audioUrl,
|
|
6389
6543
|
assetId,
|
|
6390
6544
|
elevenLabsLangCode,
|
|
6391
6545
|
elevenLabsSourceLangCode,
|
|
@@ -6397,12 +6551,13 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
6397
6551
|
wrapError(error, "Failed to create ElevenLabs dubbing job");
|
|
6398
6552
|
}
|
|
6399
6553
|
console.warn("\u23F3 Waiting for dubbing to complete...");
|
|
6554
|
+
const dubbingPollTimeoutSeconds = options.dubbingPollTimeoutSeconds ?? DEFAULT_DUBBING_POLL_TIMEOUT_SECONDS;
|
|
6555
|
+
const maxPollAttempts = Math.max(1, Math.ceil(dubbingPollTimeoutSeconds * 1e3 / DUBBING_POLL_INTERVAL_MS));
|
|
6400
6556
|
let dubbingStatus = "dubbing";
|
|
6401
6557
|
let pollAttempts = 0;
|
|
6402
|
-
const maxPollAttempts = 180;
|
|
6403
6558
|
let targetLanguages = [];
|
|
6404
|
-
while (dubbingStatus
|
|
6405
|
-
await sleep(
|
|
6559
|
+
while (dubbingStatus !== "dubbed" && pollAttempts < maxPollAttempts) {
|
|
6560
|
+
await sleep(DUBBING_POLL_INTERVAL_MS);
|
|
6406
6561
|
pollAttempts++;
|
|
6407
6562
|
try {
|
|
6408
6563
|
const statusResult = await checkElevenLabsDubbingStatus({
|
|
@@ -6411,12 +6566,12 @@ async function translateAudio(assetId, toLanguageCode, options = {}) {
|
|
|
6411
6566
|
});
|
|
6412
6567
|
dubbingStatus = statusResult.status;
|
|
6413
6568
|
targetLanguages = statusResult.targetLanguages;
|
|
6414
|
-
if (dubbingStatus === "failed") {
|
|
6415
|
-
throw new Error("ElevenLabs dubbing job failed");
|
|
6416
|
-
}
|
|
6417
6569
|
} catch (error) {
|
|
6418
6570
|
wrapError(error, "Failed to check dubbing status");
|
|
6419
6571
|
}
|
|
6572
|
+
if (dubbingStatus === "failed") {
|
|
6573
|
+
throw new MuxAiError("ElevenLabs reported that the dubbing job failed.", { type: "processing_error" });
|
|
6574
|
+
}
|
|
6420
6575
|
}
|
|
6421
6576
|
if (dubbingStatus !== "dubbed") {
|
|
6422
6577
|
throw new MuxAiError("Audio translation timed out or failed. Please try again.", { type: "timeout_error", retryable: true });
|
|
@@ -6563,13 +6718,6 @@ var DEFAULT_TRANSLATION_CHUNKING = {
|
|
|
6563
6718
|
maxCuesPerChunk: 80,
|
|
6564
6719
|
maxCueTextTokensPerChunk: 2e3
|
|
6565
6720
|
};
|
|
6566
|
-
var TOKEN_USAGE_FIELDS = [
|
|
6567
|
-
"inputTokens",
|
|
6568
|
-
"outputTokens",
|
|
6569
|
-
"totalTokens",
|
|
6570
|
-
"reasoningTokens",
|
|
6571
|
-
"cachedInputTokens"
|
|
6572
|
-
];
|
|
6573
6721
|
var TranslationChunkValidationError = class extends Error {
|
|
6574
6722
|
constructor(message) {
|
|
6575
6723
|
super(message);
|
|
@@ -6600,9 +6748,6 @@ function shouldSplitChunkTranslationError(error) {
|
|
|
6600
6748
|
}
|
|
6601
6749
|
return NoObjectGeneratedError.isInstance(error) || TypeValidationError.isInstance(error) || isTranslationChunkValidationError(error);
|
|
6602
6750
|
}
|
|
6603
|
-
function isDefinedTokenUsageValue(value) {
|
|
6604
|
-
return typeof value === "number";
|
|
6605
|
-
}
|
|
6606
6751
|
function resolveTranslationChunkingOptions(options) {
|
|
6607
6752
|
const targetChunkDurationSeconds = Math.max(
|
|
6608
6753
|
1,
|
|
@@ -6629,14 +6774,27 @@ function resolveTranslationChunkingOptions(options) {
|
|
|
6629
6774
|
)
|
|
6630
6775
|
};
|
|
6631
6776
|
}
|
|
6632
|
-
function
|
|
6633
|
-
|
|
6634
|
-
|
|
6635
|
-
|
|
6636
|
-
|
|
6777
|
+
function collectSettledTranslationsOrRethrow(outcomes, priorUsage) {
|
|
6778
|
+
const results = [];
|
|
6779
|
+
const failures = [];
|
|
6780
|
+
for (const outcome of outcomes) {
|
|
6781
|
+
if (outcome.status === "fulfilled") {
|
|
6782
|
+
results.push(outcome.value);
|
|
6783
|
+
} else {
|
|
6784
|
+
failures.push(outcome.reason);
|
|
6637
6785
|
}
|
|
6638
|
-
|
|
6639
|
-
|
|
6786
|
+
}
|
|
6787
|
+
if (failures.length > 0) {
|
|
6788
|
+
const partialUsage = [...priorUsage, ...results.map((result) => result.usage)];
|
|
6789
|
+
for (const failure of failures.slice(1)) {
|
|
6790
|
+
const usage = getErrorTokenUsage(failure);
|
|
6791
|
+
if (usage) {
|
|
6792
|
+
partialUsage.push(usage);
|
|
6793
|
+
}
|
|
6794
|
+
}
|
|
6795
|
+
rethrowWithTokenUsage(failures[0], partialUsage);
|
|
6796
|
+
}
|
|
6797
|
+
return results;
|
|
6640
6798
|
}
|
|
6641
6799
|
function createTranslationChunkRequest(id, cues, cueBlocks) {
|
|
6642
6800
|
return {
|
|
@@ -6912,24 +7070,28 @@ async function translateChunkWithFallback({
|
|
|
6912
7070
|
wrapError(error, `Chunk ${chunk.id} failed for ${Math.round(chunk.startTime)}s-${Math.round(chunk.endTime)}s`);
|
|
6913
7071
|
}
|
|
6914
7072
|
const [leftChunk, rightChunk] = splitTranslationChunkAtMidpoint(chunk);
|
|
6915
|
-
const
|
|
6916
|
-
|
|
6917
|
-
|
|
6918
|
-
|
|
6919
|
-
|
|
6920
|
-
|
|
6921
|
-
|
|
6922
|
-
|
|
6923
|
-
|
|
6924
|
-
|
|
6925
|
-
|
|
6926
|
-
|
|
6927
|
-
|
|
6928
|
-
|
|
6929
|
-
|
|
6930
|
-
|
|
6931
|
-
|
|
6932
|
-
|
|
7073
|
+
const failedAttemptUsage = getErrorTokenUsage(error);
|
|
7074
|
+
const [leftResult, rightResult] = collectSettledTranslationsOrRethrow(
|
|
7075
|
+
await Promise.allSettled([
|
|
7076
|
+
translateChunkWithFallback({
|
|
7077
|
+
chunk: leftChunk,
|
|
7078
|
+
fromLanguageCode,
|
|
7079
|
+
toLanguageCode,
|
|
7080
|
+
provider,
|
|
7081
|
+
modelId,
|
|
7082
|
+
credentials
|
|
7083
|
+
}),
|
|
7084
|
+
translateChunkWithFallback({
|
|
7085
|
+
chunk: rightChunk,
|
|
7086
|
+
fromLanguageCode,
|
|
7087
|
+
toLanguageCode,
|
|
7088
|
+
provider,
|
|
7089
|
+
modelId,
|
|
7090
|
+
credentials
|
|
7091
|
+
})
|
|
7092
|
+
]),
|
|
7093
|
+
failedAttemptUsage ? [failedAttemptUsage] : []
|
|
7094
|
+
);
|
|
6933
7095
|
return {
|
|
6934
7096
|
translatedVtt: concatenateVttSegments([leftResult.translatedVtt, rightResult.translatedVtt]),
|
|
6935
7097
|
usage: aggregateTokenUsage([leftResult.usage, rightResult.usage]),
|
|
@@ -6970,17 +7132,20 @@ async function translateCaptionTrack({
|
|
|
6970
7132
|
let totalUnexpectedKeyCount = 0;
|
|
6971
7133
|
for (let index = 0; index < chunkPlan.chunks.length; index += resolvedChunking.maxConcurrentTranslations) {
|
|
6972
7134
|
const batch = chunkPlan.chunks.slice(index, index + resolvedChunking.maxConcurrentTranslations);
|
|
6973
|
-
const batchResults =
|
|
6974
|
-
|
|
6975
|
-
(
|
|
6976
|
-
chunk
|
|
6977
|
-
|
|
6978
|
-
|
|
6979
|
-
|
|
6980
|
-
|
|
6981
|
-
|
|
6982
|
-
|
|
6983
|
-
|
|
7135
|
+
const batchResults = collectSettledTranslationsOrRethrow(
|
|
7136
|
+
await Promise.allSettled(
|
|
7137
|
+
batch.map(
|
|
7138
|
+
(chunk) => translateChunkWithFallback({
|
|
7139
|
+
chunk,
|
|
7140
|
+
fromLanguageCode,
|
|
7141
|
+
toLanguageCode,
|
|
7142
|
+
provider,
|
|
7143
|
+
modelId,
|
|
7144
|
+
credentials
|
|
7145
|
+
})
|
|
7146
|
+
)
|
|
7147
|
+
),
|
|
7148
|
+
usageByChunk
|
|
6984
7149
|
);
|
|
6985
7150
|
translatedSegments.push(...batchResults.map((result) => result.translatedVtt));
|
|
6986
7151
|
usageByChunk.push(...batchResults.map((result) => result.usage));
|
|
@@ -7033,6 +7198,14 @@ async function uploadVttToS3({
|
|
|
7033
7198
|
}
|
|
7034
7199
|
async function translateCaptions(assetId, trackId, toLanguageCode, options) {
|
|
7035
7200
|
"use workflow";
|
|
7201
|
+
const collectedUsage = [];
|
|
7202
|
+
try {
|
|
7203
|
+
return await translateCaptionsInternal(assetId, trackId, toLanguageCode, options, collectedUsage);
|
|
7204
|
+
} catch (error) {
|
|
7205
|
+
rethrowWithTokenUsage(error, collectedUsage);
|
|
7206
|
+
}
|
|
7207
|
+
}
|
|
7208
|
+
async function translateCaptionsInternal(assetId, trackId, toLanguageCode, options, collectedUsage) {
|
|
7036
7209
|
const {
|
|
7037
7210
|
provider = "openai",
|
|
7038
7211
|
model,
|
|
@@ -7111,6 +7284,7 @@ async function translateCaptions(assetId, trackId, toLanguageCode, options) {
|
|
|
7111
7284
|
});
|
|
7112
7285
|
translatedVtt = result.translatedVtt;
|
|
7113
7286
|
usage = result.usage;
|
|
7287
|
+
collectedUsage.push(result.usage);
|
|
7114
7288
|
scrubbedCueCounts = result.scrubbedCueCounts;
|
|
7115
7289
|
unexpectedKeyCount = result.unexpectedKeyCount;
|
|
7116
7290
|
} catch (error) {
|